docs: add SQL-only schema update script for v2610.1
update(2610.1).sql brings an existing database up to the current schema without PHP: base tables (CREATE IF NOT EXISTS), the v2603.x migrations (last_event_at column, uq_event removal) and the v2610.1 index changes (idx_occurred, idx_src_ip_occ, drop idx_src_ip). All steps are idempotent via information_schema checks + PREPARE/EXECUTE, so the file is safe to re-run and portable across MySQL 8 and MariaDB.
This commit is contained in:
daprogs committed
1 parent
5649930060
commit
d382c77995
2 files changed
+165
-2
No files matched your search
@@ -42,6 +42,7 @@ Each portspoof_py instance runs independently and exposes a JSON API. portspoof_
|
||||
portspoof_concentrator/
|
||||
├── config.php Database credentials and app-wide constants
|
||||
├── schema.sql MySQL DDL (applied by setup.php)
|
||||
├── update(2610.1).sql SQL-only schema update for existing databases (this release)
|
||||
├── setup.php One-time install / migration script
|
||||
├── index.php Aggregated dashboard
|
||||
├── nodes.php Add / edit / delete portspoof_py nodes
|
||||
@@ -501,13 +502,21 @@ curl -H "Authorization: Bearer your-token" "https://yourserver/api/connections.p
|
||||
|
||||
## Upgrading
|
||||
|
||||
If you deployed before `last_event_at` was added to the schema, run `setup.php` again:
|
||||
When upgrading, migrate the database to the new schema with either of:
|
||||
|
||||
**Via the PHP setup script:**
|
||||
|
||||
```bash
|
||||
php /var/www/portspoof_concentrator/setup.php
|
||||
```
|
||||
|
||||
It will add the missing column, drop the legacy `uq_event` unique key, and add the `occurred_at` / `(src_ip, occurred_at)` indexes if present. No data is lost.
|
||||
**Via SQL (no PHP required):**
|
||||
|
||||
```bash
|
||||
mysql -u portspoof -p portspoof_concentrator < "update(2610.1).sql"
|
||||
```
|
||||
|
||||
Both apply all pending migrations idempotently (the `last_event_at` column, the legacy `uq_event` key removal, and the `occurred_at` / `(src_ip, occurred_at)` indexes) and can be re-run safely. No data is lost. Note that on a large `connections` table the `ADD KEY` steps take time proportional to the table size.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
-- ============================================================================
|
||||
-- portspoof_concentrator — database update to v2610.1
|
||||
--
|
||||
-- Brings an existing database schema up to v2610.1. Safe to run from any
|
||||
-- previous version and safe to run more than once: every step is idempotent
|
||||
-- and prints a status line for what it did.
|
||||
--
|
||||
-- Usage (note the database name — the script uses DATABASE() in its checks):
|
||||
--
|
||||
-- mysql -u portspoof -p portspoof_concentrator < "update(2610.1).sql"
|
||||
--
|
||||
-- Note: on a large connections table the ADD KEY steps take time proportional
|
||||
-- to the table size (online DDL, but they scan all existing rows). Consider
|
||||
-- running during a quiet window.
|
||||
-- ============================================================================
|
||||
|
||||
-- ----------------------------------------------------------------------------
|
||||
-- 1. Base tables — no-op if they already exist (a brand-new database is
|
||||
-- brought fully up to date by this file alone)
|
||||
-- ----------------------------------------------------------------------------
|
||||
|
||||
CREATE TABLE IF NOT EXISTS nodes (
|
||||
id INT UNSIGNED NOT NULL AUTO_INCREMENT,
|
||||
name VARCHAR(100) NOT NULL,
|
||||
api_url VARCHAR(255) NOT NULL COMMENT 'e.g. https://192.168.1.10:8080',
|
||||
username VARCHAR(100) NOT NULL DEFAULT 'admin',
|
||||
password VARCHAR(255) NOT NULL,
|
||||
verify_ssl TINYINT(1) NOT NULL DEFAULT 0,
|
||||
enabled TINYINT(1) NOT NULL DEFAULT 1,
|
||||
last_fetched_at DATETIME NULL,
|
||||
last_event_at DATETIME(6) NULL COMMENT 'timestamp of the newest ingested event from this node',
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY uq_api_url (api_url)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS connections (
|
||||
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
|
||||
node_id INT UNSIGNED NOT NULL,
|
||||
occurred_at DATETIME(6) NOT NULL COMMENT 'timestamp from portspoof_py',
|
||||
src_ip VARCHAR(45) NOT NULL,
|
||||
src_port SMALLINT UNSIGNED NOT NULL,
|
||||
dst_port SMALLINT UNSIGNED NOT NULL,
|
||||
banner_hex TEXT NULL,
|
||||
banner_len INT UNSIGNED NOT NULL DEFAULT 0,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (id),
|
||||
KEY idx_node_occurred (node_id, occurred_at),
|
||||
KEY idx_occurred (occurred_at),
|
||||
KEY idx_src_ip_occ (src_ip, occurred_at),
|
||||
KEY idx_dst_port (dst_port),
|
||||
CONSTRAINT fk_conn_node FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS settings (
|
||||
key_name VARCHAR(64) NOT NULL,
|
||||
value TEXT NOT NULL,
|
||||
PRIMARY KEY (key_name)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
|
||||
INSERT IGNORE INTO settings (key_name, value) VALUES ('retention_days', '7');
|
||||
INSERT IGNORE INTO settings (key_name, value) VALUES ('frequent_ip_threshold', '5');
|
||||
|
||||
-- ----------------------------------------------------------------------------
|
||||
-- 2. v2603.x migration: nodes.last_event_at (the per-node deduplication
|
||||
-- cursor). No-op for databases created with the current schema.
|
||||
-- ----------------------------------------------------------------------------
|
||||
|
||||
SET @sql = IF(
|
||||
(SELECT COUNT(*) FROM information_schema.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'nodes'
|
||||
AND COLUMN_NAME = 'last_event_at') = 0,
|
||||
'ALTER TABLE nodes ADD COLUMN last_event_at DATETIME(6) NULL COMMENT ''timestamp of the newest ingested event from this node'' AFTER last_fetched_at',
|
||||
'SELECT ''nodes.last_event_at: already present, skipped'' AS status'
|
||||
);
|
||||
PREPARE stmt FROM @sql;
|
||||
EXECUTE stmt;
|
||||
DEALLOCATE PREPARE stmt;
|
||||
|
||||
-- ----------------------------------------------------------------------------
|
||||
-- 3. v2603.x migration: drop the legacy uq_event unique key
|
||||
-- ----------------------------------------------------------------------------
|
||||
|
||||
SET @sql = IF(
|
||||
(SELECT COUNT(*) FROM information_schema.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'connections'
|
||||
AND INDEX_NAME = 'uq_event') > 0,
|
||||
'ALTER TABLE connections DROP INDEX uq_event',
|
||||
'SELECT ''connections.uq_event: not present, skipped'' AS status'
|
||||
);
|
||||
PREPARE stmt FROM @sql;
|
||||
EXECUTE stmt;
|
||||
DEALLOCATE PREPARE stmt;
|
||||
|
||||
-- ----------------------------------------------------------------------------
|
||||
-- 4. v2610.1: idx_occurred — lets the time-window queries (dashboard stats,
|
||||
-- recent connections, JSON API, purge) use an index instead of scanning
|
||||
-- the whole connections table
|
||||
-- ----------------------------------------------------------------------------
|
||||
|
||||
SET @sql = IF(
|
||||
(SELECT COUNT(*) FROM information_schema.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'connections'
|
||||
AND INDEX_NAME = 'idx_occurred') = 0,
|
||||
'ALTER TABLE connections ADD KEY idx_occurred (occurred_at)',
|
||||
'SELECT ''connections.idx_occurred: already present, skipped'' AS status'
|
||||
);
|
||||
PREPARE stmt FROM @sql;
|
||||
EXECUTE stmt;
|
||||
DEALLOCATE PREPARE stmt;
|
||||
|
||||
-- ----------------------------------------------------------------------------
|
||||
-- 5. v2610.1: idx_src_ip_occ — makes the frequent_ips() GROUP BY a covering
|
||||
-- index scan. Must run before step 6 so src_ip coverage is never lost.
|
||||
-- ----------------------------------------------------------------------------
|
||||
|
||||
SET @sql = IF(
|
||||
(SELECT COUNT(*) FROM information_schema.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'connections'
|
||||
AND INDEX_NAME = 'idx_src_ip_occ') = 0,
|
||||
'ALTER TABLE connections ADD KEY idx_src_ip_occ (src_ip, occurred_at)',
|
||||
'SELECT ''connections.idx_src_ip_occ: already present, skipped'' AS status'
|
||||
);
|
||||
PREPARE stmt FROM @sql;
|
||||
EXECUTE stmt;
|
||||
DEALLOCATE PREPARE stmt;
|
||||
|
||||
-- ----------------------------------------------------------------------------
|
||||
-- 6. v2610.1: drop the now-redundant standalone idx_src_ip (fully covered
|
||||
-- by idx_src_ip_occ) — one less index to maintain on every insert
|
||||
-- ----------------------------------------------------------------------------
|
||||
|
||||
SET @sql = IF(
|
||||
(SELECT COUNT(*) FROM information_schema.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'connections'
|
||||
AND INDEX_NAME = 'idx_src_ip') > 0,
|
||||
'ALTER TABLE connections DROP INDEX idx_src_ip',
|
||||
'SELECT ''connections.idx_src_ip: not present, skipped'' AS status'
|
||||
);
|
||||
PREPARE stmt FROM @sql;
|
||||
EXECUTE stmt;
|
||||
DEALLOCATE PREPARE stmt;
|
||||
|
||||
-- ============================================================================
|
||||
-- Done. Verify with:
|
||||
-- SHOW INDEX FROM connections;
|
||||
-- Expected indexes: PRIMARY, idx_node_occurred, idx_occurred,
|
||||
-- idx_src_ip_occ, idx_dst_port
|
||||
-- ============================================================================
|
||||
Reference in new issue
Block a user