From d382c7799514404a025e0b35f43d30c0bb488d33 Mon Sep 17 00:00:00 2001 From: daprogs Date: Mon, 19 Oct 2026 12:12:00 -0400 Subject: [PATCH] docs: add SQL-only schema update script for v2610.1 update(2610.1).sql brings an existing database up to the current schema without PHP: base tables (CREATE IF NOT EXISTS), the v2603.x migrations (last_event_at column, uq_event removal) and the v2610.1 index changes (idx_occurred, idx_src_ip_occ, drop idx_src_ip). All steps are idempotent via information_schema checks + PREPARE/EXECUTE, so the file is safe to re-run and portable across MySQL 8 and MariaDB. --- README.md | 13 +++- update(2610.1).sql | 154 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 update(2610.1).sql diff --git a/README.md b/README.md index e83f7d2..78aebab 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ Each portspoof_py instance runs independently and exposes a JSON API. portspoof_ portspoof_concentrator/ ├── config.php Database credentials and app-wide constants ├── schema.sql MySQL DDL (applied by setup.php) +├── update(2610.1).sql SQL-only schema update for existing databases (this release) ├── setup.php One-time install / migration script ├── index.php Aggregated dashboard ├── nodes.php Add / edit / delete portspoof_py nodes @@ -501,13 +502,21 @@ curl -H "Authorization: Bearer your-token" "https://yourserver/api/connections.p ## Upgrading -If you deployed before `last_event_at` was added to the schema, run `setup.php` again: +When upgrading, migrate the database to the new schema with either of: + +**Via the PHP setup script:** ```bash php /var/www/portspoof_concentrator/setup.php ``` -It will add the missing column, drop the legacy `uq_event` unique key, and add the `occurred_at` / `(src_ip, occurred_at)` indexes if present. No data is lost. +**Via SQL (no PHP required):** + +```bash +mysql -u portspoof -p portspoof_concentrator < "update(2610.1).sql" +``` + +Both apply all pending migrations idempotently (the `last_event_at` column, the legacy `uq_event` key removal, and the `occurred_at` / `(src_ip, occurred_at)` indexes) and can be re-run safely. No data is lost. Note that on a large `connections` table the `ADD KEY` steps take time proportional to the table size. --- diff --git a/update(2610.1).sql b/update(2610.1).sql new file mode 100644 index 0000000..82a88c7 --- /dev/null +++ b/update(2610.1).sql @@ -0,0 +1,154 @@ +-- ============================================================================ +-- portspoof_concentrator — database update to v2610.1 +-- +-- Brings an existing database schema up to v2610.1. Safe to run from any +-- previous version and safe to run more than once: every step is idempotent +-- and prints a status line for what it did. +-- +-- Usage (note the database name — the script uses DATABASE() in its checks): +-- +-- mysql -u portspoof -p portspoof_concentrator < "update(2610.1).sql" +-- +-- Note: on a large connections table the ADD KEY steps take time proportional +-- to the table size (online DDL, but they scan all existing rows). Consider +-- running during a quiet window. +-- ============================================================================ + +-- ---------------------------------------------------------------------------- +-- 1. Base tables — no-op if they already exist (a brand-new database is +-- brought fully up to date by this file alone) +-- ---------------------------------------------------------------------------- + +CREATE TABLE IF NOT EXISTS nodes ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT, + name VARCHAR(100) NOT NULL, + api_url VARCHAR(255) NOT NULL COMMENT 'e.g. https://192.168.1.10:8080', + username VARCHAR(100) NOT NULL DEFAULT 'admin', + password VARCHAR(255) NOT NULL, + verify_ssl TINYINT(1) NOT NULL DEFAULT 0, + enabled TINYINT(1) NOT NULL DEFAULT 1, + last_fetched_at DATETIME NULL, + last_event_at DATETIME(6) NULL COMMENT 'timestamp of the newest ingested event from this node', + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id), + UNIQUE KEY uq_api_url (api_url) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE IF NOT EXISTS connections ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + node_id INT UNSIGNED NOT NULL, + occurred_at DATETIME(6) NOT NULL COMMENT 'timestamp from portspoof_py', + src_ip VARCHAR(45) NOT NULL, + src_port SMALLINT UNSIGNED NOT NULL, + dst_port SMALLINT UNSIGNED NOT NULL, + banner_hex TEXT NULL, + banner_len INT UNSIGNED NOT NULL DEFAULT 0, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id), + KEY idx_node_occurred (node_id, occurred_at), + KEY idx_occurred (occurred_at), + KEY idx_src_ip_occ (src_ip, occurred_at), + KEY idx_dst_port (dst_port), + CONSTRAINT fk_conn_node FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE IF NOT EXISTS settings ( + key_name VARCHAR(64) NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY (key_name) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +INSERT IGNORE INTO settings (key_name, value) VALUES ('retention_days', '7'); +INSERT IGNORE INTO settings (key_name, value) VALUES ('frequent_ip_threshold', '5'); + +-- ---------------------------------------------------------------------------- +-- 2. v2603.x migration: nodes.last_event_at (the per-node deduplication +-- cursor). No-op for databases created with the current schema. +-- ---------------------------------------------------------------------------- + +SET @sql = IF( + (SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'nodes' + AND COLUMN_NAME = 'last_event_at') = 0, + 'ALTER TABLE nodes ADD COLUMN last_event_at DATETIME(6) NULL COMMENT ''timestamp of the newest ingested event from this node'' AFTER last_fetched_at', + 'SELECT ''nodes.last_event_at: already present, skipped'' AS status' +); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +-- ---------------------------------------------------------------------------- +-- 3. v2603.x migration: drop the legacy uq_event unique key +-- ---------------------------------------------------------------------------- + +SET @sql = IF( + (SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'connections' + AND INDEX_NAME = 'uq_event') > 0, + 'ALTER TABLE connections DROP INDEX uq_event', + 'SELECT ''connections.uq_event: not present, skipped'' AS status' +); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +-- ---------------------------------------------------------------------------- +-- 4. v2610.1: idx_occurred — lets the time-window queries (dashboard stats, +-- recent connections, JSON API, purge) use an index instead of scanning +-- the whole connections table +-- ---------------------------------------------------------------------------- + +SET @sql = IF( + (SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'connections' + AND INDEX_NAME = 'idx_occurred') = 0, + 'ALTER TABLE connections ADD KEY idx_occurred (occurred_at)', + 'SELECT ''connections.idx_occurred: already present, skipped'' AS status' +); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +-- ---------------------------------------------------------------------------- +-- 5. v2610.1: idx_src_ip_occ — makes the frequent_ips() GROUP BY a covering +-- index scan. Must run before step 6 so src_ip coverage is never lost. +-- ---------------------------------------------------------------------------- + +SET @sql = IF( + (SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'connections' + AND INDEX_NAME = 'idx_src_ip_occ') = 0, + 'ALTER TABLE connections ADD KEY idx_src_ip_occ (src_ip, occurred_at)', + 'SELECT ''connections.idx_src_ip_occ: already present, skipped'' AS status' +); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +-- ---------------------------------------------------------------------------- +-- 6. v2610.1: drop the now-redundant standalone idx_src_ip (fully covered +-- by idx_src_ip_occ) — one less index to maintain on every insert +-- ---------------------------------------------------------------------------- + +SET @sql = IF( + (SELECT COUNT(*) FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'connections' + AND INDEX_NAME = 'idx_src_ip') > 0, + 'ALTER TABLE connections DROP INDEX idx_src_ip', + 'SELECT ''connections.idx_src_ip: not present, skipped'' AS status' +); +PREPARE stmt FROM @sql; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +-- ============================================================================ +-- Done. Verify with: +-- SHOW INDEX FROM connections; +-- Expected indexes: PRIMARY, idx_node_occurred, idx_occurred, +-- idx_src_ip_occ, idx_dst_port +-- ============================================================================