Compare commits

..
1 Commits
Author SHA1 Message Date
daprogs d382c77995 docs: add SQL-only schema update script for v2610.1
update(2610.1).sql brings an existing database up to the current schema
without PHP: base tables (CREATE IF NOT EXISTS), the v2603.x migrations
(last_event_at column, uq_event removal) and the v2610.1 index changes
(idx_occurred, idx_src_ip_occ, drop idx_src_ip). All steps are idempotent
via information_schema checks + PREPARE/EXECUTE, so the file is safe to
re-run and portable across MySQL 8 and MariaDB.
2026-10-19 12:12:00 -04:00
2 changed files with 165 additions and 2 deletions

No files matched your search

+11 -2
View File
@@ -42,6 +42,7 @@ Each portspoof_py instance runs independently and exposes a JSON API. portspoof_
portspoof_concentrator/
├── config.php Database credentials and app-wide constants
├── schema.sql MySQL DDL (applied by setup.php)
├── update(2610.1).sql SQL-only schema update for existing databases (this release)
├── setup.php One-time install / migration script
├── index.php Aggregated dashboard
├── nodes.php Add / edit / delete portspoof_py nodes
@@ -501,13 +502,21 @@ curl -H "Authorization: Bearer your-token" "https://yourserver/api/connections.p
## Upgrading
If you deployed before `last_event_at` was added to the schema, run `setup.php` again:
When upgrading, migrate the database to the new schema with either of:
**Via the PHP setup script:**
```bash
php /var/www/portspoof_concentrator/setup.php
```
It will add the missing column, drop the legacy `uq_event` unique key, and add the `occurred_at` / `(src_ip, occurred_at)` indexes if present. No data is lost.
**Via SQL (no PHP required):**
```bash
mysql -u portspoof -p portspoof_concentrator < "update(2610.1).sql"
```
Both apply all pending migrations idempotently (the `last_event_at` column, the legacy `uq_event` key removal, and the `occurred_at` / `(src_ip, occurred_at)` indexes) and can be re-run safely. No data is lost. Note that on a large `connections` table the `ADD KEY` steps take time proportional to the table size.
---
+154
View File
@@ -0,0 +1,154 @@
-- ============================================================================
-- portspoof_concentrator — database update to v2610.1
--
-- Brings an existing database schema up to v2610.1. Safe to run from any
-- previous version and safe to run more than once: every step is idempotent
-- and prints a status line for what it did.
--
-- Usage (note the database name — the script uses DATABASE() in its checks):
--
-- mysql -u portspoof -p portspoof_concentrator < "update(2610.1).sql"
--
-- Note: on a large connections table the ADD KEY steps take time proportional
-- to the table size (online DDL, but they scan all existing rows). Consider
-- running during a quiet window.
-- ============================================================================
-- ----------------------------------------------------------------------------
-- 1. Base tables — no-op if they already exist (a brand-new database is
-- brought fully up to date by this file alone)
-- ----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS nodes (
id INT UNSIGNED NOT NULL AUTO_INCREMENT,
name VARCHAR(100) NOT NULL,
api_url VARCHAR(255) NOT NULL COMMENT 'e.g. https://192.168.1.10:8080',
username VARCHAR(100) NOT NULL DEFAULT 'admin',
password VARCHAR(255) NOT NULL,
verify_ssl TINYINT(1) NOT NULL DEFAULT 0,
enabled TINYINT(1) NOT NULL DEFAULT 1,
last_fetched_at DATETIME NULL,
last_event_at DATETIME(6) NULL COMMENT 'timestamp of the newest ingested event from this node',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (id),
UNIQUE KEY uq_api_url (api_url)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
CREATE TABLE IF NOT EXISTS connections (
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
node_id INT UNSIGNED NOT NULL,
occurred_at DATETIME(6) NOT NULL COMMENT 'timestamp from portspoof_py',
src_ip VARCHAR(45) NOT NULL,
src_port SMALLINT UNSIGNED NOT NULL,
dst_port SMALLINT UNSIGNED NOT NULL,
banner_hex TEXT NULL,
banner_len INT UNSIGNED NOT NULL DEFAULT 0,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (id),
KEY idx_node_occurred (node_id, occurred_at),
KEY idx_occurred (occurred_at),
KEY idx_src_ip_occ (src_ip, occurred_at),
KEY idx_dst_port (dst_port),
CONSTRAINT fk_conn_node FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
CREATE TABLE IF NOT EXISTS settings (
key_name VARCHAR(64) NOT NULL,
value TEXT NOT NULL,
PRIMARY KEY (key_name)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
INSERT IGNORE INTO settings (key_name, value) VALUES ('retention_days', '7');
INSERT IGNORE INTO settings (key_name, value) VALUES ('frequent_ip_threshold', '5');
-- ----------------------------------------------------------------------------
-- 2. v2603.x migration: nodes.last_event_at (the per-node deduplication
-- cursor). No-op for databases created with the current schema.
-- ----------------------------------------------------------------------------
SET @sql = IF(
(SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'nodes'
AND COLUMN_NAME = 'last_event_at') = 0,
'ALTER TABLE nodes ADD COLUMN last_event_at DATETIME(6) NULL COMMENT ''timestamp of the newest ingested event from this node'' AFTER last_fetched_at',
'SELECT ''nodes.last_event_at: already present, skipped'' AS status'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
-- ----------------------------------------------------------------------------
-- 3. v2603.x migration: drop the legacy uq_event unique key
-- ----------------------------------------------------------------------------
SET @sql = IF(
(SELECT COUNT(*) FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'connections'
AND INDEX_NAME = 'uq_event') > 0,
'ALTER TABLE connections DROP INDEX uq_event',
'SELECT ''connections.uq_event: not present, skipped'' AS status'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
-- ----------------------------------------------------------------------------
-- 4. v2610.1: idx_occurred — lets the time-window queries (dashboard stats,
-- recent connections, JSON API, purge) use an index instead of scanning
-- the whole connections table
-- ----------------------------------------------------------------------------
SET @sql = IF(
(SELECT COUNT(*) FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'connections'
AND INDEX_NAME = 'idx_occurred') = 0,
'ALTER TABLE connections ADD KEY idx_occurred (occurred_at)',
'SELECT ''connections.idx_occurred: already present, skipped'' AS status'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
-- ----------------------------------------------------------------------------
-- 5. v2610.1: idx_src_ip_occ — makes the frequent_ips() GROUP BY a covering
-- index scan. Must run before step 6 so src_ip coverage is never lost.
-- ----------------------------------------------------------------------------
SET @sql = IF(
(SELECT COUNT(*) FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'connections'
AND INDEX_NAME = 'idx_src_ip_occ') = 0,
'ALTER TABLE connections ADD KEY idx_src_ip_occ (src_ip, occurred_at)',
'SELECT ''connections.idx_src_ip_occ: already present, skipped'' AS status'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
-- ----------------------------------------------------------------------------
-- 6. v2610.1: drop the now-redundant standalone idx_src_ip (fully covered
-- by idx_src_ip_occ) — one less index to maintain on every insert
-- ----------------------------------------------------------------------------
SET @sql = IF(
(SELECT COUNT(*) FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'connections'
AND INDEX_NAME = 'idx_src_ip') > 0,
'ALTER TABLE connections DROP INDEX idx_src_ip',
'SELECT ''connections.idx_src_ip: not present, skipped'' AS status'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
-- ============================================================================
-- Done. Verify with:
-- SHOW INDEX FROM connections;
-- Expected indexes: PRIMARY, idx_node_occurred, idx_occurred,
-- idx_src_ip_occ, idx_dst_port
-- ============================================================================