portspoof_concentrator/update(2610.1).sql

-- ============================================================================
-- portspoof_concentrator — database update to v2610.1
--
-- Brings an existing database schema up to v2610.1. Safe to run from any
-- previous version and safe to run more than once: every step is idempotent
-- and prints a status line for what it did.
--
-- Usage (note the database name — the script uses DATABASE() in its checks):
--
--   mysql -u portspoof -p portspoof_concentrator < "update(2610.1).sql"
--
-- Note: on a large connections table the ADD KEY steps take time proportional
-- to the table size (online DDL, but they scan all existing rows). Consider
-- running during a quiet window.
-- ============================================================================

-- ----------------------------------------------------------------------------
-- 1. Base tables — no-op if they already exist (a brand-new database is
--    brought fully up to date by this file alone)
-- ----------------------------------------------------------------------------

CREATE TABLE IF NOT EXISTS nodes (
    id          INT UNSIGNED    NOT NULL AUTO_INCREMENT,
    name        VARCHAR(100)    NOT NULL,
    api_url     VARCHAR(255)    NOT NULL COMMENT 'e.g. https://192.168.1.10:8080',
    username    VARCHAR(100)    NOT NULL DEFAULT 'admin',
    password    VARCHAR(255)    NOT NULL,
    verify_ssl  TINYINT(1)      NOT NULL DEFAULT 0,
    enabled     TINYINT(1)      NOT NULL DEFAULT 1,
    last_fetched_at DATETIME    NULL,
    last_event_at   DATETIME(6) NULL COMMENT 'timestamp of the newest ingested event from this node',
    created_at  DATETIME        NOT NULL DEFAULT CURRENT_TIMESTAMP,
    PRIMARY KEY (id),
    UNIQUE KEY uq_api_url (api_url)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

CREATE TABLE IF NOT EXISTS connections (
    id          BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
    node_id     INT UNSIGNED    NOT NULL,
    occurred_at DATETIME(6)     NOT NULL COMMENT 'timestamp from portspoof_py',
    src_ip      VARCHAR(45)     NOT NULL,
    src_port    SMALLINT UNSIGNED NOT NULL,
    dst_port    SMALLINT UNSIGNED NOT NULL,
    banner_hex  TEXT            NULL,
    banner_len  INT UNSIGNED    NOT NULL DEFAULT 0,
    created_at  DATETIME        NOT NULL DEFAULT CURRENT_TIMESTAMP,
    PRIMARY KEY (id),
    KEY idx_node_occurred (node_id, occurred_at),
    KEY idx_occurred (occurred_at),
    KEY idx_src_ip_occ (src_ip, occurred_at),
    KEY idx_dst_port (dst_port),
    CONSTRAINT fk_conn_node FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

CREATE TABLE IF NOT EXISTS settings (
    key_name    VARCHAR(64)  NOT NULL,
    value       TEXT         NOT NULL,
    PRIMARY KEY (key_name)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

INSERT IGNORE INTO settings (key_name, value) VALUES ('retention_days', '7');
INSERT IGNORE INTO settings (key_name, value) VALUES ('frequent_ip_threshold', '5');

-- ----------------------------------------------------------------------------
-- 2. v2603.x migration: nodes.last_event_at (the per-node deduplication
--    cursor). No-op for databases created with the current schema.
-- ----------------------------------------------------------------------------

SET @sql = IF(
    (SELECT COUNT(*) FROM information_schema.COLUMNS
     WHERE TABLE_SCHEMA = DATABASE()
       AND TABLE_NAME = 'nodes'
       AND COLUMN_NAME = 'last_event_at') = 0,
    'ALTER TABLE nodes ADD COLUMN last_event_at DATETIME(6) NULL COMMENT ''timestamp of the newest ingested event from this node'' AFTER last_fetched_at',
    'SELECT ''nodes.last_event_at: already present, skipped'' AS status'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;

-- ----------------------------------------------------------------------------
-- 3. v2603.x migration: drop the legacy uq_event unique key
-- ----------------------------------------------------------------------------

SET @sql = IF(
    (SELECT COUNT(*) FROM information_schema.STATISTICS
     WHERE TABLE_SCHEMA = DATABASE()
       AND TABLE_NAME = 'connections'
       AND INDEX_NAME = 'uq_event') > 0,
    'ALTER TABLE connections DROP INDEX uq_event',
    'SELECT ''connections.uq_event: not present, skipped'' AS status'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;

-- ----------------------------------------------------------------------------
-- 4. v2610.1: idx_occurred — lets the time-window queries (dashboard stats,
--    recent connections, JSON API, purge) use an index instead of scanning
--    the whole connections table
-- ----------------------------------------------------------------------------

SET @sql = IF(
    (SELECT COUNT(*) FROM information_schema.STATISTICS
     WHERE TABLE_SCHEMA = DATABASE()
       AND TABLE_NAME = 'connections'
       AND INDEX_NAME = 'idx_occurred') = 0,
    'ALTER TABLE connections ADD KEY idx_occurred (occurred_at)',
    'SELECT ''connections.idx_occurred: already present, skipped'' AS status'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;

-- ----------------------------------------------------------------------------
-- 5. v2610.1: idx_src_ip_occ — makes the frequent_ips() GROUP BY a covering
--    index scan. Must run before step 6 so src_ip coverage is never lost.
-- ----------------------------------------------------------------------------

SET @sql = IF(
    (SELECT COUNT(*) FROM information_schema.STATISTICS
     WHERE TABLE_SCHEMA = DATABASE()
       AND TABLE_NAME = 'connections'
       AND INDEX_NAME = 'idx_src_ip_occ') = 0,
    'ALTER TABLE connections ADD KEY idx_src_ip_occ (src_ip, occurred_at)',
    'SELECT ''connections.idx_src_ip_occ: already present, skipped'' AS status'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;

-- ----------------------------------------------------------------------------
-- 6. v2610.1: drop the now-redundant standalone idx_src_ip (fully covered
--    by idx_src_ip_occ) — one less index to maintain on every insert
-- ----------------------------------------------------------------------------

SET @sql = IF(
    (SELECT COUNT(*) FROM information_schema.STATISTICS
     WHERE TABLE_SCHEMA = DATABASE()
       AND TABLE_NAME = 'connections'
       AND INDEX_NAME = 'idx_src_ip') > 0,
    'ALTER TABLE connections DROP INDEX idx_src_ip',
    'SELECT ''connections.idx_src_ip: not present, skipped'' AS status'
);
PREPARE stmt FROM @sql;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;

-- ============================================================================
-- Done. Verify with:
--   SHOW INDEX FROM connections;
-- Expected indexes: PRIMARY, idx_node_occurred, idx_occurred,
--                   idx_src_ip_occ, idx_dst_port
-- ============================================================================