perf: add occurred_at indexes to connections
Time-window queries (dashboard stats, recent connections, JSON API, purge) had no usable index and full-scanned the connections table on every load, so the dashboard slowed down as data accumulated. - Add idx_occurred (occurred_at) for counts, MAX, ORDER BY ... LIMIT, and the purge delete - Add idx_src_ip_occ (src_ip, occurred_at) so frequent_ips() runs as a covering index scan - Drop the now-redundant standalone idx_src_ip Migrations in setup.php are idempotent (information_schema checks) and portable across MySQL 8 and MariaDB.
This commit is contained in:
1 parent
05e9fbe76e
commit
849360e61c
3 files changed
+41
-3
No files matched your search
@@ -225,7 +225,7 @@ One row per connection event ingested from any node.
|
|||||||
| `banner_len` | `INT UNSIGNED` | Length of the banner in bytes |
|
| `banner_len` | `INT UNSIGNED` | Length of the banner in bytes |
|
||||||
| `created_at` | `DATETIME` | Row insertion time |
|
| `created_at` | `DATETIME` | Row insertion time |
|
||||||
|
|
||||||
**Indexes:** `(node_id, occurred_at)` for cursor lookups and per-node queries; `src_ip` and `dst_port` for the top-N aggregations.
|
**Indexes:** `(node_id, occurred_at)` for cursor lookups and per-node queries; `occurred_at` for time-window queries, the recent-connections sort, and the purge; `(src_ip, occurred_at)` for the frequent-IP aggregation; `dst_port` for the top-port aggregation.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -507,7 +507,7 @@ If you deployed before `last_event_at` was added to the schema, run `setup.php`
|
|||||||
php /var/www/portspoof_concentrator/setup.php
|
php /var/www/portspoof_concentrator/setup.php
|
||||||
```
|
```
|
||||||
|
|
||||||
It will add the missing column and drop the legacy `uq_event` unique key if present. No data is lost.
|
It will add the missing column, drop the legacy `uq_event` unique key, and add the `occurred_at` / `(src_ip, occurred_at)` indexes if present. No data is lost.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
+2
-1
@@ -27,7 +27,8 @@ CREATE TABLE IF NOT EXISTS connections (
|
|||||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
PRIMARY KEY (id),
|
PRIMARY KEY (id),
|
||||||
KEY idx_node_occurred (node_id, occurred_at),
|
KEY idx_node_occurred (node_id, occurred_at),
|
||||||
KEY idx_src_ip (src_ip),
|
KEY idx_occurred (occurred_at),
|
||||||
|
KEY idx_src_ip_occ (src_ip, occurred_at),
|
||||||
KEY idx_dst_port (dst_port),
|
KEY idx_dst_port (dst_port),
|
||||||
CONSTRAINT fk_conn_node FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE
|
CONSTRAINT fk_conn_node FOREIGN KEY (node_id) REFERENCES nodes (id) ON DELETE CASCADE
|
||||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||||
|
|||||||
@@ -41,6 +41,43 @@ if ($row > 0) {
|
|||||||
echo "Dropped legacy uq_event unique key.\n";
|
echo "Dropped legacy uq_event unique key.\n";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Migration: add occurred_at index if upgrading from an older schema.
|
||||||
|
// Time-window queries (dashboard, JSON API, purge) full-scanned the table
|
||||||
|
// without it, so page loads slowed down as the table grew.
|
||||||
|
$row = $pdo->query(
|
||||||
|
"SELECT COUNT(*) FROM information_schema.STATISTICS
|
||||||
|
WHERE TABLE_SCHEMA = DATABASE()
|
||||||
|
AND TABLE_NAME = 'connections'
|
||||||
|
AND INDEX_NAME = 'idx_occurred'"
|
||||||
|
)->fetchColumn();
|
||||||
|
if ($row == 0) {
|
||||||
|
$pdo->exec("ALTER TABLE connections ADD KEY idx_occurred (occurred_at)");
|
||||||
|
echo "Added idx_occurred index.\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
// Migration: replace the standalone src_ip index with (src_ip, occurred_at)
|
||||||
|
// so frequent_ips() resolves as a covering index scan.
|
||||||
|
$row = $pdo->query(
|
||||||
|
"SELECT COUNT(*) FROM information_schema.STATISTICS
|
||||||
|
WHERE TABLE_SCHEMA = DATABASE()
|
||||||
|
AND TABLE_NAME = 'connections'
|
||||||
|
AND INDEX_NAME = 'idx_src_ip_occ'"
|
||||||
|
)->fetchColumn();
|
||||||
|
if ($row == 0) {
|
||||||
|
$pdo->exec("ALTER TABLE connections ADD KEY idx_src_ip_occ (src_ip, occurred_at)");
|
||||||
|
echo "Added idx_src_ip_occ index.\n";
|
||||||
|
}
|
||||||
|
$row = $pdo->query(
|
||||||
|
"SELECT COUNT(*) FROM information_schema.STATISTICS
|
||||||
|
WHERE TABLE_SCHEMA = DATABASE()
|
||||||
|
AND TABLE_NAME = 'connections'
|
||||||
|
AND INDEX_NAME = 'idx_src_ip'"
|
||||||
|
)->fetchColumn();
|
||||||
|
if ($row > 0) {
|
||||||
|
$pdo->exec("ALTER TABLE connections DROP INDEX idx_src_ip");
|
||||||
|
echo "Dropped redundant idx_src_ip index.\n";
|
||||||
|
}
|
||||||
|
|
||||||
// Migration: create settings table and seed default retention if upgrading
|
// Migration: create settings table and seed default retention if upgrading
|
||||||
$pdo->exec(
|
$pdo->exec(
|
||||||
"CREATE TABLE IF NOT EXISTS settings (
|
"CREATE TABLE IF NOT EXISTS settings (
|
||||||
|
|||||||
Reference in new issue
Block a user