From 7308aad1a8626575a81dc6f9b3ed974f2b229901 Mon Sep 17 00:00:00 2001 From: aiooss-anssi Date: Mon, 18 Nov 2024 11:40:00 +0100 Subject: [PATCH] Improve timestamps precision, fix missing events --- suricata/suricata-eve-sqlite-output/src/database.rs | 8 ++++---- suricata/suricata-eve-sqlite-output/src/schema.sql | 5 +++-- webapp/main.py | 2 +- webapp/static/js/flowdisplay.js | 6 +++--- webapp/static/js/flowlist.js | 9 +++++---- 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/suricata/suricata-eve-sqlite-output/src/database.rs b/suricata/suricata-eve-sqlite-output/src/database.rs index 8d622b9..980ecf7 100644 --- a/suricata/suricata-eve-sqlite-output/src/database.rs +++ b/suricata/suricata-eve-sqlite-output/src/database.rs @@ -68,28 +68,28 @@ fn write_event(transaction: &Transaction, buf: &str) -> Result { transaction.execute( "INSERT OR IGNORE INTO alert (flow_id, timestamp, extra_data) \ - values(?1->>'flow_id', (UNIXEPOCH(SUBSTR(?1->>'timestamp', 1, 26), 'subsec') * 1000), json_extract(?1, '$.' || ?2))", + values(?1->>'flow_id', (UNIXEPOCH(SUBSTR(?1->>'timestamp', 1, 19))*1000000 + SUBSTR(?1->>'timestamp', 21, 6)), json_extract(?1, '$.' || ?2))", (buf, event_type), ) }, "anomaly" => { transaction.execute( "INSERT OR IGNORE INTO anomaly (flow_id, timestamp, extra_data) \ - values(?1->>'flow_id', (UNIXEPOCH(SUBSTR(?1->>'timestamp', 1, 26), 'subsec') * 1000), json_extract(?1, '$.' || ?2))", + values(?1->>'flow_id', (UNIXEPOCH(SUBSTR(?1->>'timestamp', 1, 19))*1000000 + SUBSTR(?1->>'timestamp', 21, 6)), json_extract(?1, '$.' || ?2))", (buf, event_type), ) }, "fileinfo" => { transaction.execute( "INSERT OR IGNORE INTO fileinfo (flow_id, timestamp, extra_data) \ - values(?1->>'flow_id', (UNIXEPOCH(SUBSTR(?1->>'timestamp', 1, 26), 'subsec') * 1000), json_extract(?1, '$.' || ?2))", + values(?1->>'flow_id', (UNIXEPOCH(SUBSTR(?1->>'timestamp', 1, 19))*1000000 + SUBSTR(?1->>'timestamp', 21, 6)), json_extract(?1, '$.' || ?2))", (buf, event_type), ) }, _ => { transaction.execute( "INSERT OR IGNORE INTO 'app-event' (flow_id, timestamp, app_proto, extra_data) \ - values(?1->>'flow_id', (UNIXEPOCH(SUBSTR(?1->>'timestamp', 1, 26), 'subsec') * 1000), ?2, json_extract(?1, '$.' || ?2))", + values(?1->>'flow_id', (UNIXEPOCH(SUBSTR(?1->>'timestamp', 1, 19))*1000000 + SUBSTR(?1->>'timestamp', 21, 6)), ?2, json_extract(?1, '$.' || ?2))", (buf, event_type), ) } diff --git a/suricata/suricata-eve-sqlite-output/src/schema.sql b/suricata/suricata-eve-sqlite-output/src/schema.sql index 1759207..ec21401 100644 --- a/suricata/suricata-eve-sqlite-output/src/schema.sql +++ b/suricata/suricata-eve-sqlite-output/src/schema.sql @@ -2,10 +2,11 @@ -- SPDX-License-Identifier: GPL-2.0-or-later CREATE TABLE IF NOT EXISTS "flow" ( id INTEGER NOT NULL PRIMARY KEY, + -- SQLite UNIXEPOCH currently has only millisecond precision using "subsec", which is not enough ts_start INTEGER GENERATED ALWAYS - AS (UNIXEPOCH(SUBSTR(extra_data->>'start', 1, 26), 'subsec') * 1000) STORED, + AS (UNIXEPOCH(SUBSTR(extra_data->>'start', 1, 19))*1000000 + SUBSTR(extra_data->>'start', 21, 6)) STORED, ts_end INTEGER GENERATED ALWAYS - AS (UNIXEPOCH(SUBSTR(extra_data->>'end', 1, 26), 'subsec') * 1000) STORED, + AS (UNIXEPOCH(SUBSTR(extra_data->>'end', 1, 19))*1000000 + SUBSTR(extra_data->>'end', 21, 6)) STORED, src_ip TEXT NOT NULL, src_port INTEGER, src_ipport TEXT GENERATED ALWAYS diff --git a/webapp/main.py b/webapp/main.py index 5ef55b8..068b5c7 100644 --- a/webapp/main.py +++ b/webapp/main.py @@ -38,7 +38,7 @@ async def index(request): async def api_flow_list(request): # Parse GET arguments - ts_to = request.query_params.get("to", str(int(1e13))) + ts_to = request.query_params.get("to", str(int(1e16))) services = request.query_params.getlist("service") app_proto = request.query_params.get("app_proto") search = request.query_params.get("search") diff --git a/webapp/static/js/flowdisplay.js b/webapp/static/js/flowdisplay.js index db54697..fcb7452 100644 --- a/webapp/static/js/flowdisplay.js +++ b/webapp/static/js/flowdisplay.js @@ -204,9 +204,9 @@ class FlowDisplay { second: 'numeric', fractionalSecondDigits: 3 } - const dateStart = new Date(flow.flow.ts_start) + const dateStart = new Date(flow.flow.ts_start / 1000) const formatedDateStart = new Intl.DateTimeFormat(undefined, dateParams).format(dateStart) - const dateEnd = new Date(flow.flow.ts_end) + const dateEnd = new Date(flow.flow.ts_end / 1000) const formatedDateEnd = new Intl.DateTimeFormat(undefined, dateParams).format(dateEnd) // Change document title @@ -220,7 +220,7 @@ class FlowDisplay { document.getElementById('display-flow-pcap').parentNode.classList.toggle('d-none', !flow.flow.pcap_filename) if (this.tickLength > 0) { document.getElementById('display-flow-tick').classList.remove('d-none') - const tick = ((flow.flow.ts_start / 1000 - this.startTs) / this.tickLength).toFixed(3) + const tick = ((flow.flow.ts_start / 1000000 - this.startTs) / this.tickLength).toFixed(3) document.querySelector('#display-flow-tick > a > span').textContent = tick document.querySelector('#display-flow-tick > a').dataset.ts = flow.flow.ts_start } diff --git a/webapp/static/js/flowlist.js b/webapp/static/js/flowlist.js index 4c439c7..844a9f2 100644 --- a/webapp/static/js/flowlist.js +++ b/webapp/static/js/flowlist.js @@ -118,7 +118,7 @@ class FlowList { const untilTick = Number(e.target.value) const url = new URL(document.location) if (untilTick) { - url.searchParams.set('to', Math.floor(((untilTick + 1) * (this.tickLength || 1) + this.startTs)) * 1000) + url.searchParams.set('to', Math.floor(((untilTick + 1) * (this.tickLength || 1) + this.startTs)) * 1000000) } else { url.searchParams.delete('to') e.target.value = null @@ -236,6 +236,7 @@ class FlowList { * @returns Pretty string representation */ pprintDelay (delay) { + delay = delay / 1000 if (delay > 1000) { delay = delay / 1000 return `${delay.toPrecision(3)} s` @@ -348,7 +349,7 @@ class FlowList { async fillFlowsList (flows, tags) { const flowList = document.getElementById('flow-list') flows.forEach((flow) => { - const date = new Date(flow.ts_start) + const date = new Date(flow.ts_start / 1000) const startDate = new Intl.DateTimeFormat( undefined, { hour: 'numeric', minute: 'numeric', second: 'numeric', fractionalSecondDigits: 1 } @@ -362,7 +363,7 @@ class FlowList { // Create tick element on new tick if (this.tickLength > 0) { - const tick = Math.floor((flow.ts_start / 1000 - this.startTs) / this.tickLength) + const tick = Math.floor((flow.ts_start / 1000000 - this.startTs) / this.tickLength) if (tick !== this.lastTick) { const tickEl = document.createElement('span') tickEl.classList.add('list-group-item', 'sticky-top', 'pt-3', 'pb-1', 'px-2', 'border-0', 'border-bottom', 'bg-light-subtle', 'text-center', 'fw-semibold') @@ -459,7 +460,7 @@ class FlowList { // Update time filter state if (toTs) { - const toTick = (Number(toTs) / 1000 - this.startTs) / (this.tickLength || 1) - 1 + const toTick = (Number(toTs) / 1000000 - this.startTs) / (this.tickLength || 1) - 1 document.getElementById('filter-time-until').value = toTick } document.getElementById('filter-time-until').classList.toggle('is-active', toTs)