Improve timestamps precision, fix missing events

ANSSI-FR · Nov 18, 2024 · 7308aad · 7308aad
1 parent c469a53
commit 7308aad
Show file tree

Hide file tree

Showing 5 changed files with 16 additions and 14 deletions.
diff --git a/suricata/suricata-eve-sqlite-output/src/database.rs b/suricata/suricata-eve-sqlite-output/src/database.rs
@@ -68,28 +68,28 @@ fn write_event(transaction: &Transaction, buf: &str) -> Result<usize, rusqlite::
         "alert" => {
             transaction.execute(
                 "INSERT OR IGNORE INTO alert (flow_id, timestamp, extra_data) \
-                values(?1->>'flow_id', (UNIXEPOCH(SUBSTR(?1->>'timestamp', 1, 26), 'subsec') * 1000), json_extract(?1, '$.' || ?2))",
+                values(?1->>'flow_id', (UNIXEPOCH(SUBSTR(?1->>'timestamp', 1, 19))*1000000 + SUBSTR(?1->>'timestamp', 21, 6)), json_extract(?1, '$.' || ?2))",
                 (buf, event_type),
             )
         },
         "anomaly" => {
             transaction.execute(
                 "INSERT OR IGNORE INTO anomaly (flow_id, timestamp, extra_data) \
-                values(?1->>'flow_id', (UNIXEPOCH(SUBSTR(?1->>'timestamp', 1, 26), 'subsec') * 1000), json_extract(?1, '$.' || ?2))",
+                values(?1->>'flow_id', (UNIXEPOCH(SUBSTR(?1->>'timestamp', 1, 19))*1000000 + SUBSTR(?1->>'timestamp', 21, 6)), json_extract(?1, '$.' || ?2))",
                 (buf, event_type),
             )
         },
         "fileinfo" => {
             transaction.execute(
                 "INSERT OR IGNORE INTO fileinfo (flow_id, timestamp, extra_data) \
-                values(?1->>'flow_id', (UNIXEPOCH(SUBSTR(?1->>'timestamp', 1, 26), 'subsec') * 1000), json_extract(?1, '$.' || ?2))",
+                values(?1->>'flow_id', (UNIXEPOCH(SUBSTR(?1->>'timestamp', 1, 19))*1000000 + SUBSTR(?1->>'timestamp', 21, 6)), json_extract(?1, '$.' || ?2))",
                 (buf, event_type),
             )
         },
         _ => {
             transaction.execute(
                 "INSERT OR IGNORE INTO 'app-event' (flow_id, timestamp, app_proto, extra_data) \
-                values(?1->>'flow_id', (UNIXEPOCH(SUBSTR(?1->>'timestamp', 1, 26), 'subsec') * 1000), ?2, json_extract(?1, '$.' || ?2))",
+                values(?1->>'flow_id', (UNIXEPOCH(SUBSTR(?1->>'timestamp', 1, 19))*1000000 + SUBSTR(?1->>'timestamp', 21, 6)), ?2, json_extract(?1, '$.' || ?2))",
                 (buf, event_type),
             )
         }

diff --git a/suricata/suricata-eve-sqlite-output/src/schema.sql b/suricata/suricata-eve-sqlite-output/src/schema.sql
@@ -2,10 +2,11 @@
 -- SPDX-License-Identifier: GPL-2.0-or-later
 CREATE TABLE IF NOT EXISTS "flow" (
     id INTEGER NOT NULL PRIMARY KEY,
+    -- SQLite UNIXEPOCH currently has only millisecond precision using "subsec", which is not enough
     ts_start INTEGER GENERATED ALWAYS
-        AS (UNIXEPOCH(SUBSTR(extra_data->>'start', 1, 26), 'subsec') * 1000) STORED,
+        AS (UNIXEPOCH(SUBSTR(extra_data->>'start', 1, 19))*1000000 + SUBSTR(extra_data->>'start', 21, 6)) STORED,
     ts_end INTEGER GENERATED ALWAYS
-        AS (UNIXEPOCH(SUBSTR(extra_data->>'end', 1, 26), 'subsec') * 1000) STORED,
+        AS (UNIXEPOCH(SUBSTR(extra_data->>'end', 1, 19))*1000000 + SUBSTR(extra_data->>'end', 21, 6)) STORED,
     src_ip TEXT NOT NULL,
     src_port INTEGER,
     src_ipport TEXT GENERATED ALWAYS

diff --git a/webapp/main.py b/webapp/main.py
@@ -38,7 +38,7 @@ async def index(request):
 
 async def api_flow_list(request):
     # Parse GET arguments
-    ts_to = request.query_params.get("to", str(int(1e13)))
+    ts_to = request.query_params.get("to", str(int(1e16)))
     services = request.query_params.getlist("service")
     app_proto = request.query_params.get("app_proto")
     search = request.query_params.get("search")

diff --git a/webapp/static/js/flowdisplay.js b/webapp/static/js/flowdisplay.js
@@ -204,9 +204,9 @@ class FlowDisplay {
       second: 'numeric',
       fractionalSecondDigits: 3
     }
-    const dateStart = new Date(flow.flow.ts_start)
+    const dateStart = new Date(flow.flow.ts_start / 1000)
     const formatedDateStart = new Intl.DateTimeFormat(undefined, dateParams).format(dateStart)
-    const dateEnd = new Date(flow.flow.ts_end)
+    const dateEnd = new Date(flow.flow.ts_end / 1000)
     const formatedDateEnd = new Intl.DateTimeFormat(undefined, dateParams).format(dateEnd)
 
     // Change document title
@@ -220,7 +220,7 @@ class FlowDisplay {
     document.getElementById('display-flow-pcap').parentNode.classList.toggle('d-none', !flow.flow.pcap_filename)
     if (this.tickLength > 0) {
       document.getElementById('display-flow-tick').classList.remove('d-none')
-      const tick = ((flow.flow.ts_start / 1000 - this.startTs) / this.tickLength).toFixed(3)
+      const tick = ((flow.flow.ts_start / 1000000 - this.startTs) / this.tickLength).toFixed(3)
       document.querySelector('#display-flow-tick > a > span').textContent = tick
       document.querySelector('#display-flow-tick > a').dataset.ts = flow.flow.ts_start
     }

diff --git a/webapp/static/js/flowlist.js b/webapp/static/js/flowlist.js
@@ -118,7 +118,7 @@ class FlowList {
       const untilTick = Number(e.target.value)
       const url = new URL(document.location)
       if (untilTick) {
-        url.searchParams.set('to', Math.floor(((untilTick + 1) * (this.tickLength || 1) + this.startTs)) * 1000)
+        url.searchParams.set('to', Math.floor(((untilTick + 1) * (this.tickLength || 1) + this.startTs)) * 1000000)
       } else {
         url.searchParams.delete('to')
         e.target.value = null
@@ -236,6 +236,7 @@ class FlowList {
    * @returns Pretty string representation
    */
   pprintDelay (delay) {
+    delay = delay / 1000
     if (delay > 1000) {
       delay = delay / 1000
       return `${delay.toPrecision(3)} s`
@@ -348,7 +349,7 @@ class FlowList {
   async fillFlowsList (flows, tags) {
     const flowList = document.getElementById('flow-list')
     flows.forEach((flow) => {
-      const date = new Date(flow.ts_start)
+      const date = new Date(flow.ts_start / 1000)
       const startDate = new Intl.DateTimeFormat(
         undefined,
         { hour: 'numeric', minute: 'numeric', second: 'numeric', fractionalSecondDigits: 1 }
@@ -362,7 +363,7 @@ class FlowList {
 
       // Create tick element on new tick
       if (this.tickLength > 0) {
-        const tick = Math.floor((flow.ts_start / 1000 - this.startTs) / this.tickLength)
+        const tick = Math.floor((flow.ts_start / 1000000 - this.startTs) / this.tickLength)
         if (tick !== this.lastTick) {
           const tickEl = document.createElement('span')
           tickEl.classList.add('list-group-item', 'sticky-top', 'pt-3', 'pb-1', 'px-2', 'border-0', 'border-bottom', 'bg-light-subtle', 'text-center', 'fw-semibold')
@@ -459,7 +460,7 @@ class FlowList {
 
       // Update time filter state
       if (toTs) {
-        const toTick = (Number(toTs) / 1000 - this.startTs) / (this.tickLength || 1) - 1
+        const toTick = (Number(toTs) / 1000000 - this.startTs) / (this.tickLength || 1) - 1
         document.getElementById('filter-time-until').value = toTick
       }
       document.getElementById('filter-time-until').classList.toggle('is-active', toTs)