Skip to content

Commit

Permalink
Implement aggregated page for all alert families
Browse files Browse the repository at this point in the history
  • Loading branch information
@cardigliano
cardigliano committed Nov 12, 2024
1 parent 6c80655 commit 2aa46f9
Show file tree
Hide file tree
Showing 11 changed files with 235 additions and 211 deletions.
138 changes: 108 additions & 30 deletions scripts/lua/modules/alert_store/alert_store.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,16 +119,6 @@ end

-- ##############################################

-- @brief Converts interface IDs into their database type
-- Normal interface IDs are untouched.
-- The system interface ID is converted from -1 to (u_int16_t)-1 to handle everything as unsigned integer
function alert_store:_convert_ifid(ifid)
-- The system interface ID becomes (u_int16_t)-1
return 0xFFFF & tonumber(ifid)
end

-- ##############################################

-- @brief Check if the submitted fields are avalid (i.e., they are not injection attempts)
function alert_store:_valid_fields(fields)
local f = fields:split(",") or {fields}
Expand All @@ -147,23 +137,31 @@ end

-- ##############################################

-- Get the system ifid
function alert_store:get_system_ifid()
-- The System Interface has the id -1 and in u_int16_t is 65535
return 65535
-- @brief Converts interface IDs into their database type
-- Normal interface IDs are untouched.
-- The system interface ID is converted from -1 to (u_int16_t)-1 to handle everything as unsigned integer
function alert_store:_convert_ifid(ifid)
-- The system interface ID becomes (u_int16_t)-1
return 0xFFFF & tonumber(ifid)
end

-- ##############################################

-- @brief ifid
-- @brief Get current ifid
function alert_store:get_ifid()
local ifid = _GET["ifid"] or interface.getId()

local ifid = (_GET and _GET["ifid"]) or interface.getId()
ifid = tonumber(ifid)
return ifid
end

-- ##############################################

-- @bridf convert ifid to db notation of ifid
function alert_store:ifid_2_db_ifid(ifid)
-- The System Interface has the id -1 and in u_int16_t is 65535
if ifid == -1 then
ifid = self:get_system_ifid()
if ifid == getSystemInterfaceId() then
-- The System Interface has the id -1 and in u_int16_t is 65535
ifid = 65535
end

return ifid
Expand Down Expand Up @@ -918,8 +916,8 @@ function alert_store:add_order_by(sort_column, sort_order)
user = _SESSION["user"]
end

ntop.setCache(string.format(ALERT_SORTING_ORDER, self:get_ifid(), user, _GET["page"]), sort_order)
ntop.setCache(string.format(ALERT_SORTING_COLUMN, self:get_ifid(), user, _GET["page"]), sort_column)
ntop.setCache(string.format(ALERT_SORTING_ORDER, self:ifid_2_db_ifid(self:get_ifid()), user, _GET["page"]), sort_order)
ntop.setCache(string.format(ALERT_SORTING_COLUMN, self:ifid_2_db_ifid(self:get_ifid()), user, _GET["page"]), sort_column)
end

-- Creating the order by if not defined and valid
Expand All @@ -945,20 +943,98 @@ end

-- ##############################################

function alert_store:_build_insert_query(alert, write_table, engaged, rowid)
traceError(TRACE_NORMAL, TRACE_CONSOLE, "alert_store: _build_insert_query not defined for " .. self:get_family())
return ""
end

-- ##############################################

function alert_store:insert(alert)
traceError(TRACE_NORMAL, TRACE_CONSOLE, "alert_store:insert")
local write_table = self:get_write_table_name()

if write_table then

local extra_columns = ""
local extra_values = ""
if ntop.isClickHouseEnabled() then
extra_columns = "rowid, "
extra_values = "generateUUIDv4(), "
end

-- Note: alert.require_attention depends on HostAlert::autoAck() for
-- host alerts, for all other families the default is set to false in
-- OtherAlertableEntity::triggerAlert()

local alert_status = alert_consts.alert_status.historical.alert_status_id
if not alert.require_attention then
alert_status = alert_consts.alert_status.acknowledged.alert_status_id
end

local insert_stmt = self:_build_insert_query(alert, self:get_write_table_name(), alert_status, extra_columns, extra_values)
local ifid = ternary(self:get_ifid() == getSystemInterfaceId(), getSystemInterfaceId(), nil)
-- traceError(TRACE_NORMAL, TRACE_CONSOLE, insert_stmt)
return interface.alert_store_query(insert_stmt, ifid)
end

traceError(TRACE_NORMAL, TRACE_CONSOLE, "alert_store: write_table not defined for " .. self:get_family())
return false
end

-- ##############################################

function alert_store:insert_engaged(alert)
local engaged_write_table = self:get_engaged_write_table_name()

if engaged_write_table then

local extra_columns
local extra_values
if ntop.isClickHouseEnabled() then
extra_columns = "rowid, "
extra_values = string.format("concat('00000000-0000-0000-0000-', toFixedString(hex(%u), 12)), ", alert.rowid)
else
extra_columns = "rowid, "
extra_values = string.format("%u, ", alert.rowid)
end

local alert_status = alert_consts.alert_status.engaged.alert_status_id

local insert_stmt = self:_build_insert_query(alert, engaged_write_table, alert_status, extra_columns, extra_values)
local ifid = ternary(self:get_ifid() == getSystemInterfaceId(), getSystemInterfaceId(), nil)
-- traceError(TRACE_NORMAL, TRACE_CONSOLE, insert_stmt)
return interface.alert_store_query(insert_stmt, ifid)
end

traceError(TRACE_NORMAL, TRACE_CONSOLE, "alert_store: engaged_write_table not defined for " .. self:get_family())
return false
end

-- ##############################################

function alert_store:delete_engaged(alert)
local engaged_write_table = self:get_engaged_write_table_name()

if alert.rowid == nil then
-- rowid not defined, probably this comes from a store (not a release for an engaged)
-- traceError(TRACE_NORMAL, TRACE_CONSOLE, "alert_store: rowid not defined for " .. self:get_family())
return
end

if engaged_write_table then
local delete_stmt

if ntop.isClickHouseEnabled() then
delete_stmt = string.format("ALTER TABLE %s DELETE WHERE rowid = concat('00000000-0000-0000-0000-', toFixedString(hex(%u), 12))", engaged_write_table, alert.rowid)
else
delete_stmt = string.format("DELETE FROM %s WHERE rowid = %u", engaged_write_table, alert.rowid)
end
-- traceError(TRACE_NORMAL, TRACE_CONSOLE, delete_stmt)
local ifid = ternary(self:get_ifid() == getSystemInterfaceId(), getSystemInterfaceId(), nil)
return interface.alert_store_query(delete_stmt, ifid)
end

traceError(TRACE_NORMAL, TRACE_CONSOLE, "alert_store: engaged_write_table not defined for " .. self:get_family())
return false
end

Expand All @@ -970,14 +1046,16 @@ function alert_store:delete()
local where_clause = self:build_where_clause(true)

-- Prepare the final query
local q
local delete_stmt
if ntop.isClickHouseEnabled() then
q = string.format("ALTER TABLE `%s` DELETE WHERE %s ", table_name, where_clause)
delete_stmt = string.format("ALTER TABLE `%s` DELETE WHERE %s ", table_name, where_clause)
else
q = string.format("DELETE FROM `%s` WHERE %s ", table_name, where_clause)
delete_stmt = string.format("DELETE FROM `%s` WHERE %s ", table_name, where_clause)
end

local res = interface.alert_store_query(q)
local ifid = ternary(self:get_ifid() == getSystemInterfaceId(), getSystemInterfaceId(), nil)
local res = interface.alert_store_query(delete_stmt, ifid)

return res and table.len(res) == 0
end

Expand Down Expand Up @@ -1963,7 +2041,7 @@ function alert_store:get_earliest_available_epoch(status)
-- Add filters (only needed for the status, must ignore all other filters)
self:add_status_filter(status)
local cached_epoch_key =
string.format(EARLIEST_AVAILABLE_EPOCH_CACHE_KEY, self:get_ifid(), table_name, self._status)
string.format(EARLIEST_AVAILABLE_EPOCH_CACHE_KEY, self:ifid_2_db_ifid(self:get_ifid()), table_name, self._status)
local earliest = 0

-- Check if epoch has already been cached
Expand Down Expand Up @@ -2028,7 +2106,7 @@ end

-- @brief Add filters according to what is specified inside the REST API
function alert_store:add_request_filters(is_write)
local ifid = self:get_ifid()
local ifid = self:ifid_2_db_ifid(self:get_ifid())
local status = _GET["status"] -- Tab: engaged, require-attention (hitorical), all (any)
local epoch_begin = tonumber(_GET["epoch_begin"])
local epoch_end = tonumber(_GET["epoch_end"])
Expand All @@ -2043,7 +2121,7 @@ function alert_store:add_request_filters(is_write)
local description = _GET["description"]

-- Remember the score filter (see also alert_stats.lua)
local alert_score_cached = string.format(ALERT_SCORE_FILTER_KEY, self:get_ifid())
local alert_score_cached = string.format(ALERT_SCORE_FILTER_KEY, self:ifid_2_db_ifid(self:get_ifid()))

if isEmptyString(score) then
ntop.delCache(alert_score_cached)
Expand All @@ -2065,7 +2143,7 @@ function alert_store:add_request_filters(is_write)

if (ntop.isClickHouseEnabled()) then
-- Clickhouse db has the column 'interface_id', filter by that per interface
if ifid ~= self:get_system_ifid() then
if ifid ~= self:ifid_2_db_ifid(getSystemInterfaceId()) then
self:add_filter_condition_list('interface_id', ifid, 'number')
end
self:add_filter_condition_list('rowid', rowid, 'string')
Expand Down
3 changes: 2 additions & 1 deletion scripts/lua/modules/alert_store/all_alert_store.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,8 @@ function all_alert_store:select_request(filter, select_fields)
if ntop.isClickHouseEnabled() and not is_system_interface then
-- Add the system interface to show alerts both on the selected
-- interface and non-interface related.
self:add_filter_condition_list('interface_id', self:get_system_ifid(), 'number')
self:add_filter_condition_list('interface_id',
self:ifid_2_db_ifid(getSystemInterfaceId()), 'number')
end

-- Add limits and sort criteria
Expand Down
30 changes: 17 additions & 13 deletions scripts/lua/modules/alert_store/am_alert_store.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,20 +26,29 @@ local am_alert_store = classes.class(alert_store)
function am_alert_store:init(args)
self.super:init()

self._table_name = "active_monitoring_alerts"
if ntop.isClickHouseEnabled() then
self._table_name = "active_monitoring_alerts_view"
self._write_table_name = "active_monitoring_alerts"
self._engaged_write_table_name = "engaged_active_monitoring_alerts"
else
self._table_name = "active_monitoring_alerts_view"
self._write_table_name = "active_monitoring_alerts"
self._engaged_write_table_name = "mem_db.engaged_active_monitoring_alerts"
end

self._alert_entity = alert_entities.am_host
end

-- ##############################################

--@brief ifid
function am_alert_store:get_ifid()
return self:get_system_ifid()
return getSystemInterfaceId()
end

-- ##############################################

function am_alert_store:insert(alert)
function am_alert_store:_build_insert_query(alert, write_table, alert_status, extra_columns, extra_values)
local resolved_ip
local resolved_name
local measurement
Expand All @@ -59,21 +68,16 @@ function am_alert_store:insert(alert)
end
end

local extra_columns = ""
local extra_values = ""
if(ntop.isClickHouseEnabled()) then
extra_columns = "rowid, "
extra_values = "generateUUIDv4(), "
end

local insert_stmt = string.format("INSERT INTO %s "..
"(%salert_id, interface_id, tstamp, tstamp_end, severity, score, resolved_ip, resolved_name, "..
"(%salert_id, alert_status, require_attention, interface_id, tstamp, tstamp_end, severity, score, resolved_ip, resolved_name, "..
"measurement, measure_threshold, measure_value, json) "..
"VALUES (%s%u, %d, %u, %u, %u, %u, '%s', '%s', '%s', %u, %f, '%s'); ",
self._table_name,
"VALUES (%s%u, %u, %u, %d, %u, %u, %u, %u, '%s', '%s', '%s', %u, %f, '%s'); ",
write_table,
extra_columns,
extra_values,
alert.alert_id,
alert_status,
ternary(alert.require_attention, 1, 0),
self:_convert_ifid(getSystemInterfaceId()),
alert.tstamp,
alert.tstamp_end,
Expand Down
72 changes: 1 addition & 71 deletions scripts/lua/modules/alert_store/host_alert_store.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ end

-- ##############################################

function host_alert_store:_build_insert_query(alert, write_table, engaged, rowid)
function host_alert_store:_build_insert_query(alert, write_table, alert_status, extra_columns, extra_values)
local is_attacker = ternary(alert.is_attacker, 1, 0)
local is_victim = ternary(alert.is_victim, 1, 0)
local is_client = ternary(alert.is_client, 1, 0)
Expand All @@ -122,30 +122,6 @@ function host_alert_store:_build_insert_query(alert, write_table, engaged, rowid
end
end

local extra_columns = ""
local extra_values = ""

if ntop.isClickHouseEnabled() then
extra_columns = "rowid, "
if rowid then
extra_values = string.format("concat('00000000-0000-0000-0000-', toFixedString(hex(%u), 12)), ", rowid)
else
extra_values = "generateUUIDv4(), "
end
else
if rowid then
extra_columns = "rowid, "
extra_values = string.format("%u, ", rowid)
end
end

local alert_status = 0
if engaged then
alert_status = alert_consts.alert_status.engaged.alert_status_id
elseif not alert.require_attention then
alert_status = alert_consts.alert_status.acknowledged.alert_status_id
end

-- In case of some parameter empty, do not insert the alert
if not check_alert_params(alert) then
return
Expand Down Expand Up @@ -183,52 +159,6 @@ end

-- ##############################################

function host_alert_store:insert(alert)

-- traceError(TRACE_NORMAL, TRACE_CONSOLE, insert_stmt)

local insert_stmt = self:_build_insert_query(alert, self:get_write_table_name(), false, nil)
return interface.alert_store_query(insert_stmt)
end

-- ##############################################

function host_alert_store:insert_engaged(alert)

-- traceError(TRACE_NORMAL, TRACE_CONSOLE, "host_alert_store:insert_engaged")

local engaged_write_table = self:get_engaged_write_table_name()

if engaged_write_table then
local insert_stmt = self:_build_insert_query(alert, engaged_write_table, true, alert.rowid)
return interface.alert_store_query(insert_stmt)
end

return false
end

-- ##############################################

function host_alert_store:delete_engaged(alert)

-- traceError(TRACE_NORMAL, TRACE_CONSOLE, "host_alert_store:delete_engaged")

local engaged_write_table = self:get_engaged_write_table_name()

if engaged_write_table then
local q

if ntop.isClickHouseEnabled() then
q = string.format("ALTER TABLE %s DELETE WHERE rowid = concat('00000000-0000-0000-0000-', toFixedString(hex(%u), 12))", engaged_write_table, alert.rowid)
else
q = string.format("DELETE FROM %s WHERE rowid = %u", engaged_write_table, alert.rowid)
end
interface.alert_store_query(q)
end
end

-- ##############################################

-- @brief Performs a query for the top hosts by alert count
function host_alert_store:top_ip_historical()
-- Preserve all the filters currently set
Expand Down
Loading

0 comments on commit 2aa46f9

Please sign in to comment.