From 0fd5a5a289acd5a14a2a42f245b0aed628a1eca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Strzali=C5=84ski?= Date: Fri, 17 May 2024 12:54:35 +0200 Subject: [PATCH] Reroute Kibana Alerts searches to Elastic (#132) We need to reroute `/_search` request related to Kibana Alerts to Elastic. This is 3rd attempt to implement this bypass. This attempt has a minimal impact on the search pipeline. --- http_requests/kibana-alert.http | 64 ++++++++++++ quesma/quesma/matchers.go | 59 +++++++++++ quesma/quesma/matchers_test.go | 167 ++++++++++++++++++++++++++++++++ quesma/quesma/router.go | 5 +- 4 files changed, 292 insertions(+), 3 deletions(-) create mode 100644 http_requests/kibana-alert.http create mode 100644 quesma/quesma/matchers_test.go diff --git a/http_requests/kibana-alert.http b/http_requests/kibana-alert.http new file mode 100644 index 000000000..88b35c5a8 --- /dev/null +++ b/http_requests/kibana-alert.http @@ -0,0 +1,64 @@ +POST http://localhost:8080/_search +Content-Type: application/json + +{ + "aggs": { + "endpoint_alert_count": { + "cardinality": { + "field": "event.id" + } + } + }, + "pit": { + "id": "gcSHBAEvLmludGVybmFsLmFsZXJ0cy1zZWN1cml0eS5hbGVydHMtZGVmYXVsdC0wMDAwMDEWRWdvdFQwblRUN0tNaFk4SWc3TDRSQQAWMEdVOVNnVk1TV0t3ckRxbUpkb3BzZwAAAAAAAASdvBZGQXQwWTUyTVRKQ29zaDJ1elRhWFR3AAEWRWdvdFQwblRUN0tNaFk4SWc3TDRSQQAA" + }, + "query": { + "bool": { + "filter": [ + { + "bool": { + "should": [ + { + "match_phrase": { + "event.module": "endpoint" + } + } + ] + } + }, + { + "bool": { + "should": [ + { + "match_phrase": { + "kibana.alert.rule.parameters.immutable": "true" + } + } + ] + } + }, + { + "range": { + "@timestamp": { + "gte": "now-3h", + "lte": "now" + } + } + } + ] + } + }, + "size": 1000, + "sort": [ + { + "@timestamp": { + "format": "strict_date_optional_time_nanos", + "order": "asc" + } + }, + { + "_shard_doc": "desc" + } + ], + "track_total_hits": false +} \ No newline at end of file diff --git a/quesma/quesma/matchers.go b/quesma/quesma/matchers.go index b6e757088..b0529a2ae 100644 --- a/quesma/quesma/matchers.go +++ b/quesma/quesma/matchers.go @@ -1,6 +1,7 @@ package quesma import ( + "encoding/json" "mitmproxy/quesma/elasticsearch" "mitmproxy/quesma/logger" "mitmproxy/quesma/quesma/config" @@ -74,3 +75,61 @@ func matchedAgainstPattern(configuration config.QuesmaConfiguration) mux.MatchPr } } } + +// Returns false if the body contains a Kibana alert related field. +func matchAgainstKibanaAlerts() mux.MatchPredicate { + return func(m map[string]string, body string) bool { + + if body == "" { + return true + } + + // https://www.elastic.co/guide/en/security/current/alert-schema.html + + var query map[string]interface{} + err := json.Unmarshal([]byte(body), &query) + if err != nil { + logger.Warn().Msgf("error parsing json %v", err) + return true + } + + var findKibanaAlertField func(node interface{}) bool + + findKibanaAlertField = func(node interface{}) bool { + + if node == nil { + return false + } + + switch nodeValue := node.(type) { + + case map[string]interface{}: + + for k, v := range nodeValue { + + if strings.Contains(k, "kibana.alert.") { + return true + } + + if findKibanaAlertField(v) { + return true + } + } + + case []interface{}: + + for _, i := range nodeValue { + if findKibanaAlertField(i) { + return true + } + } + + } + return false + } + + q := query["query"].(map[string]interface{}) + + return !findKibanaAlertField(q) + } +} diff --git a/quesma/quesma/matchers_test.go b/quesma/quesma/matchers_test.go new file mode 100644 index 000000000..2aab01367 --- /dev/null +++ b/quesma/quesma/matchers_test.go @@ -0,0 +1,167 @@ +package quesma + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +const kibanaAlerts = `{ + "aggs": { + "endpoint_alert_count": { + "cardinality": { + "field": "event.id" + } + } + }, + "pit": { + "id": "gcSHBAEvLmludGVybmFsLmFsZXJ0cy1zZWN1cml0eS5hbGVydHMtZGVmYXVsdC0wMDAwMDEWRWdvdFQwblRUN0tNaFk4SWc3TDRSQQAWMEdVOVNnVk1TV0t3ckRxbUpkb3BzZwAAAAAAAASdvBZGQXQwWTUyTVRKQ29zaDJ1elRhWFR3AAEWRWdvdFQwblRUN0tNaFk4SWc3TDRSQQAA" + }, + "query": { + "bool": { + "filter": [ + { + "bool": { + "should": [ + { + "match_phrase": { + "event.module": "endpoint" + } + } + ] + } + }, + { + "bool": { + "should": [ + { + "match_phrase": { + "kibana.alert.rule.parameters.immutable": "true" + } + } + ] + } + }, + { + "range": { + "@timestamp": { + "gte": "now-3h", + "lte": "now" + } + } + } + ] + } + }, + "size": 1000, + "sort": [ + { + "@timestamp": { + "format": "strict_date_optional_time_nanos", + "order": "asc" + } + }, + { + "_shard_doc": "desc" + } + ], + "track_total_hits": false +} +` + +const nonKibanaAlerts = ` +{ + "_source": false, + "fields": [ + { + "field": "*", + "include_unmapped": "true" + }, + { + "field": "@timestamp", + "format": "strict_date_optional_time" + } + ], + "highlight": { + "fields": { + "*": {} + }, + "fragment_size": 2147483647, + "post_tags": [ + "@/kibana-highlighted-field@" + ], + "pre_tags": [ + "@kibana-highlighted-field@" + ] + }, + "query": { + "bool": { + "filter": [ + { + "multi_match": { + "lenient": true, + "query": "user", + "type": "best_fields" + } + }, + { + "range": { + "@timestamp": { + "format": "strict_date_optional_time", + "gte": "2022-01-23T14:43:19.481Z", + "lte": "2025-01-23T14:58:19.481Z" + } + } + } + ], + "must": [], + "must_not": [], + "should": [] + } + }, + "runtime_mappings": {}, + "script_fields": {}, + "size": 500, + "sort": [ + { + "@timestamp": { + "format": "strict_date_optional_time", + "order": "desc", + "unmapped_type": "boolean" + } + }, + { + "_doc": { + "order": "desc", + "unmapped_type": "boolean" + } + } + ], + "stored_fields": [ + "*" + ], + "track_total_hits": false, + "version": true +} + +` + +func TestMatchAgainstKibanaAlerts(t *testing.T) { + + tests := []struct { + name string + body string + expected bool + }{ + {"kibana alerts", kibanaAlerts, false}, + {"non kibana alerts", nonKibanaAlerts, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := matchAgainstKibanaAlerts()(nil, tt.body) + assert.Equal(t, tt.expected, actual) + }) + + } + +} diff --git a/quesma/quesma/router.go b/quesma/quesma/router.go index 62de2e355..be9700f0c 100644 --- a/quesma/quesma/router.go +++ b/quesma/quesma/router.go @@ -142,9 +142,8 @@ func configureRouter(cfg config.QuesmaConfiguration, lm *clickhouse.LogManager, } }) - router.RegisterPathMatcher(routes.GlobalSearchPath, []string{"GET", "POST"}, func(_ map[string]string, _ string) bool { - return true // for now, always route to Quesma, in the near future: combine results from both sources - }, func(ctx context.Context, body string, _ string, params map[string]string, _ http.Header, _ url.Values) (*mux.Result, error) { + router.RegisterPathMatcher(routes.GlobalSearchPath, []string{"GET", "POST"}, matchAgainstKibanaAlerts(), func(ctx context.Context, body string, _ string, params map[string]string, _ http.Header, _ url.Values) (*mux.Result, error) { + responseBody, err := queryRunner.handleSearch(ctx, "*", []byte(body)) if err != nil { if errors.Is(errIndexNotExists, err) {