diff --git a/http_requests/search-terms-missing.http b/http_requests/search-terms-missing.http new file mode 100644 index 000000000..12f4a18ab --- /dev/null +++ b/http_requests/search-terms-missing.http @@ -0,0 +1,14 @@ +GET http://localhost:8080/windows_logs/_search +Content-Type: application/json + + +{ + "aggs": { + "tags": { + "terms": { + "field": "registry::key", + "missing": "n/a" + } + } + } +} \ No newline at end of file diff --git a/quesma/queryparser/aggregation_parser.go b/quesma/queryparser/aggregation_parser.go index 98fb65951..12c5b265e 100644 --- a/quesma/queryparser/aggregation_parser.go +++ b/quesma/queryparser/aggregation_parser.go @@ -737,8 +737,23 @@ func (cw *ClickhouseQueryTranslator) tryBucketAggregation(currentAggr *aggrQuery isEmptyGroupBy := len(currentAggr.SelectCommand.GroupBy) == 0 - currentAggr.SelectCommand.GroupBy = append(currentAggr.SelectCommand.GroupBy, cw.parseFieldField(terms, termsType)) - currentAggr.SelectCommand.Columns = append(currentAggr.SelectCommand.Columns, cw.parseFieldField(terms, termsType)) + // parse missing paremeter + var missingPlaceholder string + if m, ok := terms.(QueryMap); ok { + if m["missing"] != nil { + missingPlaceholder = m["missing"].(string) + } + } + + fieldExpression := cw.parseFieldField(terms, termsType) + + // apply missing placeholder if it is set + if missingPlaceholder != "" { + fieldExpression = model.NewFunction("COALESCE", fieldExpression, model.NewLiteral("'"+missingPlaceholder+"'")) + } + + currentAggr.SelectCommand.GroupBy = append(currentAggr.SelectCommand.GroupBy, fieldExpression) + currentAggr.SelectCommand.Columns = append(currentAggr.SelectCommand.Columns, fieldExpression) orderByAdded := false size := 10 @@ -751,6 +766,7 @@ func (cw *ClickhouseQueryTranslator) tryBucketAggregation(currentAggr *aggrQuery logger.WarnWithCtx(cw.Ctx).Msgf("size is not an float64, but %T, value: %v. Using default", sizeRaw, sizeRaw) } } + } currentAggr.SelectCommand.Limit = size currentAggr.SelectCommand.OrderBy = append(currentAggr.SelectCommand.OrderBy, model.NewSortByCountColumn(model.DescOrder)) @@ -758,7 +774,7 @@ func (cw *ClickhouseQueryTranslator) tryBucketAggregation(currentAggr *aggrQuery } delete(queryMap, termsType) if !orderByAdded { - currentAggr.SelectCommand.OrderBy = append(currentAggr.SelectCommand.OrderBy, model.NewOrderByExprWithoutOrder(cw.parseFieldField(terms, termsType))) + currentAggr.SelectCommand.OrderBy = append(currentAggr.SelectCommand.OrderBy, model.NewOrderByExprWithoutOrder(fieldExpression)) } return success, 1, nil /* will remove later diff --git a/quesma/testdata/aggregation_requests.go b/quesma/testdata/aggregation_requests.go index 1d446d2b0..491abc2fa 100644 --- a/quesma/testdata/aggregation_requests.go +++ b/quesma/testdata/aggregation_requests.go @@ -2414,16 +2414,16 @@ var AggregationTests = []AggregationTestCase{ }, }, ExpectedSQLs: []string{ - `SELECT "event.dataset", toInt64(toUnixTimestamp64Milli("@timestamp") / 60000), count() ` + + `SELECT COALESCE("event.dataset",'unknown'), toInt64(toUnixTimestamp64Milli("@timestamp") / 60000), count() ` + `FROM ` + QuotedTableName + ` ` + `WHERE ("@timestamp">parseDateTime64BestEffort('2024-01-25T14:53:59.033Z') AND "@timestamp"<=parseDateTime64BestEffort('2024-01-25T15:08:59.033Z')) ` + - `GROUP BY "event.dataset", toInt64(toUnixTimestamp64Milli("@timestamp") / 60000) ` + - `ORDER BY "event.dataset", toInt64(toUnixTimestamp64Milli("@timestamp") / 60000)`, - `SELECT "event.dataset", count() FROM ` + QuotedTableName + ` ` + + `GROUP BY COALESCE("event.dataset",'unknown'), toInt64(toUnixTimestamp64Milli("@timestamp") / 60000) ` + + `ORDER BY COALESCE("event.dataset",'unknown'), toInt64(toUnixTimestamp64Milli("@timestamp") / 60000)`, + `SELECT COALESCE("event.dataset",'unknown'), count() FROM ` + QuotedTableName + ` ` + `WHERE ("@timestamp">parseDateTime64BestEffort('2024-01-25T14:53:59.033Z') ` + `AND "@timestamp"<=parseDateTime64BestEffort('2024-01-25T15:08:59.033Z')) ` + - `GROUP BY "event.dataset" ` + - `ORDER BY "event.dataset"`, + `GROUP BY COALESCE("event.dataset",'unknown') ` + + `ORDER BY COALESCE("event.dataset",'unknown')`, }, }, { // [14], "old" test, also can be found in testdata/requests.go TestAsyncSearch[5] diff --git a/quesma/testdata/requests.go b/quesma/testdata/requests.go index f1825dadd..c07961fe1 100644 --- a/quesma/testdata/requests.go +++ b/quesma/testdata/requests.go @@ -727,18 +727,18 @@ var TestsAsyncSearch = []AsyncSearchTestCase{ "no comment yet", model.SearchQueryInfo{Typ: model.Normal}, []string{ - `SELECT "event.dataset", ` + + `SELECT COALESCE("event.dataset",'unknown'), ` + groupBySQL("@timestamp", clickhouse.DateTime64, time.Minute) + `, count() FROM ` + QuotedTableName + ` ` + `WHERE ("@timestamp".*parseDateTime64BestEffort('2024-01-25T1.:..:59.033Z') ` + `AND "@timestamp".*parseDateTime64BestEffort('2024-01-25T1.:..:59.033Z')) ` + - `GROUP BY "event.dataset", ` + groupBySQL("@timestamp", clickhouse.DateTime64, time.Minute) + ` ` + - `ORDER BY "event.dataset", ` + groupBySQL("@timestamp", clickhouse.DateTime64, time.Minute), - `SELECT "event.dataset", count() FROM ` + QuotedTableName + ` ` + + `GROUP BY COALESCE("event.dataset",'unknown'), ` + groupBySQL("@timestamp", clickhouse.DateTime64, time.Minute) + ` ` + + `ORDER BY COALESCE("event.dataset",'unknown'), ` + groupBySQL("@timestamp", clickhouse.DateTime64, time.Minute), + `SELECT COALESCE("event.dataset",'unknown'), count() FROM ` + QuotedTableName + ` ` + `WHERE ("@timestamp".*parseDateTime64BestEffort('2024-01-25T1.:..:59.033Z') ` + `AND "@timestamp".*parseDateTime64BestEffort('2024-01-25T1.:..:59.033Z')) ` + - `GROUP BY "event.dataset" ` + - `ORDER BY "event.dataset"`, + `GROUP BY COALESCE("event.dataset",'unknown') ` + + `ORDER BY COALESCE("event.dataset",'unknown')`, }, true, },