Terms aggregation - add support to 'missing' parameter (#482)

It adds support to 'missing' parameter. https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-terms-aggregation.html#_missing_value_5 @trzysiek Please take a look.
QuesmaOrg · Jul 8, 2024 · e5c4000 · e5c4000
1 parent 0ddad27
commit e5c4000
Show file tree

Hide file tree

Showing 4 changed files with 45 additions and 15 deletions.
diff --git a/http_requests/search-terms-missing.http b/http_requests/search-terms-missing.http
@@ -0,0 +1,14 @@
+GET http://localhost:8080/windows_logs/_search
+Content-Type: application/json
+
+
+{
+  "aggs": {
+    "tags": {
+      "terms": {
+        "field": "registry::key",
+        "missing": "n/a"
+      }
+    }
+  }
+}
diff --git a/quesma/queryparser/aggregation_parser.go b/quesma/queryparser/aggregation_parser.go
@@ -737,8 +737,23 @@ func (cw *ClickhouseQueryTranslator) tryBucketAggregation(currentAggr *aggrQuery
 
 			isEmptyGroupBy := len(currentAggr.SelectCommand.GroupBy) == 0
 
-			currentAggr.SelectCommand.GroupBy = append(currentAggr.SelectCommand.GroupBy, cw.parseFieldField(terms, termsType))
-			currentAggr.SelectCommand.Columns = append(currentAggr.SelectCommand.Columns, cw.parseFieldField(terms, termsType))
+			// parse missing paremeter
+			var missingPlaceholder string
+			if m, ok := terms.(QueryMap); ok {
+				if m["missing"] != nil {
+					missingPlaceholder = m["missing"].(string)
+				}
+			}
+
+			fieldExpression := cw.parseFieldField(terms, termsType)
+
+			// apply missing placeholder if it is set
+			if missingPlaceholder != "" {
+				fieldExpression = model.NewFunction("COALESCE", fieldExpression, model.NewLiteral("'"+missingPlaceholder+"'"))
+			}
+
+			currentAggr.SelectCommand.GroupBy = append(currentAggr.SelectCommand.GroupBy, fieldExpression)
+			currentAggr.SelectCommand.Columns = append(currentAggr.SelectCommand.Columns, fieldExpression)
 
 			orderByAdded := false
 			size := 10
@@ -751,14 +766,15 @@ func (cw *ClickhouseQueryTranslator) tryBucketAggregation(currentAggr *aggrQuery
 							logger.WarnWithCtx(cw.Ctx).Msgf("size is not an float64, but %T, value: %v. Using default", sizeRaw, sizeRaw)
 						}
 					}
+
 				}
 				currentAggr.SelectCommand.Limit = size
 				currentAggr.SelectCommand.OrderBy = append(currentAggr.SelectCommand.OrderBy, model.NewSortByCountColumn(model.DescOrder))
 				orderByAdded = true
 			}
 			delete(queryMap, termsType)
 			if !orderByAdded {
-				currentAggr.SelectCommand.OrderBy = append(currentAggr.SelectCommand.OrderBy, model.NewOrderByExprWithoutOrder(cw.parseFieldField(terms, termsType)))
+				currentAggr.SelectCommand.OrderBy = append(currentAggr.SelectCommand.OrderBy, model.NewOrderByExprWithoutOrder(fieldExpression))
 			}
 			return success, 1, nil
 			/* will remove later

diff --git a/quesma/testdata/aggregation_requests.go b/quesma/testdata/aggregation_requests.go
@@ -2414,16 +2414,16 @@ var AggregationTests = []AggregationTestCase{
 			},
 		},
 		ExpectedSQLs: []string{
-			`SELECT "event.dataset", toInt64(toUnixTimestamp64Milli("@timestamp") / 60000), count() ` +
+			`SELECT COALESCE("event.dataset",'unknown'), toInt64(toUnixTimestamp64Milli("@timestamp") / 60000), count() ` +
 				`FROM ` + QuotedTableName + ` ` +
 				`WHERE ("@timestamp">parseDateTime64BestEffort('2024-01-25T14:53:59.033Z') AND "@timestamp"<=parseDateTime64BestEffort('2024-01-25T15:08:59.033Z')) ` +
-				`GROUP BY "event.dataset", toInt64(toUnixTimestamp64Milli("@timestamp") / 60000) ` +
-				`ORDER BY "event.dataset", toInt64(toUnixTimestamp64Milli("@timestamp") / 60000)`,
-			`SELECT "event.dataset", count() FROM ` + QuotedTableName + ` ` +
+				`GROUP BY COALESCE("event.dataset",'unknown'), toInt64(toUnixTimestamp64Milli("@timestamp") / 60000) ` +
+				`ORDER BY COALESCE("event.dataset",'unknown'), toInt64(toUnixTimestamp64Milli("@timestamp") / 60000)`,
+			`SELECT COALESCE("event.dataset",'unknown'), count() FROM ` + QuotedTableName + ` ` +
 				`WHERE ("@timestamp">parseDateTime64BestEffort('2024-01-25T14:53:59.033Z') ` +
 				`AND "@timestamp"<=parseDateTime64BestEffort('2024-01-25T15:08:59.033Z')) ` +
-				`GROUP BY "event.dataset" ` +
-				`ORDER BY "event.dataset"`,
+				`GROUP BY COALESCE("event.dataset",'unknown') ` +
+				`ORDER BY COALESCE("event.dataset",'unknown')`,
 		},
 	},
 	{ // [14], "old" test, also can be found in testdata/requests.go TestAsyncSearch[5]

diff --git a/quesma/testdata/requests.go b/quesma/testdata/requests.go
@@ -727,18 +727,18 @@ var TestsAsyncSearch = []AsyncSearchTestCase{
 		"no comment yet",
 		model.SearchQueryInfo{Typ: model.Normal},
 		[]string{
-			`SELECT "event.dataset", ` +
+			`SELECT COALESCE("event.dataset",'unknown'), ` +
 				groupBySQL("@timestamp", clickhouse.DateTime64, time.Minute) +
 				`, count() FROM ` + QuotedTableName + ` ` +
 				`WHERE ("@timestamp".*parseDateTime64BestEffort('2024-01-25T1.:..:59.033Z') ` +
 				`AND "@timestamp".*parseDateTime64BestEffort('2024-01-25T1.:..:59.033Z')) ` +
-				`GROUP BY "event.dataset", ` + groupBySQL("@timestamp", clickhouse.DateTime64, time.Minute) + ` ` +
-				`ORDER BY "event.dataset", ` + groupBySQL("@timestamp", clickhouse.DateTime64, time.Minute),
-			`SELECT "event.dataset", count() FROM ` + QuotedTableName + ` ` +
+				`GROUP BY COALESCE("event.dataset",'unknown'), ` + groupBySQL("@timestamp", clickhouse.DateTime64, time.Minute) + ` ` +
+				`ORDER BY COALESCE("event.dataset",'unknown'), ` + groupBySQL("@timestamp", clickhouse.DateTime64, time.Minute),
+			`SELECT COALESCE("event.dataset",'unknown'), count() FROM ` + QuotedTableName + ` ` +
 				`WHERE ("@timestamp".*parseDateTime64BestEffort('2024-01-25T1.:..:59.033Z') ` +
 				`AND "@timestamp".*parseDateTime64BestEffort('2024-01-25T1.:..:59.033Z')) ` +
-				`GROUP BY "event.dataset" ` +
-				`ORDER BY "event.dataset"`,
+				`GROUP BY COALESCE("event.dataset",'unknown') ` +
+				`ORDER BY COALESCE("event.dataset",'unknown')`,
 		},
 		true,
 	},