Made tests pass

quesma/eql/query_translator.go

@@ -30,7 +30,7 @@ func (cw *ClickhouseEQLQueryTranslator) applySizeLimit(size int) int {
 	return size
 }
 
-func (cw *ClickhouseEQLQueryTranslator) BuildNRowsQuery(fieldName string, simpleQuery queryparser.SimpleQuery, limit int) *model.Query {
+func (cw *ClickhouseEQLQueryTranslator) BuildNRowsQuery(fieldName string, simpleQuery model.SimpleQuery, limit int) *model.Query {
 	suffixClauses := make([]string, 0)
 	if len(simpleQuery.SortFields) > 0 {
 		suffixClauses = append(suffixClauses, "ORDER BY "+strings.Join(simpleQuery.SortFields, ", "))
@@ -88,13 +88,13 @@ func (cw *ClickhouseEQLQueryTranslator) MakeSearchResponse(ResultSet []model.Que
 	}, nil
 }
 
-func (cw *ClickhouseEQLQueryTranslator) ParseQuery(queryAsJson string) (query queryparser.SimpleQuery, searchQueryInfo model.SearchQueryInfo, highlighter model.Highlighter, err error) {
+func (cw *ClickhouseEQLQueryTranslator) ParseQuery(queryAsJson string) (query model.SimpleQuery, searchQueryInfo model.SearchQueryInfo, highlighter model.Highlighter, err error) {
 
 	// no highlighting here
 	highlighter = queryparser.NewEmptyHighlighter()
 
 	searchQueryInfo.Typ = model.ListAllFields
-	query.Sql = queryparser.Statement{}
+	query.Sql = model.Statement{}
 
 	queryAsMap := make(map[string]interface{})
 	err = json.Unmarshal([]byte(queryAsJson), &queryAsMap)
@@ -158,7 +158,7 @@ func (cw *ClickhouseEQLQueryTranslator) MakeResponseAggregation(aggregations []m
 	panic("EQL does not support aggregations")
 }
 
-func (cw *ClickhouseEQLQueryTranslator) BuildFacetsQuery(fieldName string, simpleQuery queryparser.SimpleQuery, limit int) *model.Query {
+func (cw *ClickhouseEQLQueryTranslator) BuildFacetsQuery(fieldName string, simpleQuery model.SimpleQuery, limit int) *model.Query {
 	panic("EQL does not support facets")
 }
 

quesma/model/simple_query.go

@@ -20,17 +20,6 @@ func NewSimpleQueryWithFieldName(sql Statement, canParse bool, fieldName string)
 	return SimpleQuery{Sql: sql, CanParse: canParse, FieldName: fieldName}
 }
 
-func (sq *SimpleQuery) CombineWheresWith(ctx context.Context, sq2 SimpleQuery) {
-	sq.Sql = And([]Statement{sq.Sql, sq2.Sql})
-	sq.CanParse = sq.CanParse && sq2.CanParse
-	if len(sq.FieldName) > 0 && len(sq2.FieldName) > 0 && sq.FieldName != sq2.FieldName {
-		logger.WarnWithCtx(ctx).Msgf("combining 2 where clauses with different field names: %s, %s, where queries: %v %v", sq.FieldName, sq2.FieldName, sq, sq2)
-	}
-	if len(sq.FieldName) == 0 && len(sq2.FieldName) > 0 {
-		sq.FieldName = sq2.FieldName
-	}
-}
-
 type Statement struct {
 	Stmt       string
 	IsCompound bool // "a" -> not compound, "a AND b" -> compound. Used to not make unnecessary brackets (not always, but usually)
@@ -60,17 +49,6 @@ func Or(orStmts []Statement) Statement {
 	return combineStatements(orStmts, "OR")
 }
 
-func FilterNonEmpty(slice []Statement) []Statement {
-	i := 0
-	for _, el := range slice {
-		if len(el.Stmt) > 0 {
-			slice[i] = el
-			i++
-		}
-	}
-	return slice[:i]
-}
-
 // sep = "AND" or "OR"
 func combineStatements(stmts []Statement, sep string) Statement {
 	stmts = FilterNonEmpty(stmts)
@@ -95,6 +73,33 @@ func combineStatements(stmts []Statement, sep string) Statement {
 	return NewSimpleStatement("")
 }
 
+func CombineWheres(ctx context.Context, where1, where2 SimpleQuery) SimpleQuery {
+	combined := SimpleQuery{
+		Sql:      And([]Statement{where1.Sql, where2.Sql}),
+		CanParse: where1.CanParse && where2.CanParse,
+	}
+	if len(where1.FieldName) > 0 && len(where2.FieldName) > 0 && where1.FieldName != where2.FieldName {
+		logger.WarnWithCtx(ctx).Msgf("combining 2 where clauses with different field names: %s, %s, where queries: %v %v", where1.FieldName, where2.FieldName, where1, where2)
+	}
+	if len(where1.FieldName) > 0 {
+		combined.FieldName = where1.FieldName
+	} else {
+		combined.FieldName = where2.FieldName
+	}
+	return combined
+}
+
+func FilterNonEmpty(slice []Statement) []Statement {
+	i := 0
+	for _, el := range slice {
+		if len(el.Stmt) > 0 {
+			slice[i] = el
+			i++
+		}
+	}
+	return slice[:i]
+}
+
 // used to combine statements with AND/OR
 // [a, b, a AND b] ==> ["a", "b", "(a AND b)"]
 func quoteWithBracketsIfCompound(slice []Statement) []Statement {

quesma/queryparser/aggregation_parser.go

@@ -351,7 +351,7 @@ func (cw *ClickhouseQueryTranslator) parseAggregation(currentAggr *aggrQueryBuil
 		if filter, ok := filterRaw.(QueryMap); ok {
 			filterOnThisLevel = true
 			currentAggr.Type = metrics_aggregations.NewCount(cw.Ctx)
-			currentAggr.whereBuilder.CombineWheresWith(cw.Ctx, cw.parseQueryMap(filter))
+			currentAggr.whereBuilder = model.CombineWheres(cw.Ctx, currentAggr.whereBuilder, cw.parseQueryMap(filter))
 			*resultAccumulator = append(*resultAccumulator, currentAggr.buildCountAggregation(metadata))
 		} else {
 			logger.WarnWithCtx(cw.Ctx).Msgf("filter is not a map, but %T, value: %v. Skipping", filterRaw, filterRaw)
@@ -403,7 +403,7 @@ func (cw *ClickhouseQueryTranslator) parseAggregation(currentAggr *aggrQueryBuil
 	}
 	delete(queryMap, "aggs") // no-op if no "aggs"
 
-	if bucketAggrPresent && !isRange {
+	if bucketAggrPresent && !aggsHandledSeparately {
 		// range aggregation has separate, optimized handling
 		*resultAccumulator = append(*resultAccumulator, currentAggr.buildBucketAggregation(metadata))
 	}
@@ -704,13 +704,17 @@ func (cw *ClickhouseQueryTranslator) tryBucketAggregation(currentAggr *aggrQuery
 	}
 	if boolRaw, ok := queryMap["bool"]; ok {
 		if Bool, ok := boolRaw.(QueryMap); ok {
-			currentAggr.whereBuilder.CombineWheresWith(cw.Ctx, cw.parseBool(Bool))
+			currentAggr.whereBuilder = model.CombineWheres(cw.Ctx, currentAggr.whereBuilder, cw.parseBool(Bool))
 		} else {
 			logger.WarnWithCtx(cw.Ctx).Msgf("bool is not a map, but %T, value: %v. Skipping", boolRaw, boolRaw)
 		}
 		delete(queryMap, "bool")
 		return
 	}
+	if isFilters, aggregation := cw.parseFilters(queryMap); isFilters {
+		currentAggr.Type = aggregation
+		return
+	}
 	success = false
 	return
 }

quesma/queryparser/filters_aggregation.go

@@ -1,6 +1,7 @@
 package queryparser
 
 import (
+	"fmt"
 	"github.com/barkimedes/go-deepcopy"
 	"mitmproxy/quesma/logger"
 	"mitmproxy/quesma/model"
@@ -18,8 +19,20 @@ func (cw *ClickhouseQueryTranslator) parseFilters(queryMap QueryMap) (success bo
 		logger.WarnWithCtx(cw.Ctx).Msgf("filters is not a map, but %T, value: %v. Using empty.", filtersRaw, filtersRaw)
 		return false, bucket_aggregations.NewFiltersEmpty(cw.Ctx)
 	}
-	filters := make([]bucket_aggregations.Filter, 0, len(filtersMap))
-	for name, filter := range filtersMap {
+	nested, exists := filtersMap["filters"]
+	if !exists {
+		logger.WarnWithCtx(cw.Ctx).Msgf("filters is not a map, but %T, value: %v. Skipping filters.", filtersRaw, filtersRaw)
+		return false, bucket_aggregations.NewFiltersEmpty(cw.Ctx)
+	}
+	nestedMap, ok := nested.(QueryMap)
+	if !ok {
+		logger.WarnWithCtx(cw.Ctx).Msgf("filters is not a map, but %T, value: %v. Skipping filters.", nested, nested)
+		return false, bucket_aggregations.NewFiltersEmpty(cw.Ctx)
+	}
+
+	filters := make([]bucket_aggregations.Filter, 0, len(nestedMap))
+	for name, filter := range nestedMap {
+		fmt.Println(name)
 		filterMap, ok := filter.(QueryMap)
 		if !ok {
 			logger.WarnWithCtx(cw.Ctx).Msgf("filter is not a map, but %T, value: %v. Skipping.", filter, filter)
@@ -32,13 +45,15 @@ func (cw *ClickhouseQueryTranslator) parseFilters(queryMap QueryMap) (success bo
 
 func (cw *ClickhouseQueryTranslator) processFiltersAggregation(aggrBuilder *aggrQueryBuilder,
 	aggr bucket_aggregations.Filters, queryMap QueryMap, resultAccumulator *[]model.Query) error {
+	whereBeforeNesting := aggrBuilder.whereBuilder
+	aggrBuilder.Aggregators[len(aggrBuilder.Aggregators)-1].Filters = true
 	for _, filter := range aggr.Filters {
 		// newBuilder := aggrBuilder.clone()
 		// newBuilder.Type = bucket_aggregations.NewFilters(cw.Ctx, []bucket_aggregations.Filter{filter})
 		// newBuilder.whereBuilder.CombineWheresWith(filter.Sql)
 		// newBuilder.Aggregators = append(aggrBuilder.Aggregators, model.NewAggregatorEmpty(filter.Name))
 		aggrBuilder.Type = aggr
-		aggrBuilder.whereBuilder.CombineWheresWith(cw.Ctx, filter.Sql)
+		aggrBuilder.whereBuilder = model.CombineWheres(cw.Ctx, aggrBuilder.whereBuilder, filter.Sql)
 		aggrBuilder.Aggregators = append(aggrBuilder.Aggregators, model.NewAggregatorEmpty(filter.Name))
 		*resultAccumulator = append(*resultAccumulator, aggrBuilder.buildBucketAggregation(nil)) // nil for now, will be changed
 		if aggs, ok := queryMap["aggs"].(QueryMap); ok {
@@ -53,6 +68,8 @@ func (cw *ClickhouseQueryTranslator) processFiltersAggregation(aggrBuilder *aggr
 				logger.ErrorWithCtx(cw.Ctx).Msgf("deepcopy 'aggs' map error: %v. Skipping. aggs: %v", errAggs, aggs)
 			}
 		}
+		aggrBuilder.Aggregators = aggrBuilder.Aggregators[:len(aggrBuilder.Aggregators)-1]
+		aggrBuilder.whereBuilder = whereBeforeNesting
 	}
 	delete(queryMap, "filters")
 	return nil

quesma/queryparser/range_aggregation.go

@@ -85,7 +85,8 @@ func (cw *ClickhouseQueryTranslator) processRangeAggregation(currentAggr *aggrQu
 	// Range aggregation with subaggregations should be a quite rare case, so I'm leaving that for later.
 	whereBeforeNesting := currentAggr.whereBuilder
 	for _, interval := range Range.Intervals {
-		currentAggr.whereBuilder.CombineWheresWith(cw.Ctx,
+		currentAggr.whereBuilder = model.CombineWheres(
+			cw.Ctx, currentAggr.whereBuilder,
 			model.NewSimpleQuery(model.NewSimpleStatement(interval.ToWhereClause(Range.QuotedFieldName)), true),
 		)
 		currentAggr.Aggregators = append(currentAggr.Aggregators, model.NewAggregatorEmpty(interval.String()))

quesma/quesma/query_translator.go

@@ -17,12 +17,12 @@ import (
 // 2. ClickhouseEQLQueryTranslator (implements only a subset of methods)
 
 type IQueryTranslator interface {
-	ParseQuery(queryAsJson string) (queryparser.SimpleQuery, model.SearchQueryInfo, model.Highlighter, error)
+	ParseQuery(queryAsJson string) (model.SimpleQuery, model.SearchQueryInfo, model.Highlighter, error)
 	ParseAggregationJson(aggregationJson string) ([]model.Query, error)
 
 	BuildSimpleCountQuery(whereClause string) *model.Query
-	BuildNRowsQuery(fieldName string, simpleQuery queryparser.SimpleQuery, limit int) *model.Query
-	BuildFacetsQuery(fieldName string, simpleQuery queryparser.SimpleQuery, limit int) *model.Query
+	BuildNRowsQuery(fieldName string, simpleQuery model.SimpleQuery, limit int) *model.Query
+	BuildFacetsQuery(fieldName string, simpleQuery model.SimpleQuery, limit int) *model.Query
 
 	MakeSearchResponse(ResultSet []model.QueryResultRow, query model.Query) (*model.SearchResp, error)
 	MakeResponseAggregation(aggregations []model.Query, aggregationResults [][]model.QueryResultRow) *model.SearchResp

quesma/quesma/search.go

@@ -210,7 +210,7 @@ func (q *QueryRunner) handleSearchCommon(ctx context.Context, indexPattern strin
 		if table == nil {
 			continue
 		}
-		var simpleQuery queryparser.SimpleQuery
+		var simpleQuery model.SimpleQuery
 
 		queryTranslator = NewQueryTranslator(ctx, queryLanguage, table, q.logManager, q.DateMathRenderer)
 
@@ -435,7 +435,7 @@ func (q *QueryRunner) addAsyncQueryContext(ctx context.Context, cancel context.C
 
 func (q *QueryRunner) makeBasicQuery(ctx context.Context,
 	queryTranslator IQueryTranslator, table *clickhouse.Table,
-	simpleQuery queryparser.SimpleQuery, queryInfo model.SearchQueryInfo, highlighter model.Highlighter) (*model.Query, []string) {
+	simpleQuery model.SimpleQuery, queryInfo model.SearchQueryInfo, highlighter model.Highlighter) (*model.Query, []string) {
 	var fullQuery *model.Query
 	var columns []string
 	switch queryInfo.Typ {
@@ -526,7 +526,7 @@ func (q *QueryRunner) Close() {
 	logger.Info().Msg("queryRunner Stopped")
 }
 
-func (q *QueryRunner) findNonexistingProperties(queryInfo model.SearchQueryInfo, simpleQuery queryparser.SimpleQuery, table *clickhouse.Table) []string {
+func (q *QueryRunner) findNonexistingProperties(queryInfo model.SearchQueryInfo, simpleQuery model.SimpleQuery, table *clickhouse.Table) []string {
 	var results = make([]string, 0)
 	var allReferencedFields = make([]string, 0)
 	allReferencedFields = append(allReferencedFields, queryInfo.RequestedFields...)