From 905f4b3db715e4e305d2a0dafdf5bec0630e61d2 Mon Sep 17 00:00:00 2001 From: Krzysztof Kiewicz Date: Fri, 7 Jun 2024 15:17:34 +0200 Subject: [PATCH] Query column as expression 4 (#243) TL;DR * Change all our left expressions from plain strings to `model.SelectColumn` (`FromClause` and a couple of complex SQLs). We're currently thinking what this `FromClause` should ultimately be, so that'll probably change, but still it's a step forward. * Add pretty print for errors in all tests with `sqlmock` --------------------------------------------- Some older story: So I basically wanted to improve my last `GROUP BY` change just a little bit, by making just a few expressions like that ``` Select := fmt.Sprintf("count(if(%s<=%f, 1, NULL))/count(*)*100", strconv.Quote(getFirstFieldName()), cutValue) query.Columns = append(query.Columns, model.SelectColumn{Expression: aexp.SQL{Query: Select}}) ``` properly typed, instead of using `aexp.SQL{}`. But the scope of this PR kind of spiraled out of control, as those small changes made some other part of code not compile, and that repeated a lot of times. But it's not that bad, all these changes change code from untyped `string` field representation to our typed system. One most notable change is `Query.FromClause` also becomes of `model.SelectColumn` type instead of `string`. I didn't expect that, but that makes sense. This string often looked like this: ``` query.FromClause = fmt.Sprintf( "(SELECT %s, ROW_NUMBER() OVER (PARTITION BY %s ORDER BY %s %s) AS %s FROM %s WHERE %s)", fieldsAsString, partitionBy, strconv.Quote(metricsAggr.SortBy), metricsAggr.Order, model.RowNumberColumnName, query.FromClause, b.whereBuilder.WhereClauseAsString(), ) ``` so it's probably a good idea to make it typed as well. I think that maybe using `aexp.AExp` as its type might be a bit more natural, than `model.SelectColumn`, but even if you agree, that's a small not very important change that can be done in another PR. --------- Co-authored-by: Jacek Migdal --- quesma/clickhouse/insert_test.go | 16 +- quesma/clickhouse/table.go | 7 + quesma/clickhouse/util.go | 22 +- quesma/model/bucket_aggregations/range.go | 43 +-- quesma/model/query.go | 61 ++++- quesma/queryparser/aexp/model.go | 2 +- quesma/queryparser/aggregation_parser.go | 259 +++++++++--------- quesma/queryparser/aggregation_parser_test.go | 83 +++--- .../aggregation_percentile_parser.go | 11 +- .../aggregation_percentile_parser_test.go | 7 +- quesma/queryparser/query_parser.go | 2 +- quesma/queryparser/query_parser_test.go | 6 +- quesma/queryparser/query_translator.go | 49 ++-- quesma/queryparser/query_translator_test.go | 2 +- quesma/queryparser/query_util/query_util.go | 4 +- quesma/queryparser/range_aggregation.go | 23 +- .../top_metrics_aggregation_parser.go | 24 +- quesma/quesma/schema_transformer.go | 2 +- quesma/quesma/schema_transformer_test.go | 22 +- quesma/quesma/search_norace_test.go | 14 +- quesma/quesma/search_opensearch_test.go | 13 +- quesma/quesma/search_test.go | 58 +--- quesma/quesma/termsenum/terms_enum_test.go | 3 +- quesma/testdata/aggregation_requests.go | 218 ++++++++------- .../kibana-visualize/aggregation_requests.go | 36 +-- .../aggregation_requests.go | 40 +-- .../pipeline_aggregation_requests.go | 174 ++++++------ quesma/testdata/opensearch_requests.go | 12 +- quesma/testdata/requests.go | 26 +- .../requests_with_special_characters.go | 18 +- quesma/util/utils.go | 20 ++ 31 files changed, 668 insertions(+), 609 deletions(-) diff --git a/quesma/clickhouse/insert_test.go b/quesma/clickhouse/insert_test.go index cfefbd970..f585a97f3 100644 --- a/quesma/clickhouse/insert_test.go +++ b/quesma/clickhouse/insert_test.go @@ -8,6 +8,7 @@ import ( "mitmproxy/quesma/concurrent" "mitmproxy/quesma/quesma/config" "mitmproxy/quesma/quesma/types" + "mitmproxy/quesma/util" "slices" "strconv" "strings" @@ -204,8 +205,7 @@ func TestProcessInsertQuery(t *testing.T) { for index2, config := range configs { for index3, lm := range logManagers(config) { t.Run("case insertTest["+strconv.Itoa(index1)+"], config["+strconv.Itoa(index2)+"], logManager["+strconv.Itoa(index3)+"]", func(t *testing.T) { - db, mock, err := sqlmock.New() - assert.NoError(t, err) + db, mock := util.InitSqlMockWithPrettyPrint(t) lm.lm.chDb = db defer db.Close() @@ -224,7 +224,7 @@ func TestProcessInsertQuery(t *testing.T) { mock.ExpectExec(expectedInserts[2*index1+1]).WillReturnResult(sqlmock.NewResult(1, 1)) } - err = lm.lm.ProcessInsertQuery(ctx, tableName, []types.JSON{types.MustJSON(tt.insertJson)}) + err := lm.lm.ProcessInsertQuery(ctx, tableName, []types.JSON{types.MustJSON(tt.insertJson)}) assert.NoError(t, err) if err := mock.ExpectationsWereMet(); err != nil { t.Fatal("there were unfulfilled expections:", err) @@ -247,8 +247,7 @@ func TestInsertVeryBigIntegers(t *testing.T) { // big integer as a schema field for i, bigInt := range bigInts { t.Run("big integer schema field: "+bigInt, func(t *testing.T) { - db, mock, err := sqlmock.New() - assert.NoError(t, err) + db, mock := util.InitSqlMockWithPrettyPrint(t) lm := NewLogManagerEmpty() lm.chDb = db defer db.Close() @@ -256,7 +255,7 @@ func TestInsertVeryBigIntegers(t *testing.T) { mock.ExpectExec(`CREATE TABLE IF NOT EXISTS "` + tableName).WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec(expectedInsertJsons[i]).WillReturnResult(sqlmock.NewResult(0, 0)) - err = lm.ProcessInsertQuery(context.Background(), tableName, []types.JSON{types.MustJSON(fmt.Sprintf(`{"severity":"sev","int": %s}`, bigInt))}) + err := lm.ProcessInsertQuery(context.Background(), tableName, []types.JSON{types.MustJSON(fmt.Sprintf(`{"severity":"sev","int": %s}`, bigInt))}) assert.NoError(t, err) if err := mock.ExpectationsWereMet(); err != nil { t.Fatal("there were unfulfilled expections:", err) @@ -274,8 +273,7 @@ func TestInsertVeryBigIntegers(t *testing.T) { for i, bigInt := range bigInts { t.Run("big integer attribute field: "+bigInt, func(t *testing.T) { - db, mock, err := sqlmock.New() - assert.NoError(t, err) + db, mock := util.InitSqlMockWithPrettyPrint(t) lm := NewLogManagerEmpty() lm.chDb = db var ptr = atomic.Pointer[TableMap]{} @@ -288,7 +286,7 @@ func TestInsertVeryBigIntegers(t *testing.T) { bigIntAsInt, _ := strconv.ParseInt(bigInt, 10, 64) fmt.Printf(`{"severity":"sev","int": %d}\n`, bigIntAsInt) - err = lm.ProcessInsertQuery(context.Background(), tableName, []types.JSON{types.MustJSON(fmt.Sprintf(`{"severity":"sev","int": %d}`, bigIntAsInt))}) + err := lm.ProcessInsertQuery(context.Background(), tableName, []types.JSON{types.MustJSON(fmt.Sprintf(`{"severity":"sev","int": %d}`, bigIntAsInt))}) assert.NoError(t, err) if err := mock.ExpectationsWereMet(); err != nil { t.Fatal("there were unfulfilled expections:", err) diff --git a/quesma/clickhouse/table.go b/quesma/clickhouse/table.go index 4c055bc4b..3bbebfd49 100644 --- a/quesma/clickhouse/table.go +++ b/quesma/clickhouse/table.go @@ -181,6 +181,13 @@ func (t *Table) GetDateTimeType(ctx context.Context, fieldName string) DateTimeT return Invalid } +func (t *Table) GetDateTimeTypeFromSelectColumn(ctx context.Context, col model.SelectColumn) DateTimeType { + if exp, ok := col.Expression.(aexp.TableColumnExp); ok { + return t.GetDateTimeType(ctx, exp.ColumnName) + } + return Invalid +} + // applyIndexConfig applies full text search and alias configuration to the table func (t *Table) applyIndexConfig(configuration config.QuesmaConfiguration) { for _, c := range t.Cols { diff --git a/quesma/clickhouse/util.go b/quesma/clickhouse/util.go index dbef2900f..1a67854c4 100644 --- a/quesma/clickhouse/util.go +++ b/quesma/clickhouse/util.go @@ -5,6 +5,8 @@ import ( "encoding/json" "fmt" "mitmproxy/quesma/logger" + "mitmproxy/quesma/model" + "mitmproxy/quesma/queryparser/aexp" "strings" "time" ) @@ -112,15 +114,25 @@ func PrettyJson(jsonStr string) string { // TimestampGroupBy returns string to be used in the select part of Clickhouse query, when grouping by timestamp interval. // e.g. // - timestampGroupBy("@timestamp", DateTime64, 30 seconds) --> toInt64(toUnixTimestamp64Milli(`@timestamp`)/30000) -// - timestampGroupBy("@timestamp", DateTime, 30 seconds) --> toInt64(toUnixTimestamp(`@timestamp`)/30.0) -func TimestampGroupBy(timestampFieldName string, typ DateTimeType, groupByInterval time.Duration) string { +// - timestampGroupBy("@timestamp", DateTime, 30 seconds) --> toInt64(toUnixTimestamp(`@timestamp`)/30) +func TimestampGroupBy(timestampField model.SelectColumn, typ DateTimeType, groupByInterval time.Duration) aexp.AExp { + + createAExp := func(innerFuncName string, interval int64) aexp.AExp { + return aexp.Function("toInt64", aexp.NewComposite( + aexp.Function(innerFuncName, timestampField.Expression), + aexp.String("/"), + aexp.Literal(interval), + )) + } + switch typ { case DateTime64: - return fmt.Sprintf("toInt64(toUnixTimestamp64Milli(`%s`)/%d)", timestampFieldName, groupByInterval.Milliseconds()) + // as string: fmt.Sprintf("toInt64(toUnixTimestamp(`%s`)/%f)", timestampFieldName, groupByInterval.Seconds()) + return createAExp("toUnixTimestamp64Milli", groupByInterval.Milliseconds()) case DateTime: - return fmt.Sprintf("toInt64(toUnixTimestamp(`%s`)/%f)", timestampFieldName, groupByInterval.Seconds()) + return createAExp("toUnixTimestamp", groupByInterval.Milliseconds()/1000) default: logger.Error().Msgf("invalid timestamp fieldname: %s", timestampFieldName) - return "invalid" + return aexp.Literal("invalid") // maybe create new type InvalidExpr? } } diff --git a/quesma/model/bucket_aggregations/range.go b/quesma/model/bucket_aggregations/range.go index 381856e6c..ac78a601f 100644 --- a/quesma/model/bucket_aggregations/range.go +++ b/quesma/model/bucket_aggregations/range.go @@ -6,6 +6,7 @@ import ( "math" "mitmproxy/quesma/logger" "mitmproxy/quesma/model" + "mitmproxy/quesma/queryparser/aexp" wc "mitmproxy/quesma/queryparser/where_clause" "strconv" "strings" @@ -30,28 +31,34 @@ func (interval Interval) String() string { } // ToSQLSelectQuery returns count(...) where ... is a condition for the interval, just like we want it in SQL's SELECT -func (interval Interval) ToSQLSelectQuery(quotedFieldName string) string { - var sqlLeft, sqlRight, sql string +func (interval Interval) ToSQLSelectQuery(col model.SelectColumn) model.SelectColumn { + var sqlLeft, sqlRight, sql aexp.AExp if !interval.IsOpeningBoundInfinite() { - sqlLeft = quotedFieldName + ">=" + strconv.FormatFloat(interval.Begin, 'f', -1, 64) + sqlLeft = aexp.Infix(col.Expression, ">=", aexp.Literal(interval.Begin)) } if !interval.IsClosingBoundInfinite() { - sqlRight = quotedFieldName + "<" + strconv.FormatFloat(interval.End, 'f', -1, 64) + sqlRight = aexp.Infix(col.Expression, "<", aexp.Literal(interval.End)) } switch { - case sqlLeft != "" && sqlRight != "": - sql = sqlLeft + " AND " + sqlRight - case sqlLeft != "": + case sqlLeft != nil && sqlRight != nil: + sql = aexp.Infix(sqlLeft, "AND", sqlRight) + case sqlLeft != nil: sql = sqlLeft - case sqlRight != "": + case sqlRight != nil: sql = sqlRight default: - return "count()" + return model.SelectColumn{Expression: aexp.Function("count")} } - return "count(if(" + sql + ", 1, NULL))" + // count(if(sql, 1, NULL)) + return model.SelectColumn{Expression: aexp.Function("count", aexp.Function("if", sql, aexp.Literal(1), aexp.String("NULL")))} } -func (interval Interval) ToWhereClause(fieldName string) wc.Statement { // returns a condition for the interval, just like we want it in SQL's WHERE +func (interval Interval) ToWhereClause(field model.SelectColumn) wc.Statement { // returns a condition for the interval, just like we want it in SQL's WHERE + fieldName := field.SQL() // TODO a) this should be improved b) unify SelectColumn and ColumnRef? + if unquoted, err := strconv.Unquote(fieldName); err == nil { + fieldName = unquoted + } + var sqlLeft, sqlRight wc.Statement if !interval.IsOpeningBoundInfinite() { sqlLeft = wc.NewInfixOp(wc.NewColumnRef(fieldName), ">=", wc.NewLiteral(strconv.FormatFloat(interval.Begin, 'f', -1, 64))) @@ -100,20 +107,20 @@ func (interval Interval) floatToString(number float64) string { } type Range struct { - ctx context.Context - QuotedFieldName string - Intervals []Interval + ctx context.Context + Col model.SelectColumn + Intervals []Interval // defines what response should look like // https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-range-aggregation.html#_keyed_response_4 Keyed bool } -func NewRange(ctx context.Context, quotedFieldName string, intervals []Interval, keyed bool) Range { - return Range{ctx, quotedFieldName, intervals, keyed} +func NewRange(ctx context.Context, col model.SelectColumn, intervals []Interval, keyed bool) Range { + return Range{ctx, col, intervals, keyed} } -func NewRangeWithDefaultKeyed(ctx context.Context, quotedFieldName string, intervals []Interval) Range { - return Range{ctx, quotedFieldName, intervals, keyedDefaultValue} +func NewRangeWithDefaultKeyed(ctx context.Context, col model.SelectColumn, intervals []Interval) Range { + return Range{ctx, col, intervals, keyedDefaultValue} } func (query Range) IsBucketAggregation() bool { diff --git a/quesma/model/query.go b/quesma/model/query.go index 1b1e19662..b37a888d2 100644 --- a/quesma/model/query.go +++ b/quesma/model/query.go @@ -31,15 +31,16 @@ type ( Columns []SelectColumn // Columns to select, including aliases GroupBy []SelectColumn // if not empty, we do GROUP BY GroupBy... OrderBy []SelectColumn // if not empty, we do ORDER BY OrderBy... + FromClause SelectColumn // usually just "tableName", or databaseName."tableName". Sometimes a subquery e.g. (SELECT ...) WhereClause where_clause.Statement // "WHERE ..." until next clause like GROUP BY/ORDER BY, etc. Limit int // LIMIT clause, noLimit (0) means no limit - FromClause string // usually just "tableName", or databaseName."tableName". Sometimes a subquery e.g. (SELECT ...) - CanParse bool // true <=> query is valid + CanParse bool // true <=> query is valid // Eventually we should merge this two QueryInfoType SearchQueryType Type QueryType + TableName string Highlighter Highlighter NoDBQuery bool // true <=> we don't need query to DB here, true in some pipeline aggregations @@ -95,6 +96,14 @@ func NewSortByCountColumn(desc bool) SelectColumn { return SelectColumn{Expression: aexp.NewComposite(aexp.Count(), aexp.String(order))} } +func NewSelectColumnTableField(fieldName string) SelectColumn { + return SelectColumn{Expression: aexp.TableColumn(fieldName)} +} + +func NewSelectColumnString(s string) SelectColumn { + return SelectColumn{Expression: aexp.StringExp{Value: s}} +} + func (c SelectColumn) SQL() string { if c.Expression == nil { @@ -126,7 +135,6 @@ var NoMetadataField JsonMap = nil // returns string with SQL query func (q *Query) String(ctx context.Context) string { - var sb strings.Builder sb.WriteString("SELECT ") if q.IsDistinct { @@ -147,7 +155,7 @@ func (q *Query) String(ctx context.Context) string { sb.WriteString(strings.Join(columns, ", ")) sb.WriteString(" FROM ") - sb.WriteString(q.FromClause) + sb.WriteString(q.FromClause.SQL()) if q.WhereClause != nil { sb.WriteString(" WHERE ") @@ -279,6 +287,51 @@ func (q *Query) ApplyAliases(cfg map[string]config.IndexConfiguration, resolvedT } } +// TODO change whereClause type string -> some typed +func (q *Query) NewSelectColumnSubselectWithRowNumber(selectFields []SelectColumn, groupByFields []SelectColumn, + whereClause string, orderByField string, orderByDesc bool) SelectColumn { + + const additionalArrayLength = 6 + /* used to be as string: + fromSelect := fmt.Sprintf( + "(SELECT %s, ROW_NUMBER() OVER (PARTITION BY %s ORDER BY %s %s) AS %s FROM %s WHERE %s)", + fieldsAsString, fieldsAsString, orderField, asc/desc, + model.RowNumberColumnName, query.FromClause, b.whereBuilder.WhereClauseAsString(), + ) + */ + + fromSelect := make([]aexp.AExp, 0, 2*(len(selectFields)+len(groupByFields))+additionalArrayLength) // +6 without ORDER BY, +8 with ORDER BY + fromSelect = append(fromSelect, aexp.String("SELECT")) + for _, field := range selectFields { + fromSelect = append(fromSelect, field.Expression) + fromSelect = append(fromSelect, aexp.String(",")) + } + + // Maybe keep this ROW_NUMBER as SelectColumn? It'd introduce some problems, because it's not in schema. + // Sticking to simpler solution now. + fromSelect = append(fromSelect, aexp.String("ROW_NUMBER() OVER (PARTITION BY")) + for i, field := range groupByFields { + fromSelect = append(fromSelect, field.Expression) + if i != len(groupByFields)-1 { + fromSelect = append(fromSelect, aexp.String(",")) + } + } + if orderByField != "" { + fromSelect = append(fromSelect, aexp.String("ORDER BY")) + fromSelect = append(fromSelect, NewSortColumn(orderByField, orderByDesc).Expression) + } + fromSelect = append(fromSelect, aexp.String(") AS")) + fromSelect = append(fromSelect, aexp.Literal(RowNumberColumnName)) + fromSelect = append(fromSelect, aexp.String("FROM")) + fromSelect = append(fromSelect, q.FromClause.Expression) + + if whereClause != "" { + fromSelect = append(fromSelect, aexp.String("WHERE "+whereClause)) + } + + return SelectColumn{Expression: aexp.Function("", aexp.NewComposite(fromSelect...))} +} + // Aggregator is always initialized as "empty", so with SplitOverHowManyFields == 0, Keyed == false, Filters == false. // It's updated after construction, during further processing of aggregations. type Aggregator struct { diff --git a/quesma/queryparser/aexp/model.go b/quesma/queryparser/aexp/model.go index f8f46916e..e9fc9731d 100644 --- a/quesma/queryparser/aexp/model.go +++ b/quesma/queryparser/aexp/model.go @@ -144,7 +144,7 @@ type AExpVisitor interface { VisitFunction(e FunctionExp) interface{} VisitMultiFunction(e MultiFunctionExp) interface{} VisitLiteral(l LiteralExp) interface{} - VisitString(l StringExp) interface{} + VisitString(e StringExp) interface{} VisitComposite(e CompositeExp) interface{} VisitInfix(e InfixExp) interface{} VisitSQL(s SQL) interface{} diff --git a/quesma/queryparser/aggregation_parser.go b/quesma/queryparser/aggregation_parser.go index de78b18a5..fae9b99ff 100644 --- a/quesma/queryparser/aggregation_parser.go +++ b/quesma/queryparser/aggregation_parser.go @@ -28,7 +28,7 @@ type aggrQueryBuilder struct { type metricsAggregation struct { AggrType string - FieldNames []string // on these fields we're doing aggregation. Array, because e.g. 'top_hits' can have multiple fields + Fields []model.SelectColumn // on these fields we're doing aggregation. Array, because e.g. 'top_hits' can have multiple fields FieldType clickhouse.DateTimeType // field type of FieldNames[0]. If it's a date field, a slightly different response is needed Percentiles map[string]float64 // Only for percentiles aggregation Keyed bool // Only for percentiles aggregation @@ -85,26 +85,22 @@ func (b *aggrQueryBuilder) buildBucketAggregation(metadata model.JsonMap) model. return query } func (b *aggrQueryBuilder) buildMetricsAggregation(metricsAggr metricsAggregation, metadata model.JsonMap) model.Query { - getFirstFieldName := func() string { - if len(metricsAggr.FieldNames) > 0 { - return metricsAggr.FieldNames[0] + getFirstField := func() model.SelectColumn { + if len(metricsAggr.Fields) > 0 { + return metricsAggr.Fields[0] } logger.ErrorWithCtx(b.ctx).Msg("No field names in metrics aggregation. Using empty.") - return "" + return model.SelectColumn{} + } + getFirstExpression := func() aexp.AExp { + return getFirstField().Expression } query := b.buildAggregationCommon(metadata) switch metricsAggr.AggrType { case "sum", "min", "max", "avg": - - // TODO firstFieldName can be an SQL expression or field name - if strings.Contains(getFirstFieldName(), "(") { - query.Columns = append(query.Columns, model.SelectColumn{Expression: aexp.Function(metricsAggr.AggrType+"OrNull", aexp.SQL{Query: getFirstFieldName()})}) - } else { - query.Columns = append(query.Columns, model.SelectColumn{Expression: aexp.Function(metricsAggr.AggrType+"OrNull", aexp.TableColumn(getFirstFieldName()))}) - } - + query.Columns = append(query.Columns, model.SelectColumn{Expression: aexp.Function(metricsAggr.AggrType+"OrNull", getFirstExpression())}) case "quantile": // Sorting here useful mostly for determinism in tests. // It wasn't there before, and everything worked fine. We could safely remove it, if needed. @@ -115,47 +111,37 @@ func (b *aggrQueryBuilder) buildMetricsAggregation(metricsAggr metricsAggregatio query.Columns = append(query.Columns, model.SelectColumn{ Expression: aexp.MultiFunctionExp{ Name: "quantiles", - Args: []aexp.AExp{aexp.Literal(percentAsFloat), aexp.TableColumn(getFirstFieldName())}}, + Args: []aexp.AExp{aexp.Literal(percentAsFloat), getFirstExpression()}}, Alias: fmt.Sprintf("quantile_%s", usersPercent), }) } case "cardinality": - query.Columns = append(query.Columns, model.SelectColumn{Expression: aexp.Count(aexp.NewComposite(aexp.Symbol("DISTINCT"), aexp.TableColumn(getFirstFieldName())))}) + query.Columns = append(query.Columns, + model.SelectColumn{Expression: aexp.Count(aexp.NewComposite(aexp.Symbol("DISTINCT"), getFirstExpression()))}) case "value_count": query.Columns = append(query.Columns, model.SelectColumn{Expression: aexp.Count()}) case "stats": - fieldName := getFirstFieldName() + expr := getFirstExpression() - query.Columns = append(query.Columns, model.SelectColumn{Expression: aexp.Count(aexp.TableColumn(fieldName))}, - model.SelectColumn{Expression: aexp.Function("minOrNull", aexp.TableColumn(fieldName))}, - model.SelectColumn{Expression: aexp.Function("maxOrNull", aexp.TableColumn(fieldName))}, - model.SelectColumn{Expression: aexp.Function("avgOrNull", aexp.TableColumn(fieldName))}, - model.SelectColumn{Expression: aexp.Function("sumOrNull", aexp.TableColumn(fieldName))}) + query.Columns = append(query.Columns, model.SelectColumn{Expression: aexp.Count(expr)}, + model.SelectColumn{Expression: aexp.Function("minOrNull", expr)}, + model.SelectColumn{Expression: aexp.Function("maxOrNull", expr)}, + model.SelectColumn{Expression: aexp.Function("avgOrNull", expr)}, + model.SelectColumn{Expression: aexp.Function("sumOrNull", expr)}) case "top_hits": // TODO add/restore tests for top_hits. E.g. we missed WHERE in FROM below, so the SQL might not be correct - for _, field := range metricsAggr.FieldNames { - query.Columns = append(query.Columns, model.SelectColumn{Expression: aexp.TableColumn(field)}) - } - - partitionByArr := make([]string, 0, len(b.Query.GroupBy)) - for _, groupByField := range b.Query.GroupBy { - partitionByArr = append(partitionByArr, groupByField.SQL()) - } - partitionBy := strings.Join(partitionByArr, ", ") - innerFields := strings.Join(quoteArray(metricsAggr.FieldNames), ", ") // need those fields in the inner clause - - whereString := "" - if b.whereBuilder.WhereClause != nil { - whereString = " WHERE " + b.whereBuilder.WhereClauseAsString() - } - - query.FromClause = fmt.Sprintf( - "(SELECT %s, ROW_NUMBER() OVER (PARTITION BY %s) AS %s FROM %s%s)", - innerFields, partitionBy, model.RowNumberColumnName, query.FromClause, whereString, - ) + query.Columns = append(query.Columns, metricsAggr.Fields...) + /* + query.FromClause = fmt.Sprintf( + "(SELECT %s, ROW_NUMBER() OVER (PARTITION BY %s) AS %s FROM %s%s)", + metricsAggr.Fields, partitionBy, model.RowNumberColumnName, query.FromClause, whereString, + ) + */ + query.FromClause = query.NewSelectColumnSubselectWithRowNumber( + metricsAggr.Fields, b.GroupBy, b.whereBuilder.WhereClauseAsString(), "", true) query.WhereClause = model.And([]where_clause.Statement{ query.WhereClause, where_clause.NewInfixOp( @@ -176,53 +162,59 @@ func (b *aggrQueryBuilder) buildMetricsAggregation(metricsAggr metricsAggregatio case "desc": ordFunc = `minOrNull` } - var topSelectFields []string - innerFields := append(metricsAggr.FieldNames, metricsAggr.SortBy) - for _, field := range innerFields { - topSelectFields = append(topSelectFields, fmt.Sprintf(`%s("%s") AS "windowed_%s"`, ordFunc, field, field)) - } - for _, field := range topSelectFields { - query.Columns = append(query.Columns, model.SelectColumn{Expression: aexp.SQL{Query: field}}) + innerFields := append(metricsAggr.Fields, model.NewSelectColumnTableField(metricsAggr.SortBy)) + for _, field := range innerFields { + // as string: fmt.Sprintf(`%s("%s") AS "windowed_%s"`, ordFunc, field, field)) + fieldName := field.SQL() + if unquoted, err := strconv.Unquote(fieldName); err == nil { + fieldName = unquoted + } + query.Columns = append(query.Columns, + model.SelectColumn{Expression: aexp.Function(ordFunc, field.Expression), Alias: "windowed_" + fieldName}) } - partitionByArr := make([]string, 0, len(b.Query.GroupBy)) - for _, groupByField := range b.Query.GroupBy { - partitionByArr = append(partitionByArr, groupByField.SQL()) - } - partitionBy := strings.Join(partitionByArr, ", ") - fieldsAsString := strings.Join(quoteArray(innerFields), ", ") // need those fields in the inner clause - query.FromClause = fmt.Sprintf( - "(SELECT %s, ROW_NUMBER() OVER (PARTITION BY %s ORDER BY %s %s) AS %s FROM %s WHERE %s)", - fieldsAsString, partitionBy, - strconv.Quote(metricsAggr.SortBy), metricsAggr.Order, - model.RowNumberColumnName, query.FromClause, b.whereBuilder.WhereClauseAsString(), + query.FromClause = query.NewSelectColumnSubselectWithRowNumber( + innerFields, b.Query.GroupBy, b.whereBuilder.WhereClauseAsString(), + metricsAggr.SortBy, strings.ToLower(metricsAggr.Order) == "desc", ) - query.WhereClause = model.And([]where_clause.Statement{query.WhereClause, where_clause.NewInfixOp(where_clause.NewColumnRef(model.RowNumberColumnName), "<=", where_clause.NewLiteral(strconv.Itoa(metricsAggr.Size)))}) + query.WhereClause = model.And([]where_clause.Statement{query.WhereClause, + where_clause.NewInfixOp(where_clause.NewColumnRef(model.RowNumberColumnName), "<=", where_clause.NewLiteral(strconv.Itoa(metricsAggr.Size)))}) } else { query.Limit = metricsAggr.Size - for _, f := range metricsAggr.FieldNames { - query.Columns = append(query.Columns, model.SelectColumn{Expression: aexp.TableColumn(f)}) - } + query.Columns = append(query.Columns, metricsAggr.Fields...) if metricsAggr.sortByExists() { query.Columns = append(query.Columns, model.SelectColumn{Expression: aexp.TableColumn(metricsAggr.SortBy)}) query.OrderBy = append(query.OrderBy, model.NewSortColumn(metricsAggr.SortBy, strings.ToLower(metricsAggr.Order) == "desc")) } } case "percentile_ranks": - for _, cutValueAsString := range metricsAggr.FieldNames[1:] { - cutValue, _ := strconv.ParseFloat(cutValueAsString, 64) - Select := fmt.Sprintf("count(if(%s<=%f, 1, NULL))/count(*)*100", strconv.Quote(getFirstFieldName()), cutValue) + for _, cutValueAsString := range metricsAggr.Fields[1:] { + cutValue, _ := strconv.ParseFloat(cutValueAsString.SQL(), 64) + + // full exp we create below looks like this: + // fmt.Sprintf("count(if(%s<=%f, 1, NULL))/count(*)*100", strconv.Quote(getFirstFieldName()), cutValue) - query.Columns = append(query.Columns, model.SelectColumn{Expression: aexp.SQL{Query: Select}}) + ifExp := aexp.Function( + "if", + aexp.Infix(getFirstExpression(), "<=", aexp.Literal(cutValue)), + aexp.Literal(1), + aexp.String("NULL"), + ) + firstCountExp := aexp.Function("count", ifExp) + twoCountsExp := aexp.Infix(firstCountExp, "/", aexp.Count(aexp.Wildcard)) + + query.Columns = append(query.Columns, model.SelectColumn{ + Expression: aexp.Infix(twoCountsExp, "*", aexp.Literal(100)), + }) } case "extended_stats": - fieldNameBare := getFirstFieldName() + expr := getFirstExpression() // add column with fn applied to field addColumn := func(funcName string) { - query.Columns = append(query.Columns, model.SelectColumn{Expression: aexp.Function(funcName, aexp.TableColumn(fieldNameBare))}) + query.Columns = append(query.Columns, model.SelectColumn{Expression: aexp.Function(funcName, expr)}) } addColumn("count") @@ -231,7 +223,7 @@ func (b *aggrQueryBuilder) buildMetricsAggregation(metricsAggr metricsAggregatio addColumn("avgOrNull") addColumn("sumOrNull") - query.Columns = append(query.Columns, model.SelectColumn{Expression: aexp.Function("sumOrNull", aexp.Infix(aexp.TableColumn(fieldNameBare), "*", aexp.TableColumn(fieldNameBare)))}) + query.Columns = append(query.Columns, model.SelectColumn{Expression: aexp.Function("sumOrNull", aexp.Infix(expr, "*", expr))}) addColumn("varPop") addColumn("varSamp") @@ -276,7 +268,8 @@ func (b *aggrQueryBuilder) buildMetricsAggregation(metricsAggr metricsAggregatio func (cw *ClickhouseQueryTranslator) ParseAggregationJson(body types.JSON) ([]model.Query, error) { queryAsMap := body.Clone() currentAggr := aggrQueryBuilder{} - currentAggr.FromClause = cw.Table.FullTableName() + currentAggr.FromClause = model.NewSelectColumnString(cw.Table.FullTableName()) + currentAggr.TableName = cw.Table.FullTableName() currentAggr.ctx = cw.Ctx if queryPartRaw, ok := queryAsMap["query"]; ok { if queryPart, ok := queryPartRaw.(QueryMap); ok { @@ -524,12 +517,12 @@ func (cw *ClickhouseQueryTranslator) tryMetricsAggregation(queryMap QueryMap) (m metricsAggregations := []string{"sum", "avg", "min", "max", "cardinality", "value_count", "stats"} for k, v := range queryMap { if slices.Contains(metricsAggregations, k) { - fieldName, isFieldNameFromScript := cw.parseFieldFieldMaybeScript(v, k) + field, isFromScript := cw.parseFieldFieldMaybeScript(v, k) return metricsAggregation{ AggrType: k, - FieldNames: []string{fieldName}, - FieldType: cw.Table.GetDateTimeType(cw.Ctx, fieldName), - IsFieldNameCompound: isFieldNameFromScript, + Fields: []model.SelectColumn{field}, + FieldType: cw.Table.GetDateTimeTypeFromSelectColumn(cw.Ctx, field), + IsFieldNameCompound: isFromScript, }, true } } @@ -539,11 +532,11 @@ func (cw *ClickhouseQueryTranslator) tryMetricsAggregation(queryMap QueryMap) (m if !ok { logger.WarnWithCtx(cw.Ctx).Msgf("percentiles is not a map, but %T, value: %v. Using empty map.", percentile, percentile) } - fieldName, keyed, percentiles := cw.parsePercentilesAggregation(percentileMap) + field, keyed, percentiles := cw.parsePercentilesAggregation(percentileMap) return metricsAggregation{ AggrType: "quantile", - FieldNames: []string{fieldName}, - FieldType: cw.Table.GetDateTimeType(cw.Ctx, fieldName), + Fields: []model.SelectColumn{field}, + FieldType: cw.Table.GetDateTimeTypeFromSelectColumn(cw.Ctx, field), Percentiles: percentiles, Keyed: keyed, }, true @@ -563,12 +556,13 @@ func (cw *ClickhouseQueryTranslator) tryMetricsAggregation(queryMap QueryMap) (m if !ok { logger.WarnWithCtx(cw.Ctx).Msgf("can't parse top_hits' fields. top_hits type: %T, value: %v. Using empty fields.", topHits, topHits) } - fieldsAsStrings := make([]string, 0, len(fields)) - for i, v := range fields { - if vAsString, ok := v.(string); ok { - fieldsAsStrings = append(fieldsAsStrings, vAsString) + cols := make([]model.SelectColumn, 0, len(fields)) + for i, fieldNameRaw := range fields { + if fieldName, ok := fieldNameRaw.(string); ok { + cols = append(cols, model.NewSelectColumnTableField(fieldName)) } else { - logger.WarnWithCtx(cw.Ctx).Msgf("field %d in top_hits is not a string. Field's type: %T, value: %v. Skipping.", i, v, v) + logger.WarnWithCtx(cw.Ctx).Msgf("field %d in top_hits is not a string. Field's type: %T, value: %v. Skipping.", + i, fieldNameRaw, fieldNameRaw) } } @@ -580,17 +574,17 @@ func (cw *ClickhouseQueryTranslator) tryMetricsAggregation(queryMap QueryMap) (m logger.WarnWithCtx(cw.Ctx).Msgf("top_hits is not a map, but %T, value: %v. Using default size.", topHits, topHits) } return metricsAggregation{ - AggrType: "top_hits", - FieldNames: fieldsAsStrings, - FieldType: metricsAggregationDefaultFieldType, // don't need to check, it's unimportant for this aggregation - Size: size, + AggrType: "top_hits", + Fields: cols, + FieldType: metricsAggregationDefaultFieldType, // don't need to check, it's unimportant for this aggregation + Size: size, }, true } // Shortcut here. Percentile_ranks has "field" and a list of "values" // I'm keeping all of them in `fieldNames' array for "simplicity". if percentileRanks, ok := queryMap["percentile_ranks"]; ok { - fieldNames := []string{cw.parseFieldField(percentileRanks, "percentile_ranks")} + fields := []model.SelectColumn{cw.parseFieldField(percentileRanks, "percentile_ranks")} var cutValues []any if values, exists := percentileRanks.(QueryMap)["values"]; exists { cutValues, ok = values.([]any) @@ -603,9 +597,9 @@ func (cw *ClickhouseQueryTranslator) tryMetricsAggregation(queryMap QueryMap) (m for _, cutValue := range cutValues { switch cutValueTyped := cutValue.(type) { case float64: - fieldNames = append(fieldNames, strconv.FormatFloat(cutValueTyped, 'f', -1, 64)) + fields = append(fields, model.NewSelectColumnString(strconv.FormatFloat(cutValueTyped, 'f', -1, 64))) case int64: - fieldNames = append(fieldNames, strconv.FormatInt(cutValueTyped, 10)) + fields = append(fields, model.NewSelectColumnString(strconv.FormatInt(cutValueTyped, 10))) default: logger.WarnWithCtx(cw.Ctx).Msgf("cutValue in percentile_ranks is not a number, but %T, value: %v. Skipping.", cutValue, cutValue) } @@ -620,10 +614,10 @@ func (cw *ClickhouseQueryTranslator) tryMetricsAggregation(queryMap QueryMap) (m keyed = keyedDefaultValuePercentileRanks } return metricsAggregation{ - AggrType: "percentile_ranks", - FieldNames: fieldNames, - FieldType: metricsAggregationDefaultFieldType, // don't need to check, it's unimportant for this aggregation - Keyed: keyed, + AggrType: "percentile_ranks", + Fields: fields, + FieldType: metricsAggregationDefaultFieldType, // don't need to check, it's unimportant for this aggregation + Keyed: keyed, }, true } @@ -643,9 +637,9 @@ func (cw *ClickhouseQueryTranslator) tryMetricsAggregation(queryMap QueryMap) (m } } return metricsAggregation{ - AggrType: "extended_stats", - FieldNames: []string{cw.parseFieldField(extendedStats, "extended_stats")}, - sigma: sigma, + AggrType: "extended_stats", + Fields: []model.SelectColumn{cw.parseFieldField(extendedStats, "extended_stats")}, + sigma: sigma, }, true } @@ -665,13 +659,7 @@ func (cw *ClickhouseQueryTranslator) tryBucketAggregation(currentAggr *aggrQuery if !ok { logger.WarnWithCtx(cw.Ctx).Msgf("date_histogram is not a map, but %T, value: %v", histogramRaw, histogramRaw) } - fieldName, isFieldNameFromScript := cw.parseFieldFieldMaybeScript(histogram, "histogram") - var fieldNameProperlyQuoted string - if isFieldNameFromScript { - fieldNameProperlyQuoted = fieldName - } else { - fieldNameProperlyQuoted = strconv.Quote(fieldName) - } + var interval float64 intervalRaw, ok := histogram["interval"] if !ok { @@ -695,17 +683,24 @@ func (cw *ClickhouseQueryTranslator) tryBucketAggregation(currentAggr *aggrQuery minDocCount := cw.parseMinDocCount(histogram) currentAggr.Type = bucket_aggregations.NewHistogram(cw.Ctx, interval, minDocCount) - var groupByStr string + field, _ := cw.parseFieldFieldMaybeScript(histogram, "histogram") + var col model.SelectColumn if interval != 1.0 { - groupByStr = fmt.Sprintf("floor(%s / %f) * %f", fieldNameProperlyQuoted, interval, interval) + // col as string is: fmt.Sprintf("floor(%s / %f) * %f", fieldNameProperlyQuoted, interval, interval) + col = model.SelectColumn{Expression: aexp.NewComposite( + aexp.Infix( + aexp.Function("floor", aexp.Infix(field.Expression, "/", aexp.Literal(interval))), + "*", + aexp.Literal(interval), + ), + )} } else { - groupByStr = fieldNameProperlyQuoted + col = field } - currentAggr.GroupBy = append(currentAggr.GroupBy, model.SelectColumn{Expression: aexp.SQL{Query: groupByStr}}) - currentAggr.OrderBy = append(currentAggr.OrderBy, model.SelectColumn{Expression: aexp.SQL{Query: groupByStr}}) - - currentAggr.Columns = append(currentAggr.Columns, model.SelectColumn{Expression: aexp.SQL{Query: groupByStr}}) + currentAggr.Columns = append(currentAggr.Columns, col) + currentAggr.GroupBy = append(currentAggr.GroupBy, col) + currentAggr.OrderBy = append(currentAggr.OrderBy, col) delete(queryMap, "histogram") return success, 1, 1, 1, nil @@ -719,10 +714,9 @@ func (cw *ClickhouseQueryTranslator) tryBucketAggregation(currentAggr *aggrQuery currentAggr.Type = bucket_aggregations.NewDateHistogram(cw.Ctx, minDocCount, cw.extractInterval(dateHistogram)) histogramPartOfQuery := cw.createHistogramPartOfQuery(dateHistogram) - currentAggr.GroupBy = append(currentAggr.GroupBy, model.SelectColumn{Expression: aexp.SQL{Query: histogramPartOfQuery}}) - currentAggr.OrderBy = append(currentAggr.OrderBy, model.SelectColumn{Expression: aexp.SQL{Query: histogramPartOfQuery}}) - - currentAggr.Columns = append(currentAggr.Columns, model.SelectColumn{Expression: aexp.SQL{Query: histogramPartOfQuery}}) + currentAggr.Columns = append(currentAggr.Columns, model.SelectColumn{Expression: histogramPartOfQuery}) + currentAggr.GroupBy = append(currentAggr.GroupBy, model.SelectColumn{Expression: histogramPartOfQuery}) + currentAggr.OrderBy = append(currentAggr.OrderBy, model.SelectColumn{Expression: histogramPartOfQuery}) delete(queryMap, "date_histogram") return success, 1, 1, 1, nil @@ -733,8 +727,8 @@ func (cw *ClickhouseQueryTranslator) tryBucketAggregation(currentAggr *aggrQuery isEmptyGroupBy := len(currentAggr.GroupBy) == 0 - currentAggr.GroupBy = append(currentAggr.GroupBy, model.SelectColumn{Expression: aexp.TableColumn(cw.parseFieldField(terms, termsType))}) - currentAggr.Columns = append(currentAggr.Columns, model.SelectColumn{Expression: aexp.TableColumn(cw.parseFieldField(terms, termsType))}) + currentAggr.GroupBy = append(currentAggr.GroupBy, cw.parseFieldField(terms, termsType)) + currentAggr.Columns = append(currentAggr.Columns, cw.parseFieldField(terms, termsType)) orderByAdded := false size := 10 @@ -754,7 +748,7 @@ func (cw *ClickhouseQueryTranslator) tryBucketAggregation(currentAggr *aggrQuery } delete(queryMap, termsType) if !orderByAdded { - currentAggr.OrderBy = append(currentAggr.OrderBy, model.SelectColumn{Expression: aexp.TableColumn(cw.parseFieldField(terms, termsType))}) + currentAggr.OrderBy = append(currentAggr.OrderBy, cw.parseFieldField(terms, termsType)) } return success, 1, 1, 1, nil /* will remove later @@ -802,8 +796,7 @@ func (cw *ClickhouseQueryTranslator) tryBucketAggregation(currentAggr *aggrQuery } fieldsNr = len(terms) for _, term := range terms { - fieldName := cw.parseFieldField(term, "multi_terms") - column := model.SelectColumn{Expression: aexp.TableColumn(fieldName)} + column := cw.parseFieldField(term, "multi_terms") currentAggr.Columns = append(currentAggr.GroupBy, column) currentAggr.GroupBy = append(currentAggr.GroupBy, column) if !orderByAdded { @@ -902,22 +895,22 @@ func (cw *ClickhouseQueryTranslator) tryBucketAggregation(currentAggr *aggrQuery } // parseFieldField returns field 'field' from shouldBeMap, which should be a string. Logs some warnings in case of errors, and returns "" then -func (cw *ClickhouseQueryTranslator) parseFieldField(shouldBeMap any, aggregationType string) string { +func (cw *ClickhouseQueryTranslator) parseFieldField(shouldBeMap any, aggregationType string) model.SelectColumn { Map, ok := shouldBeMap.(QueryMap) if !ok { logger.WarnWithCtx(cw.Ctx).Msgf("%s aggregation is not a map, but %T, value: %v", aggregationType, shouldBeMap, shouldBeMap) - return "" + return model.SelectColumn{} } if fieldRaw, ok := Map["field"]; ok { if field, ok := fieldRaw.(string); ok { - return cw.Table.ResolveField(cw.Ctx, field) + return model.NewSelectColumnTableField(cw.Table.ResolveField(cw.Ctx, field)) // remove this resolve? we do all transforms after parsing is done? } else { logger.WarnWithCtx(cw.Ctx).Msgf("field is not a string, but %T, value: %v", fieldRaw, fieldRaw) } } else { logger.WarnWithCtx(cw.Ctx).Msgf("field not found in %s aggregation: %v", aggregationType, Map) } - return "" + return model.SelectColumn{} } func (cw *ClickhouseQueryTranslator) parseIntField(queryMap QueryMap, fieldName string, defaultValue int) int { @@ -931,8 +924,7 @@ func (cw *ClickhouseQueryTranslator) parseIntField(queryMap QueryMap, fieldName } // parseFieldFieldMaybeScript is basically almost a copy of parseFieldField above, but it also handles a basic script, if "field" is missing. -func (cw *ClickhouseQueryTranslator) parseFieldFieldMaybeScript(shouldBeMap any, aggregationType string) (field string, isFromScript bool) { - isFromScript = false +func (cw *ClickhouseQueryTranslator) parseFieldFieldMaybeScript(shouldBeMap any, aggregationType string) (field model.SelectColumn, isFromScript bool) { Map, ok := shouldBeMap.(QueryMap) if !ok { logger.WarnWithCtx(cw.Ctx).Msgf("%s aggregation is not a map, but %T, value: %v", aggregationType, shouldBeMap, shouldBeMap) @@ -940,23 +932,22 @@ func (cw *ClickhouseQueryTranslator) parseFieldFieldMaybeScript(shouldBeMap any, } // maybe "field" field if fieldRaw, ok := Map["field"]; ok { - if field, ok = fieldRaw.(string); ok { - return + if field, ok := fieldRaw.(string); ok { + return model.NewSelectColumnTableField(cw.Table.ResolveField(cw.Ctx, field)), true // remove this resolve? we do all transforms after parsing is done? } else { logger.WarnWithCtx(cw.Ctx).Msgf("field is not a string, but %T, value: %v", fieldRaw, fieldRaw) } } // else: maybe script - if fieldName, ok := cw.parseFieldFromScriptField(Map); ok { - return fmt.Sprintf("toHour(`%s`)", fieldName), true + if field, isFromScript = cw.parseFieldFromScriptField(Map); !isFromScript { + logger.WarnWithCtx(cw.Ctx).Msgf("field not found in %s aggregation: %v", aggregationType, Map) } - - logger.WarnWithCtx(cw.Ctx).Msgf("field not found in %s aggregation: %v", aggregationType, Map) return } -func (cw *ClickhouseQueryTranslator) parseFieldFromScriptField(queryMap QueryMap) (fieldName string, success bool) { +// parseFieldFromScriptField returns (field, true) if parsing succeeded, (model.SelectColumn{}, false) otherwise. +func (cw *ClickhouseQueryTranslator) parseFieldFromScriptField(queryMap QueryMap) (field model.SelectColumn, success bool) { scriptRaw, exists := queryMap["script"] if !exists { return @@ -981,7 +972,7 @@ func (cw *ClickhouseQueryTranslator) parseFieldFromScriptField(queryMap QueryMap wantedRegex := regexp.MustCompile(`^doc\['(\w+)']\.value\.(?:getHour\(\)|hourOfDay)$`) matches := wantedRegex.FindStringSubmatch(source) if len(matches) == 2 { - return matches[1], true + return model.SelectColumn{Expression: aexp.Function("toHour", aexp.TableColumn(matches[1]))}, true } return } diff --git a/quesma/queryparser/aggregation_parser_test.go b/quesma/queryparser/aggregation_parser_test.go index a533136f6..a4617d12a 100644 --- a/quesma/queryparser/aggregation_parser_test.go +++ b/quesma/queryparser/aggregation_parser_test.go @@ -8,6 +8,7 @@ import ( "mitmproxy/quesma/clickhouse" "mitmproxy/quesma/concurrent" "mitmproxy/quesma/model" + "mitmproxy/quesma/queryparser/aexp" "mitmproxy/quesma/quesma/config" "mitmproxy/quesma/quesma/types" "mitmproxy/quesma/testdata" @@ -156,9 +157,10 @@ var aggregationTests = []struct { []string{ `SELECT count() FROM ` + tableNameQuoted, `SELECT "FlightDelayType", count() FROM ` + tableNameQuoted + ` GROUP BY "FlightDelayType" ORDER BY "FlightDelayType"`, - "SELECT \"FlightDelayType\", toInt64(toUnixTimestamp64Milli(`timestamp`)/10800000), count() FROM " + tableNameQuoted + " " + - "GROUP BY \"FlightDelayType\", toInt64(toUnixTimestamp64Milli(`timestamp`)/10800000) " + - "ORDER BY \"FlightDelayType\", toInt64(toUnixTimestamp64Milli(`timestamp`)/10800000)", + `SELECT "FlightDelayType", toInt64(toUnixTimestamp64Milli("timestamp") / 10800000), count() ` + + `FROM ` + tableNameQuoted + " " + + `GROUP BY "FlightDelayType", toInt64(toUnixTimestamp64Milli("timestamp") / 10800000) ` + + `ORDER BY "FlightDelayType", toInt64(toUnixTimestamp64Milli("timestamp") / 10800000)`, }, }, { // [3] @@ -304,15 +306,15 @@ var aggregationTests = []struct { `SELECT "OriginAirportID", "DestAirportID", count() FROM ` + tableNameQuoted + ` ` + `GROUP BY "OriginAirportID", "DestAirportID" ORDER BY "OriginAirportID", "DestAirportID"`, `SELECT "OriginAirportID", "DestAirportID", "DestLocation" ` + - `FROM (SELECT "DestLocation", ROW_NUMBER() ` + - `OVER (PARTITION BY "OriginAirportID", "DestAirportID") AS row_number ` + + `FROM (SELECT "DestLocation" , ROW_NUMBER() ` + + `OVER (PARTITION BY "OriginAirportID" , "DestAirportID" ) AS 'row_number' ` + `FROM "logs-generic-default") ` + `WHERE "row_number"<=1 ` + `GROUP BY "OriginAirportID", "DestAirportID" ` + `ORDER BY "OriginAirportID", "DestAirportID"`, `SELECT "OriginAirportID", "OriginLocation", "Origin" ` + - `FROM (SELECT "OriginLocation", "Origin", ROW_NUMBER() ` + - `OVER (PARTITION BY "OriginAirportID") AS row_number ` + + `FROM (SELECT "OriginLocation" , "Origin" , ROW_NUMBER() ` + + `OVER (PARTITION BY "OriginAirportID" ) AS 'row_number' ` + `FROM "logs-generic-default") ` + `WHERE "row_number"<=1 ` + `GROUP BY "OriginAirportID" ` + @@ -352,9 +354,10 @@ var aggregationTests = []struct { }`, []string{ `SELECT count() FROM ` + tableNameQuoted, - `SELECT "category", ` + "toInt64(toUnixTimestamp64Milli(`order_date`)/86400000), count() FROM " + tableNameQuoted + ` ` + - `GROUP BY "category", ` + "toInt64(toUnixTimestamp64Milli(`order_date`)/86400000) " + - `ORDER BY "category", ` + "toInt64(toUnixTimestamp64Milli(`order_date`)/86400000)", + `SELECT "category", toInt64(toUnixTimestamp64Milli("order_date") / 86400000), count() ` + + `FROM ` + tableNameQuoted + ` ` + + `GROUP BY "category", toInt64(toUnixTimestamp64Milli("order_date") / 86400000) ` + + `ORDER BY "category", toInt64(toUnixTimestamp64Milli("order_date") / 86400000)`, `SELECT "category", count() FROM ` + tableNameQuoted + ` GROUP BY "category" ORDER BY "category"`, }, }, @@ -476,31 +479,31 @@ var aggregationTests = []struct { []string{ `SELECT count() FROM ` + tableNameQuoted, `SELECT count() FROM ` + tableNameQuoted + ` WHERE "taxful_total_price" > '250'`, - "SELECT toInt64(toUnixTimestamp64Milli(`order_date`)/43200000), " + + `SELECT toInt64(toUnixTimestamp64Milli("order_date") / 43200000), ` + `maxOrNull("order_date") AS "windowed_order_date", maxOrNull("order_date") AS "windowed_order_date" ` + - `FROM (SELECT "order_date", "order_date", ROW_NUMBER() OVER ` + - "(PARTITION BY toInt64(toUnixTimestamp64Milli(`order_date`)/43200000) " + - `ORDER BY "order_date" asc) AS row_number ` + + `FROM (SELECT "order_date" , "order_date" , ROW_NUMBER() OVER ` + + `(PARTITION BY toInt64(toUnixTimestamp64Milli("order_date") / 43200000) ` + + `ORDER BY "order_date" ASC ) AS 'row_number' ` + `FROM ` + tableNameQuoted + ` ` + `WHERE "taxful_total_price" > '250') ` + `WHERE ("taxful_total_price" > '250' AND "row_number"<=10) ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`order_date`)/43200000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`order_date`)/43200000)", - "SELECT toInt64(toUnixTimestamp64Milli(`order_date`)/43200000), " + + `GROUP BY toInt64(toUnixTimestamp64Milli("order_date") / 43200000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("order_date") / 43200000)`, + `SELECT toInt64(toUnixTimestamp64Milli("order_date") / 43200000), ` + `maxOrNull("taxful_total_price") AS "windowed_taxful_total_price", maxOrNull("order_date") AS "windowed_order_date" ` + - `FROM (SELECT "taxful_total_price", "order_date", ROW_NUMBER() OVER ` + - "(PARTITION BY toInt64(toUnixTimestamp64Milli(`order_date`)/43200000) " + - `ORDER BY "order_date" asc) AS row_number ` + + `FROM (SELECT "taxful_total_price" , "order_date" , ROW_NUMBER() OVER ` + + `(PARTITION BY toInt64(toUnixTimestamp64Milli("order_date") / 43200000) ` + + `ORDER BY "order_date" ASC ) AS 'row_number' ` + `FROM ` + tableNameQuoted + ` ` + `WHERE "taxful_total_price" > '250') ` + `WHERE ("taxful_total_price" > '250' AND "row_number"<=10) ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`order_date`)/43200000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`order_date`)/43200000)", - "SELECT toInt64(toUnixTimestamp64Milli(`order_date`)/43200000), count() " + + `GROUP BY toInt64(toUnixTimestamp64Milli("order_date") / 43200000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("order_date") / 43200000)`, + `SELECT toInt64(toUnixTimestamp64Milli("order_date") / 43200000), count() ` + `FROM ` + tableNameQuoted + ` ` + `WHERE "taxful_total_price" > '250' ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`order_date`)/43200000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`order_date`)/43200000)", + `GROUP BY toInt64(toUnixTimestamp64Milli("order_date") / 43200000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("order_date") / 43200000)`, }, }, { // [12] @@ -717,26 +720,30 @@ func Test_parseFieldFromScriptField(t *testing.T) { goodQueryMap := func(sourceField string) QueryMap { return QueryMap{"script": QueryMap{"source": sourceField}} } + + empty := model.SelectColumn{} testcases := []struct { queryMap QueryMap - expectedMatch string + expectedMatch model.SelectColumn expectedSuccess bool }{ - {goodQueryMap("doc['field1'].value.getHour()"), "field1", true}, - {goodQueryMap("doc['field1'].value.getHour() + doc['field2'].value.getHour()"), "", false}, - {goodQueryMap("doc['field1'].value.hourOfDay"), "field1", true}, - {goodQueryMap("doc['field1'].value"), "", false}, - {goodQueryMap("value.getHour() + doc['field2'].value.getHour()"), "", false}, - {QueryMap{}, "", false}, - {QueryMap{"script": QueryMap{}}, "", false}, - {QueryMap{"script": QueryMap{"source": ""}}, "", false}, - {QueryMap{"script": "script"}, "", false}, - {QueryMap{"script": QueryMap{"source": 1}}, "", false}, + {goodQueryMap("doc['field1'].value.getHour()"), + model.SelectColumn{Expression: aexp.Function("toHour", aexp.TableColumnExp{ColumnName: "field1"})}, true}, + {goodQueryMap("doc['field1'].value.getHour() + doc['field2'].value.getHour()"), empty, false}, + {goodQueryMap("doc['field1'].value.hourOfDay"), + model.SelectColumn{Expression: aexp.Function("toHour", aexp.TableColumnExp{ColumnName: "field1"})}, true}, + {goodQueryMap("doc['field1'].value"), empty, false}, + {goodQueryMap("value.getHour() + doc['field2'].value.getHour()"), empty, false}, + {QueryMap{}, empty, false}, + {QueryMap{"script": QueryMap{}}, empty, false}, + {QueryMap{"script": QueryMap{"source": empty}}, empty, false}, + {QueryMap{"script": "script"}, empty, false}, + {QueryMap{"script": QueryMap{"source": 1}}, empty, false}, } cw := ClickhouseQueryTranslator{Ctx: context.Background()} for _, tc := range testcases { - fieldName, success := cw.parseFieldFromScriptField(tc.queryMap) + field, success := cw.parseFieldFromScriptField(tc.queryMap) assert.Equal(t, tc.expectedSuccess, success) - assert.Equal(t, tc.expectedMatch, fieldName) + assert.Equal(t, tc.expectedMatch, field) } } diff --git a/quesma/queryparser/aggregation_percentile_parser.go b/quesma/queryparser/aggregation_percentile_parser.go index fe73cfc4b..829d316f2 100644 --- a/quesma/queryparser/aggregation_percentile_parser.go +++ b/quesma/queryparser/aggregation_percentile_parser.go @@ -3,6 +3,7 @@ package queryparser import ( "fmt" "mitmproxy/quesma/logger" + "mitmproxy/quesma/model" ) const maxPrecision = 0.999999 @@ -19,8 +20,8 @@ var defaultPercentiles = map[string]float64{ const keyedDefaultValue = true -func (cw *ClickhouseQueryTranslator) parsePercentilesAggregation(queryMap QueryMap) (fieldName string, keyed bool, percentiles map[string]float64) { - fieldName = cw.parseFieldField(queryMap, "percentile") +func (cw *ClickhouseQueryTranslator) parsePercentilesAggregation(queryMap QueryMap) (field model.SelectColumn, keyed bool, percentiles map[string]float64) { + field = cw.parseFieldField(queryMap, "percentile") if keyedQueryMap, ok := queryMap["keyed"]; ok { if keyed, ok = keyedQueryMap.(bool); !ok { logger.WarnWithCtx(cw.Ctx).Msgf("keyed specified for percentiles aggregation is not a boolean. Querymap: %v", queryMap) @@ -32,12 +33,12 @@ func (cw *ClickhouseQueryTranslator) parsePercentilesAggregation(queryMap QueryM percents, ok := queryMap["percents"] if !ok { - return fieldName, keyed, defaultPercentiles + return field, keyed, defaultPercentiles } userInput, ok := percents.([]interface{}) if !ok { logger.WarnWithCtx(cw.Ctx).Msgf("percents specified for percentiles aggregation is not an array. Querymap: %v", queryMap) - return fieldName, keyed, defaultPercentiles + return field, keyed, defaultPercentiles } userSpecifiedPercents := make(map[string]float64, len(userInput)) for _, p := range userInput { @@ -53,5 +54,5 @@ func (cw *ClickhouseQueryTranslator) parsePercentilesAggregation(queryMap QueryM } userSpecifiedPercents[asString] = asFloat } - return fieldName, keyed, userSpecifiedPercents + return field, keyed, userSpecifiedPercents } diff --git a/quesma/queryparser/aggregation_percentile_parser_test.go b/quesma/queryparser/aggregation_percentile_parser_test.go index a69d4f762..03c03a8d3 100644 --- a/quesma/queryparser/aggregation_percentile_parser_test.go +++ b/quesma/queryparser/aggregation_percentile_parser_test.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/stretchr/testify/assert" "mitmproxy/quesma/clickhouse" + "mitmproxy/quesma/model" "testing" ) @@ -13,8 +14,8 @@ func Test_parsePercentilesAggregationWithDefaultPercents(t *testing.T) { "field": "custom_name", } cw := &ClickhouseQueryTranslator{Table: &clickhouse.Table{}, Ctx: context.Background()} - fieldName, _, userSpecifiedPercents := cw.parsePercentilesAggregation(payload) - assert.Equal(t, "custom_name", fieldName) + field, _, userSpecifiedPercents := cw.parsePercentilesAggregation(payload) + assert.Equal(t, model.NewSelectColumnTableField("custom_name"), field) assert.Equal(t, defaultPercentiles, userSpecifiedPercents) } @@ -43,7 +44,7 @@ func Test_parsePercentilesAggregationWithUserSpecifiedPercents(t *testing.T) { } cw := &ClickhouseQueryTranslator{Table: &clickhouse.Table{}, Ctx: context.Background()} fieldName, _, parsedMap := cw.parsePercentilesAggregation(payload) - assert.Equal(t, "custom_name", fieldName) + assert.Equal(t, model.NewSelectColumnTableField("custom_name"), fieldName) parsedMapKeys := make([]string, 0, len(parsedMap)) for k := range parsedMap { diff --git a/quesma/queryparser/query_parser.go b/quesma/queryparser/query_parser.go index 16a74ab16..26405d096 100644 --- a/quesma/queryparser/query_parser.go +++ b/quesma/queryparser/query_parser.go @@ -84,7 +84,7 @@ func (cw *ClickhouseQueryTranslator) makeBasicQuery( case model.Facets, model.FacetsNumeric: // queryInfo = (Facets, fieldName, Limit results, Limit last rows to look into) - fullQuery = cw.BuildFacetsQuery(queryInfo.FieldName, simpleQuery.WhereClause) + fullQuery = cw.BuildFacetsQuery(queryInfo.FieldName, simpleQuery) if len(fullQuery.Columns) > 0 { fullQuery.Columns[0].Alias = "key" diff --git a/quesma/queryparser/query_parser_test.go b/quesma/queryparser/query_parser_test.go index ccd37e674..2a9e4070d 100644 --- a/quesma/queryparser/query_parser_test.go +++ b/quesma/queryparser/query_parser_test.go @@ -66,7 +66,7 @@ func TestQueryParserStringAttrConfig(t *testing.T) { assert.Contains(t, query.String(context.Background()), wantedSQL, "query contains wanted sql") } assert.True(t, query.CanParse, "can parse") - assert.Equal(t, strconv.Quote(testdata.TableName), query.FromClause) + assert.Equal(t, model.NewSelectColumnString(strconv.Quote(testdata.TableName)), query.FromClause) assert.Equal(t, []model.SelectColumn{{Expression: aexp.Wildcard}}, query.Columns) }) } @@ -102,7 +102,7 @@ func TestQueryParserNoFullTextFields(t *testing.T) { assert.Contains(t, query.String(context.Background()), wantedSQL, "query contains wanted sql") } assert.True(t, query.CanParse, "can parse") - assert.Equal(t, strconv.Quote(testdata.TableName), query.FromClause) + assert.Equal(t, model.NewSelectColumnString(strconv.Quote(testdata.TableName)), query.FromClause) assert.Equal(t, []model.SelectColumn{{Expression: aexp.Wildcard}}, query.Columns) }) } @@ -137,7 +137,7 @@ func TestQueryParserNoAttrsConfig(t *testing.T) { assert.Contains(t, query.String(context.Background()), wantedSQL, "query contains wanted sql") } assert.True(t, query.CanParse, "can parse") - assert.Equal(t, strconv.Quote(testdata.TableName), query.FromClause) + assert.Equal(t, model.NewSelectColumnString(strconv.Quote(testdata.TableName)), query.FromClause) assert.Equal(t, []model.SelectColumn{{Expression: aexp.Wildcard}}, query.Columns) }) } diff --git a/quesma/queryparser/query_translator.go b/quesma/queryparser/query_translator.go index c2595946c..2eee6fb49 100644 --- a/quesma/queryparser/query_translator.go +++ b/quesma/queryparser/query_translator.go @@ -560,7 +560,8 @@ func (cw *ClickhouseQueryTranslator) BuildSimpleCountQuery(whereClause where_cla return &model.Query{ Columns: []model.SelectColumn{{Expression: aexp.Count()}}, WhereClause: whereClause, - FromClause: cw.Table.FullTableName(), + FromClause: model.NewSelectColumnString(cw.Table.FullTableName()), + TableName: cw.Table.FullTableName(), CanParse: true, } } @@ -575,7 +576,8 @@ func (cw *ClickhouseQueryTranslator) BuildAutocompleteQuery(fieldName string, wh Columns: []model.SelectColumn{{Expression: aexp.TableColumn(fieldName)}}, WhereClause: whereClause, Limit: limit, - FromClause: cw.Table.FullTableName(), + FromClause: model.NewSelectColumnString(cw.Table.FullTableName()), + TableName: cw.Table.FullTableName(), CanParse: true, } } @@ -592,26 +594,26 @@ func (cw *ClickhouseQueryTranslator) BuildAutocompleteSuggestionsQuery(fieldName Columns: []model.SelectColumn{{Expression: aexp.TableColumn(fieldName)}}, WhereClause: whereClause, Limit: limit, - FromClause: cw.Table.FullTableName(), + FromClause: model.NewSelectColumnString(cw.Table.FullTableName()), + TableName: cw.Table.FullTableName(), CanParse: true, } } -func (cw *ClickhouseQueryTranslator) BuildFacetsQuery(fieldName string, whereClause where_clause.Statement) *model.Query { - innerQuery := model.Query{ - WhereClause: whereClause, - Columns: []model.SelectColumn{{Expression: aexp.TableColumn(fieldName)}}, - Limit: facetsSampleSize, - FromClause: cw.Table.FullTableName(), - CanParse: true, - } - +func (cw *ClickhouseQueryTranslator) BuildFacetsQuery(fieldName string, simpleQuery model.SimpleQuery) *model.Query { + // FromClause: (SELECT fieldName FROM table WHERE whereClause LIMIT facetsSampleSize) return &model.Query{ - Columns: []model.SelectColumn{{Expression: aexp.TableColumn(fieldName)}, {Expression: aexp.Count()}}, - GroupBy: []model.SelectColumn{{Expression: aexp.TableColumn(fieldName)}}, - OrderBy: []model.SelectColumn{model.NewSortByCountColumn(true)}, - FromClause: "(" + innerQuery.String(cw.Ctx) + ")", - CanParse: true, + Columns: []model.SelectColumn{{Expression: aexp.TableColumn(fieldName)}, {Expression: aexp.Count()}}, + GroupBy: []model.SelectColumn{{Expression: aexp.TableColumn(fieldName)}}, + OrderBy: []model.SelectColumn{model.NewSortByCountColumn(true)}, + FromClause: model.SelectColumn{Expression: aexp.NewComposite( + aexp.String("(SELECT"), aexp.TableColumn(fieldName), + aexp.SQL{Query: "FROM " + cw.Table.FullTableName()}, + aexp.SQL{Query: "WHERE " + simpleQuery.WhereClauseAsString()}, + aexp.String("LIMIT"), aexp.Literal(facetsSampleSize), + aexp.String(")"))}, + TableName: cw.Table.FullTableName(), + CanParse: true, } } @@ -623,24 +625,25 @@ func (cw *ClickhouseQueryTranslator) BuildTimestampQuery(timestampFieldName stri WhereClause: whereClause, OrderBy: []model.SelectColumn{model.NewSortColumn(timestampFieldName, !earliest)}, Limit: 1, - FromClause: cw.Table.FullTableName(), + FromClause: model.NewSelectColumnString(cw.Table.FullTableName()), + TableName: cw.Table.FullTableName(), CanParse: true, } } -func (cw *ClickhouseQueryTranslator) createHistogramPartOfQuery(queryMap QueryMap) string { +func (cw *ClickhouseQueryTranslator) createHistogramPartOfQuery(queryMap QueryMap) aexp.AExp { const defaultDateTimeType = clickhouse.DateTime64 - fieldName := cw.parseFieldField(queryMap, "histogram") + field := cw.parseFieldField(queryMap, "histogram") interval, err := kibana.ParseInterval(cw.extractInterval(queryMap)) if err != nil { logger.ErrorWithCtx(cw.Ctx).Msg(err.Error()) } - dateTimeType := cw.Table.GetDateTimeType(cw.Ctx, fieldName) + dateTimeType := cw.Table.GetDateTimeTypeFromSelectColumn(cw.Ctx, field) if dateTimeType == clickhouse.Invalid { - logger.ErrorWithCtx(cw.Ctx).Msgf("invalid date type for field %v. Using DateTime64 as default.", fieldName) + logger.ErrorWithCtx(cw.Ctx).Msgf("invalid date type for field %+v. Using DateTime64 as default.", field) dateTimeType = defaultDateTimeType } - return clickhouse.TimestampGroupBy(fieldName, dateTimeType, interval) + return clickhouse.TimestampGroupBy(field, dateTimeType, interval) } // sortInTopologicalOrder sorts all our queries to DB, which we send to calculate response for a single query request. diff --git a/quesma/queryparser/query_translator_test.go b/quesma/queryparser/query_translator_test.go index 138a68266..f422f1497 100644 --- a/quesma/queryparser/query_translator_test.go +++ b/quesma/queryparser/query_translator_test.go @@ -495,7 +495,7 @@ func TestMakeResponseAsyncSearchQueryIsProperJson(t *testing.T) { cw := ClickhouseQueryTranslator{ClickhouseLM: lm, Table: table, Ctx: context.Background()} queries := []*model.Query{ cw.BuildAutocompleteSuggestionsQuery("@", "", 0), - cw.BuildFacetsQuery("@", nil), + cw.BuildFacetsQuery("@", model.SimpleQuery{}), // queryTranslator.BuildTimestampQuery("@", "@", "", true), TODO uncomment when add unification for this query type } types := []model.SearchQueryType{model.ListAllFields, model.ListByField} diff --git a/quesma/queryparser/query_util/query_util.go b/quesma/queryparser/query_util/query_util.go index c2f26408d..d184674e1 100644 --- a/quesma/queryparser/query_util/query_util.go +++ b/quesma/queryparser/query_util/query_util.go @@ -26,13 +26,13 @@ func BuildNRowsQuery(ctx context.Context, tableName string, fieldName string, qu } else { col = model.SelectColumn{Expression: aexp.TableColumn(fieldName)} } - return &model.Query{ Columns: []model.SelectColumn{col}, WhereClause: query.WhereClause, OrderBy: query.OrderBy, Limit: applySizeLimit(ctx, limit), - FromClause: tableName, + FromClause: model.NewSelectColumnString(tableName), + TableName: tableName, CanParse: true, } } diff --git a/quesma/queryparser/range_aggregation.go b/quesma/queryparser/range_aggregation.go index 8579b10e6..745daf72e 100644 --- a/quesma/queryparser/range_aggregation.go +++ b/quesma/queryparser/range_aggregation.go @@ -5,12 +5,10 @@ import ( "mitmproxy/quesma/logger" "mitmproxy/quesma/model" "mitmproxy/quesma/model/bucket_aggregations" - "mitmproxy/quesma/queryparser/aexp" - "strconv" ) func (cw *ClickhouseQueryTranslator) parseRangeAggregation(rangePart QueryMap) bucket_aggregations.Range { - fieldName := cw.parseFieldField(rangePart, "range") + field := cw.parseFieldField(rangePart, "range") var ranges []any if rangesRaw, ok := rangePart["ranges"]; ok { ranges, ok = rangesRaw.([]any) @@ -46,13 +44,12 @@ func (cw *ClickhouseQueryTranslator) parseRangeAggregation(rangePart QueryMap) b } if keyedRaw, exists := rangePart["keyed"]; exists { if keyed, ok := keyedRaw.(bool); ok { - // TODO we should not store Quoted field name in the range struct - return bucket_aggregations.NewRange(cw.Ctx, strconv.Quote(fieldName), intervals, keyed) + return bucket_aggregations.NewRange(cw.Ctx, field, intervals, keyed) } else { logger.WarnWithCtx(cw.Ctx).Msgf("keyed is not a bool, but %T, value: %v", keyedRaw, keyedRaw) } } - return bucket_aggregations.NewRangeWithDefaultKeyed(cw.Ctx, strconv.Quote(fieldName), intervals) + return bucket_aggregations.NewRangeWithDefaultKeyed(cw.Ctx, field, intervals) } func (cw *ClickhouseQueryTranslator) processRangeAggregation(currentAggr *aggrQueryBuilder, Range bucket_aggregations.Range, @@ -60,10 +57,7 @@ func (cw *ClickhouseQueryTranslator) processRangeAggregation(currentAggr *aggrQu // build this aggregation for _, interval := range Range.Intervals { - - // TODO XXXX - currentAggr.Columns = append(currentAggr.Columns, model.SelectColumn{Expression: aexp.SQL{Query: interval.ToSQLSelectQuery(Range.QuotedFieldName)}}) - + currentAggr.Columns = append(currentAggr.Columns, interval.ToSQLSelectQuery(Range.Col)) } if !Range.Keyed { // there's a difference in output structure whether the range is keyed or not @@ -87,16 +81,9 @@ func (cw *ClickhouseQueryTranslator) processRangeAggregation(currentAggr *aggrQu // Range aggregation with subaggregations should be a quite rare case, so I'm leaving that for later. whereBeforeNesting := currentAggr.whereBuilder for _, interval := range Range.Intervals { - var fieldName string - if f, err := strconv.Unquote(Range.QuotedFieldName); err != nil { - logger.Error().Msgf("Unquoting field name in range aggregation failed: %v", err) // fallback to what we have... - fieldName = Range.QuotedFieldName - } else { - fieldName = f - } currentAggr.whereBuilder = model.CombineWheres( cw.Ctx, currentAggr.whereBuilder, - model.NewSimpleQuery(interval.ToWhereClause(fieldName), true), + model.NewSimpleQuery(interval.ToWhereClause(Range.Col), true), ) currentAggr.Aggregators = append(currentAggr.Aggregators, model.NewAggregator(interval.String())) aggsCopy, err := deepcopy.Anything(aggs) diff --git a/quesma/queryparser/top_metrics_aggregation_parser.go b/quesma/queryparser/top_metrics_aggregation_parser.go index d21cb08e6..84e54270c 100644 --- a/quesma/queryparser/top_metrics_aggregation_parser.go +++ b/quesma/queryparser/top_metrics_aggregation_parser.go @@ -3,10 +3,11 @@ package queryparser import ( "context" "mitmproxy/quesma/logger" + "mitmproxy/quesma/model" ) func (cw *ClickhouseQueryTranslator) ParseTopMetricsAggregation(queryMap QueryMap) metricsAggregation { - var fieldNames []string + var fields []model.SelectColumn metrics, exists := queryMap["metrics"] if exists { var fieldList []interface{} @@ -15,7 +16,7 @@ func (cw *ClickhouseQueryTranslator) ParseTopMetricsAggregation(queryMap QueryMa } else { fieldList = append(fieldList, metrics) } - fieldNames = cw.getFieldNames(fieldList) + fields = cw.getFieldNames(fieldList) } else { logger.WarnWithCtx(cw.Ctx).Msg("no metrics field found in query") } @@ -43,12 +44,12 @@ func (cw *ClickhouseQueryTranslator) ParseTopMetricsAggregation(queryMap QueryMa size = defaultSize } return metricsAggregation{ - AggrType: "top_metrics", - FieldNames: fieldNames, - FieldType: metricsAggregationDefaultFieldType, // don't need to check, it's unimportant for this aggregation - SortBy: sortBy, - Size: size, - Order: order, + AggrType: "top_metrics", + Fields: fields, + FieldType: metricsAggregationDefaultFieldType, // don't need to check, it's unimportant for this aggregation + SortBy: sortBy, + Size: size, + Order: order, } } @@ -63,16 +64,15 @@ func getFirstKeyValue(ctx context.Context, queryMap QueryMap) (string, string) { return "", "" } -func (cw *ClickhouseQueryTranslator) getFieldNames(fields []interface{}) []string { - var fieldNames []string +func (cw *ClickhouseQueryTranslator) getFieldNames(fields []interface{}) (cols []model.SelectColumn) { for _, field := range fields { if fName, ok := field.(QueryMap)["field"]; ok { if fieldName, ok := fName.(string); ok { - fieldNames = append(fieldNames, cw.Table.ResolveField(cw.Ctx, fieldName)) + cols = append(cols, model.NewSelectColumnTableField(cw.Table.ResolveField(cw.Ctx, fieldName))) } else { logger.WarnWithCtx(cw.Ctx).Msgf("field %v is not a string (type: %T). Might be correct, might not. Check it out.", fName, fName) } } } - return fieldNames + return cols } diff --git a/quesma/quesma/schema_transformer.go b/quesma/quesma/schema_transformer.go index 2cde41dc0..b468351f4 100644 --- a/quesma/quesma/schema_transformer.go +++ b/quesma/quesma/schema_transformer.go @@ -86,7 +86,7 @@ func (s *SchemaCheckPass) applyIpTransformations(query model.Query) (model.Query whereVisitor := &WhereVisitor{} query.WhereClause.Accept(whereVisitor) - fromTable := getFromTable(query.FromClause) + fromTable := getFromTable(query.TableName) mappedType := s.cfg[fromTable].TypeMappings[strings.Trim(whereVisitor.lhs, "\"")] if mappedType != "ip" { diff --git a/quesma/quesma/schema_transformer_test.go b/quesma/quesma/schema_transformer_test.go index 6e36d99f0..86e6033a1 100644 --- a/quesma/quesma/schema_transformer_test.go +++ b/quesma/quesma/schema_transformer_test.go @@ -29,7 +29,8 @@ func Test_ipRangeTransform(t *testing.T) { expectedQueries := []model.Query{ { - FromClause: "kibana_sample_data_logs", + FromClause: model.NewSelectColumnString("kibana_sample_data_logs"), + TableName: "kibana_sample_data_logs", Columns: []model.SelectColumn{{ Expression: aexp.Wildcard, }, @@ -49,7 +50,8 @@ func Test_ipRangeTransform(t *testing.T) { }, }, { - FromClause: "kibana_sample_data_logs", + FromClause: model.NewSelectColumnString("kibana_sample_data_logs"), + TableName: "kibana_sample_data_logs", Columns: []model.SelectColumn{{ Expression: aexp.Wildcard, }, @@ -61,7 +63,8 @@ func Test_ipRangeTransform(t *testing.T) { }, }, { - FromClause: "kibana_sample_data_logs", + FromClause: model.NewSelectColumnString("kibana_sample_data_logs"), + TableName: "kibana_sample_data_logs", Columns: []model.SelectColumn{{ Expression: aexp.Wildcard, }, @@ -81,9 +84,12 @@ func Test_ipRangeTransform(t *testing.T) { }, }, } + queries := [][]model.Query{ { - {FromClause: "kibana_sample_data_logs", + { + FromClause: model.NewSelectColumnString("kibana_sample_data_logs"), + TableName: "kibana_sample_data_logs", Columns: []model.SelectColumn{{ Expression: aexp.Wildcard, }, @@ -96,7 +102,9 @@ func Test_ipRangeTransform(t *testing.T) { }, }, { - {FromClause: "kibana_sample_data_logs", + { + FromClause: model.NewSelectColumnString("kibana_sample_data_logs"), + TableName: "kibana_sample_data_logs", Columns: []model.SelectColumn{{ Expression: aexp.Wildcard, }, @@ -109,7 +117,9 @@ func Test_ipRangeTransform(t *testing.T) { }, }, { - {FromClause: "kibana_sample_data_logs", + { + FromClause: model.NewSelectColumnString("kibana_sample_data_logs"), + TableName: "kibana_sample_data_logs", Columns: []model.SelectColumn{{ Expression: aexp.Wildcard, }, diff --git a/quesma/quesma/search_norace_test.go b/quesma/quesma/search_norace_test.go index 2887180d5..aa91b2e5c 100644 --- a/quesma/quesma/search_norace_test.go +++ b/quesma/quesma/search_norace_test.go @@ -8,7 +8,6 @@ package quesma import ( "context" - "github.com/DATA-DOG/go-sqlmock" "github.com/stretchr/testify/assert" "math/rand" "mitmproxy/quesma/clickhouse" @@ -21,6 +20,7 @@ import ( "mitmproxy/quesma/telemetry" "mitmproxy/quesma/testdata" "mitmproxy/quesma/tracing" + "mitmproxy/quesma/util" "testing" "time" ) @@ -34,12 +34,8 @@ func TestAllUnsupportedQueryTypesAreProperlyRecorded(t *testing.T) { if tt.QueryType == "script" { t.Skip("Only 1 test. We can't deal with scripts inside queries yet. It fails very early, during JSON unmarshalling, so we can't even know the type of aggregation.") } - db, _, err := sqlmock.New() - if err != nil { - t.Fatal(err) - } + db, _ := util.InitSqlMockWithPrettyPrint(t) defer db.Close() - assert.NoError(t, err) lm := clickhouse.NewLogManagerWithConnection(db, table) cfg := config.QuesmaConfiguration{IndexConfig: map[string]config.IndexConfiguration{tableName: {Enabled: true}}} @@ -89,12 +85,8 @@ func TestDifferentUnsupportedQueries(t *testing.T) { testCounts[randInt]++ } - db, _, err := sqlmock.New() - if err != nil { - t.Fatal(err) - } + db, _ := util.InitSqlMockWithPrettyPrint(t) defer db.Close() - assert.NoError(t, err) lm := clickhouse.NewLogManagerWithConnection(db, table) cfg := config.QuesmaConfiguration{IndexConfig: map[string]config.IndexConfiguration{tableName: {Enabled: true}}} diff --git a/quesma/quesma/search_opensearch_test.go b/quesma/quesma/search_opensearch_test.go index a1130f110..d3357d9cd 100644 --- a/quesma/quesma/search_opensearch_test.go +++ b/quesma/quesma/search_opensearch_test.go @@ -2,7 +2,6 @@ package quesma import ( "context" - "fmt" "github.com/DATA-DOG/go-sqlmock" "github.com/stretchr/testify/assert" "mitmproxy/quesma/clickhouse" @@ -35,13 +34,8 @@ func TestSearchOpensearch(t *testing.T) { for i, tt := range testdata.OpensearchSearchTests { t.Run(strconv.Itoa(i)+tt.Name, func(t *testing.T) { - queryMatcher := sqlmock.QueryMatcherFunc(func(expectedSQL, actualSQL string) error { - fmt.Printf("actual SQL: %s\n", actualSQL) - return sqlmock.QueryMatcherRegexp.Match(expectedSQL, actualSQL) - }) - db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(queryMatcher)) + db, mock := util.InitSqlMockWithPrettyPrint(t) mock.MatchExpectationsInOrder(false) - assert.NoError(t, err) defer db.Close() lm := clickhouse.NewLogManagerWithConnection(db, concurrent.NewMapWith(tableName, &table)) managementConsole := ui.NewQuesmaManagementConsole(cfg, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent()) @@ -59,7 +53,7 @@ func TestSearchOpensearch(t *testing.T) { mock.ExpectQuery(testdata.EscapeBrackets(wantedRegex)).WillReturnRows(sqlmock.NewRows([]string{"@timestamp", "host.name"})) } queryRunner := NewQueryRunner(lm, cfg, nil, managementConsole) - _, err = queryRunner.handleSearch(ctx, tableName, types.MustJSON(tt.QueryJson)) + _, err := queryRunner.handleSearch(ctx, tableName, types.MustJSON(tt.QueryJson)) assert.NoError(t, err) if err = mock.ExpectationsWereMet(); err != nil { @@ -174,8 +168,7 @@ func TestHighlighter(t *testing.T) { Created: true, } - db, mock, err := sqlmock.New() - assert.NoError(t, err) + db, mock := util.InitSqlMockWithPrettyPrint(t) defer db.Close() lm := clickhouse.NewLogManagerWithConnection(db, concurrent.NewMapWith(tableName, &table)) managementConsole := ui.NewQuesmaManagementConsole(cfg, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent()) diff --git a/quesma/quesma/search_test.go b/quesma/quesma/search_test.go index ee60fa47f..b9110b18a 100644 --- a/quesma/quesma/search_test.go +++ b/quesma/quesma/search_test.go @@ -16,6 +16,7 @@ import ( "mitmproxy/quesma/telemetry" "mitmproxy/quesma/testdata" "mitmproxy/quesma/tracing" + "mitmproxy/quesma/util" "strconv" "strings" "testing" @@ -74,17 +75,9 @@ func TestAsyncSearchHandler(t *testing.T) { }) for i, tt := range testdata.TestsAsyncSearch { t.Run(strconv.Itoa(i)+tt.Name, func(t *testing.T) { - queryMatcher := sqlmock.QueryMatcherFunc(func(expectedSQL, actualSQL string) error { - fmt.Printf("actual SQL: %s\n", actualSQL) - return sqlmock.QueryMatcherRegexp.Match(expectedSQL, actualSQL) - }) - db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(queryMatcher)) + db, mock := util.InitSqlMockWithPrettyPrint(t) mock.MatchExpectationsInOrder(false) - if err != nil { - t.Fatal(err) - } defer db.Close() - assert.NoError(t, err) lm := clickhouse.NewLogManagerWithConnection(db, table) managementConsole := ui.NewQuesmaManagementConsole(cfg, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent()) @@ -103,7 +96,7 @@ func TestAsyncSearchHandler(t *testing.T) { mock.ExpectQuery(wantedRegex).WillReturnRows(sqlmock.NewRows([]string{"@timestamp", "host.name"})) } queryRunner := NewQueryRunner(lm, cfg, nil, managementConsole) - _, err = queryRunner.handleAsyncSearch(ctx, tableName, types.MustJSON(tt.QueryJson), defaultAsyncSearchTimeout, true) + _, err := queryRunner.handleAsyncSearch(ctx, tableName, types.MustJSON(tt.QueryJson), defaultAsyncSearchTimeout, true) assert.NoError(t, err) if err := mock.ExpectationsWereMet(); err != nil { @@ -128,13 +121,9 @@ func TestAsyncSearchHandlerSpecialCharacters(t *testing.T) { for i, tt := range testdata.AggregationTestsWithSpecialCharactersInFieldNames { t.Run(strconv.Itoa(i), func(t *testing.T) { - db, mock, err := sqlmock.New() + db, mock := util.InitSqlMockWithPrettyPrint(t) mock.MatchExpectationsInOrder(false) - if err != nil { - t.Fatal(err) - } defer db.Close() - assert.NoError(t, err) lm := clickhouse.NewLogManagerWithConnection(db, concurrent.NewMapWith(tableName, &table)) managementConsole := ui.NewQuesmaManagementConsole(cfg, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent()) @@ -143,7 +132,7 @@ func TestAsyncSearchHandlerSpecialCharacters(t *testing.T) { } queryRunner := NewQueryRunner(lm, cfg, nil, managementConsole) - _, err = queryRunner.handleAsyncSearch(ctx, tableName, types.MustJSON(tt.QueryRequestJson), defaultAsyncSearchTimeout, true) + _, err := queryRunner.handleAsyncSearch(ctx, tableName, types.MustJSON(tt.QueryRequestJson), defaultAsyncSearchTimeout, true) assert.NoError(t, err) if err = mock.ExpectationsWereMet(); err != nil { @@ -172,13 +161,8 @@ func TestSearchHandler(t *testing.T) { cfg := config.QuesmaConfiguration{IndexConfig: map[string]config.IndexConfiguration{tableName: {Enabled: true}}} for _, tt := range testdata.TestsSearch { t.Run(tt.Name, func(t *testing.T) { - - db, mock, err := sqlmock.New() - if err != nil { - t.Fatal(err) - } + db, mock := util.InitSqlMockWithPrettyPrint(t) defer db.Close() - assert.NoError(t, err) lm := clickhouse.NewLogManagerWithConnection(db, table) managementConsole := ui.NewQuesmaManagementConsole(cfg, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent()) @@ -201,12 +185,8 @@ func TestSearchHandlerNoAttrsConfig(t *testing.T) { cfg := config.QuesmaConfiguration{IndexConfig: map[string]config.IndexConfiguration{tableName: {Enabled: true}}} for _, tt := range testdata.TestsSearchNoAttrs { t.Run(tt.Name, func(t *testing.T) { - db, mock, err := sqlmock.New() - if err != nil { - t.Fatal(err) - } + db, mock := util.InitSqlMockWithPrettyPrint(t) defer db.Close() - assert.NoError(t, err) lm := clickhouse.NewLogManagerWithConnection(db, table) managementConsole := ui.NewQuesmaManagementConsole(cfg, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent()) @@ -227,13 +207,9 @@ func TestAsyncSearchFilter(t *testing.T) { cfg := config.QuesmaConfiguration{IndexConfig: map[string]config.IndexConfiguration{tableName: {Enabled: true}}} for _, tt := range testdata.TestSearchFilter { t.Run(tt.Name, func(t *testing.T) { - db, mock, err := sqlmock.New() + db, mock := util.InitSqlMockWithPrettyPrint(t) mock.MatchExpectationsInOrder(false) - if err != nil { - t.Fatal(err) - } defer db.Close() - assert.NoError(t, err) lm := clickhouse.NewLogManagerWithConnection(db, table) managementConsole := ui.NewQuesmaManagementConsole(cfg, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent()) @@ -295,19 +271,16 @@ func TestHandlingDateTimeFields(t *testing.T) { }` } expectedSelectStatementRegex := map[string]string{ - dateTimeTimestampField: "SELECT toInt64(toUnixTimestamp(`timestamp`)/60.000000), count() FROM", - dateTime64TimestampField: "SELECT toInt64(toUnixTimestamp64Milli(`timestamp64`)/60000), count() FROM", - dateTime64OurTimestampField: "SELECT toInt64(toUnixTimestamp64Milli(`@timestamp`)/60000), count() FROM", + dateTimeTimestampField: `SELECT toInt64(toUnixTimestamp("timestamp") / 60), count() FROM`, + dateTime64TimestampField: `SELECT toInt64(toUnixTimestamp64Milli("timestamp64") / 60000), count() FROM`, + dateTime64OurTimestampField: `SELECT toInt64(toUnixTimestamp64Milli("@timestamp") / 60000), count() FROM`, } - db, mock, err := sqlmock.New() + db, mock := util.InitSqlMockWithPrettyPrint(t) // queries can be run in parallel mock.MatchExpectationsInOrder(false) - if err != nil { - t.Fatal(err) - } defer db.Close() lm := clickhouse.NewLogManagerWithConnection(db, concurrent.NewMapWith(tableName, &table)) managementConsole := ui.NewQuesmaManagementConsole(cfg, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent()) @@ -362,12 +335,8 @@ func TestNumericFacetsQueries(t *testing.T) { for i, tt := range testdata.TestsNumericFacets { for _, handlerName := range handlers { t.Run(strconv.Itoa(i)+tt.Name, func(t *testing.T) { - db, mock, err := sqlmock.New() - if err != nil { - t.Fatal(err) - } + db, mock := util.InitSqlMockWithPrettyPrint(t) defer db.Close() - assert.NoError(t, err) lm := clickhouse.NewLogManagerWithConnection(db, table) managementConsole := ui.NewQuesmaManagementConsole(cfg, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent()) @@ -381,6 +350,7 @@ func TestNumericFacetsQueries(t *testing.T) { queryRunner := NewQueryRunner(lm, cfg, nil, managementConsole) var response []byte + var err error if handlerName == "handleSearch" { response, err = queryRunner.handleSearch(ctx, tableName, types.MustJSON(tt.QueryJson)) } else if handlerName == "handleAsyncSearch" { diff --git a/quesma/quesma/termsenum/terms_enum_test.go b/quesma/quesma/termsenum/terms_enum_test.go index ccf6afc29..5b3ced561 100644 --- a/quesma/quesma/termsenum/terms_enum_test.go +++ b/quesma/quesma/termsenum/terms_enum_test.go @@ -15,6 +15,7 @@ import ( "mitmproxy/quesma/quesma/ui" "mitmproxy/quesma/telemetry" "mitmproxy/quesma/tracing" + "mitmproxy/quesma/util" "regexp" "testing" ) @@ -76,7 +77,7 @@ func TestHandleTermsEnumRequest(t *testing.T) { } managementConsole := ui.NewQuesmaManagementConsole(config.QuesmaConfiguration{}, nil, nil, make(<-chan tracing.LogWithLevel, 50000), telemetry.NewPhoneHomeEmptyAgent()) - db, mock, _ := sqlmock.New() + db, mock := util.InitSqlMockWithPrettyPrint(t) defer db.Close() lm := clickhouse.NewLogManagerWithConnection(db, concurrent.NewMapWith(testTableName, table)) qt := &queryparser.ClickhouseQueryTranslator{ClickhouseLM: lm, Table: table, Ctx: context.Background()} diff --git a/quesma/testdata/aggregation_requests.go b/quesma/testdata/aggregation_requests.go index 4c7ad503b..f87605f36 100644 --- a/quesma/testdata/aggregation_requests.go +++ b/quesma/testdata/aggregation_requests.go @@ -3,10 +3,16 @@ package testdata import ( "mitmproxy/quesma/clickhouse" "mitmproxy/quesma/model" + "mitmproxy/quesma/queryparser/aexp" "time" ) -var timestampGroupByClause = clickhouse.TimestampGroupBy("@timestamp", clickhouse.DateTime64, 30*time.Second) +var timestampGroupByClause = aexp.RenderSQL(clickhouse.TimestampGroupBy( + model.NewSelectColumnTableField("@timestamp"), clickhouse.DateTime64, 30*time.Second)) + +func groupBySQL(fieldName string, typ clickhouse.DateTimeType, groupByInterval time.Duration) string { + return aexp.RenderSQL(clickhouse.TimestampGroupBy(model.NewSelectColumnTableField(fieldName), typ, groupByInterval)) +} var AggregationTests = []AggregationTestCase{ { // [0] @@ -531,12 +537,12 @@ var AggregationTests = []AggregationTestCase{ `SELECT count() FROM ` + QuotedTableName + ` ` + `WHERE ("timestamp">=parseDateTime64BestEffort('2024-02-02T13:47:16.029Z') ` + `AND "timestamp"<=parseDateTime64BestEffort('2024-02-09T13:47:16.029Z'))`, - `SELECT "FlightDelayType", ` + clickhouse.TimestampGroupBy("timestamp", clickhouse.DateTime64, 3*time.Hour) + `, count() ` + + `SELECT "FlightDelayType", ` + groupBySQL("timestamp", clickhouse.DateTime64, 3*time.Hour) + `, count() ` + `FROM ` + QuotedTableName + ` ` + `WHERE ("timestamp">=parseDateTime64BestEffort('2024-02-02T13:47:16.029Z') ` + `AND "timestamp"<=parseDateTime64BestEffort('2024-02-09T13:47:16.029Z')) ` + - `GROUP BY "FlightDelayType", ` + clickhouse.TimestampGroupBy("timestamp", clickhouse.DateTime64, 3*time.Hour) + ` ` + - `ORDER BY "FlightDelayType", ` + clickhouse.TimestampGroupBy("timestamp", clickhouse.DateTime64, 3*time.Hour), + `GROUP BY "FlightDelayType", ` + groupBySQL("timestamp", clickhouse.DateTime64, 3*time.Hour) + ` ` + + `ORDER BY "FlightDelayType", ` + groupBySQL("timestamp", clickhouse.DateTime64, 3*time.Hour), `SELECT "FlightDelayType", count() FROM ` + QuotedTableName + ` ` + `WHERE ("timestamp">=parseDateTime64BestEffort('2024-02-02T13:47:16.029Z') ` + `AND "timestamp"<=parseDateTime64BestEffort('2024-02-09T13:47:16.029Z')) ` + @@ -1693,12 +1699,12 @@ var AggregationTests = []AggregationTestCase{ `WHERE ("host.name" iLIKE '%prometheus%' ` + `AND ("@timestamp"<=parseDateTime64BestEffort('2024-02-09T16:36:49.940Z') ` + `AND "@timestamp">=parseDateTime64BestEffort('2024-02-02T16:36:49.940Z')))`, - `SELECT "severity", toInt64(toUnixTimestamp64Milli(` + "`@timestamp`" + `)/10800000), count() FROM ` + QuotedTableName + ` ` + + `SELECT "severity", toInt64(toUnixTimestamp64Milli("@timestamp") / 10800000), count() FROM ` + QuotedTableName + ` ` + `WHERE ("host.name" iLIKE '%prometheus%' ` + `AND ("@timestamp">=parseDateTime64BestEffort('2024-02-02T16:36:49.940Z') ` + `AND "@timestamp"<=parseDateTime64BestEffort('2024-02-09T16:36:49.940Z'))) ` + - `GROUP BY "severity", toInt64(toUnixTimestamp64Milli(` + "`@timestamp`)/10800000)" + ` ` + - `ORDER BY "severity", toInt64(toUnixTimestamp64Milli(` + "`@timestamp`)/10800000)", + `GROUP BY "severity", toInt64(toUnixTimestamp64Milli("@timestamp") / 10800000)` + ` ` + + `ORDER BY "severity", toInt64(toUnixTimestamp64Milli("@timestamp") / 10800000)`, `SELECT "severity", count() FROM ` + QuotedTableName + ` ` + `WHERE ("host.name" iLIKE '%prometheus%' ` + `AND ("@timestamp">=parseDateTime64BestEffort('2024-02-02T16:36:49.940Z') ` + @@ -1977,33 +1983,33 @@ var AggregationTests = []AggregationTestCase{ []string{ `SELECT count() FROM ` + QuotedTableName + ` WHERE ("order_date">=parseDateTime64BestEffort('2024-02-06T09:59:57.034Z') ` + `AND "order_date"<=parseDateTime64BestEffort('2024-02-13T09:59:57.034Z'))`, - "SELECT toInt64(toUnixTimestamp64Milli(`order_date`)/43200000), " + `maxOrNull("order_date") AS "windowed_order_date", ` + + `SELECT toInt64(toUnixTimestamp64Milli("order_date") / 43200000), maxOrNull("order_date") AS "windowed_order_date", ` + `maxOrNull("order_date") AS "windowed_order_date" FROM ` + - `(SELECT "order_date", "order_date", ROW_NUMBER() OVER ` + - "(PARTITION BY toInt64(toUnixTimestamp64Milli(`order_date`)/43200000) " + - `ORDER BY "order_date" asc) AS row_number FROM ` + QuotedTableName + " " + + `(SELECT "order_date" , "order_date" , ROW_NUMBER() OVER ` + + `(PARTITION BY toInt64(toUnixTimestamp64Milli("order_date") / 43200000) ` + + `ORDER BY "order_date" ASC ) AS 'row_number' FROM ` + QuotedTableName + " " + `WHERE (("order_date">=parseDateTime64BestEffort('2024-02-06T09:59:57.034Z') AND ` + `"order_date"<=parseDateTime64BestEffort('2024-02-13T09:59:57.034Z')) AND "taxful_total_price" > '250')) ` + `WHERE ((("order_date">=parseDateTime64BestEffort('2024-02-06T09:59:57.034Z') AND ` + `"order_date"<=parseDateTime64BestEffort('2024-02-13T09:59:57.034Z')) AND "taxful_total_price" > '250') AND "row_number"<=10) ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`order_date`)/43200000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`order_date`)/43200000)", - "SELECT toInt64(toUnixTimestamp64Milli(`order_date`)/43200000), " + `maxOrNull("taxful_total_price") AS "windowed_taxful_total_price", ` + + `GROUP BY toInt64(toUnixTimestamp64Milli("order_date") / 43200000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("order_date") / 43200000)`, + `SELECT toInt64(toUnixTimestamp64Milli("order_date") / 43200000), maxOrNull("taxful_total_price") AS "windowed_taxful_total_price", ` + `maxOrNull("order_date") AS "windowed_order_date" FROM ` + - `(SELECT "taxful_total_price", "order_date", ROW_NUMBER() OVER ` + - "(PARTITION BY toInt64(toUnixTimestamp64Milli(`order_date`)/43200000) " + - `ORDER BY "order_date" asc) AS row_number FROM ` + QuotedTableName + " " + + `(SELECT "taxful_total_price" , "order_date" , ROW_NUMBER() OVER ` + + `(PARTITION BY toInt64(toUnixTimestamp64Milli("order_date") / 43200000) ` + + `ORDER BY "order_date" ASC ) AS 'row_number' FROM ` + QuotedTableName + " " + `WHERE (("order_date">=parseDateTime64BestEffort('2024-02-06T09:59:57.034Z') AND ` + `"order_date"<=parseDateTime64BestEffort('2024-02-13T09:59:57.034Z')) AND "taxful_total_price" > '250')) ` + `WHERE ((("order_date">=parseDateTime64BestEffort('2024-02-06T09:59:57.034Z') AND ` + `"order_date"<=parseDateTime64BestEffort('2024-02-13T09:59:57.034Z')) AND "taxful_total_price" > '250') AND "row_number"<=10) ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`order_date`)/43200000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`order_date`)/43200000)", - "SELECT toInt64(toUnixTimestamp64Milli(`order_date`)/43200000), count() FROM " + QuotedTableName + " " + + `GROUP BY toInt64(toUnixTimestamp64Milli("order_date") / 43200000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("order_date") / 43200000)`, + `SELECT toInt64(toUnixTimestamp64Milli("order_date") / 43200000), count() FROM ` + QuotedTableName + " " + `WHERE (("order_date">=parseDateTime64BestEffort('2024-02-06T09:59:57.034Z') AND ` + `"order_date"<=parseDateTime64BestEffort('2024-02-13T09:59:57.034Z')) AND "taxful_total_price" > '250') ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`order_date`)/43200000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`order_date`)/43200000)", + `GROUP BY toInt64(toUnixTimestamp64Milli("order_date") / 43200000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("order_date") / 43200000)`, `SELECT count() FROM ` + QuotedTableName + ` WHERE (("order_date">=parseDateTime64BestEffort('2024-02-06T09:59:57.034Z') ` + `AND "order_date"<=parseDateTime64BestEffort('2024-02-13T09:59:57.034Z')) AND "taxful_total_price" > '250')`, }, @@ -2397,11 +2403,11 @@ var AggregationTests = []AggregationTestCase{ `SELECT count() FROM ` + QuotedTableName + ` ` + `WHERE ("@timestamp">parseDateTime64BestEffort('2024-01-25T14:53:59.033Z') ` + `AND "@timestamp"<=parseDateTime64BestEffort('2024-01-25T15:08:59.033Z'))`, - `SELECT "event.dataset", ` + "toInt64(toUnixTimestamp64Milli(`@timestamp`)/60000), count() " + + `SELECT "event.dataset", toInt64(toUnixTimestamp64Milli("@timestamp") / 60000), count() ` + `FROM ` + QuotedTableName + ` ` + `WHERE ("@timestamp">parseDateTime64BestEffort('2024-01-25T14:53:59.033Z') AND "@timestamp"<=parseDateTime64BestEffort('2024-01-25T15:08:59.033Z')) ` + - `GROUP BY "event.dataset", ` + "toInt64(toUnixTimestamp64Milli(`@timestamp`)/60000) " + - `ORDER BY "event.dataset", ` + "toInt64(toUnixTimestamp64Milli(`@timestamp`)/60000)", + `GROUP BY "event.dataset", toInt64(toUnixTimestamp64Milli("@timestamp") / 60000) ` + + `ORDER BY "event.dataset", toInt64(toUnixTimestamp64Milli("@timestamp") / 60000)`, `SELECT "event.dataset", count() FROM ` + QuotedTableName + ` ` + `WHERE ("@timestamp">parseDateTime64BestEffort('2024-01-25T14:53:59.033Z') ` + `AND "@timestamp"<=parseDateTime64BestEffort('2024-01-25T15:08:59.033Z')) ` + @@ -2635,19 +2641,19 @@ var AggregationTests = []AggregationTestCase{ `SELECT count() FROM ` + QuotedTableName + ` ` + `WHERE ("order_date">=parseDateTime64BestEffort('2024-02-19T17:40:56.351Z') ` + `AND "order_date"<=parseDateTime64BestEffort('2024-02-26T17:40:56.351Z'))`, - `SELECT ` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 24*time.Hour) + `, ` + + `SELECT ` + groupBySQL("order_date", clickhouse.DateTime64, 24*time.Hour) + `, ` + `sumOrNull("taxful_total_price") ` + `FROM ` + QuotedTableName + ` ` + `WHERE ("order_date">=parseDateTime64BestEffort('2024-02-19T17:40:56.351Z') ` + `AND "order_date"<=parseDateTime64BestEffort('2024-02-26T17:40:56.351Z')) ` + - `GROUP BY ` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 24*time.Hour) + ` ` + - `ORDER BY ` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 24*time.Hour), - `SELECT ` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 24*time.Hour) + `, count() ` + + `GROUP BY ` + groupBySQL("order_date", clickhouse.DateTime64, 24*time.Hour) + ` ` + + `ORDER BY ` + groupBySQL("order_date", clickhouse.DateTime64, 24*time.Hour), + `SELECT ` + groupBySQL("order_date", clickhouse.DateTime64, 24*time.Hour) + `, count() ` + `FROM ` + QuotedTableName + ` ` + `WHERE ("order_date">=parseDateTime64BestEffort('2024-02-19T17:40:56.351Z') ` + `AND "order_date"<=parseDateTime64BestEffort('2024-02-26T17:40:56.351Z')) ` + - `GROUP BY ` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 24*time.Hour) + ` ` + - `ORDER BY ` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 24*time.Hour), + `GROUP BY ` + groupBySQL("order_date", clickhouse.DateTime64, 24*time.Hour) + ` ` + + `ORDER BY ` + groupBySQL("order_date", clickhouse.DateTime64, 24*time.Hour), }, }, { // [16] @@ -2926,27 +2932,27 @@ var AggregationTests = []AggregationTestCase{ `SELECT count() FROM ` + QuotedTableName + ` ` + `WHERE ("order_date">=parseDateTime64BestEffort('2024-02-22T18:47:34.149Z') ` + `AND "order_date"<=parseDateTime64BestEffort('2024-02-29T18:47:34.149Z'))`, - `SELECT ` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 12*time.Hour) + `, ` + + `SELECT ` + groupBySQL("order_date", clickhouse.DateTime64, 12*time.Hour) + `, ` + `sumOrNull("taxful_total_price") ` + `FROM ` + QuotedTableName + ` ` + `WHERE (("order_date">=parseDateTime64BestEffort('2024-02-22T18:47:34.149Z') ` + `AND "order_date"<=parseDateTime64BestEffort('2024-02-29T18:47:34.149Z')) ` + `AND "products.product_name" ILIKE '%watch%') ` + - `GROUP BY ` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 12*time.Hour) + ` ` + - `ORDER BY ` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 12*time.Hour), - `SELECT ` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 12*time.Hour) + `, count() ` + + `GROUP BY ` + groupBySQL("order_date", clickhouse.DateTime64, 12*time.Hour) + ` ` + + `ORDER BY ` + groupBySQL("order_date", clickhouse.DateTime64, 12*time.Hour), + `SELECT ` + groupBySQL("order_date", clickhouse.DateTime64, 12*time.Hour) + `, count() ` + `FROM ` + QuotedTableName + ` ` + `WHERE (("order_date">=parseDateTime64BestEffort('2024-02-22T18:47:34.149Z') ` + `AND "order_date"<=parseDateTime64BestEffort('2024-02-29T18:47:34.149Z')) ` + `AND "products.product_name" ILIKE '%watch%') ` + - `GROUP BY ` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 12*time.Hour) + ` ` + - `ORDER BY ` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 12*time.Hour), - `SELECT ` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 12*time.Hour) + `, count() ` + + `GROUP BY ` + groupBySQL("order_date", clickhouse.DateTime64, 12*time.Hour) + ` ` + + `ORDER BY ` + groupBySQL("order_date", clickhouse.DateTime64, 12*time.Hour), + `SELECT ` + groupBySQL("order_date", clickhouse.DateTime64, 12*time.Hour) + `, count() ` + `FROM ` + QuotedTableName + ` ` + `WHERE ("order_date">=parseDateTime64BestEffort('2024-02-22T18:47:34.149Z') ` + `AND "order_date"<=parseDateTime64BestEffort('2024-02-29T18:47:34.149Z')) ` + - `GROUP BY ` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 12*time.Hour) + ` ` + - `ORDER BY ` + clickhouse.TimestampGroupBy("order_date", clickhouse.DateTime64, 12*time.Hour), + `GROUP BY ` + groupBySQL("order_date", clickhouse.DateTime64, 12*time.Hour) + ` ` + + `ORDER BY ` + groupBySQL("order_date", clickhouse.DateTime64, 12*time.Hour), }, }, { // [18] @@ -3175,7 +3181,7 @@ var AggregationTests = []AggregationTestCase{ `AND "order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z')) ` + `OR ("order_date">=parseDateTime64BestEffort('2024-02-15T21:57:36.376Z') ` + `AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z')))`, - "SELECT toInt64(toUnixTimestamp64Milli(`order_date`)/86400000), " + `sumOrNull("taxful_total_price") ` + + `SELECT toInt64(toUnixTimestamp64Milli("order_date") / 86400000), sumOrNull("taxful_total_price") ` + `FROM ` + QuotedTableName + ` ` + `WHERE ((("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') ` + `AND "order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z')) ` + @@ -3183,9 +3189,9 @@ var AggregationTests = []AggregationTestCase{ `AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z'))) ` + `AND ("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') ` + `AND "order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z'))) ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`order_date`)/86400000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`order_date`)/86400000)", - "SELECT toInt64(toUnixTimestamp64Milli(`order_date`)/86400000), " + + `GROUP BY toInt64(toUnixTimestamp64Milli("order_date") / 86400000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("order_date") / 86400000)`, + `SELECT toInt64(toUnixTimestamp64Milli("order_date") / 86400000), ` + `sumOrNull("taxful_total_price") ` + `FROM ` + QuotedTableName + ` ` + `WHERE ((("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') ` + @@ -3194,9 +3200,9 @@ var AggregationTests = []AggregationTestCase{ `AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z'))) ` + `AND ("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') ` + `AND "order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z'))) ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`order_date`)/86400000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`order_date`)/86400000)", - "SELECT toInt64(toUnixTimestamp64Milli(`order_date`)/86400000), count() " + + `GROUP BY toInt64(toUnixTimestamp64Milli("order_date") / 86400000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("order_date") / 86400000)`, + `SELECT toInt64(toUnixTimestamp64Milli("order_date") / 86400000), count() ` + `FROM ` + QuotedTableName + ` ` + `WHERE ((("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') ` + `AND "order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z')) ` + @@ -3204,8 +3210,8 @@ var AggregationTests = []AggregationTestCase{ `AND "order_date">=parseDateTime64BestEffort('2024-02-15T21:57:36.376Z'))) ` + `AND ("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') ` + `AND "order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z'))) ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`order_date`)/86400000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`order_date`)/86400000)", + `GROUP BY toInt64(toUnixTimestamp64Milli("order_date") / 86400000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("order_date") / 86400000)`, `SELECT count() ` + `FROM ` + QuotedTableName + ` ` + `WHERE ((("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') ` + @@ -3214,7 +3220,7 @@ var AggregationTests = []AggregationTestCase{ `AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z'))) ` + `AND ("order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z') ` + `AND "order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z')))`, - "SELECT toInt64(toUnixTimestamp64Milli(`order_date`)/86400000), " + + `SELECT toInt64(toUnixTimestamp64Milli("order_date") / 86400000), ` + `sumOrNull("taxful_total_price") ` + `FROM ` + QuotedTableName + ` ` + `WHERE ((("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') ` + @@ -3223,9 +3229,9 @@ var AggregationTests = []AggregationTestCase{ `AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z'))) ` + `AND ("order_date">=parseDateTime64BestEffort('2024-02-15T21:57:36.376Z') ` + `AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z'))) ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`order_date`)/86400000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`order_date`)/86400000)", - "SELECT toInt64(toUnixTimestamp64Milli(`order_date`)/86400000), " + + `GROUP BY toInt64(toUnixTimestamp64Milli("order_date") / 86400000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("order_date") / 86400000)`, + `SELECT toInt64(toUnixTimestamp64Milli("order_date") / 86400000), ` + `sumOrNull("taxful_total_price") ` + `FROM ` + QuotedTableName + ` ` + `WHERE ((("order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z') ` + @@ -3234,9 +3240,9 @@ var AggregationTests = []AggregationTestCase{ `AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z'))) ` + `AND ("order_date">=parseDateTime64BestEffort('2024-02-15T21:57:36.376Z') ` + `AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z'))) ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`order_date`)/86400000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`order_date`)/86400000)", - "SELECT toInt64(toUnixTimestamp64Milli(`order_date`)/86400000), count() " + + `GROUP BY toInt64(toUnixTimestamp64Milli("order_date") / 86400000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("order_date") / 86400000)`, + `SELECT toInt64(toUnixTimestamp64Milli("order_date") / 86400000), count() ` + `FROM ` + QuotedTableName + ` ` + `WHERE ((("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') ` + `AND "order_date"<=parseDateTime64BestEffort('2024-02-29T21:57:36.376Z')) ` + @@ -3244,8 +3250,8 @@ var AggregationTests = []AggregationTestCase{ `AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z'))) ` + `AND ("order_date">=parseDateTime64BestEffort('2024-02-15T21:57:36.376Z') ` + `AND "order_date"<=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z'))) ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`order_date`)/86400000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`order_date`)/86400000)", + `GROUP BY toInt64(toUnixTimestamp64Milli("order_date") / 86400000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("order_date") / 86400000)`, `SELECT count() ` + `FROM ` + QuotedTableName + ` ` + `WHERE ((("order_date">=parseDateTime64BestEffort('2024-02-22T21:57:36.376Z') ` + @@ -3363,12 +3369,12 @@ var AggregationTests = []AggregationTestCase{ `SELECT count() FROM ` + QuotedTableName + ` ` + `WHERE (toUnixTimestamp64Milli("@timestamp")>=1.709815794995e+12 ` + `AND toUnixTimestamp64Milli("@timestamp")<=1.709816694995e+12)`, - `SELECT ` + clickhouse.TimestampGroupBy("@timestamp", clickhouse.DateTime64, 15*time.Second) + `, count() ` + + `SELECT ` + groupBySQL("@timestamp", clickhouse.DateTime64, 15*time.Second) + `, count() ` + `FROM ` + QuotedTableName + ` ` + `WHERE (toUnixTimestamp64Milli("@timestamp")>=1.709815794995e+12 ` + `AND toUnixTimestamp64Milli("@timestamp")<=1.709816694995e+12) ` + - `GROUP BY ` + clickhouse.TimestampGroupBy("@timestamp", clickhouse.DateTime64, 15*time.Second) + ` ` + - `ORDER BY ` + clickhouse.TimestampGroupBy("@timestamp", clickhouse.DateTime64, 15*time.Second), + `GROUP BY ` + groupBySQL("@timestamp", clickhouse.DateTime64, 15*time.Second) + ` ` + + `ORDER BY ` + groupBySQL("@timestamp", clickhouse.DateTime64, 15*time.Second), `SELECT count() FROM ` + QuotedTableName + ` ` + `WHERE (toUnixTimestamp64Milli("@timestamp")>=1.709815794995e+12 ` + `AND toUnixTimestamp64Milli("@timestamp")<=1.709816694995e+12)`, @@ -3840,16 +3846,16 @@ var AggregationTests = []AggregationTestCase{ ExpectedSQLs: []string{ `SELECT count() FROM ` + QuotedTableName + ` WHERE ("timestamp">=parseDateTime64BestEffort('2024-04-16T12:15:11.790Z') ` + `AND "timestamp"<=parseDateTime64BestEffort('2024-04-16T12:30:11.790Z'))`, - `SELECT count(if("bytes_gauge">=0 AND "bytes_gauge"<1000, 1, NULL)), ` + - `count(if("bytes_gauge">=1000 AND "bytes_gauge"<2000, 1, NULL)), ` + - `count(if("bytes_gauge">=-5.5, 1, NULL)), ` + - `count(if("bytes_gauge"<6.555, 1, NULL)), ` + + `SELECT count(if("bytes_gauge" >= 0.000000 AND "bytes_gauge" < 1000.000000, 1, NULL)), ` + + `count(if("bytes_gauge" >= 1000.000000 AND "bytes_gauge" < 2000.000000, 1, NULL)), ` + + `count(if("bytes_gauge" >= -5.500000, 1, NULL)), ` + + `count(if("bytes_gauge" < 6.555000, 1, NULL)), ` + `count(), count() FROM ` + QuotedTableName + ` WHERE ("timestamp">=parseDateTime64BestEffort('2024-04-16T12:15:11.790Z') ` + `AND "timestamp"<=parseDateTime64BestEffort('2024-04-16T12:30:11.790Z'))`, - `SELECT count(if("bytes_gauge">=0 AND "bytes_gauge"<1000, 1, NULL)), ` + - `count(if("bytes_gauge">=1000 AND "bytes_gauge"<2000, 1, NULL)), ` + - `count(if("bytes_gauge">=-5.5, 1, NULL)), ` + - `count(if("bytes_gauge"<6.555, 1, NULL)), ` + + `SELECT count(if("bytes_gauge" >= 0.000000 AND "bytes_gauge" < 1000.000000, 1, NULL)), ` + + `count(if("bytes_gauge" >= 1000.000000 AND "bytes_gauge" < 2000.000000, 1, NULL)), ` + + `count(if("bytes_gauge" >= -5.500000, 1, NULL)), ` + + `count(if("bytes_gauge" < 6.555000, 1, NULL)), ` + `count(), count() FROM ` + QuotedTableName + ` WHERE ("timestamp">=parseDateTime64BestEffort('2024-04-16T12:15:11.790Z') ` + `AND "timestamp"<=parseDateTime64BestEffort('2024-04-16T12:30:11.790Z'))`, }, @@ -4184,14 +4190,14 @@ var AggregationTests = []AggregationTestCase{ }, ExpectedSQLs: []string{ `SELECT count() FROM ` + QuotedTableName, - "SELECT toInt64(toUnixTimestamp64Milli(`@timestamp`)/79200000), " + `count(DISTINCT "host.name") ` + + `SELECT toInt64(toUnixTimestamp64Milli("@timestamp") / 79200000), count(DISTINCT "host.name") ` + `FROM ` + QuotedTableName + " " + - "GROUP BY toInt64(toUnixTimestamp64Milli(`@timestamp`)/79200000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`@timestamp`)/79200000)", - "SELECT toInt64(toUnixTimestamp64Milli(`@timestamp`)/79200000), count() " + + `GROUP BY toInt64(toUnixTimestamp64Milli("@timestamp") / 79200000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("@timestamp") / 79200000)`, + `SELECT toInt64(toUnixTimestamp64Milli("@timestamp") / 79200000), count() ` + `FROM ` + QuotedTableName + " " + - "GROUP BY toInt64(toUnixTimestamp64Milli(`@timestamp`)/79200000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`@timestamp`)/79200000)", + `GROUP BY toInt64(toUnixTimestamp64Milli("@timestamp") / 79200000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("@timestamp") / 79200000)`, }, }, { // [25] @@ -4470,12 +4476,12 @@ var AggregationTests = []AggregationTestCase{ `SELECT count() FROM ` + QuotedTableName + ` ` + `WHERE ("timestamp">=parseDateTime64BestEffort('2024-05-10T14:29:02.900Z') ` + `AND "timestamp"<=parseDateTime64BestEffort('2024-05-10T14:44:02.900Z'))`, - "SELECT toInt64(toUnixTimestamp64Milli(`timestamp`)/30000), count() " + + `SELECT toInt64(toUnixTimestamp64Milli("timestamp") / 30000), count() ` + `FROM ` + QuotedTableName + ` ` + `WHERE ("timestamp">=parseDateTime64BestEffort('2024-05-10T14:29:02.900Z') ` + `AND "timestamp"<=parseDateTime64BestEffort('2024-05-10T14:44:02.900Z')) ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`timestamp`)/30000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`timestamp`)/30000)", + `GROUP BY toInt64(toUnixTimestamp64Milli("timestamp") / 30000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("timestamp") / 30000)`, }, }, { // [27] @@ -5326,33 +5332,33 @@ var AggregationTests = []AggregationTestCase{ {{Cols: []model.QueryResultCol{model.NewQueryResultCol("hits", uint64(2167))}}}, { {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`@timestamp`)/86400000)", int64(1715212800000/86400000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("@timestamp") / 86400000)`, int64(1715212800000/86400000)), model.NewQueryResultCol(`"windowed_message"`, 5), model.NewQueryResultCol(`minOrNull("order_date")`, "2024-05-09T23:52:48Z"), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`@timestamp`)/86400000)", int64(1716336000000/86400000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("@timestamp") / 86400000)`, int64(1716336000000/86400000)), model.NewQueryResultCol(`windowed_message`, 30), model.NewQueryResultCol(`minOrNull("order_date")`, "2024-05-22T10:20:38Z"), }}, }, { {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`@timestamp`)/86400000)", int64(1715212800000/86400000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("@timestamp") / 86400000)`, int64(1715212800000/86400000)), model.NewQueryResultCol(`count()`, 146), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`@timestamp`)/86400000)", int64(1716336000000/86400000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("@timestamp") / 86400000)`, int64(1716336000000/86400000)), model.NewQueryResultCol(`count()`, 58), }}, }, { {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`@timestamp`)/86400000)", int64(1715212800000/86400000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("@timestamp") / 86400000)`, int64(1715212800000/86400000)), model.NewQueryResultCol(`count()`, 146), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`@timestamp`)/86400000)", int64(1716336000000/86400000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("@timestamp") / 86400000)`, int64(1716336000000/86400000)), model.NewQueryResultCol(`count()`, 58), }}, }, @@ -5360,30 +5366,30 @@ var AggregationTests = []AggregationTestCase{ ExpectedSQLs: []string{ `SELECT count() ` + `FROM ` + QuotedTableName, - "SELECT toInt64(toUnixTimestamp64Milli(`@timestamp`)/86400000), " + + `SELECT toInt64(toUnixTimestamp64Milli("@timestamp") / 86400000), ` + `minOrNull("message") AS "windowed_message", ` + `minOrNull("order_date") AS "windowed_order_date" ` + - `FROM (SELECT "message", "order_date", ROW_NUMBER() OVER ` + - "(PARTITION BY toInt64(toUnixTimestamp64Milli(`@timestamp`)/86400000) " + - `ORDER BY "order_date" desc) ` + - `AS row_number ` + + `FROM (SELECT "message" , "order_date" , ROW_NUMBER() OVER ` + + `(PARTITION BY toInt64(toUnixTimestamp64Milli("@timestamp") / 86400000) ` + + `ORDER BY "order_date" DESC ) ` + + `AS 'row_number' ` + `FROM ` + QuotedTableName + ` ` + `WHERE "message" IS NOT NULL) ` + `WHERE ("message" IS NOT NULL ` + `AND "row_number"<=1) ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`@timestamp`)/86400000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`@timestamp`)/86400000)", - "SELECT toInt64(toUnixTimestamp64Milli(`@timestamp`)/86400000), " + + `GROUP BY toInt64(toUnixTimestamp64Milli("@timestamp") / 86400000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("@timestamp") / 86400000)`, + `SELECT toInt64(toUnixTimestamp64Milli("@timestamp") / 86400000), ` + "count() " + `FROM ` + QuotedTableName + ` ` + `WHERE "message" IS NOT NULL ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`@timestamp`)/86400000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`@timestamp`)/86400000)", - "SELECT toInt64(toUnixTimestamp64Milli(`@timestamp`)/86400000), " + + `GROUP BY toInt64(toUnixTimestamp64Milli("@timestamp") / 86400000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("@timestamp") / 86400000)`, + `SELECT toInt64(toUnixTimestamp64Milli("@timestamp") / 86400000), ` + "count() " + `FROM ` + QuotedTableName + ` ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`@timestamp`)/86400000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`@timestamp`)/86400000)", + `GROUP BY toInt64(toUnixTimestamp64Milli("@timestamp") / 86400000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("@timestamp") / 86400000)`, }, }, { // [32] @@ -5662,7 +5668,7 @@ var AggregationTests = []AggregationTestCase{ `FROM ` + QuotedTableName + ` ` + `WHERE ("timestamp">=parseDateTime64BestEffort('2024-05-21T21:35:34.210Z') ` + `AND "timestamp"<=parseDateTime64BestEffort('2024-05-22T12:35:34.210Z'))`, - "SELECT toInt64(toUnixTimestamp64Milli(`timestamp`)/600000), " + + `SELECT toInt64(toUnixTimestamp64Milli("timestamp") / 600000), ` + `count("bytes"), ` + `minOrNull("bytes"), ` + `maxOrNull("bytes"), ` + @@ -5676,9 +5682,9 @@ var AggregationTests = []AggregationTestCase{ `FROM ` + QuotedTableName + ` ` + `WHERE ("timestamp">=parseDateTime64BestEffort('2024-05-21T21:35:34.210Z') ` + `AND "timestamp"<=parseDateTime64BestEffort('2024-05-22T12:35:34.210Z')) ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", - "SELECT toInt64(toUnixTimestamp64Milli(`timestamp`)/600000), " + + `GROUP BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, + `SELECT toInt64(toUnixTimestamp64Milli("timestamp") / 600000), ` + `count("bytes"), ` + `minOrNull("bytes"), ` + `maxOrNull("bytes"), ` + @@ -5692,15 +5698,15 @@ var AggregationTests = []AggregationTestCase{ `FROM ` + QuotedTableName + ` ` + `WHERE ("timestamp">=parseDateTime64BestEffort('2024-05-21T21:35:34.210Z') ` + `AND "timestamp"<=parseDateTime64BestEffort('2024-05-22T12:35:34.210Z')) ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", - "SELECT toInt64(toUnixTimestamp64Milli(`timestamp`)/600000), " + + `GROUP BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, + `SELECT toInt64(toUnixTimestamp64Milli("timestamp") / 600000), ` + `count() ` + `FROM ` + QuotedTableName + ` ` + `WHERE ("timestamp">=parseDateTime64BestEffort('2024-05-21T21:35:34.210Z') ` + `AND "timestamp"<=parseDateTime64BestEffort('2024-05-22T12:35:34.210Z')) ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", + `GROUP BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, }, }, { // [33] diff --git a/quesma/testdata/kibana-visualize/aggregation_requests.go b/quesma/testdata/kibana-visualize/aggregation_requests.go index 8ad85f96f..608c5155d 100644 --- a/quesma/testdata/kibana-visualize/aggregation_requests.go +++ b/quesma/testdata/kibana-visualize/aggregation_requests.go @@ -222,19 +222,19 @@ var AggregationTests = []testdata.AggregationTestCase{ `FROM ` + testdata.QuotedTableName + ` ` + `WHERE ("@timestamp">=parseDateTime64BestEffort('2024-05-27T11:59:56.627Z') ` + `AND "@timestamp"<=parseDateTime64BestEffort('2024-05-27T12:14:56.627Z'))`, - "SELECT toInt64(toUnixTimestamp64Milli(`@timestamp`)/30000), " + + `SELECT toInt64(toUnixTimestamp64Milli("@timestamp") / 30000), ` + `"severity", "source", count() ` + `FROM ` + testdata.QuotedTableName + ` ` + `WHERE ("@timestamp">=parseDateTime64BestEffort('2024-05-27T11:59:56.627Z') ` + `AND "@timestamp"<=parseDateTime64BestEffort('2024-05-27T12:14:56.627Z')) ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`@timestamp`)/30000), " + `"severity", "source" ` + - "ORDER BY toInt64(toUnixTimestamp64Milli(`@timestamp`)/30000), " + `"severity", "source"`, - "SELECT toInt64(toUnixTimestamp64Milli(`@timestamp`)/30000), count() " + + `GROUP BY toInt64(toUnixTimestamp64Milli("@timestamp") / 30000), ` + `"severity", "source" ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("@timestamp") / 30000), ` + `"severity", "source"`, + `SELECT toInt64(toUnixTimestamp64Milli("@timestamp") / 30000), count() ` + `FROM ` + testdata.QuotedTableName + ` ` + `WHERE ("@timestamp">=parseDateTime64BestEffort('2024-05-27T11:59:56.627Z') ` + `AND "@timestamp"<=parseDateTime64BestEffort('2024-05-27T12:14:56.627Z')) ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`@timestamp`)/30000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`@timestamp`)/30000)", + `GROUP BY toInt64(toUnixTimestamp64Milli("@timestamp") / 30000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("@timestamp") / 30000)`, }, }, { // [1] @@ -392,10 +392,10 @@ var AggregationTests = []testdata.AggregationTestCase{ }, ExpectedSQLs: []string{ `SELECT count() FROM ` + testdata.QuotedTableName, - `SELECT "message", "host.name", ` + "toInt64(toUnixTimestamp64Milli(`@timestamp`)/30000), count() " + + `SELECT "message", "host.name", toInt64(toUnixTimestamp64Milli("@timestamp") / 30000), count() ` + `FROM ` + testdata.QuotedTableName + ` ` + - `GROUP BY "message", "host.name", ` + "toInt64(toUnixTimestamp64Milli(`@timestamp`)/30000) " + - `ORDER BY "message", "host.name", ` + "toInt64(toUnixTimestamp64Milli(`@timestamp`)/30000)", + `GROUP BY "message", "host.name", toInt64(toUnixTimestamp64Milli("@timestamp") / 30000) ` + + `ORDER BY "message", "host.name", toInt64(toUnixTimestamp64Milli("@timestamp") / 30000)`, `SELECT "message", "host.name", count() ` + `FROM ` + testdata.QuotedTableName + ` ` + `GROUP BY "message", "host.name" ` + @@ -633,14 +633,14 @@ var AggregationTests = []testdata.AggregationTestCase{ }, ExpectedSQLs: []string{ `SELECT count() FROM ` + testdata.QuotedTableName, - `SELECT "severity", "source", ` + "toInt64(toUnixTimestamp64Milli(`@timestamp`)/30000), " + `count(DISTINCT "severity") ` + + `SELECT "severity", "source", toInt64(toUnixTimestamp64Milli("@timestamp") / 30000), count(DISTINCT "severity") ` + `FROM ` + testdata.QuotedTableName + ` ` + - `GROUP BY "severity", "source", ` + "toInt64(toUnixTimestamp64Milli(`@timestamp`)/30000) " + - `ORDER BY "severity", "source", ` + "toInt64(toUnixTimestamp64Milli(`@timestamp`)/30000)", - `SELECT "severity", "source", ` + "toInt64(toUnixTimestamp64Milli(`@timestamp`)/30000), count() " + + `GROUP BY "severity", "source", toInt64(toUnixTimestamp64Milli("@timestamp") / 30000) ` + + `ORDER BY "severity", "source", toInt64(toUnixTimestamp64Milli("@timestamp") / 30000)`, + `SELECT "severity", "source", toInt64(toUnixTimestamp64Milli("@timestamp") / 30000), count() ` + `FROM ` + testdata.QuotedTableName + ` ` + - `GROUP BY "severity", "source", ` + "toInt64(toUnixTimestamp64Milli(`@timestamp`)/30000) " + - `ORDER BY "severity", "source", ` + "toInt64(toUnixTimestamp64Milli(`@timestamp`)/30000)", + `GROUP BY "severity", "source", toInt64(toUnixTimestamp64Milli("@timestamp") / 30000) ` + + `ORDER BY "severity", "source", toInt64(toUnixTimestamp64Milli("@timestamp") / 30000)`, `SELECT "severity", "source", count(DISTINCT "severity") ` + `FROM ` + testdata.QuotedTableName + ` ` + `GROUP BY "severity", "source" ` + @@ -836,10 +836,10 @@ var AggregationTests = []testdata.AggregationTestCase{ }, ExpectedSQLs: []string{ `SELECT count() FROM ` + testdata.QuotedTableName, - `SELECT "Cancelled", "AvgTicketPrice", ` + "toInt64(toUnixTimestamp64Milli(`@timestamp`)/30000), count() " + + `SELECT "Cancelled", "AvgTicketPrice", toInt64(toUnixTimestamp64Milli("@timestamp") / 30000), count() ` + `FROM ` + testdata.QuotedTableName + ` ` + - `GROUP BY "Cancelled", "AvgTicketPrice", ` + "toInt64(toUnixTimestamp64Milli(`@timestamp`)/30000) " + - `ORDER BY "Cancelled", "AvgTicketPrice", ` + "toInt64(toUnixTimestamp64Milli(`@timestamp`)/30000)", + `GROUP BY "Cancelled", "AvgTicketPrice", toInt64(toUnixTimestamp64Milli("@timestamp") / 30000) ` + + `ORDER BY "Cancelled", "AvgTicketPrice", toInt64(toUnixTimestamp64Milli("@timestamp") / 30000)`, `SELECT "Cancelled", "AvgTicketPrice", count() ` + `FROM ` + testdata.QuotedTableName + ` ` + `GROUP BY "Cancelled", "AvgTicketPrice" ` + diff --git a/quesma/testdata/opensearch-visualize/aggregation_requests.go b/quesma/testdata/opensearch-visualize/aggregation_requests.go index c6f6a9592..0d651a14f 100644 --- a/quesma/testdata/opensearch-visualize/aggregation_requests.go +++ b/quesma/testdata/opensearch-visualize/aggregation_requests.go @@ -133,7 +133,7 @@ var AggregationTests = []testdata.AggregationTestCase{ `WHERE (("epoch_time">='2024-04-27T14:25:59.383Z' AND "epoch_time"<='2024-04-27T14:40:59.383Z') AND "ftd_session_time"<1000)`, `SELECT count(DISTINCT "ftd_session_time") FROM "logs-generic-default" ` + `WHERE (("epoch_time">='2024-04-27T14:25:59.383Z' AND "epoch_time"<='2024-04-27T14:40:59.383Z') AND "ftd_session_time">=-100)`, - `SELECT count(if("ftd_session_time"<1000, 1, NULL)), count(if("ftd_session_time">=-100, 1, NULL)), count() ` + + `SELECT count(if("ftd_session_time" < 1000.000000, 1, NULL)), count(if("ftd_session_time" >= -100.000000, 1, NULL)), count() ` + `FROM "logs-generic-default" WHERE ("epoch_time">='2024-04-27T14:25:59.383Z' AND "epoch_time"<='2024-04-27T14:40:59.383Z')`, }, }, @@ -471,8 +471,8 @@ var AggregationTests = []testdata.AggregationTestCase{ `SELECT sumOrNull("properties.entry_time") FROM "logs-generic-default" ` + `WHERE (("epoch_time">='2024-04-28T14:34:22.674Z' AND "epoch_time"<='2024-04-28T14:49:22.674Z') ` + `AND "epoch_time_original">=1000)`, - `SELECT count(if("epoch_time_original">=0 AND "epoch_time_original"<1000, 1, NULL)), ` + - `count(if("epoch_time_original">=1000, 1, NULL)), count() FROM "logs-generic-default" ` + + `SELECT count(if("epoch_time_original" >= 0.000000 AND "epoch_time_original" < 1000.000000, 1, NULL)), ` + + `count(if("epoch_time_original" >= 1000.000000, 1, NULL)), count() FROM "logs-generic-default" ` + `WHERE ("epoch_time">='2024-04-28T14:34:22.674Z' AND "epoch_time"<='2024-04-28T14:49:22.674Z')`, }, }, @@ -607,16 +607,16 @@ var AggregationTests = []testdata.AggregationTestCase{ ExpectedSQLs: []string{ `SELECT count() FROM ` + testdata.QuotedTableName + ` ` + `WHERE ("epoch_time">='2024-04-18T04:40:12.252Z' AND "epoch_time"<='2024-05-03T04:40:12.252Z')`, - "SELECT quantiles(0.500000)(\"properties::entry_time\") AS \"quantile_50\" " + + `SELECT quantiles(0.500000)("properties::entry_time") AS "quantile_50" ` + `FROM ` + testdata.QuotedTableName + ` ` + `WHERE (("epoch_time">='2024-04-18T04:40:12.252Z' AND "epoch_time"<='2024-05-03T04:40:12.252Z') ` + `AND ("properties::exoestimation_connection_speedinkbps">=0 AND "properties::exoestimation_connection_speedinkbps"<1000))`, - "SELECT quantiles(0.500000)(\"properties::entry_time\") AS \"quantile_50\" " + + `SELECT quantiles(0.500000)("properties::entry_time") AS "quantile_50" ` + `FROM ` + testdata.QuotedTableName + ` ` + `WHERE (("epoch_time">='2024-04-18T04:40:12.252Z' AND "epoch_time"<='2024-05-03T04:40:12.252Z') ` + `AND ("properties::exoestimation_connection_speedinkbps">=1000 AND "properties::exoestimation_connection_speedinkbps"<2000))`, - `SELECT count(if("properties::exoestimation_connection_speedinkbps">=0 AND "properties::exoestimation_connection_speedinkbps"<1000, 1, NULL)), ` + - `count(if("properties::exoestimation_connection_speedinkbps">=1000 AND "properties::exoestimation_connection_speedinkbps"<2000, 1, NULL)), ` + + `SELECT count(if("properties::exoestimation_connection_speedinkbps" >= 0.000000 AND "properties::exoestimation_connection_speedinkbps" < 1000.000000, 1, NULL)), ` + + `count(if("properties::exoestimation_connection_speedinkbps" >= 1000.000000 AND "properties::exoestimation_connection_speedinkbps" < 2000.000000, 1, NULL)), ` + `count() ` + `FROM ` + testdata.QuotedTableName + ` ` + `WHERE ("epoch_time">='2024-04-18T04:40:12.252Z' AND "epoch_time"<='2024-05-03T04:40:12.252Z')`, @@ -1264,16 +1264,16 @@ var AggregationTests = []testdata.AggregationTestCase{ }, ExpectedSQLs: []string{ `SELECT count() FROM ` + testdata.QuotedTableName, - "SELECT toInt64(toUnixTimestamp64Milli(`timestamp`)/3600000), " + - `count(if("AvgTicketPrice"<=0.000000, 1, NULL))/count(*)*100, ` + - `count(if("AvgTicketPrice"<=50000.000000, 1, NULL))/count(*)*100 ` + + `SELECT toInt64(toUnixTimestamp64Milli("timestamp") / 3600000), ` + + `count(if("AvgTicketPrice" <= 0.000000, 1, NULL)) / count(*) * 100, ` + + `count(if("AvgTicketPrice" <= 50000.000000, 1, NULL)) / count(*) * 100 ` + `FROM ` + testdata.QuotedTableName + ` ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`timestamp`)/3600000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`timestamp`)/3600000)", - "SELECT toInt64(toUnixTimestamp64Milli(`timestamp`)/3600000), count() " + + `GROUP BY toInt64(toUnixTimestamp64Milli("timestamp") / 3600000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("timestamp") / 3600000)`, + `SELECT toInt64(toUnixTimestamp64Milli("timestamp") / 3600000), count() ` + `FROM ` + testdata.QuotedTableName + ` ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`timestamp`)/3600000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`timestamp`)/3600000)", + `GROUP BY toInt64(toUnixTimestamp64Milli("timestamp") / 3600000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("timestamp") / 3600000)`, }, }, { // [8] @@ -1374,8 +1374,8 @@ var AggregationTests = []testdata.AggregationTestCase{ }, ExpectedSQLs: []string{ `SELECT count() FROM ` + testdata.QuotedTableName, - "SELECT maxOrNull(toHour(`timestamp`)) FROM " + testdata.QuotedTableName, - "SELECT minOrNull(toHour(`timestamp`)) FROM " + testdata.QuotedTableName, + `SELECT maxOrNull(toHour("timestamp")) FROM ` + testdata.QuotedTableName, + `SELECT minOrNull(toHour("timestamp")) FROM ` + testdata.QuotedTableName, }, }, { // [9] @@ -1483,9 +1483,9 @@ var AggregationTests = []testdata.AggregationTestCase{ }, ExpectedSQLs: []string{ `SELECT count() FROM ` + testdata.QuotedTableName, - "SELECT toHour(`timestamp`), count() FROM " + testdata.QuotedTableName + " " + - "GROUP BY toHour(`timestamp`) " + - "ORDER BY toHour(`timestamp`)", + `SELECT toHour("timestamp"), count() FROM ` + testdata.QuotedTableName + " " + + `GROUP BY toHour("timestamp") ` + + `ORDER BY toHour("timestamp")`, }, }, } diff --git a/quesma/testdata/opensearch-visualize/pipeline_aggregation_requests.go b/quesma/testdata/opensearch-visualize/pipeline_aggregation_requests.go index 109fb1ad5..f21308b9d 100644 --- a/quesma/testdata/opensearch-visualize/pipeline_aggregation_requests.go +++ b/quesma/testdata/opensearch-visualize/pipeline_aggregation_requests.go @@ -854,45 +854,45 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{ {}, // NoDBQuery { {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1715196000000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1715196000000/600000)), model.NewQueryResultCol("count()", 19.0), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1715196600000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1715196600000/600000)), model.NewQueryResultCol("count()", 19.0), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1715198400000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1715198400000/600000)), model.NewQueryResultCol("count()", 20.0), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1715199000000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1715199000000/600000)), model.NewQueryResultCol("count()", 32.0), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1715199600000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1715199600000/600000)), model.NewQueryResultCol("count()", 27.0), }}, }, { {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1715196000000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1715196000000/600000)), model.NewQueryResultCol("count()", 1), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1715196600000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1715196600000/600000)), model.NewQueryResultCol("count()", 1), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1715198400000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1715198400000/600000)), model.NewQueryResultCol("count()", 1), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1715199000000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1715199000000/600000)), model.NewQueryResultCol("count()", 4), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1715199600000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1715199600000/600000)), model.NewQueryResultCol("count()", 3), }}, }, @@ -901,16 +901,16 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{ `SELECT count() ` + `FROM ` + testdata.QuotedTableName, `NoDBQuery`, - "SELECT toInt64(toUnixTimestamp64Milli(`timestamp`)/600000), " + - "sumOrNull(toHour(`timestamp`)) " + + `SELECT toInt64(toUnixTimestamp64Milli("timestamp") / 600000), ` + + `sumOrNull(toHour("timestamp")) ` + "FROM " + testdata.QuotedTableName + " " + - "GROUP BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", - "SELECT toInt64(toUnixTimestamp64Milli(`timestamp`)/600000), " + + `GROUP BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, + `SELECT toInt64(toUnixTimestamp64Milli("timestamp") / 600000), ` + "count() " + "FROM " + testdata.QuotedTableName + " " + - "GROUP BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", + `GROUP BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, }, }, { // [6] @@ -1121,43 +1121,43 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{ {}, // NoDBQuery { {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1714869000000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1714869000000/600000)), model.NewQueryResultCol("count()", 2), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1714869600000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1714869600000/600000)), model.NewQueryResultCol("count()", 0), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1714878600000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1714878600000/600000)), model.NewQueryResultCol("count()", 0), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1714879200000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1714879200000/600000)), model.NewQueryResultCol("count()", 2), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1714879800000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1714879800000/600000)), model.NewQueryResultCol("count()", 6), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1714880400000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1714880400000/600000)), model.NewQueryResultCol("count()", 2), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1714881000000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1714881000000/600000)), model.NewQueryResultCol("count()", 2), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1714881600000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1714881600000/600000)), model.NewQueryResultCol("count()", 0), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1714882200000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1714882200000/600000)), model.NewQueryResultCol("count()", 2), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1714882800000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1714882800000/600000)), model.NewQueryResultCol("count()", 0), }}, }, @@ -1166,10 +1166,10 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{ `SELECT count() FROM ` + testdata.QuotedTableName, `NoDBQuery`, `NoDBQuery`, - "SELECT toInt64(toUnixTimestamp64Milli(`timestamp`)/600000), count() " + + `SELECT toInt64(toUnixTimestamp64Milli("timestamp") / 600000), count() ` + `FROM ` + testdata.QuotedTableName + ` ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", + `GROUP BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, }, }, { // [7] @@ -1630,45 +1630,45 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{ {}, // NoDBQuery { {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1715196000000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1715196000000/600000)), model.NewQueryResultCol("count()", 19.0), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1715196600000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1715196600000/600000)), model.NewQueryResultCol("count()", 19.0), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1715198400000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1715198400000/600000)), model.NewQueryResultCol("count()", 20.0), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1715199000000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1715199000000/600000)), model.NewQueryResultCol("count()", 32.0), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1715199600000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1715199600000/600000)), model.NewQueryResultCol("count()", 27.0), }}, }, { {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1715196000000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1715196000000/600000)), model.NewQueryResultCol("count()", 1), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1715196600000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1715196600000/600000)), model.NewQueryResultCol("count()", 1), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1715198400000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1715198400000/600000)), model.NewQueryResultCol("count()", 1), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1715199000000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1715199000000/600000)), model.NewQueryResultCol("count()", 4), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1715199600000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1715199600000/600000)), model.NewQueryResultCol("count()", 3), }}, }, @@ -1677,16 +1677,16 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{ `SELECT count() ` + `FROM ` + testdata.QuotedTableName, `NoDBQuery`, - "SELECT toInt64(toUnixTimestamp64Milli(`timestamp`)/600000), " + - "sumOrNull(toHour(`timestamp`)) " + + `SELECT toInt64(toUnixTimestamp64Milli("timestamp") / 600000), ` + + `sumOrNull(toHour("timestamp")) ` + "FROM " + testdata.QuotedTableName + " " + - "GROUP BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", - "SELECT toInt64(toUnixTimestamp64Milli(`timestamp`)/600000), " + + `GROUP BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, + `SELECT toInt64(toUnixTimestamp64Milli("timestamp") / 600000), ` + "count() " + "FROM " + testdata.QuotedTableName + " " + - "GROUP BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", + `GROUP BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, }, }, { // [10] @@ -1897,43 +1897,43 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{ {}, // NoDBQuery { {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1714869000000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1714869000000/600000)), model.NewQueryResultCol("count()", 2), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1714869600000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1714869600000/600000)), model.NewQueryResultCol("count()", 0), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1714878600000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1714878600000/600000)), model.NewQueryResultCol("count()", 0), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1714879200000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1714879200000/600000)), model.NewQueryResultCol("count()", 2), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1714879800000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1714879800000/600000)), model.NewQueryResultCol("count()", 6), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1714880400000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1714880400000/600000)), model.NewQueryResultCol("count()", 2), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1714881000000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1714881000000/600000)), model.NewQueryResultCol("count()", 2), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1714881600000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1714881600000/600000)), model.NewQueryResultCol("count()", 0), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1714882200000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1714882200000/600000)), model.NewQueryResultCol("count()", 2), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1714882800000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1714882800000/600000)), model.NewQueryResultCol("count()", 0), }}, }, @@ -1942,10 +1942,10 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{ `SELECT count() FROM ` + testdata.QuotedTableName, `NoDBQuery`, `NoDBQuery`, - "SELECT toInt64(toUnixTimestamp64Milli(`timestamp`)/600000), count() " + + `SELECT toInt64(toUnixTimestamp64Milli("timestamp") / 600000), count() ` + `FROM ` + testdata.QuotedTableName + ` ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", + `GROUP BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, }, }, { // [11] @@ -2070,10 +2070,10 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{ `SELECT count() ` + `FROM ` + testdata.QuotedTableName, `NoDBQuery`, - "SELECT toInt64(toUnixTimestamp64Milli(`timestamp`)/600000), count() " + + `SELECT toInt64(toUnixTimestamp64Milli("timestamp") / 600000), count() ` + `FROM ` + testdata.QuotedTableName + ` ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", + `GROUP BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, }, }, { // [12] @@ -2228,14 +2228,14 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{ `SELECT count() ` + `FROM ` + testdata.QuotedTableName, `NoDBQuery`, - "SELECT toInt64(toUnixTimestamp64Milli(`timestamp`)/600000), " + `maxOrNull("bytes") ` + + `SELECT toInt64(toUnixTimestamp64Milli("timestamp") / 600000), maxOrNull("bytes") ` + `FROM ` + testdata.QuotedTableName + ` ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", - "SELECT toInt64(toUnixTimestamp64Milli(`timestamp`)/600000), count() " + + `GROUP BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, + `SELECT toInt64(toUnixTimestamp64Milli("timestamp") / 600000), count() ` + `FROM ` + testdata.QuotedTableName + ` ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", + `GROUP BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, }, }, /* TODO need fix for date_range and subaggregations. Same one, as already merged ~1-2 weeks ago for range. It's WIP. @@ -2701,14 +2701,14 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{ `SELECT count() ` + `FROM ` + testdata.QuotedTableName, `NoDBQuery`, - "SELECT toInt64(toUnixTimestamp64Milli(`timestamp`)/600000), " + `"bytes", count() ` + + `SELECT toInt64(toUnixTimestamp64Milli("timestamp") / 600000), "bytes", count() ` + `FROM ` + testdata.QuotedTableName + ` ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000), " + `"bytes" ` + - "ORDER BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000), " + `"bytes"`, - "SELECT toInt64(toUnixTimestamp64Milli(`timestamp`)/600000), count() " + + `GROUP BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000), "bytes" ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000), "bytes"`, + `SELECT toInt64(toUnixTimestamp64Milli("timestamp") / 600000), count() ` + `FROM ` + testdata.QuotedTableName + ` ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", + `GROUP BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, }, }, { // [15] @@ -3588,29 +3588,29 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{ {}, // NoDBQuery { {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1716231600000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1716231600000/600000)), model.NewQueryResultCol(`minOrNull("memory")`, nil), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1716276600000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1716276600000/600000)), model.NewQueryResultCol(`minOrNull("memory")`, 121360.0), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1716277200000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1716277200000/600000)), model.NewQueryResultCol(`minOrNull("memory")`, nil), }}, }, { {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1716231600000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1716231600000/600000)), model.NewQueryResultCol("count()", 1), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1716276600000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1716276600000/600000)), model.NewQueryResultCol("count()", 4), }}, {Cols: []model.QueryResultCol{ - model.NewQueryResultCol("toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", int64(1716277200000/600000)), + model.NewQueryResultCol(`toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, int64(1716277200000/600000)), model.NewQueryResultCol("count()", 1), }}, }, @@ -3619,16 +3619,16 @@ var PipelineAggregationTests = []testdata.AggregationTestCase{ `SELECT count() ` + `FROM ` + testdata.QuotedTableName, `NoDBQuery`, - "SELECT toInt64(toUnixTimestamp64Milli(`timestamp`)/600000), " + + `SELECT toInt64(toUnixTimestamp64Milli("timestamp") / 600000), ` + `minOrNull("memory") ` + `FROM ` + testdata.QuotedTableName + ` ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", - "SELECT toInt64(toUnixTimestamp64Milli(`timestamp`)/600000), " + + `GROUP BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, + `SELECT toInt64(toUnixTimestamp64Milli("timestamp") / 600000), ` + "count() " + "FROM " + testdata.QuotedTableName + " " + - "GROUP BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`timestamp`)/600000)", + `GROUP BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("timestamp") / 600000)`, }, }, { // [20] diff --git a/quesma/testdata/opensearch_requests.go b/quesma/testdata/opensearch_requests.go index da2b07207..0adbb3e74 100644 --- a/quesma/testdata/opensearch_requests.go +++ b/quesma/testdata/opensearch_requests.go @@ -84,12 +84,12 @@ var OpensearchSearchTests = []SearchTestCase{ "SELECT count() FROM " + QuotedTableName + ` ` + `WHERE ("-@timestamp".=parseDateTime64BestEffort('2024-04-04T13:..:18.149Z') ` + `AND "-@timestamp".=parseDateTime64BestEffort('2024-04-04T13:..:18.149Z'))`, - "SELECT toInt64(toUnixTimestamp64Milli(`-@timestamp`)/30000), count() " + + `SELECT toInt64(toUnixTimestamp64Milli("-@timestamp") / 30000), count() ` + `FROM ` + QuotedTableName + ` ` + `WHERE ("-@timestamp".=parseDateTime64BestEffort('2024-04-04T13:..:18.149Z') ` + `AND "-@timestamp".=parseDateTime64BestEffort('2024-04-04T13:..:18.149Z')) ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`-@timestamp`)/30000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`-@timestamp`)/30000)", + `GROUP BY toInt64(toUnixTimestamp64Milli("-@timestamp") / 30000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("-@timestamp") / 30000)`, `SELECT.*"-@bytes".*FROM ` + QuotedTableName + ` ` + `WHERE ("-@timestamp".=parseDateTime64BestEffort('2024-04-04T13:..:18.149Z') ` + `AND "-@timestamp".=parseDateTime64BestEffort('2024-04-04T13:..:18.149Z')) ` + @@ -175,11 +175,11 @@ var OpensearchSearchTests = []SearchTestCase{ "SELECT count() FROM " + QuotedTableName + ` ` + `WHERE ("-@timestamp".=parseDateTime64BestEffort('2024-04-04T13:..:18.149Z') ` + `AND "-@timestamp".=parseDateTime64BestEffort('2024-04-04T13:..:18.149Z'))`, - "SELECT toInt64(toUnixTimestamp64Milli(`-@timestamp`)/30000), count() FROM " + QuotedTableName + ` ` + + `SELECT toInt64(toUnixTimestamp64Milli("-@timestamp") / 30000), count() FROM ` + QuotedTableName + ` ` + `WHERE ("-@timestamp".=parseDateTime64BestEffort('2024-04-04T13:..:18.149Z') ` + `AND "-@timestamp".=parseDateTime64BestEffort('2024-04-04T13:..:18.149Z')) ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`-@timestamp`)/30000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`-@timestamp`)/30000)", + `GROUP BY toInt64(toUnixTimestamp64Milli("-@timestamp") / 30000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("-@timestamp") / 30000)`, }, }, } diff --git a/quesma/testdata/requests.go b/quesma/testdata/requests.go index 3cada663b..4d7990461 100644 --- a/quesma/testdata/requests.go +++ b/quesma/testdata/requests.go @@ -147,7 +147,7 @@ var TestsAsyncSearch = []AsyncSearchTestCase{ }`, "no comment yet", model.SearchQueryInfo{Typ: model.Facets, FieldName: "host.name", I1: 10, I2: 5000}, - []string{`SELECT "host.name" AS "key", count() AS "doc_count" FROM (SELECT "host.name" FROM "logs-generic-default" WHERE (("@timestamp".=parseDateTime64BestEffort('2024-01-23T11:27:16.820Z') AND "@timestamp".=parseDateTime64BestEffort('2024-01-23T11:42:16.820Z')) AND "message" iLIKE '%user%') LIMIT ` + queryparserFacetsSampleSize + `) GROUP BY "host.name" ORDER BY count() DESC`}, + []string{`SELECT "host.name" AS "key", count() AS "doc_count" FROM (SELECT "host.name" FROM "logs-generic-default" WHERE (("@timestamp".=parseDateTime64BestEffort('2024-01-23T11:27:16.820Z') AND "@timestamp".=parseDateTime64BestEffort('2024-01-23T11:42:16.820Z')) AND "message" iLIKE '%user%') LIMIT ` + queryparserFacetsSampleSize + ` ) GROUP BY "host.name" ORDER BY count() DESC`}, true, }, { // [1] @@ -662,13 +662,13 @@ var TestsAsyncSearch = []AsyncSearchTestCase{ model.SearchQueryInfo{Typ: model.ListByField, RequestedFields: []string{"@timestamp"}, FieldName: "@timestamp"}, []string{ `SELECT count() FROM ` + QuotedTableName + ` WHERE ("message" iLIKE '%user%' AND ("@timestamp".=parseDateTime64BestEffort('2024-01-23T14:..:19.481Z') AND "@timestamp".=parseDateTime64BestEffort('2024-01-23T14:..:19.481Z')))`, - `SELECT toInt64(toUnixTimestamp64Milli(` + "`@timestamp`" + `)/30000), count() ` + + `SELECT toInt64(toUnixTimestamp64Milli("@timestamp") / 30000), count() ` + `FROM ` + QuotedTableName + ` ` + `WHERE ("message" iLIKE '%user%' ` + `AND ("@timestamp".=parseDateTime64BestEffort('2024-01-23T14:..:19.481Z') ` + `AND "@timestamp".=parseDateTime64BestEffort('2024-01-23T14:..:19.481Z'))) ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`@timestamp`)/30000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`@timestamp`)/30000)", + `GROUP BY toInt64(toUnixTimestamp64Milli("@timestamp") / 30000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("@timestamp") / 30000)`, }, true, }, @@ -710,12 +710,12 @@ var TestsAsyncSearch = []AsyncSearchTestCase{ []string{ `SELECT count() FROM ` + QuotedTableName + ` WHERE ("@timestamp".*parseDateTime64BestEffort('2024-01-25T..:..:59.033Z') AND "@timestamp".*parseDateTime64BestEffort('2024-01-25T..:..:59.033Z'))`, `SELECT "event.dataset", ` + - clickhouse.TimestampGroupBy("@timestamp", clickhouse.DateTime64, time.Minute) + + groupBySQL("@timestamp", clickhouse.DateTime64, time.Minute) + `, count() FROM ` + QuotedTableName + ` ` + `WHERE ("@timestamp".*parseDateTime64BestEffort('2024-01-25T1.:..:59.033Z') ` + `AND "@timestamp".*parseDateTime64BestEffort('2024-01-25T1.:..:59.033Z')) ` + - `GROUP BY "event.dataset", ` + clickhouse.TimestampGroupBy("@timestamp", clickhouse.DateTime64, time.Minute) + ` ` + - `ORDER BY "event.dataset", ` + clickhouse.TimestampGroupBy("@timestamp", clickhouse.DateTime64, time.Minute), + `GROUP BY "event.dataset", ` + groupBySQL("@timestamp", clickhouse.DateTime64, time.Minute) + ` ` + + `ORDER BY "event.dataset", ` + groupBySQL("@timestamp", clickhouse.DateTime64, time.Minute), `SELECT "event.dataset", count() FROM ` + QuotedTableName + ` ` + `WHERE ("@timestamp".*parseDateTime64BestEffort('2024-01-25T1.:..:59.033Z') ` + `AND "@timestamp".*parseDateTime64BestEffort('2024-01-25T1.:..:59.033Z')) ` + @@ -2082,10 +2082,10 @@ var TestSearchFilter = []SearchTestCase{ //}, []string{ "SELECT count() FROM " + QuotedTableName, - "SELECT " + clickhouse.TimestampGroupBy("@timestamp", clickhouse.DateTime64, 30*time.Second) + ", count() " + + "SELECT " + groupBySQL("@timestamp", clickhouse.DateTime64, 30*time.Second) + ", count() " + "FROM " + QuotedTableName + " " + - "GROUP BY " + clickhouse.TimestampGroupBy("@timestamp", clickhouse.DateTime64, 30*time.Second) + " " + - "ORDER BY " + clickhouse.TimestampGroupBy("@timestamp", clickhouse.DateTime64, 30*time.Second), + "GROUP BY " + groupBySQL("@timestamp", clickhouse.DateTime64, 30*time.Second) + " " + + "ORDER BY " + groupBySQL("@timestamp", clickhouse.DateTime64, 30*time.Second), }, }, { // [1] @@ -2145,11 +2145,11 @@ var TestSearchFilter = []SearchTestCase{ //}, []string{ "SELECT count() FROM " + QuotedTableName + ` WHERE "@timestamp">subDate(now(), INTERVAL 15 minute)`, - "SELECT " + clickhouse.TimestampGroupBy("@timestamp", clickhouse.DateTime64, 30*time.Second) + `, count() ` + + "SELECT " + groupBySQL("@timestamp", clickhouse.DateTime64, 30*time.Second) + `, count() ` + `FROM ` + QuotedTableName + ` ` + `WHERE "@timestamp">subDate(now(), INTERVAL 15 minute) ` + - `GROUP BY ` + clickhouse.TimestampGroupBy("@timestamp", clickhouse.DateTime64, 30*time.Second) + ` ` + - `ORDER BY ` + clickhouse.TimestampGroupBy("@timestamp", clickhouse.DateTime64, 30*time.Second), + `GROUP BY ` + groupBySQL("@timestamp", clickhouse.DateTime64, 30*time.Second) + ` ` + + `ORDER BY ` + groupBySQL("@timestamp", clickhouse.DateTime64, 30*time.Second), }, }, { // [2] diff --git a/quesma/testdata/requests_with_special_characters.go b/quesma/testdata/requests_with_special_characters.go index 9a842e4bb..edc86e673 100644 --- a/quesma/testdata/requests_with_special_characters.go +++ b/quesma/testdata/requests_with_special_characters.go @@ -145,20 +145,20 @@ var AggregationTestsWithSpecialCharactersInFieldNames = []AggregationTestCase{ ExpectedResults: [][]model.QueryResultRow{}, // checking only the SQLs is enough for now ExpectedSQLs: []string{ `SELECT count() FROM ` + QuotedTableName + ` WHERE "message\$\*\%\:\;" IS NOT NULL`, - `SELECT toInt64(toUnixTimestamp64Milli(` + "`-@timestamp`" + `)/43200000), ` + + `SELECT toInt64(toUnixTimestamp64Milli("-@timestamp") / 43200000), ` + `minOrNull("-@bytes") AS "windowed_-@bytes", ` + `minOrNull("-@timestamp") AS "windowed_-@timestamp" ` + - `FROM (SELECT "-@bytes", "-@timestamp", ROW_NUMBER() ` + - `OVER (PARTITION BY toInt64(toUnixTimestamp64Milli(` + "`-@timestamp`)/43200000) ORDER BY " + `"-@timestamp" desc) ` + - `AS row_number FROM ` + QuotedTableName + ` WHERE "message\$\*\%\:\;" IS NOT NULL) ` + + `FROM (SELECT "-@bytes" , "-@timestamp" , ROW_NUMBER() ` + + `OVER (PARTITION BY toInt64(toUnixTimestamp64Milli("-@timestamp") / 43200000) ORDER BY "-@timestamp" DESC ) ` + + `AS 'row_number' FROM ` + QuotedTableName + ` WHERE "message\$\*\%\:\;" IS NOT NULL) ` + `WHERE ("message\$\*\%\:\;" IS NOT NULL ` + `AND "row_number"<=1) ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`-@timestamp`)/43200000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`-@timestamp`)/43200000)", - "SELECT toInt64(toUnixTimestamp64Milli(`-@timestamp`)/43200000), count() FROM " + QuotedTableName + ` ` + + `GROUP BY toInt64(toUnixTimestamp64Milli("-@timestamp") / 43200000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("-@timestamp") / 43200000)`, + `SELECT toInt64(toUnixTimestamp64Milli("-@timestamp") / 43200000), count() FROM ` + QuotedTableName + ` ` + `WHERE "message\$\*\%\:\;\" IS NOT NULL ` + - "GROUP BY toInt64(toUnixTimestamp64Milli(`-@timestamp`)/43200000) " + - "ORDER BY toInt64(toUnixTimestamp64Milli(`-@timestamp`)/43200000)", + `GROUP BY toInt64(toUnixTimestamp64Milli("-@timestamp") / 43200000) ` + + `ORDER BY toInt64(toUnixTimestamp64Milli("-@timestamp") / 43200000)`, }, }, } diff --git a/quesma/util/utils.go b/quesma/util/utils.go index e875a7422..eee037326 100644 --- a/quesma/util/utils.go +++ b/quesma/util/utils.go @@ -3,8 +3,10 @@ package util import ( "bytes" "context" + "database/sql" "encoding/json" "fmt" + "github.com/DATA-DOG/go-sqlmock" "github.com/k0kubun/pp" "io" "log" @@ -653,3 +655,21 @@ func ExtractNumeric64Maybe(value any) (asFloat64 float64, success bool) { } return 0.0, false } + +func InitSqlMockWithPrettyPrint(t *testing.T) (*sql.DB, sqlmock.Sqlmock) { + queryMatcher := sqlmock.QueryMatcherFunc(func(expectedSQL, actualSQL string) error { + matchErr := sqlmock.QueryMatcherRegexp.Match(expectedSQL, actualSQL) + if matchErr != nil { + pp.Println("-- Expected:") + fmt.Printf("%s\n", SqlPrettyPrint([]byte(expectedSQL))) + pp.Println("---- Actual:") + fmt.Printf("%s\n", SqlPrettyPrint([]byte(actualSQL))) + } + return matchErr + }) + db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(queryMatcher)) + if err != nil { + t.Fatal(err) + } + return db, mock +}