From 3e63212dcd40f8edb22f4e9bd3ab3ad00f8e2167 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Strzali=C5=84ski?= Date: Fri, 12 Jul 2024 13:08:24 +0200 Subject: [PATCH] Array types - tests (#501) Co-authored-by: Grzegorz Piwowarek --- quesma/quesma/schema_array_transformer.go | 24 ++- quesma/quesma/schema_transformer_test.go | 195 ++++++++++++++++++++++ 2 files changed, 206 insertions(+), 13 deletions(-) diff --git a/quesma/quesma/schema_array_transformer.go b/quesma/quesma/schema_array_transformer.go index d9f770893..3c6410982 100644 --- a/quesma/quesma/schema_array_transformer.go +++ b/quesma/quesma/schema_array_transformer.go @@ -21,11 +21,11 @@ import ( type ArrayTypeVisitor struct { tableName string table *clickhouse.Table + schema schema.Schema // deps schemaRegistry schema.Registry logManager *clickhouse.LogManager - schema schema.Schema } func (v *ArrayTypeVisitor) visitChildren(args []model.Expr) []model.Expr { @@ -56,6 +56,7 @@ func (v *ArrayTypeVisitor) dbColumnType(fieldName string) string { } func (v *ArrayTypeVisitor) VisitLiteral(e model.LiteralExpr) interface{} { return e } + func (v *ArrayTypeVisitor) VisitInfix(e model.InfixExpr) interface{} { column, ok := e.Left.(model.ColumnRef) @@ -90,12 +91,10 @@ func (v *ArrayTypeVisitor) VisitInfix(e model.InfixExpr) interface{} { } func (v *ArrayTypeVisitor) VisitPrefixExpr(e model.PrefixExpr) interface{} { - args := v.visitChildren(e.Args) - return model.NewPrefixExpr(e.Op, args) - } + func (v *ArrayTypeVisitor) VisitFunction(e model.FunctionExpr) interface{} { if len(e.Args) == 1 { @@ -123,52 +122,51 @@ func (v *ArrayTypeVisitor) VisitFunction(e model.FunctionExpr) interface{} { args := v.visitChildren(e.Args) return model.NewFunction(e.Name, args...) } -func (v *ArrayTypeVisitor) VisitColumnRef(e model.ColumnRef) interface{} { +func (v *ArrayTypeVisitor) VisitColumnRef(e model.ColumnRef) interface{} { return e } func (v *ArrayTypeVisitor) VisitNestedProperty(e model.NestedProperty) interface{} { - return model.NestedProperty{ ColumnRef: e.ColumnRef.Accept(v).(model.ColumnRef), PropertyName: e.PropertyName.Accept(v).(model.LiteralExpr), } } + func (v *ArrayTypeVisitor) VisitArrayAccess(e model.ArrayAccess) interface{} { return model.ArrayAccess{ ColumnRef: e.ColumnRef.Accept(v).(model.ColumnRef), Index: e.Index.Accept(v).(model.Expr), } } -func (v *ArrayTypeVisitor) VisitMultiFunction(e model.MultiFunctionExpr) interface{} { +func (v *ArrayTypeVisitor) VisitMultiFunction(e model.MultiFunctionExpr) interface{} { args := v.visitChildren(e.Args) return model.MultiFunctionExpr{Name: e.Name, Args: args} } func (v *ArrayTypeVisitor) VisitString(e model.StringExpr) interface{} { return e } -func (v *ArrayTypeVisitor) VisitOrderByExpr(e model.OrderByExpr) interface{} { +func (v *ArrayTypeVisitor) VisitOrderByExpr(e model.OrderByExpr) interface{} { exprs := v.visitChildren(e.Exprs) - return model.NewOrderByExpr(exprs, e.Direction) - } -func (v *ArrayTypeVisitor) VisitDistinctExpr(e model.DistinctExpr) interface{} { +func (v *ArrayTypeVisitor) VisitDistinctExpr(e model.DistinctExpr) interface{} { return model.NewDistinctExpr(e.Expr.Accept(v).(model.Expr)) } + func (v *ArrayTypeVisitor) VisitTableRef(e model.TableRef) interface{} { return model.NewTableRef(e.Name) } + func (v *ArrayTypeVisitor) VisitAliasedExpr(e model.AliasedExpr) interface{} { return model.NewAliasedExpr(e.Expr.Accept(v).(model.Expr), e.Alias) } -func (v *ArrayTypeVisitor) VisitWindowFunction(e model.WindowFunction) interface{} { +func (v *ArrayTypeVisitor) VisitWindowFunction(e model.WindowFunction) interface{} { return model.NewWindowFunction(e.Name, v.visitChildren(e.Args), v.visitChildren(e.PartitionBy), e.OrderBy.Accept(v).(model.OrderByExpr)) - } func (v *ArrayTypeVisitor) VisitSelectCommand(e model.SelectCommand) interface{} { diff --git a/quesma/quesma/schema_transformer_test.go b/quesma/quesma/schema_transformer_test.go index 8e7b10f86..cc284abbc 100644 --- a/quesma/quesma/schema_transformer_test.go +++ b/quesma/quesma/schema_transformer_test.go @@ -254,3 +254,198 @@ func Test_ipRangeTransform(t *testing.T) { assert.Equal(t, expectedQueries[k].SelectCommand.String(), resultQueries[0].SelectCommand.String()) } } + +func Test_arrayType(t *testing.T) { + + indexConfig := map[string]config.IndexConfiguration{ + "kibana_sample_data_ecommerce": { + Name: "kibana_sample_data_ecommerce", + Enabled: true, + }, + } + cfg := config.QuesmaConfiguration{ + IndexConfig: indexConfig, + } + + tableDiscovery := + fixedTableProvider{tables: map[string]schema.Table{ + "kibana_sample_data_ecommerce": {Columns: map[string]schema.Column{ + "products::name": {Name: "products::name", Type: "keyword"}, + "products::quantity": {Name: "products::quantity", Type: "long"}, + "products::sku": {Name: "products::sku", Type: "keyword"}, + "order_date": {Name: "order_date", Type: "timestamp"}, + }}, + }} + + tableDefinition := clickhouse.Table{ + Name: "kibana_sample_data_ecommerce", + Config: clickhouse.NewDefaultCHConfig(), + Cols: map[string]*clickhouse.Column{ + "products::name": {Name: "products::name", Type: clickhouse.NewBaseType("Array(String)")}, + "products::quantity": {Name: "products::quantity", Type: clickhouse.NewBaseType("Array(Int64)")}, + "products::sku": {Name: "products::sku", Type: clickhouse.NewBaseType("Array(String)")}, + "order_date": {Name: "order_date", Type: clickhouse.NewBaseType("DateTime64")}, + }, + } + + lm := clickhouse.NewLogManagerEmpty() + + td, err := lm.GetTableDefinitions() + if err != nil { + t.Fatal(err) + } + td.Store("kibana_sample_data_ecommerce", &tableDefinition) + + s := schema.NewSchemaRegistry(tableDiscovery, cfg, clickhouse.SchemaTypeAdapter{}) + transform := &SchemaCheckPass{cfg: indexConfig, schemaRegistry: s, logManager: lm} + + tests := []struct { + name string + query *model.Query + expected *model.Query + }{ + { + name: "simple array", + query: &model.Query{ + TableName: "kibana_sample_data_logs", + SelectCommand: model.SelectCommand{ + FromClause: model.NewTableRef("kibana_sample_data_logs"), + Columns: []model.Expr{model.NewWildcardExpr}, + }, + }, + expected: &model.Query{ + TableName: "kibana_sample_data_logs", + SelectCommand: model.SelectCommand{ + FromClause: model.NewTableRef("kibana_sample_data_logs"), + Columns: []model.Expr{model.NewWildcardExpr}, + }, + }, + }, + + { + name: "arrayReduce", + //SELECT "order_date", sumOrNull("products::quantity") FROM "kibana_sample_data_ecommerce" GROUP BY "order_date" + query: &model.Query{ + TableName: "kibana_sample_data_ecommerce", + SelectCommand: model.SelectCommand{ + FromClause: model.NewTableRef("kibana_sample_data_ecommerce"), + Columns: []model.Expr{ + model.NewColumnRef("order_date"), + model.NewFunction("sumOrNull", model.NewColumnRef("products::quantity")), + }, + GroupBy: []model.Expr{model.NewColumnRef("order_date")}, + }, + }, + //SELECT "order_date", sumOrNull(arrayReduce('sumOrNull',"products::quantity")) FROM "kibana_sample_data_ecommerce" GROUP BY "order_date" + expected: &model.Query{ + TableName: "kibana_sample_data_ecommerce", + SelectCommand: model.SelectCommand{ + FromClause: model.NewTableRef("kibana_sample_data_ecommerce"), + Columns: []model.Expr{ + model.NewColumnRef("order_date"), + model.NewFunction("sumOrNull", model.NewFunction("arrayReduce", model.NewLiteral("'sumOrNull'"), model.NewColumnRef("products::quantity"))), + }, + GroupBy: []model.Expr{model.NewColumnRef("order_date")}, + }, + }, + }, + + { + name: "ilike array", + //SELECT "order_date", count() FROM "kibana_sample_data_ecommerce" WHERE "products::name" ILIKE '%bag% GROUP BY "order_date" + //SELECT "order_date", count() FROM "kibana_sample_data_ecommerce" WHERE arrayExists((x) -> x ILIKE '%bag%',"products::product_name") GROUP BY "order_date" + + query: &model.Query{ + TableName: "kibana_sample_data_ecommerce", + SelectCommand: model.SelectCommand{ + FromClause: model.NewTableRef("kibana_sample_data_ecommerce"), + Columns: []model.Expr{ + model.NewColumnRef("order_date"), + model.NewFunction("count"), + }, + WhereClause: model.NewInfixExpr( + model.NewColumnRef("products::name"), + "ILIKE", + model.NewLiteral("%foo%"), + ), + GroupBy: []model.Expr{model.NewColumnRef("order_date")}, + }, + }, + expected: &model.Query{ + TableName: "kibana_sample_data_ecommerce", + SelectCommand: model.SelectCommand{ + FromClause: model.NewTableRef("kibana_sample_data_ecommerce"), + Columns: []model.Expr{ + model.NewColumnRef("order_date"), + model.NewFunction("count"), + }, + WhereClause: model.NewFunction( + "arrayExists", + model.NewLambdaExpr([]string{"x"}, model.NewInfixExpr(model.NewLiteral("x"), "ILIKE", model.NewLiteral("%foo%"))), + model.NewColumnRef("products::name")), + GroupBy: []model.Expr{model.NewColumnRef("order_date")}, + }, + }, + }, + + //SELECT "order_date", count() FROM "kibana_sample_data_ecommerce" WHERE "products.sku" = 'XYZ' group by "order_date" + //SELECT "order_date", count() FROM "kibana_sample_data_ecommerce" WHERE has("products.sku",'XYZ') group by "order_date" + + { + name: "equals array", + query: &model.Query{ + TableName: "kibana_sample_data_ecommerce", + SelectCommand: model.SelectCommand{ + FromClause: model.NewTableRef("kibana_sample_data_ecommerce"), + Columns: []model.Expr{ + model.NewColumnRef("order_date"), + model.NewFunction("count"), + }, + WhereClause: model.NewInfixExpr( + model.NewColumnRef("products::sku"), + "=", + model.NewLiteral("'XYZ'"), + ), + GroupBy: []model.Expr{model.NewColumnRef("order_date")}, + }, + }, + expected: &model.Query{ + TableName: "kibana_sample_data_ecommerce", + SelectCommand: model.SelectCommand{ + FromClause: model.NewTableRef("kibana_sample_data_ecommerce"), + Columns: []model.Expr{ + model.NewColumnRef("order_date"), + model.NewFunction("count"), + }, + WhereClause: model.NewFunction( + "has", + model.NewColumnRef("products::sku"), + model.NewLiteral("'XYZ'")), + GroupBy: []model.Expr{model.NewColumnRef("order_date")}, + }, + }, + }, + } + + asString := func(query *model.Query) string { + return query.SelectCommand.String() + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual, err := transform.Transform([]*model.Query{tt.query}) + assert.NoError(t, err) + + if err != nil { + t.Fatal(err) + } + + assert.True(t, len(actual) == 1, "len queries == 1") + + expectedJson := asString(tt.expected) + actualJson := asString(actual[0]) + + assert.Equal(t, expectedJson, actualJson) + }) + } +}