Skip to content

Commit

Permalink
Array types - tests (#501)
Browse files Browse the repository at this point in the history 
Co-authored-by: Grzegorz Piwowarek <[email protected]>
  • Loading branch information
@nablaone @pivovarit
nablaone and pivovarit authored Jul 12, 2024
1 parent e42d684 commit 3e63212
Show file tree
Hide file tree
Showing 2 changed files with 206 additions and 13 deletions.
24 changes: 11 additions & 13 deletions quesma/quesma/schema_array_transformer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,11 @@ import (
type ArrayTypeVisitor struct {
tableName string
table *clickhouse.Table
schema schema.Schema

// deps
schemaRegistry schema.Registry
logManager *clickhouse.LogManager
schema schema.Schema
}

func (v *ArrayTypeVisitor) visitChildren(args []model.Expr) []model.Expr {
Expand Down Expand Up @@ -56,6 +56,7 @@ func (v *ArrayTypeVisitor) dbColumnType(fieldName string) string {
}

func (v *ArrayTypeVisitor) VisitLiteral(e model.LiteralExpr) interface{} { return e }

func (v *ArrayTypeVisitor) VisitInfix(e model.InfixExpr) interface{} {

column, ok := e.Left.(model.ColumnRef)
Expand Down Expand Up @@ -90,12 +91,10 @@ func (v *ArrayTypeVisitor) VisitInfix(e model.InfixExpr) interface{} {
}

func (v *ArrayTypeVisitor) VisitPrefixExpr(e model.PrefixExpr) interface{} {

args := v.visitChildren(e.Args)

return model.NewPrefixExpr(e.Op, args)

}

func (v *ArrayTypeVisitor) VisitFunction(e model.FunctionExpr) interface{} {

if len(e.Args) == 1 {
Expand Down Expand Up @@ -123,52 +122,51 @@ func (v *ArrayTypeVisitor) VisitFunction(e model.FunctionExpr) interface{} {
args := v.visitChildren(e.Args)
return model.NewFunction(e.Name, args...)
}
func (v *ArrayTypeVisitor) VisitColumnRef(e model.ColumnRef) interface{} {

func (v *ArrayTypeVisitor) VisitColumnRef(e model.ColumnRef) interface{} {
return e
}

func (v *ArrayTypeVisitor) VisitNestedProperty(e model.NestedProperty) interface{} {

return model.NestedProperty{
ColumnRef: e.ColumnRef.Accept(v).(model.ColumnRef),
PropertyName: e.PropertyName.Accept(v).(model.LiteralExpr),
}
}

func (v *ArrayTypeVisitor) VisitArrayAccess(e model.ArrayAccess) interface{} {
return model.ArrayAccess{
ColumnRef: e.ColumnRef.Accept(v).(model.ColumnRef),
Index: e.Index.Accept(v).(model.Expr),
}
}
func (v *ArrayTypeVisitor) VisitMultiFunction(e model.MultiFunctionExpr) interface{} {

func (v *ArrayTypeVisitor) VisitMultiFunction(e model.MultiFunctionExpr) interface{} {
args := v.visitChildren(e.Args)
return model.MultiFunctionExpr{Name: e.Name, Args: args}
}

func (v *ArrayTypeVisitor) VisitString(e model.StringExpr) interface{} { return e }
func (v *ArrayTypeVisitor) VisitOrderByExpr(e model.OrderByExpr) interface{} {

func (v *ArrayTypeVisitor) VisitOrderByExpr(e model.OrderByExpr) interface{} {
exprs := v.visitChildren(e.Exprs)

return model.NewOrderByExpr(exprs, e.Direction)

}
func (v *ArrayTypeVisitor) VisitDistinctExpr(e model.DistinctExpr) interface{} {

func (v *ArrayTypeVisitor) VisitDistinctExpr(e model.DistinctExpr) interface{} {
return model.NewDistinctExpr(e.Expr.Accept(v).(model.Expr))
}

func (v *ArrayTypeVisitor) VisitTableRef(e model.TableRef) interface{} {
return model.NewTableRef(e.Name)
}

func (v *ArrayTypeVisitor) VisitAliasedExpr(e model.AliasedExpr) interface{} {
return model.NewAliasedExpr(e.Expr.Accept(v).(model.Expr), e.Alias)
}
func (v *ArrayTypeVisitor) VisitWindowFunction(e model.WindowFunction) interface{} {

func (v *ArrayTypeVisitor) VisitWindowFunction(e model.WindowFunction) interface{} {
return model.NewWindowFunction(e.Name, v.visitChildren(e.Args), v.visitChildren(e.PartitionBy), e.OrderBy.Accept(v).(model.OrderByExpr))

}

func (v *ArrayTypeVisitor) VisitSelectCommand(e model.SelectCommand) interface{} {
Expand Down
195 changes: 195 additions & 0 deletions quesma/quesma/schema_transformer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,3 +254,198 @@ func Test_ipRangeTransform(t *testing.T) {
assert.Equal(t, expectedQueries[k].SelectCommand.String(), resultQueries[0].SelectCommand.String())
}
}

func Test_arrayType(t *testing.T) {

indexConfig := map[string]config.IndexConfiguration{
"kibana_sample_data_ecommerce": {
Name: "kibana_sample_data_ecommerce",
Enabled: true,
},
}
cfg := config.QuesmaConfiguration{
IndexConfig: indexConfig,
}

tableDiscovery :=
fixedTableProvider{tables: map[string]schema.Table{
"kibana_sample_data_ecommerce": {Columns: map[string]schema.Column{
"products::name": {Name: "products::name", Type: "keyword"},
"products::quantity": {Name: "products::quantity", Type: "long"},
"products::sku": {Name: "products::sku", Type: "keyword"},
"order_date": {Name: "order_date", Type: "timestamp"},
}},
}}

tableDefinition := clickhouse.Table{
Name: "kibana_sample_data_ecommerce",
Config: clickhouse.NewDefaultCHConfig(),
Cols: map[string]*clickhouse.Column{
"products::name": {Name: "products::name", Type: clickhouse.NewBaseType("Array(String)")},
"products::quantity": {Name: "products::quantity", Type: clickhouse.NewBaseType("Array(Int64)")},
"products::sku": {Name: "products::sku", Type: clickhouse.NewBaseType("Array(String)")},
"order_date": {Name: "order_date", Type: clickhouse.NewBaseType("DateTime64")},
},
}

lm := clickhouse.NewLogManagerEmpty()

td, err := lm.GetTableDefinitions()
if err != nil {
t.Fatal(err)
}
td.Store("kibana_sample_data_ecommerce", &tableDefinition)

s := schema.NewSchemaRegistry(tableDiscovery, cfg, clickhouse.SchemaTypeAdapter{})
transform := &SchemaCheckPass{cfg: indexConfig, schemaRegistry: s, logManager: lm}

tests := []struct {
name string
query *model.Query
expected *model.Query
}{
{
name: "simple array",
query: &model.Query{
TableName: "kibana_sample_data_logs",
SelectCommand: model.SelectCommand{
FromClause: model.NewTableRef("kibana_sample_data_logs"),
Columns: []model.Expr{model.NewWildcardExpr},
},
},
expected: &model.Query{
TableName: "kibana_sample_data_logs",
SelectCommand: model.SelectCommand{
FromClause: model.NewTableRef("kibana_sample_data_logs"),
Columns: []model.Expr{model.NewWildcardExpr},
},
},
},

{
name: "arrayReduce",
//SELECT "order_date", sumOrNull("products::quantity") FROM "kibana_sample_data_ecommerce" GROUP BY "order_date"
query: &model.Query{
TableName: "kibana_sample_data_ecommerce",
SelectCommand: model.SelectCommand{
FromClause: model.NewTableRef("kibana_sample_data_ecommerce"),
Columns: []model.Expr{
model.NewColumnRef("order_date"),
model.NewFunction("sumOrNull", model.NewColumnRef("products::quantity")),
},
GroupBy: []model.Expr{model.NewColumnRef("order_date")},
},
},
//SELECT "order_date", sumOrNull(arrayReduce('sumOrNull',"products::quantity")) FROM "kibana_sample_data_ecommerce" GROUP BY "order_date"
expected: &model.Query{
TableName: "kibana_sample_data_ecommerce",
SelectCommand: model.SelectCommand{
FromClause: model.NewTableRef("kibana_sample_data_ecommerce"),
Columns: []model.Expr{
model.NewColumnRef("order_date"),
model.NewFunction("sumOrNull", model.NewFunction("arrayReduce", model.NewLiteral("'sumOrNull'"), model.NewColumnRef("products::quantity"))),
},
GroupBy: []model.Expr{model.NewColumnRef("order_date")},
},
},
},

{
name: "ilike array",
//SELECT "order_date", count() FROM "kibana_sample_data_ecommerce" WHERE "products::name" ILIKE '%bag% GROUP BY "order_date"
//SELECT "order_date", count() FROM "kibana_sample_data_ecommerce" WHERE arrayExists((x) -> x ILIKE '%bag%',"products::product_name") GROUP BY "order_date"

query: &model.Query{
TableName: "kibana_sample_data_ecommerce",
SelectCommand: model.SelectCommand{
FromClause: model.NewTableRef("kibana_sample_data_ecommerce"),
Columns: []model.Expr{
model.NewColumnRef("order_date"),
model.NewFunction("count"),
},
WhereClause: model.NewInfixExpr(
model.NewColumnRef("products::name"),
"ILIKE",
model.NewLiteral("%foo%"),
),
GroupBy: []model.Expr{model.NewColumnRef("order_date")},
},
},
expected: &model.Query{
TableName: "kibana_sample_data_ecommerce",
SelectCommand: model.SelectCommand{
FromClause: model.NewTableRef("kibana_sample_data_ecommerce"),
Columns: []model.Expr{
model.NewColumnRef("order_date"),
model.NewFunction("count"),
},
WhereClause: model.NewFunction(
"arrayExists",
model.NewLambdaExpr([]string{"x"}, model.NewInfixExpr(model.NewLiteral("x"), "ILIKE", model.NewLiteral("%foo%"))),
model.NewColumnRef("products::name")),
GroupBy: []model.Expr{model.NewColumnRef("order_date")},
},
},
},

//SELECT "order_date", count() FROM "kibana_sample_data_ecommerce" WHERE "products.sku" = 'XYZ' group by "order_date"
//SELECT "order_date", count() FROM "kibana_sample_data_ecommerce" WHERE has("products.sku",'XYZ') group by "order_date"

{
name: "equals array",
query: &model.Query{
TableName: "kibana_sample_data_ecommerce",
SelectCommand: model.SelectCommand{
FromClause: model.NewTableRef("kibana_sample_data_ecommerce"),
Columns: []model.Expr{
model.NewColumnRef("order_date"),
model.NewFunction("count"),
},
WhereClause: model.NewInfixExpr(
model.NewColumnRef("products::sku"),
"=",
model.NewLiteral("'XYZ'"),
),
GroupBy: []model.Expr{model.NewColumnRef("order_date")},
},
},
expected: &model.Query{
TableName: "kibana_sample_data_ecommerce",
SelectCommand: model.SelectCommand{
FromClause: model.NewTableRef("kibana_sample_data_ecommerce"),
Columns: []model.Expr{
model.NewColumnRef("order_date"),
model.NewFunction("count"),
},
WhereClause: model.NewFunction(
"has",
model.NewColumnRef("products::sku"),
model.NewLiteral("'XYZ'")),
GroupBy: []model.Expr{model.NewColumnRef("order_date")},
},
},
},
}

asString := func(query *model.Query) string {
return query.SelectCommand.String()
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
actual, err := transform.Transform([]*model.Query{tt.query})
assert.NoError(t, err)

if err != nil {
t.Fatal(err)
}

assert.True(t, len(actual) == 1, "len queries == 1")

expectedJson := asString(tt.expected)
actualJson := asString(actual[0])

assert.Equal(t, expectedJson, actualJson)
})
}
}

0 comments on commit 3e63212

Please sign in to comment.