Skip to content

Commit

Permalink
Using schema on query path, final round of tests update (#444)
Browse files Browse the repository at this point in the history
  • Loading branch information
@pdelewski
pdelewski authored Jul 5, 2024
1 parent 5f1b9de commit 0a70f98
Show file tree
Hide file tree
Showing 10 changed files with 550 additions and 91 deletions.
87 changes: 73 additions & 14 deletions quesma/queryparser/aggregation_parser_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,23 @@ import (
const tableName = "logs-generic-default"
const tableNameQuoted = `"` + tableName + `"`

type staticRegistry struct {
tables map[schema.TableName]schema.Schema
}

func (e staticRegistry) AllSchemas() map[schema.TableName]schema.Schema {
if e.tables != nil {
return e.tables
} else {
return map[schema.TableName]schema.Schema{}
}
}

func (e staticRegistry) FindSchema(name schema.TableName) (schema.Schema, bool) {
s, found := e.tables[name]
return s, found
}

// Simple unit tests, testing only "aggs" part of the request json query
var aggregationTests = []struct {
aggregationJson string
Expand Down Expand Up @@ -564,7 +581,26 @@ func TestAggregationParser(t *testing.T) {
t.Fatal(err)
}
lm := clickhouse.NewLogManager(concurrent.NewMapWith(tableName, table), config.QuesmaConfiguration{})
cw := ClickhouseQueryTranslator{ClickhouseLM: lm, Table: table, Ctx: context.Background()}
s := staticRegistry{
tables: map[schema.TableName]schema.Schema{
"logs-generic-default": {
Fields: map[schema.FieldName]schema.Field{
"host.name": {PropertyName: "host.name", InternalPropertyName: "host.name", Type: schema.TypeObject},
"type": {PropertyName: "type", InternalPropertyName: "type", Type: schema.TypeText},
"name": {PropertyName: "name", InternalPropertyName: "name", Type: schema.TypeText},
"content": {PropertyName: "content", InternalPropertyName: "content", Type: schema.TypeText},
"message": {PropertyName: "message", InternalPropertyName: "message", Type: schema.TypeText},
"host_name.keyword": {PropertyName: "host_name.keyword", InternalPropertyName: "host_name.keyword", Type: schema.TypeKeyword},
"FlightDelay": {PropertyName: "FlightDelay", InternalPropertyName: "FlightDelay", Type: schema.TypeText},
"Cancelled": {PropertyName: "Cancelled", InternalPropertyName: "Cancelled", Type: schema.TypeText},
"FlightDelayMin": {PropertyName: "FlightDelayMin", InternalPropertyName: "FlightDelayMin", Type: schema.TypeText},
"_id": {PropertyName: "_id", InternalPropertyName: "_id", Type: schema.TypeText},
},
},
},
}

cw := ClickhouseQueryTranslator{ClickhouseLM: lm, Table: table, Ctx: context.Background(), SchemaRegistry: s}

for testIdx, test := range aggregationTests {
t.Run(strconv.Itoa(testIdx), func(t *testing.T) {
Expand Down Expand Up @@ -612,20 +648,25 @@ func Test2AggregationParserExternalTestcases(t *testing.T) {
Config: clickhouse.NewDefaultCHConfig(),
}
lm := clickhouse.NewLogManager(concurrent.NewMapWith(tableName, &table), config.QuesmaConfiguration{})
indexConfig := map[string]config.IndexConfiguration{
"logs-generic-default": {
Name: "logs-generic-default",
Enabled: true,

s := staticRegistry{
tables: map[schema.TableName]schema.Schema{
"logs-generic-default": {
Fields: map[schema.FieldName]schema.Field{
"host.name": {PropertyName: "host.name", InternalPropertyName: "host.name", Type: schema.TypeObject},
"type": {PropertyName: "type", InternalPropertyName: "type", Type: schema.TypeText},
"name": {PropertyName: "name", InternalPropertyName: "name", Type: schema.TypeText},
"content": {PropertyName: "content", InternalPropertyName: "content", Type: schema.TypeText},
"message": {PropertyName: "message", InternalPropertyName: "message", Type: schema.TypeText},
"host_name.keyword": {PropertyName: "host_name.keyword", InternalPropertyName: "host_name.keyword", Type: schema.TypeKeyword},
"FlightDelay": {PropertyName: "FlightDelay", InternalPropertyName: "FlightDelay", Type: schema.TypeText},
"Cancelled": {PropertyName: "Cancelled", InternalPropertyName: "Cancelled", Type: schema.TypeText},
"FlightDelayMin": {PropertyName: "FlightDelayMin", InternalPropertyName: "FlightDelayMin", Type: schema.TypeText},
"_id": {PropertyName: "_id", InternalPropertyName: "_id", Type: schema.TypeText},
},
},
},
}
cfg := config.QuesmaConfiguration{
IndexConfig: indexConfig,
}
tableDiscovery :=
fixedTableProvider{tables: map[string]schema.Table{
"logs-generic-default": {Columns: map[string]schema.Column{}},
}}
s := schema.NewSchemaRegistry(tableDiscovery, cfg, clickhouse.SchemaTypeAdapter{})
cw := ClickhouseQueryTranslator{ClickhouseLM: lm, Table: &table, Ctx: context.Background(), SchemaRegistry: s}
allTests := testdata.AggregationTests
allTests = append(allTests, opensearch_visualize.AggregationTests...)
Expand Down Expand Up @@ -759,7 +800,25 @@ func Test_parseFieldFromScriptField(t *testing.T) {
{QueryMap{"script": "script"}, nil, false},
{QueryMap{"script": QueryMap{"source": 1}}, nil, false},
}
cw := ClickhouseQueryTranslator{Ctx: context.Background()}
s := staticRegistry{
tables: map[schema.TableName]schema.Schema{
"logs-generic-default": {
Fields: map[schema.FieldName]schema.Field{
"host.name": {PropertyName: "host.name", InternalPropertyName: "host.name", Type: schema.TypeObject},
"type": {PropertyName: "type", InternalPropertyName: "type", Type: schema.TypeText},
"name": {PropertyName: "name", InternalPropertyName: "name", Type: schema.TypeText},
"content": {PropertyName: "content", InternalPropertyName: "content", Type: schema.TypeText},
"message": {PropertyName: "message", InternalPropertyName: "message", Type: schema.TypeText},
"host_name.keyword": {PropertyName: "host_name.keyword", InternalPropertyName: "host_name.keyword", Type: schema.TypeKeyword},
"FlightDelay": {PropertyName: "FlightDelay", InternalPropertyName: "FlightDelay", Type: schema.TypeText},
"Cancelled": {PropertyName: "Cancelled", InternalPropertyName: "Cancelled", Type: schema.TypeText},
"FlightDelayMin": {PropertyName: "FlightDelayMin", InternalPropertyName: "FlightDelayMin", Type: schema.TypeText},
"_id": {PropertyName: "_id", InternalPropertyName: "_id", Type: schema.TypeText},
},
},
},
}
cw := ClickhouseQueryTranslator{Ctx: context.Background(), SchemaRegistry: s}
for _, tc := range testcases {
field, success := cw.parseFieldFromScriptField(tc.queryMap)
assert.Equal(t, tc.expectedSuccess, success)
Expand Down
41 changes: 39 additions & 2 deletions quesma/queryparser/aggregation_percentile_parser_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,33 @@ import (
"github.com/stretchr/testify/assert"
"quesma/clickhouse"
"quesma/model"
"quesma/schema"
"testing"
)

func Test_parsePercentilesAggregationWithDefaultPercents(t *testing.T) {
payload := QueryMap{
"field": "custom_name",
}
cw := &ClickhouseQueryTranslator{Table: &clickhouse.Table{}, Ctx: context.Background()}
s := staticRegistry{
tables: map[schema.TableName]schema.Schema{
"logs-generic-default": {
Fields: map[schema.FieldName]schema.Field{
"host.name": {PropertyName: "host.name", InternalPropertyName: "host.name", Type: schema.TypeObject},
"type": {PropertyName: "type", InternalPropertyName: "type", Type: schema.TypeText},
"name": {PropertyName: "name", InternalPropertyName: "name", Type: schema.TypeText},
"content": {PropertyName: "content", InternalPropertyName: "content", Type: schema.TypeText},
"message": {PropertyName: "message", InternalPropertyName: "message", Type: schema.TypeText},
"host_name.keyword": {PropertyName: "host_name.keyword", InternalPropertyName: "host_name.keyword", Type: schema.TypeKeyword},
"FlightDelay": {PropertyName: "FlightDelay", InternalPropertyName: "FlightDelay", Type: schema.TypeText},
"Cancelled": {PropertyName: "Cancelled", InternalPropertyName: "Cancelled", Type: schema.TypeText},
"FlightDelayMin": {PropertyName: "FlightDelayMin", InternalPropertyName: "FlightDelayMin", Type: schema.TypeText},
"_id": {PropertyName: "_id", InternalPropertyName: "_id", Type: schema.TypeText},
},
},
},
}
cw := &ClickhouseQueryTranslator{Table: &clickhouse.Table{}, Ctx: context.Background(), SchemaRegistry: s}
field, _, userSpecifiedPercents := cw.parsePercentilesAggregation(payload)
assert.Equal(t, model.NewColumnRef("custom_name"), field)
assert.Equal(t, defaultPercentiles, userSpecifiedPercents)
Expand Down Expand Up @@ -44,7 +63,25 @@ func Test_parsePercentilesAggregationWithUserSpecifiedPercents(t *testing.T) {
for k := range expectedOutputMap {
expectedOutputMapKeys = append(expectedOutputMapKeys, k)
}
cw := &ClickhouseQueryTranslator{Table: &clickhouse.Table{}, Ctx: context.Background()}
s := staticRegistry{
tables: map[schema.TableName]schema.Schema{
"logs-generic-default": {
Fields: map[schema.FieldName]schema.Field{
"host.name": {PropertyName: "host.name", InternalPropertyName: "host.name", Type: schema.TypeObject},
"type": {PropertyName: "type", InternalPropertyName: "type", Type: schema.TypeText},
"name": {PropertyName: "name", InternalPropertyName: "name", Type: schema.TypeText},
"content": {PropertyName: "content", InternalPropertyName: "content", Type: schema.TypeText},
"message": {PropertyName: "message", InternalPropertyName: "message", Type: schema.TypeText},
"host_name.keyword": {PropertyName: "host_name.keyword", InternalPropertyName: "host_name.keyword", Type: schema.TypeKeyword},
"FlightDelay": {PropertyName: "FlightDelay", InternalPropertyName: "FlightDelay", Type: schema.TypeText},
"Cancelled": {PropertyName: "Cancelled", InternalPropertyName: "Cancelled", Type: schema.TypeText},
"FlightDelayMin": {PropertyName: "FlightDelayMin", InternalPropertyName: "FlightDelayMin", Type: schema.TypeText},
"_id": {PropertyName: "_id", InternalPropertyName: "_id", Type: schema.TypeText},
},
},
},
}
cw := &ClickhouseQueryTranslator{Table: &clickhouse.Table{}, Ctx: context.Background(), SchemaRegistry: s}
fieldName, _, parsedMap := cw.parsePercentilesAggregation(payload)
assert.Equal(t, model.NewColumnRef("custom_name"), fieldName)

Expand Down
42 changes: 40 additions & 2 deletions quesma/queryparser/query_parser_async_search_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"quesma/clickhouse"
"quesma/concurrent"
"quesma/quesma/config"
"quesma/schema"
"quesma/testdata"
"testing"
)
Expand All @@ -26,7 +27,25 @@ func TestQueryParserAsyncSearch(t *testing.T) {
Created: true,
}
lm := clickhouse.NewLogManager(concurrent.NewMapWith(tableName, &table), config.QuesmaConfiguration{})
cw := ClickhouseQueryTranslator{ClickhouseLM: lm, Table: &table, Ctx: context.Background()}
s := staticRegistry{
tables: map[schema.TableName]schema.Schema{
"logs-generic-default": {
Fields: map[schema.FieldName]schema.Field{
"host.name": {PropertyName: "host.name", InternalPropertyName: "host.name", Type: schema.TypeObject},
"type": {PropertyName: "type", InternalPropertyName: "type", Type: schema.TypeText},
"name": {PropertyName: "name", InternalPropertyName: "name", Type: schema.TypeText},
"content": {PropertyName: "content", InternalPropertyName: "content", Type: schema.TypeText},
"message": {PropertyName: "message", InternalPropertyName: "message", Type: schema.TypeText},
"host.name.keyword": {PropertyName: "host.name.keyword", InternalPropertyName: "host.name.keyword", Type: schema.TypeKeyword},
"FlightDelay": {PropertyName: "FlightDelay", InternalPropertyName: "FlightDelay", Type: schema.TypeText},
"Cancelled": {PropertyName: "Cancelled", InternalPropertyName: "Cancelled", Type: schema.TypeText},
"FlightDelayMin": {PropertyName: "FlightDelayMin", InternalPropertyName: "FlightDelayMin", Type: schema.TypeText},
},
},
},
}

cw := ClickhouseQueryTranslator{ClickhouseLM: lm, Table: &table, Ctx: context.Background(), SchemaRegistry: s}
for _, tt := range testdata.TestsAsyncSearch {
t.Run(tt.Name, func(t *testing.T) {
query, queryInfo, _ := cw.ParseQueryAsyncSearch(tt.QueryJson)
Expand All @@ -40,7 +59,26 @@ func TestQueryParserAsyncSearch(t *testing.T) {
func TestQueryParserAggregation(t *testing.T) {
table := clickhouse.NewEmptyTable("tablename")
lm := clickhouse.NewLogManager(concurrent.NewMapWith("tablename", table), config.QuesmaConfiguration{})
cw := ClickhouseQueryTranslator{ClickhouseLM: lm, Table: table, Ctx: context.Background()}
s := staticRegistry{
tables: map[schema.TableName]schema.Schema{
"tablename": {
Fields: map[schema.FieldName]schema.Field{
"host.name": {PropertyName: "host.name", InternalPropertyName: "host.name", Type: schema.TypeObject},
"type": {PropertyName: "type", InternalPropertyName: "type", Type: schema.TypeText},
"name": {PropertyName: "name", InternalPropertyName: "name", Type: schema.TypeText},
"content": {PropertyName: "content", InternalPropertyName: "content", Type: schema.TypeText},
"message": {PropertyName: "message", InternalPropertyName: "message", Type: schema.TypeText},
"host_name.keyword": {PropertyName: "host_name.keyword", InternalPropertyName: "host_name.keyword", Type: schema.TypeKeyword},
"FlightDelay": {PropertyName: "FlightDelay", InternalPropertyName: "FlightDelay", Type: schema.TypeText},
"Cancelled": {PropertyName: "Cancelled", InternalPropertyName: "Cancelled", Type: schema.TypeText},
"FlightDelayMin": {PropertyName: "FlightDelayMin", InternalPropertyName: "FlightDelayMin", Type: schema.TypeText},
"_id": {PropertyName: "_id", InternalPropertyName: "_id", Type: schema.TypeText},
},
},
},
}

cw := ClickhouseQueryTranslator{ClickhouseLM: lm, Table: table, Ctx: context.Background(), SchemaRegistry: s}
for _, tt := range testdata.AggregationTests {
t.Run(tt.TestName, func(t *testing.T) {
cw.ParseQueryAsyncSearch(tt.QueryRequestJson)
Expand Down
32 changes: 17 additions & 15 deletions quesma/queryparser/query_parser_range_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,6 @@ type parseRangeTest struct {
expectedWhere string
}

type fixedTableProvider struct {
tables map[string]schema.Table
}

func (f fixedTableProvider) TableDefinitions() map[string]schema.Table {
return f.tables
}

var parseRangeTests = []parseRangeTest{
{
"DateTime64",
Expand Down Expand Up @@ -85,14 +77,24 @@ var parseRangeTests = []parseRangeTest{
}

func Test_parseRange(t *testing.T) {
indexConfig := map[string]config.IndexConfiguration{}
cfg := config.QuesmaConfiguration{
IndexConfig: indexConfig,
s := staticRegistry{
tables: map[schema.TableName]schema.Schema{
"logs-generic-default": {
Fields: map[schema.FieldName]schema.Field{
"host.name": {PropertyName: "host.name", InternalPropertyName: "host.name", Type: schema.TypeObject},
"type": {PropertyName: "type", InternalPropertyName: "type", Type: schema.TypeText},
"name": {PropertyName: "name", InternalPropertyName: "name", Type: schema.TypeText},
"content": {PropertyName: "content", InternalPropertyName: "content", Type: schema.TypeText},
"message": {PropertyName: "message", InternalPropertyName: "message", Type: schema.TypeText},
"host_name.keyword": {PropertyName: "host_name.keyword", InternalPropertyName: "host_name.keyword", Type: schema.TypeKeyword},
"FlightDelay": {PropertyName: "FlightDelay", InternalPropertyName: "FlightDelay", Type: schema.TypeText},
"Cancelled": {PropertyName: "Cancelled", InternalPropertyName: "Cancelled", Type: schema.TypeText},
"FlightDelayMin": {PropertyName: "FlightDelayMin", InternalPropertyName: "FlightDelayMin", Type: schema.TypeText},
"_id": {PropertyName: "_id", InternalPropertyName: "_id", Type: schema.TypeText},
},
},
},
}
tableDiscovery :=
fixedTableProvider{tables: map[string]schema.Table{}}
s := schema.NewSchemaRegistry(tableDiscovery, cfg, clickhouse.SchemaTypeAdapter{})

for _, test := range parseRangeTests {
t.Run(test.name, func(t *testing.T) {
table, err := clickhouse.NewTable(test.createTableQuery, clickhouse.NewNoTimestampOnlyStringAttrCHConfig())
Expand Down
Loading

0 comments on commit 0a70f98

Please sign in to comment.