Skip to content

Commit

Permalink
Schema: decouple internal property representation from the public-fac…
Browse files Browse the repository at this point in the history
…ing property name (#437)

Decouple public-facing property names from the internal representation.
For example, property _host.name_ can be internally stored as
_host::name_, but should be stiff referencible by _host.name_

At the moment of this change, internal names are the same as
public-facing ones.
  • Loading branch information
pivovarit authored Jul 3, 2024
1 parent 7f79aa0 commit a18ddad
Show file tree
Hide file tree
Showing 5 changed files with 47 additions and 44 deletions.
20 changes: 10 additions & 10 deletions quesma/quesma/functionality/field_capabilities/field_caps_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,10 +87,10 @@ func TestFieldCaps(t *testing.T) {
tables: map[schema.TableName]schema.Schema{
"logs-generic-default": {
Fields: map[schema.FieldName]schema.Field{
"service.name": {Name: "service.name", Type: schema.TypeKeyword},
"arrayOfArraysOfStrings": {Name: "arrayOfArraysOfStrings", Type: schema.TypeKeyword},
"arrayOfTuples": {Name: "arrayOfTuples", Type: schema.TypeObject},
"host.name": {Name: "host.name", Type: schema.TypeObject},
"service.name": {PropertyName: "service.name", InternalPropertyName: "service.name", Type: schema.TypeKeyword},
"arrayOfArraysOfStrings": {PropertyName: "arrayOfArraysOfStrings", InternalPropertyName: "arrayOfArraysOfStrings", Type: schema.TypeKeyword},
"arrayOfTuples": {PropertyName: "arrayOfTuples", InternalPropertyName: "arrayOfTuples", Type: schema.TypeObject},
"host.name": {PropertyName: "host.name", InternalPropertyName: "host.name", Type: schema.TypeObject},
},
},
},
Expand Down Expand Up @@ -146,7 +146,7 @@ func TestFieldCapsWithAliases(t *testing.T) {
}, staticRegistry{
tables: map[schema.TableName]schema.Schema{
"logs-generic-default": {
Fields: map[schema.FieldName]schema.Field{"@timestamp": {Name: "@timestamp", Type: schema.TypeTimestamp}},
Fields: map[schema.FieldName]schema.Field{"@timestamp": {PropertyName: "@timestamp", InternalPropertyName: "@timestamp", Type: schema.TypeTimestamp}},
Aliases: map[schema.FieldName]schema.FieldName{"timestamp": "@timestamp"},
},
},
Expand Down Expand Up @@ -196,12 +196,12 @@ func TestFieldCapsMultipleIndexes(t *testing.T) {
tables: map[schema.TableName]schema.Schema{
"logs-1": {
Fields: map[schema.FieldName]schema.Field{
"foo.bar1": {Name: "foo.bar1", Type: schema.TypeKeyword},
"foo.bar1": {PropertyName: "foo.bar1", InternalPropertyName: "foo.bar1", Type: schema.TypeKeyword},
},
},
"logs-2": {
Fields: map[schema.FieldName]schema.Field{
"foo.bar2": {Name: "foo.bar2", Type: schema.TypeKeyword},
"foo.bar2": {PropertyName: "foo.bar2", InternalPropertyName: "foo.bar2", Type: schema.TypeKeyword},
},
},
},
Expand Down Expand Up @@ -309,17 +309,17 @@ func TestFieldCapsMultipleIndexesConflictingEntries(t *testing.T) {
tables: map[schema.TableName]schema.Schema{
"logs-1": {
Fields: map[schema.FieldName]schema.Field{
"foo.bar": {Name: "foo.bar", Type: schema.TypeKeyword},
"foo.bar": {PropertyName: "foo.bar", InternalPropertyName: "foo.bar", Type: schema.TypeKeyword},
},
},
"logs-2": {
Fields: map[schema.FieldName]schema.Field{
"foo.bar": {Name: "foo.bar", Type: schema.TypeBoolean},
"foo.bar": {PropertyName: "foo.bar", InternalPropertyName: "foo.bar", Type: schema.TypeBoolean},
},
},
"logs-3": {
Fields: map[schema.FieldName]schema.Field{
"foo.bar": {Name: "foo.bar", Type: schema.TypeBoolean},
"foo.bar": {PropertyName: "foo.bar", InternalPropertyName: "foo.bar", Type: schema.TypeBoolean},
},
},
},
Expand Down
8 changes: 4 additions & 4 deletions quesma/schema/registry.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ func (s *schemaRegistry) populateSchemaFromStaticConfiguration(indexConfiguratio
if deprecatedConfigInUse(indexConfiguration) {
for fieldName, fieldType := range indexConfiguration.TypeMappings {
if resolvedType, valid := ParseType(fieldType); valid {
fields[FieldName(fieldName)] = Field{Name: FieldName(fieldName), Type: resolvedType}
fields[FieldName(fieldName)] = Field{PropertyName: FieldName(fieldName), InternalPropertyName: FieldName(fieldName), Type: resolvedType}
} else {
logger.Warn().Msgf("invalid configuration: type %s not supported (should have been spotted when validating configuration)", fieldType)
}
Expand All @@ -95,7 +95,7 @@ func (s *schemaRegistry) populateSchemaFromStaticConfiguration(indexConfiguratio
continue
}
if resolvedType, valid := ParseType(field.Type.AsString()); valid {
fields[FieldName(field.Name.AsString())] = Field{Name: FieldName(field.Name.AsString()), Type: resolvedType}
fields[FieldName(field.Name.AsString())] = Field{PropertyName: FieldName(field.Name.AsString()), InternalPropertyName: FieldName(field.Name.AsString()), Type: resolvedType}
} else {
logger.Warn().Msgf("invalid configuration: type %s not supported (should have been spotted when validating configuration)", field.Type.AsString())
}
Expand Down Expand Up @@ -132,10 +132,10 @@ func (s *schemaRegistry) populateSchemaFromTableDefinition(definitions map[strin
for _, column := range tableDefinition.Columns {
if _, exists := fields[FieldName(column.Name)]; !exists {
if quesmaType, found2 := s.dataSourceTypeAdapter.Convert(column.Type); found2 {
fields[FieldName(column.Name)] = Field{Name: FieldName(column.Name), Type: quesmaType}
fields[FieldName(column.Name)] = Field{PropertyName: FieldName(column.Name), InternalPropertyName: FieldName(column.Name), Type: quesmaType}
} else {
logger.Debug().Msgf("type %s not supported, falling back to text", column.Type)
fields[FieldName(column.Name)] = Field{Name: FieldName(column.Name), Type: TypeText}
fields[FieldName(column.Name)] = Field{PropertyName: FieldName(column.Name), InternalPropertyName: FieldName(column.Name), Type: TypeText}
}
}
}
Expand Down
44 changes: 22 additions & 22 deletions quesma/schema/registry_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,9 @@ func Test_schemaRegistry_FindSchema(t *testing.T) {
}},
tableName: "some_table",
want: schema.Schema{Fields: map[schema.FieldName]schema.Field{
"message": {Name: "message", Type: schema.TypeKeyword},
"event_date": {Name: "event_date", Type: schema.TypeTimestamp},
"count": {Name: "count", Type: schema.TypeLong}},
"message": {PropertyName: "message", InternalPropertyName: "message", Type: schema.TypeKeyword},
"event_date": {PropertyName: "event_date", InternalPropertyName: "event_date", Type: schema.TypeTimestamp},
"count": {PropertyName: "count", InternalPropertyName: "count", Type: schema.TypeLong}},
Aliases: map[schema.FieldName]schema.FieldName{}},
exists: true,
},
Expand All @@ -65,9 +65,9 @@ func Test_schemaRegistry_FindSchema(t *testing.T) {
}},
tableName: "some_table",
want: schema.Schema{Fields: map[schema.FieldName]schema.Field{
"message": {Name: "message", Type: schema.TypeKeyword},
"event_date": {Name: "event_date", Type: schema.TypeTimestamp},
"count": {Name: "count", Type: schema.TypeLong}},
"message": {PropertyName: "message", InternalPropertyName: "message", Type: schema.TypeKeyword},
"event_date": {PropertyName: "event_date", InternalPropertyName: "event_date", Type: schema.TypeTimestamp},
"count": {PropertyName: "count", InternalPropertyName: "count", Type: schema.TypeLong}},
Aliases: map[schema.FieldName]schema.FieldName{}},
exists: true,
},
Expand All @@ -86,9 +86,9 @@ func Test_schemaRegistry_FindSchema(t *testing.T) {
}},
tableName: "some_table",
want: schema.Schema{Fields: map[schema.FieldName]schema.Field{
"message": {Name: "message", Type: schema.TypeKeyword},
"event_date": {Name: "event_date", Type: schema.TypeTimestamp},
"count": {Name: "count", Type: schema.TypeLong}},
"message": {PropertyName: "message", InternalPropertyName: "message", Type: schema.TypeKeyword},
"event_date": {PropertyName: "event_date", InternalPropertyName: "event_date", Type: schema.TypeTimestamp},
"count": {PropertyName: "count", InternalPropertyName: "count", Type: schema.TypeLong}},
Aliases: map[schema.FieldName]schema.FieldName{}},
exists: true,
},
Expand All @@ -111,9 +111,9 @@ func Test_schemaRegistry_FindSchema(t *testing.T) {
}},
tableName: "some_table",
want: schema.Schema{Fields: map[schema.FieldName]schema.Field{
"message": {Name: "message", Type: schema.TypeKeyword},
"event_date": {Name: "event_date", Type: schema.TypeTimestamp},
"count": {Name: "count", Type: schema.TypeLong}},
"message": {PropertyName: "message", InternalPropertyName: "message", Type: schema.TypeKeyword},
"event_date": {PropertyName: "event_date", InternalPropertyName: "event_date", Type: schema.TypeTimestamp},
"count": {PropertyName: "count", InternalPropertyName: "count", Type: schema.TypeLong}},
Aliases: map[schema.FieldName]schema.FieldName{}},
exists: true,
},
Expand All @@ -131,7 +131,7 @@ func Test_schemaRegistry_FindSchema(t *testing.T) {
tableDiscovery: fixedTableProvider{tables: map[string]schema.Table{}},
tableName: "some_table",
want: schema.Schema{Fields: map[schema.FieldName]schema.Field{
"message": {Name: "message", Type: schema.TypeKeyword}},
"message": {PropertyName: "message", InternalPropertyName: "message", Type: schema.TypeKeyword}},
Aliases: map[schema.FieldName]schema.FieldName{}},
exists: true,
},
Expand All @@ -155,9 +155,9 @@ func Test_schemaRegistry_FindSchema(t *testing.T) {
}}},
tableName: "some_table",
want: schema.Schema{Fields: map[schema.FieldName]schema.Field{
"message": {Name: "message", Type: schema.TypeKeyword},
"event_date": {Name: "event_date", Type: schema.TypeTimestamp},
"count": {Name: "count", Type: schema.TypeLong}},
"message": {PropertyName: "message", InternalPropertyName: "message", Type: schema.TypeKeyword},
"event_date": {PropertyName: "event_date", InternalPropertyName: "event_date", Type: schema.TypeTimestamp},
"count": {PropertyName: "count", InternalPropertyName: "count", Type: schema.TypeLong}},
Aliases: map[schema.FieldName]schema.FieldName{}},
exists: true,
},
Expand All @@ -182,9 +182,9 @@ func Test_schemaRegistry_FindSchema(t *testing.T) {
}},
tableName: "some_table",
want: schema.Schema{Fields: map[schema.FieldName]schema.Field{
"message": {Name: "message", Type: schema.TypeKeyword},
"event_date": {Name: "event_date", Type: schema.TypeTimestamp},
"count": {Name: "count", Type: schema.TypeLong}},
"message": {PropertyName: "message", InternalPropertyName: "message", Type: schema.TypeKeyword},
"event_date": {PropertyName: "event_date", InternalPropertyName: "event_date", Type: schema.TypeTimestamp},
"count": {PropertyName: "count", InternalPropertyName: "count", Type: schema.TypeLong}},
Aliases: map[schema.FieldName]schema.FieldName{
"message_alias": "message",
}},
Expand All @@ -209,9 +209,9 @@ func Test_schemaRegistry_FindSchema(t *testing.T) {
}},
tableName: "some_table",
want: schema.Schema{Fields: map[schema.FieldName]schema.Field{
"message": {Name: "message", Type: schema.TypeKeyword},
"event_date": {Name: "event_date", Type: schema.TypeTimestamp},
"count": {Name: "count", Type: schema.TypeLong}},
"message": {PropertyName: "message", InternalPropertyName: "message", Type: schema.TypeKeyword},
"event_date": {PropertyName: "event_date", InternalPropertyName: "event_date", Type: schema.TypeTimestamp},
"count": {PropertyName: "count", InternalPropertyName: "count", Type: schema.TypeLong}},
Aliases: map[schema.FieldName]schema.FieldName{
"message_alias": "message",
}},
Expand Down
7 changes: 5 additions & 2 deletions quesma/schema/schema.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,11 @@ type (
Aliases map[FieldName]FieldName
}
Field struct {
Name FieldName
Type Type
// PropertyName is how users refer to the field
PropertyName FieldName
// InternalPropertyName is how the field is represented in the data source
InternalPropertyName FieldName
Type Type
}
TableName string
FieldName string
Expand Down
12 changes: 6 additions & 6 deletions quesma/schema/schema_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,18 +26,18 @@ func TestSchema_ResolveField(t *testing.T) {
fieldName: "message",
schema: Schema{
Fields: map[FieldName]Field{
"message": {Name: "message", Type: TypeText},
"message": {PropertyName: "message", InternalPropertyName: "message", Type: TypeText},
},
},
resolvedField: Field{Name: "message", Type: TypeText},
resolvedField: Field{PropertyName: "message", InternalPropertyName: "message", Type: TypeText},
exists: true,
},
{
name: "should not resolve field",
fieldName: "foo",
schema: Schema{
Fields: map[FieldName]Field{
"message": {Name: "message", Type: TypeText},
"message": {PropertyName: "message", InternalPropertyName: "message", Type: TypeText},
},
},
resolvedField: Field{},
Expand All @@ -47,17 +47,17 @@ func TestSchema_ResolveField(t *testing.T) {
name: "should resolve aliased field",
fieldName: "message_alias",
schema: Schema{
Fields: map[FieldName]Field{"message": {Name: "message", Type: TypeText}},
Fields: map[FieldName]Field{"message": {PropertyName: "message", InternalPropertyName: "message", Type: TypeText}},
Aliases: map[FieldName]FieldName{"message_alias": "message"},
},
resolvedField: Field{Name: "message", Type: TypeText},
resolvedField: Field{PropertyName: "message", InternalPropertyName: "message", Type: TypeText},
exists: true,
},
{
name: "should not resolve aliased field",
fieldName: "message_alias",
schema: Schema{
Fields: map[FieldName]Field{"message": {Name: "message", Type: TypeText}},
Fields: map[FieldName]Field{"message": {PropertyName: "message", InternalPropertyName: "message", Type: TypeText}},
Aliases: map[FieldName]FieldName{"message_alias": "foo"},
},
resolvedField: Field{},
Expand Down

0 comments on commit a18ddad

Please sign in to comment.