From ba20b05bc8b0cbcc68fba520378cdc85610b726e Mon Sep 17 00:00:00 2001 From: Florian Bacher Date: Mon, 4 Nov 2024 21:59:35 +0100 Subject: [PATCH] [pkg/ottl]: add SliceToMap function (#35412) **Description:** This PR adds a function that converts slices to maps, as described in the linked issue. Currently still WIP, but creating a draft PR already to show how this could be implemented and used **Link to tracking Issue:** #35256 **Testing:** Added unit and end to end tests **Documentation:** Added description for the new function in the readme file --------- Signed-off-by: Florian Bacher Co-authored-by: Daniel Jaglowski Co-authored-by: Evan Bradley <11745660+evan-bradley@users.noreply.github.com> --- .chloggen/ottl-slice-to-map-function.yaml | 27 ++ pkg/ottl/e2e/e2e_test.go | 75 ++++- pkg/ottl/ottlfuncs/README.md | 62 ++++ pkg/ottl/ottlfuncs/func_slice_to_map.go | 105 +++++++ pkg/ottl/ottlfuncs/func_slice_to_map_test.go | 310 +++++++++++++++++++ pkg/ottl/ottlfuncs/functions.go | 1 + 6 files changed, 577 insertions(+), 3 deletions(-) create mode 100644 .chloggen/ottl-slice-to-map-function.yaml create mode 100644 pkg/ottl/ottlfuncs/func_slice_to_map.go create mode 100644 pkg/ottl/ottlfuncs/func_slice_to_map_test.go diff --git a/.chloggen/ottl-slice-to-map-function.yaml b/.chloggen/ottl-slice-to-map-function.yaml new file mode 100644 index 000000000000..e61f7c191296 --- /dev/null +++ b/.chloggen/ottl-slice-to-map-function.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: pkg/ottl + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add SliceToMap function + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [35256] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/pkg/ottl/e2e/e2e_test.go b/pkg/ottl/e2e/e2e_test.go index 451e577c8c4e..791550a5a062 100644 --- a/pkg/ottl/e2e/e2e_test.go +++ b/pkg/ottl/e2e/e2e_test.go @@ -57,6 +57,7 @@ func Test_e2e_editors(t *testing.T) { tCtx.GetLogRecord().Attributes().Remove("flags") tCtx.GetLogRecord().Attributes().Remove("total.string") tCtx.GetLogRecord().Attributes().Remove("foo") + tCtx.GetLogRecord().Attributes().Remove("things") }, }, { @@ -67,6 +68,15 @@ func Test_e2e_editors(t *testing.T) { tCtx.GetLogRecord().Attributes().PutStr("foo.flags", "pass") tCtx.GetLogRecord().Attributes().PutStr("foo.slice.0", "val") tCtx.GetLogRecord().Attributes().PutStr("foo.nested.test", "pass") + + tCtx.GetLogRecord().Attributes().Remove("things") + m1 := tCtx.GetLogRecord().Attributes().PutEmptyMap("things.0") + m1.PutStr("name", "foo") + m1.PutInt("value", 2) + + m2 := tCtx.GetLogRecord().Attributes().PutEmptyMap("things.1") + m2.PutStr("name", "bar") + m2.PutInt("value", 5) }, }, { @@ -84,12 +94,29 @@ func Test_e2e_editors(t *testing.T) { m.PutStr("test.foo.flags", "pass") m.PutStr("test.foo.slice.0", "val") m.PutStr("test.foo.nested.test", "pass") + + m1 := m.PutEmptyMap("test.things.0") + m1.PutStr("name", "foo") + m1.PutInt("value", 2) + + m2 := m.PutEmptyMap("test.things.1") + m2.PutStr("name", "bar") + m2.PutInt("value", 5) m.CopyTo(tCtx.GetLogRecord().Attributes()) }, }, { statement: `flatten(attributes, depth=0)`, - want: func(_ ottllog.TransformContext) {}, + want: func(tCtx ottllog.TransformContext) { + tCtx.GetLogRecord().Attributes().Remove("things") + m1 := tCtx.GetLogRecord().Attributes().PutEmptyMap("things.0") + m1.PutStr("name", "foo") + m1.PutInt("value", 2) + + m2 := tCtx.GetLogRecord().Attributes().PutEmptyMap("things.1") + m2.PutStr("name", "bar") + m2.PutInt("value", 5) + }, }, { statement: `flatten(attributes, depth=1)`, @@ -105,8 +132,17 @@ func Test_e2e_editors(t *testing.T) { m.PutStr("foo.bar", "pass") m.PutStr("foo.flags", "pass") m.PutStr("foo.slice.0", "val") - m2 := m.PutEmptyMap("foo.nested") - m2.PutStr("test", "pass") + + m1 := m.PutEmptyMap("things.0") + m1.PutStr("name", "foo") + m1.PutInt("value", 2) + + m2 := m.PutEmptyMap("things.1") + m2.PutStr("name", "bar") + m2.PutInt("value", 5) + + m3 := m.PutEmptyMap("foo.nested") + m3.PutStr("test", "pass") m.CopyTo(tCtx.GetLogRecord().Attributes()) }, }, @@ -117,6 +153,7 @@ func Test_e2e_editors(t *testing.T) { tCtx.GetLogRecord().Attributes().Remove("http.path") tCtx.GetLogRecord().Attributes().Remove("http.url") tCtx.GetLogRecord().Attributes().Remove("foo") + tCtx.GetLogRecord().Attributes().Remove("things") }, }, { @@ -131,6 +168,7 @@ func Test_e2e_editors(t *testing.T) { tCtx.GetLogRecord().Attributes().Remove("http.url") tCtx.GetLogRecord().Attributes().Remove("flags") tCtx.GetLogRecord().Attributes().Remove("foo") + tCtx.GetLogRecord().Attributes().Remove("things") }, }, { @@ -914,6 +952,28 @@ func Test_e2e_converters(t *testing.T) { m.PutStr("user_agent.version", "7.81.0") }, }, + { + statement: `set(attributes["test"], SliceToMap(attributes["things"], ["name"]))`, + want: func(tCtx ottllog.TransformContext) { + m := tCtx.GetLogRecord().Attributes().PutEmptyMap("test") + thing1 := m.PutEmptyMap("foo") + thing1.PutStr("name", "foo") + thing1.PutInt("value", 2) + + thing2 := m.PutEmptyMap("bar") + thing2.PutStr("name", "bar") + thing2.PutInt("value", 5) + }, + }, + { + statement: `set(attributes["test"], SliceToMap(attributes["things"], ["name"], ["value"]))`, + want: func(tCtx ottllog.TransformContext) { + m := tCtx.GetLogRecord().Attributes().PutEmptyMap("test") + m.PutInt("foo", 2) + m.PutInt("bar", 5) + + }, + }, } for _, tt := range tests { @@ -1112,6 +1172,15 @@ func constructLogTransformContext() ottllog.TransformContext { m2 := m.PutEmptyMap("nested") m2.PutStr("test", "pass") + s2 := logRecord.Attributes().PutEmptySlice("things") + thing1 := s2.AppendEmpty().SetEmptyMap() + thing1.PutStr("name", "foo") + thing1.PutInt("value", 2) + + thing2 := s2.AppendEmpty().SetEmptyMap() + thing2.PutStr("name", "bar") + thing2.PutInt("value", 5) + return ottllog.NewTransformContext(logRecord, scope, resource, plog.NewScopeLogs(), plog.NewResourceLogs()) } diff --git a/pkg/ottl/ottlfuncs/README.md b/pkg/ottl/ottlfuncs/README.md index ff01e7519bdd..a1ef094b4265 100644 --- a/pkg/ottl/ottlfuncs/README.md +++ b/pkg/ottl/ottlfuncs/README.md @@ -456,6 +456,7 @@ Available Converters: - [SHA1](#sha1) - [SHA256](#sha256) - [SHA512](#sha512) +- [SliceToMap](#slicetomap) - [Sort](#sort) - [SpanID](#spanid) - [Split](#split) @@ -1668,6 +1669,67 @@ Examples: - `SHA512("name")` +### SliceToMap + +`SliceToMap(target, keyPath, Optional[valuePath])` + +The `SliceToMap` converter converts a slice of objects to a map. The arguments are as follows: + +- `target`: A list of maps containing the entries to be converted. +- `keyPath`: A string array that determines the name of the keys for the map entries by pointing to the value of an attribute within each slice item. Note that +the `keyPath` must resolve to a string value, otherwise the converter will not be able to convert the item +to a map entry. +- `valuePath`: This optional string array determines which attribute should be used as the value for the map entry. If no +`valuePath` is defined, the value of the map entry will be the same as the original slice item. + +Examples: + +The examples below will convert the following input: + +```yaml +attributes: + hello: world + things: + - name: foo + value: 2 + - name: bar + value: 5 +``` + +- `SliceToMap(attributes["things"], ["name"])`: + +This converts the input above to the following: + +```yaml +attributes: + hello: world + things: + foo: + name: foo + value: 2 + bar: + name: bar + value: 5 +``` + +- `SliceToMap(attributes["things"], ["name"], ["value"])`: + +This converts the input above to the following: + +```yaml +attributes: + hello: world + things: + foo: 2 + bar: 5 +``` + +Once the `SliceToMap` function has been applied to a value, the converted entries are addressable via their keys: + +- `set(attributes["thingsMap"], SliceToMap(attributes["things"], ["name"]))` +- `set(attributes["element_1"], attributes["thingsMap"]["foo'])` +- `set(attributes["element_2"], attributes["thingsMap"]["bar'])` + ### Sort `Sort(target, Optional[order])` diff --git a/pkg/ottl/ottlfuncs/func_slice_to_map.go b/pkg/ottl/ottlfuncs/func_slice_to_map.go new file mode 100644 index 000000000000..2ed8575c0068 --- /dev/null +++ b/pkg/ottl/ottlfuncs/func_slice_to_map.go @@ -0,0 +1,105 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package ottlfuncs // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/ottlfuncs" +import ( + "fmt" + + "go.opentelemetry.io/collector/pdata/pcommon" + "golang.org/x/net/context" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" +) + +type SliceToMapArguments[K any] struct { + Target ottl.Getter[K] + KeyPath []string + ValuePath ottl.Optional[[]string] +} + +func NewSliceToMapFactory[K any]() ottl.Factory[K] { + return ottl.NewFactory("SliceToMap", &SliceToMapArguments[K]{}, sliceToMapFunction[K]) +} + +func sliceToMapFunction[K any](_ ottl.FunctionContext, oArgs ottl.Arguments) (ottl.ExprFunc[K], error) { + args, ok := oArgs.(*SliceToMapArguments[K]) + if !ok { + return nil, fmt.Errorf("SliceToMapFactory args must be of type *SliceToMapArguments[K") + } + + return getSliceToMapFunc(args.Target, args.KeyPath, args.ValuePath) +} + +func getSliceToMapFunc[K any](target ottl.Getter[K], keyPath []string, valuePath ottl.Optional[[]string]) (ottl.ExprFunc[K], error) { + if len(keyPath) == 0 { + return nil, fmt.Errorf("key path must contain at least one element") + } + return func(ctx context.Context, tCtx K) (any, error) { + val, err := target.Get(ctx, tCtx) + if err != nil { + return nil, err + } + + switch v := val.(type) { + case []any: + return sliceToMap(v, keyPath, valuePath) + case pcommon.Slice: + return sliceToMap(v.AsRaw(), keyPath, valuePath) + default: + return nil, fmt.Errorf("unsupported type provided to SliceToMap function: %T", v) + } + }, nil +} + +func sliceToMap(v []any, keyPath []string, valuePath ottl.Optional[[]string]) (any, error) { + result := make(map[string]any, len(v)) + for _, elem := range v { + e, ok := elem.(map[string]any) + if !ok { + return nil, fmt.Errorf("could not cast element '%v' to map[string]any", elem) + } + extractedKey, err := extractValue(e, keyPath) + if err != nil { + return nil, fmt.Errorf("could not extract key from element: %w", err) + } + + key, ok := extractedKey.(string) + if !ok { + return nil, fmt.Errorf("extracted key attribute is not of type string") + } + + if valuePath.IsEmpty() { + result[key] = e + continue + } + extractedValue, err := extractValue(e, valuePath.Get()) + if err != nil { + return nil, fmt.Errorf("could not extract value from element: %w", err) + } + result[key] = extractedValue + } + m := pcommon.NewMap() + if err := m.FromRaw(result); err != nil { + return nil, fmt.Errorf("could not create pcommon.Map from result: %w", err) + } + + return m, nil +} + +func extractValue(v map[string]any, path []string) (any, error) { + if len(path) == 0 { + return nil, fmt.Errorf("must provide at least one path item") + } + obj, ok := v[path[0]] + if !ok { + return nil, fmt.Errorf("provided object does not contain the path %v", path) + } + if len(path) == 1 { + return obj, nil + } + + if o, ok := obj.(map[string]any); ok { + return extractValue(o, path[1:]) + } + return nil, fmt.Errorf("provided object does not contain the path %v", path) +} diff --git a/pkg/ottl/ottlfuncs/func_slice_to_map_test.go b/pkg/ottl/ottlfuncs/func_slice_to_map_test.go new file mode 100644 index 000000000000..9ec68798356e --- /dev/null +++ b/pkg/ottl/ottlfuncs/func_slice_to_map_test.go @@ -0,0 +1,310 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package ottlfuncs + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/pdata/pcommon" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" +) + +func Test_SliceToMap(t *testing.T) { + type testCase struct { + name string + value func() any + keyPath []string + valuePath []string + want func() pcommon.Map + wantExecutionErr string + wantConfigErr string + } + tests := []testCase{ + { + name: "flat object with key path only", + keyPath: []string{"name"}, + value: func() any { + sl := pcommon.NewSlice() + thing1 := sl.AppendEmpty().SetEmptyMap() + thing1.PutStr("name", "foo") + thing1.PutInt("value", 2) + + thing2 := sl.AppendEmpty().SetEmptyMap() + thing2.PutStr("name", "bar") + thing2.PutInt("value", 5) + + return sl + }, + want: func() pcommon.Map { + m := pcommon.NewMap() + thing1 := m.PutEmptyMap("foo") + thing1.PutStr("name", "foo") + thing1.PutInt("value", 2) + + thing2 := m.PutEmptyMap("bar") + thing2.PutStr("name", "bar") + thing2.PutInt("value", 5) + + return m + }, + }, + { + name: "flat object with missing key value", + keyPath: []string{"notfound"}, + value: func() any { + sl := pcommon.NewSlice() + thing1 := sl.AppendEmpty().SetEmptyMap() + thing1.PutStr("name", "foo") + thing1.PutInt("value", 2) + + thing2 := sl.AppendEmpty().SetEmptyMap() + thing2.PutStr("name", "bar") + thing2.PutInt("value", 5) + + return sl + }, + wantExecutionErr: "could not extract key from element: provided object does not contain the path [notfound]", + }, + { + name: "flat object with key path and value path", + keyPath: []string{"name"}, + valuePath: []string{"value"}, + value: func() any { + sl := pcommon.NewSlice() + thing1 := sl.AppendEmpty().SetEmptyMap() + thing1.PutStr("name", "foo") + thing1.PutInt("value", 2) + + thing2 := sl.AppendEmpty().SetEmptyMap() + thing2.PutStr("name", "bar") + thing2.PutInt("value", 5) + + return sl + }, + want: func() pcommon.Map { + m := pcommon.NewMap() + m.PutInt("foo", 2) + m.PutInt("bar", 5) + + return m + }, + }, + { + name: "nested object with key path only", + keyPath: []string{"value", "test"}, + value: func() any { + sl := pcommon.NewSlice() + thing1 := sl.AppendEmpty().SetEmptyMap() + thing1.PutStr("name", "foo") + thing1.PutEmptyMap("value").PutStr("test", "x") + + thing2 := sl.AppendEmpty().SetEmptyMap() + thing2.PutStr("name", "bar") + thing2.PutInt("value", 5) + thing2.PutEmptyMap("value").PutStr("test", "y") + + return sl + }, + want: func() pcommon.Map { + m := pcommon.NewMap() + thing1 := m.PutEmptyMap("x") + thing1.PutStr("name", "foo") + thing1.PutEmptyMap("value").PutStr("test", "x") + + thing2 := m.PutEmptyMap("y") + thing2.PutStr("name", "bar") + thing2.PutEmptyMap("value").PutStr("test", "y") + + return m + }, + }, + { + name: "nested object with key path and value path", + keyPath: []string{"value", "test"}, + valuePath: []string{"name"}, + value: func() any { + sl := pcommon.NewSlice() + thing1 := sl.AppendEmpty().SetEmptyMap() + thing1.PutStr("name", "foo") + thing1.PutEmptyMap("value").PutStr("test", "x") + + thing2 := sl.AppendEmpty().SetEmptyMap() + thing2.PutStr("name", "bar") + thing2.PutInt("value", 5) + thing2.PutEmptyMap("value").PutStr("test", "y") + + return sl + }, + want: func() pcommon.Map { + m := pcommon.NewMap() + m.PutStr("x", "foo") + m.PutStr("y", "bar") + + return m + }, + }, + { + name: "flat object with key path resolving to non-string", + keyPath: []string{"value"}, + value: func() any { + sl := pcommon.NewSlice() + thing1 := sl.AppendEmpty().SetEmptyMap() + thing1.PutStr("name", "foo") + thing1.PutInt("value", 2) + + thing2 := sl.AppendEmpty().SetEmptyMap() + thing2.PutStr("name", "bar") + thing2.PutInt("value", 5) + + return sl + }, + wantExecutionErr: "extracted key attribute is not of type string", + }, + { + name: "nested object with value path not resolving to a value", + keyPath: []string{"value", "test"}, + valuePath: []string{"notfound"}, + value: func() any { + sl := pcommon.NewSlice() + thing1 := sl.AppendEmpty().SetEmptyMap() + thing1.PutStr("name", "foo") + thing1.PutEmptyMap("value").PutStr("test", "x") + + thing2 := sl.AppendEmpty().SetEmptyMap() + thing2.PutStr("name", "bar") + thing2.PutInt("value", 5) + thing2.PutEmptyMap("value").PutStr("test", "y") + + return sl + }, + wantExecutionErr: "could not extract value from element: provided object does not contain the path [notfound]", + }, + { + name: "nested object with value path segment resolving to non-map value", + keyPath: []string{"value", "test"}, + valuePath: []string{"name", "nothing"}, + value: func() any { + sl := pcommon.NewSlice() + thing1 := sl.AppendEmpty().SetEmptyMap() + thing1.PutStr("name", "foo") + thing1.PutEmptyMap("value").PutStr("test", "x") + + thing2 := sl.AppendEmpty().SetEmptyMap() + thing2.PutStr("name", "bar") + thing2.PutInt("value", 5) + thing2.PutEmptyMap("value").PutStr("test", "y") + + return sl + }, + wantExecutionErr: "could not extract value from element: provided object does not contain the path [name nothing]", + }, + { + name: "unsupported type", + keyPath: []string{"name"}, + value: func() any { + return pcommon.NewMap() + }, + wantExecutionErr: "unsupported type provided to SliceToMap function: pcommon.Map", + }, + { + name: "slice containing unsupported value type", + keyPath: []string{"name"}, + value: func() any { + sl := pcommon.NewSlice() + sl.AppendEmpty().SetStr("unsupported") + + return sl + }, + wantExecutionErr: "could not cast element 'unsupported' to map[string]any", + }, + { + name: "empty key path", + keyPath: []string{}, + value: func() any { + return pcommon.NewMap() + }, + wantConfigErr: "key path must contain at least one element", + }, + { + name: "mixed data types with invalid element", + keyPath: []string{"name"}, + valuePath: []string{"value"}, + value: func() any { + sl := pcommon.NewSlice() + thing1 := sl.AppendEmpty().SetEmptyMap() + thing1.PutStr("name", "foo") + thing1.PutInt("value", 2) + + sl.AppendEmpty().SetStr("nothingToSeeHere") + + thing2 := sl.AppendEmpty().SetEmptyMap() + thing2.PutStr("name", "bar") + thing2.PutInt("value", 5) + + return sl + }, + wantExecutionErr: "could not cast element 'nothingToSeeHere' to map[string]any", + }, + { + name: "nested with different value data types", + keyPath: []string{"name"}, + valuePath: []string{"value"}, + value: func() any { + sl := pcommon.NewSlice() + thing1 := sl.AppendEmpty().SetEmptyMap() + thing1.PutStr("name", "foo") + thing1.PutEmptyMap("value").PutStr("test", "value") + + thing2 := sl.AppendEmpty().SetEmptyMap() + thing2.PutStr("name", "bar") + thing2.PutInt("value", 5) + + return sl + }, + want: func() pcommon.Map { + m := pcommon.NewMap() + m.PutEmptyMap("foo").PutStr("test", "value") + m.PutInt("bar", 5) + + return m + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + valuePathOptional := ottl.Optional[[]string]{} + + if len(tt.valuePath) > 0 { + valuePathOptional = ottl.NewTestingOptional(tt.valuePath) + } + associateFunc, err := sliceToMapFunction[any](ottl.FunctionContext{}, &SliceToMapArguments[any]{ + Target: &ottl.StandardGetSetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return tt.value(), nil + }, + }, + KeyPath: tt.keyPath, + ValuePath: valuePathOptional, + }) + + if tt.wantConfigErr != "" { + require.ErrorContains(t, err, tt.wantConfigErr) + return + } + require.NoError(t, err) + + result, err := associateFunc(nil, nil) + if tt.wantExecutionErr != "" { + require.ErrorContains(t, err, tt.wantExecutionErr) + return + } + + require.NoError(t, err) + require.EqualValues(t, tt.want().AsRaw(), result.(pcommon.Map).AsRaw()) + }) + } +} diff --git a/pkg/ottl/ottlfuncs/functions.go b/pkg/ottl/ottlfuncs/functions.go index 9979c1800d3d..6fae06eb6b01 100644 --- a/pkg/ottl/ottlfuncs/functions.go +++ b/pkg/ottl/ottlfuncs/functions.go @@ -101,5 +101,6 @@ func converters[K any]() []ottl.Factory[K] { NewAppendFactory[K](), NewYearFactory[K](), NewHexFactory[K](), + NewSliceToMapFactory[K](), } }