From a9a44c32626e367f65c8c55137d595738411f55d Mon Sep 17 00:00:00 2001 From: Daniel Kuiper <44123852+kuiperda@users.noreply.github.com> Date: Tue, 1 Oct 2024 17:43:29 -0400 Subject: [PATCH] [pkg/ottl] Add ToKeyValueString Converter (#35409) **Description:** Implements ToKeyValueString OTTL Converter. **Link to tracking Issue:** #35334 **Testing:** Added unit tests and e2e **Documentation:** Added --- .chloggen/ottl-tokeyvaluestring.yaml | 27 ++ pkg/ottl/e2e/e2e_test.go | 24 ++ pkg/ottl/ottlfuncs/README.md | 39 +++ .../ottlfuncs/func_to_key_value_string.go | 122 +++++++++ .../func_to_key_value_string_test.go | 249 ++++++++++++++++++ pkg/ottl/ottlfuncs/functions.go | 1 + 6 files changed, 462 insertions(+) create mode 100644 .chloggen/ottl-tokeyvaluestring.yaml create mode 100644 pkg/ottl/ottlfuncs/func_to_key_value_string.go create mode 100644 pkg/ottl/ottlfuncs/func_to_key_value_string_test.go diff --git a/.chloggen/ottl-tokeyvaluestring.yaml b/.chloggen/ottl-tokeyvaluestring.yaml new file mode 100644 index 000000000000..a762cff62296 --- /dev/null +++ b/.chloggen/ottl-tokeyvaluestring.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: pkg/ottl + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add ToKeyValueString Converter + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [35334] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/pkg/ottl/e2e/e2e_test.go b/pkg/ottl/e2e/e2e_test.go index 592332e76585..ede1e329bdb1 100644 --- a/pkg/ottl/e2e/e2e_test.go +++ b/pkg/ottl/e2e/e2e_test.go @@ -621,6 +621,30 @@ func Test_e2e_converters(t *testing.T) { m.PutStr("k2", "v2__!__v2") }, }, + { + statement: `set(attributes["test"], ToKeyValueString(ParseKeyValue("k1=v1 k2=v2"), "=", " ", true))`, + want: func(tCtx ottllog.TransformContext) { + tCtx.GetLogRecord().Attributes().PutStr("test", "k1=v1 k2=v2") + }, + }, + { + statement: `set(attributes["test"], ToKeyValueString(ParseKeyValue("k1:v1,k2:v2", ":" , ","), ":", ",", true))`, + want: func(tCtx ottllog.TransformContext) { + tCtx.GetLogRecord().Attributes().PutStr("test", "k1:v1,k2:v2") + }, + }, + { + statement: `set(attributes["test"], ToKeyValueString(ParseKeyValue("k1=v1 k2=v2"), "!", "+", true))`, + want: func(tCtx ottllog.TransformContext) { + tCtx.GetLogRecord().Attributes().PutStr("test", "k1!v1+k2!v2") + }, + }, + { + statement: `set(attributes["test"], ToKeyValueString(ParseKeyValue("k1=v1 k2=v2=v3"), "=", " ", true))`, + want: func(tCtx ottllog.TransformContext) { + tCtx.GetLogRecord().Attributes().PutStr("test", "k1=v1 k2=\"v2=v3\"") + }, + }, { statement: `set(attributes["test"], ParseXML("This is a log message!"))`, want: func(tCtx ottllog.TransformContext) { diff --git a/pkg/ottl/ottlfuncs/README.md b/pkg/ottl/ottlfuncs/README.md index ab36043ada24..380cb41d8c50 100644 --- a/pkg/ottl/ottlfuncs/README.md +++ b/pkg/ottl/ottlfuncs/README.md @@ -459,6 +459,7 @@ Available Converters: - [String](#string) - [Substring](#substring) - [Time](#time) +- [ToKeyValueString](#tokeyvaluestring) - [TraceID](#traceid) - [TruncateTime](#truncatetime) - [Unix](#unix) @@ -1660,6 +1661,44 @@ Examples: - `Time("mercoledì set 4 2024", "%A %h %e %Y", "", "it")` - `Time("Febrero 25 lunes, 2002, 02:03:04 p.m.", "%B %d %A, %Y, %r", "America/New_York", "es-ES")` +### ToKeyValueString + +`ToKeyValueString(target, Optional[delimiter], Optional[pair_delimiter], Optional[sort_output])` + +The `ToKeyValueString` Converter takes a `pcommon.Map` and converts it to a `string` of key value pairs. + +- `target` is a Getter that returns a `pcommon.Map`. +- `delimiter` is an optional string that is used to join keys and values, the default is `=`. +- `pair_delimiter` is an optional string that is used to join key value pairs, the default is a single space (` `). +- `sort_output` is an optional bool that is used to deterministically sort the keys of the output string. It should only be used if the output is required to be in the same order each time, as it introduces some performance overhead. + +For example, the following map `{"k1":"v1","k2":"v2","k3":"v3"}` will use default delimiters and be converted into the following string: + +``` +`k1=v1 k2=v2 k3=v3` +``` + +**Note:** Any nested arrays or maps will be represented as a JSON string. It is recommended to [flatten](#flatten) `target` before using this function. + +For example, `{"k1":"v1","k2":{"k3":"v3","k4":["v4","v5"]}}` will be converted to: + +``` +`k1=v1 k2={\"k3\":\"v3\",\"k4\":[\"v4\",\"v5\"]}` +``` + +**Note:** If any keys or values contain either delimiter, they will be double quoted. If any double quotes are present in the quoted value, they will be escaped. + +For example, `{"k1":"v1","k2":"v=2","k3"="\"v=3\""}` will be converted to: + +``` +`k1=v1 k2="v=2" k3="\"v=3\""` +``` + +Examples: + +- `ToKeyValueString(body)` +- `ToKeyValueString(body, ":", ",", true)` + ### TraceID `TraceID(bytes)` diff --git a/pkg/ottl/ottlfuncs/func_to_key_value_string.go b/pkg/ottl/ottlfuncs/func_to_key_value_string.go new file mode 100644 index 000000000000..ece12a88e1bc --- /dev/null +++ b/pkg/ottl/ottlfuncs/func_to_key_value_string.go @@ -0,0 +1,122 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package ottlfuncs // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/ottlfuncs" + +import ( + "context" + "fmt" + gosort "sort" + "strings" + + "go.opentelemetry.io/collector/pdata/pcommon" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" +) + +type ToKeyValueStringArguments[K any] struct { + Target ottl.PMapGetter[K] + Delimiter ottl.Optional[string] + PairDelimiter ottl.Optional[string] + SortOutput ottl.Optional[bool] +} + +func NewToKeyValueStringFactory[K any]() ottl.Factory[K] { + return ottl.NewFactory("ToKeyValueString", &ToKeyValueStringArguments[K]{}, createToKeyValueStringFunction[K]) +} + +func createToKeyValueStringFunction[K any](_ ottl.FunctionContext, oArgs ottl.Arguments) (ottl.ExprFunc[K], error) { + args, ok := oArgs.(*ToKeyValueStringArguments[K]) + + if !ok { + return nil, fmt.Errorf("ToKeyValueStringFactory args must be of type *ToKeyValueStringArguments[K]") + } + + return toKeyValueString[K](args.Target, args.Delimiter, args.PairDelimiter, args.SortOutput) +} + +func toKeyValueString[K any](target ottl.PMapGetter[K], d ottl.Optional[string], p ottl.Optional[string], s ottl.Optional[bool]) (ottl.ExprFunc[K], error) { + delimiter := "=" + if !d.IsEmpty() { + if d.Get() == "" { + return nil, fmt.Errorf("delimiter cannot be set to an empty string") + } + delimiter = d.Get() + } + + pairDelimiter := " " + if !p.IsEmpty() { + if p.Get() == "" { + return nil, fmt.Errorf("pair delimiter cannot be set to an empty string") + } + pairDelimiter = p.Get() + } + + if pairDelimiter == delimiter { + return nil, fmt.Errorf("pair delimiter %q cannot be equal to delimiter %q", pairDelimiter, delimiter) + } + + sortOutput := false + if !s.IsEmpty() { + sortOutput = s.Get() + } + + return func(ctx context.Context, tCtx K) (any, error) { + source, err := target.Get(ctx, tCtx) + if err != nil { + return nil, err + } + + return convertMapToKV(source, delimiter, pairDelimiter, sortOutput), nil + }, nil +} + +// convertMapToKV converts a pcommon.Map to a key value string +func convertMapToKV(target pcommon.Map, delimiter string, pairDelimiter string, sortOutput bool) string { + + var kvStrings []string + if sortOutput { + var keyValues []struct { + key string + val pcommon.Value + } + + // Sort by keys + target.Range(func(k string, v pcommon.Value) bool { + keyValues = append(keyValues, struct { + key string + val pcommon.Value + }{key: k, val: v}) + return true + }) + gosort.Slice(keyValues, func(i, j int) bool { + return keyValues[i].key < keyValues[j].key + }) + + // Convert KV pairs + for _, kv := range keyValues { + kvStrings = append(kvStrings, buildKVString(kv.key, kv.val, delimiter, pairDelimiter)) + } + } else { + target.Range(func(k string, v pcommon.Value) bool { + kvStrings = append(kvStrings, buildKVString(k, v, delimiter, pairDelimiter)) + return true + }) + } + + return strings.Join(kvStrings, pairDelimiter) +} + +func buildKVString(k string, v pcommon.Value, delimiter string, pairDelimiter string) string { + key := escapeAndQuoteKV(k, delimiter, pairDelimiter) + value := escapeAndQuoteKV(v.AsString(), delimiter, pairDelimiter) + return key + delimiter + value +} + +func escapeAndQuoteKV(s string, delimiter string, pairDelimiter string) string { + s = strings.ReplaceAll(s, `"`, `\"`) + if strings.Contains(s, pairDelimiter) || strings.Contains(s, delimiter) { + s = `"` + s + `"` + } + return s +} diff --git a/pkg/ottl/ottlfuncs/func_to_key_value_string_test.go b/pkg/ottl/ottlfuncs/func_to_key_value_string_test.go new file mode 100644 index 000000000000..981c8c8ceb8d --- /dev/null +++ b/pkg/ottl/ottlfuncs/func_to_key_value_string_test.go @@ -0,0 +1,249 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package ottlfuncs + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" +) + +func Test_toKeyValueString(t *testing.T) { + tests := []struct { + name string + target ottl.PMapGetter[any] + delimiter ottl.Optional[string] + pairDelimiter ottl.Optional[string] + expected string + }{ + { + name: "default delimiters with no nesting", + target: ottl.StandardPMapGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return map[string]any{ + "key1": "value1", + "key2": "value2", + }, nil + }, + }, + delimiter: ottl.Optional[string]{}, + pairDelimiter: ottl.Optional[string]{}, + expected: `key1=value1 key2=value2`, + }, + { + name: "custom delimiter with no nesting", + target: ottl.StandardPMapGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return map[string]any{ + "key1": "value1", + "key2": "value2", + }, nil + }, + }, + delimiter: ottl.NewTestingOptional[string](":"), + pairDelimiter: ottl.Optional[string]{}, + expected: `key1:value1 key2:value2`, + }, + { + name: "custom pair delimiter with no nesting", + target: ottl.StandardPMapGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return map[string]any{ + "key1": "value1", + "key2": "value2", + }, nil + }, + }, + delimiter: ottl.Optional[string]{}, + pairDelimiter: ottl.NewTestingOptional[string](","), + expected: `key1=value1,key2=value2`, + }, + { + name: "delimiters present in keys and values", + target: ottl.StandardPMapGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return map[string]any{ + "key 1": "value 1", + "key2=": "value2=", + }, nil + }, + }, + delimiter: ottl.Optional[string]{}, + pairDelimiter: ottl.Optional[string]{}, + expected: `"key 1"="value 1" "key2="="value2="`, + }, + { + name: "long delimiters present in keys and values", + target: ottl.StandardPMapGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return map[string]any{ + "key1": "value1", + "key2,,,": "value2,,,,,,", + }, nil + }, + }, + delimiter: ottl.NewTestingOptional[string](",,,"), + pairDelimiter: ottl.Optional[string]{}, + expected: `key1,,,value1 "key2,,,",,,"value2,,,,,,"`, + }, + { + name: "delimiters and quotes present in keys and values", + target: ottl.StandardPMapGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return map[string]any{ + "key 1": "value 1", + "key2\"=": "value2\"=", + "key\"3": "value\"3", + }, nil + }, + }, + delimiter: ottl.Optional[string]{}, + pairDelimiter: ottl.Optional[string]{}, + expected: `"key 1"="value 1" key\"3=value\"3 "key2\"="="value2\"="`, + }, + { + name: "nested", + target: ottl.StandardPMapGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return map[string]any{ + "key1": "value1", + "key2": map[string]any{ + "key3": "value3", + "key4": map[string]any{ + "key5": "value5", + "key6": []any{"value6a", "value6b"}, + }, + }, + "key7": []any{"value7", []any{"value8a", map[string]any{"key8b": "value8b"}}}, + }, nil + }, + }, + delimiter: ottl.Optional[string]{}, + pairDelimiter: ottl.Optional[string]{}, + expected: `key1=value1 key2={\"key3\":\"value3\",\"key4\":{\"key5\":\"value5\",\"key6\":[\"value6a\",\"value6b\"]}} key7=[\"value7\",[\"value8a\",{\"key8b\":\"value8b\"}]]`, + }, + { + name: "nested with delimiter present", + target: ottl.StandardPMapGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return map[string]any{ + "key1": "value1", + "key2": map[string]any{ + "key3": "value3", + "key4": map[string]any{ + "key5": "value=5", + "key6": []any{"value6a", "value6b"}, + }, + }, + "key7": []any{"value7", []any{"value8a", map[string]any{"key 8b": "value8b"}}}, + }, nil + }, + }, + delimiter: ottl.Optional[string]{}, + pairDelimiter: ottl.Optional[string]{}, + expected: `key1=value1 key2="{\"key3\":\"value3\",\"key4\":{\"key5\":\"value=5\",\"key6\":[\"value6a\",\"value6b\"]}}" key7="[\"value7\",[\"value8a\",{\"key 8b\":\"value8b\"}]]"`, + }, + { + name: "nested with delimiter and quotes present", + target: ottl.StandardPMapGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return map[string]any{ + "key1": "value1\"", + "key2": map[string]any{ + "key3": "value3", + "key4": map[string]any{ + "key5": "value=5\"", + "key6": []any{"value6a", "value6b"}, + }, + }, + "key7": []any{"value7", []any{"value8a", map[string]any{"key 8b\"": "value8b"}}}, + }, nil + }, + }, + delimiter: ottl.Optional[string]{}, + pairDelimiter: ottl.Optional[string]{}, + expected: `key1=value1\" key2="{\"key3\":\"value3\",\"key4\":{\"key5\":\"value=5\\"\",\"key6\":[\"value6a\",\"value6b\"]}}" key7="[\"value7\",[\"value8a\",{\"key 8b\\"\":\"value8b\"}]]"`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + exprFunc, err := toKeyValueString[any](tt.target, tt.delimiter, tt.pairDelimiter, ottl.NewTestingOptional[bool](true)) + assert.NoError(t, err) + + result, err := exprFunc(context.Background(), nil) + assert.NoError(t, err) + + actual, ok := result.(string) + assert.True(t, ok) + + assert.Equal(t, tt.expected, actual) + }) + } +} + +func Test_toKeyValueString_equal_delimiters(t *testing.T) { + target := ottl.StandardPMapGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return map[string]any{ + "key1": "value1", + "key2": "value2", + }, nil + }, + } + delimiter := ottl.NewTestingOptional[string]("=") + pairDelimiter := ottl.NewTestingOptional[string]("=") + _, err := toKeyValueString[any](target, delimiter, pairDelimiter, ottl.NewTestingOptional[bool](false)) + assert.Error(t, err) + + delimiter = ottl.NewTestingOptional[string](" ") + _, err = toKeyValueString[any](target, delimiter, ottl.Optional[string]{}, ottl.NewTestingOptional[bool](false)) + assert.Error(t, err) +} + +func Test_toKeyValueString_bad_target(t *testing.T) { + target := ottl.StandardPMapGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return 1, nil + }, + } + delimiter := ottl.NewTestingOptional[string]("=") + pairDelimiter := ottl.NewTestingOptional[string]("!") + exprFunc, err := toKeyValueString[any](target, delimiter, pairDelimiter, ottl.NewTestingOptional[bool](false)) + assert.NoError(t, err) + _, err = exprFunc(context.Background(), nil) + assert.Error(t, err) +} + +func Test_toKeyValueString_empty_target(t *testing.T) { + target := ottl.StandardPMapGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "", nil + }, + } + delimiter := ottl.NewTestingOptional[string]("=") + pairDelimiter := ottl.NewTestingOptional[string]("!") + exprFunc, err := toKeyValueString[any](target, delimiter, pairDelimiter, ottl.NewTestingOptional[bool](false)) + assert.NoError(t, err) + _, err = exprFunc(context.Background(), nil) + assert.Error(t, err) +} + +func Test_toKeyValueString_empty_delimiters(t *testing.T) { + target := ottl.StandardPMapGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "a=b c=d", nil + }, + } + delimiter := ottl.NewTestingOptional[string]("") + + _, err := toKeyValueString[any](target, delimiter, ottl.Optional[string]{}, ottl.NewTestingOptional[bool](false)) + assert.ErrorContains(t, err, "delimiter cannot be set to an empty string") + + _, err = toKeyValueString[any](target, ottl.Optional[string]{}, delimiter, ottl.NewTestingOptional[bool](false)) + assert.ErrorContains(t, err, "pair delimiter cannot be set to an empty string") +} diff --git a/pkg/ottl/ottlfuncs/functions.go b/pkg/ottl/ottlfuncs/functions.go index 99bcd1ad3b8f..5e3aa6741cad 100644 --- a/pkg/ottl/ottlfuncs/functions.go +++ b/pkg/ottl/ottlfuncs/functions.go @@ -84,6 +84,7 @@ func converters[K any]() []ottl.Factory[K] { NewStringFactory[K](), NewSubstringFactory[K](), NewTimeFactory[K](), + NewToKeyValueStringFactory[K](), NewTruncateTimeFactory[K](), NewTraceIDFactory[K](), NewUnixFactory[K](),