From c28e23a9596660a92c7fb4961532bc6300882307 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 1 Dec 2023 10:43:23 -0800 Subject: [PATCH 01/16] Document new `mapping.omit_attributes_prefix` config option --- exporter/elasticsearchexporter/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/exporter/elasticsearchexporter/README.md b/exporter/elasticsearchexporter/README.md index caed01f44022..395141b036fc 100644 --- a/exporter/elasticsearchexporter/README.md +++ b/exporter/elasticsearchexporter/README.md @@ -76,6 +76,8 @@ This exporter supports sending OpenTelemetry logs to [Elasticsearch](https://www will reject documents that have duplicate fields. - `dedot` (default=true): When enabled attributes with `.` will be split into proper json objects. + - `omit_attributes_prefix` (default=false): Omit the `Attributes.` string prefixed to field names for + log and span attributes. - `sending_queue` - `enabled` (default = false) - `num_consumers` (default = 10): Number of consumers that dequeue batches; ignored if `enabled` is `false` From c5d79417138002718377cdb1a6d1d5075ee666f5 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 1 Dec 2023 10:43:45 -0800 Subject: [PATCH 02/16] Define new `mapping.omit_attributes_prefix` config option --- exporter/elasticsearchexporter/config.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/exporter/elasticsearchexporter/config.go b/exporter/elasticsearchexporter/config.go index 2240bf3acd2a..54ab30419ca7 100644 --- a/exporter/elasticsearchexporter/config.go +++ b/exporter/elasticsearchexporter/config.go @@ -167,6 +167,9 @@ type MappingsSettings struct { Dedup bool `mapstructure:"dedup"` Dedot bool `mapstructure:"dedot"` + + // Omit "Attributes." prefix on fields. + OmitAttributesPrefix bool `mapstructure:"omit_attributes_prefix"` } type MappingMode int From a6ff601733445657e880c73a1a7bdd9785bf259f Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 1 Dec 2023 10:45:00 -0800 Subject: [PATCH 03/16] Add ability to merge documents --- .../elasticsearchexporter/internal/objmodel/objmodel.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/exporter/elasticsearchexporter/internal/objmodel/objmodel.go b/exporter/elasticsearchexporter/internal/objmodel/objmodel.go index a26260144fa1..045bd31fe188 100644 --- a/exporter/elasticsearchexporter/internal/objmodel/objmodel.go +++ b/exporter/elasticsearchexporter/internal/objmodel/objmodel.go @@ -244,6 +244,14 @@ func (doc *Document) Serialize(w io.Writer, dedot bool) error { return doc.iterJSON(v, dedot) } +// MergeFrom merges the fields of anotherDoc into doc, overwriting any existing fields. +func (doc *Document) MergeFrom(anotherDoc Document) { + for _, field := range anotherDoc.fields { + doc.Add(field.key, field.value) + } + doc.Dedup() +} + func (doc *Document) iterJSON(v *json.Visitor, dedot bool) error { if dedot { return doc.iterJSONDedot(v) From 233e552022df636324e6bfd5b8e8d53a2d337f11 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 1 Dec 2023 10:45:35 -0800 Subject: [PATCH 04/16] Use new `mapping.omit_attributes_prefix` option while encoding --- .../elasticsearchexporter/logs_exporter.go | 6 +++++- exporter/elasticsearchexporter/model.go | 18 ++++++++++++++---- .../elasticsearchexporter/trace_exporter.go | 6 +++++- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/exporter/elasticsearchexporter/logs_exporter.go b/exporter/elasticsearchexporter/logs_exporter.go index b74a5f942ae2..b2c4410ee692 100644 --- a/exporter/elasticsearchexporter/logs_exporter.go +++ b/exporter/elasticsearchexporter/logs_exporter.go @@ -53,7 +53,11 @@ func newLogsExporter(logger *zap.Logger, cfg *Config) (*elasticsearchLogsExporte maxAttempts = cfg.Retry.MaxRequests } - model := &encodeModel{dedup: cfg.Mapping.Dedup, dedot: cfg.Mapping.Dedot} + model := &encodeModel{ + dedup: cfg.Mapping.Dedup, + dedot: cfg.Mapping.Dedot, + omitAttributesPrefix: cfg.Mapping.OmitAttributesPrefix, + } indexStr := cfg.LogsIndex if cfg.Index != "" { diff --git a/exporter/elasticsearchexporter/model.go b/exporter/elasticsearchexporter/model.go index c727de45e644..ca57e5c2ddf1 100644 --- a/exporter/elasticsearchexporter/model.go +++ b/exporter/elasticsearchexporter/model.go @@ -28,8 +28,9 @@ type mappingModel interface { // // See: https://github.com/open-telemetry/oteps/blob/master/text/logs/0097-log-data-model.md type encodeModel struct { - dedup bool - dedot bool + dedup bool + dedot bool + omitAttributesPrefix bool } const ( @@ -47,7 +48,7 @@ func (m *encodeModel) encodeLog(resource pcommon.Resource, record plog.LogRecord document.AddString("SeverityText", record.SeverityText()) document.AddInt("SeverityNumber", int64(record.SeverityNumber())) document.AddAttribute("Body", record.Body()) - document.AddAttributes("Attributes", record.Attributes()) + m.encodeAttributes(&document, record.Attributes()) document.AddAttributes("Resource", resource.Attributes()) document.AddAttributes("Scope", scopeToAttributes(scope)) @@ -74,7 +75,7 @@ func (m *encodeModel) encodeSpan(resource pcommon.Resource, span ptrace.Span, sc document.AddInt("TraceStatus", int64(span.Status().Code())) document.AddString("TraceStatusDescription", span.Status().Message()) document.AddString("Link", spanLinksToString(span.Links())) - document.AddAttributes("Attributes", span.Attributes()) + m.encodeAttributes(&document, span.Attributes()) document.AddAttributes("Resource", resource.Attributes()) document.AddEvents("Events", span.Events()) document.AddInt("Duration", durationAsMicroseconds(span.StartTimestamp().AsTime(), span.EndTimestamp().AsTime())) // unit is microseconds @@ -91,6 +92,15 @@ func (m *encodeModel) encodeSpan(resource pcommon.Resource, span ptrace.Span, sc return buf.Bytes(), err } +func (m *encodeModel) encodeAttributes(document *objmodel.Document, attributes pcommon.Map) { + if m.omitAttributesPrefix { + rawDoc := objmodel.DocumentFromAttributes(attributes) + document.MergeFrom(rawDoc) + } else { + document.AddAttributes("Attributes", attributes) + } +} + func spanLinksToString(spanLinkSlice ptrace.SpanLinkSlice) string { linkArray := make([]map[string]any, 0, spanLinkSlice.Len()) for i := 0; i < spanLinkSlice.Len(); i++ { diff --git a/exporter/elasticsearchexporter/trace_exporter.go b/exporter/elasticsearchexporter/trace_exporter.go index bfa3c485271f..0470b13c5191 100644 --- a/exporter/elasticsearchexporter/trace_exporter.go +++ b/exporter/elasticsearchexporter/trace_exporter.go @@ -49,7 +49,11 @@ func newTracesExporter(logger *zap.Logger, cfg *Config) (*elasticsearchTracesExp maxAttempts = cfg.Retry.MaxRequests } - model := &encodeModel{dedup: cfg.Mapping.Dedup, dedot: cfg.Mapping.Dedot} + model := &encodeModel{ + dedup: cfg.Mapping.Dedup, + dedot: cfg.Mapping.Dedot, + omitAttributesPrefix: cfg.Mapping.OmitAttributesPrefix, + } return &elasticsearchTracesExporter{ logger: logger, From 30ca34a597e676654a748b924e94eefe5cf2ff03 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 1 Dec 2023 12:11:05 -0800 Subject: [PATCH 05/16] Adding CHANGELOG entry --- .chloggen/exp-es-omit-attributes-prefix.yaml | 27 ++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100755 .chloggen/exp-es-omit-attributes-prefix.yaml diff --git a/.chloggen/exp-es-omit-attributes-prefix.yaml b/.chloggen/exp-es-omit-attributes-prefix.yaml new file mode 100755 index 000000000000..a8a5df9416c5 --- /dev/null +++ b/.chloggen/exp-es-omit-attributes-prefix.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: elasticsearchexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add `mapping.omit_attributes_prefix` configuration option + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [26647] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] From 5d37b8181d2089d4fc84f32ff650579b48b05627 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 1 Dec 2023 12:14:02 -0800 Subject: [PATCH 06/16] Update unit tests for config --- exporter/elasticsearchexporter/config_test.go | 14 ++++++++------ .../elasticsearchexporter/testdata/config.yaml | 2 ++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/exporter/elasticsearchexporter/config_test.go b/exporter/elasticsearchexporter/config_test.go index 91ccfe638ef8..7899d50adfee 100644 --- a/exporter/elasticsearchexporter/config_test.go +++ b/exporter/elasticsearchexporter/config_test.go @@ -134,9 +134,10 @@ func TestLoadConfig(t *testing.T) { MaxInterval: 1 * time.Minute, }, Mapping: MappingsSettings{ - Mode: "ecs", - Dedup: true, - Dedot: true, + Mode: "ecs", + Dedup: true, + Dedot: true, + OmitAttributesPrefix: false, }, LogstashFormat: LogstashFormatSettings{ Enabled: false, @@ -184,9 +185,10 @@ func TestLoadConfig(t *testing.T) { MaxInterval: 1 * time.Minute, }, Mapping: MappingsSettings{ - Mode: "ecs", - Dedup: true, - Dedot: true, + Mode: "ecs", + Dedup: true, + Dedot: true, + OmitAttributesPrefix: true, }, LogstashFormat: LogstashFormatSettings{ Enabled: false, diff --git a/exporter/elasticsearchexporter/testdata/config.yaml b/exporter/elasticsearchexporter/testdata/config.yaml index 792640222adb..42ac4171880f 100644 --- a/exporter/elasticsearchexporter/testdata/config.yaml +++ b/exporter/elasticsearchexporter/testdata/config.yaml @@ -40,6 +40,8 @@ elasticsearch/log: max_requests: 5 sending_queue: enabled: true + mapping: + omit_attributes_prefix: true elasticsearch/logstash_format: endpoints: [http://localhost:9200] logstash_format: From 51bef51d8ec0605cac9196b5fc0f60568ba8e565 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 1 Dec 2023 13:12:02 -0800 Subject: [PATCH 07/16] Simplify implementation --- .../elasticsearchexporter/internal/objmodel/objmodel.go | 8 -------- exporter/elasticsearchexporter/model.go | 6 ++++-- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/exporter/elasticsearchexporter/internal/objmodel/objmodel.go b/exporter/elasticsearchexporter/internal/objmodel/objmodel.go index 045bd31fe188..a26260144fa1 100644 --- a/exporter/elasticsearchexporter/internal/objmodel/objmodel.go +++ b/exporter/elasticsearchexporter/internal/objmodel/objmodel.go @@ -244,14 +244,6 @@ func (doc *Document) Serialize(w io.Writer, dedot bool) error { return doc.iterJSON(v, dedot) } -// MergeFrom merges the fields of anotherDoc into doc, overwriting any existing fields. -func (doc *Document) MergeFrom(anotherDoc Document) { - for _, field := range anotherDoc.fields { - doc.Add(field.key, field.value) - } - doc.Dedup() -} - func (doc *Document) iterJSON(v *json.Visitor, dedot bool) error { if dedot { return doc.iterJSONDedot(v) diff --git a/exporter/elasticsearchexporter/model.go b/exporter/elasticsearchexporter/model.go index ca57e5c2ddf1..fc0f0052e0af 100644 --- a/exporter/elasticsearchexporter/model.go +++ b/exporter/elasticsearchexporter/model.go @@ -94,8 +94,10 @@ func (m *encodeModel) encodeSpan(resource pcommon.Resource, span ptrace.Span, sc func (m *encodeModel) encodeAttributes(document *objmodel.Document, attributes pcommon.Map) { if m.omitAttributesPrefix { - rawDoc := objmodel.DocumentFromAttributes(attributes) - document.MergeFrom(rawDoc) + attributes.Range(func(k string, v pcommon.Value) bool { + document.AddAttribute(k, v) + return true + }) } else { document.AddAttributes("Attributes", attributes) } From 161ebdbd3dc62018b0bdd02728e2b5f9ab9d947f Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 1 Dec 2023 14:10:02 -0800 Subject: [PATCH 08/16] Add unit test --- exporter/elasticsearchexporter/model_test.go | 49 ++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/exporter/elasticsearchexporter/model_test.go b/exporter/elasticsearchexporter/model_test.go index e76d1f39e491..076a22ea21d5 100644 --- a/exporter/elasticsearchexporter/model_test.go +++ b/exporter/elasticsearchexporter/model_test.go @@ -8,9 +8,13 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/ptrace" semconv "go.opentelemetry.io/collector/semconv/v1.18.0" + + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/elasticsearchexporter/internal/objmodel" ) var expectedSpanBody = `{"@timestamp":"2023-04-19T03:04:05.000000006Z","Attributes.service.instance.id":"23","Duration":1000000,"EndTimestamp":"2023-04-19T03:04:06.000000006Z","Events.fooEvent.evnetMockBar":"bar","Events.fooEvent.evnetMockFoo":"foo","Events.fooEvent.time":"2023-04-19T03:04:05.000000006Z","Kind":"SPAN_KIND_CLIENT","Link":"[{\"attribute\":{},\"spanID\":\"\",\"traceID\":\"01020304050607080807060504030200\"}]","Name":"client span","Resource.cloud.platform":"aws_elastic_beanstalk","Resource.cloud.provider":"aws","Resource.deployment.environment":"BETA","Resource.service.instance.id":"23","Resource.service.name":"some-service","Resource.service.version":"env-version-1234","Scope.lib-foo":"lib-bar","Scope.name":"io.opentelemetry.rabbitmq-2.7","Scope.version":"1.30.0-alpha","SpanId":"1920212223242526","TraceId":"01020304050607080807060504030201","TraceStatus":2,"TraceStatusDescription":"Test"}` @@ -63,3 +67,48 @@ func mockResourceSpans() ptrace.Traces { event.Attributes().PutStr("evnetMockBar", "bar") return traces } + +func TestEncodeAttributes(t *testing.T) { + t.Parallel() + + attributes := pcommon.NewMap() + err := attributes.FromRaw(map[string]any{ + "s": "baz", + "o": map[string]any{ + "sub_i": 19, + }, + }) + require.NoError(t, err) + + tests := map[string]struct { + omitAttributesPrefix bool + want func() objmodel.Document + }{ + "omit": { + omitAttributesPrefix: true, + want: func() objmodel.Document { + return objmodel.DocumentFromAttributes(attributes) + }, + }, + "keep": { + omitAttributesPrefix: false, + want: func() objmodel.Document { + doc := objmodel.Document{} + doc.AddAttributes("Attributes", attributes) + return doc + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + m := encodeModel{ + omitAttributesPrefix: test.omitAttributesPrefix, + } + + doc := objmodel.Document{} + m.encodeAttributes(&doc, attributes) + require.Equal(t, test.want(), doc) + }) + } +} From 115f33282cc177a34bf9884c1b58942a66b56460 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 4 Dec 2023 16:44:29 -0800 Subject: [PATCH 09/16] Update CHANGELOG entry --- .chloggen/exp-es-omit-attributes-prefix.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.chloggen/exp-es-omit-attributes-prefix.yaml b/.chloggen/exp-es-omit-attributes-prefix.yaml index a8a5df9416c5..30a598e53047 100755 --- a/.chloggen/exp-es-omit-attributes-prefix.yaml +++ b/.chloggen/exp-es-omit-attributes-prefix.yaml @@ -7,7 +7,7 @@ change_type: enhancement component: elasticsearchexporter # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). -note: Add `mapping.omit_attributes_prefix` configuration option +note: Add `mapping.mode: raw` configuration option # Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. issues: [26647] @@ -15,7 +15,13 @@ issues: [26647] # (Optional) One or more lines of additional information to render under the primary note. # These lines will be padded with 2 spaces and then inserted directly into the document. # Use pipe (|) for multiline entries. -subtext: +subtext: | + Setting `mapping.mode: raw` in the Elasticsearch exporter's configuration + will result logs and traces being indexed into Elasticsearch with their + attribute fields directly at the root level of the document instead inside an + `Attributes` object. Similarly, this setting will also result in traces being + indexed into Elasticsearch with their event fields directly at the root level + of the document instead of inside an `Events` object. # If your change doesn't affect end users or the exported elements of any package, # you should instead start your pull request title with [chore] or use the "Skip Changelog" label. From a870aaf86c0679e15896cb925383073b8ddf5a2e Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 4 Dec 2023 16:44:51 -0800 Subject: [PATCH 10/16] Add raw mapping mode --- exporter/elasticsearchexporter/config.go | 11 +++ .../elasticsearchexporter/logs_exporter.go | 6 +- exporter/elasticsearchexporter/model.go | 27 ++++--- exporter/elasticsearchexporter/model_test.go | 77 +++++++++++++++++-- .../elasticsearchexporter/trace_exporter.go | 6 +- 5 files changed, 103 insertions(+), 24 deletions(-) diff --git a/exporter/elasticsearchexporter/config.go b/exporter/elasticsearchexporter/config.go index 54ab30419ca7..fc1a80e17535 100644 --- a/exporter/elasticsearchexporter/config.go +++ b/exporter/elasticsearchexporter/config.go @@ -178,6 +178,7 @@ type MappingMode int const ( MappingNone MappingMode = iota MappingECS + MappingRaw ) var ( @@ -191,6 +192,8 @@ func (m MappingMode) String() string { return "" case MappingECS: return "ecs" + case MappingRaw: + return "raw" default: return "" } @@ -201,6 +204,7 @@ var mappingModes = func() map[string]MappingMode { for _, m := range []MappingMode{ MappingNone, MappingECS, + MappingRaw, } { table[strings.ToLower(m.String())] = m } @@ -234,3 +238,10 @@ func (cfg *Config) Validate() error { return nil } + +// MappingMode returns the mapping.mode defined in the given cfg +// object. This method must be called after cfg.Validate() has been +// called without returning an error. +func (cfg *Config) MappingMode() MappingMode { + return mappingModes[cfg.Mapping.Mode] +} diff --git a/exporter/elasticsearchexporter/logs_exporter.go b/exporter/elasticsearchexporter/logs_exporter.go index b2c4410ee692..2c94d816418d 100644 --- a/exporter/elasticsearchexporter/logs_exporter.go +++ b/exporter/elasticsearchexporter/logs_exporter.go @@ -54,9 +54,9 @@ func newLogsExporter(logger *zap.Logger, cfg *Config) (*elasticsearchLogsExporte } model := &encodeModel{ - dedup: cfg.Mapping.Dedup, - dedot: cfg.Mapping.Dedot, - omitAttributesPrefix: cfg.Mapping.OmitAttributesPrefix, + dedup: cfg.Mapping.Dedup, + dedot: cfg.Mapping.Dedot, + mode: cfg.MappingMode(), } indexStr := cfg.LogsIndex diff --git a/exporter/elasticsearchexporter/model.go b/exporter/elasticsearchexporter/model.go index fc0f0052e0af..bbdfedcb10fc 100644 --- a/exporter/elasticsearchexporter/model.go +++ b/exporter/elasticsearchexporter/model.go @@ -28,9 +28,9 @@ type mappingModel interface { // // See: https://github.com/open-telemetry/oteps/blob/master/text/logs/0097-log-data-model.md type encodeModel struct { - dedup bool - dedot bool - omitAttributesPrefix bool + dedup bool + dedot bool + mode MappingMode } const ( @@ -77,7 +77,7 @@ func (m *encodeModel) encodeSpan(resource pcommon.Resource, span ptrace.Span, sc document.AddString("Link", spanLinksToString(span.Links())) m.encodeAttributes(&document, span.Attributes()) document.AddAttributes("Resource", resource.Attributes()) - document.AddEvents("Events", span.Events()) + m.encodeEvents(&document, span.Events()) document.AddInt("Duration", durationAsMicroseconds(span.StartTimestamp().AsTime(), span.EndTimestamp().AsTime())) // unit is microseconds document.AddAttributes("Scope", scopeToAttributes(scope)) @@ -93,14 +93,19 @@ func (m *encodeModel) encodeSpan(resource pcommon.Resource, span ptrace.Span, sc } func (m *encodeModel) encodeAttributes(document *objmodel.Document, attributes pcommon.Map) { - if m.omitAttributesPrefix { - attributes.Range(func(k string, v pcommon.Value) bool { - document.AddAttribute(k, v) - return true - }) - } else { - document.AddAttributes("Attributes", attributes) + key := "Attributes" + if m.mode == MappingRaw { + key = "" } + document.AddAttributes(key, attributes) +} + +func (m *encodeModel) encodeEvents(document *objmodel.Document, events ptrace.SpanEventSlice) { + key := "Events" + if m.mode == MappingRaw { + key = "" + } + document.AddEvents(key, events) } func spanLinksToString(spanLinkSlice ptrace.SpanLinkSlice) string { diff --git a/exporter/elasticsearchexporter/model_test.go b/exporter/elasticsearchexporter/model_test.go index 076a22ea21d5..6d6f87dccaa5 100644 --- a/exporter/elasticsearchexporter/model_test.go +++ b/exporter/elasticsearchexporter/model_test.go @@ -4,6 +4,7 @@ package elasticsearchexporter import ( + "fmt" "testing" "time" @@ -81,17 +82,25 @@ func TestEncodeAttributes(t *testing.T) { require.NoError(t, err) tests := map[string]struct { - omitAttributesPrefix bool - want func() objmodel.Document + mappingMode MappingMode + want func() objmodel.Document }{ - "omit": { - omitAttributesPrefix: true, + "raw": { + mappingMode: MappingRaw, want: func() objmodel.Document { return objmodel.DocumentFromAttributes(attributes) }, }, - "keep": { - omitAttributesPrefix: false, + "none": { + mappingMode: MappingNone, + want: func() objmodel.Document { + doc := objmodel.Document{} + doc.AddAttributes("Attributes", attributes) + return doc + }, + }, + "ecs": { + mappingMode: MappingECS, want: func() objmodel.Document { doc := objmodel.Document{} doc.AddAttributes("Attributes", attributes) @@ -103,7 +112,7 @@ func TestEncodeAttributes(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { m := encodeModel{ - omitAttributesPrefix: test.omitAttributesPrefix, + mode: test.mappingMode, } doc := objmodel.Document{} @@ -112,3 +121,57 @@ func TestEncodeAttributes(t *testing.T) { }) } } + +func TestEncodeEvents(t *testing.T) { + t.Parallel() + + events := ptrace.NewSpanEventSlice() + events.EnsureCapacity(4) + for i := 0; i < 4; i++ { + event := events.AppendEmpty() + event.SetTimestamp(pcommon.NewTimestampFromTime(time.Now().Add(time.Duration(i) * time.Minute))) + event.SetName(fmt.Sprintf("event_%d", i)) + } + + tests := map[string]struct { + mappingMode MappingMode + want func() objmodel.Document + }{ + "raw": { + mappingMode: MappingRaw, + want: func() objmodel.Document { + doc := objmodel.Document{} + doc.AddEvents("", events) + return doc + }, + }, + "none": { + mappingMode: MappingNone, + want: func() objmodel.Document { + doc := objmodel.Document{} + doc.AddEvents("Events", events) + return doc + }, + }, + "ecs": { + mappingMode: MappingECS, + want: func() objmodel.Document { + doc := objmodel.Document{} + doc.AddEvents("Events", events) + return doc + }, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + m := encodeModel{ + mode: test.mappingMode, + } + + doc := objmodel.Document{} + m.encodeEvents(&doc, events) + require.Equal(t, test.want(), doc) + }) + } +} diff --git a/exporter/elasticsearchexporter/trace_exporter.go b/exporter/elasticsearchexporter/trace_exporter.go index 0470b13c5191..0d5f0e28bc75 100644 --- a/exporter/elasticsearchexporter/trace_exporter.go +++ b/exporter/elasticsearchexporter/trace_exporter.go @@ -50,9 +50,9 @@ func newTracesExporter(logger *zap.Logger, cfg *Config) (*elasticsearchTracesExp } model := &encodeModel{ - dedup: cfg.Mapping.Dedup, - dedot: cfg.Mapping.Dedot, - omitAttributesPrefix: cfg.Mapping.OmitAttributesPrefix, + dedup: cfg.Mapping.Dedup, + dedot: cfg.Mapping.Dedot, + mode: cfg.MappingMode(), } return &elasticsearchTracesExporter{ From 0139e753d2f524387ad85e4c2a82dbc46248de68 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 4 Dec 2023 16:50:27 -0800 Subject: [PATCH 11/16] Update README --- exporter/elasticsearchexporter/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/exporter/elasticsearchexporter/README.md b/exporter/elasticsearchexporter/README.md index 395141b036fc..ff8b80f73fee 100644 --- a/exporter/elasticsearchexporter/README.md +++ b/exporter/elasticsearchexporter/README.md @@ -68,6 +68,9 @@ This exporter supports sending OpenTelemetry logs to [Elasticsearch](https://www - `ecs`: Try to map fields defined in the [OpenTelemetry Semantic Conventions](https://github.com/open-telemetry/semantic-conventions) to [Elastic Common Schema (ECS)](https://www.elastic.co/guide/en/ecs/current/index.html). + - `raw`: Omit the `Attributes.` string prefixed to field names for log and + span attributes as well as omit the `Events.` string prefixed to + field names for span events. - `fields` (optional): Configure additional fields mappings. - `file` (optional): Read additional field mappings from the provided YAML file. - `dedup` (default=true): Try to find and remove duplicate fields/attributes @@ -76,8 +79,6 @@ This exporter supports sending OpenTelemetry logs to [Elasticsearch](https://www will reject documents that have duplicate fields. - `dedot` (default=true): When enabled attributes with `.` will be split into proper json objects. - - `omit_attributes_prefix` (default=false): Omit the `Attributes.` string prefixed to field names for - log and span attributes. - `sending_queue` - `enabled` (default = false) - `num_consumers` (default = 10): Number of consumers that dequeue batches; ignored if `enabled` is `false` From d7b8b4e29e082d60f9516e333f60c300d3adb1ac Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 7 Dec 2023 03:03:42 -0800 Subject: [PATCH 12/16] Replace mapping.omit_attributes_prefix: true with mapping.mode: raw --- exporter/elasticsearchexporter/config.go | 3 --- exporter/elasticsearchexporter/config_test.go | 14 ++++++-------- .../elasticsearchexporter/testdata/config.yaml | 2 +- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/exporter/elasticsearchexporter/config.go b/exporter/elasticsearchexporter/config.go index fc1a80e17535..f05b597f4fc1 100644 --- a/exporter/elasticsearchexporter/config.go +++ b/exporter/elasticsearchexporter/config.go @@ -167,9 +167,6 @@ type MappingsSettings struct { Dedup bool `mapstructure:"dedup"` Dedot bool `mapstructure:"dedot"` - - // Omit "Attributes." prefix on fields. - OmitAttributesPrefix bool `mapstructure:"omit_attributes_prefix"` } type MappingMode int diff --git a/exporter/elasticsearchexporter/config_test.go b/exporter/elasticsearchexporter/config_test.go index 7899d50adfee..ecd50bde5a6c 100644 --- a/exporter/elasticsearchexporter/config_test.go +++ b/exporter/elasticsearchexporter/config_test.go @@ -134,10 +134,9 @@ func TestLoadConfig(t *testing.T) { MaxInterval: 1 * time.Minute, }, Mapping: MappingsSettings{ - Mode: "ecs", - Dedup: true, - Dedot: true, - OmitAttributesPrefix: false, + Mode: "ecs", + Dedup: true, + Dedot: true, }, LogstashFormat: LogstashFormatSettings{ Enabled: false, @@ -185,10 +184,9 @@ func TestLoadConfig(t *testing.T) { MaxInterval: 1 * time.Minute, }, Mapping: MappingsSettings{ - Mode: "ecs", - Dedup: true, - Dedot: true, - OmitAttributesPrefix: true, + Mode: "raw", + Dedup: true, + Dedot: true, }, LogstashFormat: LogstashFormatSettings{ Enabled: false, diff --git a/exporter/elasticsearchexporter/testdata/config.yaml b/exporter/elasticsearchexporter/testdata/config.yaml index 42ac4171880f..068519b94b22 100644 --- a/exporter/elasticsearchexporter/testdata/config.yaml +++ b/exporter/elasticsearchexporter/testdata/config.yaml @@ -41,7 +41,7 @@ elasticsearch/log: sending_queue: enabled: true mapping: - omit_attributes_prefix: true + mode: raw elasticsearch/logstash_format: endpoints: [http://localhost:9200] logstash_format: From 13ed3dbcbbaffb749ebed42f132d6634742b2dac Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 18 Dec 2023 13:50:20 -0800 Subject: [PATCH 13/16] Adding separate test case for format: raw --- exporter/elasticsearchexporter/config_test.go | 11 ++++++++++- exporter/elasticsearchexporter/testdata/config.yaml | 6 ++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/exporter/elasticsearchexporter/config_test.go b/exporter/elasticsearchexporter/config_test.go index ecd50bde5a6c..91022d71b36d 100644 --- a/exporter/elasticsearchexporter/config_test.go +++ b/exporter/elasticsearchexporter/config_test.go @@ -85,6 +85,10 @@ func TestLoadConfig(t *testing.T) { defaultLogstashFormatCfg.(*Config).Endpoints = []string{"http://localhost:9200"} defaultLogstashFormatCfg.(*Config).LogstashFormat.Enabled = true + defaultRawCfg := createDefaultConfig() + defaultRawCfg.(*Config).Endpoints = []string{"http://localhost:9200"} + defaultRawCfg.(*Config).Mapping.Mode = "raw" + tests := []struct { configFile string id component.ID @@ -184,7 +188,7 @@ func TestLoadConfig(t *testing.T) { MaxInterval: 1 * time.Minute, }, Mapping: MappingsSettings{ - Mode: "raw", + Mode: "ecs", Dedup: true, Dedot: true, }, @@ -200,6 +204,11 @@ func TestLoadConfig(t *testing.T) { configFile: "config.yaml", expected: defaultLogstashFormatCfg, }, + { + id: component.NewIDWithName(metadata.Type, "raw"), + configFile: "config.yaml", + expected: defaultRawCfg, + }, } for _, tt := range tests { diff --git a/exporter/elasticsearchexporter/testdata/config.yaml b/exporter/elasticsearchexporter/testdata/config.yaml index 068519b94b22..3054df30355c 100644 --- a/exporter/elasticsearchexporter/testdata/config.yaml +++ b/exporter/elasticsearchexporter/testdata/config.yaml @@ -40,9 +40,11 @@ elasticsearch/log: max_requests: 5 sending_queue: enabled: true - mapping: - mode: raw elasticsearch/logstash_format: endpoints: [http://localhost:9200] logstash_format: enabled: true +elasticsearch/raw: + endpoints: [http://localhost:9200] + mapping: + mode: raw From a6f614610f2e15d238b8bba24f99554bdb361658 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 16 Jan 2024 09:09:20 -0800 Subject: [PATCH 14/16] Escape text in CHANGELOG entry --- .chloggen/exp-es-omit-attributes-prefix.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.chloggen/exp-es-omit-attributes-prefix.yaml b/.chloggen/exp-es-omit-attributes-prefix.yaml index 30a598e53047..24d646b3d8a1 100755 --- a/.chloggen/exp-es-omit-attributes-prefix.yaml +++ b/.chloggen/exp-es-omit-attributes-prefix.yaml @@ -7,7 +7,7 @@ change_type: enhancement component: elasticsearchexporter # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). -note: Add `mapping.mode: raw` configuration option +note: "Add `mapping.mode: raw` configuration option" # Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. issues: [26647] From c9f6fd5a227d437cbbb77b0f1045745fc55b258c Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Wed, 17 Jan 2024 09:10:52 +0100 Subject: [PATCH 15/16] make gci --- exporter/elasticsearchexporter/model_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/exporter/elasticsearchexporter/model_test.go b/exporter/elasticsearchexporter/model_test.go index 6d6f87dccaa5..dc318df6e768 100644 --- a/exporter/elasticsearchexporter/model_test.go +++ b/exporter/elasticsearchexporter/model_test.go @@ -10,7 +10,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/ptrace" semconv "go.opentelemetry.io/collector/semconv/v1.18.0" From 1ec5255d180f7414241e0027d08f0ec94d3a9104 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 17 Jan 2024 14:57:17 -0800 Subject: [PATCH 16/16] Fix unit tests --- exporter/elasticsearchexporter/logs_exporter_test.go | 2 +- exporter/elasticsearchexporter/traces_exporter_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/exporter/elasticsearchexporter/logs_exporter_test.go b/exporter/elasticsearchexporter/logs_exporter_test.go index 705fbaf08924..cf83b736604a 100644 --- a/exporter/elasticsearchexporter/logs_exporter_test.go +++ b/exporter/elasticsearchexporter/logs_exporter_test.go @@ -122,7 +122,7 @@ func TestExporter_New(t *testing.T) { cfg.Mapping.Dedot = false cfg.Mapping.Dedup = true }), - want: successWithInternalModel(&encodeModel{dedot: false, dedup: true}), + want: successWithInternalModel(&encodeModel{dedot: false, dedup: true, mode: MappingECS}), }, } diff --git a/exporter/elasticsearchexporter/traces_exporter_test.go b/exporter/elasticsearchexporter/traces_exporter_test.go index f3722bfbe5cb..c42e5d58f6b0 100644 --- a/exporter/elasticsearchexporter/traces_exporter_test.go +++ b/exporter/elasticsearchexporter/traces_exporter_test.go @@ -99,7 +99,7 @@ func TestTracesExporter_New(t *testing.T) { cfg.Mapping.Dedot = false cfg.Mapping.Dedup = true }), - want: successWithInternalModel(&encodeModel{dedot: false, dedup: true}), + want: successWithInternalModel(&encodeModel{dedot: false, dedup: true, mode: MappingECS}), }, }