From 57af894d367744be7f3423aaa435d97d228ecf36 Mon Sep 17 00:00:00 2001 From: Brian Derr Date: Tue, 10 Dec 2024 15:53:10 -0800 Subject: [PATCH] [exporter/splunkhecexporter] Add `OtelAttrsToHec` struct and related config fields (#35476) **Description:** Adding a new config fields `otel_attrs_to_hec_metadata/*` to replace the `hec_metadata_to_otel_attrs/*` config fields. **Link to tracking Issue:** fixes #35092 **Testing:** No new tests added, modified existing tests. **Documentation:** Adding field-level description to exporter README.md. --- .chloggen/iss-35092.yaml | 28 +++++ exporter/splunkhecexporter/README.md | 8 ++ exporter/splunkhecexporter/config.go | 5 + exporter/splunkhecexporter/config_test.go | 6 + exporter/splunkhecexporter/factory.go | 6 + .../splunkhecexporter/logdata_to_splunk.go | 39 +++++- .../logdata_to_splunk_test.go | 116 ++++++++++++++++-- .../splunkhecexporter/testdata/config.yaml | 5 + internal/splunk/common.go | 10 ++ 9 files changed, 208 insertions(+), 15 deletions(-) create mode 100644 .chloggen/iss-35092.yaml diff --git a/.chloggen/iss-35092.yaml b/.chloggen/iss-35092.yaml new file mode 100644 index 000000000000..79fddc3f42dd --- /dev/null +++ b/.chloggen/iss-35092.yaml @@ -0,0 +1,28 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: deprecation + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: splunkhecexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add `otel_attrs_to_hec_metadata/*` config fields to replace `hec_metadata_to_otel_attrs/*` fields. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [35092] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: | + `otel_attrs_to_hec_metadata/*` config fields will replace the `hec_metadata_to_otel_attrs/*` fields in a later release. + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/exporter/splunkhecexporter/README.md b/exporter/splunkhecexporter/README.md index 5db67fce7696..29a25f8ac7fe 100644 --- a/exporter/splunkhecexporter/README.md +++ b/exporter/splunkhecexporter/README.md @@ -54,10 +54,18 @@ The following configuration options can also be configured: - `health_path` (default = '/services/collector/health'): The path reporting [health checks](https://docs.splunk.com/Documentation/Splunk/9.0.1/RESTREF/RESTinput#services.2Fcollector.2Fhealth). - `health_check_enabled` (default = false): Whether to perform Splunk HEC Health Check during the exporter's startup. - `export_raw` (default = false): send only the log's body, targeting a Splunk HEC raw endpoint. +- `otel_attrs_to_hec_metadata/source` (default = 'com.splunk.source'): Specifies the mapping of a specific unified model attribute value to the standard source field of a HEC event. +- `otel_attrs_to_hec_metadata/sourcetype` (default = 'com.splunk.sourcetype'): Specifies the mapping of a specific unified model attribute value to the standard sourcetype field of a HEC event. +- `otel_attrs_to_hec_metadata/index` (default = 'com.splunk.index'): Specifies the mapping of a specific unified model attribute value to the standard index field of a HEC event. +- `otel_attrs_to_hec_metadata/host` (default = 'host.name'): Specifies the mapping of a specific unified model attribute value to the standard host field and the `host.name` field of a HEC event. - `hec_metadata_to_otel_attrs/source` (default = 'com.splunk.source'): Specifies the mapping of a specific unified model attribute value to the standard source field of a HEC event. + **Deprecated** (v0.116.0): prefer `otel_attrs_to_hec_metadata/source`. - `hec_metadata_to_otel_attrs/sourcetype` (default = 'com.splunk.sourcetype'): Specifies the mapping of a specific unified model attribute value to the standard sourcetype field of a HEC event. + **Deprecated** (v0.116.0): prefer `otel_attrs_to_hec_metadata/sourcetype`. - `hec_metadata_to_otel_attrs/index` (default = 'com.splunk.index'): Specifies the mapping of a specific unified model attribute value to the standard index field of a HEC event. + **Deprecated** (v0.116.0): prefer `otel_attrs_to_hec_metadata/index`. - `hec_metadata_to_otel_attrs/host` (default = 'host.name'): Specifies the mapping of a specific unified model attribute value to the standard host field and the `host.name` field of a HEC event. + **Deprecated** (v0.116.0): prefer `otel_attrs_to_hec_metadata/host`. - `otel_to_hec_fields/severity_text` (default = `otel.log.severity.text`): Specifies the name of the field to map the severity text field of log events. - `otel_to_hec_fields/severity_number` (default = `otel.log.severity.number`): Specifies the name of the field to map the severity number field of log events. - `otel_to_hec_fields/name` (default = `"otel.log.name`): Specifies the name of the field to map the name field of log events. diff --git a/exporter/splunkhecexporter/config.go b/exporter/splunkhecexporter/config.go index 0d8c3e4a3e0f..9e435af573fb 100644 --- a/exporter/splunkhecexporter/config.go +++ b/exporter/splunkhecexporter/config.go @@ -116,7 +116,12 @@ type Config struct { // App version is used to track telemetry information for Splunk App's using HEC by App version. Defaults to the current OpenTelemetry Collector Contrib build version. SplunkAppVersion string `mapstructure:"splunk_app_version"` + + // OtelAttrsToHec creates a mapping from attributes to HEC specific metadata: source, sourcetype, index and host. + OtelAttrsToHec splunk.HecToOtelAttrs `mapstructure:"otel_attrs_to_hec_metadata"` + // HecToOtelAttrs creates a mapping from attributes to HEC specific metadata: source, sourcetype, index and host. + // Deprecated: [v0.113.0] Use OtelAttrsToHec instead. HecToOtelAttrs splunk.HecToOtelAttrs `mapstructure:"hec_metadata_to_otel_attrs"` // HecFields creates a mapping from attributes to HEC fields. HecFields OtelToHecFields `mapstructure:"otel_to_hec_fields"` diff --git a/exporter/splunkhecexporter/config_test.go b/exporter/splunkhecexporter/config_test.go index a8290bd7eab6..a9a2959f4a00 100644 --- a/exporter/splunkhecexporter/config_test.go +++ b/exporter/splunkhecexporter/config_test.go @@ -102,6 +102,12 @@ func TestLoadConfig(t *testing.T) { MaxSizeItems: 10, }, }, + OtelAttrsToHec: splunk.HecToOtelAttrs{ + Source: "mysource", + SourceType: "mysourcetype", + Index: "myindex", + Host: "myhost", + }, HecToOtelAttrs: splunk.HecToOtelAttrs{ Source: "mysource", SourceType: "mysourcetype", diff --git a/exporter/splunkhecexporter/factory.go b/exporter/splunkhecexporter/factory.go index a5d82775b28e..38f8d570d147 100644 --- a/exporter/splunkhecexporter/factory.go +++ b/exporter/splunkhecexporter/factory.go @@ -86,6 +86,12 @@ func createDefaultConfig() component.Config { MaxContentLengthMetrics: defaultContentLengthMetricsLimit, MaxContentLengthTraces: defaultContentLengthTracesLimit, MaxEventSize: defaultMaxEventSize, + OtelAttrsToHec: splunk.HecToOtelAttrs{ + Source: splunk.DefaultSourceLabel, + SourceType: splunk.DefaultSourceTypeLabel, + Index: splunk.DefaultIndexLabel, + Host: conventions.AttributeHostName, + }, HecToOtelAttrs: splunk.HecToOtelAttrs{ Source: splunk.DefaultSourceLabel, SourceType: splunk.DefaultSourceTypeLabel, diff --git a/exporter/splunkhecexporter/logdata_to_splunk.go b/exporter/splunkhecexporter/logdata_to_splunk.go index a89ff89ec8f6..96ad419a2a5b 100644 --- a/exporter/splunkhecexporter/logdata_to_splunk.go +++ b/exporter/splunkhecexporter/logdata_to_splunk.go @@ -23,6 +23,33 @@ const ( traceIDFieldKey = "trace_id" ) +// copyOtelAttrs copies values from HecToOtelAttrs to OtelAttrsToHec struct. +func copyOtelAttrs(config *Config) { + defaultCfg := createDefaultConfig().(*Config) + if config.OtelAttrsToHec.Equal(defaultCfg.OtelAttrsToHec) { + if !config.HecToOtelAttrs.Equal(defaultCfg.HecToOtelAttrs) { + // Copy settings to ease deprecation of HecToOtelAttrs. + config.OtelAttrsToHec = config.HecToOtelAttrs + } + } else { + if !config.HecToOtelAttrs.Equal(defaultCfg.HecToOtelAttrs) { + // Replace all default fields in OtelAttrsToHec. + if config.OtelAttrsToHec.Source == defaultCfg.OtelAttrsToHec.Source { + config.OtelAttrsToHec.Source = config.HecToOtelAttrs.Source + } + if config.OtelAttrsToHec.SourceType == defaultCfg.OtelAttrsToHec.SourceType { + config.OtelAttrsToHec.SourceType = config.HecToOtelAttrs.SourceType + } + if config.OtelAttrsToHec.Index == defaultCfg.OtelAttrsToHec.Index { + config.OtelAttrsToHec.Index = config.HecToOtelAttrs.Index + } + if config.OtelAttrsToHec.Host == defaultCfg.OtelAttrsToHec.Host { + config.OtelAttrsToHec.Host = config.HecToOtelAttrs.Host + } + } + } +} + func mapLogRecordToSplunkEvent(res pcommon.Resource, lr plog.LogRecord, config *Config) *splunk.Event { body := lr.Body().AsRaw() if body == nil || body == "" { @@ -30,15 +57,19 @@ func mapLogRecordToSplunkEvent(res pcommon.Resource, lr plog.LogRecord, config * return nil } + // Manage the deprecation of HecToOtelAttrs config parameters. + // TODO: remove this once HecToOtelAttrs is removed from Config. + copyOtelAttrs(config) + host := unknownHostName source := config.Source sourcetype := config.SourceType index := config.Index fields := map[string]any{} - sourceKey := config.HecToOtelAttrs.Source - sourceTypeKey := config.HecToOtelAttrs.SourceType - indexKey := config.HecToOtelAttrs.Index - hostKey := config.HecToOtelAttrs.Host + sourceKey := config.OtelAttrsToHec.Source + sourceTypeKey := config.OtelAttrsToHec.SourceType + indexKey := config.OtelAttrsToHec.Index + hostKey := config.OtelAttrsToHec.Host severityTextKey := config.HecFields.SeverityText severityNumberKey := config.HecFields.SeverityNumber if spanID := lr.SpanID(); !spanID.IsEmpty() { diff --git a/exporter/splunkhecexporter/logdata_to_splunk_test.go b/exporter/splunkhecexporter/logdata_to_splunk_test.go index 6c4303f48c83..a8bd780c4fbb 100644 --- a/exporter/splunkhecexporter/logdata_to_splunk_test.go +++ b/exporter/splunkhecexporter/logdata_to_splunk_test.go @@ -14,6 +14,100 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/internal/splunk" ) +func Test_copyOtelAttrs(t *testing.T) { + tests := []struct { + name string + configDataFn func() *Config + wantConfigDataFn func() *Config + }{ + { + name: "defaults", + configDataFn: func() *Config { + return createDefaultConfig().(*Config) + }, + wantConfigDataFn: func() *Config { + return createDefaultConfig().(*Config) + }, + }, + { + name: "override hec_metadata_to_otel_attrs", + configDataFn: func() *Config { + cfg := createDefaultConfig().(*Config) + + cfg.HecToOtelAttrs.Index = "testIndex" + cfg.HecToOtelAttrs.Source = "testSource" + cfg.HecToOtelAttrs.SourceType = "testSourceType" + cfg.HecToOtelAttrs.Host = "testHost" + + return cfg + }, + wantConfigDataFn: func() *Config { + cfg := createDefaultConfig().(*Config) + + cfg.HecToOtelAttrs.Index = "testIndex" + cfg.HecToOtelAttrs.Source = "testSource" + cfg.HecToOtelAttrs.SourceType = "testSourceType" + cfg.HecToOtelAttrs.Host = "testHost" + + cfg.OtelAttrsToHec.Index = "testIndex" + cfg.OtelAttrsToHec.Source = "testSource" + cfg.OtelAttrsToHec.SourceType = "testSourceType" + cfg.OtelAttrsToHec.Host = "testHost" + + return cfg + }, + }, + { + name: "partial otel_attrs_to_hec_metadata", + configDataFn: func() *Config { + cfg := createDefaultConfig().(*Config) + + cfg.OtelAttrsToHec.Source = "testSource" + cfg.OtelAttrsToHec.Index = "testIndex" + + return cfg + }, + wantConfigDataFn: func() *Config { + cfg := createDefaultConfig().(*Config) + + cfg.OtelAttrsToHec.Source = "testSource" + cfg.OtelAttrsToHec.Index = "testIndex" + + return cfg + }, + }, + { + name: "prefer otel_attrs_to_hec_metadata", + configDataFn: func() *Config { + cfg := createDefaultConfig().(*Config) + + cfg.HecToOtelAttrs.Index = "hecIndex" + + cfg.OtelAttrsToHec.Index = "otelIndex" + + return cfg + }, + wantConfigDataFn: func() *Config { + cfg := createDefaultConfig().(*Config) + + cfg.HecToOtelAttrs.Index = "hecIndex" + + cfg.OtelAttrsToHec.Index = "otelIndex" + + return cfg + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := tt.configDataFn() + copyOtelAttrs(cfg) + assert.Equal(t, tt.wantConfigDataFn(), cfg) + }) + } +} + func Test_mapLogRecordToSplunkEvent(t *testing.T) { ts := pcommon.Timestamp(123) @@ -153,18 +247,18 @@ func Test_mapLogRecordToSplunkEvent(t *testing.T) { }, logResourceFn: pcommon.NewResource, configDataFn: func() *Config { - return &Config{ - HecToOtelAttrs: splunk.HecToOtelAttrs{ - Source: "mysource", - SourceType: "mysourcetype", - Index: "myindex", - Host: "myhost", - }, - HecFields: OtelToHecFields{ - SeverityNumber: "myseveritynum", - SeverityText: "myseverity", - }, + config := createDefaultConfig().(*Config) + config.HecToOtelAttrs = splunk.HecToOtelAttrs{ + Source: "mysource", + SourceType: "mysourcetype", + Index: "myindex", + Host: "myhost", + } + config.HecFields = OtelToHecFields{ + SeverityNumber: "myseveritynum", + SeverityText: "myseverity", } + return config }, wantSplunkEvents: []*splunk.Event{ func() *splunk.Event { diff --git a/exporter/splunkhecexporter/testdata/config.yaml b/exporter/splunkhecexporter/testdata/config.yaml index 54b7040dbf8d..5e6afc0e058b 100644 --- a/exporter/splunkhecexporter/testdata/config.yaml +++ b/exporter/splunkhecexporter/testdata/config.yaml @@ -33,6 +33,11 @@ splunk_hec/allsettings: max_size_items: 10 splunk_app_name: "OpenTelemetry-Collector Splunk Exporter" splunk_app_version: "v0.0.1" + otel_attrs_to_hec_metadata: + source: "mysource" + sourcetype: "mysourcetype" + index: "myindex" + host: "myhost" hec_metadata_to_otel_attrs: source: "mysource" sourcetype: "mysourcetype" diff --git a/internal/splunk/common.go b/internal/splunk/common.go index c7cc3c2ecb40..e57ecaf53e15 100644 --- a/internal/splunk/common.go +++ b/internal/splunk/common.go @@ -139,6 +139,16 @@ type HecToOtelAttrs struct { Host string `mapstructure:"host"` } +func (h HecToOtelAttrs) Equal(o HecToOtelAttrs) bool { + if h.Host != o.Host || + h.Source != o.Source || + h.SourceType != o.SourceType || + h.Index != o.Index { + return false + } + return true +} + type AckRequest struct { Acks []uint64 `json:"acks"` }