From d3f6bd12f7cdb8ee972b8f15e21d5a7eba8d5fbd Mon Sep 17 00:00:00 2001 From: hgaol Date: Wed, 11 Dec 2024 22:49:06 +0800 Subject: [PATCH] fix nit issues and update README --- .chloggen/36650.yaml | 2 +- pkg/translator/azure/resourcelogs_to_logs.go | 16 +++++++------- .../azure/resourcelogs_to_logs_test.go | 22 +++++++++---------- pkg/translator/azure/resources_to_traces.go | 8 +++---- .../azurelogs/resourcelogs_to_logs.go | 16 +++++++------- .../azurelogs/resourcelogs_to_logs_test.go | 22 +++++++++---------- receiver/azureeventhubreceiver/README.md | 10 ++------- .../azureresourcelogs_unmarshaler.go | 12 +++++----- .../azureresourcetraces_unmarshaler.go | 6 ++--- receiver/azureeventhubreceiver/config.go | 2 +- receiver/azureeventhubreceiver/factory.go | 6 ++--- 11 files changed, 58 insertions(+), 64 deletions(-) diff --git a/.chloggen/36650.yaml b/.chloggen/36650.yaml index 5d9f53e4c088..7fdb5eab0c99 100644 --- a/.chloggen/36650.yaml +++ b/.chloggen/36650.yaml @@ -7,7 +7,7 @@ change_type: enhancement component: azureeventhubreceiver # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). -note: support more time format +note: support providing one or more time formats for timestamp parsing # Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. issues: [36650] diff --git a/pkg/translator/azure/resourcelogs_to_logs.go b/pkg/translator/azure/resourcelogs_to_logs.go index 008c30fc59e9..5b608570612a 100644 --- a/pkg/translator/azure/resourcelogs_to_logs.go +++ b/pkg/translator/azure/resourcelogs_to_logs.go @@ -72,9 +72,9 @@ type azureLogRecord struct { var _ plog.Unmarshaler = (*ResourceLogsUnmarshaler)(nil) type ResourceLogsUnmarshaler struct { - Version string - Logger *zap.Logger - TimeFormat []string + Version string + Logger *zap.Logger + TimeFormats []string } func (r ResourceLogsUnmarshaler) UnmarshalLogs(buf []byte) (plog.Logs, error) { @@ -107,7 +107,7 @@ func (r ResourceLogsUnmarshaler) UnmarshalLogs(buf []byte) (plog.Logs, error) { for i := 0; i < len(logs); i++ { log := logs[i] - nanos, err := getTimestamp(log, r.TimeFormat) + nanos, err := getTimestamp(log, r.TimeFormats...) if err != nil { r.Logger.Warn("Unable to convert timestamp from log", zap.String("timestamp", log.Time)) continue @@ -131,11 +131,11 @@ func (r ResourceLogsUnmarshaler) UnmarshalLogs(buf []byte) (plog.Logs, error) { return l, nil } -func getTimestamp(record azureLogRecord, format []string) (pcommon.Timestamp, error) { +func getTimestamp(record azureLogRecord, formats ...string) (pcommon.Timestamp, error) { if record.Time != "" { - return asTimestamp(record.Time, format) + return asTimestamp(record.Time, formats...) } else if record.Timestamp != "" { - return asTimestamp(record.Timestamp, format) + return asTimestamp(record.Timestamp, formats...) } return 0, errMissingTimestamp @@ -144,7 +144,7 @@ func getTimestamp(record azureLogRecord, format []string) (pcommon.Timestamp, er // asTimestamp will parse an ISO8601 string into an OpenTelemetry // nanosecond timestamp. If the string cannot be parsed, it will // return zero and the error. -func asTimestamp(s string, formats []string) (pcommon.Timestamp, error) { +func asTimestamp(s string, formats ...string) (pcommon.Timestamp, error) { var err error var t time.Time // Try parsing with provided formats first diff --git a/pkg/translator/azure/resourcelogs_to_logs_test.go b/pkg/translator/azure/resourcelogs_to_logs_test.go index b30e41fb1247..83a9540bce2a 100644 --- a/pkg/translator/azure/resourcelogs_to_logs_test.go +++ b/pkg/translator/azure/resourcelogs_to_logs_test.go @@ -27,7 +27,7 @@ var testBuildInfo = component.BuildInfo{ var minimumLogRecord = func() plog.LogRecord { lr := plog.NewLogs().ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - ts, _ := asTimestamp("2022-11-11T04:48:27.6767145Z", nil) + ts, _ := asTimestamp("2022-11-11T04:48:27.6767145Z") lr.SetTimestamp(ts) lr.Attributes().PutStr(azureOperationName, "SecretGet") lr.Attributes().PutStr(azureCategory, "AuditEvent") @@ -38,7 +38,7 @@ var minimumLogRecord = func() plog.LogRecord { var maximumLogRecord1 = func() plog.LogRecord { lr := plog.NewLogs().ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - ts, _ := asTimestamp("2022-11-11T04:48:27.6767145Z", nil) + ts, _ := asTimestamp("2022-11-11T04:48:27.6767145Z") lr.SetTimestamp(ts) lr.SetSeverityNumber(plog.SeverityNumberWarn) lr.SetSeverityText("Warning") @@ -72,7 +72,7 @@ var maximumLogRecord2 = func() []plog.LogRecord { lr := sl.LogRecords().AppendEmpty() lr2 := sl.LogRecords().AppendEmpty() - ts, _ := asTimestamp("2022-11-11T04:48:29.6767145Z", nil) + ts, _ := asTimestamp("2022-11-11T04:48:29.6767145Z") lr.SetTimestamp(ts) lr.SetSeverityNumber(plog.SeverityNumberWarn) lr.SetSeverityText("Warning") @@ -98,7 +98,7 @@ var maximumLogRecord2 = func() []plog.LogRecord { m.PutDouble("float", 41.3) m.PutBool("bool", true) - ts, _ = asTimestamp("2022-11-11T04:48:31.6767145Z", nil) + ts, _ = asTimestamp("2022-11-11T04:48:31.6767145Z") lr2.SetTimestamp(ts) lr2.SetSeverityNumber(plog.SeverityNumberWarn) lr2.SetSeverityText("Warning") @@ -131,7 +131,7 @@ var maximumLogRecord2 = func() []plog.LogRecord { var badLevelLogRecord = func() plog.LogRecord { lr := plog.NewLogs().ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - ts, _ := asTimestamp("2023-10-26T14:22:43.3416357Z", nil) + ts, _ := asTimestamp("2023-10-26T14:22:43.3416357Z") lr.SetTimestamp(ts) lr.SetSeverityNumber(plog.SeverityNumberTrace4) lr.SetSeverityText("4") @@ -171,7 +171,7 @@ var badLevelLogRecord = func() plog.LogRecord { var badTimeLogRecord = func() plog.LogRecord { lr := plog.NewLogs().ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - ts, _ := asTimestamp("2021-10-14T22:17:11+00:00", nil) + ts, _ := asTimestamp("2021-10-14T22:17:11+00:00") lr.SetTimestamp(ts) lr.Attributes().PutStr(azureOperationName, "ApplicationGatewayAccess") @@ -213,29 +213,29 @@ var badTimeLogRecord = func() plog.LogRecord { func TestAsTimestamp(t *testing.T) { timestamp := "2022-11-11T04:48:27.6767145Z" - nanos, err := asTimestamp(timestamp, nil) + nanos, err := asTimestamp(timestamp) assert.NoError(t, err) assert.Less(t, pcommon.Timestamp(0), nanos) timestamp = "11/20/2024 13:57:18" - nanos, err = asTimestamp(timestamp, []string{"01/02/2006 15:04:05"}) + nanos, err = asTimestamp(timestamp, "01/02/2006 15:04:05") assert.NoError(t, err) assert.Less(t, pcommon.Timestamp(0), nanos) // time_format set, but fallback to iso8601 and succeeded to parse timestamp = "2022-11-11T04:48:27.6767145Z" - nanos, err = asTimestamp(timestamp, []string{"01/02/2006 15:04:05"}) + nanos, err = asTimestamp(timestamp, "01/02/2006 15:04:05") assert.NoError(t, err) assert.Less(t, pcommon.Timestamp(0), nanos) // time_format set, but all failed to parse timestamp = "11/20/2024 13:57:18" - nanos, err = asTimestamp(timestamp, []string{"2006-01-02 15:04:05"}) + nanos, err = asTimestamp(timestamp, "2006-01-02 15:04:05") assert.Error(t, err) assert.Equal(t, pcommon.Timestamp(0), nanos) timestamp = "invalid-time" - nanos, err = asTimestamp(timestamp, nil) + nanos, err = asTimestamp(timestamp, nil...) assert.Error(t, err) assert.Equal(t, pcommon.Timestamp(0), nanos) } diff --git a/pkg/translator/azure/resources_to_traces.go b/pkg/translator/azure/resources_to_traces.go index 9f8feb240722..2f92d013248c 100644 --- a/pkg/translator/azure/resources_to_traces.go +++ b/pkg/translator/azure/resources_to_traces.go @@ -62,9 +62,9 @@ type azureTracesRecord struct { var _ ptrace.Unmarshaler = (*TracesUnmarshaler)(nil) type TracesUnmarshaler struct { - Version string - Logger *zap.Logger - TimeFormat []string + Version string + Logger *zap.Logger + TimeFormats []string } func (r TracesUnmarshaler) UnmarshalTraces(buf []byte) (ptrace.Traces, error) { @@ -96,7 +96,7 @@ func (r TracesUnmarshaler) UnmarshalTraces(buf []byte) (ptrace.Traces, error) { resource.Attributes().PutStr("service.name", azureTrace.AppRoleName) - nanos, err := asTimestamp(azureTrace.Time, r.TimeFormat) + nanos, err := asTimestamp(azureTrace.Time, r.TimeFormats...) if err != nil { r.Logger.Warn("Invalid Timestamp", zap.String("time", azureTrace.Time)) continue diff --git a/pkg/translator/azurelogs/resourcelogs_to_logs.go b/pkg/translator/azurelogs/resourcelogs_to_logs.go index 4e45d5c6500e..437df4096163 100644 --- a/pkg/translator/azurelogs/resourcelogs_to_logs.go +++ b/pkg/translator/azurelogs/resourcelogs_to_logs.go @@ -76,9 +76,9 @@ type azureLogRecord struct { var _ plog.Unmarshaler = (*ResourceLogsUnmarshaler)(nil) type ResourceLogsUnmarshaler struct { - Version string - Logger *zap.Logger - TimeFormat []string + Version string + Logger *zap.Logger + TimeFormats []string } func (r ResourceLogsUnmarshaler) UnmarshalLogs(buf []byte) (plog.Logs, error) { @@ -110,7 +110,7 @@ func (r ResourceLogsUnmarshaler) UnmarshalLogs(buf []byte) (plog.Logs, error) { for i := 0; i < len(logs); i++ { log := logs[i] - nanos, err := getTimestamp(log, r.TimeFormat) + nanos, err := getTimestamp(log, r.TimeFormats...) if err != nil { r.Logger.Warn("Unable to convert timestamp from log", zap.String("timestamp", log.Time)) continue @@ -138,11 +138,11 @@ func (r ResourceLogsUnmarshaler) UnmarshalLogs(buf []byte) (plog.Logs, error) { return l, nil } -func getTimestamp(record azureLogRecord, formats []string) (pcommon.Timestamp, error) { +func getTimestamp(record azureLogRecord, formats ...string) (pcommon.Timestamp, error) { if record.Time != "" { - return asTimestamp(record.Time, formats) + return asTimestamp(record.Time, formats...) } else if record.Timestamp != "" { - return asTimestamp(record.Timestamp, formats) + return asTimestamp(record.Timestamp, formats...) } return 0, errMissingTimestamp @@ -151,7 +151,7 @@ func getTimestamp(record azureLogRecord, formats []string) (pcommon.Timestamp, e // asTimestamp will parse an ISO8601 string into an OpenTelemetry // nanosecond timestamp. If the string cannot be parsed, it will // return zero and the error. -func asTimestamp(s string, formats []string) (pcommon.Timestamp, error) { +func asTimestamp(s string, formats ...string) (pcommon.Timestamp, error) { var err error var t time.Time // Try parsing with provided formats first diff --git a/pkg/translator/azurelogs/resourcelogs_to_logs_test.go b/pkg/translator/azurelogs/resourcelogs_to_logs_test.go index 6da186cb81b1..9e0fbcf05f28 100644 --- a/pkg/translator/azurelogs/resourcelogs_to_logs_test.go +++ b/pkg/translator/azurelogs/resourcelogs_to_logs_test.go @@ -27,7 +27,7 @@ var testBuildInfo = component.BuildInfo{ var minimumLogRecord = func() plog.LogRecord { lr := plog.NewLogs().ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - ts, _ := asTimestamp("2022-11-11T04:48:27.6767145Z", nil) + ts, _ := asTimestamp("2022-11-11T04:48:27.6767145Z") lr.SetTimestamp(ts) lr.Attributes().PutStr(conventions.AttributeCloudProvider, conventions.AttributeCloudProviderAzure) lr.Attributes().PutStr(conventions.AttributeCloudResourceID, "/RESOURCE_ID") @@ -44,7 +44,7 @@ var minimumLogRecord = func() plog.LogRecord { var maximumLogRecord1 = func() plog.LogRecord { lr := plog.NewLogs().ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - ts, _ := asTimestamp("2022-11-11T04:48:27.6767145Z", nil) + ts, _ := asTimestamp("2022-11-11T04:48:27.6767145Z") lr.SetTimestamp(ts) lr.SetSeverityNumber(plog.SeverityNumberWarn) lr.SetSeverityText("Warning") @@ -82,7 +82,7 @@ var maximumLogRecord2 = func() []plog.LogRecord { lr := sl.LogRecords().AppendEmpty() lr2 := sl.LogRecords().AppendEmpty() - ts, _ := asTimestamp("2022-11-11T04:48:29.6767145Z", nil) + ts, _ := asTimestamp("2022-11-11T04:48:29.6767145Z") lr.SetTimestamp(ts) lr.SetSeverityNumber(plog.SeverityNumberWarn) lr.SetSeverityText("Warning") @@ -112,7 +112,7 @@ var maximumLogRecord2 = func() []plog.LogRecord { properties.PutDouble("float", 41.3) properties.PutBool("bool", true) - ts, _ = asTimestamp("2022-11-11T04:48:31.6767145Z", nil) + ts, _ = asTimestamp("2022-11-11T04:48:31.6767145Z") lr2.SetTimestamp(ts) lr2.SetSeverityNumber(plog.SeverityNumberWarn) lr2.SetSeverityText("Warning") @@ -149,7 +149,7 @@ var maximumLogRecord2 = func() []plog.LogRecord { var badLevelLogRecord = func() plog.LogRecord { lr := plog.NewLogs().ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - ts, _ := asTimestamp("2023-10-26T14:22:43.3416357Z", nil) + ts, _ := asTimestamp("2023-10-26T14:22:43.3416357Z") lr.SetTimestamp(ts) lr.SetSeverityNumber(plog.SeverityNumberTrace4) lr.SetSeverityText("4") @@ -193,7 +193,7 @@ var badLevelLogRecord = func() plog.LogRecord { var badTimeLogRecord = func() plog.LogRecord { lr := plog.NewLogs().ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() - ts, _ := asTimestamp("2021-10-14T22:17:11+00:00", nil) + ts, _ := asTimestamp("2021-10-14T22:17:11+00:00") lr.SetTimestamp(ts) lr.Attributes().PutStr(conventions.AttributeCloudProvider, conventions.AttributeCloudProviderAzure) @@ -239,29 +239,29 @@ var badTimeLogRecord = func() plog.LogRecord { func TestAsTimestamp(t *testing.T) { timestamp := "2022-11-11T04:48:27.6767145Z" - nanos, err := asTimestamp(timestamp, nil) + nanos, err := asTimestamp(timestamp) assert.NoError(t, err) assert.Less(t, pcommon.Timestamp(0), nanos) timestamp = "11/20/2024 13:57:18" - nanos, err = asTimestamp(timestamp, []string{"01/02/2006 15:04:05"}) + nanos, err = asTimestamp(timestamp, "01/02/2006 15:04:05") assert.NoError(t, err) assert.Less(t, pcommon.Timestamp(0), nanos) // time_format set, but fallback to iso8601 and succeeded to parse timestamp = "2022-11-11T04:48:27.6767145Z" - nanos, err = asTimestamp(timestamp, []string{"01/02/2006 15:04:05"}) + nanos, err = asTimestamp(timestamp, "01/02/2006 15:04:05") assert.NoError(t, err) assert.Less(t, pcommon.Timestamp(0), nanos) // time_format set, but all failed to parse timestamp = "11/20/2024 13:57:18" - nanos, err = asTimestamp(timestamp, []string{"2006-01-02 15:04:05"}) + nanos, err = asTimestamp(timestamp, "2006-01-02 15:04:05") assert.Error(t, err) assert.Equal(t, pcommon.Timestamp(0), nanos) timestamp = "invalid-time" - nanos, err = asTimestamp(timestamp, nil) + nanos, err = asTimestamp(timestamp) assert.Error(t, err) assert.Equal(t, pcommon.Timestamp(0), nanos) } diff --git a/receiver/azureeventhubreceiver/README.md b/receiver/azureeventhubreceiver/README.md index 79dfc9479529..bad2b947a5d8 100644 --- a/receiver/azureeventhubreceiver/README.md +++ b/receiver/azureeventhubreceiver/README.md @@ -50,18 +50,12 @@ attribute names are copied without any changes. Default: `false` (semantic conventions are not applied) -### time_format (optional) +### time_formats (optional) All supported time format for logs, metrics and traces. Default is `nil` (unset), which means using the current iso8601 parser. The format is based on https://pkg.go.dev/time#Layout. If no time-zone info, will use UTC time. If all failed, it will use iso8601 format to parse. Default: `nil` -### time_offset (optional) - -The offset hours to parsed time. Mainly for cases when there's no time-zone info in time string. - -Default: `0`. - ### Example Configuration ```yaml @@ -73,7 +67,7 @@ receivers: offset: "1234-5566" format: "azure" # optional - time_format: + time_formats: # All supported time format. Default is empty string array, which means using the current iso8601 parser. The format is based on https://pkg.go.dev/time#Layout. If no time-zone info, will use UTC time. logs: ["01/02/2006 15:04:05","2006-01-02 15:04:05","2006-01-02T15:04:05Z07:00"] metrics: ["01/02/2006 15:04:05"] diff --git a/receiver/azureeventhubreceiver/azureresourcelogs_unmarshaler.go b/receiver/azureeventhubreceiver/azureresourcelogs_unmarshaler.go index 8a3db44b0c1a..d7845de6201e 100644 --- a/receiver/azureeventhubreceiver/azureresourcelogs_unmarshaler.go +++ b/receiver/azureeventhubreceiver/azureresourcelogs_unmarshaler.go @@ -25,17 +25,17 @@ func newAzureResourceLogsUnmarshaler(buildInfo component.BuildInfo, logger *zap. if applySemanticConventions { return AzureResourceLogsEventUnmarshaler{ unmarshaler: &azurelogs.ResourceLogsUnmarshaler{ - Version: buildInfo.Version, - Logger: logger, - TimeFormat: timeFormat, + Version: buildInfo.Version, + Logger: logger, + TimeFormats: timeFormat, }, } } return AzureResourceLogsEventUnmarshaler{ unmarshaler: &azure.ResourceLogsUnmarshaler{ - Version: buildInfo.Version, - Logger: logger, - TimeFormat: timeFormat, + Version: buildInfo.Version, + Logger: logger, + TimeFormats: timeFormat, }, } } diff --git a/receiver/azureeventhubreceiver/azureresourcetraces_unmarshaler.go b/receiver/azureeventhubreceiver/azureresourcetraces_unmarshaler.go index 815817fa162e..6ff4213d8874 100755 --- a/receiver/azureeventhubreceiver/azureresourcetraces_unmarshaler.go +++ b/receiver/azureeventhubreceiver/azureresourcetraces_unmarshaler.go @@ -19,9 +19,9 @@ type azureTracesEventUnmarshaler struct { func newAzureTracesUnmarshaler(buildInfo component.BuildInfo, logger *zap.Logger, timeFormat []string) eventTracesUnmarshaler { return azureTracesEventUnmarshaler{ unmarshaler: &azure.TracesUnmarshaler{ - Version: buildInfo.Version, - Logger: logger, - TimeFormat: timeFormat, + Version: buildInfo.Version, + Logger: logger, + TimeFormats: timeFormat, }, } } diff --git a/receiver/azureeventhubreceiver/config.go b/receiver/azureeventhubreceiver/config.go index 09cc02c7e31c..b7db1541a38d 100644 --- a/receiver/azureeventhubreceiver/config.go +++ b/receiver/azureeventhubreceiver/config.go @@ -32,7 +32,7 @@ type Config struct { Format string `mapstructure:"format"` ConsumerGroup string `mapstructure:"group"` ApplySemanticConventions bool `mapstructure:"apply_semantic_conventions"` - TimeFormat TimeFormat `mapstructure:"time_format"` + TimeFormats TimeFormat `mapstructure:"time_formats"` } type TimeFormat struct { diff --git a/receiver/azureeventhubreceiver/factory.go b/receiver/azureeventhubreceiver/factory.go index 7e858e4353ee..cc0ee1098f1f 100644 --- a/receiver/azureeventhubreceiver/factory.go +++ b/receiver/azureeventhubreceiver/factory.go @@ -110,21 +110,21 @@ func (f *eventhubReceiverFactory) getReceiver( if logFormat(receiverConfig.Format) == rawLogFormat { logsUnmarshaler = newRawLogsUnmarshaler(settings.Logger) } else { - logsUnmarshaler = newAzureResourceLogsUnmarshaler(settings.BuildInfo, settings.Logger, receiverConfig.ApplySemanticConventions, receiverConfig.TimeFormat.Logs) + logsUnmarshaler = newAzureResourceLogsUnmarshaler(settings.BuildInfo, settings.Logger, receiverConfig.ApplySemanticConventions, receiverConfig.TimeFormats.Logs) } case pipeline.SignalMetrics: if logFormat(receiverConfig.Format) == rawLogFormat { metricsUnmarshaler = nil err = errors.New("raw format not supported for Metrics") } else { - metricsUnmarshaler = newAzureResourceMetricsUnmarshaler(settings.BuildInfo, settings.Logger, receiverConfig.TimeFormat.Metrics) + metricsUnmarshaler = newAzureResourceMetricsUnmarshaler(settings.BuildInfo, settings.Logger, receiverConfig.TimeFormats.Metrics) } case pipeline.SignalTraces: if logFormat(receiverConfig.Format) == rawLogFormat { tracesUnmarshaler = nil err = errors.New("raw format not supported for Traces") } else { - tracesUnmarshaler = newAzureTracesUnmarshaler(settings.BuildInfo, settings.Logger, receiverConfig.TimeFormat.Traces) + tracesUnmarshaler = newAzureTracesUnmarshaler(settings.BuildInfo, settings.Logger, receiverConfig.TimeFormats.Traces) } }