diff --git a/.chloggen/logs-for-libhoneyreceiver.yaml b/.chloggen/logs-for-libhoneyreceiver.yaml new file mode 100644 index 000000000000..dc90a4fbe50b --- /dev/null +++ b/.chloggen/logs-for-libhoneyreceiver.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: libhoneyreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Implement log signal for libhoney receiver + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [36693] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] \ No newline at end of file diff --git a/receiver/libhoneyreceiver/README.md b/receiver/libhoneyreceiver/README.md index a87c8735d5d0..a765c45383f4 100644 --- a/receiver/libhoneyreceiver/README.md +++ b/receiver/libhoneyreceiver/README.md @@ -45,20 +45,21 @@ The following setting is required for refinery traffic since: - "/1/batch" include_metadata: true auth_api: https://api.honeycomb.io - resources: - service_name: service_name - scopes: - library_name: library.name - library_version: library.version - attributes: - trace_id: trace_id - parent_id: parent_id - span_id: span_id - name: name - error: error - spankind: span.kind - durationFields: - - duration_ms + fields: + resources: + service_name: service_name + scopes: + library_name: library.name + library_version: library.version + attributes: + trace_id: trace_id + parent_id: parent_id + span_id: span_id + name: name + error: error + spankind: span.kind + durationFields: + - duration_ms ``` ### Telemetry data types supported diff --git a/receiver/libhoneyreceiver/config.go b/receiver/libhoneyreceiver/config.go index abfd6476dbd1..8290df733f84 100644 --- a/receiver/libhoneyreceiver/config.go +++ b/receiver/libhoneyreceiver/config.go @@ -11,16 +11,16 @@ import ( "go.opentelemetry.io/collector/config/confighttp" "go.opentelemetry.io/collector/confmap" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/libhoneyreceiver/internal/simplespan" ) // Config represents the receiver config settings within the collector's config.yaml type Config struct { - HTTP *HTTPConfig `mapstructure:"http"` - AuthAPI string `mapstructure:"auth_api"` - Wrapper string `mapstructure:"wrapper"` - Resources ResourcesConfig `mapstructure:"resources"` - Scopes ScopesConfig `mapstructure:"scopes"` - Attributes AttributesConfig `mapstructure:"attributes"` + HTTP *HTTPConfig `mapstructure:"http"` + AuthAPI string `mapstructure:"auth_api"` + Wrapper string `mapstructure:"wrapper"` + FieldMapConfig simplespan.FieldMapConfig `mapstructure:"fields"` } type HTTPConfig struct { @@ -30,25 +30,6 @@ type HTTPConfig struct { TracesURLPaths []string `mapstructure:"traces_url_paths,omitempty"` } -type ResourcesConfig struct { - ServiceName string `mapstructure:"service_name"` -} - -type ScopesConfig struct { - LibraryName string `mapstructure:"library_name"` - LibraryVersion string `mapstructure:"library_version"` -} - -type AttributesConfig struct { - TraceID string `mapstructure:"trace_id"` - ParentID string `mapstructure:"parent_id"` - SpanID string `mapstructure:"span_id"` - Name string `mapstructure:"name"` - Error string `mapstructure:"error"` - SpanKind string `mapstructure:"spankind"` - DurationFields []string `mapstructure:"durationFields"` -} - func (cfg *Config) Validate() error { if cfg.HTTP == nil { return errors.New("must specify at least one protocol when using the arbitrary JSON receiver") diff --git a/receiver/libhoneyreceiver/factory.go b/receiver/libhoneyreceiver/factory.go index 4d0d0fa25cfa..d90eef5a532e 100644 --- a/receiver/libhoneyreceiver/factory.go +++ b/receiver/libhoneyreceiver/factory.go @@ -14,6 +14,7 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/internal/sharedcomponent" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/libhoneyreceiver/internal/metadata" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/libhoneyreceiver/internal/simplespan" ) const ( @@ -44,21 +45,23 @@ func createDefaultConfig() component.Config { TracesURLPaths: defaultTracesURLPaths, }, AuthAPI: "", - Resources: ResourcesConfig{ - ServiceName: "service.name", - }, - Scopes: ScopesConfig{ - LibraryName: "library.name", - LibraryVersion: "library.version", - }, - Attributes: AttributesConfig{ - TraceID: "trace.trace_id", - SpanID: "trace.span_id", - ParentID: "trace.parent_id", - Name: "name", - Error: "error", - SpanKind: "span.kind", - DurationFields: durationFieldsArr, + FieldMapConfig: simplespan.FieldMapConfig{ + Resources: simplespan.ResourcesConfig{ + ServiceName: "service.name", + }, + Scopes: simplespan.ScopesConfig{ + LibraryName: "library.name", + LibraryVersion: "library.version", + }, + Attributes: simplespan.AttributesConfig{ + TraceID: "trace.trace_id", + SpanID: "trace.span_id", + ParentID: "trace.parent_id", + Name: "name", + Error: "error", + SpanKind: "span.kind", + DurationFields: durationFieldsArr, + }, }, } } diff --git a/receiver/libhoneyreceiver/internal/simplespan/simplespan.go b/receiver/libhoneyreceiver/internal/simplespan/simplespan.go new file mode 100644 index 000000000000..a8355391763f --- /dev/null +++ b/receiver/libhoneyreceiver/internal/simplespan/simplespan.go @@ -0,0 +1,187 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package simplespan // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/libhoneyreceiver/internal/simplespan" + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "time" + + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/pdata/ptrace" + "go.uber.org/zap" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/libhoneyreceiver/internal/eventtime" +) + +type FieldMapConfig struct { + Resources ResourcesConfig `mapstructure:"resources"` + Scopes ScopesConfig `mapstructure:"scopes"` + Attributes AttributesConfig `mapstructure:"attributes"` +} + +type ResourcesConfig struct { + ServiceName string `mapstructure:"service_name"` +} + +type ScopesConfig struct { + LibraryName string `mapstructure:"library_name"` + LibraryVersion string `mapstructure:"library_version"` +} + +type AttributesConfig struct { + TraceID string `mapstructure:"trace_id"` + ParentID string `mapstructure:"parent_id"` + SpanID string `mapstructure:"span_id"` + Name string `mapstructure:"name"` + Error string `mapstructure:"error"` + SpanKind string `mapstructure:"spankind"` + DurationFields []string `mapstructure:"durationFields"` +} + +type SimpleSpan struct { + Samplerate int `json:"samplerate" msgpack:"samplerate"` + MsgPackTimestamp *time.Time `msgpack:"time"` + Time string `json:"time"` // should not be trusted. use MsgPackTimestamp + Data map[string]interface{} `json:"data" msgpack:"data"` +} + +// Overrides unmarshall to make sure the MsgPackTimestamp is set +func (s *SimpleSpan) UnmarshalJSON(j []byte) error { + type _simpleSpan SimpleSpan + tstr := eventtime.GetEventTimeDefaultString() + tzero := time.Time{} + tmp := _simpleSpan{Time: "none", MsgPackTimestamp: &tzero, Samplerate: 1} + + err := json.Unmarshal(j, &tmp) + if err != nil { + return err + } + if tmp.MsgPackTimestamp.IsZero() && tmp.Time == "none" { + // neither timestamp was set. give it right now. + tmp.Time = tstr + tnow := time.Now() + tmp.MsgPackTimestamp = &tnow + } + if tmp.MsgPackTimestamp.IsZero() { + propertime := eventtime.GetEventTime(tmp.Time) + tmp.MsgPackTimestamp = &propertime + } + + *s = SimpleSpan(tmp) + return nil +} + +func (s *SimpleSpan) DebugString() string { + return fmt.Sprintf("%#v", s) +} + +// returns log until we add the trace parser +func (s *SimpleSpan) SignalType() (string, error) { + return "log", nil +} + +func (s *SimpleSpan) GetService(fields FieldMapConfig, seen *ServiceHistory, dataset string) (string, error) { + if serviceName, ok := s.Data[fields.Resources.ServiceName]; ok { + seen.NameCount[serviceName.(string)] += 1 + return serviceName.(string), nil + } + return dataset, errors.New("no service.name found in event") +} + +func (s *SimpleSpan) GetScope(fields FieldMapConfig, seen *ScopeHistory, serviceName string) (string, error) { + if scopeLibraryName, ok := s.Data[fields.Scopes.LibraryName]; ok { + scopeKey := serviceName + scopeLibraryName.(string) + if _, ok := seen.Scope[scopeKey]; ok { + // if we've seen it, we don't expect it to be different right away so we'll just return it. + return scopeKey, nil + } + // otherwise, we need to make a new found scope + scopeLibraryVersion := "unset" + if scopeLibVer, ok := s.Data[fields.Scopes.LibraryVersion]; ok { + scopeLibraryVersion = scopeLibVer.(string) + } + newScope := SimpleScope{ + ServiceName: serviceName, // we only set the service name once. If the same library comes from multiple services in the same batch, we're in trouble. + LibraryName: scopeLibraryName.(string), + LibraryVersion: scopeLibraryVersion, + ScopeSpans: ptrace.NewSpanSlice(), + ScopeLogs: plog.NewLogRecordSlice(), + } + seen.Scope[scopeKey] = newScope + return scopeKey, nil + } + return "libhoney.receiver", errors.New("library name not found") +} + +type SimpleScope struct { + ServiceName string + LibraryName string + LibraryVersion string + ScopeSpans ptrace.SpanSlice + ScopeLogs plog.LogRecordSlice +} + +type ScopeHistory struct { + Scope map[string]SimpleScope // key here is service.name+library.name +} +type ServiceHistory struct { + NameCount map[string]int +} + +func (s *SimpleSpan) ToPLogRecord(newLog *plog.LogRecord, already_used_fields *[]string, logger zap.Logger) error { + time_ns := s.MsgPackTimestamp.UnixNano() + logger.Debug("processing log with", zap.Int64("timestamp", time_ns)) + newLog.SetTimestamp(pcommon.Timestamp(time_ns)) + + if logSevCode, ok := s.Data["severity_code"]; ok { + logSevInt := int32(logSevCode.(int64)) + newLog.SetSeverityNumber(plog.SeverityNumber(logSevInt)) + } + + if logSevText, ok := s.Data["severity_text"]; ok { + newLog.SetSeverityText(logSevText.(string)) + } + + if logFlags, ok := s.Data["flags"]; ok { + logFlagsUint := uint32(logFlags.(uint64)) + newLog.SetFlags(plog.LogRecordFlags(logFlagsUint)) + } + + // undoing this is gonna be complicated: https://github.com/honeycombio/husky/blob/91c0498333cd9f5eed1fdb8544ca486db7dea565/otlp/logs.go#L61 + if logBody, ok := s.Data["body"]; ok { + newLog.Body().SetStr(logBody.(string)) + } + + newLog.Attributes().PutInt("SampleRate", int64(s.Samplerate)) + + logFieldsAlready := []string{"severity_text", "severity_code", "flags", "body"} + for k, v := range s.Data { + if slices.Contains(*already_used_fields, k) { + continue + } + if slices.Contains(logFieldsAlready, k) { + continue + } + switch v := v.(type) { + case string: + newLog.Attributes().PutStr(k, v) + case int: + newLog.Attributes().PutInt(k, int64(v)) + case int64, int16, int32: + intv := v.(int64) + newLog.Attributes().PutInt(k, intv) + case float64: + newLog.Attributes().PutDouble(k, v) + case bool: + newLog.Attributes().PutBool(k, v) + default: + logger.Warn("Span data type issue", zap.Int64("timestamp", time_ns), zap.String("key", k)) + } + } + return nil +} diff --git a/receiver/libhoneyreceiver/libhoneyparser.go b/receiver/libhoneyreceiver/libhoneyparser.go index d58cac623c60..35ba4c2eeaeb 100644 --- a/receiver/libhoneyreceiver/libhoneyparser.go +++ b/receiver/libhoneyreceiver/libhoneyparser.go @@ -4,22 +4,17 @@ package libhoneyreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/libhoneyreceiver" import ( - "encoding/json" - "errors" "fmt" "mime" "net/http" "net/url" - "slices" - "time" - "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/plog" "go.opentelemetry.io/collector/pdata/ptrace" semconv "go.opentelemetry.io/collector/semconv/v1.16.0" "go.uber.org/zap" - "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/libhoneyreceiver/internal/eventtime" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/libhoneyreceiver/internal/simplespan" ) func readContentType(resp http.ResponseWriter, req *http.Request) (encoder, bool) { @@ -63,7 +58,6 @@ func handleUnmatchedContentType(resp http.ResponseWriter) { writeResponse(resp, "text/plain", status, []byte(fmt.Sprintf("%v unsupported media type, supported: [%s, %s]", status, jsonContentType, pbContentType))) } -// taken from refinery https://github.com/honeycombio/refinery/blob/v2.6.1/route/route.go#L964-L974 func getDatasetFromRequest(path string) (string, error) { if path == "" { return "", fmt.Errorf("missing dataset name") @@ -75,164 +69,27 @@ func getDatasetFromRequest(path string) (string, error) { return dataset, nil } -type simpleSpan struct { - Samplerate int `json:"samplerate" msgpack:"samplerate"` - MsgPackTimestamp *time.Time `msgpack:"time"` - Time string `json:"time"` // should not be trusted. use MsgPackTimestamp - Data map[string]interface{} `json:"data" msgpack:"data"` -} - -// Overrides unmarshall to make sure the MsgPackTimestamp is set -func (s *simpleSpan) UnmarshalJSON(j []byte) error { - type _simpleSpan simpleSpan - tstr := eventtime.GetEventTimeDefaultString() - tzero := time.Time{} - tmp := _simpleSpan{Time: "none", MsgPackTimestamp: &tzero, Samplerate: 1} - - err := json.Unmarshal(j, &tmp) - if err != nil { - return err - } - if tmp.MsgPackTimestamp.IsZero() && tmp.Time == "none" { - // neither timestamp was set. give it right now. - tmp.Time = tstr - tnow := time.Now() - tmp.MsgPackTimestamp = &tnow - } - if tmp.MsgPackTimestamp.IsZero() { - propertime := eventtime.GetEventTime(tmp.Time) - tmp.MsgPackTimestamp = &propertime - } - - *s = simpleSpan(tmp) - return nil -} - -func (s *simpleSpan) DebugString() string { - return fmt.Sprintf("%#v", s) -} - -// returns log until we add the trace parser -func (s *simpleSpan) SignalType() (string, error) { - return "log", nil -} - -func (s *simpleSpan) GetService(cfg Config, seen *serviceHistory, dataset string) (string, error) { - if serviceName, ok := s.Data[cfg.Resources.ServiceName]; ok { - seen.NameCount[serviceName.(string)] += 1 - return serviceName.(string), nil - } - return dataset, errors.New("no service.name found in event") -} - -func (s *simpleSpan) GetScope(cfg Config, seen *scopeHistory, serviceName string) (string, error) { - if scopeLibraryName, ok := s.Data[cfg.Scopes.LibraryName]; ok { - scopeKey := serviceName + scopeLibraryName.(string) - if _, ok := seen.Scope[scopeKey]; ok { - // if we've seen it, we don't expect it to be different right away so we'll just return it. - return scopeKey, nil - } - // otherwise, we need to make a new found scope - scopeLibraryVersion := "unset" - if scopeLibVer, ok := s.Data[cfg.Scopes.LibraryVersion]; ok { - scopeLibraryVersion = scopeLibVer.(string) - } - newScope := simpleScope{ - ServiceName: serviceName, // we only set the service name once. If the same library comes from multiple services in the same batch, we're in trouble. - LibraryName: scopeLibraryName.(string), - LibraryVersion: scopeLibraryVersion, - ScopeSpans: ptrace.NewSpanSlice(), - ScopeLogs: plog.NewLogRecordSlice(), - } - seen.Scope[scopeKey] = newScope - return scopeKey, nil - } - return "libhoney.receiver", errors.New("library name not found") -} - -type simpleScope struct { - ServiceName string - LibraryName string - LibraryVersion string - ScopeSpans ptrace.SpanSlice - ScopeLogs plog.LogRecordSlice -} - -type scopeHistory struct { - Scope map[string]simpleScope // key here is service.name+library.name -} -type serviceHistory struct { - NameCount map[string]int -} - -func (s *simpleSpan) ToPLogRecord(newLog *plog.LogRecord, already_used_fields *[]string, cfg Config, logger zap.Logger) error { - time_ns := s.MsgPackTimestamp.UnixNano() - logger.Debug("processing log with", zap.Int64("timestamp", time_ns)) - newLog.SetTimestamp(pcommon.Timestamp(time_ns)) - - if logSevCode, ok := s.Data["severity_code"]; ok { - logSevInt := int32(logSevCode.(int64)) - newLog.SetSeverityNumber(plog.SeverityNumber(logSevInt)) - } - - if logSevText, ok := s.Data["severity_text"]; ok { - newLog.SetSeverityText(logSevText.(string)) - } - - if logFlags, ok := s.Data["flags"]; ok { - logFlagsUint := uint32(logFlags.(uint64)) - newLog.SetFlags(plog.LogRecordFlags(logFlagsUint)) - } - - // undoing this is gonna be complicated: https://github.com/honeycombio/husky/blob/91c0498333cd9f5eed1fdb8544ca486db7dea565/otlp/logs.go#L61 - if logBody, ok := s.Data["body"]; ok { - newLog.Body().SetStr(logBody.(string)) - } - - newLog.Attributes().PutInt("SampleRate", int64(s.Samplerate)) - - logFieldsAlready := []string{"severity_text", "severity_code", "flags", "body"} - for k, v := range s.Data { - if slices.Contains(*already_used_fields, k) { - continue - } - if slices.Contains(logFieldsAlready, k) { - continue - } - switch v := v.(type) { - case string: - newLog.Attributes().PutStr(k, v) - case int: - newLog.Attributes().PutInt(k, int64(v)) - case int64, int16, int32: - intv := v.(int64) - newLog.Attributes().PutInt(k, intv) - case float64: - newLog.Attributes().PutDouble(k, v) - case bool: - newLog.Attributes().PutBool(k, v) - default: - logger.Warn("Span data type issue", zap.Int64("timestamp", time_ns), zap.String("key", k)) - } - } - return nil -} - -func toPsomething(dataset string, ss []simpleSpan, cfg Config, logger zap.Logger) (plog.Logs, error) { - foundServices := serviceHistory{} +func toPsomething(dataset string, ss []simplespan.SimpleSpan, cfg Config, logger zap.Logger) (plog.Logs, error) { + foundServices := simplespan.ServiceHistory{} foundServices.NameCount = make(map[string]int) - foundScopes := scopeHistory{} - foundScopes.Scope = make(map[string]simpleScope) - - foundScopes.Scope = make(map[string]simpleScope) // a list of already seen scopes - foundScopes.Scope["libhoney.receiver"] = simpleScope{dataset, "libhoney.receiver", "1.0.0", ptrace.NewSpanSlice(), plog.NewLogRecordSlice()} // seed a default - - already_used_fields := []string{cfg.Resources.ServiceName, cfg.Scopes.LibraryName, cfg.Scopes.LibraryVersion} - already_used_fields = append(already_used_fields, cfg.Attributes.Name, - cfg.Attributes.TraceID, cfg.Attributes.ParentID, cfg.Attributes.SpanID, - cfg.Attributes.Error, cfg.Attributes.SpanKind, + foundScopes := simplespan.ScopeHistory{} + foundScopes.Scope = make(map[string]simplespan.SimpleScope) + + foundScopes.Scope = make(map[string]simplespan.SimpleScope) // a list of already seen scopes + foundScopes.Scope["libhoney.receiver"] = simplespan.SimpleScope{ + ServiceName: dataset, + LibraryName: "libhoney.receiver", + LibraryVersion: "1.0.0", + ScopeSpans: ptrace.NewSpanSlice(), + ScopeLogs: plog.NewLogRecordSlice(), + } // seed a default + + already_used_fields := []string{cfg.FieldMapConfig.Resources.ServiceName, cfg.FieldMapConfig.Scopes.LibraryName, cfg.FieldMapConfig.Scopes.LibraryVersion} + already_used_fields = append(already_used_fields, cfg.FieldMapConfig.Attributes.Name, + cfg.FieldMapConfig.Attributes.TraceID, cfg.FieldMapConfig.Attributes.ParentID, cfg.FieldMapConfig.Attributes.SpanID, + cfg.FieldMapConfig.Attributes.Error, cfg.FieldMapConfig.Attributes.SpanKind, ) - already_used_fields = append(already_used_fields, cfg.Attributes.DurationFields...) + already_used_fields = append(already_used_fields, cfg.FieldMapConfig.Attributes.DurationFields...) for _, span := range ss { action, err := span.SignalType() @@ -243,10 +100,10 @@ func toPsomething(dataset string, ss []simpleSpan, cfg Config, logger zap.Logger case "span": // not implemented case "log": - logService, _ := span.GetService(cfg, &foundServices, dataset) - logScopeKey, _ := span.GetScope(cfg, &foundScopes, logService) // adds a new found scope if needed + logService, _ := span.GetService(cfg.FieldMapConfig, &foundServices, dataset) + logScopeKey, _ := span.GetScope(cfg.FieldMapConfig, &foundScopes, logService) // adds a new found scope if needed newLog := foundScopes.Scope[logScopeKey].ScopeLogs.AppendEmpty() - span.ToPLogRecord(&newLog, &already_used_fields, cfg, logger) + span.ToPLogRecord(&newLog, &already_used_fields, logger) if err != nil { logger.Warn("log could not be converted from libhoney to plog", zap.String("span.object", span.DebugString())) } diff --git a/receiver/libhoneyreceiver/receiver.go b/receiver/libhoneyreceiver/receiver.go index 8fc0b37dda7f..bde46bca8364 100644 --- a/receiver/libhoneyreceiver/receiver.go +++ b/receiver/libhoneyreceiver/receiver.go @@ -24,6 +24,7 @@ import ( "go.uber.org/zap" "github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/errorutil" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/libhoneyreceiver/internal/simplespan" ) type libhoneyReceiver struct { @@ -195,7 +196,7 @@ func (r *libhoneyReceiver) handleSomething(resp http.ResponseWriter, req *http.R errorutil.HTTPError(resp, err) } - simpleSpans := make([]simpleSpan, 0) + simpleSpans := make([]simplespan.SimpleSpan, 0) switch req.Header.Get("Content-Type") { case "application/x-msgpack", "application/msgpack": decoder := msgpack.NewDecoder(bytes.NewReader(body))