diff --git a/.chloggen/exporter-dropped-metric.yaml b/.chloggen/exporter-dropped-metric.yaml new file mode 100644 index 00000000000..79ad4aa28dd --- /dev/null +++ b/.chloggen/exporter-dropped-metric.yaml @@ -0,0 +1,25 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver) +component: exporterhelper + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Adds `exporter_dropped_spans`, `exporter_dropped_metric_points`, and `exporter_dropped_log_records` to record when an exporter has dropped data forever. + +# One or more tracking issues or pull requests related to the change +issues: [11077] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/exporter/exporterhelper/documentation.md b/exporter/exporterhelper/documentation.md index a82163a2bfa..c59edb046dc 100644 --- a/exporter/exporterhelper/documentation.md +++ b/exporter/exporterhelper/documentation.md @@ -6,6 +6,30 @@ The following telemetry is emitted by this component. +### otelcol_exporter_dropped_log_records + +Number of log records dropped after failing to export. + +| Unit | Metric Type | Value Type | Monotonic | +| ---- | ----------- | ---------- | --------- | +| {records} | Sum | Int | true | + +### otelcol_exporter_dropped_metric_points + +Number of metric points dropped after failing to export. + +| Unit | Metric Type | Value Type | Monotonic | +| ---- | ----------- | ---------- | --------- | +| {datapoints} | Sum | Int | true | + +### otelcol_exporter_dropped_spans + +Number of spans dropped after failing to export. + +| Unit | Metric Type | Value Type | Monotonic | +| ---- | ----------- | ---------- | --------- | +| {spans} | Sum | Int | true | + ### otelcol_exporter_enqueue_failed_log_records Number of log records failed to be added to the sending queue. diff --git a/exporter/exporterhelper/internal/base_exporter.go b/exporter/exporterhelper/internal/base_exporter.go index 922dbc9b34a..15a33cbd223 100644 --- a/exporter/exporterhelper/internal/base_exporter.go +++ b/exporter/exporterhelper/internal/base_exporter.go @@ -136,6 +136,7 @@ func NewBaseExporter(set exporter.Settings, signal component.DataType, osf Obsre func (be *BaseExporter) Send(ctx context.Context, req internal.Request) error { err := be.QueueSender.Send(ctx, req) if err != nil { + recordDroppedMetric(ctx, be.Obsrep, int64(req.ItemsCount())) be.Set.Logger.Error("Exporting failed. Rejecting data."+be.ExportFailureMessage, zap.Error(err), zap.Int("rejected_items", req.ItemsCount())) } @@ -339,3 +340,15 @@ func CheckStatus(t *testing.T, sd sdktrace.ReadOnlySpan, err error) { require.Equal(t, codes.Unset, sd.Status().Code, "SpanData %v", sd) } } + +func recordDroppedMetric(ctx context.Context, obsrep *ObsReport, count int64) { + if obsrep.DataType == component.DataTypeTraces { + obsrep.TelemetryBuilder.ExporterDroppedSpans.Add(ctx, count) + } + if obsrep.DataType == component.DataTypeMetrics { + obsrep.TelemetryBuilder.ExporterDroppedMetricPoints.Add(ctx, count) + } + if obsrep.DataType == component.DataTypeLogs { + obsrep.TelemetryBuilder.ExporterDroppedLogRecords.Add(ctx, count) + } +} diff --git a/exporter/exporterhelper/internal/metadata/generated_telemetry.go b/exporter/exporterhelper/internal/metadata/generated_telemetry.go index b61d6cda6f2..6b8fab021c3 100644 --- a/exporter/exporterhelper/internal/metadata/generated_telemetry.go +++ b/exporter/exporterhelper/internal/metadata/generated_telemetry.go @@ -30,6 +30,9 @@ func Tracer(settings component.TelemetrySettings) trace.Tracer { // as defined in metadata and user config. type TelemetryBuilder struct { meter metric.Meter + ExporterDroppedLogRecords metric.Int64Counter + ExporterDroppedMetricPoints metric.Int64Counter + ExporterDroppedSpans metric.Int64Counter ExporterEnqueueFailedLogRecords metric.Int64Counter ExporterEnqueueFailedMetricPoints metric.Int64Counter ExporterEnqueueFailedSpans metric.Int64Counter @@ -100,6 +103,24 @@ func NewTelemetryBuilder(settings component.TelemetrySettings, options ...Teleme } builder.meters[configtelemetry.LevelBasic] = LeveledMeter(settings, configtelemetry.LevelBasic) var err, errs error + builder.ExporterDroppedLogRecords, err = builder.meters[configtelemetry.LevelBasic].Int64Counter( + "otelcol_exporter_dropped_log_records", + metric.WithDescription("Number of log records dropped after failing to export."), + metric.WithUnit("{records}"), + ) + errs = errors.Join(errs, err) + builder.ExporterDroppedMetricPoints, err = builder.meters[configtelemetry.LevelBasic].Int64Counter( + "otelcol_exporter_dropped_metric_points", + metric.WithDescription("Number of metric points dropped after failing to export."), + metric.WithUnit("{datapoints}"), + ) + errs = errors.Join(errs, err) + builder.ExporterDroppedSpans, err = builder.meters[configtelemetry.LevelBasic].Int64Counter( + "otelcol_exporter_dropped_spans", + metric.WithDescription("Number of spans dropped after failing to export."), + metric.WithUnit("{spans}"), + ) + errs = errors.Join(errs, err) builder.ExporterEnqueueFailedLogRecords, err = builder.meters[configtelemetry.LevelBasic].Int64Counter( "otelcol_exporter_enqueue_failed_log_records", metric.WithDescription("Number of log records failed to be added to the sending queue."), diff --git a/exporter/exporterhelper/internal/queue_sender.go b/exporter/exporterhelper/internal/queue_sender.go index 60a94966336..66d77be4d5b 100644 --- a/exporter/exporterhelper/internal/queue_sender.go +++ b/exporter/exporterhelper/internal/queue_sender.go @@ -97,6 +97,7 @@ func NewQueueSender(q exporterqueue.Queue[internal.Request], set exporter.Settin consumeFunc := func(ctx context.Context, req internal.Request) error { err := qs.NextSender.Send(ctx, req) if err != nil { + recordDroppedMetric(ctx, qs.obsrep, int64(req.ItemsCount())) set.Logger.Error("Exporting failed. Dropping data."+exportFailureMessage, zap.Error(err), zap.Int("dropped_items", req.ItemsCount())) } diff --git a/exporter/exporterhelper/metadata.yaml b/exporter/exporterhelper/metadata.yaml index 9156cb95c80..09ddf1aa4e5 100644 --- a/exporter/exporterhelper/metadata.yaml +++ b/exporter/exporterhelper/metadata.yaml @@ -99,3 +99,27 @@ telemetry: gauge: value_type: int async: true + + exporter_dropped_spans: + enabled: true + description: Number of spans dropped after failing to export. + unit: "{spans}" + sum: + value_type: int + monotonic: true + + exporter_dropped_metric_points: + enabled: true + description: Number of metric points dropped after failing to export. + unit: "{datapoints}" + sum: + value_type: int + monotonic: true + + exporter_dropped_log_records: + enabled: true + description: Number of log records dropped after failing to export. + unit: "{records}" + sum: + value_type: int + monotonic: true \ No newline at end of file