From 1cb3d76188c449f52b0930eed56f4fcf32d27a11 Mon Sep 17 00:00:00 2001 From: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> Date: Fri, 6 Sep 2024 12:15:35 -0600 Subject: [PATCH 1/4] Add exporter dropped metrics --- exporter/exporterhelper/common.go | 13 ++++++++++ exporter/exporterhelper/documentation.md | 24 +++++++++++++++++++ .../internal/metadata/generated_telemetry.go | 21 ++++++++++++++++ exporter/exporterhelper/metadata.yaml | 24 +++++++++++++++++++ exporter/exporterhelper/queue_sender.go | 1 + 5 files changed, 83 insertions(+) diff --git a/exporter/exporterhelper/common.go b/exporter/exporterhelper/common.go index e33205f4317..54012f0e018 100644 --- a/exporter/exporterhelper/common.go +++ b/exporter/exporterhelper/common.go @@ -309,6 +309,7 @@ func newBaseExporter(set exporter.Settings, signal component.DataType, osf obsre func (be *baseExporter) send(ctx context.Context, req Request) error { err := be.queueSender.send(ctx, req) if err != nil { + recordDroppedMetric(ctx, be.obsrep, int64(req.ItemsCount())) be.set.Logger.Error("Exporting failed. Rejecting data."+be.exportFailureMessage, zap.Error(err), zap.Int("rejected_items", req.ItemsCount())) } @@ -349,3 +350,15 @@ func (be *baseExporter) Shutdown(ctx context.Context) error { // Last shutdown the wrapped exporter itself. be.ShutdownFunc.Shutdown(ctx)) } + +func recordDroppedMetric(ctx context.Context, obsrep *obsReport, count int64) { + if obsrep.dataType == component.DataTypeTraces { + obsrep.telemetryBuilder.ExporterDroppedSpans.Add(ctx, count) + } + if obsrep.dataType == component.DataTypeMetrics { + obsrep.telemetryBuilder.ExporterDroppedMetricPoints.Add(ctx, count) + } + if obsrep.dataType == component.DataTypeLogs { + obsrep.telemetryBuilder.ExporterDroppedLogRecords.Add(ctx, count) + } +} diff --git a/exporter/exporterhelper/documentation.md b/exporter/exporterhelper/documentation.md index a82163a2bfa..09df4200816 100644 --- a/exporter/exporterhelper/documentation.md +++ b/exporter/exporterhelper/documentation.md @@ -6,6 +6,30 @@ The following telemetry is emitted by this component. +### otelcol_exporter_dropped_log_records + +Number of log records dropped after failing to export. + +| Unit | Metric Type | Value Type | Monotonic | +| ---- | ----------- | ---------- | --------- | +| {records} | Sum | Int | true | + +### otelcol_exporter_dropped_metric_points + +Number of metric points dropped after failing to export. + +| Unit | Metric Type | Value Type | Monotonic | +| ---- | ----------- | ---------- | --------- | +| {records} | Sum | Int | true | + +### otelcol_exporter_dropped_spans + +Number of spans dropped after failing to export. + +| Unit | Metric Type | Value Type | Monotonic | +| ---- | ----------- | ---------- | --------- | +| {records} | Sum | Int | true | + ### otelcol_exporter_enqueue_failed_log_records Number of log records failed to be added to the sending queue. diff --git a/exporter/exporterhelper/internal/metadata/generated_telemetry.go b/exporter/exporterhelper/internal/metadata/generated_telemetry.go index a35625ac0a5..70d4c3a9052 100644 --- a/exporter/exporterhelper/internal/metadata/generated_telemetry.go +++ b/exporter/exporterhelper/internal/metadata/generated_telemetry.go @@ -30,6 +30,9 @@ func Tracer(settings component.TelemetrySettings) trace.Tracer { // as defined in metadata and user config. type TelemetryBuilder struct { meter metric.Meter + ExporterDroppedLogRecords metric.Int64Counter + ExporterDroppedMetricPoints metric.Int64Counter + ExporterDroppedSpans metric.Int64Counter ExporterEnqueueFailedLogRecords metric.Int64Counter ExporterEnqueueFailedMetricPoints metric.Int64Counter ExporterEnqueueFailedSpans metric.Int64Counter @@ -92,6 +95,24 @@ func NewTelemetryBuilder(settings component.TelemetrySettings, options ...teleme } builder.meters[configtelemetry.LevelBasic] = LeveledMeter(settings, configtelemetry.LevelBasic) var err, errs error + builder.ExporterDroppedLogRecords, err = builder.meters[configtelemetry.LevelBasic].Int64Counter( + "otelcol_exporter_dropped_log_records", + metric.WithDescription("Number of log records dropped after failing to export."), + metric.WithUnit("{records}"), + ) + errs = errors.Join(errs, err) + builder.ExporterDroppedMetricPoints, err = builder.meters[configtelemetry.LevelBasic].Int64Counter( + "otelcol_exporter_dropped_metric_points", + metric.WithDescription("Number of metric points dropped after failing to export."), + metric.WithUnit("{records}"), + ) + errs = errors.Join(errs, err) + builder.ExporterDroppedSpans, err = builder.meters[configtelemetry.LevelBasic].Int64Counter( + "otelcol_exporter_dropped_spans", + metric.WithDescription("Number of spans dropped after failing to export."), + metric.WithUnit("{records}"), + ) + errs = errors.Join(errs, err) builder.ExporterEnqueueFailedLogRecords, err = builder.meters[configtelemetry.LevelBasic].Int64Counter( "otelcol_exporter_enqueue_failed_log_records", metric.WithDescription("Number of log records failed to be added to the sending queue."), diff --git a/exporter/exporterhelper/metadata.yaml b/exporter/exporterhelper/metadata.yaml index 9156cb95c80..0c6cfe0cb0c 100644 --- a/exporter/exporterhelper/metadata.yaml +++ b/exporter/exporterhelper/metadata.yaml @@ -99,3 +99,27 @@ telemetry: gauge: value_type: int async: true + + exporter_dropped_spans: + enabled: true + description: Number of spans dropped after failing to export. + unit: "{records}" + sum: + value_type: int + monotonic: true + + exporter_dropped_metric_points: + enabled: true + description: Number of metric points dropped after failing to export. + unit: "{records}" + sum: + value_type: int + monotonic: true + + exporter_dropped_log_records: + enabled: true + description: Number of log records dropped after failing to export. + unit: "{records}" + sum: + value_type: int + monotonic: true \ No newline at end of file diff --git a/exporter/exporterhelper/queue_sender.go b/exporter/exporterhelper/queue_sender.go index 060edab813a..6ea780161ff 100644 --- a/exporter/exporterhelper/queue_sender.go +++ b/exporter/exporterhelper/queue_sender.go @@ -89,6 +89,7 @@ func newQueueSender(q exporterqueue.Queue[Request], set exporter.Settings, numCo consumeFunc := func(ctx context.Context, req Request) error { err := qs.nextSender.send(ctx, req) if err != nil { + recordDroppedMetric(ctx, qs.obsrep, int64(req.ItemsCount())) set.Logger.Error("Exporting failed. Dropping data."+exportFailureMessage, zap.Error(err), zap.Int("dropped_items", req.ItemsCount())) } From b1bf50cab88859a422ab17de7f396d2600009637 Mon Sep 17 00:00:00 2001 From: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> Date: Mon, 16 Sep 2024 15:12:59 -0600 Subject: [PATCH 2/4] Apply suggestions from code review Co-authored-by: Alex Boten <223565+codeboten@users.noreply.github.com> --- exporter/exporterhelper/metadata.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exporter/exporterhelper/metadata.yaml b/exporter/exporterhelper/metadata.yaml index 0c6cfe0cb0c..09ddf1aa4e5 100644 --- a/exporter/exporterhelper/metadata.yaml +++ b/exporter/exporterhelper/metadata.yaml @@ -103,7 +103,7 @@ telemetry: exporter_dropped_spans: enabled: true description: Number of spans dropped after failing to export. - unit: "{records}" + unit: "{spans}" sum: value_type: int monotonic: true @@ -111,7 +111,7 @@ telemetry: exporter_dropped_metric_points: enabled: true description: Number of metric points dropped after failing to export. - unit: "{records}" + unit: "{datapoints}" sum: value_type: int monotonic: true From 67e6d318b00d3815693bad57ba19b4c792712f2b Mon Sep 17 00:00:00 2001 From: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> Date: Mon, 16 Sep 2024 15:14:06 -0600 Subject: [PATCH 3/4] Changelog --- .chloggen/exporter-dropped-metric.yaml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .chloggen/exporter-dropped-metric.yaml diff --git a/.chloggen/exporter-dropped-metric.yaml b/.chloggen/exporter-dropped-metric.yaml new file mode 100644 index 00000000000..79ad4aa28dd --- /dev/null +++ b/.chloggen/exporter-dropped-metric.yaml @@ -0,0 +1,25 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver) +component: exporterhelper + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Adds `exporter_dropped_spans`, `exporter_dropped_metric_points`, and `exporter_dropped_log_records` to record when an exporter has dropped data forever. + +# One or more tracking issues or pull requests related to the change +issues: [11077] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] From 3eab07b8305b3dc23c91e2db8beade645c9043bf Mon Sep 17 00:00:00 2001 From: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> Date: Thu, 19 Sep 2024 10:50:49 -0600 Subject: [PATCH 4/4] make gogenerate --- exporter/exporterhelper/documentation.md | 4 ++-- .../exporterhelper/internal/metadata/generated_telemetry.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/exporter/exporterhelper/documentation.md b/exporter/exporterhelper/documentation.md index 09df4200816..c59edb046dc 100644 --- a/exporter/exporterhelper/documentation.md +++ b/exporter/exporterhelper/documentation.md @@ -20,7 +20,7 @@ Number of metric points dropped after failing to export. | Unit | Metric Type | Value Type | Monotonic | | ---- | ----------- | ---------- | --------- | -| {records} | Sum | Int | true | +| {datapoints} | Sum | Int | true | ### otelcol_exporter_dropped_spans @@ -28,7 +28,7 @@ Number of spans dropped after failing to export. | Unit | Metric Type | Value Type | Monotonic | | ---- | ----------- | ---------- | --------- | -| {records} | Sum | Int | true | +| {spans} | Sum | Int | true | ### otelcol_exporter_enqueue_failed_log_records diff --git a/exporter/exporterhelper/internal/metadata/generated_telemetry.go b/exporter/exporterhelper/internal/metadata/generated_telemetry.go index c269a3cd449..6b8fab021c3 100644 --- a/exporter/exporterhelper/internal/metadata/generated_telemetry.go +++ b/exporter/exporterhelper/internal/metadata/generated_telemetry.go @@ -112,13 +112,13 @@ func NewTelemetryBuilder(settings component.TelemetrySettings, options ...Teleme builder.ExporterDroppedMetricPoints, err = builder.meters[configtelemetry.LevelBasic].Int64Counter( "otelcol_exporter_dropped_metric_points", metric.WithDescription("Number of metric points dropped after failing to export."), - metric.WithUnit("{records}"), + metric.WithUnit("{datapoints}"), ) errs = errors.Join(errs, err) builder.ExporterDroppedSpans, err = builder.meters[configtelemetry.LevelBasic].Int64Counter( "otelcol_exporter_dropped_spans", metric.WithDescription("Number of spans dropped after failing to export."), - metric.WithUnit("{records}"), + metric.WithUnit("{spans}"), ) errs = errors.Join(errs, err) builder.ExporterEnqueueFailedLogRecords, err = builder.meters[configtelemetry.LevelBasic].Int64Counter(