From 6027c1ae76f2969ee41dcd894011eea3832391fe Mon Sep 17 00:00:00 2001 From: Alex Boten Date: Fri, 1 Dec 2023 07:34:05 -0800 Subject: [PATCH] add option for resource attributes in metrics for prometheus exporter (#4733) * add option for resource attributes in metrics for prometheus exporter This PR adds the `WithResourceAsConstantLabels` option to the Prometheus exporter to allow users to configure resource attributes to be applied on every metric. Fixes #4732 Signed-off-by: Alex Boten * add test, changelog Signed-off-by: Alex Boten * add test for including only a subset of tags, dont use a ptr Signed-off-by: Alex Boten * Update exporters/prometheus/config.go Co-authored-by: David Ashpole * include feedback from review Signed-off-by: Alex Boten * cache results Signed-off-by: Alex Boten * removed map in favour of single keyVals Signed-off-by: Alex Boten * Update exporters/prometheus/config.go Co-authored-by: Tyler Yahn * move check outside the createResourceAttributes and rename func Signed-off-by: Alex Boten --------- Signed-off-by: Alex Boten Co-authored-by: David Ashpole Co-authored-by: Tyler Yahn --- CHANGELOG.md | 4 + exporters/prometheus/config.go | 27 ++++-- exporters/prometheus/exporter.go | 82 +++++++++++++------ exporters/prometheus/exporter_test.go | 40 +++++++++ .../with_allow_resource_attributes_filter.txt | 9 ++ .../with_resource_attributes_filter.txt | 9 ++ 6 files changed, 137 insertions(+), 34 deletions(-) create mode 100755 exporters/prometheus/testdata/with_allow_resource_attributes_filter.txt create mode 100755 exporters/prometheus/testdata/with_resource_attributes_filter.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b62be0c145..f6a05cca4a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - Improve `go.opentelemetry.io/otel/propagation.TraceContext`'s performance. (#4721) +### Added + +- Add `WithResourceAsConstantLabels` option to apply resource attributes for every metric emitted by the Prometheus exporter. (#4733) + ## [1.21.0/0.44.0] 2023-11-16 ### Removed diff --git a/exporters/prometheus/config.go b/exporters/prometheus/config.go index fe14212784d..03ce27b131e 100644 --- a/exporters/prometheus/config.go +++ b/exporters/prometheus/config.go @@ -19,18 +19,20 @@ import ( "github.com/prometheus/client_golang/prometheus" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/sdk/metric" ) // config contains options for the exporter. type config struct { - registerer prometheus.Registerer - disableTargetInfo bool - withoutUnits bool - withoutCounterSuffixes bool - readerOpts []metric.ManualReaderOption - disableScopeInfo bool - namespace string + registerer prometheus.Registerer + disableTargetInfo bool + withoutUnits bool + withoutCounterSuffixes bool + readerOpts []metric.ManualReaderOption + disableScopeInfo bool + namespace string + resourceAttributesFilter attribute.Filter } // newConfig creates a validated config configured with options. @@ -151,3 +153,14 @@ func WithNamespace(ns string) Option { return cfg }) } + +// WithResourceAsConstantLabels configures the Exporter to add the resource attributes the +// resourceFilter returns true for as attributes on all exported metrics. +// +// The does not affect the target info generated from resource attributes. +func WithResourceAsConstantLabels(resourceFilter attribute.Filter) Option { + return optionFunc(func(cfg config) config { + cfg.resourceAttributesFilter = resourceFilter + return cfg + }) +} diff --git a/exporters/prometheus/exporter.go b/exporters/prometheus/exporter.go index 92651c38cec..16df309be44 100644 --- a/exporters/prometheus/exporter.go +++ b/exporters/prometheus/exporter.go @@ -78,14 +78,21 @@ func (e *Exporter) MarshalLog() interface{} { var _ metric.Reader = &Exporter{} +// keyVals is used to store resource attribute key value pairs. +type keyVals struct { + keys []string + vals []string +} + // collector is used to implement prometheus.Collector. type collector struct { reader metric.Reader - withoutUnits bool - withoutCounterSuffixes bool - disableScopeInfo bool - namespace string + withoutUnits bool + withoutCounterSuffixes bool + disableScopeInfo bool + namespace string + resourceAttributesFilter attribute.Filter mu sync.Mutex // mu protects all members below from the concurrent access. disableTargetInfo bool @@ -93,6 +100,7 @@ type collector struct { scopeInfos map[instrumentation.Scope]prometheus.Metric scopeInfosInvalid map[instrumentation.Scope]struct{} metricFamilies map[string]*dto.MetricFamily + resourceKeyVals keyVals } // prometheus counters MUST have a _total suffix by default: @@ -109,15 +117,16 @@ func New(opts ...Option) (*Exporter, error) { reader := metric.NewManualReader(cfg.readerOpts...) collector := &collector{ - reader: reader, - disableTargetInfo: cfg.disableTargetInfo, - withoutUnits: cfg.withoutUnits, - withoutCounterSuffixes: cfg.withoutCounterSuffixes, - disableScopeInfo: cfg.disableScopeInfo, - scopeInfos: make(map[instrumentation.Scope]prometheus.Metric), - scopeInfosInvalid: make(map[instrumentation.Scope]struct{}), - metricFamilies: make(map[string]*dto.MetricFamily), - namespace: cfg.namespace, + reader: reader, + disableTargetInfo: cfg.disableTargetInfo, + withoutUnits: cfg.withoutUnits, + withoutCounterSuffixes: cfg.withoutCounterSuffixes, + disableScopeInfo: cfg.disableScopeInfo, + scopeInfos: make(map[instrumentation.Scope]prometheus.Metric), + scopeInfosInvalid: make(map[instrumentation.Scope]struct{}), + metricFamilies: make(map[string]*dto.MetricFamily), + namespace: cfg.namespace, + resourceAttributesFilter: cfg.resourceAttributesFilter, } if err := cfg.registerer.Register(collector); err != nil { @@ -181,6 +190,10 @@ func (c *collector) Collect(ch chan<- prometheus.Metric) { ch <- c.targetInfo } + if c.resourceAttributesFilter != nil && len(c.resourceKeyVals.keys) == 0 { + c.createResourceAttributes(metrics.Resource) + } + for _, scopeMetrics := range metrics.ScopeMetrics { var keys, values [2]string @@ -219,26 +232,26 @@ func (c *collector) Collect(ch chan<- prometheus.Metric) { switch v := m.Data.(type) { case metricdata.Histogram[int64]: - addHistogramMetric(ch, v, m, keys, values, name) + addHistogramMetric(ch, v, m, keys, values, name, c.resourceKeyVals) case metricdata.Histogram[float64]: - addHistogramMetric(ch, v, m, keys, values, name) + addHistogramMetric(ch, v, m, keys, values, name, c.resourceKeyVals) case metricdata.Sum[int64]: - addSumMetric(ch, v, m, keys, values, name) + addSumMetric(ch, v, m, keys, values, name, c.resourceKeyVals) case metricdata.Sum[float64]: - addSumMetric(ch, v, m, keys, values, name) + addSumMetric(ch, v, m, keys, values, name, c.resourceKeyVals) case metricdata.Gauge[int64]: - addGaugeMetric(ch, v, m, keys, values, name) + addGaugeMetric(ch, v, m, keys, values, name, c.resourceKeyVals) case metricdata.Gauge[float64]: - addGaugeMetric(ch, v, m, keys, values, name) + addGaugeMetric(ch, v, m, keys, values, name, c.resourceKeyVals) } } } } -func addHistogramMetric[N int64 | float64](ch chan<- prometheus.Metric, histogram metricdata.Histogram[N], m metricdata.Metrics, ks, vs [2]string, name string) { +func addHistogramMetric[N int64 | float64](ch chan<- prometheus.Metric, histogram metricdata.Histogram[N], m metricdata.Metrics, ks, vs [2]string, name string, resourceKV keyVals) { // TODO(https://github.com/open-telemetry/opentelemetry-go/issues/3163): support exemplars for _, dp := range histogram.DataPoints { - keys, values := getAttrs(dp.Attributes, ks, vs) + keys, values := getAttrs(dp.Attributes, ks, vs, resourceKV) desc := prometheus.NewDesc(name, m.Description, keys, nil) buckets := make(map[float64]uint64, len(dp.Bounds)) @@ -257,14 +270,14 @@ func addHistogramMetric[N int64 | float64](ch chan<- prometheus.Metric, histogra } } -func addSumMetric[N int64 | float64](ch chan<- prometheus.Metric, sum metricdata.Sum[N], m metricdata.Metrics, ks, vs [2]string, name string) { +func addSumMetric[N int64 | float64](ch chan<- prometheus.Metric, sum metricdata.Sum[N], m metricdata.Metrics, ks, vs [2]string, name string, resourceKV keyVals) { valueType := prometheus.CounterValue if !sum.IsMonotonic { valueType = prometheus.GaugeValue } for _, dp := range sum.DataPoints { - keys, values := getAttrs(dp.Attributes, ks, vs) + keys, values := getAttrs(dp.Attributes, ks, vs, resourceKV) desc := prometheus.NewDesc(name, m.Description, keys, nil) m, err := prometheus.NewConstMetric(desc, valueType, float64(dp.Value), values...) @@ -276,9 +289,9 @@ func addSumMetric[N int64 | float64](ch chan<- prometheus.Metric, sum metricdata } } -func addGaugeMetric[N int64 | float64](ch chan<- prometheus.Metric, gauge metricdata.Gauge[N], m metricdata.Metrics, ks, vs [2]string, name string) { +func addGaugeMetric[N int64 | float64](ch chan<- prometheus.Metric, gauge metricdata.Gauge[N], m metricdata.Metrics, ks, vs [2]string, name string, resourceKV keyVals) { for _, dp := range gauge.DataPoints { - keys, values := getAttrs(dp.Attributes, ks, vs) + keys, values := getAttrs(dp.Attributes, ks, vs, resourceKV) desc := prometheus.NewDesc(name, m.Description, keys, nil) m, err := prometheus.NewConstMetric(desc, prometheus.GaugeValue, float64(dp.Value), values...) @@ -293,7 +306,7 @@ func addGaugeMetric[N int64 | float64](ch chan<- prometheus.Metric, gauge metric // getAttrs parses the attribute.Set to two lists of matching Prometheus-style // keys and values. It sanitizes invalid characters and handles duplicate keys // (due to sanitization) by sorting and concatenating the values following the spec. -func getAttrs(attrs attribute.Set, ks, vs [2]string) ([]string, []string) { +func getAttrs(attrs attribute.Set, ks, vs [2]string, resourceKV keyVals) ([]string, []string) { keysMap := make(map[string][]string) itr := attrs.Iter() for itr.Next() { @@ -321,11 +334,17 @@ func getAttrs(attrs attribute.Set, ks, vs [2]string) ([]string, []string) { keys = append(keys, ks[:]...) values = append(values, vs[:]...) } + + for idx := range resourceKV.keys { + keys = append(keys, resourceKV.keys[idx]) + values = append(values, resourceKV.vals[idx]) + } + return keys, values } func createInfoMetric(name, description string, res *resource.Resource) (prometheus.Metric, error) { - keys, values := getAttrs(*res.Set(), [2]string{}, [2]string{}) + keys, values := getAttrs(*res.Set(), [2]string{}, [2]string{}, keyVals{}) desc := prometheus.NewDesc(name, description, keys, nil) return prometheus.NewConstMetric(desc, prometheus.GaugeValue, float64(1), values...) } @@ -473,6 +492,15 @@ func (c *collector) metricType(m metricdata.Metrics) *dto.MetricType { return nil } +func (c *collector) createResourceAttributes(res *resource.Resource) { + c.mu.Lock() + defer c.mu.Unlock() + + resourceAttrs, _ := res.Set().Filter(c.resourceAttributesFilter) + resourceKeys, resourceValues := getAttrs(resourceAttrs, [2]string{}, [2]string{}, keyVals{}) + c.resourceKeyVals = keyVals{keys: resourceKeys, vals: resourceValues} +} + func (c *collector) scopeInfo(scope instrumentation.Scope) (prometheus.Metric, error) { c.mu.Lock() defer c.mu.Unlock() diff --git a/exporters/prometheus/exporter_test.go b/exporters/prometheus/exporter_test.go index bd31657825e..cb402e10ced 100644 --- a/exporters/prometheus/exporter_test.go +++ b/exporters/prometheus/exporter_test.go @@ -368,6 +368,46 @@ func TestPrometheusExporter(t *testing.T) { counter.Add(ctx, 9, opt) }, }, + { + name: "with resource attributes filter", + expectedFile: "testdata/with_resource_attributes_filter.txt", + options: []Option{ + WithResourceAsConstantLabels(attribute.NewDenyKeysFilter()), + }, + recordMetrics: func(ctx context.Context, meter otelmetric.Meter) { + opt := otelmetric.WithAttributes( + attribute.Key("A").String("B"), + attribute.Key("C").String("D"), + attribute.Key("E").Bool(true), + attribute.Key("F").Int(42), + ) + counter, err := meter.Float64Counter("foo", otelmetric.WithDescription("a simple counter")) + require.NoError(t, err) + counter.Add(ctx, 5, opt) + counter.Add(ctx, 10.1, opt) + counter.Add(ctx, 9.8, opt) + }, + }, + { + name: "with some resource attributes filter", + expectedFile: "testdata/with_allow_resource_attributes_filter.txt", + options: []Option{ + WithResourceAsConstantLabels(attribute.NewAllowKeysFilter("service.name")), + }, + recordMetrics: func(ctx context.Context, meter otelmetric.Meter) { + opt := otelmetric.WithAttributes( + attribute.Key("A").String("B"), + attribute.Key("C").String("D"), + attribute.Key("E").Bool(true), + attribute.Key("F").Int(42), + ) + counter, err := meter.Float64Counter("foo", otelmetric.WithDescription("a simple counter")) + require.NoError(t, err) + counter.Add(ctx, 5, opt) + counter.Add(ctx, 5.9, opt) + counter.Add(ctx, 5.3, opt) + }, + }, } for _, tc := range testCases { diff --git a/exporters/prometheus/testdata/with_allow_resource_attributes_filter.txt b/exporters/prometheus/testdata/with_allow_resource_attributes_filter.txt new file mode 100755 index 00000000000..9a4ca7793ea --- /dev/null +++ b/exporters/prometheus/testdata/with_allow_resource_attributes_filter.txt @@ -0,0 +1,9 @@ +# HELP foo_total a simple counter +# TYPE foo_total counter +foo_total{A="B",C="D",E="true",F="42",otel_scope_name="testmeter",otel_scope_version="v0.1.0",service_name="prometheus_test"} 16.2 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1 diff --git a/exporters/prometheus/testdata/with_resource_attributes_filter.txt b/exporters/prometheus/testdata/with_resource_attributes_filter.txt new file mode 100755 index 00000000000..b3ab4f80fab --- /dev/null +++ b/exporters/prometheus/testdata/with_resource_attributes_filter.txt @@ -0,0 +1,9 @@ +# HELP foo_total a simple counter +# TYPE foo_total counter +foo_total{A="B",C="D",E="true",F="42",otel_scope_name="testmeter",otel_scope_version="v0.1.0",service_name="prometheus_test",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 24.9 +# HELP otel_scope_info Instrumentation Scope metadata +# TYPE otel_scope_info gauge +otel_scope_info{otel_scope_name="testmeter",otel_scope_version="v0.1.0"} 1 +# HELP target_info Target metadata +# TYPE target_info gauge +target_info{service_name="prometheus_test",telemetry_sdk_language="go",telemetry_sdk_name="opentelemetry",telemetry_sdk_version="latest"} 1