chore: Udate OTel and prometheus components (#46)

* collector: update components to use OTel 0.111.0

This commit makes the following changes:
- Updates OTel components
- Removes the untyped metrics hack, and replace it with the support
  upstream (using metric metadata instead of an additional attribute per
  sample)
- Several test updates

This change brings our fork much closer to upstream OTel and Prometheus.
This should be pretty helpful when unforking.

Change-Id: I0cfcc6a3c6bd3deba498e2592884acedda6dcab5

* confgenerator: update tests and add validation of scrape configs

This change updates the confgenerator to match the updated prometheus
library changes

Change-Id: Ie73edd0c7f1e1047a0e9e1576786e6c24922a0ff

* confgenerator: update the self metrics to flect changes in OTel

This change does the following:
- Update OTel to 0.113.0
- Remove the featuregate from the exporter for untyped double write
- Update selfmetrics (same changes done in Ops Agent)'
- Misc fixes on the fork to get tests passing - mostly updating to match
  contrib

Change-Id: I56c1d1408aec3d2c7a14fc1d83905aa2dc178a4b

* confgenerator: clean up unnecessary bits in the generated prom config

Change-Id: Ic2f04cf1a32d56f697836f8d399c4c985641af43

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21.11 as builder
+FROM golang:1.22.7 as builder
 WORKDIR /sidecar
 COPY . .
 
@@ -10,6 +10,7 @@ RUN apt update && apt install -y make
 RUN make build
 
 FROM alpine:latest
+RUN apk update && apk upgrade
 RUN apk add --no-cache ca-certificates
 COPY --from=builder /sidecar/bin/rungmpcol /rungmpcol
 COPY --from=builder /sidecar/bin/run-gmp-entrypoint /run-gmp-entrypoint

collector/exporter/googlemanagedprometheusexporter/config.go

@@ -9,42 +9,37 @@ import (
 	"github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/collector"
 	"github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/collector/googlemanagedprometheus"
 	"go.opentelemetry.io/collector/exporter/exporterhelper"
-	"go.opentelemetry.io/collector/featuregate"
+	"go.opentelemetry.io/collector/pdata/pmetric"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus"
 )
 
 // Config defines configuration for Google Cloud Managed Service for Prometheus exporter.
 type Config struct {
 	GMPConfig `mapstructure:",squash"`
 
 	// Timeout for all API calls. If not set, defaults to 12 seconds.
-	exporterhelper.TimeoutSettings `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct.
-	exporterhelper.QueueSettings   `mapstructure:"sending_queue"`
+	TimeoutSettings exporterhelper.TimeoutConfig `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct.
+	QueueSettings   exporterhelper.QueueConfig   `mapstructure:"sending_queue"`
 }
 
 // GMPConfig is a subset of the collector config applicable to the GMP exporter.
 type GMPConfig struct {
 	ProjectID    string       `mapstructure:"project"`
 	UserAgent    string       `mapstructure:"user_agent"`
 	MetricConfig MetricConfig `mapstructure:"metric"`
-
-	// Setting UntypedDoubleExport to true makes the collector double write prometheus
-	// untyped metrics to GMP similar to the GMP collector. That is, it writes it once as
-	// a gauge with the metric name suffix `unknown` and once as a counter with the
-	// metric name suffix `unknown:counter`.
-	// For the counter, if the point value is smaller than the previous point in the series
-	// it is considered a reset point.
-	UntypedDoubleExport bool `mapstructure:"untyped_double_export"`
 }
 
 type MetricConfig struct {
 	// Prefix configures the prefix of metrics sent to GoogleManagedPrometheus.  Defaults to prometheus.googleapis.com.
 	// Changing this prefix is not recommended, as it may cause metrics to not be queryable with promql in the Cloud Monitoring UI.
-	Prefix       string                         `mapstructure:"prefix"`
-	ClientConfig collector.ClientConfig         `mapstructure:",squash"`
-	Config       googlemanagedprometheus.Config `mapstructure:",squash"`
+	Prefix          string                         `mapstructure:"prefix"`
+	ClientConfig    collector.ClientConfig         `mapstructure:",squash"`
+	Config          googlemanagedprometheus.Config `mapstructure:",squash"`
+	ResourceFilters []collector.ResourceFilter     `mapstructure:"resource_filters"`
 }
 
-func (c *GMPConfig) toCollectorConfig() (collector.Config, error) {
+func (c *GMPConfig) toCollectorConfig() collector.Config {
 	// start with whatever the default collector config is.
 	cfg := collector.DefaultConfig()
 	cfg.MetricConfig.Prefix = c.MetricConfig.Prefix
@@ -55,30 +50,27 @@ func (c *GMPConfig) toCollectorConfig() (collector.Config, error) {
 	cfg.MetricConfig.InstrumentationLibraryLabels = false
 	cfg.MetricConfig.ServiceResourceLabels = false
 	// Update metric naming to match GMP conventions
-	cfg.MetricConfig.GetMetricName = c.MetricConfig.Config.GetMetricName
+	cfg.MetricConfig.GetMetricName = func(baseName string, metric pmetric.Metric) (string, error) {
+		compliantName := prometheus.BuildCompliantName(metric, "", c.MetricConfig.Config.AddMetricSuffixes)
+		return googlemanagedprometheus.GetMetricName(baseName, compliantName, metric)
+	}
 	// Map to the prometheus_target monitored resource
 	cfg.MetricConfig.MapMonitoredResource = c.MetricConfig.Config.MapToPrometheusTarget
+	cfg.MetricConfig.ExtraMetrics = c.MetricConfig.Config.ExtraMetrics
 	cfg.MetricConfig.EnableSumOfSquaredDeviation = true
 	// map the GMP config's fields to the collector config
 	cfg.ProjectID = c.ProjectID
 	cfg.UserAgent = c.UserAgent
 	cfg.MetricConfig.ClientConfig = c.MetricConfig.ClientConfig
-	if c.UntypedDoubleExport {
-		err := featuregate.GlobalRegistry().Set("gcp.untypedDoubleExport", true)
-		if err != nil {
-			return cfg, err
-		}
-	}
-
-	return cfg, nil
+	cfg.MetricConfig.ResourceFilters = c.MetricConfig.ResourceFilters
+	return cfg
 }
 
 func (cfg *Config) Validate() error {
-	collectorConfig, err := cfg.toCollectorConfig()
-	if err != nil {
-		return fmt.Errorf("error setting featuregate option: %w", err)
+	if err := collector.ValidateConfig(cfg.toCollectorConfig()); err != nil {
+		return fmt.Errorf("exporter settings are invalid :%w", err)
 	}
-	if err := collector.ValidateConfig(collectorConfig); err != nil {
+	if err := cfg.MetricConfig.Config.Validate(); err != nil {
 		return fmt.Errorf("exporter settings are invalid :%w", err)
 	}
 	return nil

collector/exporter/googlemanagedprometheusexporter/config_test.go

@@ -37,7 +37,7 @@ func TestLoadConfig(t *testing.T) {
 	r1 := cfg.Exporters[component.NewIDWithName(metadata.Type, "customname")].(*Config)
 	assert.Equal(t, r1,
 		&Config{
-			TimeoutSettings: exporterhelper.TimeoutSettings{
+			TimeoutSettings: exporterhelper.TimeoutConfig{
 				Timeout: 20 * time.Second,
 			},
 			GMPConfig: GMPConfig{
@@ -54,7 +54,7 @@ func TestLoadConfig(t *testing.T) {
 					Prefix: "my-metric-domain.com",
 				},
 			},
-			QueueSettings: exporterhelper.QueueSettings{
+			QueueSettings: exporterhelper.QueueConfig{
 				Enabled:      true,
 				NumConsumers: 2,
 				QueueSize:    10,

collector/exporter/googlemanagedprometheusexporter/factory.go

@@ -12,6 +12,7 @@ import (
 	"github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/collector"
 	"github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/collector/googlemanagedprometheus"
 	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/consumer"
 	"go.opentelemetry.io/collector/exporter"
 	"go.opentelemetry.io/collector/exporter/exporterhelper"
 
@@ -34,8 +35,8 @@ func NewFactory() exporter.Factory {
 // createDefaultConfig creates the default configuration for exporter.
 func createDefaultConfig() component.Config {
 	return &Config{
-		TimeoutSettings: exporterhelper.TimeoutSettings{Timeout: defaultTimeout},
-		QueueSettings:   exporterhelper.NewDefaultQueueSettings(),
+		TimeoutSettings: exporterhelper.TimeoutConfig{Timeout: defaultTimeout},
+		QueueSettings:   exporterhelper.NewDefaultQueueConfig(),
 		GMPConfig: GMPConfig{
 			MetricConfig: MetricConfig{
 				Config: googlemanagedprometheus.DefaultConfig(),
@@ -47,28 +48,26 @@ func createDefaultConfig() component.Config {
 // createMetricsExporter creates a metrics exporter based on this config.
 func createMetricsExporter(
 	ctx context.Context,
-	params exporter.CreateSettings,
+	params exporter.Settings,
 	cfg component.Config) (exporter.Metrics, error) {
 	eCfg := cfg.(*Config)
-
-	// We turn off normalization for serverless environments.
-	collectorConfig, err := eCfg.GMPConfig.toCollectorConfig()
-	if err != nil {
-		return nil, err
-	}
+	collectorConfig := eCfg.GMPConfig.toCollectorConfig()
 	collectorConfig.MetricConfig.CumulativeNormalization = false
-	mExp, err := collector.NewGoogleCloudMetricsExporter(ctx, collectorConfig, params.TelemetrySettings.Logger, params.BuildInfo.Version, eCfg.Timeout)
+	mExp, err := collector.NewGoogleCloudMetricsExporter(ctx, collectorConfig, params.TelemetrySettings.Logger, params.TelemetrySettings.MeterProvider, params.BuildInfo.Version, eCfg.TimeoutSettings.Timeout)
 	if err != nil {
 		return nil, err
 	}
-	return exporterhelper.NewMetricsExporter(
+	return exporterhelper.NewMetrics(
 		ctx,
 		params,
 		cfg,
 		mExp.PushMetrics,
+		exporterhelper.WithStart(mExp.Start),
 		exporterhelper.WithShutdown(mExp.Shutdown),
 		// Disable exporterhelper Timeout, since we are using a custom mechanism
 		// within exporter itself
-		exporterhelper.WithTimeout(exporterhelper.TimeoutSettings{Timeout: 0}),
-		exporterhelper.WithQueue(eCfg.QueueSettings))
+		exporterhelper.WithTimeout(exporterhelper.TimeoutConfig{Timeout: 0}),
+		exporterhelper.WithQueue(eCfg.QueueSettings),
+		exporterhelper.WithCapabilities(consumer.Capabilities{MutatesData: true}),
+	)
 }

collector/exporter/googlemanagedprometheusexporter/factory_test.go

@@ -30,11 +30,11 @@ func TestCreateExporter(t *testing.T) {
 	eCfg := cfg.(*Config)
 	eCfg.ProjectID = "test"
 
-	te, err := factory.CreateTracesExporter(ctx, exportertest.NewNopCreateSettings(), eCfg)
+	te, err := factory.CreateTracesExporter(ctx, exportertest.NewNopSettings(), eCfg)
 	assert.NoError(t, err)
 	assert.NotNil(t, te, "failed to create trace exporter")
 
-	me, err := factory.CreateMetricsExporter(ctx, exportertest.NewNopCreateSettings(), eCfg)
+	me, err := factory.CreateMetricsExporter(ctx, exportertest.NewNopSettings(), eCfg)
 	assert.NoError(t, err)
 	assert.NotNil(t, me, "failed to create metrics exporter")
 }

collector/internal/tools/go.mod

@@ -1,7 +1,8 @@
 module github.com/GoogleCloudPlatform/opentelemetry-operations-collector
 
-go 1.21
-toolchain go1.22.5
+go 1.22.0
+
+toolchain go1.23.1
 
 require (
 	github.com/client9/misspell v0.3.4
@@ -55,7 +56,7 @@ require (
 	github.com/go-toolsmith/astp v1.1.0 // indirect
 	github.com/go-toolsmith/strparse v1.1.0 // indirect
 	github.com/go-toolsmith/typep v1.1.0 // indirect
-	github.com/go-viper/mapstructure/v2 v2.0.0 // indirect
+	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
 	github.com/go-xmlfmt/xmlfmt v1.1.2 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
@@ -148,7 +149,7 @@ require (
 	github.com/sourcegraph/go-diff v0.7.0 // indirect
 	github.com/spf13/afero v1.9.5 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
-	github.com/spf13/cobra v1.6.1 // indirect
+	github.com/spf13/cobra v1.8.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.15.0 // indirect
@@ -170,24 +171,24 @@ require (
 	github.com/yagipy/maintidx v1.0.0 // indirect
 	github.com/yeya24/promlinter v0.2.0 // indirect
 	gitlab.com/bosi/decorder v0.2.3 // indirect
-	go.opentelemetry.io/collector/confmap v0.102.1 // indirect
-	go.opentelemetry.io/collector/confmap/provider/fileprovider v0.102.1 // indirect
-	go.opentelemetry.io/collector/pdata v1.9.0 // indirect
-	go.opentelemetry.io/collector/receiver v0.102.1 // indirect
-	go.opentelemetry.io/collector/semconv v0.102.1 // indirect
+	go.opentelemetry.io/collector/confmap v1.17.0 // indirect
+	go.opentelemetry.io/collector/confmap/provider/fileprovider v1.17.0 // indirect
+	go.opentelemetry.io/collector/pdata v1.17.0 // indirect
+	go.opentelemetry.io/collector/receiver v0.111.0 // indirect
+	go.opentelemetry.io/collector/semconv v0.111.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
 	golang.org/x/exp/typeparams v0.0.0-20230321023759-10a507213a29 // indirect
 	golang.org/x/mod v0.17.0 // indirect
-	golang.org/x/net v0.26.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/sys v0.21.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/net v0.30.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
 	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect
-	google.golang.org/grpc v1.64.1 // indirect
-	google.golang.org/protobuf v1.34.1 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240930140551-af27646dc61f // indirect
+	google.golang.org/grpc v1.67.1 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect