pkg/translator/prometheus: Allow not normalizing UTF8 characters

.chloggen/allowutf-prometheustranslator.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: pkg/translator/prometheus
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Allow UTF-8 characters in Prometheus metric names and labels.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [35459]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user, api]

exporter/prometheusexporter/README.md

@@ -60,6 +60,10 @@ Given the example, metrics will be available at `https://1.2.3.4:1234/metrics`.
 
 OpenTelemetry metric names and attributes are normalized to be compliant with Prometheus naming rules. [Details on this normalization process are described in the Prometheus translator module](../../pkg/translator/prometheus/).
 
+Prometheus 2.55.0 introduced support for UTF-8 characters behind the feature-flag `utf8-names`. Prometheus 3.0.0 and later accept UTF-8 by default. This means that name and attribute normalization is not required if you're using those versions. To allow UTF-8 characters to be exposed without normalization, start the collector with the feature gate: `--feature-gates=pkg.translator.prometheus.allow_utf8`.
+
+The scraper must include `scaping=allow-utf-8` in the `Accept` header for UTF-8 characters to be exposed.
+
 ## Setting resource attributes as metric labels
 
 By default, resource attributes are added to a special metric called `target_info`. To select and group by metrics by resource attributes, you [need to do join on `target_info`](https://prometheus.io/docs/prometheus/latest/querying/operators/#many-to-one-and-one-to-many-vector-matches). For example, to select metrics with `k8s_namespace_name` attribute equal to `my-namespace`:

exporter/prometheusexporter/prometheus.go

@@ -11,9 +11,12 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/prometheus/common/model"
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/exporter"
 	"go.opentelemetry.io/collector/pdata/pmetric"
+
+	prometheustranslator "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus"
 )
 
 type prometheusExporter struct {
@@ -30,6 +33,10 @@ type prometheusExporter struct {
 var errBlankPrometheusAddress = errors.New("expecting a non-blank address to run the Prometheus metrics handler")
 
 func newPrometheusExporter(config *Config, set exporter.Settings) (*prometheusExporter, error) {
+	if prometheustranslator.AllowUTF8FeatureGate.IsEnabled() {
+		model.NameValidationScheme = model.UTF8Validation
+	}
+
 	addr := strings.TrimSpace(config.Endpoint)
 	if strings.TrimSpace(config.Endpoint) == "" {
 		return nil, errBlankPrometheusAddress

exporter/prometheusremotewriteexporter/README.md

@@ -111,6 +111,8 @@ To enable it run collector with enabled feature gate `exporter.prometheusremotew
 
 OpenTelemetry metric names and attributes are normalized to be compliant with Prometheus naming rules. [Details on this normalization process are described in the Prometheus translator module](../../pkg/translator/prometheus/).
 
+Prometheus 2.55.0 introduced support for UTF-8 characters behind the feature-flag `utf8-names`. Prometheus 3.0.0 and later accept UTF-8 by default. This means that name and attribute normalization is not required if you're using those versions. To allow UTF-8 characters to be exposed without normalization, start the collector with the feature gate: `--feature-gates=pkg.translator.prometheus.allow_utf8`.
+
 ## Setting resource attributes as metric labels
 
 By default, resource attributes are added to a special metric called `target_info`. To select and group by metrics by resource attributes, you [need to do join on `target_info`](https://prometheus.io/docs/prometheus/latest/querying/operators/#many-to-one-and-one-to-many-vector-matches). For example, to select metrics with `k8s_namespace_name` attribute equal to `my-namespace`:

exporter/prometheusremotewriteexporter/exporter.go

@@ -18,6 +18,7 @@ import (
 	"github.com/cenkalti/backoff/v4"
 	"github.com/gogo/protobuf/proto"
 	"github.com/golang/snappy"
+	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/prompb"
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/config/confighttp"
@@ -88,6 +89,10 @@ func newPRWTelemetry(set exporter.Settings) (prwTelemetry, error) {
 
 // newPRWExporter initializes a new prwExporter instance and sets fields accordingly.
 func newPRWExporter(cfg *Config, set exporter.Settings) (*prwExporter, error) {
+	if prometheustranslator.AllowUTF8FeatureGate.IsEnabled() {
+		model.NameValidationScheme = model.UTF8Validation
+	}
+
 	sanitizedLabels, err := validateAndSanitizeExternalLabels(cfg)
 	if err != nil {
 		return nil, err

exporter/prometheusremotewriteexporter/go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.111.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus v0.111.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite v0.111.0
+	github.com/prometheus/common v0.60.0
 	github.com/prometheus/prometheus v0.54.1
 	github.com/stretchr/testify v1.9.0
 	github.com/tidwall/wal v1.1.7
@@ -55,7 +56,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.60.0 // indirect
 	github.com/rs/cors v1.11.1 // indirect
 	github.com/tidwall/gjson v1.10.2 // indirect
 	github.com/tidwall/match v1.1.1 // indirect

pkg/translator/prometheus/feature_gates.go

@@ -0,0 +1,53 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package prometheus // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus"
+
+import (
+	"go.opentelemetry.io/collector/featuregate"
+)
+
+var (
+	dropSanitizationGate = featuregate.GlobalRegistry().MustRegister(
+		"pkg.translator.prometheus.PermissiveLabelSanitization",
+		featuregate.StageAlpha,
+		featuregate.WithRegisterDescription("Controls whether to change labels starting with '_' to 'key_'."),
+		featuregate.WithRegisterReferenceURL("https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/8950"),
+	)
+
+	normalizeNameGate = featuregate.GlobalRegistry().MustRegister(
+		"pkg.translator.prometheus.NormalizeName",
+		featuregate.StageBeta,
+		featuregate.WithRegisterDescription("Controls whether metrics names are automatically normalized to follow Prometheus naming convention. Attention: if 'pkg.translator.prometheus.allowUTF8' is enabled, UTF-8 characters will not be normalized."),
+		featuregate.WithRegisterReferenceURL("https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/8950"),
+	)
+
+	// AllowUTF8FeatureGate could be a private variable, it is used mostly to toogle the
+	// value of a global variable and it's intended usage is:
+	/*
+		import "github.com/prometheus/common/model"
+
+		if AllowUTF8FeatureGate.IsEnabled() {
+			model.NameValidationScheme = model.UTF8Validation
+		}
+	*/
+	// We could do this with a init function in the translator package, but we can't guarantee that
+	// the featuregate.GlobalRegistry is initialized before the init function here.
+	//
+	// Components that want to respect this feature gate behavior should check if it is enabled and
+	// set the model.NameValidationScheme accordingly.
+	//
+	// WARNING: Since it's a global variable, if one component enables it then it works for all components
+	// that are using this package.
+	AllowUTF8FeatureGate = featuregate.GlobalRegistry().MustRegister(
+		"pkg.translator.prometheus.allowUTF8",
+		featuregate.StageAlpha,
+		featuregate.WithRegisterDescription("When enabled, UTF-8 characters will not be translated to underscores."),
+		featuregate.WithRegisterReferenceURL("https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/35459"),
+		featuregate.WithRegisterFromVersion("v0.110.0"),
+	)
+)
+
+func init() {
+
+}

pkg/translator/prometheus/normalize_label.go

@@ -6,15 +6,6 @@ package prometheus // import "github.com/open-telemetry/opentelemetry-collector-
 import (
 	"strings"
 	"unicode"
-
-	"go.opentelemetry.io/collector/featuregate"
-)
-
-var dropSanitizationGate = featuregate.GlobalRegistry().MustRegister(
-	"pkg.translator.prometheus.PermissiveLabelSanitization",
-	featuregate.StageAlpha,
-	featuregate.WithRegisterDescription("Controls whether to change labels starting with '_' to 'key_'."),
-	featuregate.WithRegisterReferenceURL("https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/8950"),
 )
 
 // Normalizes the specified label to follow Prometheus label names standard
@@ -31,8 +22,10 @@ func NormalizeLabel(label string) string {
 		return label
 	}
 
-	// Replace all non-alphanumeric runes with underscores
-	label = strings.Map(sanitizeRune, label)
+	if !AllowUTF8FeatureGate.IsEnabled() {
+		// Replace all non-alphanumeric runes with underscores
+		label = strings.Map(sanitizeRune, label)
+	}
 
 	// If label starts with a number, prepend with "key_"
 	if unicode.IsDigit(rune(label[0])) {

pkg/translator/prometheus/normalize_label_test.go

@@ -11,25 +11,68 @@ import (
 	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/common/testutil"
 )
 
-func TestSanitize(t *testing.T) {
+func TestNormalizeLabel(t *testing.T) {
+	testCases := []struct {
+		label           string
+		utfAllowed      bool
+		dropSantization bool
+		expected        string
+	}{
+		{
+			label:    "",
+			expected: "",
+		},
+		{
+			label:    "test",
+			expected: "test",
+		},
+		{
+			label:    "__test",
+			expected: "__test",
+		},
+		{
+			label:    "_test",
+			expected: "key_test",
+		},
+		{
+			label:           "_test",
+			dropSantization: true,
+			expected:        "_test",
+		},
+		{
+			label:    "0test",
+			expected: "key_0test",
+		},
+		{
+			label:           "0test",
+			dropSantization: true,
+			expected:        "key_0test",
+		},
+		{
+			label:    "test_/",
+			expected: "test__",
+		},
+		{
+			label:      "test_/",
+			utfAllowed: true,
+			expected:   "test_/",
+		},
+		{
+			label:    "test.test",
+			expected: "test_test",
+		},
+		{
+			label:      "test.test",
+			utfAllowed: true,
+			expected:   "test.test",
+		},
+	}
 
-	defer testutil.SetFeatureGateForTest(t, dropSanitizationGate, false)()
-
-	require.Equal(t, "", NormalizeLabel(""), "")
-	require.Equal(t, "key_test", NormalizeLabel("_test"))
-	require.Equal(t, "key_0test", NormalizeLabel("0test"))
-	require.Equal(t, "test", NormalizeLabel("test"))
-	require.Equal(t, "test__", NormalizeLabel("test_/"))
-	require.Equal(t, "__test", NormalizeLabel("__test"))
-}
-
-func TestSanitizeDropSanitization(t *testing.T) {
-
-	defer testutil.SetFeatureGateForTest(t, dropSanitizationGate, true)()
-
-	require.Equal(t, "", NormalizeLabel(""))
-	require.Equal(t, "_test", NormalizeLabel("_test"))
-	require.Equal(t, "key_0test", NormalizeLabel("0test"))
-	require.Equal(t, "test", NormalizeLabel("test"))
-	require.Equal(t, "__test", NormalizeLabel("__test"))
+	for _, tc := range testCases {
+		t.Run(tc.label, func(t *testing.T) {
+			testutil.SetFeatureGateForTest(t, dropSanitizationGate, tc.dropSantization)
+			testutil.SetFeatureGateForTest(t, AllowUTF8FeatureGate, tc.utfAllowed)
+			require.Equal(t, tc.expected, NormalizeLabel(tc.label))
+		})
+	}
 }