From efdf2e48bf552347f399f5080beb54df0bd5593c Mon Sep 17 00:00:00 2001 From: Daniel Jaglowski Date: Fri, 13 Dec 2024 16:27:57 -0500 Subject: [PATCH] [connector/routing] Deprecate `match_once` parameter (#36824) This PR deprecates the `match_once` parameter. It defines a multi-step process which hopefully gives users plenty of time to make necessary changes. It also provides several detailed examples of how to migrate a configuration. Resolves #29882 --- .chloggen/deprecate-matchonce.yaml | 27 ++++ connector/routingconnector/README.md | 172 ++++++++++++++++++++++++++ connector/routingconnector/logs.go | 5 + connector/routingconnector/metrics.go | 5 + connector/routingconnector/traces.go | 5 + 5 files changed, 214 insertions(+) create mode 100644 .chloggen/deprecate-matchonce.yaml diff --git a/.chloggen/deprecate-matchonce.yaml b/.chloggen/deprecate-matchonce.yaml new file mode 100644 index 000000000000..b8a761765d59 --- /dev/null +++ b/.chloggen/deprecate-matchonce.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: deprecation + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: connector/routing + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Deprecate `match_once` parameter. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [29882] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/connector/routingconnector/README.md b/connector/routingconnector/README.md index bfc4c26b2111..9235cc114b67 100644 --- a/connector/routingconnector/README.md +++ b/connector/routingconnector/README.md @@ -26,6 +26,21 @@ Routes logs, metrics or traces based on resource attributes to specific pipelines using [OpenTelemetry Transformation Language (OTTL)](../../pkg/ottl/README.md) statements as routing conditions. +## Notice + +The `match_once` field is deprecated as of `v0.115.0`. The deprecation schedule is planned as follows: + +- `v0.115.0`: The field is deprecated. If `false` is used, a warning will be logged. +- `v0.116.0`: The default value will change from `false` to `true`. If `false` is used, an error will be logged. +- `v0.117.0`: The field will be disconnected from behavior of the connector. If used (either `false` or `true`), an error will be logged. +- `v0.119.0`: The field will be removed. + +### Migration + +It is recommended to set `match_once: true` until `v0.116.0` and then remove all usage of the field before `v0.119.0`. + +For detailed guidance on how to migrate configuration from `match_once: false` to `match_once: true`, see [Config Migration](#config-migration.md). + ## Configuration If you are not already familiar with connectors, you may find it helpful to first visit the [Connectors README]. @@ -285,6 +300,163 @@ service: exporters: [file/ecorp] ``` +## Config Migration + +The following examples demonstrate some strategies for migrating a configuration to `match_once: true`. + +### Example without `default_pipelines` + +If not using `default_pipelines`, you may be able to split the router into multiple parallel routers. +In the following example, the `"env"` and `"region"` are not directly related. + +```yaml +routing: + match_once: false + table: + - condition: attributes["env"] == "prod" + pipelines: [ logs/prod ] + - condition: attributes["env"] == "dev" + pipelines: [ logs/dev ] + - condition: attributes["region"] == "east" + pipelines: [ logs/east ] + - condition: attributes["region"] == "west" + pipelines: [ logs/west ] + +service: + pipelines: + logs/in::exporters: [routing] + logs/prod::receivers: [routing] + logs/dev::receivers: [routing] + logs/east::receivers: [routing] + logs/west::receivers: [routing] +``` + +Therefore, the same behavior can be achieved using separate routers. Listing both routers in the pipeline configuration will +result in each receiving an independent handle to the data. The same data can then match routes in both routers. + +```yaml +routing/env: + match_once: true + table: + - condition: attributes["env"] == "prod" + pipelines: [ logs/prod ] + - condition: attributes["env"] == "dev" + pipelines: [ logs/dev ] +routing/region: + match_once: true + table: + - condition: attributes["region"] == "east" + pipelines: [ logs/east ] + - condition: attributes["region"] == "west" + pipelines: [ logs/west ] + +service: + pipelines: + logs/in::exporters: [routing/env, routing/region] + logs/prod::receivers: [routing/env] + logs/dev::receivers: [routing/env] + logs/east::receivers: [routing/region] + logs/west::receivers: [routing/region] +``` + +### Example with `default_pipelines` + +The following example demonstrates strategies for migrating to `match_once: true` while using `default_pipelines`. + +```yaml +routing: + match_once: false + default_pipelines: [ logs/default ] + table: + - condition: attributes["env"] == "prod" + pipelines: [ logs/prod ] + - condition: attributes["env"] == "dev" + pipelines: [ logs/dev ] + - condition: attributes["region"] == "east" + pipelines: [ logs/east ] + - condition: attributes["region"] == "west" + pipelines: [ logs/west ] + +service: + pipelines: + logs/in::exporters: [routing] + logs/default::receivers: [routing] + logs/prod::receivers: [routing] + logs/dev::receivers: [routing] + logs/east::receivers: [routing] + logs/west::receivers: [routing] +``` + +If the number of routes are limited, you may be able to articulate a route for each combination of conditions. This avoids the need to change any pipelines. + +```yaml +routing: + match_once: true + default_pipelines: [ logs/default ] + table: + - condition: attributes["env"] == "prod" and attributes["region"] == "east" + pipelines: [ logs/prod, logs/east ] + - condition: attributes["env"] == "prod" and attributes["region"] == "west" + pipelines: [ logs/prod, logs/west ] + - condition: attributes["env"] == "dev" and attributes["region"] == "east" + pipelines: [ logs/dev, logs/east ] + - condition: attributes["env"] == "dev" and attributes["region"] == "west" + pipelines: [ logs/dev, logs/west ] + +service: + pipelines: + logs/in::exporters: [routing] + logs/default::receivers: [routing] + logs/prod::receivers: [routing] + logs/dev::receivers: [routing] + logs/east::receivers: [routing] + logs/west::receivers: [routing] +``` + +A more general solution is to use a layered approach. In this design, the first layer is a single router that sorts data according to whether it matches +_any route_ or _no route_. This allows the second layer to work without `default_pipelines`. The downside to this approach is that the set of conditions +in the first and second layers must be kept in sync. + +```yaml +# First layer separates logs that match no routes +routing: + match_once: true + default_pipelines: [ logs/default ] + table: # all routes forward to second layer + - condition: attributes["env"] == "prod" + pipelines: [ logs/env, logs/region ] + - condition: attributes["env"] == "dev" + pipelines: [ logs/env, logs/region ] + - condition: attributes["region"] == "east" + pipelines: [ logs/env, logs/region ] + - condition: attributes["region"] == "west" + pipelines: [ logs/env, logs/region ] + +# Second layer routes logs based on environment and region +routing/env: + match_once: true + table: + - condition: attributes["env"] == "prod" + pipelines: [ logs/prod ] + - condition: attributes["env"] == "dev" + pipelines: [ logs/dev ] +routing/region: + match_once: true + table: + - condition: attributes["region"] == "east" + pipelines: [ logs/east ] + - condition: attributes["region"] == "west" + pipelines: [ logs/west ] + +service: + pipelines: + logs/in::exporters: [routing] + logs/prod::receivers: [routing/env] + logs/dev::receivers: [routing/env] + logs/east::receivers: [routing/region] + logs/west::receivers: [routing/region] +``` + [Connectors README]:https://github.com/open-telemetry/opentelemetry-collector/blob/main/connector/README.md [OTTL]: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/ottl/README.md diff --git a/connector/routingconnector/logs.go b/connector/routingconnector/logs.go index cb1cc213b06f..d488ae79fc65 100644 --- a/connector/routingconnector/logs.go +++ b/connector/routingconnector/logs.go @@ -35,6 +35,11 @@ func newLogsConnector( ) (*logsConnector, error) { cfg := config.(*Config) + // TODO update log from warning to error in v0.116.0 + if !cfg.MatchOnce { + set.Logger.Warn("The 'match_once' field has been deprecated. Set to 'true' to suppress this warning.") + } + lr, ok := logs.(connector.LogsRouterAndConsumer) if !ok { return nil, errUnexpectedConsumer diff --git a/connector/routingconnector/metrics.go b/connector/routingconnector/metrics.go index 92bd654caa47..dfb45c953352 100644 --- a/connector/routingconnector/metrics.go +++ b/connector/routingconnector/metrics.go @@ -36,6 +36,11 @@ func newMetricsConnector( ) (*metricsConnector, error) { cfg := config.(*Config) + // TODO update log from warning to error in v0.116.0 + if !cfg.MatchOnce { + set.Logger.Warn("The 'match_once' field has been deprecated. Set to 'true' to suppress this warning.") + } + mr, ok := metrics.(connector.MetricsRouterAndConsumer) if !ok { return nil, errUnexpectedConsumer diff --git a/connector/routingconnector/traces.go b/connector/routingconnector/traces.go index d679442c7f33..aadac9615264 100644 --- a/connector/routingconnector/traces.go +++ b/connector/routingconnector/traces.go @@ -35,6 +35,11 @@ func newTracesConnector( ) (*tracesConnector, error) { cfg := config.(*Config) + // TODO update log from warning to error in v0.116.0 + if !cfg.MatchOnce { + set.Logger.Warn("The 'match_once' field has been deprecated. Set to 'true' to suppress this warning.") + } + tr, ok := traces.(connector.TracesRouterAndConsumer) if !ok { return nil, errUnexpectedConsumer