Create a skeleton for the 'blobuploadprocessor'.

.chloggen/blobuploadprocessor.yaml

@@ -0,0 +1,6 @@
+change_type: new_component
+component: blobuploadprocessor
+note: Processor for uploading pieces of a signal to blob storage and replacing with a ref
+issues: [33737]
+subtext: Skeleton implementation at the moment.
+change_logs: []

.github/CODEOWNERS

@@ -171,6 +171,7 @@ pkg/translator/zipkin/                                              @open-teleme
 pkg/winperfcounters/                                                @open-telemetry/collector-contrib-approvers @dashpole @Mrod1598 @alxbl @pjanotti
 
 processor/attributesprocessor/                                      @open-telemetry/collector-contrib-approvers @boostchicken
+connector/blobuploadprocessor/                                      @open-telemetry/collector-contrib-approvers @michaelsafyan @dashpole @braydonk
 processor/coralogixprocessor/                                       @open-telemetry/collector-contrib-approvers @crobert-1 @galrose
 processor/cumulativetodeltaprocessor/                               @open-telemetry/collector-contrib-approvers @TylerHelmuth
 processor/deltatocumulativeprocessor/                               @open-telemetry/collector-contrib-approvers @sh0rez @RichieSams @jpkrohling

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -166,6 +166,7 @@ body:
       - pkg/translator/zipkin
       - pkg/winperfcounters
       - processor/attributes
+      - processor/blobupload
       - processor/coralogix
       - processor/cumulativetodelta
       - processor/deltatocumulative

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -20,6 +20,7 @@ body:
       - confmap/provider/aesprovider
       - confmap/provider/s3provider
       - confmap/provider/secretsmanagerprovider
+      - connector/blobupload
       - connector/count
       - connector/datadog
       - connector/exceptions

.github/ISSUE_TEMPLATE/other.yaml

@@ -20,6 +20,7 @@ body:
       - confmap/provider/aesprovider
       - confmap/provider/s3provider
       - confmap/provider/secretsmanagerprovider
+      - connector/blobupload
       - connector/count
       - connector/datadog
       - connector/exceptions

.github/ISSUE_TEMPLATE/unmaintained.yaml

@@ -25,6 +25,7 @@ body:
       - confmap/provider/aesprovider
       - confmap/provider/s3provider
       - confmap/provider/secretsmanagerprovider
+      - connector/blobupload
       - connector/count
       - connector/datadog
       - connector/exceptions

cmd/otelcontribcol/builder-config.yaml

@@ -102,6 +102,7 @@ processors:
   - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.112.0
   - gomod: go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.112.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.112.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/blobuploadprocessor v0.112.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/cumulativetodeltaprocessor v0.112.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor v0.112.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatorateprocessor v0.112.0
@@ -352,6 +353,7 @@ replaces:
   - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/elasticsearchreceiver => ../../receiver/elasticsearchreceiver
   - github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricsgenerationprocessor => ../../processor/metricsgenerationprocessor
   - github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor => ../../processor/attributesprocessor
+  - github.com/open-telemetry/opentelemetry-collector-contrib/processor/blobuploadprocessor => ../../processor/blobuploadprocessor
   - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/sqlqueryreceiver => ../../receiver/sqlqueryreceiver
   - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/purefareceiver => ../../receiver/purefareceiver
   - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/purefbreceiver => ../../receiver/purefbreceiver

processor/blobuploadprocessor/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

processor/blobuploadprocessor/README.md

@@ -0,0 +1,96 @@
+# Blob Upload Connector
+
+<!-- status autogenerated section -->
+| Status        |           |
+| ------------- |-----------|
+| Stability     | [development]: traces, logs   |
+| Distributions | [contrib] |
+| Issues        | [![Open issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aopen%20label%3Aprocessor%2Fblobupload%20&label=open&color=orange&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aopen+is%3Aissue+label%3Aprocessor%2Fblobupload) [![Closed issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aclosed%20label%3Aprocessor%2Fblobupload%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aclosed+is%3Aissue+label%3Aprocessor%2Fblobupload) |
+| [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner)    | [@michaelsafyan](https://www.github.com/michaelsafyan), [@dashpole](https://www.github.com/dashpole), [@braydonk](https://www.github.com/braydonk) |
+
+[development]: https://github.com/open-telemetry/opentelemetry-collector#development
+[contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib
+<!-- end autogenerated section -->
+
+## Status
+
+Under development.
+
+This code is a skeleton only. Follow implementation at [#33737](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/33737) ("New component: Blob Uploader Connector").
+
+## What It Does
+
+1. Queues an upload for specified pieces of a signal to a configured blob storage destination.
+2. Replaces the original data with a reference to where the blob was written.
+3. Forwards the modified signal to subsequent pipeline (to be exported per normal).
+
+## Motivation
+
+1. Handling overly large pieces of a signal that may be too big for a typical ops backend
+2. Handling sensitive information by sending to an alternative system with tighter ACLs
+
+## Config
+
+The configuration will look roughly as follows:
+
+```
+blobuploadconnector:
+  common: ...
+  traces:  ...
+  logs: ...
+```
+
+As shown above, there will be some set of common configuration that applies by default to all signal types.
+
+Each signal type will have its own config for how it will be processed.
+
+### Common Config
+
+The common configuration governs things like how to upload the data:
+
+```
+blobuploadconnector:
+  common:
+    upload:
+      queue_size: 100
+      timeout_nanos: 5000
+  ...
+```
+
+The `queue_size` indicates the maximum number of parallel uploads; the `timeout_nanos` specifies the timeout on uploading.
+
+### Traces Config
+
+The traces configuration governs how to match trace spans and span events. It will look roughly as follows:
+
+```
+blobuploadconnector:
+  traces:
+    # matching attributes on spans
+    attributes: ...
+    # matching (groups of) span events and their attributes
+    events: ...
+  ...
+```
+
+### Logs Config
+
+The logs configuration governs how to match logs. It will look roughly as follows:
+
+```
+blobuploadconnector:
+  logs:
+    groups:
+    - name: ...
+      match: ...
+      body: ...
+      attributes: ...
+  ...
+```
+
+Each named group will match logs based on `match`. For matched logs, `attributes` will govern whether/how to upload attributes while `body` will govern whether/how to upload portions of the body.
+
+## See Also
+
+- [#1428](https://github.com/open-telemetry/semantic-conventions/issues/1428) - "Seeking input on proposal for 'reference'-type attribute values"
+- [#33737](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/33737) - "New component: Blob Uploader Connector"

processor/blobuploadprocessor/config.go

@@ -0,0 +1,31 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package blobuploadprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/blobuploadprocessor"
+
+import (
+	"go.opentelemetry.io/collector/component"
+)
+
+// Config defines the overall configuration for this component.
+type Config struct {
+	Common       *CommonConfig `mapstructure:"common"`
+	TracesConfig *TracesConfig `mapstructure:"traces"`
+	LogsConfig   *LogsConfig   `mapstructure:"logs"`
+}
+
+// createTypedDefaultConfig instantiates a default, valid config.
+func createTypedDefaultConfig() *Config {
+	return &Config{}
+}
+
+// createDefaultConfig erases the type to allow use in component.
+func createDefaultConfig() component.Config {
+	return createTypedDefaultConfig()
+}
+
+// Validate ensures that the configuration is valid.
+func (c *Config) Validate() error {
+	// TODO: implement validation in subsequent PRs.
+	return nil
+}

processor/blobuploadprocessor/config_common.go

@@ -0,0 +1,7 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package blobuploadprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/blobuploadprocessor"
+
+// CommonConfig defines configuration that applies to multiple signal types.
+type CommonConfig struct{}

processor/blobuploadprocessor/config_common_test.go

@@ -0,0 +1,4 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package blobuploadprocessor

processor/blobuploadprocessor/config_logs.go

@@ -0,0 +1,7 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package blobuploadprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/blobuploadprocessor"
+
+// LogsConfig defines configuration for the logs signal type.
+type LogsConfig struct{}

processor/blobuploadprocessor/config_logs_test.go

@@ -0,0 +1,4 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package blobuploadprocessor

processor/blobuploadprocessor/config_test.go

@@ -0,0 +1,16 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package blobuploadprocessor
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDefaultConfigIsValid(t *testing.T) {
+	cfg := createTypedDefaultConfig()
+	assert.NotNil(t, cfg)
+	assert.NoError(t, cfg.Validate())
+}

processor/blobuploadprocessor/config_traces.go

@@ -0,0 +1,7 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package blobuploadprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/blobuploadprocessor"
+
+// TracesConfig defines configuration for the traces signal type.
+type TracesConfig struct{}

processor/blobuploadprocessor/config_traces_test.go

@@ -0,0 +1,4 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package blobuploadprocessor

processor/blobuploadprocessor/deps.go

@@ -0,0 +1,19 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package blobuploadprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/blobuploadprocessor"
+
+// Deps provides a means of mocking out dependencies in tests.
+type deps interface {
+	depsUnexported()
+}
+
+// Default implementation of deps.
+type depsImpl struct{}
+
+func (*depsImpl) depsUnexported() {}
+
+// NewDeps provides access to the default, real version of deps.
+func newDeps() deps {
+	return &depsImpl{}
+}

processor/blobuploadprocessor/doc.go

@@ -0,0 +1,6 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+//go:generate mdatagen metadata.yaml
+
+package blobuploadprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/blobuploadprocessor"

processor/blobuploadprocessor/factory.go

@@ -0,0 +1,29 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package blobuploadprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/blobuploadprocessor"
+
+import (
+	"go.opentelemetry.io/collector/processor"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/processor/blobuploadprocessor/internal/metadata"
+)
+
+// NewFactoryWithDeps instantiates the factory with dependency injection, allowing
+// for a more testable interface that allows global functions/objects to be swapped out.
+func newFactoryWithDeps(d deps) processor.Factory {
+	return processor.NewFactory(
+		metadata.Type,
+		createDefaultConfig,
+		processor.WithTraces(
+			createTracesProcessor(d),
+			metadata.TracesStability),
+		processor.WithLogs(
+			createLogsProcessor(d),
+			metadata.LogsStability))
+}
+
+// NewFactory is used by the OTel collector to instantiate this component.
+func NewFactory() processor.Factory {
+	return newFactoryWithDeps(newDeps())
+}