Create a skeleton for the 'blobuploadprocessor'.

.chloggen/blobuploadconnector.yaml

@@ -0,0 +1,6 @@
+change_type: new_component
+component: blobuploadconnector
+note: Connector for uploading and replacing pieces of a signal to blob storage
+issues: [33737]
+subtext: Skeleton implementation at the moment.
+change_logs: []

.github/CODEOWNERS

@@ -25,6 +25,7 @@ confmap/provider/aesprovider/                                       @open-teleme
 confmap/provider/s3provider/                                        @open-telemetry/collector-contrib-approvers @Aneurysm9
 confmap/provider/secretsmanagerprovider/                            @open-telemetry/collector-contrib-approvers @driverpt @atoulme
 
+connector/blobuploadconnector/                                      @open-telemetry/collector-contrib-approvers @michaelsafyan @dashpole
 connector/countconnector/                                           @open-telemetry/collector-contrib-approvers @djaglowski @jpkrohling
 connector/datadogconnector/                                         @open-telemetry/collector-contrib-approvers @mx-psi @dineshg13 @ankitpatel96
 connector/exceptionsconnector/                                      @open-telemetry/collector-contrib-approvers @jpkrohling @marctc

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -26,6 +26,7 @@ body:
       - confmap/provider/aesprovider
       - confmap/provider/s3provider
       - confmap/provider/secretsmanagerprovider
+      - connector/blobupload
       - connector/count
       - connector/datadog
       - connector/exceptions

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -20,6 +20,7 @@ body:
       - confmap/provider/aesprovider
       - confmap/provider/s3provider
       - confmap/provider/secretsmanagerprovider
+      - connector/blobupload
       - connector/count
       - connector/datadog
       - connector/exceptions

.github/ISSUE_TEMPLATE/other.yaml

@@ -20,6 +20,7 @@ body:
       - confmap/provider/aesprovider
       - confmap/provider/s3provider
       - confmap/provider/secretsmanagerprovider
+      - connector/blobupload
       - connector/count
       - connector/datadog
       - connector/exceptions

.github/ISSUE_TEMPLATE/unmaintained.yaml

@@ -25,6 +25,7 @@ body:
       - confmap/provider/aesprovider
       - confmap/provider/s3provider
       - confmap/provider/secretsmanagerprovider
+      - connector/blobupload
       - connector/count
       - connector/datadog
       - connector/exceptions

cmd/otelcontribcol/builder-config.yaml

@@ -220,6 +220,7 @@ receivers:
 
 connectors:
   - gomod: go.opentelemetry.io/collector/connector/forwardconnector v0.112.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/connector/blobuploadconnector v0.112.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector v0.112.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/connector/datadogconnector v0.112.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/connector/exceptionsconnector v0.112.0
@@ -451,6 +452,7 @@ replaces:
   - github.com/open-telemetry/opentelemetry-collector-contrib/exporter/syslogexporter => ../../exporter/syslogexporter
   - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest => ../../pkg/pdatatest
   - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil => ../../pkg/pdatautil
+  - github.com/open-telemetry/opentelemetry-collector-contrib/connector/blobuploadconnector => ../../connector/blobuploadconnector
   - github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector => ../../connector/countconnector
   - github.com/open-telemetry/opentelemetry-collector-contrib/connector/datadogconnector => ../../connector/datadogconnector
   - github.com/open-telemetry/opentelemetry-collector-contrib/connector/exceptionsconnector => ../../connector/exceptionsconnector

connector/blobuploadconnector/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

connector/blobuploadconnector/README.md

@@ -0,0 +1,87 @@
+# Blob Upload Connector
+
+<!-- status autogenerated section -->
+<!-- end autogenerated section -->
+
+## Status
+
+Under development.
+
+This code is a skeleton only. Follow implementation at [#33737](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/33737) ("New component: Blob Uploader Connector").
+
+## What It Does
+
+1. Queues an upload for specified pieces of a signal to a configured blob storage destination.
+2. Replaces the original data with a reference to where the blob was written.
+3. Forwards the modified signal to subsequent pipeline (to be exported per normal).
+
+## Motivation
+
+1. Handling overly large pieces of a signal that may be too big for a typical ops backend
+2. Handling sensitive information by sending to an alternative system with tighter ACLs
+
+## Config
+
+The configuration will look roughly as follows:
+
+```
+blobuploadconnector:
+  common: ...
+  traces:  ...
+  logs: ...
+```
+
+As shown above, there will be some set of common configuration that applies by default to all signal types.
+
+Each signal type will have its own config for how it will be processed.
+
+### Common Config
+
+The common configuration governs things like how to upload the data:
+
+```
+blobuploadconnector:
+  common:
+    upload:
+      queue_size: 100
+      timeout_nanos: 5000
+  ...
+```
+
+The `queue_size` indicates the maximum number of parallel uploads; the `timeout_nanos` specifies the timeout on uploading.
+
+### Traces Config
+
+The traces configuration governs how to match trace spans and span events. It will look roughly as follows:
+
+```
+blobuploadconnector:
+  traces:
+    # matching attributes on spans
+    attributes: ...
+    # matching (groups of) span events and their attributes
+    events: ...
+  ...
+```
+
+### Logs Config
+
+The logs configuration governs how to match logs. It will look roughly as follows:
+
+```
+blobuploadconnector:
+  logs:
+    groups:
+    - name: ...
+      match: ...
+      body: ...
+      attributes: ...
+  ...
+```
+
+Each named group will match logs based on `match`. For matched logs, `attributes` will govern whether/how to upload attributes while `body` will govern whether/how to upload portions of the body.
+
+## See Also
+
+- [#1428](https://github.com/open-telemetry/semantic-conventions/issues/1428) - "Seeking input on proposal for 'reference'-type attribute values"
+- [#33737](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/33737) - "New component: Blob Uploader Connector"

connector/blobuploadconnector/config.go

@@ -0,0 +1,22 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package blobuploadconnector
+
+// Config defines the overall configuration for this component.
+type Config struct {
+	Common       *CommonConfig `mapstructure:"common"`
+	TracesConfig *TracesConfig `mapstructure:"traces"`
+	LogsConfig   *LogsConfig   `mapstructure:"logs"`
+}
+
+// NewDefaultConfig instantiates a default, valid config.
+func NewDefaultConfig() *Config {
+	return &Config{}
+}
+
+// Validate ensures that the configuration is valid.
+func (c *Config) Validate() error {
+	// TODO: implement validation in subsequent PRs.
+	return nil
+}

connector/blobuploadconnector/config_common.go

@@ -0,0 +1,7 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package blobuploadconnector
+
+// CommonConfig defines configuration that applies to multiple signal types.
+type CommonConfig struct{}

connector/blobuploadconnector/config_common_test.go

@@ -0,0 +1,4 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package blobuploadconnector

connector/blobuploadconnector/config_logs.go

@@ -0,0 +1,7 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package blobuploadconnector
+
+// LogsConfig defines configuration for the logs signal type.
+type LogsConfig struct{}

connector/blobuploadconnector/config_logs_test.go

@@ -0,0 +1,4 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package blobuploadconnector

connector/blobuploadconnector/config_test.go

@@ -0,0 +1,16 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package blobuploadconnector
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDefaultConfigIsValid(t *testing.T) {
+	cfg := NewDefaultConfig()
+	assert.NotNil(t, cfg)
+	assert.NoError(t, cfg.Validate())
+}

connector/blobuploadconnector/config_traces.go

@@ -0,0 +1,7 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package blobuploadconnector
+
+// TracesConfig defines configuration for the traces signal type.
+type TracesConfig struct{}

connector/blobuploadconnector/config_traces_test.go

@@ -0,0 +1,4 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package blobuploadconnector

connector/blobuploadconnector/deps.go

@@ -0,0 +1,19 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package blobuploadconnector
+
+// Deps provides a means of mocking out dependencies in tests.
+type Deps interface {
+	depsUnexported()
+}
+
+// Default implementation of Deps.
+type depsImpl struct{}
+
+func (*depsImpl) depsUnexported() {}
+
+// NewDeps provides access to the default, real version of deps.
+func NewDeps() Deps {
+	return &depsImpl{}
+}

connector/blobuploadconnector/doc.go

@@ -0,0 +1,6 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+//go:generate mdatagen metadata.yaml
+
+package blobuploadconnector

connector/blobuploadconnector/factory.go

@@ -0,0 +1,34 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package blobuploadconnector
+
+import (
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/connector"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/connector/blobuploadconnector/internal/metadata"
+)
+
+func createDefaultConfig() component.Config {
+	return NewDefaultConfig()
+}
+
+// NewFactoryWithDeps instantiates the factory with dependency injection, allowing
+// for a more testable interface that allows global functions/objects to be swapped out.
+func NewFactoryWithDeps(deps Deps) connector.Factory {
+	return connector.NewFactory(
+		metadata.Type,
+		createDefaultConfig,
+		connector.WithTracesToTraces(
+			createTracesToTracesConnector(deps),
+			metadata.TracesToTracesStability),
+		connector.WithLogsToLogs(
+			createLogsToLogsConnector(deps),
+			metadata.LogsToLogsStability))
+}
+
+// NewFactory is used by the OTel collector to instantiate this component.
+func NewFactory() connector.Factory {
+	return NewFactoryWithDeps(NewDeps())
+}