From ecdbb535c791fe785d46240af54d9782105b3447 Mon Sep 17 00:00:00 2001 From: Ankit Patel <8731662+ankitpatel96@users.noreply.github.com> Date: Mon, 20 May 2024 13:33:02 -0700 Subject: [PATCH] Documentation improvements - Internal Architecture Doc + Package level comments (#10068) #### Documentation Creates an internal architecture file. In it is a diagram of the startup flow of the collector as well as links to key files / packages. I also added package level comments to some key packages. I wrote some other documentation in https://github.com/open-telemetry/opentelemetry-collector/pull/10029 but split the PRs up. --------- Co-authored-by: Pablo Baeyens --- CONTRIBUTING.md | 2 ++ component/component.go | 2 ++ docs/internal-architecture.md | 53 +++++++++++++++++++++++++++++++++ otelcol/collector.go | 8 +++-- service/internal/graph/graph.go | 9 ++++++ 5 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 docs/internal-architecture.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fb881c33d16..85528d9a6c3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -108,6 +108,8 @@ in advance so that maintainers can decide if the proposal is a good fit for this repository. This will help avoid situations when you spend significant time on something that maintainers may decide this repo is not the right place for. +If you're new to the Collector, the [internal architecture](docs/internal-architecture.md) documentation may be helpful. + Follow the instructions below to create your PR. ### Fork diff --git a/component/component.go b/component/component.go index c7f6408680a..794fc9235a9 100644 --- a/component/component.go +++ b/component/component.go @@ -1,6 +1,8 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 +// Package component outlines the abstraction of components within the OpenTelemetry Collector. It provides details on the component +// lifecycle as well as defining the interface that components must fulfill. package component // import "go.opentelemetry.io/collector/component" import ( diff --git a/docs/internal-architecture.md b/docs/internal-architecture.md new file mode 100644 index 00000000000..c3668901017 --- /dev/null +++ b/docs/internal-architecture.md @@ -0,0 +1,53 @@ +## Internal architecture + +This document describes the Collector internal architecture and startup flow. It can be helpful if you are starting to contribute to the Collector codebase. + +For the end-user focused architecture document, please see the [opentelemetry.io's Architecture documentation](https://opentelemetry.io/docs/collector/architecture/). While it is end user focused, it's still a good place to start if you're trying to learn about the Collector codebase. + +### Startup Diagram +```mermaid +flowchart TD + A("`**command.NewCommand**`") -->|1| B("`**updateSettingsUsingFlags**`") + A --> |2| C("`**NewCollector** + Creates and returns a new instance of Collector`") + A --> |3| D("`**Collector.Run** + Starts the collector and blocks until it shuts down`") + D --> E("`**setupConfigurationComponents**`") + E --> |1| F("`**getConfMap**`") + E ---> |2| G("`**Service.New** + Initializes telemetry, then initializes the pipelines`") + E --> |3| Q("`**Service.Start** + 1. Start all extensions. + 2. Notify extensions about Collector configuration + 3. Start all pipelines. + 4. Notify extensions that the pipeline is ready. + `") + Q --> R("`**Graph.StartAll** + Calls Start on each component in reverse topological order`") + G --> H("`**initExtensionsAndPipeline** + Creates extensions and then builds the pipeline graph`") + H --> I("`**Graph.Build** + Converts the settings to an internal graph representation`") + I --> |1| J("`**createNodes** + Builds the node objects from pipeline configuration and adds to graph. Also validates connectors`") + I --> |2| K("`**createEdges** + Iterates through the pipelines and creates edges between components`") + I --> |3| L("`**buildComponents** + Topological sort the graph, and create each component in reverse order`") + L --> M(Receiver Factory) & N(Processor Factory) & O(Exporter Factory) & P(Connector Factory) +``` +### Where to start to read the code +Here is a brief list of useful and/or important files and interfaces that you may find valuable to glance through. +Most of these have package-level documentation and function/struct-level comments that help explain the Collector! + +- [collector.go](../otelcol/collector.go) +- [graph.go](../service/internal/graph/graph.go) +- [component.go](../component/component.go) + +#### Factories +Each component type contains a `Factory` interface along with its corresponding `NewFactory` function. +Implementations of new components use this `NewFactory` function in their implementation to register key functions with +the Collector. An example of this is in [receiver.go](../receiver/receiver.go). + +For example, the Collector uses this interface to give receivers a handle to a `nextConsumer` - +which represents where the receiver will send its data next in its telemetry pipeline. diff --git a/otelcol/collector.go b/otelcol/collector.go index 3376f3a0c35..64339a4040d 100644 --- a/otelcol/collector.go +++ b/otelcol/collector.go @@ -1,8 +1,9 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -// Package service handles the command-line, configuration, and runs the -// OpenTelemetry Collector. +// Package otelcol handles the command-line, configuration, and runs the OpenTelemetry Collector. +// It contains the main [Collector] struct and its constructor [NewCollector]. +// [Collector.Run] starts the Collector and then blocks until it shuts down. package otelcol // import "go.opentelemetry.io/collector/otelcol" import ( @@ -167,7 +168,7 @@ func (col *Collector) Shutdown() { } } -// setupConfigurationComponents loads the config and starts the components. If all the steps succeeds it +// setupConfigurationComponents loads the config, creates the graph, and starts the components. If all the steps succeeds it // sets the col.service with the service currently running. func (col *Collector) setupConfigurationComponents(ctx context.Context) error { col.setCollectorState(StateStarting) @@ -282,6 +283,7 @@ func newFallbackLogger(options []zap.Option) (*zap.Logger, error) { // Consecutive calls to Run are not allowed, Run shouldn't be called once a collector is shut down. // Sets up the control logic for config reloading and shutdown. func (col *Collector) Run(ctx context.Context) error { + // setupConfigurationComponents is the "main" function responsible for startup if err := col.setupConfigurationComponents(ctx); err != nil { col.setCollectorState(StateClosed) logger, loggerErr := newFallbackLogger(col.set.LoggingOptions) diff --git a/service/internal/graph/graph.go b/service/internal/graph/graph.go index 525c269a6de..c95d1f54b7e 100644 --- a/service/internal/graph/graph.go +++ b/service/internal/graph/graph.go @@ -1,6 +1,15 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 +// Package graph contains the internal graph representation of the pipelines. +// +// [Build] is the constructor for a [Graph] object. The method calls out to helpers that transform the graph from a config +// to a DAG of components. The configuration undergoes additional validation here as well, and is used to instantiate +// the components of the pipeline. +// +// [Graph.StartAll] starts all components in each pipeline. +// +// [Graph.ShutdownAll] stops all components in each pipeline. package graph // import "go.opentelemetry.io/collector/service/internal/graph" import (