loki.source.doker: port latest Promtail changes (#5570)

Signed-off-by: Paschalis Tsilias <[email protected]>
Co-authored-by: Piotr <[email protected]>
Co-authored-by: Clayton Cornell <[email protected]>

CHANGELOG.md

@@ -71,6 +71,9 @@ Main (unreleased)
 
 - Improved performance of `pyroscope.scrape` component when working with a large number of targets. (@cyriltovena)
 
+- The `loki.source.docker` component now allows connecting to Docker daemons
+  over HTTP(S) and setting up TLS credentials. (@tpaschalis)
+
 v0.37.2 (2023-10-16)
 -----------------
 

component/loki/source/docker/docker.go

@@ -1,8 +1,12 @@
 package docker
 
+// NOTE: This code is adapted from Promtail (90a1d4593e2d690b37333386383870865fe177bf).
+
 import (
 	"context"
 	"fmt"
+	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -12,12 +16,15 @@ import (
 	"github.com/docker/docker/client"
 	"github.com/go-kit/log"
 	"github.com/grafana/agent/component"
+	types "github.com/grafana/agent/component/common/config"
 	"github.com/grafana/agent/component/common/loki"
 	"github.com/grafana/agent/component/common/loki/positions"
 	flow_relabel "github.com/grafana/agent/component/common/relabel"
 	"github.com/grafana/agent/component/discovery"
 	dt "github.com/grafana/agent/component/loki/source/docker/internal/dockertarget"
+	"github.com/grafana/agent/pkg/build"
 	"github.com/grafana/agent/pkg/flow/logging/level"
+	"github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/model/relabel"
 )
@@ -33,6 +40,8 @@ func init() {
 	})
 }
 
+var userAgent = fmt.Sprintf("GrafanaAgent/%s", build.Version)
+
 const (
 	dockerLabel                = model.MetaLabelPrefix + "docker_"
 	dockerLabelContainerPrefix = dockerLabel + "container_"
@@ -42,11 +51,43 @@ const (
 // Arguments holds values which are used to configure the loki.source.docker
 // component.
 type Arguments struct {
-	Host         string              `river:"host,attr"`
-	Targets      []discovery.Target  `river:"targets,attr"`
-	ForwardTo    []loki.LogsReceiver `river:"forward_to,attr"`
-	Labels       map[string]string   `river:"labels,attr,optional"`
-	RelabelRules flow_relabel.Rules  `river:"relabel_rules,attr,optional"`
+	Host             string                  `river:"host,attr"`
+	Targets          []discovery.Target      `river:"targets,attr"`
+	ForwardTo        []loki.LogsReceiver     `river:"forward_to,attr"`
+	Labels           map[string]string       `river:"labels,attr,optional"`
+	RelabelRules     flow_relabel.Rules      `river:"relabel_rules,attr,optional"`
+	HTTPClientConfig *types.HTTPClientConfig `river:"http_client_config,block,optional"`
+	RefreshInterval  time.Duration           `river:"refresh_interval,attr,optional"`
+}
+
+// GetDefaultArguments return an instance of Arguments with the optional fields
+// initialized.
+func GetDefaultArguments() Arguments {
+	return Arguments{
+		HTTPClientConfig: types.CloneDefaultHTTPClientConfig(),
+		RefreshInterval:  60 * time.Second,
+	}
+}
+
+// SetToDefault implements river.Defaulter.
+func (a *Arguments) SetToDefault() {
+	*a = GetDefaultArguments()
+}
+
+// Validate implements river.Validator.
+func (a *Arguments) Validate() error {
+	if _, err := url.Parse(a.Host); err != nil {
+		return fmt.Errorf("failed to parse Docker host %q: %w", a.Host, err)
+	}
+	// We must explicitly Validate because HTTPClientConfig is squashed and it won't run otherwise
+	if a.HTTPClientConfig != nil {
+		if a.RefreshInterval <= 0 {
+			return fmt.Errorf("refresh_interval must be positive, got %q", a.RefreshInterval)
+		}
+		return a.HTTPClientConfig.Validate()
+	}
+
+	return nil
 }
 
 var (
@@ -220,10 +261,36 @@ func (c *Component) getManagerOptions(args Arguments) (*options, error) {
 		return c.lastOptions, nil
 	}
 
+	hostURL, err := url.Parse(args.Host)
+	if err != nil {
+		return c.lastOptions, err
+	}
+
 	opts := []client.Opt{
 		client.WithHost(args.Host),
 		client.WithAPIVersionNegotiation(),
 	}
+
+	// There are other protocols than HTTP supported by the Docker daemon, like
+	// unix, which are not supported by the HTTP client. Passing HTTP client
+	// options to the Docker client makes those non-HTTP requests fail.
+	if hostURL.Scheme == "http" || hostURL.Scheme == "https" {
+		rt, err := config.NewRoundTripperFromConfig(*args.HTTPClientConfig.Convert(), "docker_sd")
+		if err != nil {
+			return c.lastOptions, err
+		}
+		opts = append(opts,
+			client.WithHTTPClient(&http.Client{
+				Transport: rt,
+				Timeout:   args.RefreshInterval,
+			}),
+			client.WithScheme(hostURL.Scheme),
+			client.WithHTTPHeaders(map[string]string{
+				"User-Agent": userAgent,
+			}),
+		)
+	}
+
 	client, err := client.NewClientWithOpts(opts...)
 	if err != nil {
 		level.Error(c.opts.Logger).Log("msg", "could not create new Docker client", "err", err)
@@ -241,7 +308,7 @@ func (c *Component) getManagerOptions(args Arguments) (*options, error) {
 func (c *Component) DebugInfo() interface{} {
 	var res readerDebugInfo
 	for _, tgt := range c.manager.targets() {
-		details := tgt.Details().(map[string]string)
+		details := tgt.Details()
 		res.TargetsInfo = append(res.TargetsInfo, targetInfo{
 			Labels:     tgt.LabelsStr(),
 			ID:         details["id"],

component/loki/source/docker/internal/dockertarget/metrics.go

@@ -1,8 +1,8 @@
 package dockertarget
 
-// This code is copied from Promtail. The dockertarget package is used to
-// configure and run the targets that can read logs from Docker containers and
-// forward them to other loki components.
+// NOTE: This code is adapted from Promtail (90a1d4593e2d690b37333386383870865fe177bf).
+// The dockertarget package is used to configure and run the targets that can
+// read logs from Docker containers and forward them to other loki components.
 
 import "github.com/prometheus/client_golang/prometheus"
 

component/loki/source/docker/internal/dockertarget/target.go

@@ -1,8 +1,8 @@
 package dockertarget
 
-// This code is copied from Promtail. The dockertarget package is used to
-// configure and run the targets that can read logs from Docker containers and
-// forward them to other loki components.
+// NOTE: This code is adapted from Promtail (90a1d4593e2d690b37333386383870865fe177bf).
+// The dockertarget package is used to configure and run the targets that can
+// read logs from Docker containers and forward them to other loki components.
 
 import (
 	"bufio"
@@ -261,12 +261,13 @@ func (t *Target) Hash() uint64 {
 	return uint64(t.labels.Fingerprint())
 }
 
+// Path returns the target's container name.
 func (t *Target) Path() string {
 	return t.containerName
 }
 
 // Details returns target-specific details.
-func (t *Target) Details() interface{} {
+func (t *Target) Details() map[string]string {
 	var errMsg string
 	if t.err != nil {
 		errMsg = t.err.Error()

component/loki/source/docker/internal/dockertarget/target_test.go

@@ -1,8 +1,8 @@
 package dockertarget
 
-// This code is copied from Promtail. The dockertarget package is used to
-// configure and run the targets that can read logs from Docker containers and
-// forward them to other loki components.
+// NOTE: This code is adapted from Promtail (90a1d4593e2d690b37333386383870865fe177bf).
+// The dockertarget package is used to configure and run the targets that can
+// read logs from Docker containers and forward them to other loki components.
 
 import (
 	"encoding/json"

component/loki/source/docker/runner.go

@@ -1,5 +1,7 @@
 package docker
 
+// NOTE: This code is adapted from Promtail (90a1d4593e2d690b37333386383870865fe177bf).
+
 import (
 	"context"
 	"sync"

docs/sources/flow/reference/components/loki.source.docker.md

@@ -41,11 +41,67 @@ Name            | Type                 | Description          | Default | Requir
 `forward_to`    | `list(LogsReceiver)` | List of receivers to send log entries to. | | yes
 `labels`        | `map(string)`        | The default set of labels to apply on entries. | `"{}"` | no
 `relabel_rules` | `RelabelRules`       | Relabeling rules to apply on log entries. | `"{}"` | no
+`refresh_interval` | `duration`        | The refresh interval to use when connecting to the Docker daemon over HTTP(S). | `"60s"` | no
 
 ## Blocks
 
-The `loki.source.docker` component doesn't support any inner blocks and is
-configured fully through arguments.
+The following blocks are supported inside the definition of `loki.source.docker`:
+
+Hierarchy | Block | Description | Required
+--------- | ----- | ----------- | --------
+client | [client][] | HTTP client settings when connecting to the endpoint. | no
+client > basic_auth | [basic_auth][] | Configure basic_auth for authenticating to the endpoint. | no
+client > authorization | [authorization][] | Configure generic authorization to the endpoint. | no
+client > oauth2 | [oauth2][] | Configure OAuth2 for authenticating to the endpoint. | no
+client > oauth2 > tls_config | [tls_config][] | Configure TLS settings for connecting to the endpoint. | no
+client > tls_config | [tls_config][] | Configure TLS settings for connecting to the endpoint. | no
+
+The `>` symbol indicates deeper levels of nesting. For example, `client >
+basic_auth` refers to an `basic_auth` block defined inside a `client` block.
+
+These blocks are only applicable when connecting to a Docker daemon over HTTP
+or HTTPS and has no effect when connecting via a `unix:///` socket
+
+[client]: #client-block
+[basic_auth]: #basic_auth-block
+[authorization]: #authorization-block
+[oauth2]: #oauth2-block
+[tls_config]: #tls_config-block
+
+### client block
+
+The `client` block configures settings used to connect to HTTP(S) Docker
+daemons.
+
+{{< docs/shared lookup="flow/reference/components/http-client-config-block.md" source="agent" version="<AGENT_VERSION>" >}}
+
+### basic_auth block
+
+The `basic_auth` block configures basic authentication for HTTP(S) Docker
+daemons.
+
+{{< docs/shared lookup="flow/reference/components/basic-auth-block.md" source="agent" version="<AGENT_VERSION>" >}}
+
+### authorization block
+
+The `authorization` block configures custom authorization to use for the Docker
+daemon.
+
+{{< docs/shared lookup="flow/reference/components/authorization-block.md" source="agent" version="<AGENT_VERSION>" >}}
+
+### oauth2 block
+
+The `oauth2` block configures OAuth2 authorization to use for the Docker
+daemon.
+
+{{< docs/shared lookup="flow/reference/components/oauth2-block.md" source="agent" version="<AGENT_VERSION>" >}}
+
+### tls_config block
+
+The `tls_config` block configures TLS settings for connecting to HTTPS Docker
+daemons.
+
+{{< docs/shared lookup="flow/reference/components/tls-config-block.md" source="agent" version="<AGENT_VERSION>" >}}
 
 ## Exported fields