diff --git a/CHANGELOG.md b/CHANGELOG.md index f740a3dc4f60..1b3af915d7e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,11 +105,15 @@ v0.39.0 (2024-01-09) - Add `max_cache_size` to `prometheus.relabel` to allow configurability instead of hard coded 100,000. (@mattdurham) - Add support for `http_sd_config` within a `scrape_config` for prometheus to flow config conversion. (@erikbaranowski) + - `discovery.lightsail` now supports additional parameters for configuring HTTP client settings. (@ptodev) - Add `sample_age_limit` to remote_write config to drop samples older than a specified duration. (@marctc) - Handle paths in the Kubelet URL for `discovery.kubelet`. (@petewall) +- `loki.source.docker` now deduplicates targets which report the same container + ID. (@tpaschalis) + ### Bugfixes - Update `pyroscope.ebpf` to fix a logical bug causing to profile to many kthreads instead of regular processes https://github.com/grafana/pyroscope/pull/2778 (@korniltsev) diff --git a/Makefile b/Makefile index 024a624d2223..5b7d4f424759 100644 --- a/Makefile +++ b/Makefile @@ -174,7 +174,7 @@ lint: agentlint # more without -race for packages that have known race detection issues. test: $(GO_ENV) go test $(GO_FLAGS) -race $(shell go list ./... | grep -v /integration-tests/) - $(GO_ENV) go test $(GO_FLAGS) ./pkg/integrations/node_exporter ./pkg/logs ./pkg/operator ./pkg/util/k8s ./component/otelcol/processor/tail_sampling ./component/loki/source/file + $(GO_ENV) go test $(GO_FLAGS) ./pkg/integrations/node_exporter ./pkg/logs ./pkg/operator ./pkg/util/k8s ./component/otelcol/processor/tail_sampling ./component/loki/source/file ./component/loki/source/docker test-packages: docker pull $(BUILD_IMAGE) diff --git a/component/loki/source/docker/docker.go b/component/loki/source/docker/docker.go index 400b1d30b5e6..193f27f1d7a2 100644 --- a/component/loki/source/docker/docker.go +++ b/component/loki/source/docker/docker.go @@ -215,6 +215,7 @@ func (c *Component) Update(args component.Arguments) error { // Convert input targets into targets to give to tailer. targets := make([]*dt.Target, 0, len(newArgs.Targets)) + seenTargets := make(map[string]struct{}, len(newArgs.Targets)) for _, target := range newArgs.Targets { containerID, ok := target[dockerLabelContainerID] @@ -222,6 +223,10 @@ func (c *Component) Update(args component.Arguments) error { level.Debug(c.opts.Logger).Log("msg", "docker target did not include container ID label:"+dockerLabelContainerID) continue } + if _, seen := seenTargets[containerID]; seen { + continue + } + seenTargets[containerID] = struct{}{} var labels = make(model.LabelSet) for k, v := range target { diff --git a/component/loki/source/docker/docker_test.go b/component/loki/source/docker/docker_test.go index 51c2a4568cff..c4b99c47388c 100644 --- a/component/loki/source/docker/docker_test.go +++ b/component/loki/source/docker/docker_test.go @@ -1,3 +1,5 @@ +//go:build !race + package docker import ( @@ -5,9 +7,11 @@ import ( "testing" "time" + "github.com/grafana/agent/component" "github.com/grafana/agent/pkg/flow/componenttest" "github.com/grafana/agent/pkg/util" "github.com/grafana/river" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/require" ) @@ -33,3 +37,39 @@ func Test(t *testing.T) { require.NoError(t, ctrl.WaitRunning(time.Minute)) } + +func TestDuplicateTargets(t *testing.T) { + // Use host that works on all platforms (including Windows). + var cfg = ` + host = "tcp://127.0.0.1:9376" + targets = [ + {__meta_docker_container_id = "foo", __meta_docker_port_private = "8080"}, + {__meta_docker_container_id = "foo", __meta_docker_port_private = "8081"}, + ] + forward_to = [] + ` + + var args Arguments + err := river.Unmarshal([]byte(cfg), &args) + require.NoError(t, err) + + ctrl, err := componenttest.NewControllerFromID(util.TestLogger(t), "loki.source.docker") + require.NoError(t, err) + + go func() { + err := ctrl.Run(context.Background(), args) + require.NoError(t, err) + }() + + require.NoError(t, ctrl.WaitRunning(time.Minute)) + + cmp, err := New(component.Options{ + ID: "loki.source.docker.test", + Logger: util.TestFlowLogger(t), + Registerer: prometheus.NewRegistry(), + DataPath: t.TempDir(), + }, args) + require.NoError(t, err) + + require.Len(t, cmp.manager.tasks, 1) +} diff --git a/docs/sources/flow/reference/components/loki.source.docker.md b/docs/sources/flow/reference/components/loki.source.docker.md index cbf77163d646..3dd327f70a80 100644 --- a/docs/sources/flow/reference/components/loki.source.docker.md +++ b/docs/sources/flow/reference/components/loki.source.docker.md @@ -131,6 +131,14 @@ fully qualified name) to store its _positions file_. The positions file stores the read offsets so that if there is a component or Agent restart, `loki.source.docker` can pick up tailing from the same spot. +If the target's argument contains multiple entries with the same container +ID (for example as a result of `discovery.docker` picking up multiple exposed +ports or networks), `loki.source.docker` will deduplicate them, and only keep +the first of each container ID instances, based on the +`__meta_docker_container_id` label. As such, the Docker daemon is queried +for each container ID only once, and only one target will be available in the +component's debug info. + ## Example This example collects log entries from the files specified in the `targets`