Merge branch 'main' into configurable-default-instance-label-for-scra…

…ped-k8s-logs

.github/workflows/trivy.yml

@@ -26,7 +26,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@d710430a6722f083d3b36b8339ff66b32f22ee55
+        uses: aquasecurity/trivy-action@b2933f565dbc598b29947660e66259e3c7bc8561
         with:
           image-ref: 'grafana/agent:main'
           format: 'template'

CHANGELOG.md

@@ -10,6 +10,9 @@ internal API changes are not present.
 Main (unreleased)
 -----------------
 
+v0.41.0 (2024-05-31)
+--------------------
+
 ### Breaking changes
 
 - The default listen port for `otelcol.receiver.opencensus` has changed from
@@ -18,20 +21,6 @@ Main (unreleased)
 - The default sync interval for `mimir.rules.kubernetes` has changed from `30s`
   to `5m` to reduce load on Mimir. (@56quarters)
 
-- `prometheus.exporter.postgres` has been updated to the latest upstream
-  version which changes the set of exported metrics. The following metrics were
-  removed: `pg_stat_database_session_time`, `pg_stat_database_sessions`,
-  `pg_stat_database_sessions_abandoned`, `pg_stat_database_sessions_fatal`,
-  `pg_stat_database_sessions_killed`, `pg_stat_database_idle_in_transaction_time`,
-  `pg_stat_database_checksum_failures`, `pg_stat_database_checksum_last_failure`,
-  `pg_stat_database_active_time`. The following metrics were
-  renamed: `pg_stat_bgwriter_buffers_alloc`, `pg_stat_bgwriter_buffers_backend`,
-  `pg_stat_bgwriter_buffers_backend_fsync`, `pg_stat_bgwriter_buffers_checkpoint`,
-  `pg_stat_bgwriter_buffers_clean`, `pg_stat_bgwriter_checkpoint_sync_time`,
-  `pg_stat_bgwriter_checkpoint_write_time`, `pg_stat_bgwriter_checkpoints_req`,
-  `pg_stat_bgwriter_checkpoints_timed`, `pg_stat_bgwriter_maxwritten_clean`,
-  `pg_stat_bgwriter_stats_reset` - the new names include the `_total` suffix. (@thampiotr)
-
 ### Enhancements
 
 - Add support for importing folders as single module to `import.file`. (@wildum)
@@ -44,6 +33,8 @@ Main (unreleased)
 - Increased the alert interval and renamed the `ClusterSplitBrain` alert to `ClusterNodeCountMismatch` in the Grafana
   Agent Mixin to better match the alert conditions. (@thampiotr)
 
+- Not restart tailers in `loki.source.kubernetes` component by above-average time deltas if K8s version is >= 1.29.1 (@hainenber)
+
 - Add conversion from static to flow mode for `loki.source.windowsevent` via `legacy_bookmark_path`. (@mattdurham)
 
 - Add ability to convert static mode positions file to `loki.source.file` compatible via `legacy_positions_file` argument. (@mattdurham)
@@ -62,19 +53,19 @@ Main (unreleased)
 
 - A new `loki.rules.kubernetes` component that discovers `PrometheusRule` Kubernetes resources and loads them into a Loki Ruler instance. (@EStork09)
 
-- Add `beyla.ebpf` component to automatically instrument services with eBPF. (@marctc)
+- A new `snmp_context` configuration argument for `prometheus.exporter.snmp` and the `snmp` Static mode integration.
+  It overrides the `context_name` parameter in the SNMP configuration file. (@ptodev)
 
 ### Bugfixes
 
-- Fix an issue where JSON string array elements were not parsed correctly in `loki.source.cloudflare`. (@thampiotr)
+- Fix panic for `prometheus.exporter.snmp` and snmp_exporter integration
+  introduced in v0.40.5 with a version upgrade. This was due to an
+  uninitialized new metric for the exporter. (@erikbaranowski)
 
+- Fix an issue where JSON string array elements were not parsed correctly in `loki.source.cloudflare`. (@thampiotr)
 
 - Fix SSRF vulnerability in `faro.receiver` by disabling source map download. (@hainenber)
 
-- Fix an issue where the azure exporter was not correctly gathering subscription scoped metrics when only one region was configured (@kgeckhart)
-
-- Update gcp_exporter to a newer version with a patch for incorrect delta histograms (@kgeckhart)
-
 - Fix an issue where the default values of some component's arguments change
   whenever that argument is explicitly configured. This issue only affected a
   small subset of arguments across 15 components. (@erikbaranowski, @rfratto)
@@ -85,8 +76,10 @@ Main (unreleased)
 
 - The `import.git` config block did not work with branches or tags this now fixes that behavior. (@mattdurham)
 
-- Fixed an issue where creating a `prometheus.exporter.postgres` component with
-  multiple `data_source_names` would result in an error. (@thampiotr)
+- Fix an issue on Windows where uninstalling Alloy did not remove it from the
+  Add/Remove programs list. (@rfratto)
+
+- Fix a bug where a topic was claimed by the wrong consumer type in `otelcol.receiver.kafka`. (@wildum)
 
 ### Other changes
 
@@ -96,6 +89,68 @@ Main (unreleased)
 
 - Resync defaults for `otelcol.exporter.otlp` and `otelcol.exporter.otlphttp` with upstream. (@hainenber)
 
+v0.40.5 (2024-05-15)
+--------------------
+
+### Breaking changes
+
+- `prometheus.exporter.postgres` has been updated to the latest upstream
+  version which changes the set of exported metrics. The following metrics were
+  removed: `pg_stat_database_session_time`, `pg_stat_database_sessions`,
+  `pg_stat_database_sessions_abandoned`, `pg_stat_database_sessions_fatal`,
+  `pg_stat_database_sessions_killed`, `pg_stat_database_idle_in_transaction_time`,
+  `pg_stat_database_checksum_failures`, `pg_stat_database_checksum_last_failure`,
+  `pg_stat_database_active_time`. The following metrics were
+  renamed: `pg_stat_bgwriter_buffers_alloc`, `pg_stat_bgwriter_buffers_backend`,
+  `pg_stat_bgwriter_buffers_backend_fsync`, `pg_stat_bgwriter_buffers_checkpoint`,
+  `pg_stat_bgwriter_buffers_clean`, `pg_stat_bgwriter_checkpoint_sync_time`,
+  `pg_stat_bgwriter_checkpoint_write_time`, `pg_stat_bgwriter_checkpoints_req`,
+  `pg_stat_bgwriter_checkpoints_timed`, `pg_stat_bgwriter_maxwritten_clean`,
+  `pg_stat_bgwriter_stats_reset` - the new names include the `_total` suffix. (@thampiotr)
+
+### Bugfixes
+
+- Fix an issue where the azure exporter was not correctly gathering subscription scoped metrics when only one region was configured (@kgeckhart)
+
+- Fixed an issue where creating a `prometheus.exporter.postgres` component with
+  multiple `data_source_names` would result in an error. (@thampiotr)
+
+- Fix a bug with the logs pipeline in static mode which prevented it from shutting down cleanly.
+
+### Other changes
+
+- Updating SNMP exporter from v0.24.1 to v0.26.0.
+
+v0.40.4 (2024-04-12)
+--------------------
+
+### Security fixes
+
+- Fixes following vulnerabilities (@ptodev)
+  * [CVE-2024-27304](https://github.com/advisories/GHSA-mrww-27vc-gghv)
+  * [CVE-2024-27289](https://github.com/advisories/GHSA-m7wr-2xf7-cm9p)
+  * [CVE-2024-28180](https://github.com/advisories/GHSA-c5q2-7r4c-mv6g)
+  * [CVE-2024-24786](https://github.com/advisories/GHSA-8r3f-844c-mc37)
+
+### Enhancements
+
+- Update `prometheus.exporter.kafka` with the following functionalities (@wildum):
+  * GSSAPI config
+  * enable/disable PA_FX_FAST
+  * set a TLS server name
+  * show the offset/lag for all consumer group or only the connected ones
+  * set the minimum number of topics to monitor
+  * enable/disable auto-creation of requested topics if they don't already exist
+  * regex to exclude topics / groups 
+  * added metric kafka_broker_info
+
+- In `prometheus.exporter.kafka`, the interpolation table used to compute estimated lag metrics is now pruned
+  on `metadata_refresh_interval` instead of `prune_interval_seconds`. (@wildum)
+
+### Bugfixes
+
+- Update gcp_exporter to a newer version with a patch for incorrect delta histograms (@kgeckhart)
+
 v0.40.3 (2024-03-14)
 --------------------
 

CODEOWNERS

@@ -18,5 +18,4 @@
 /docs/sources/ @clayton-cornell
 
 # Components:
-/internal/component/pyroscope/ @grafana/grafana-agent-profiling-maintainers
-/internal/component/beyla/     @marctc
+/component/pyroscope/ @grafana/grafana-agent-profiling-maintainers

Makefile

@@ -163,7 +163,7 @@ lint: agentlint
 # more without -race for packages that have known race detection issues.
 test:
 	$(GO_ENV) go test $(GO_FLAGS) -race $(shell go list ./... | grep -v /integration-tests/)
-	$(GO_ENV) go test $(GO_FLAGS) ./internal/static/integrations/node_exporter ./internal/static/logs ./internal/static/operator ./internal/util/k8s ./internal/component/otelcol/processor/tail_sampling ./internal/component/loki/source/file ./internal/component/loki/source/docker
+	$(GO_ENV) go test $(GO_FLAGS) ./static/integrations/node_exporter ./static/logs ./static/operator ./internal/util/k8s ./internal/component/otelcol/processor/tail_sampling ./internal/component/loki/source/file ./internal/component/loki/source/docker
 
 test-packages:
 	docker pull $(BUILD_IMAGE)
@@ -272,7 +272,7 @@ ifeq ($(USE_CONTAINER),1)
 	$(RERUN_IN_CONTAINER)
 else
 	bash ./tools/generate-crds.bash
-	gen-crd-api-reference-docs -config tools/gen-crd-docs/config.json -api-dir "github.com/grafana/agent/internal/static/operator/apis/monitoring/" -out-file docs/sources/operator/api.md -template-dir tools/gen-crd-docs/template
+	gen-crd-api-reference-docs -config tools/gen-crd-docs/config.json -api-dir "github.com/grafana/agent/static/operator/apis/monitoring/" -out-file docs/sources/operator/api.md -template-dir tools/gen-crd-docs/template
 endif
 
 generate-drone:
@@ -304,7 +304,7 @@ generate-protos:
 ifeq ($(USE_CONTAINER),1)
 	$(RERUN_IN_CONTAINER)
 else
-	go generate ./internal/static/agentproto/
+	go generate ./static/agentproto/
 endif
 
 generate-ui:

README.md

@@ -1,3 +1,16 @@
+> **Deprecation notice**: Grafana Agent has been deprecated and is now in
+> Long-Term Support mode. We recommend migrating to the new [Grafana Alloy]
+> collector, which is built on the foundation of Grafana Agent Flow.
+>
+> For more information, read our blog posts about Alloy and how to easily
+> migrate from Agent to Alloy:
+>
+> * [Alloy announcement blog post](https://grafana.com/blog/2024/04/09/grafana-alloy-opentelemetry-collector-with-prometheus-pipelines/)
+> * [Alloy FAQ](https://grafana.com/blog/2024/04/09/grafana-agent-to-grafana-alloy-opentelemetry-collector-faq/)
+> * [Migrate to Alloy](https://grafana.com/docs/alloy/latest/tasks/migrate/)
+>
+> [Grafana Alloy]: https://github.com/grafana/alloy
+
 <p align="center"><img src="docs/sources/assets/logo_and_name.png" alt="Grafana Agent logo"></p>
 
 Grafana Agent is an OpenTelemetry Collector distribution with configuration

cmd/grafana-agent-flow/main.go

@@ -16,7 +16,7 @@ import (
 	_ "github.com/prometheus/prometheus/discovery/install"
 
 	// Register integrations
-	_ "github.com/grafana/agent/internal/static/integrations/install"
+	_ "github.com/grafana/agent/static/integrations/install"
 
 	// Embed a set of fallback X.509 trusted roots
 	// Allows the app to work correctly even when the OS does not provide a verifier or systems roots pool

cmd/grafana-agent-operator/main.go

@@ -8,9 +8,9 @@ import (
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/grafana/agent/internal/build"
-	"github.com/grafana/agent/internal/static/operator"
-	"github.com/grafana/agent/internal/static/operator/logutil"
 	util_log "github.com/grafana/agent/internal/util/log"
+	"github.com/grafana/agent/static/operator"
+	"github.com/grafana/agent/static/operator/logutil"
 	controller "sigs.k8s.io/controller-runtime"
 
 	// Needed for clients.

cmd/grafana-agent/entrypoint.go

@@ -18,14 +18,14 @@ import (
 	"github.com/go-kit/log/level"
 	"github.com/gorilla/mux"
 	"github.com/grafana/agent/internal/agentseed"
-	"github.com/grafana/agent/internal/static/config"
-	"github.com/grafana/agent/internal/static/logs"
-	"github.com/grafana/agent/internal/static/metrics"
-	"github.com/grafana/agent/internal/static/metrics/instance"
-	"github.com/grafana/agent/internal/static/server"
-	"github.com/grafana/agent/internal/static/supportbundle"
-	"github.com/grafana/agent/internal/static/traces"
 	"github.com/grafana/agent/internal/usagestats"
+	"github.com/grafana/agent/static/config"
+	"github.com/grafana/agent/static/logs"
+	"github.com/grafana/agent/static/metrics"
+	"github.com/grafana/agent/static/metrics/instance"
+	"github.com/grafana/agent/static/server"
+	"github.com/grafana/agent/static/supportbundle"
+	"github.com/grafana/agent/static/traces"
 	"github.com/grafana/dskit/signals"
 	"github.com/oklog/run"
 	"github.com/prometheus/client_golang/prometheus"

cmd/grafana-agent/main.go

@@ -9,9 +9,9 @@ import (
 	"github.com/grafana/agent/internal/boringcrypto"
 	"github.com/grafana/agent/internal/build"
 	"github.com/grafana/agent/internal/flowmode"
-	"github.com/grafana/agent/internal/static/config"
-	"github.com/grafana/agent/internal/static/server"
 	util_log "github.com/grafana/agent/internal/util/log"
+	"github.com/grafana/agent/static/config"
+	"github.com/grafana/agent/static/server"
 
 	"github.com/prometheus/client_golang/prometheus"
 
@@ -20,7 +20,7 @@ import (
 	_ "github.com/prometheus/prometheus/discovery/install"
 
 	// Register integrations
-	_ "github.com/grafana/agent/internal/static/integrations/install"
+	_ "github.com/grafana/agent/static/integrations/install"
 
 	// Embed a set of fallback X.509 trusted roots
 	// Allows the app to work correctly even when the OS does not provide a verifier or systems roots pool

cmd/grafana-agent/service_windows.go

@@ -8,9 +8,9 @@ import (
 	"os"
 
 	"github.com/go-kit/log/level"
-	"github.com/grafana/agent/internal/static/config"
-	"github.com/grafana/agent/internal/static/server"
 	util_log "github.com/grafana/agent/internal/util/log"
+	"github.com/grafana/agent/static/config"
+	"github.com/grafana/agent/static/server"
 
 	"golang.org/x/sys/windows/svc"
 )

cmd/grafana-agentctl/main.go

@@ -13,23 +13,23 @@ import (
 	"syscall"
 
 	"github.com/grafana/agent/internal/build"
-	"github.com/grafana/agent/internal/static/agentctl/waltools"
-	"github.com/grafana/agent/internal/static/config"
-	"github.com/grafana/agent/internal/static/logs"
+	"github.com/grafana/agent/static/agentctl/waltools"
+	"github.com/grafana/agent/static/config"
+	"github.com/grafana/agent/static/logs"
 	"github.com/olekukonko/tablewriter"
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
-	"github.com/grafana/agent/internal/static/agentctl"
-	"github.com/grafana/agent/internal/static/client"
+	"github.com/grafana/agent/static/agentctl"
+	"github.com/grafana/agent/static/client"
 	"github.com/spf13/cobra"
 
 	// Register Prometheus SD components
 	_ "github.com/prometheus/prometheus/discovery/install"
 
 	// Register integrations
-	_ "github.com/grafana/agent/internal/static/integrations/install"
+	_ "github.com/grafana/agent/static/integrations/install"
 
 	// Needed for operator-detach
 	"k8s.io/apimachinery/pkg/fields"

docs/Makefile

@@ -11,11 +11,11 @@ include docs.mk
 docs: check-cloudwatch-integration
 
 check-cloudwatch-integration:
-	$(PODMAN) run -v $(shell git rev-parse --show-toplevel):/repo -v $(shell pwd):/docs -w /repo golang:1.22.1-bullseye go run internal/static/integrations/cloudwatch_exporter/docs/doc.go check /docs/sources/static/configuration/integrations/cloudwatch-exporter-config.md
-	$(PODMAN) run -v $(shell git rev-parse --show-toplevel):/repo -v $(shell pwd):/docs -w /repo golang:1.22.1-bullseye go run internal/static/integrations/cloudwatch_exporter/docs/doc.go check /docs/sources/flow/reference/components/prometheus.exporter.cloudwatch.md
+	$(PODMAN) run -v $(shell git rev-parse --show-toplevel):/repo -v $(shell pwd):/docs -w /repo golang:1.22.1-bullseye go run static/integrations/cloudwatch_exporter/docs/doc.go check /docs/sources/static/configuration/integrations/cloudwatch-exporter-config.md
+	$(PODMAN) run -v $(shell git rev-parse --show-toplevel):/repo -v $(shell pwd):/docs -w /repo golang:1.22.1-bullseye go run static/integrations/cloudwatch_exporter/docs/doc.go check /docs/sources/flow/reference/components/prometheus.exporter.cloudwatch.md
 
 generate-cloudwatch-integration:
-	$(PODMAN) run -v $(shell git rev-parse --show-toplevel):/repo -v $(shell pwd):/docs -w /repo golang:1.22.1-bullseye go run internal/static/integrations/cloudwatch_exporter/docs/doc.go generate
+	$(PODMAN) run -v $(shell git rev-parse --show-toplevel):/repo -v $(shell pwd):/docs -w /repo golang:1.22.1-bullseye go run static/integrations/cloudwatch_exporter/docs/doc.go generate
 
 sources/assets/hierarchy.svg: sources/operator/hierarchy.dot
 	cat $< | $(PODMAN) run --rm -i nshine/dot dot -Tsvg > $@

docs/developer/release/8-update-helm-charts.md

@@ -30,7 +30,7 @@ Our Helm charts require some version updates as well.
 
 4. Create a branch from `main` for [grafana/agent](https://github.com/grafana/agent).
 
-5. Update the helm chart code in `$agentRepo/operations/helm`:
+5. Update the helm chart code in `operations/helm/charts/grafana-agent`:
 
    1. Update `Chart.yaml` with the new helm version and app version.
    2. Update `CHANGELOG.md` with a new section for the helm version.

docs/docs.mk

@@ -65,6 +65,11 @@ ifeq ($(origin HUGO_REFLINKSERRORLEVEL), undefined)
 export HUGO_REFLINKSERRORLEVEL := WARNING
 endif
 
+# Whether to pull the latest container image before running the container.
+ifeq ($(origin PULL), undefined)
+export PULL := true
+endif
+
 .PHONY: docs-rm
 docs-rm: ## Remove the docs container.
 	$(PODMAN) rm -f $(DOCS_CONTAINER)
@@ -81,13 +86,12 @@ make-docs:
 	fi
 
 .PHONY: docs
-docs: ## Serve documentation locally, which includes pulling the latest `DOCS_IMAGE` (default: `grafana/docs-base:latest`) container image. See also `docs-no-pull`.
+docs: ## Serve documentation locally, which includes pulling the latest `DOCS_IMAGE` (default: `grafana/docs-base:latest`) container image. To not pull the image, set `PULL=false`.
+ifeq ($(PULL), true)
 docs: docs-pull make-docs
-	$(CURDIR)/make-docs $(PROJECTS)
-
-.PHONY: docs-no-pull
-docs-no-pull: ## Serve documentation locally without pulling the `DOCS_IMAGE` (default: `grafana/docs-base:latest`) container image.
-docs-no-pull: make-docs
+else
+docs: make-docs
+endif
 	$(CURDIR)/make-docs $(PROJECTS)
 
 .PHONY: docs-debug
@@ -96,13 +100,19 @@ docs-debug: make-docs
 	WEBSITE_EXEC='hugo server --bind 0.0.0.0 --port 3002 --debug' $(CURDIR)/make-docs $(PROJECTS)
 
 .PHONY: doc-validator
-doc-validator: ## Run doc-validator on the entire docs folder.
+doc-validator: ## Run doc-validator on the entire docs folder which includes pulling the latest `DOC_VALIDATOR_IMAGE` (default: `grafana/doc-validator:latest`) container image. To not pull the image, set `PULL=false`.
 doc-validator: make-docs
+ifeq ($(PULL), true)
+	$(PODMAN) pull -q $(DOC_VALIDATOR_IMAGE)
+endif
 	DOCS_IMAGE=$(DOC_VALIDATOR_IMAGE) $(CURDIR)/make-docs $(PROJECTS)
 
 .PHONY: vale
-vale: ## Run vale on the entire docs folder.
+vale: ## Run vale on the entire docs folder which includes pulling the latest `VALE_IMAGE` (default: `grafana/vale:latest`) container image. To not pull the image, set `PULL=false`.
 vale: make-docs
+ifeq ($(PULL), true)
+	$(PODMAN) pull -q $(VALE_IMAGE)
+endif
 	DOCS_IMAGE=$(VALE_IMAGE) $(CURDIR)/make-docs $(PROJECTS)
 
 .PHONY: update