Add sample_age_limit to remote_write config to drop old samples (#6052

)

CHANGELOG.md

@@ -82,6 +82,7 @@ Main (unreleased)
 
 - Add support for `http_sd_config` within a `scrape_config` for prometheus to flow config conversion. (@erikbaranowski)
 - `discovery.lightsail` now supports additional parameters for configuring HTTP client settings. (@ptodev)
+- Add `sample_age_limit` to remote_write config to drop samples older than a specified duration. (@marctc)
 
 ### Bugfixes
 

component/prometheus/remotewrite/types.go

@@ -35,6 +35,7 @@ var (
 		MinBackoff:        30 * time.Millisecond,
 		MaxBackoff:        5 * time.Second,
 		RetryOnHTTP429:    true,
+		SampleAgeLimit:    0,
 	}
 
 	DefaultMetadataOptions = MetadataOptions{
@@ -141,6 +142,7 @@ type QueueOptions struct {
 	MinBackoff        time.Duration `river:"min_backoff,attr,optional"`
 	MaxBackoff        time.Duration `river:"max_backoff,attr,optional"`
 	RetryOnHTTP429    bool          `river:"retry_on_http_429,attr,optional"`
+	SampleAgeLimit    time.Duration `river:"sample_age_limit,attr,optional"`
 }
 
 // SetToDefault implements river.Defaulter.
@@ -164,6 +166,7 @@ func (r *QueueOptions) toPrometheusType() config.QueueConfig {
 		MinBackoff:        model.Duration(r.MinBackoff),
 		MaxBackoff:        model.Duration(r.MaxBackoff),
 		RetryOnRateLimit:  r.RetryOnHTTP429,
+		SampleAgeLimit:    model.Duration(r.SampleAgeLimit),
 	}
 }
 

converter/internal/prometheusconvert/component/remote_write.go

@@ -96,6 +96,7 @@ func toQueueOptions(queueConfig *prom_config.QueueConfig) *remotewrite.QueueOpti
 		MinBackoff:        time.Duration(queueConfig.MinBackoff),
 		MaxBackoff:        time.Duration(queueConfig.MaxBackoff),
 		RetryOnHTTP429:    queueConfig.RetryOnRateLimit,
+		SampleAgeLimit:    time.Duration(queueConfig.SampleAgeLimit),
 	}
 }
 

converter/internal/staticconvert/testdata-v2/integrations_v2.river

@@ -1,6 +1,6 @@
 prometheus.remote_write "metrics_default" {
 	endpoint {
-		name = "default-8be96f"
+		name = "default-149bbd"
 		url  = "http://localhost:9009/api/prom/push"
 
 		queue_config { }

converter/internal/staticconvert/testdata-v2/unsupported.river

@@ -1,6 +1,6 @@
 prometheus.remote_write "metrics_default" {
 	endpoint {
-		name = "default-8be96f"
+		name = "default-149bbd"
 		url  = "http://localhost:9009/api/prom/push"
 
 		queue_config { }

converter/internal/staticconvert/testdata/prom_remote_write.river

@@ -1,6 +1,6 @@
 prometheus.remote_write "metrics_test1" {
 	endpoint {
-		name = "test1-8be96f"
+		name = "test1-149bbd"
 		url  = "http://localhost:9009/api/prom/push"
 
 		queue_config { }
@@ -11,7 +11,7 @@ prometheus.remote_write "metrics_test1" {
 
 prometheus.remote_write "metrics_test2" {
 	endpoint {
-		name           = "test2-533083"
+		name           = "test2-c6d55a"
 		url            = "http://localhost:9010/api/prom/push"
 		send_exemplars = false
 
@@ -23,7 +23,7 @@ prometheus.remote_write "metrics_test2" {
 
 prometheus.remote_write "metrics_test3" {
 	endpoint {
-		name = "test3-a3c419"
+		name = "test3-aa96fd"
 		url  = "http://localhost:9011/api/prom/push"
 
 		queue_config { }
@@ -32,7 +32,7 @@ prometheus.remote_write "metrics_test3" {
 	}
 
 	endpoint {
-		name = "test3-41df1c"
+		name = "test3-a93240"
 		url  = "http://localhost:9012/api/prom/push"
 
 		queue_config {
@@ -45,7 +45,7 @@ prometheus.remote_write "metrics_test3" {
 
 prometheus.remote_write "metrics_test4_sigv4_defaults" {
 	endpoint {
-		name = "test4_sigv4_defaults-c42e88"
+		name = "test4_sigv4_defaults-f815bf"
 		url  = "http://localhost:9012/api/prom/push"
 
 		queue_config { }
@@ -58,7 +58,7 @@ prometheus.remote_write "metrics_test4_sigv4_defaults" {
 
 prometheus.remote_write "metrics_test5_sigv4_explicit" {
 	endpoint {
-		name = "test5_sigv4_explicit-050ad5"
+		name = "test5_sigv4_explicit-bc8fca"
 		url  = "http://localhost:9012/api/prom/push"
 
 		queue_config { }
@@ -77,7 +77,7 @@ prometheus.remote_write "metrics_test5_sigv4_explicit" {
 
 prometheus.remote_write "metrics_test6_azuread_defaults" {
 	endpoint {
-		name = "test6_azuread_defaults-fbed02"
+		name = "test6_azuread_defaults-cc4e7e"
 		url  = "http://localhost:9012/api/prom/push"
 
 		queue_config { }
@@ -94,7 +94,7 @@ prometheus.remote_write "metrics_test6_azuread_defaults" {
 
 prometheus.remote_write "metrics_test7_azuread_explicit" {
 	endpoint {
-		name = "test7_azuread_explicit-416842"
+		name = "test7_azuread_explicit-9e1a3e"
 		url  = "http://localhost:9012/api/prom/push"
 
 		queue_config { }

converter/internal/staticconvert/testdata/prom_scrape.river

@@ -91,7 +91,7 @@ prometheus.relabel "metrics_agent_promobee" {
 
 prometheus.remote_write "metrics_agent" {
 	endpoint {
-		name = "agent-6ea089"
+		name = "agent-36127e"
 		url  = "https://prometheus-us-central1.grafana.net/api/prom/push"
 
 		basic_auth {
@@ -103,6 +103,7 @@ prometheus.remote_write "metrics_agent" {
 			max_shards          = 10
 			batch_send_deadline = "3m0s"
 			max_backoff         = "10s"
+			sample_age_limit    = "50s"
 		}
 
 		metadata_config { }

converter/internal/staticconvert/testdata/prom_scrape.yaml

@@ -19,6 +19,7 @@ metrics:
         batch_send_deadline: 3m
         max_shards: 10
         max_backoff: 10s
+        sample_age_limit: 50s
       basic_auth:
         username: 11111
         password: my-secret-password-here

converter/internal/staticconvert/testdata/promtail_prom.river

@@ -18,7 +18,7 @@ prometheus.scrape "metrics_name_jobName" {
 
 prometheus.remote_write "metrics_name" {
 	endpoint {
-		name = "name-8be96f"
+		name = "name-149bbd"
 		url  = "http://localhost:9009/api/prom/push"
 
 		queue_config { }

converter/internal/staticconvert/testdata/sanitize.river

@@ -1,6 +1,6 @@
 prometheus.remote_write "metrics_integrations" {
 	endpoint {
-		name = "integrations-717d0f"
+		name = "integrations-ce3432"
 		url  = "https://region.grafana.net/api/prom/push"
 
 		basic_auth {

converter/internal/staticconvert/testdata/unsupported.river

@@ -8,7 +8,7 @@ prometheus.scrape "metrics_agent_prometheus" {
 
 prometheus.remote_write "metrics_agent" {
 	endpoint {
-		name = "agent-d885f6"
+		name = "agent-eea444"
 		url  = "https://prometheus-us-central1.grafana.net/api/prom/push"
 
 		queue_config { }
@@ -41,7 +41,7 @@ prometheus.scrape "integrations_statsd_exporter" {
 
 prometheus.remote_write "integrations" {
 	endpoint {
-		name = "agent-d885f6"
+		name = "agent-eea444"
 		url  = "https://prometheus-us-central1.grafana.net/api/prom/push"
 
 		queue_config { }

docs/sources/flow/reference/components/prometheus.remote_write.md

@@ -165,6 +165,7 @@ Name | Type | Description | Default | Required
 `min_backoff` | `duration` | Initial retry delay. The backoff time gets doubled for each retry. | `"30ms"` | no
 `max_backoff` | `duration` | Maximum retry delay. | `"5s"` | no
 `retry_on_http_429` | `bool` | Retry when an HTTP 429 status code is received. | `true` | no
+`sample_age_limit` | `duration` | Maximum age of samples to send. | `"0s"` | no
 
 Each queue then manages a number of concurrent _shards_ which is responsible
 for sending a fraction of data to their respective endpoints. The number of
@@ -191,6 +192,10 @@ responses should be treated as recoverable errors; other `HTTP 4xx` status code
 responses are never considered recoverable errors. When `retry_on_http_429` is
 enabled, `Retry-After` response headers from the servers are honored.
 
+The `sample_age_limit` argument specifies the maximum age of samples to send. Any
+samples older than the limit are dropped and won't be sent to the remote storage.
+The default value is `0s`, which means that all samples are sent (feature is disabled).
+
 ### metadata_config block
 
 Name | Type | Description | Default | Required

go.mod

@@ -142,7 +142,7 @@ require (
 	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.66.0
 	github.com/prometheus-operator/prometheus-operator/pkg/client v0.66.0
 	github.com/prometheus/blackbox_exporter v0.24.1-0.20230623125439-bd22efa1c900
-	github.com/prometheus/client_golang v1.17.0
+	github.com/prometheus/client_golang v1.18.0
 	github.com/prometheus/client_model v0.5.0
 	github.com/prometheus/common v0.45.0
 	github.com/prometheus/consul_exporter v0.8.0
@@ -216,7 +216,7 @@ require (
 	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa
 	golang.org/x/net v0.18.0
 	golang.org/x/oauth2 v0.13.0
-	golang.org/x/sys v0.14.1-0.20231108175955-e4099bfacb8c
+	golang.org/x/sys v0.15.0
 	golang.org/x/text v0.14.0
 	golang.org/x/time v0.3.0
 	google.golang.org/api v0.147.0
@@ -677,6 +677,14 @@ replace (
 	k8s.io/klog/v2 => github.com/simonpasquier/klog-gokit/v3 v3.3.0
 )
 
+// TODO(marctc): remove replace directive once:
+//
+// * There is a release of Prometheus which contains
+// prometheus/prometheus#13002
+// We use the last v1-related tag as the replace statement does not work for v2
+// tags without the v2 suffix to the module root.
+replace github.com/prometheus/prometheus => github.com/grafana/prometheus v1.8.2-0.20240105105355-3e2c486167d2 // grafana/prometheus@drop-old-inmemory-samples-squashed-2
+
 replace gopkg.in/yaml.v2 => github.com/rfratto/go-yaml v0.0.0-20211119180816-77389c3526dc
 
 // Replace directives from Loki

go.sum

@@ -1080,6 +1080,8 @@ github.com/grafana/opentelemetry-collector/service v0.0.0-20231018134914-c0109e0
 github.com/grafana/opentelemetry-collector/service v0.0.0-20231018134914-c0109e052230/go.mod h1:kBdpzrqR2wJkOdg50yzp4dv+2XBMyeqTgF4lCx0hSpQ=
 github.com/grafana/postgres_exporter v0.8.1-0.20210722175051-db35d7c2f520 h1:HnFWqxhoSF3WC7sKAdMZ+SRXvHLVZlZ3sbQjuUlTqkw=
 github.com/grafana/postgres_exporter v0.8.1-0.20210722175051-db35d7c2f520/go.mod h1:+HPXgiOV0InDHcZ2jNijL1SOKvo0eEPege5fQA0+ICI=
+github.com/grafana/prometheus v1.8.2-0.20240105105355-3e2c486167d2 h1:eJD8U9G91ID/pKsLjJnjqve8yv1NiE/l6dGYnwchPVM=
+github.com/grafana/prometheus v1.8.2-0.20240105105355-3e2c486167d2/go.mod h1:SRw624aMAxTfryAcP8rOjg4S/sHHaetx2lyJJ2nM83g=
 github.com/grafana/pyroscope-go/godeltaprof v0.1.3 h1:eunWpv1B3Z7ZK9o4499EmQGlY+CsDmSZ4FbxjRx37uk=
 github.com/grafana/pyroscope-go/godeltaprof v0.1.3/go.mod h1:1HSPtjU8vLG0jE9JrTdzjgFqdJ/VgN7fvxBNq3luJko=
 github.com/grafana/pyroscope/api v0.2.0 h1:TzOxL0s6SiaLEy944ZAKgHcx/JDRJXu4O8ObwkqR6p4=
@@ -1957,8 +1959,8 @@ github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr
 github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ=
-github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
-github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
+github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
+github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA=
 github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
@@ -2010,8 +2012,6 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
 github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
 github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
-github.com/prometheus/prometheus v0.48.1 h1:CTszphSNTXkuCG6O0IfpKdHcJkvvnAAE1GbELKS+NFk=
-github.com/prometheus/prometheus v0.48.1/go.mod h1:SRw624aMAxTfryAcP8rOjg4S/sHHaetx2lyJJ2nM83g=
 github.com/prometheus/snmp_exporter v0.24.1 h1:AihTbJHurMo8bjtjJde8U+4gMEvpvYvT21Xbd4SzJgY=
 github.com/prometheus/snmp_exporter v0.24.1/go.mod h1:j6uIGkdR0DXvKn7HJtSkeDj//UY0sWmdd6XhvdBjln0=
 github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI=
@@ -2807,8 +2807,8 @@ golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.14.1-0.20231108175955-e4099bfacb8c h1:3kC/TjQ+xzIblQv39bCOyRk8fbEeJcDHwbyxPUU2BpA=
-golang.org/x/sys v0.14.1-0.20231108175955-e4099bfacb8c/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=