Add sample_age_limit to remote_write config to drop old samples

CHANGELOG.md

@@ -82,6 +82,7 @@ Main (unreleased)
 
 - Add support for `http_sd_config` within a `scrape_config` for prometheus to flow config conversion. (@erikbaranowski)
 - `discovery.lightsail` now supports additional parameters for configuring HTTP client settings. (@ptodev)
+- Add `sample_age_limit` to remote_write config to drop samples older than a specified duration. (@marctc)
 
 ### Bugfixes
 

component/prometheus/remotewrite/types.go

@@ -35,6 +35,7 @@ var (
 		MinBackoff:        30 * time.Millisecond,
 		MaxBackoff:        5 * time.Second,
 		RetryOnHTTP429:    true,
+		SampleAgeLimit:    0,
 	}
 
 	DefaultMetadataOptions = MetadataOptions{
@@ -141,6 +142,7 @@ type QueueOptions struct {
 	MinBackoff        time.Duration `river:"min_backoff,attr,optional"`
 	MaxBackoff        time.Duration `river:"max_backoff,attr,optional"`
 	RetryOnHTTP429    bool          `river:"retry_on_http_429,attr,optional"`
+	SampleAgeLimit    time.Duration `river:"sample_age_limit,attr,optional"`
 }
 
 // SetToDefault implements river.Defaulter.
@@ -164,6 +166,7 @@ func (r *QueueOptions) toPrometheusType() config.QueueConfig {
 		MinBackoff:        model.Duration(r.MinBackoff),
 		MaxBackoff:        model.Duration(r.MaxBackoff),
 		RetryOnRateLimit:  r.RetryOnHTTP429,
+		SampleAgeLimit:    model.Duration(r.SampleAgeLimit),
 	}
 }
 

converter/internal/prometheusconvert/component/remote_write.go

@@ -96,6 +96,7 @@ func toQueueOptions(queueConfig *prom_config.QueueConfig) *remotewrite.QueueOpti
 		MinBackoff:        time.Duration(queueConfig.MinBackoff),
 		MaxBackoff:        time.Duration(queueConfig.MaxBackoff),
 		RetryOnHTTP429:    queueConfig.RetryOnRateLimit,
+		SampleAgeLimit:    time.Duration(queueConfig.SampleAgeLimit),
 	}
 }
 

docs/sources/flow/reference/components/prometheus.remote_write.md

@@ -165,6 +165,7 @@ Name | Type | Description | Default | Required
 `min_backoff` | `duration` | Initial retry delay. The backoff time gets doubled for each retry. | `"30ms"` | no
 `max_backoff` | `duration` | Maximum retry delay. | `"5s"` | no
 `retry_on_http_429` | `bool` | Retry when an HTTP 429 status code is received. | `true` | no
+`sample_age_limit` | `duration` | Maximum age of samples to send. | `"0s"` | no
 
 Each queue then manages a number of concurrent _shards_ which is responsible
 for sending a fraction of data to their respective endpoints. The number of
@@ -191,6 +192,10 @@ responses should be treated as recoverable errors; other `HTTP 4xx` status code
 responses are never considered recoverable errors. When `retry_on_http_429` is
 enabled, `Retry-After` response headers from the servers are honored.
 
+The `sample_age_limit` argument specifies the maximum age of samples to send. Any
+samples older than the limit are dropped and won't be sent to the remote storage.
+
+
 ### metadata_config block
 
 Name | Type | Description | Default | Required

go.mod

@@ -677,6 +677,14 @@ replace (
 	k8s.io/klog/v2 => github.com/simonpasquier/klog-gokit/v3 v3.3.0
 )
 
+// TODO(marctc): remove replace directive once:
+//
+// * There is a release of Prometheus which contains
+// prometheus/prometheus#13002
+// We use the last v1-related tag as the replace statement does not work for v2
+// tags without the v2 suffix to the module root.
+replace github.com/prometheus/prometheus => github.com/grafana/prometheus v1.8.2-0.20240105105355-3e2c486167d2 // grafana/prometheus@drop-old-inmemory-samples-squashed-2
+
 replace gopkg.in/yaml.v2 => github.com/rfratto/go-yaml v0.0.0-20211119180816-77389c3526dc
 
 // Replace directives from Loki

go.sum

@@ -1080,6 +1080,8 @@ github.com/grafana/opentelemetry-collector/service v0.0.0-20231018134914-c0109e0
 github.com/grafana/opentelemetry-collector/service v0.0.0-20231018134914-c0109e052230/go.mod h1:kBdpzrqR2wJkOdg50yzp4dv+2XBMyeqTgF4lCx0hSpQ=
 github.com/grafana/postgres_exporter v0.8.1-0.20210722175051-db35d7c2f520 h1:HnFWqxhoSF3WC7sKAdMZ+SRXvHLVZlZ3sbQjuUlTqkw=
 github.com/grafana/postgres_exporter v0.8.1-0.20210722175051-db35d7c2f520/go.mod h1:+HPXgiOV0InDHcZ2jNijL1SOKvo0eEPege5fQA0+ICI=
+github.com/grafana/prometheus v1.8.2-0.20240105105355-3e2c486167d2 h1:eJD8U9G91ID/pKsLjJnjqve8yv1NiE/l6dGYnwchPVM=
+github.com/grafana/prometheus v1.8.2-0.20240105105355-3e2c486167d2/go.mod h1:SRw624aMAxTfryAcP8rOjg4S/sHHaetx2lyJJ2nM83g=
 github.com/grafana/pyroscope-go/godeltaprof v0.1.3 h1:eunWpv1B3Z7ZK9o4499EmQGlY+CsDmSZ4FbxjRx37uk=
 github.com/grafana/pyroscope-go/godeltaprof v0.1.3/go.mod h1:1HSPtjU8vLG0jE9JrTdzjgFqdJ/VgN7fvxBNq3luJko=
 github.com/grafana/pyroscope/api v0.2.0 h1:TzOxL0s6SiaLEy944ZAKgHcx/JDRJXu4O8ObwkqR6p4=
@@ -2010,8 +2012,6 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
 github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
 github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
-github.com/prometheus/prometheus v0.48.1 h1:CTszphSNTXkuCG6O0IfpKdHcJkvvnAAE1GbELKS+NFk=
-github.com/prometheus/prometheus v0.48.1/go.mod h1:SRw624aMAxTfryAcP8rOjg4S/sHHaetx2lyJJ2nM83g=
 github.com/prometheus/snmp_exporter v0.24.1 h1:AihTbJHurMo8bjtjJde8U+4gMEvpvYvT21Xbd4SzJgY=
 github.com/prometheus/snmp_exporter v0.24.1/go.mod h1:j6uIGkdR0DXvKn7HJtSkeDj//UY0sWmdd6XhvdBjln0=
 github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI=