From c43819cb8fedb351bfa7c6839240bddf6dfdf3ae Mon Sep 17 00:00:00 2001 From: David Cook Date: Thu, 18 Apr 2024 09:35:52 -0500 Subject: [PATCH] Add GC subcommand (#3027) * Enable GC in aggregator graceful shutdown test * Add garbage collector subcommand and Docker image --- .github/workflows/ci-build.yml | 1 + .../workflows/push-docker-images-release.yml | 2 + Dockerfile | 1 + README.md | 9 +- aggregator/src/binaries.rs | 1 + aggregator/src/binaries/aggregator.rs | 25 +-- aggregator/src/binaries/garbage_collector.rs | 172 ++++++++++++++++++ aggregator/src/main.rs | 37 ++-- .../tests/integration/graceful_shutdown.rs | 42 ++++- docker-bake.hcl | 23 +++ docs/DEPLOYING.md | 11 ++ .../advanced_config/garbage_collector.yaml | 89 +++++++++ .../basic_config/garbage_collector.yaml | 25 +++ 13 files changed, 390 insertions(+), 48 deletions(-) create mode 100644 aggregator/src/binaries/garbage_collector.rs create mode 100644 docs/samples/advanced_config/garbage_collector.yaml create mode 100644 docs/samples/basic_config/garbage_collector.yaml diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index d80b816ea..ab4c3ee1f 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -158,6 +158,7 @@ jobs: targets: janus load: true - run: docker run --rm janus_aggregator --help + - run: docker run --rm janus_garbage_collector --help - run: docker run --rm janus_aggregation_job_creator --help - run: docker run --rm janus_aggregation_job_driver --help - run: docker run --rm janus_collection_job_driver --help diff --git a/.github/workflows/push-docker-images-release.yml b/.github/workflows/push-docker-images-release.yml index 6c6698e0d..3137805b7 100644 --- a/.github/workflows/push-docker-images-release.yml +++ b/.github/workflows/push-docker-images-release.yml @@ -55,6 +55,7 @@ jobs: password: ${{ steps.gcp-auth-private.outputs.access_token }} - run: docker push us-west2-docker.pkg.dev/janus-artifacts/janus/janus_aggregator:${{ steps.get_version.outputs.VERSION }} + - run: docker push us-west2-docker.pkg.dev/janus-artifacts/janus/janus_garbage_collector:${{ steps.get_version.outputs.VERSION }} - run: docker push us-west2-docker.pkg.dev/janus-artifacts/janus/janus_aggregation_job_creator:${{ steps.get_version.outputs.VERSION }} - run: docker push us-west2-docker.pkg.dev/janus-artifacts/janus/janus_aggregation_job_driver:${{ steps.get_version.outputs.VERSION }} - run: docker push us-west2-docker.pkg.dev/janus-artifacts/janus/janus_collection_job_driver:${{ steps.get_version.outputs.VERSION }} @@ -80,6 +81,7 @@ jobs: password: ${{ steps.gcp-auth-public.outputs.access_token }} - run: docker push us-west2-docker.pkg.dev/divviup-artifacts-public/janus/janus_aggregator:${{ steps.get_version.outputs.VERSION }} + - run: docker push us-west2-docker.pkg.dev/divviup-artifacts-public/janus/janus_garbage_collector:${{ steps.get_version.outputs.VERSION }} - run: docker push us-west2-docker.pkg.dev/divviup-artifacts-public/janus/janus_aggregation_job_creator:${{ steps.get_version.outputs.VERSION }} - run: docker push us-west2-docker.pkg.dev/divviup-artifacts-public/janus/janus_aggregation_job_driver:${{ steps.get_version.outputs.VERSION }} - run: docker push us-west2-docker.pkg.dev/divviup-artifacts-public/janus/janus_collection_job_driver:${{ steps.get_version.outputs.VERSION }} diff --git a/Dockerfile b/Dockerfile index a507ae13b..09b52b673 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,6 +46,7 @@ ARG GIT_REVISION=unknown LABEL revision ${GIT_REVISION} COPY --from=builder /src/target/release/janus_aggregator /janus_aggregator RUN ln -s /janus_aggregator /aggregator && \ + ln -s /janus_aggregator /garbage_collector && \ ln -s /janus_aggregator /aggregation_job_creator && \ ln -s /janus_aggregator /aggregation_job_driver && \ ln -s /janus_aggregator /collection_job_driver && \ diff --git a/README.md b/README.md index cb54240bf..fcf70c6f9 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ should not be depended on directly. If you find yourself needing to depend on it directly while using any other Janus crates, open a bug report. The following crates are stable on their external configuration, CLI arguments, -and HTTP API. Their Rust API may arbitrarily change and should not be depended +and HTTP API. Their Rust API may arbitrarily change and should not be depended on. They are not published to crates.io. - `janus_aggregator` - `janus_tools` @@ -105,9 +105,10 @@ subtle incompatibilities between the two that will cause tests to fail. ### Container image To build container images, run `docker buildx bake --load`. This will produce -images tagged `janus_aggregator`, `janus_aggregation_job_creator`, -`janus_aggregation_job_driver`, `janus_collection_job_driver`, `janus_cli`, -`janus_db_migrator`, `janus_interop_client`, `janus_interop_aggregator`, and +images tagged `janus_aggregator`, `janus_garbage_collector`, +`janus_aggregation_job_creator`, `janus_aggregation_job_driver`, +`janus_collection_job_driver`, `janus_cli`, `janus_db_migrator`, +`janus_interop_client`, `janus_interop_aggregator`, and `janus_interop_collector` by default. Pre-built container images are available at diff --git a/aggregator/src/binaries.rs b/aggregator/src/binaries.rs index f20ff58ae..40dc29c18 100644 --- a/aggregator/src/binaries.rs +++ b/aggregator/src/binaries.rs @@ -2,4 +2,5 @@ pub mod aggregation_job_creator; pub mod aggregation_job_driver; pub mod aggregator; pub mod collection_job_driver; +pub mod garbage_collector; pub mod janus_cli; diff --git a/aggregator/src/binaries/aggregator.rs b/aggregator/src/binaries/aggregator.rs index 3160ab077..826b5e8f9 100644 --- a/aggregator/src/binaries/aggregator.rs +++ b/aggregator/src/binaries/aggregator.rs @@ -1,5 +1,6 @@ use crate::{ - aggregator::{self, garbage_collector::GarbageCollector, http_handlers::aggregator_handler}, + aggregator::{self, http_handlers::aggregator_handler}, + binaries::garbage_collector::run_garbage_collector, binary_utils::{setup_server, BinaryContext, BinaryOptions, CommonBinaryOptions}, cache::GlobalHpkeKeypairCache, config::{BinaryConfig, CommonConfig, TaskprovConfig}, @@ -22,8 +23,8 @@ use std::{ pin::Pin, }; use std::{iter::Iterator, net::SocketAddr, sync::Arc, time::Duration}; -use tokio::{join, sync::watch, time::interval}; -use tracing::{error, info}; +use tokio::{join, sync::watch}; +use tracing::info; use trillium::Handler; use trillium_router::router; use url::Url; @@ -77,24 +78,10 @@ async fn run_aggregator( let datastore = Arc::clone(&datastore); let gc_config = config.garbage_collection.take(); let meter = meter.clone(); + let stopper = stopper.clone(); async move { if let Some(gc_config) = gc_config { - let gc = GarbageCollector::new( - datastore, - &meter, - gc_config.report_limit, - gc_config.aggregation_limit, - gc_config.collection_limit, - gc_config.tasks_per_tx, - gc_config.concurrent_tx_limit, - ); - let mut interval = interval(Duration::from_secs(gc_config.gc_frequency_s)); - loop { - interval.tick().await; - if let Err(err) = gc.run().await { - error!(?err, "GC error"); - } - } + run_garbage_collector(datastore, gc_config, meter, stopper).await; } } }; diff --git a/aggregator/src/binaries/garbage_collector.rs b/aggregator/src/binaries/garbage_collector.rs new file mode 100644 index 000000000..58f209434 --- /dev/null +++ b/aggregator/src/binaries/garbage_collector.rs @@ -0,0 +1,172 @@ +use std::{sync::Arc, time::Duration}; + +use anyhow::Result; +use clap::Parser; +use janus_aggregator_core::datastore::Datastore; +use janus_core::time::RealClock; +use opentelemetry::metrics::Meter; +use serde::{Deserialize, Serialize}; +use tokio::time::interval; +use tracing::error; +use trillium_tokio::Stopper; + +use crate::{ + aggregator::garbage_collector::GarbageCollector, + binary_utils::{BinaryContext, BinaryOptions, CommonBinaryOptions}, + config::{BinaryConfig, CommonConfig}, +}; + +use super::aggregator::GarbageCollectorConfig; + +pub async fn main_callback(ctx: BinaryContext) -> Result<()> { + let BinaryContext { + config, + datastore, + meter, + stopper, + .. + } = ctx; + + let datastore = Arc::new(datastore); + + run_garbage_collector(datastore, config.garbage_collection, meter, stopper).await; + + Ok(()) +} + +pub(super) async fn run_garbage_collector( + datastore: Arc>, + gc_config: GarbageCollectorConfig, + meter: Meter, + stopper: Stopper, +) { + let gc = GarbageCollector::new( + datastore, + &meter, + gc_config.report_limit, + gc_config.aggregation_limit, + gc_config.collection_limit, + gc_config.tasks_per_tx, + gc_config.concurrent_tx_limit, + ); + let mut interval = interval(Duration::from_secs(gc_config.gc_frequency_s)); + while stopper.stop_future(interval.tick()).await.is_some() { + if let Err(err) = gc.run().await { + error!(?err, "GC error"); + } + } +} + +#[derive(Debug, Default, Parser)] +#[clap( + name = "garbage-collector", + about = "Janus garbage collector", + rename_all = "kebab-case", + version = env!("CARGO_PKG_VERSION"), +)] +pub struct Options { + #[clap(flatten)] + pub common: CommonBinaryOptions, +} + +impl BinaryOptions for Options { + fn common_options(&self) -> &CommonBinaryOptions { + &self.common + } +} + +/// Non-secret configuration options for a Janus garbage collector, deserialized from YAML. +/// +/// # Examples +/// +/// ``` +/// # use janus_aggregator::binaries::garbage_collector::Config; +/// let yaml_config = r#" +/// --- +/// database: +/// url: "postgres://postgres:postgres@localhost:5432/postgres" +/// logging_config: # logging_config is optional +/// force_json_output: true +/// garbage_collection: +/// gc_frequency_s: 60 +/// report_limit: 5000 +/// aggregation_limit: 500 +/// collection_limit: 50 +/// "#; +/// +/// let _decoded: Config = serde_yaml::from_str(yaml_config).unwrap(); +/// ``` +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Config { + #[serde(flatten)] + pub common_config: CommonConfig, + + pub garbage_collection: GarbageCollectorConfig, +} + +impl BinaryConfig for Config { + fn common_config(&self) -> &CommonConfig { + &self.common_config + } + + fn common_config_mut(&mut self) -> &mut CommonConfig { + &mut self.common_config + } +} + +#[cfg(test)] +mod tests { + use std::net::{Ipv4Addr, SocketAddr}; + + use clap::CommandFactory; + use janus_core::test_util::roundtrip_encoding; + + use crate::{ + binaries::aggregator::GarbageCollectorConfig, + config::{ + default_max_transaction_retries, + test_util::{generate_db_config, generate_metrics_config, generate_trace_config}, + CommonConfig, + }, + }; + + use super::{Config, Options}; + + #[test] + fn verify_app() { + Options::command().debug_assert(); + } + + #[test] + fn roundtrip_config() { + roundtrip_encoding(Config { + common_config: CommonConfig { + database: generate_db_config(), + logging_config: generate_trace_config(), + metrics_config: generate_metrics_config(), + health_check_listen_address: SocketAddr::from((Ipv4Addr::UNSPECIFIED, 8080)), + max_transaction_retries: default_max_transaction_retries(), + }, + garbage_collection: GarbageCollectorConfig { + gc_frequency_s: 60, + report_limit: 5000, + aggregation_limit: 500, + collection_limit: 50, + tasks_per_tx: 1, + concurrent_tx_limit: None, + }, + }); + } + + #[test] + fn documentation_config_examples() { + serde_yaml::from_str::(include_str!( + "../../../docs/samples/basic_config/garbage_collector.yaml" + )) + .unwrap(); + serde_yaml::from_str::(include_str!( + "../../../docs/samples/advanced_config/garbage_collector.yaml" + )) + .unwrap(); + } +} diff --git a/aggregator/src/main.rs b/aggregator/src/main.rs index a203db8d3..6b842b70b 100644 --- a/aggregator/src/main.rs +++ b/aggregator/src/main.rs @@ -2,7 +2,7 @@ use clap::{Parser, Subcommand}; use janus_aggregator::{ binaries::{ aggregation_job_creator, aggregation_job_driver, aggregator, collection_job_driver, - janus_cli, + garbage_collector, janus_cli, }, binary_utils::janus_main, }; @@ -13,6 +13,8 @@ use janus_core::time::RealClock; enum Options { #[clap(name = "aggregator")] Aggregator(aggregator::Options), + #[clap(name = "garbage_collector")] + GarbageCollector(garbage_collector::Options), #[clap(name = "aggregation_job_creator")] AggregationJobCreator(aggregation_job_creator::Options), #[clap(name = "aggregation_job_driver")] @@ -34,6 +36,8 @@ enum Options { enum Nested { #[clap(name = "aggregator")] Aggregator(aggregator::Options), + #[clap(name = "garbage_collector")] + GarbageCollector(garbage_collector::Options), #[clap(name = "aggregation_job_creator")] AggregationJobCreator(aggregation_job_creator::Options), #[clap(name = "aggregation_job_driver")] @@ -46,21 +50,20 @@ enum Nested { #[tokio::main] async fn main() -> anyhow::Result<()> { + let clock = RealClock::default(); match Options::parse() { Options::Aggregator(options) | Options::Default(Nested::Aggregator(options)) => { - janus_main( - options, - RealClock::default(), - true, - aggregator::main_callback, - ) - .await + janus_main(options, clock, true, aggregator::main_callback).await + } + Options::GarbageCollector(options) + | Options::Default(Nested::GarbageCollector(options)) => { + janus_main(options, clock, false, garbage_collector::main_callback).await } Options::AggregationJobCreator(options) | Options::Default(Nested::AggregationJobCreator(options)) => { janus_main( options, - RealClock::default(), + clock, false, aggregation_job_creator::main_callback, ) @@ -68,23 +71,11 @@ async fn main() -> anyhow::Result<()> { } Options::AggregationJobDriver(options) | Options::Default(Nested::AggregationJobDriver(options)) => { - janus_main( - options, - RealClock::default(), - true, - aggregation_job_driver::main_callback, - ) - .await + janus_main(options, clock, true, aggregation_job_driver::main_callback).await } Options::CollectionJobDriver(options) | Options::Default(Nested::CollectionJobDriver(options)) => { - janus_main( - options, - RealClock::default(), - false, - collection_job_driver::main_callback, - ) - .await + janus_main(options, clock, false, collection_job_driver::main_callback).await } Options::JanusCli(options) | Options::Default(Nested::JanusCli(options)) => { janus_cli::run(options).await diff --git a/aggregator/tests/integration/graceful_shutdown.rs b/aggregator/tests/integration/graceful_shutdown.rs index e5acb30c6..85143125b 100644 --- a/aggregator/tests/integration/graceful_shutdown.rs +++ b/aggregator/tests/integration/graceful_shutdown.rs @@ -8,8 +8,9 @@ use janus_aggregator::{ binaries::{ aggregation_job_creator::Config as AggregationJobCreatorConfig, aggregation_job_driver::Config as AggregationJobDriverConfig, - aggregator::{AggregatorApi, Config as AggregatorConfig}, + aggregator::{AggregatorApi, Config as AggregatorConfig, GarbageCollectorConfig}, collection_job_driver::Config as CollectionJobDriverConfig, + garbage_collector::Config as GarbageCollectorBinaryConfig, }, config::{ default_max_transaction_retries, BinaryConfig, CommonConfig, DbConfig, JobDriverConfig, @@ -264,7 +265,14 @@ async fn aggregator_shutdown() { max_transaction_retries: default_max_transaction_retries(), }, taskprov_config: TaskprovConfig::default(), - garbage_collection: None, + garbage_collection: Some(GarbageCollectorConfig { + gc_frequency_s: 60, + report_limit: 5000, + aggregation_limit: 500, + collection_limit: 50, + tasks_per_tx: 1, + concurrent_tx_limit: None, + }), listen_address: aggregator_listen_address, aggregator_api: Some(AggregatorApi { listen_address: Some(aggregator_api_listen_address), @@ -281,6 +289,36 @@ async fn aggregator_shutdown() { graceful_shutdown("aggregator", config).await; } +#[tokio::test(flavor = "multi_thread")] +#[cfg_attr(not(target_os = "linux"), ignore)] +async fn garbage_collector_shutdown() { + let config = GarbageCollectorBinaryConfig { + common_config: CommonConfig { + database: DbConfig { + url: "postgres://localhost".parse().unwrap(), + connection_pool_timeouts_secs: 60, + connection_pool_max_size: None, + check_schema_version: true, + tls_trust_store_path: None, + }, + logging_config: TraceConfiguration::default(), + metrics_config: MetricsConfiguration::default(), + health_check_listen_address: "127.0.0.1:9001".parse().unwrap(), + max_transaction_retries: default_max_transaction_retries(), + }, + garbage_collection: GarbageCollectorConfig { + gc_frequency_s: 60, + report_limit: 5000, + aggregation_limit: 500, + collection_limit: 50, + tasks_per_tx: 1, + concurrent_tx_limit: None, + }, + }; + + graceful_shutdown("garbage_collector", config).await; +} + #[tokio::test(flavor = "multi_thread")] #[cfg_attr(not(target_os = "linux"), ignore)] async fn aggregation_job_creator_shutdown() { diff --git a/docker-bake.hcl b/docker-bake.hcl index af949b1b4..e2a1b2710 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -21,6 +21,7 @@ group "default" { group "janus" { targets = [ "janus_aggregator", + "janus_garbage_collector", "janus_aggregation_job_creator", "janus_aggregation_job_driver", "janus_collection_job_driver", @@ -44,6 +45,7 @@ group "release" { group "janus_release" { targets = [ "janus_aggregator_release", + "janus_garbage_collector_release", "janus_aggregation_job_creator_release", "janus_aggregation_job_driver_release", "janus_collection_job_driver_release", @@ -89,6 +91,27 @@ target "janus_aggregator_release" { ] } +target "janus_garbage_collector" { + args = { + BINARY = "garbage_collector" + GIT_REVISION = "${GIT_REVISION}" + } + cache-from = [ + "type=gha,scope=main-janus", + "type=gha,scope=${GITHUB_BASE_REF}-janus", + "type=gha,scope=${GITHUB_REF_NAME}-janus", + ] + tags = ["janus_garbage_collector:${VERSION}"] +} + +target "janus_garbage_collector_release" { + inherits = ["janus_garbage_collector"] + tags = [ + "us-west2-docker.pkg.dev/janus-artifacts/janus/janus_garbage_collector:${VERSION}", + "us-west2-docker.pkg.dev/divviup-artifacts-public/janus/janus_garbage_collector:${VERSION}", + ] +} + target "janus_aggregation_job_creator" { args = { BINARY = "aggregation_job_creator" diff --git a/docs/DEPLOYING.md b/docs/DEPLOYING.md index db707a3a8..ad4145e72 100644 --- a/docs/DEPLOYING.md +++ b/docs/DEPLOYING.md @@ -13,6 +13,7 @@ - [Tracing](#tracing) - [`tokio-console`](#tokio-console) - [`aggregator` configuration](#aggregator-configuration) + - [`garbage_collector` configuration](#garbagecollector-configuration) - [`aggregation_job_creator` configuration](#aggregationjobcreator-configuration) - [`aggregation_job_driver` configuration](#aggregationjobdriver-configuration) - [`collection_job_driver` configuration](#collectionjobdriver-configuration) @@ -118,6 +119,16 @@ requests, and additional parameters to customize batching of uploaded reports into database transactions. See the [sample configuration file](samples/basic_config/aggregator.yaml) for details. +The `aggregator` component can optionally do garbage collection of old data as +well, i.e. when running a single-service helper-only aggregator. + +### `garbage_collector` configuration + +The `garbage_collector` component requires configuration parameters to determine +how frequently it scans for old data eligible for deletion, how many objects it +deletes at a time, and how much concurrency to use. See the [sample +configuration file](samples/basic_config/garbage_collector.yaml) for details. + ### `aggregation_job_creator` configuration The `aggregation_job_creator` component requires configuration parameters to diff --git a/docs/samples/advanced_config/garbage_collector.yaml b/docs/samples/advanced_config/garbage_collector.yaml new file mode 100644 index 000000000..c501e74ae --- /dev/null +++ b/docs/samples/advanced_config/garbage_collector.yaml @@ -0,0 +1,89 @@ +# Common configuration parameters: + +database: + # Database URL. (required) + url: "postgres://postgres:postgres@localhost:5432/postgres" + + # Timeout for new database connections. Defaults to 60 seconds. + connection_pool_timeout_secs: 60 + + # Maximum number of database connections. Defaults to CPUs * 4. + connection_pool_max_size: 8 + + # Flag to check if the database schema version is compatible upon startup. + # (optional, defaults to true) + check_schema_version: true + + # Path to a PEM file with root certificates to trust for TLS database + # connections. If present, TLS may be used depending on the "sslmode" + # connection string parameter, and the server's support for TLS. If absent, + # TLS will never be used. (optional) + tls_trust_store_path: /path/to/file.pem + +# Socket address for /healthz and /traceconfigz HTTP requests. Defaults to 127.0.0.1:9001. +health_check_listen_address: "0.0.0.0:8000" + +# Logging configuration. (optional) +logging_config: + # Flag to output structured logs. (optional) + force_json_output: true + + # Flag to output structured logs in Google Cloud Logging's format. (optional) + stackdriver_json_output: false + + # Configuration for the tokio-console tracing subscriber. (optional) + tokio_console_config: + # Enable the subscriber. (optional) + enabled: true + # Socket address to listen on. (optional) + listen_address: "127.0.0.1:6669" + + # OpenTelemetry tracing configuration. This can contain an "otlp" key with a + # map containing exporter configuration. (optional) + open_telemetry_config: + otlp: + # OTLP gRPC endpoint. + endpoint: "https://example.com" + # gRPC metadata to send with OTLP requests. (optional) + metadata: + key: "value" + + # Flag to write tracing spans and events to JSON files. This is compatible + # with Chrome's trace viewer, available at `chrome://tracing`, and + # Perfetto, at https://ui.perfetto.dev/. (optional) + chrome: false + +# Metrics configuration. (optional) +metrics_config: + # Metrics exporter configuration. This contains a map with single key, either + # "prometheus" or "otlp". (optional) + exporter: + prometheus: + # Address on which to listen for Prometheus metrics scrape requests. (optional) + host: "0.0.0.0" + # Port number for metrics server. (optional) + port: 9464 + + ##otlp: + ## # OTLP gRPC endpoint. + ## endpoint: "https://example.com/" + ## # gRPC metadata to send with OTLP requests. (optional) + ## metadata: + ## key: "value" + +# Garbage collector-specific parameters: +garbage_collection: + # How frequently to collect garbage, in seconds. + gc_frequency_s: 60 + + # The maximum number of client reports, per task, to delete in a single run of the garbage + # collector. + report_limit: 5000 + + # The maximum number of aggregation jobs (& related artifacts), per task, to delete in a single + # run of the garbage collector. + aggregation_limit: 500 + + # The maximum number of collection jobs (& related artifacts), per task, to delete in a single run + # of the garbage collector. + collection_limit: 50 diff --git a/docs/samples/basic_config/garbage_collector.yaml b/docs/samples/basic_config/garbage_collector.yaml new file mode 100644 index 000000000..2e750605b --- /dev/null +++ b/docs/samples/basic_config/garbage_collector.yaml @@ -0,0 +1,25 @@ +# Common configuration parameters: + +database: + # Database URL. (required) + url: "postgres://postgres:postgres@localhost:5432/postgres" + +# Socket address for /healthz and /traceconfigz HTTP requests. Defaults to 127.0.0.1:9001. +health_check_listen_address: "0.0.0.0:8000" + +# Garbage collector-specific parameters: +garbage_collection: + # How frequently to collect garbage, in seconds. + gc_frequency_s: 60 + + # The maximum number of client reports, per task, to delete in a single run of the garbage + # collector. + report_limit: 5000 + + # The maximum number of aggregation jobs (& related artifacts), per task, to delete in a single + # run of the garbage collector. + aggregation_limit: 500 + + # The maximum number of collection jobs (& related artifacts), per task, to delete in a single run + # of the garbage collector. + collection_limit: 50