diff --git a/CHANGELOG.md b/CHANGELOG.md index 1123828..f5e7ae1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ * [CHANGE] Add default instance max inflight pushes for distributors * [CHANGE] Remove mem-ballast from distributor and querier. * [CHANGE] Increase cpu requests for querier to 2. -* [CHANGE] Configure GOMAXPROCS and GOMEMLIMIT for all cortex modules based on requested cpu and memory +* [CHANGE] Configure GOMAXPROCS and GOMEMLIMIT for all cortex modules based on cpu and memory requests or limits * [CHANGE] Add default tenant shard sizes * [CHANGE] Use cortex v1.15.3 * [CHANGE] Azure storage endpoint suffix is set to `blob.core.windows.net` for backward compatibility diff --git a/cortex/alertmanager.libsonnet b/cortex/alertmanager.libsonnet index f9725ce..1e870c6 100644 --- a/cortex/alertmanager.libsonnet +++ b/cortex/alertmanager.libsonnet @@ -99,12 +99,6 @@ container.withPorts($.util.defaultPorts + mode.ports) + container.withEnvMap($.alertmanager_env_map) + container.withEnvMixin([container.envType.fromFieldPath('POD_IP', 'status.podIP')]) + - container.withEnvMixin([ - envType.withName('GOMAXPROCS') + - envType.valueFrom.resourceFieldRef.withResource('requests.cpu'), - envType.withName('GOMEMLIMIT') + - envType.valueFrom.resourceFieldRef.withResource('requests.memory'), - ]) + container.withArgsMixin( $.util.mapToFlags($.alertmanager_args) + mode.flags @@ -117,6 +111,7 @@ ) + $.util.resourcesRequests('100m', '1Gi') + $.util.readinessProbe + + $.go_container_mixin + $.jaeger_mixin else {}, diff --git a/cortex/compactor.libsonnet b/cortex/compactor.libsonnet index 2e78c4d..d12d5de 100644 --- a/cortex/compactor.libsonnet +++ b/cortex/compactor.libsonnet @@ -45,17 +45,12 @@ container.withPorts($.compactor_ports) + container.withArgsMixin($.util.mapToFlags($.compactor_args)) + container.withEnvMap($.compactor_env_map) + - container.withEnvMixin([ - envType.withName('GOMAXPROCS') + - envType.valueFrom.resourceFieldRef.withResource('requests.cpu'), - envType.withName('GOMEMLIMIT') + - envType.valueFrom.resourceFieldRef.withResource('requests.memory'), - ]) + container.withVolumeMountsMixin([volumeMount.new('compactor-data', '/data')]) + // Do not limit compactor CPU and request enough cores to honor configured max concurrency. $.util.resourcesRequests($._config.cortex_compactor_max_concurrency, '5Gi') + $.util.resourcesLimits(null, '6Gi') + $.util.readinessProbe + + $.go_container_mixin + $.jaeger_mixin, compactor_env_map:: { diff --git a/cortex/config.libsonnet b/cortex/config.libsonnet index bad810f..11c3f07 100644 --- a/cortex/config.libsonnet +++ b/cortex/config.libsonnet @@ -401,8 +401,35 @@ max_series: 4.8e+6, // Max number of series per ingester. 0 = no limit. 4.8 million is closely tied to 15Gb in requests per ingester // max_tenants: 0, // Max number of tenants per ingester. 0 = no limit. }, + + // if we disable this, we need to make sure we set the resource limits + // Disabling this can potentially increase cortex performance, + // but it will also cause performance inconsistencies + gomaxprocs_based_on_cpu_requests: true, + gomemlimit_based_on_mem_requests: true, + + gomaxprocs_resource: + if $._config.gomaxprocs_based_on_cpu_requests then + 'requests.cpu' + else + 'limits.cpu', + + gomemlimit_resource: + if $._config.gomemlimit_based_on_mem_requests then + 'requests.memory' + else + 'limits.memory', }, + go_container_mixin:: + local container = $.core.v1.container; + container.withEnvMixin([ + container.envType.withName('GOMAXPROCS') + + container.envType.valueFrom.resourceFieldRef.withResource($._config.gomaxprocs_resource), + container.envType.withName('GOMEMLIMIT') + + container.envType.valueFrom.resourceFieldRef.withResource($._config.gomemlimit_resource), + ]), + local configMap = $.core.v1.configMap, overrides_config: diff --git a/cortex/distributor.libsonnet b/cortex/distributor.libsonnet index 79c4033..2759156 100644 --- a/cortex/distributor.libsonnet +++ b/cortex/distributor.libsonnet @@ -44,16 +44,11 @@ container.new('distributor', $._images.distributor) + container.withPorts($.distributor_ports) + container.withArgsMixin($.util.mapToFlags($.distributor_args)) + - container.withEnvMixin([ - envType.withName('GOMAXPROCS') + - envType.valueFrom.resourceFieldRef.withResource('requests.cpu'), - envType.withName('GOMEMLIMIT') + - envType.valueFrom.resourceFieldRef.withResource('requests.memory'), - ]) + container.withEnvMap($.distributor_env_map) + $.util.resourcesRequests('2', '2Gi') + $.util.resourcesLimits(null, '4Gi') + $.util.readinessProbe + + $.go_container_mixin + $.jaeger_mixin, local deployment = $.apps.v1.deployment, diff --git a/cortex/flusher-job-blocks.libsonnet b/cortex/flusher-job-blocks.libsonnet index 6917a86..56264c1 100644 --- a/cortex/flusher-job-blocks.libsonnet +++ b/cortex/flusher-job-blocks.libsonnet @@ -25,11 +25,10 @@ $.util.resourcesRequests('4', '15Gi') + $.util.resourcesLimits(null, '25Gi') + $.util.readinessProbe + + $.go_container_mixin + $.jaeger_mixin, flusher_env_map:: { - GOMAXPROCS: '4', - GOMEMLIMIT: '15GiB', }, flusher_job_func(jobName, pvcName):: diff --git a/cortex/ingester.libsonnet b/cortex/ingester.libsonnet index 93f8ab5..11e22f5 100644 --- a/cortex/ingester.libsonnet +++ b/cortex/ingester.libsonnet @@ -56,15 +56,10 @@ container.withPorts($.ingester_ports) + container.withArgsMixin($.util.mapToFlags($.ingester_args)) + container.withEnvMap($.ingester_env_map) + - container.withEnvMixin([ - envType.withName('GOMAXPROCS') + - envType.valueFrom.resourceFieldRef.withResource('requests.cpu'), - envType.withName('GOMEMLIMIT') + - envType.valueFrom.resourceFieldRef.withResource('requests.memory'), - ]) + $.util.resourcesRequests('4', '15Gi') + $.util.resourcesLimits(null, '25Gi') + $.util.readinessProbe + + $.go_container_mixin + $.jaeger_mixin, ingester_deployment_labels:: {}, diff --git a/cortex/overrides-exporter.libsonnet b/cortex/overrides-exporter.libsonnet index 1f9de4e..8fbf4ac 100644 --- a/cortex/overrides-exporter.libsonnet +++ b/cortex/overrides-exporter.libsonnet @@ -20,6 +20,7 @@ container.withArgsMixin($.util.mapToFlags($.overrides_exporter_args, prefix='--')) + $.util.resourcesRequests('0.5', '0.5Gi') + $.util.readinessProbe + + $.go_container_mixin + container.mixin.readinessProbe.httpGet.withPort($.overrides_exporter_port.name), local deployment = $.apps.v1.deployment, diff --git a/cortex/querier.libsonnet b/cortex/querier.libsonnet index d58759c..15e2245 100644 --- a/cortex/querier.libsonnet +++ b/cortex/querier.libsonnet @@ -43,12 +43,7 @@ $.jaeger_mixin + $.util.readinessProbe + container.withEnvMap($.querier_env_map) + - container.withEnvMixin([ - envType.withName('GOMAXPROCS') + - envType.valueFrom.resourceFieldRef.withResource('requests.cpu'), - envType.withName('GOMEMLIMIT') + - envType.valueFrom.resourceFieldRef.withResource('requests.memory'), - ]) + + $.go_container_mixin + $.util.resourcesRequests('2', '12Gi') + $.util.resourcesLimits(null, '24Gi'), diff --git a/cortex/query-frontend.libsonnet b/cortex/query-frontend.libsonnet index e3b3667..5cbabaf 100644 --- a/cortex/query-frontend.libsonnet +++ b/cortex/query-frontend.libsonnet @@ -44,12 +44,7 @@ container.withPorts($.util.defaultPorts) + container.withArgsMixin($.util.mapToFlags($.query_frontend_args)) + container.withEnvMap($.query_frontend_env_map) + - container.withEnvMixin([ - envType.withName('GOMAXPROCS') + - envType.valueFrom.resourceFieldRef.withResource('requests.cpu'), - envType.withName('GOMEMLIMIT') + - envType.valueFrom.resourceFieldRef.withResource('requests.memory'), - ]) + + $.go_container_mixin + $.jaeger_mixin + $.util.readinessProbe + $.util.resourcesRequests('2', '600Mi') + diff --git a/cortex/query-scheduler.libsonnet b/cortex/query-scheduler.libsonnet index fab9295..8aa5cf7 100644 --- a/cortex/query-scheduler.libsonnet +++ b/cortex/query-scheduler.libsonnet @@ -19,12 +19,7 @@ container.withPorts($.util.defaultPorts) + container.withArgsMixin($.util.mapToFlags($.query_scheduler_args)) + container.withEnvMap($.query_scheduler_env_map) + - container.withEnvMixin([ - envType.withName('GOMAXPROCS') + - envType.valueFrom.resourceFieldRef.withResource('requests.cpu'), - envType.withName('GOMEMLIMIT') + - envType.valueFrom.resourceFieldRef.withResource('requests.memory'), - ]) + + $.go_container_mixin + $.jaeger_mixin + $.util.readinessProbe + $.util.resourcesRequests('2', '1Gi') + diff --git a/cortex/query-tee.libsonnet b/cortex/query-tee.libsonnet index 6a89e99..9856c34 100644 --- a/cortex/query-tee.libsonnet +++ b/cortex/query-tee.libsonnet @@ -20,13 +20,8 @@ ]) + container.withArgsMixin($.util.mapToFlags($.query_tee_args)) + container.withEnvMap($.query_tee_env_map) + - container.withEnvMixin([ - envType.withName('GOMAXPROCS') + - envType.valueFrom.resourceFieldRef.withResource('requests.cpu'), - envType.withName('GOMEMLIMIT') + - envType.valueFrom.resourceFieldRef.withResource('requests.memory'), - ]) + $.util.resourcesRequests('1', '512Mi') + + $.go_container_mixin + $.jaeger_mixin, query_tee_env_map:: { diff --git a/cortex/ruler.libsonnet b/cortex/ruler.libsonnet index 3735684..c60a874 100644 --- a/cortex/ruler.libsonnet +++ b/cortex/ruler.libsonnet @@ -40,12 +40,7 @@ container.withPorts($.util.defaultPorts) + container.withArgsMixin($.util.mapToFlags($.ruler_args)) + container.withEnvMap($.ruler_env_map) + - container.withEnvMixin([ - envType.withName('GOMAXPROCS') + - envType.valueFrom.resourceFieldRef.withResource('requests.cpu'), - envType.withName('GOMEMLIMIT') + - envType.valueFrom.resourceFieldRef.withResource('requests.memory'), - ]) + + $.go_container_mixin + $.util.resourcesRequests('2', '6Gi') + $.util.resourcesLimits('16', '16Gi') + $.util.readinessProbe + diff --git a/cortex/store-gateway.libsonnet b/cortex/store-gateway.libsonnet index 7250b25..c21ee30 100644 --- a/cortex/store-gateway.libsonnet +++ b/cortex/store-gateway.libsonnet @@ -42,12 +42,7 @@ container.withPorts($.store_gateway_ports) + container.withArgsMixin($.util.mapToFlags($.store_gateway_args)) + container.withEnvMap($.store_gateway_env_map) + - container.withEnvMixin([ - envType.withName('GOMAXPROCS') + - envType.valueFrom.resourceFieldRef.withResource('requests.cpu'), - envType.withName('GOMEMLIMIT') + - envType.valueFrom.resourceFieldRef.withResource('requests.memory'), - ]) + + $.go_container_mixin + container.withVolumeMountsMixin([volumeMount.new('store-gateway-data', '/data')]) + $.util.resourcesRequests('2', '12Gi') + $.util.resourcesLimits(null, '18Gi') + diff --git a/cortex/test-exporter.libsonnet b/cortex/test-exporter.libsonnet index 036d6fe..e7d088e 100644 --- a/cortex/test-exporter.libsonnet +++ b/cortex/test-exporter.libsonnet @@ -21,11 +21,10 @@ container.withEnvMap($.test_exporter_env_map) + $.util.resourcesRequests('100m', '100Mi') + $.util.resourcesLimits('100m', '100Mi') + + $.go_container_mixin + $.jaeger_mixin, test_exporter_env_map:: { - GOMAXPROCS: '1', - GOMEMLIMIT: '100MiB', }, local deployment = $.apps.v1.deployment,