Reorganize limits better and allow to use limits.cpu and limits.memor…

…y too (#37) Signed-off-by: Friedrich Gonzalez <[email protected]>
cortexproject · Nov 24, 2023 · 272aaee · 272aaee
1 parent 006c8fb
commit 272aaee
Show file tree

Hide file tree

Showing 15 changed files with 41 additions and 65 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,7 +5,7 @@
 * [CHANGE] Add default instance max inflight pushes for distributors
 * [CHANGE] Remove mem-ballast from distributor and querier.
 * [CHANGE] Increase cpu requests for querier to 2.
-* [CHANGE] Configure GOMAXPROCS and GOMEMLIMIT for all cortex modules based on requested cpu and memory
+* [CHANGE] Configure GOMAXPROCS and GOMEMLIMIT for all cortex modules based on cpu and memory requests or limits
 * [CHANGE] Add default tenant shard sizes
 * [CHANGE] Use cortex v1.15.3
 * [CHANGE] Azure storage endpoint suffix is set to `blob.core.windows.net` for backward compatibility

diff --git a/cortex/alertmanager.libsonnet b/cortex/alertmanager.libsonnet
@@ -99,12 +99,6 @@
       container.withPorts($.util.defaultPorts + mode.ports) +
       container.withEnvMap($.alertmanager_env_map) +
       container.withEnvMixin([container.envType.fromFieldPath('POD_IP', 'status.podIP')]) +
-      container.withEnvMixin([
-        envType.withName('GOMAXPROCS') +
-        envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
-        envType.withName('GOMEMLIMIT') +
-        envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
-      ]) +
       container.withArgsMixin(
         $.util.mapToFlags($.alertmanager_args) +
         mode.flags
@@ -117,6 +111,7 @@
       ) +
       $.util.resourcesRequests('100m', '1Gi') +
       $.util.readinessProbe +
+      $.go_container_mixin +
       $.jaeger_mixin
     else {},
 

diff --git a/cortex/compactor.libsonnet b/cortex/compactor.libsonnet
@@ -45,17 +45,12 @@
     container.withPorts($.compactor_ports) +
     container.withArgsMixin($.util.mapToFlags($.compactor_args)) +
     container.withEnvMap($.compactor_env_map) +
-    container.withEnvMixin([
-      envType.withName('GOMAXPROCS') +
-      envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
-      envType.withName('GOMEMLIMIT') +
-      envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
-    ]) +
     container.withVolumeMountsMixin([volumeMount.new('compactor-data', '/data')]) +
     // Do not limit compactor CPU and request enough cores to honor configured max concurrency.
     $.util.resourcesRequests($._config.cortex_compactor_max_concurrency, '5Gi') +
     $.util.resourcesLimits(null, '6Gi') +
     $.util.readinessProbe +
+    $.go_container_mixin +
     $.jaeger_mixin,
 
   compactor_env_map:: {

diff --git a/cortex/config.libsonnet b/cortex/config.libsonnet
@@ -401,8 +401,35 @@
       max_series: 4.8e+6,  // Max number of series per ingester. 0 = no limit. 4.8 million is closely tied to 15Gb in requests per ingester
       // max_tenants: 0,  // Max number of tenants per ingester. 0 = no limit.
     },
+
+    // if we disable this, we need to make sure we set the resource limits
+    // Disabling this can potentially increase cortex performance,
+    // but it will also cause performance inconsistencies
+    gomaxprocs_based_on_cpu_requests: true,
+    gomemlimit_based_on_mem_requests: true,
+
+    gomaxprocs_resource:
+      if $._config.gomaxprocs_based_on_cpu_requests then
+        'requests.cpu'
+      else
+        'limits.cpu',
+
+    gomemlimit_resource:
+      if $._config.gomemlimit_based_on_mem_requests then
+        'requests.memory'
+      else
+        'limits.memory',
   },
 
+  go_container_mixin::
+    local container = $.core.v1.container;
+    container.withEnvMixin([
+      container.envType.withName('GOMAXPROCS') +
+      container.envType.valueFrom.resourceFieldRef.withResource($._config.gomaxprocs_resource),
+      container.envType.withName('GOMEMLIMIT') +
+      container.envType.valueFrom.resourceFieldRef.withResource($._config.gomemlimit_resource),
+    ]),
+
   local configMap = $.core.v1.configMap,
 
   overrides_config:

diff --git a/cortex/distributor.libsonnet b/cortex/distributor.libsonnet
@@ -44,16 +44,11 @@
     container.new('distributor', $._images.distributor) +
     container.withPorts($.distributor_ports) +
     container.withArgsMixin($.util.mapToFlags($.distributor_args)) +
-    container.withEnvMixin([
-      envType.withName('GOMAXPROCS') +
-      envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
-      envType.withName('GOMEMLIMIT') +
-      envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
-    ]) +
     container.withEnvMap($.distributor_env_map) +
     $.util.resourcesRequests('2', '2Gi') +
     $.util.resourcesLimits(null, '4Gi') +
     $.util.readinessProbe +
+    $.go_container_mixin +
     $.jaeger_mixin,
 
   local deployment = $.apps.v1.deployment,

diff --git a/cortex/flusher-job-blocks.libsonnet b/cortex/flusher-job-blocks.libsonnet
@@ -25,11 +25,10 @@
     $.util.resourcesRequests('4', '15Gi') +
     $.util.resourcesLimits(null, '25Gi') +
     $.util.readinessProbe +
+    $.go_container_mixin +
     $.jaeger_mixin,
 
   flusher_env_map:: {
-    GOMAXPROCS: '4',
-    GOMEMLIMIT: '15GiB',
   },
 
   flusher_job_func(jobName, pvcName)::

diff --git a/cortex/ingester.libsonnet b/cortex/ingester.libsonnet
@@ -56,15 +56,10 @@
     container.withPorts($.ingester_ports) +
     container.withArgsMixin($.util.mapToFlags($.ingester_args)) +
     container.withEnvMap($.ingester_env_map) +
-    container.withEnvMixin([
-      envType.withName('GOMAXPROCS') +
-      envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
-      envType.withName('GOMEMLIMIT') +
-      envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
-    ]) +
     $.util.resourcesRequests('4', '15Gi') +
     $.util.resourcesLimits(null, '25Gi') +
     $.util.readinessProbe +
+    $.go_container_mixin +
     $.jaeger_mixin,
 
   ingester_deployment_labels:: {},

diff --git a/cortex/overrides-exporter.libsonnet b/cortex/overrides-exporter.libsonnet
@@ -20,6 +20,7 @@
     container.withArgsMixin($.util.mapToFlags($.overrides_exporter_args, prefix='--')) +
     $.util.resourcesRequests('0.5', '0.5Gi') +
     $.util.readinessProbe +
+    $.go_container_mixin +
     container.mixin.readinessProbe.httpGet.withPort($.overrides_exporter_port.name),
 
   local deployment = $.apps.v1.deployment,

diff --git a/cortex/querier.libsonnet b/cortex/querier.libsonnet
@@ -43,12 +43,7 @@
     $.jaeger_mixin +
     $.util.readinessProbe +
     container.withEnvMap($.querier_env_map) +
-    container.withEnvMixin([
-      envType.withName('GOMAXPROCS') +
-      envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
-      envType.withName('GOMEMLIMIT') +
-      envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
-    ]) +
+    $.go_container_mixin +
     $.util.resourcesRequests('2', '12Gi') +
     $.util.resourcesLimits(null, '24Gi'),
 

diff --git a/cortex/query-frontend.libsonnet b/cortex/query-frontend.libsonnet
@@ -44,12 +44,7 @@
     container.withPorts($.util.defaultPorts) +
     container.withArgsMixin($.util.mapToFlags($.query_frontend_args)) +
     container.withEnvMap($.query_frontend_env_map) +
-    container.withEnvMixin([
-      envType.withName('GOMAXPROCS') +
-      envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
-      envType.withName('GOMEMLIMIT') +
-      envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
-    ]) +
+    $.go_container_mixin +
     $.jaeger_mixin +
     $.util.readinessProbe +
     $.util.resourcesRequests('2', '600Mi') +

diff --git a/cortex/query-scheduler.libsonnet b/cortex/query-scheduler.libsonnet
@@ -19,12 +19,7 @@
     container.withPorts($.util.defaultPorts) +
     container.withArgsMixin($.util.mapToFlags($.query_scheduler_args)) +
     container.withEnvMap($.query_scheduler_env_map) +
-    container.withEnvMixin([
-      envType.withName('GOMAXPROCS') +
-      envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
-      envType.withName('GOMEMLIMIT') +
-      envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
-    ]) +
+    $.go_container_mixin +
     $.jaeger_mixin +
     $.util.readinessProbe +
     $.util.resourcesRequests('2', '1Gi') +

diff --git a/cortex/query-tee.libsonnet b/cortex/query-tee.libsonnet
@@ -20,13 +20,8 @@
     ]) +
     container.withArgsMixin($.util.mapToFlags($.query_tee_args)) +
     container.withEnvMap($.query_tee_env_map) +
-    container.withEnvMixin([
-      envType.withName('GOMAXPROCS') +
-      envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
-      envType.withName('GOMEMLIMIT') +
-      envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
-    ]) +
     $.util.resourcesRequests('1', '512Mi') +
+    $.go_container_mixin +
     $.jaeger_mixin,
 
   query_tee_env_map:: {

diff --git a/cortex/ruler.libsonnet b/cortex/ruler.libsonnet
@@ -40,12 +40,7 @@
       container.withPorts($.util.defaultPorts) +
       container.withArgsMixin($.util.mapToFlags($.ruler_args)) +
       container.withEnvMap($.ruler_env_map) +
-      container.withEnvMixin([
-        envType.withName('GOMAXPROCS') +
-        envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
-        envType.withName('GOMEMLIMIT') +
-        envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
-      ]) +
+      $.go_container_mixin +
       $.util.resourcesRequests('2', '6Gi') +
       $.util.resourcesLimits('16', '16Gi') +
       $.util.readinessProbe +

diff --git a/cortex/store-gateway.libsonnet b/cortex/store-gateway.libsonnet
@@ -42,12 +42,7 @@
     container.withPorts($.store_gateway_ports) +
     container.withArgsMixin($.util.mapToFlags($.store_gateway_args)) +
     container.withEnvMap($.store_gateway_env_map) +
-    container.withEnvMixin([
-      envType.withName('GOMAXPROCS') +
-      envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
-      envType.withName('GOMEMLIMIT') +
-      envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
-    ]) +
+    $.go_container_mixin +
     container.withVolumeMountsMixin([volumeMount.new('store-gateway-data', '/data')]) +
     $.util.resourcesRequests('2', '12Gi') +
     $.util.resourcesLimits(null, '18Gi') +

diff --git a/cortex/test-exporter.libsonnet b/cortex/test-exporter.libsonnet
@@ -21,11 +21,10 @@
       container.withEnvMap($.test_exporter_env_map) +
       $.util.resourcesRequests('100m', '100Mi') +
       $.util.resourcesLimits('100m', '100Mi') +
+      $.go_container_mixin +
       $.jaeger_mixin,
 
   test_exporter_env_map:: {
-    GOMAXPROCS: '1',
-    GOMEMLIMIT: '100MiB',
   },
 
   local deployment = $.apps.v1.deployment,