Configure GOMAXPROCS and GOMEMLIMIT based on requests cpu and memory

Signed-off-by: Friedrich Gonzalez <[email protected]>
cortexproject · Nov 20, 2023 · 524c3b3 · 524c3b3
1 parent c35087e
commit 524c3b3
Show file tree

Hide file tree

Showing 10 changed files with 74 additions and 24 deletions.
diff --git a/cortex/alertmanager.libsonnet b/cortex/alertmanager.libsonnet
@@ -3,6 +3,7 @@
   local volumeMount = $.core.v1.volumeMount,
   local volume = $.core.v1.volume,
   local container = $.core.v1.container,
+  local envType = container.envType,
   local statefulSet = $.apps.v1.statefulSet,
   local service = $.core.v1.service,
   local configMap = $.core.v1.configMap,
@@ -98,6 +99,12 @@
       container.withPorts($.util.defaultPorts + mode.ports) +
       container.withEnvMap($.alertmanager_env_map) +
       container.withEnvMixin([container.envType.fromFieldPath('POD_IP', 'status.podIP')]) +
+      container.withEnvMixin([
+        envType.withName('GOMAXPROCS') +
+        envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
+        envType.withName('GOMEMLIMIT') +
+        envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
+      ]) +
       container.withArgsMixin(
         $.util.mapToFlags($.alertmanager_args) +
         mode.flags
@@ -108,14 +115,12 @@
           [volumeMount.new('alertmanager-fallback-config', '/configs')]
         else []
       ) +
-      $.util.resourcesRequests('100m', '1Gi') +
+      $.util.resourcesRequests('1', '1Gi') +
       $.util.readinessProbe +
       $.jaeger_mixin
     else {},
 
   alertmanager_env_map:: {
-    GOMAXPROCS: '1',
-    GOMEMLIMIT: '1GiB',
   },
 
   alertmanager_statefulset:

diff --git a/cortex/compactor.libsonnet b/cortex/compactor.libsonnet
@@ -1,5 +1,6 @@
 {
   local container = $.core.v1.container,
+  local envType = container.envType,
   local pvc = $.core.v1.persistentVolumeClaim,
   local statefulSet = $.apps.v1.statefulSet,
   local volumeMount = $.core.v1.volumeMount,
@@ -44,16 +45,20 @@
     container.withPorts($.compactor_ports) +
     container.withArgsMixin($.util.mapToFlags($.compactor_args)) +
     container.withEnvMap($.compactor_env_map) +
+    container.withEnvMixin([
+      envType.withName('GOMAXPROCS') +
+      envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
+      envType.withName('GOMEMLIMIT') +
+      envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
+    ]) +
     container.withVolumeMountsMixin([volumeMount.new('compactor-data', '/data')]) +
     // Do not limit compactor CPU and request enough cores to honor configured max concurrency.
-    $.util.resourcesRequests($._config.cortex_compactor_max_concurrency, '6Gi') +
+    $.util.resourcesRequests($._config.cortex_compactor_max_concurrency, '5Gi') +
     $.util.resourcesLimits(null, '6Gi') +
     $.util.readinessProbe +
     $.jaeger_mixin,
 
   compactor_env_map:: {
-    GOMAXPROCS: std.toString($._config.cortex_compactor_max_concurrency),
-    GOMEMLIMIT: '5GiB',
   },
 
   newCompactorStatefulSet(name, container)::

diff --git a/cortex/distributor.libsonnet b/cortex/distributor.libsonnet
@@ -1,5 +1,6 @@
 {
   local container = $.core.v1.container,
+  local envType = container.envType,
   local containerPort = $.core.v1.containerPort,
 
   distributor_args::
@@ -35,8 +36,6 @@
     },
 
   distributor_env_map:: {
-    GOMAXPROCS: '2',
-    GOMEMLIMIT: '2GiB',
   },
 
   distributor_ports:: $.util.defaultPorts,
@@ -45,6 +44,12 @@
     container.new('distributor', $._images.distributor) +
     container.withPorts($.distributor_ports) +
     container.withArgsMixin($.util.mapToFlags($.distributor_args)) +
+    container.withEnvMixin([
+      envType.withName('GOMAXPROCS') +
+      envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
+      envType.withName('GOMEMLIMIT') +
+      envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
+    ]) +
     container.withEnvMap($.distributor_env_map) +
     $.util.resourcesRequests('2', '2Gi') +
     $.util.resourcesLimits(null, '4Gi') +

diff --git a/cortex/ingester.libsonnet b/cortex/ingester.libsonnet
@@ -49,12 +49,19 @@
 
   local name = 'ingester',
   local container = $.core.v1.container,
+  local envType = container.envType,
 
   ingester_container::
     container.new(name, $._images.ingester) +
     container.withPorts($.ingester_ports) +
     container.withArgsMixin($.util.mapToFlags($.ingester_args)) +
     container.withEnvMap($.ingester_env_map) +
+    container.withEnvMixin([
+      envType.withName('GOMAXPROCS') +
+      envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
+      envType.withName('GOMEMLIMIT') +
+      envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
+    ]) +
     $.util.resourcesRequests('4', '15Gi') +
     $.util.resourcesLimits(null, '25Gi') +
     $.util.readinessProbe +
@@ -63,8 +70,6 @@
   ingester_deployment_labels:: {},
 
   ingester_env_map:: {
-    GOMAXPROCS: '4',
-    GOMEMLIMIT: '15GiB',
   },
 
   local ingester_pvc =

diff --git a/cortex/querier.libsonnet b/cortex/querier.libsonnet
@@ -1,5 +1,6 @@
 {
   local container = $.core.v1.container,
+  local envType = container.envType,
 
   querier_args::
     $._config.grpcConfig +
@@ -32,8 +33,6 @@
   querier_ports:: $.util.defaultPorts,
 
   querier_env_map:: {
-    GOMAXPROCS: '2',
-    GOMEMLIMIT: '12GiB',
     JAEGER_REPORTER_MAX_QUEUE_SIZE: '1024',  // Default is 100.
   },
 
@@ -44,6 +43,12 @@
     $.jaeger_mixin +
     $.util.readinessProbe +
     container.withEnvMap($.querier_env_map) +
+    container.withEnvMixin([
+      envType.withName('GOMAXPROCS') +
+      envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
+      envType.withName('GOMEMLIMIT') +
+      envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
+    ]) +
     $.util.resourcesRequests('2', '12Gi') +
     $.util.resourcesLimits(null, '24Gi'),
 

diff --git a/cortex/query-frontend.libsonnet b/cortex/query-frontend.libsonnet
@@ -1,5 +1,6 @@
 {
   local container = $.core.v1.container,
+  local envType = container.envType,
 
   query_frontend_args::
     $._config.grpcConfig
@@ -43,14 +44,18 @@
     container.withPorts($.util.defaultPorts) +
     container.withArgsMixin($.util.mapToFlags($.query_frontend_args)) +
     container.withEnvMap($.query_frontend_env_map) +
+    container.withEnvMixin([
+      envType.withName('GOMAXPROCS') +
+      envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
+      envType.withName('GOMEMLIMIT') +
+      envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
+    ]) +
     $.jaeger_mixin +
     $.util.readinessProbe +
     $.util.resourcesRequests('2', '600Mi') +
     $.util.resourcesLimits(null, '1200Mi'),
 
   query_frontend_env_map:: {
-    GOMAXPROCS: '2',
-    GOMEMLIMIT: '600MiB',
   },
 
   local deployment = $.apps.v1.deployment,

diff --git a/cortex/query-scheduler.libsonnet b/cortex/query-scheduler.libsonnet
@@ -3,6 +3,7 @@
 {
   local container = $.core.v1.container,
   local deployment = $.apps.v1.deployment,
+  local envType = container.envType,
   local service = $.core.v1.service,
 
   query_scheduler_args+::
@@ -18,6 +19,12 @@
     container.withPorts($.util.defaultPorts) +
     container.withArgsMixin($.util.mapToFlags($.query_scheduler_args)) +
     container.withEnvMap($.query_scheduler_env_map) +
+    container.withEnvMixin([
+      envType.withName('GOMAXPROCS') +
+      envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
+      envType.withName('GOMEMLIMIT') +
+      envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
+    ]) +
     $.jaeger_mixin +
     $.util.readinessProbe +
     $.util.resourcesRequests('2', '1Gi') +
@@ -32,8 +39,6 @@
     deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1),
 
   query_scheduler_env_map:: {
-    GOMAXPROCS: '2',
-    GOMEMLIMIT: '1GiB',
   },
 
   query_scheduler_deployment: if !$._config.query_scheduler_enabled then {} else

diff --git a/cortex/query-tee.libsonnet b/cortex/query-tee.libsonnet
@@ -2,6 +2,7 @@
   local container = $.core.v1.container,
   local containerPort = $.core.v1.containerPort,
   local deployment = $.apps.v1.deployment,
+  local envType = container.envType,
   local service = $.core.v1.service,
   local servicePort = $.core.v1.servicePort,
 
@@ -19,12 +20,16 @@
     ]) +
     container.withArgsMixin($.util.mapToFlags($.query_tee_args)) +
     container.withEnvMap($.query_tee_env_map) +
+    container.withEnvMixin([
+      envType.withName('GOMAXPROCS') +
+      envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
+      envType.withName('GOMEMLIMIT') +
+      envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
+    ]) +
     $.util.resourcesRequests('1', '512Mi') +
     $.jaeger_mixin,
 
   query_tee_env_map:: {
-    GOMAXPROCS: '1',
-    GOMEMLIMIT: '512MiB',
   },
 
   query_tee_deployment: if !($._config.query_tee_enabled) then {} else

diff --git a/cortex/ruler.libsonnet b/cortex/ruler.libsonnet
@@ -1,5 +1,6 @@
 {
   local container = $.core.v1.container,
+  local envType = container.envType,
 
   ruler_args::
     $._config.grpcConfig +
@@ -39,7 +40,13 @@
       container.withPorts($.util.defaultPorts) +
       container.withArgsMixin($.util.mapToFlags($.ruler_args)) +
       container.withEnvMap($.ruler_env_map) +
-      $.util.resourcesRequests('1', '6Gi') +
+      container.withEnvMixin([
+        envType.withName('GOMAXPROCS') +
+        envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
+        envType.withName('GOMEMLIMIT') +
+        envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
+      ]) +
+      $.util.resourcesRequests('2', '6Gi') +
       $.util.resourcesLimits('16', '16Gi') +
       $.util.readinessProbe +
       $.jaeger_mixin
@@ -58,8 +65,6 @@
     else {},
 
   ruler_env_map:: {
-    GOMAXPROCS: '2',
-    GOMEMLIMIT: '6GiB',
   },
 
   local service = $.core.v1.service,

diff --git a/cortex/store-gateway.libsonnet b/cortex/store-gateway.libsonnet
@@ -1,5 +1,6 @@
 {
   local container = $.core.v1.container,
+  local envType = container.envType,
   local podDisruptionBudget = $.policy.v1.podDisruptionBudget,
   local pvc = $.core.v1.persistentVolumeClaim,
   local statefulSet = $.apps.v1.statefulSet,
@@ -41,15 +42,19 @@
     container.withPorts($.store_gateway_ports) +
     container.withArgsMixin($.util.mapToFlags($.store_gateway_args)) +
     container.withEnvMap($.store_gateway_env_map) +
+    container.withEnvMixin([
+      envType.withName('GOMAXPROCS') +
+      envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
+      envType.withName('GOMEMLIMIT') +
+      envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
+    ]) +
     container.withVolumeMountsMixin([volumeMount.new('store-gateway-data', '/data')]) +
-    $.util.resourcesRequests('1', '12Gi') +
+    $.util.resourcesRequests('2', '12Gi') +
     $.util.resourcesLimits(null, '18Gi') +
     $.util.readinessProbe +
     $.jaeger_mixin,
 
   store_gateway_env_map:: {
-    GOMAXPROCS: '2',
-    GOMEMLIMIT: '12GiB',
   },
 
   newStoreGatewayStatefulSet(name, container)::