Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reorganize limits better and allow to use limits.cpu and limits.memory too #37

Merged
merged 1 commit into from
Nov 24, 2023
Merged

Reorganize limits better and allow to use limits.cpu and limits.memory too #37

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
* [CHANGE] Add default instance max inflight pushes for distributors
* [CHANGE] Remove mem-ballast from distributor and querier.
* [CHANGE] Increase cpu requests for querier to 2.
* [CHANGE] Configure GOMAXPROCS and GOMEMLIMIT for all cortex modules based on requested cpu and memory
* [CHANGE] Configure GOMAXPROCS and GOMEMLIMIT for all cortex modules based on cpu and memory requests or limits
* [CHANGE] Add default tenant shard sizes
* [CHANGE] Use cortex v1.15.3
* [CHANGE] Azure storage endpoint suffix is set to `blob.core.windows.net` for backward compatibility
Expand Down
7 changes: 1 addition & 6 deletions cortex/alertmanager.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,12 +99,6 @@
container.withPorts($.util.defaultPorts + mode.ports) +
container.withEnvMap($.alertmanager_env_map) +
container.withEnvMixin([container.envType.fromFieldPath('POD_IP', 'status.podIP')]) +
container.withEnvMixin([
envType.withName('GOMAXPROCS') +
envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
envType.withName('GOMEMLIMIT') +
envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
]) +
container.withArgsMixin(
$.util.mapToFlags($.alertmanager_args) +
mode.flags
Expand All @@ -117,6 +111,7 @@
) +
$.util.resourcesRequests('100m', '1Gi') +
$.util.readinessProbe +
$.go_container_mixin +
$.jaeger_mixin
else {},

Expand Down
7 changes: 1 addition & 6 deletions cortex/compactor.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,17 +45,12 @@
container.withPorts($.compactor_ports) +
container.withArgsMixin($.util.mapToFlags($.compactor_args)) +
container.withEnvMap($.compactor_env_map) +
container.withEnvMixin([
envType.withName('GOMAXPROCS') +
envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
envType.withName('GOMEMLIMIT') +
envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
]) +
container.withVolumeMountsMixin([volumeMount.new('compactor-data', '/data')]) +
// Do not limit compactor CPU and request enough cores to honor configured max concurrency.
$.util.resourcesRequests($._config.cortex_compactor_max_concurrency, '5Gi') +
$.util.resourcesLimits(null, '6Gi') +
$.util.readinessProbe +
$.go_container_mixin +
$.jaeger_mixin,

compactor_env_map:: {
Expand Down
27 changes: 27 additions & 0 deletions cortex/config.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -401,8 +401,35 @@
max_series: 4.8e+6, // Max number of series per ingester. 0 = no limit. 4.8 million is closely tied to 15Gb in requests per ingester
// max_tenants: 0, // Max number of tenants per ingester. 0 = no limit.
},

// if we disable this, we need to make sure we set the resource limits
// Disabling this can potentially increase cortex performance,
// but it will also cause performance inconsistencies
gomaxprocs_based_on_cpu_requests: true,
gomemlimit_based_on_mem_requests: true,

gomaxprocs_resource:
if $._config.gomaxprocs_based_on_cpu_requests then
'requests.cpu'
else
'limits.cpu',

gomemlimit_resource:
if $._config.gomemlimit_based_on_mem_requests then
'requests.memory'
else
'limits.memory',
},

go_container_mixin::
local container = $.core.v1.container;
container.withEnvMixin([
container.envType.withName('GOMAXPROCS') +
container.envType.valueFrom.resourceFieldRef.withResource($._config.gomaxprocs_resource),
container.envType.withName('GOMEMLIMIT') +
container.envType.valueFrom.resourceFieldRef.withResource($._config.gomemlimit_resource),
]),

local configMap = $.core.v1.configMap,

overrides_config:
Expand Down
7 changes: 1 addition & 6 deletions cortex/distributor.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,16 +44,11 @@
container.new('distributor', $._images.distributor) +
container.withPorts($.distributor_ports) +
container.withArgsMixin($.util.mapToFlags($.distributor_args)) +
container.withEnvMixin([
envType.withName('GOMAXPROCS') +
envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
envType.withName('GOMEMLIMIT') +
envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
]) +
container.withEnvMap($.distributor_env_map) +
$.util.resourcesRequests('2', '2Gi') +
$.util.resourcesLimits(null, '4Gi') +
$.util.readinessProbe +
$.go_container_mixin +
$.jaeger_mixin,

local deployment = $.apps.v1.deployment,
Expand Down
3 changes: 1 addition & 2 deletions cortex/flusher-job-blocks.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,10 @@
$.util.resourcesRequests('4', '15Gi') +
$.util.resourcesLimits(null, '25Gi') +
$.util.readinessProbe +
$.go_container_mixin +
$.jaeger_mixin,

flusher_env_map:: {
GOMAXPROCS: '4',
GOMEMLIMIT: '15GiB',
},

flusher_job_func(jobName, pvcName)::
Expand Down
7 changes: 1 addition & 6 deletions cortex/ingester.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,15 +56,10 @@
container.withPorts($.ingester_ports) +
container.withArgsMixin($.util.mapToFlags($.ingester_args)) +
container.withEnvMap($.ingester_env_map) +
container.withEnvMixin([
envType.withName('GOMAXPROCS') +
envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
envType.withName('GOMEMLIMIT') +
envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
]) +
$.util.resourcesRequests('4', '15Gi') +
$.util.resourcesLimits(null, '25Gi') +
$.util.readinessProbe +
$.go_container_mixin +
$.jaeger_mixin,

ingester_deployment_labels:: {},
Expand Down
1 change: 1 addition & 0 deletions cortex/overrides-exporter.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
container.withArgsMixin($.util.mapToFlags($.overrides_exporter_args, prefix='--')) +
$.util.resourcesRequests('0.5', '0.5Gi') +
$.util.readinessProbe +
$.go_container_mixin +
container.mixin.readinessProbe.httpGet.withPort($.overrides_exporter_port.name),

local deployment = $.apps.v1.deployment,
Expand Down
7 changes: 1 addition & 6 deletions cortex/querier.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,12 +43,7 @@
$.jaeger_mixin +
$.util.readinessProbe +
container.withEnvMap($.querier_env_map) +
container.withEnvMixin([
envType.withName('GOMAXPROCS') +
envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
envType.withName('GOMEMLIMIT') +
envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
]) +
$.go_container_mixin +
$.util.resourcesRequests('2', '12Gi') +
$.util.resourcesLimits(null, '24Gi'),

Expand Down
7 changes: 1 addition & 6 deletions cortex/query-frontend.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,12 +44,7 @@
container.withPorts($.util.defaultPorts) +
container.withArgsMixin($.util.mapToFlags($.query_frontend_args)) +
container.withEnvMap($.query_frontend_env_map) +
container.withEnvMixin([
envType.withName('GOMAXPROCS') +
envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
envType.withName('GOMEMLIMIT') +
envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
]) +
$.go_container_mixin +
$.jaeger_mixin +
$.util.readinessProbe +
$.util.resourcesRequests('2', '600Mi') +
Expand Down
7 changes: 1 addition & 6 deletions cortex/query-scheduler.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,7 @@
container.withPorts($.util.defaultPorts) +
container.withArgsMixin($.util.mapToFlags($.query_scheduler_args)) +
container.withEnvMap($.query_scheduler_env_map) +
container.withEnvMixin([
envType.withName('GOMAXPROCS') +
envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
envType.withName('GOMEMLIMIT') +
envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
]) +
$.go_container_mixin +
$.jaeger_mixin +
$.util.readinessProbe +
$.util.resourcesRequests('2', '1Gi') +
Expand Down
7 changes: 1 addition & 6 deletions cortex/query-tee.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,8 @@
]) +
container.withArgsMixin($.util.mapToFlags($.query_tee_args)) +
container.withEnvMap($.query_tee_env_map) +
container.withEnvMixin([
envType.withName('GOMAXPROCS') +
envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
envType.withName('GOMEMLIMIT') +
envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
]) +
$.util.resourcesRequests('1', '512Mi') +
$.go_container_mixin +
$.jaeger_mixin,

query_tee_env_map:: {
Expand Down
7 changes: 1 addition & 6 deletions cortex/ruler.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,12 +40,7 @@
container.withPorts($.util.defaultPorts) +
container.withArgsMixin($.util.mapToFlags($.ruler_args)) +
container.withEnvMap($.ruler_env_map) +
container.withEnvMixin([
envType.withName('GOMAXPROCS') +
envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
envType.withName('GOMEMLIMIT') +
envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
]) +
$.go_container_mixin +
$.util.resourcesRequests('2', '6Gi') +
$.util.resourcesLimits('16', '16Gi') +
$.util.readinessProbe +
Expand Down
7 changes: 1 addition & 6 deletions cortex/store-gateway.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,12 +42,7 @@
container.withPorts($.store_gateway_ports) +
container.withArgsMixin($.util.mapToFlags($.store_gateway_args)) +
container.withEnvMap($.store_gateway_env_map) +
container.withEnvMixin([
envType.withName('GOMAXPROCS') +
envType.valueFrom.resourceFieldRef.withResource('requests.cpu'),
envType.withName('GOMEMLIMIT') +
envType.valueFrom.resourceFieldRef.withResource('requests.memory'),
]) +
$.go_container_mixin +
container.withVolumeMountsMixin([volumeMount.new('store-gateway-data', '/data')]) +
$.util.resourcesRequests('2', '12Gi') +
$.util.resourcesLimits(null, '18Gi') +
Expand Down
3 changes: 1 addition & 2 deletions cortex/test-exporter.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,10 @@
container.withEnvMap($.test_exporter_env_map) +
$.util.resourcesRequests('100m', '100Mi') +
$.util.resourcesLimits('100m', '100Mi') +
$.go_container_mixin +
$.jaeger_mixin,

test_exporter_env_map:: {
GOMAXPROCS: '1',
GOMEMLIMIT: '100MiB',
},

local deployment = $.apps.v1.deployment,
Expand Down