From 4bde8b0629b72a708ceb5c641613875f84f57b38 Mon Sep 17 00:00:00 2001 From: Narsing Metpally Date: Wed, 25 Sep 2024 12:43:12 -0600 Subject: [PATCH] Increase CortexProvisioningTooManyWrites alert threshold to 160k (#60) * Increase CortexProvisioningTooManyWrites alert threshold to 160k --- CHANGELOG.md | 1 + cortex-mixin/alerts/alerts.libsonnet | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6895ec..74252e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ * [CHANGE] Remove chunks support for dashboards * [CHANGE] Target 3M memory series per ingester instead of 1.5M * [CHANGE] Update jsonnet-libs to Fri Jul 19 12:51:49 2024 #57 +* [CHANGE] Increase CortexProvisioningTooManyWrites alert threshold to 160e3 * [ENHANCEMENT] Configure `-ingester.client.grpc-compression` to be `snappy-block` * [ENHANCEMENT] Support Grafana 11 in Cortex Service Scaling Dashboard diff --git a/cortex-mixin/alerts/alerts.libsonnet b/cortex-mixin/alerts/alerts.libsonnet index 7145d02..ec44565 100644 --- a/cortex-mixin/alerts/alerts.libsonnet +++ b/cortex-mixin/alerts/alerts.libsonnet @@ -407,9 +407,9 @@ }, { alert: 'CortexProvisioningTooManyWrites', - // 80k writes / s per ingester max. + // 160k writes / s per ingester max. expr: ||| - avg by (%s) (rate(cortex_ingester_ingested_samples_total[1m])) > 80e3 + avg by (%s) (rate(cortex_ingester_ingested_samples_total[1m])) > 160e3 ||| % $._config.alert_aggregation_labels, 'for': '15m', labels: {