From c924e39dacf376c72770b6862eaaeff07a3a3d92 Mon Sep 17 00:00:00 2001 From: haasad Date: Wed, 10 Apr 2024 00:35:27 +0200 Subject: [PATCH 1/2] Use amtool in the alertmanager pod instead of silencing job --- .../how-tos/network/migrate-to-cilium.adoc | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/docs/modules/ROOT/pages/how-tos/network/migrate-to-cilium.adoc b/docs/modules/ROOT/pages/how-tos/network/migrate-to-cilium.adoc index 31dece09..265a8676 100644 --- a/docs/modules/ROOT/pages/how-tos/network/migrate-to-cilium.adoc +++ b/docs/modules/ROOT/pages/how-tos/network/migrate-to-cilium.adoc @@ -12,8 +12,17 @@ IMPORTANT: Make sure that your `$KUBECONFIG` points to the cluster you want to migrate before starting. -:duration: +120 minutes -include::partial$create-alertmanager-silence-all-projectsyn.adoc[] +. Silence all Project Syn alerts ++ +[source,bash] +---- +silence_id=$( + kubectl --as=cluster-admin -n openshift-monitoring exec \ + sts/alertmanager-main -- amtool --alertmanager.url=http://localhost:9093 \ + silence add alertname!=Watchdog --duration="3h" -c "cilium migration" +) +echo $silence_id +---- . Select cluster + @@ -268,4 +277,10 @@ include::partial$enable-argocd-autosync.adoc[] == Cleanup alert silence -include::partial$remove-alertmanager-silence-all-projectsyn.adoc[] +. Expire the silence ++ +[source,bash] +---- +kubectl --as=cluster-admin -n openshift-monitoring exec sts/alertmanager-main --\ + amtool --alertmanager.url=http://localhost:9093 silence expire $silence_id +---- From c97a6b9257e6016d1d8e47a0f7dff2f51d6e61c6 Mon Sep 17 00:00:00 2001 From: haasad Date: Mon, 15 Apr 2024 13:53:16 +0200 Subject: [PATCH 2/2] Move amtool commands to their own partials --- .../how-tos/network/migrate-to-cilium.adoc | 23 ++++--------------- .../ROOT/partials/create-amtool-silence.adoc | 12 ++++++++++ .../ROOT/partials/expire-amtool-silence.adoc | 8 +++++++ 3 files changed, 25 insertions(+), 18 deletions(-) create mode 100644 docs/modules/ROOT/partials/create-amtool-silence.adoc create mode 100644 docs/modules/ROOT/partials/expire-amtool-silence.adoc diff --git a/docs/modules/ROOT/pages/how-tos/network/migrate-to-cilium.adoc b/docs/modules/ROOT/pages/how-tos/network/migrate-to-cilium.adoc index 265a8676..18f58dca 100644 --- a/docs/modules/ROOT/pages/how-tos/network/migrate-to-cilium.adoc +++ b/docs/modules/ROOT/pages/how-tos/network/migrate-to-cilium.adoc @@ -12,17 +12,10 @@ IMPORTANT: Make sure that your `$KUBECONFIG` points to the cluster you want to migrate before starting. -. Silence all Project Syn alerts -+ -[source,bash] ----- -silence_id=$( - kubectl --as=cluster-admin -n openshift-monitoring exec \ - sts/alertmanager-main -- amtool --alertmanager.url=http://localhost:9093 \ - silence add alertname!=Watchdog --duration="3h" -c "cilium migration" -) -echo $silence_id ----- +:alert_statement: alertname!=Watchdog +:duration: 3h +:comment: cilium migration +include::partial$create-amtool-silence.adoc[] . Select cluster + @@ -277,10 +270,4 @@ include::partial$enable-argocd-autosync.adoc[] == Cleanup alert silence -. Expire the silence -+ -[source,bash] ----- -kubectl --as=cluster-admin -n openshift-monitoring exec sts/alertmanager-main --\ - amtool --alertmanager.url=http://localhost:9093 silence expire $silence_id ----- +include::partial$expire-amtool-silence.adoc[] diff --git a/docs/modules/ROOT/partials/create-amtool-silence.adoc b/docs/modules/ROOT/partials/create-amtool-silence.adoc new file mode 100644 index 00000000..24e4c5ea --- /dev/null +++ b/docs/modules/ROOT/partials/create-amtool-silence.adoc @@ -0,0 +1,12 @@ +// see https://manpages.debian.org/unstable/prometheus-alertmanager/amtool.1.en.html#silence_add_%5B%3Cflags%3E%5D_%5B%3Cmatcher-groups%3E...%5D +. Create alertmanager silence ++ +[source,bash,subs="attributes+"] +---- +silence_id=$( + kubectl --as=cluster-admin -n openshift-monitoring exec \ + sts/alertmanager-main -- amtool --alertmanager.url=http://localhost:9093 \ + silence add {alert_statement} --duration="{duration}" -c "{comment}" +) +echo $silence_id +---- diff --git a/docs/modules/ROOT/partials/expire-amtool-silence.adoc b/docs/modules/ROOT/partials/expire-amtool-silence.adoc new file mode 100644 index 00000000..d44446e5 --- /dev/null +++ b/docs/modules/ROOT/partials/expire-amtool-silence.adoc @@ -0,0 +1,8 @@ +// see https://manpages.debian.org/unstable/prometheus-alertmanager/amtool.1.en.html#silence_expire_%5B%3Csilence-ids%3E...%5D +. Expire alertmanager silence ++ +[source,bash] +---- +kubectl --as=cluster-admin -n openshift-monitoring exec sts/alertmanager-main --\ + amtool --alertmanager.url=http://localhost:9093 silence expire $silence_id +----