From 834b2ce24b9cfeb96d7141e4f9c4d62fb81f15e6 Mon Sep 17 00:00:00 2001 From: "Al hel md. shahriar zaman" Date: Mon, 23 Dec 2024 16:27:00 +0600 Subject: [PATCH 1/3] new json line added in the ndjason file --- infrastructure/monitoring/kibana/config.ndjson | 1 + 1 file changed, 1 insertion(+) diff --git a/infrastructure/monitoring/kibana/config.ndjson b/infrastructure/monitoring/kibana/config.ndjson index f79787241..859588a23 100644 --- a/infrastructure/monitoring/kibana/config.ndjson +++ b/infrastructure/monitoring/kibana/config.ndjson @@ -1,6 +1,7 @@ {"attributes":{"actions":[{"actionRef":"preconfigured:preconfigured-alert-history-es-index","actionTypeId":".index","group":"metrics.inventory_threshold.fired","params":{"documents":["{\"@timestamp\":\"2024-08-06T07:57:35.644Z\",\"tags\":\"{{rule.tags}}\",\"rule\":{\"id\":\"{{rule.id}}\",\"name\":\"{{rule.name}}\",\"params\":{\"{{rule__type}}\":\"{{params}}\"},\"space\":\"{{rule.spaceId}}\",\"type\":\"{{rule.type}}\"},\"kibana\":{\"alert\":{\"id\":\"{{alert.id}}\",\"context\":{\"{{rule__type}}\":\"{{context}}\"},\"actionGroup\":\"{{alert.actionGroup}}\",\"actionGroupName\":\"{{alert.actionGroupName}}\"}},\"event\":{\"kind\":\"alert\"}}"]}}],"alertTypeId":"metrics.alert.inventory.threshold","apiKey":null,"apiKeyOwner":null,"consumer":"alerts","createdAt":"2023-11-17T12:01:46.420Z","createdBy":"elastic","enabled":false,"executionStatus":{"error":null,"lastExecutionDate":"2024-08-06T08:02:44.568Z","status":"pending"},"legacyId":null,"meta":{"versionApiKeyLastmodified":"7.17.18"},"muteAll":false,"mutedInstanceIds":[],"name":"Available disk space in root file system","notifyWhen":"onActionGroupChange","params":{"alertOnNoData":true,"criteria":[{"comparator":">=","customMetric":{"aggregation":"max","field":"system.filesystem.used.pct","id":"alert-custom-metric","type":"custom"},"metric":"custom","threshold":[0.7],"timeSize":1,"timeUnit":"h","warningComparator":">=","warningThreshold":[0.5]}],"filterQuery":"{\"bool\":{\"should\":[{\"match_phrase\":{\"system.filesystem.mount_point\":\"/hostfs\"}}],\"minimum_should_match\":1}}","filterQueryText":"system.filesystem.mount_point : \"/hostfs\"","nodeType":"host","sourceId":"default"},"schedule":{"interval":"1h"},"scheduledTaskId":null,"tags":["infra","opencrvs-builtin"],"throttle":null,"updatedAt":"2024-08-06T08:01:56.542Z","updatedBy":"elastic"},"coreMigrationVersion":"7.17.18","id":"14778650-8541-11ee-9002-2f37fdc4e5d5","migrationVersion":{"alert":"7.16.0"},"references":[],"sort":[1722931316554,643],"type":"alert","updated_at":"2024-08-06T08:01:56.554Z","version":"WzM5MywxXQ=="} {"attributes":{"actions":[{"actionRef":"preconfigured:preconfigured-alert-history-es-index","actionTypeId":".index","group":"threshold_met","params":{"documents":["{\"@timestamp\":\"2022-04-18T07:05:33.819Z\",\"tags\":\"{{rule.tags}}\",\"rule\":{\"id\":\"{{rule.id}}\",\"name\":\"{{rule.name}}\",\"params\":{\"{{rule__type}}\":\"{{params}}\"},\"space\":\"{{rule.spaceId}}\",\"type\":\"{{rule.type}}\"},\"kibana\":{\"alert\":{\"id\":\"{{alert.id}}\",\"context\":{\"{{rule__type}}\":\"{{context}}\"},\"actionGroup\":\"{{alert.actionGroup}}\",\"actionGroupName\":\"{{alert.actionGroupName}}\"}},\"event\":{\"kind\":\"alert\"}}"]}}],"alertTypeId":"apm.error_rate","apiKey":null,"apiKeyOwner":null,"consumer":"alerts","createdAt":"2022-06-01T11:30:27.033Z","createdBy":"opencrvs-admin","enabled":false,"executionStatus":{"error":null,"lastExecutionDate":"2024-02-07T08:28:08.400Z","status":"pending"},"legacyId":null,"meta":{"versionApiKeyLastmodified":"7.17.0"},"muteAll":false,"mutedInstanceIds":[],"name":"Error in service","notifyWhen":"onActionGroupChange","params":{"environment":"ENVIRONMENT_ALL","threshold":1,"windowSize":1,"windowUnit":"m"},"schedule":{"interval":"1m"},"scheduledTaskId":null,"tags":[],"throttle":null,"updatedAt":"2024-02-05T03:00:20.633Z","updatedBy":"opencrvs-admin"},"coreMigrationVersion":"7.17.0","id":"3b6722e0-e19e-11ec-ba8e-51649755648d","migrationVersion":{"alert":"7.16.0"},"references":[],"sort":[1707273006619,214975],"type":"alert","updated_at":"2024-02-07T02:30:06.619Z","version":"WzQ5MjAzNCwxOV0="} {"attributes":{"buildNum":46534,"defaultIndex":"metricbeat-*"},"coreMigrationVersion":"7.17.0","id":"7.17.0","migrationVersion":{"config":"7.13.0"},"references":[],"sort":[1707273006619,216009],"type":"config","updated_at":"2024-02-07T02:30:06.619Z","version":"WzQ5MjQyOCwxOV0="} +{"attributes":{"actions":[{"actionRef":"preconfigured:preconfigured-alert-history-es-index","actionTypeId":".index","frequency":{"notifyWhen":"onActionGroupChange","summary":false,"throttle":null},"group":"logs.threshold.fired","params":{"documents":["{\"@timestamp\":\"2024-12-23T09:50:26.528Z\",\"tags\":\"{{rule.tags}}\",\"rule\":{\"id\":\"{{rule.id}}\",\"name\":\"{{rule.name}}\",\"params\":{\"{{rule__type}}\":\"{{rule.params}}\"},\"space\":\"{{rule.spaceId}}\",\"type\":\"{{rule.type}}\"},\"kibana\":{\"alert\":{\"id\":\"{{alert.id}}\",\"context\":{\"{{rule__type}}\":\"{{context}}\"},\"actionGroup\":\"{{alert.actionGroup}}\",\"actionGroupName\":\"{{alert.actionGroupName}}\"}},\"event\":{\"kind\":\"alert\"}}"]},"uuid":"1504b514-82e5-47de-a6ef-60795788f7e6"}],"alertTypeId":"logs.alert.document.count","apiKey":null,"apiKeyCreatedByUser":null,"apiKeyOwner":null,"consumer":"alerts","createdAt":"2024-12-23T10:16:28.758Z","createdBy":"opencrvs-admin","enabled":false,"executionStatus":{"error":null,"lastExecutionDate":"2024-12-23T10:20:10.118Z","status":"pending","warning":null},"lastRun":{"alertsCount":{"active":1,"ignored":0,"new":1,"recovered":0},"outcome":"succeeded","outcomeMsg":null,"outcomeOrder":0,"warning":null},"legacyId":null,"meta":{"versionApiKeyLastmodified":"8.14.3"},"monitoring":{"run":{"calculated_metrics":{"p50":529,"p95":565,"p99":565,"success_ratio":1},"history":[{"duration":493,"success":true,"timestamp":1734948992101},{"duration":565,"success":true,"timestamp":1734949097318}],"last_run":{"metrics":{"duration":565,"gap_duration_s":null,"total_alerts_created":null,"total_alerts_detected":null,"total_indexing_duration_ms":null,"total_search_duration_ms":null},"timestamp":"2024-12-23T10:18:17.318Z"}}},"muteAll":false,"mutedInstanceIds":[],"name":"Error while backup ","nextRun":"2024-12-23T11:18:17.247Z","notifyWhen":null,"params":{"count":{"comparator":"more than or equals","value":1},"criteria":[{"comparator":"equals","field":"log.file.path","value":"/var/log/opencrvs-backup.error.log"}],"logView":{"logViewId":"log-view-reference-0","type":"log-view-reference"},"timeSize":1,"timeUnit":"h"},"revision":1,"running":false,"schedule":{"interval":"1h"},"scheduledTaskId":null,"snoozeSchedule":[],"tags":[],"throttle":null,"updatedAt":"2024-12-23T10:17:24.571Z","updatedBy":"opencrvs-admin"},"coreMigrationVersion":"8.8.0","created_at":"2024-12-23T10:17:24.572Z","id":"8caf3676-e78a-4e7c-ba85-c150df55901b","managed":false,"references":[{"id":"default","name":"param:log-view-reference-0","type":"infrastructure-monitoring-log-view"}],"sort":[1734949097887,115],"type":"alert","typeMigrationVersion":"10.1.0","updated_at":"2024-12-23T10:18:17.887Z","version":"WzgyNDExMyw0MDRd"} {"attributes":{"actions":[{"actionRef":"preconfigured:preconfigured-alert-history-es-index","actionTypeId":".index","group":"logs.threshold.fired","params":{"documents":["{\"@timestamp\":\"2023-11-22T08:25:47.329Z\",\"tags\":\"{{rule.tags}}\",\"rule\":{\"id\":\"{{rule.id}}\",\"name\":\"{{rule.name}}\",\"params\":{\"{{rule__type}}\":\"{{params}}\"},\"space\":\"{{rule.spaceId}}\",\"type\":\"{{rule.type}}\"},\"kibana\":{\"alert\":{\"id\":\"{{alert.id}}\",\"context\":{\"{{rule__type}}\":\"{{context}}\"},\"actionGroup\":\"{{alert.actionGroup}}\",\"actionGroupName\":\"{{alert.actionGroupName}}\"}},\"event\":{\"kind\":\"alert\"}}"]}}],"alertTypeId":"logs.alert.document.count","apiKey":null,"apiKeyOwner":null,"consumer":"alerts","createdAt":"2023-11-22T08:32:38.272Z","createdBy":"elastic","enabled":false,"executionStatus":{"error":null,"lastExecutionDate":"2024-02-07T08:28:08.400Z","status":"pending"},"legacyId":null,"meta":{"versionApiKeyLastmodified":"7.17.0"},"muteAll":false,"mutedInstanceIds":[],"name":"Error in backup logs","notifyWhen":"onActionGroupChange","params":{"count":{"comparator":"more than or equals","value":1},"criteria":[{"comparator":"matches","field":"message","value":"error"},{"comparator":"equals","field":"log.file.path","value":"/var/log/opencrvs-backup.log"}],"timeSize":1,"timeUnit":"h"},"schedule":{"interval":"1h"},"scheduledTaskId":null,"tags":[],"throttle":null,"updatedAt":"2024-02-07T02:27:22.558Z","updatedBy":"elastic"},"coreMigrationVersion":"7.17.0","id":"b166fcb0-8911-11ee-8111-2f3be9e93efc","migrationVersion":{"alert":"7.16.0"},"references":[],"sort":[1707294436464,232766],"type":"alert","updated_at":"2024-02-07T08:27:16.464Z","version":"WzQ5NTQzNCwxOV0="} {"attributes":{"anomalyThreshold":50,"description":"","fields":{"container":"container.id","host":"host.name","message":["message","@message"],"pod":"kubernetes.pod.uid","tiebreaker":"_doc","timestamp":"@timestamp"},"inventoryDefaultView":"0","logColumns":[{"timestampColumn":{"id":"5e7f964a-be8a-40d8-88d2-fbcfbdca0e2f"}},{"fieldColumn":{"field":"event.dataset","id":" eb9777a8-fcd3-420e-ba7d-172fff6da7a2"}},{"messageColumn":{"id":"b645d6da-824b-4723-9a2a-e8cece1645c0"}}],"logIndices":{"indexName":"logs-*,filebeat-*,kibana_sample_data_logs*,logstash*,ecs-logstash*","type":"index_name"},"metricAlias":"metrics-*,metricbeat-*","metricsExplorerDefaultView":"0","name":"Default"},"coreMigrationVersion":"7.17.0","id":"default","migrationVersion":{"infrastructure-ui-source":"7.16.2"},"references":[],"sort":[1707273006619,217714],"type":"infrastructure-ui-source","updated_at":"2024-02-07T02:30:06.619Z","version":"WzQ5MzAyNywxOV0="} {"attributes":{"actions":[{"actionRef":"preconfigured:preconfigured-alert-history-es-index","actionTypeId":".index","group":"query matched","params":{"documents":["{\"@timestamp\":\"2023-11-20T10:19:30.521Z\",\"tags\":\"{{rule.tags}}\",\"rule\":{\"id\":\"{{rule.id}}\",\"name\":\"{{rule.name}}\",\"params\":{\"{{rule__type}}\":\"{{params}}\"},\"space\":\"{{rule.spaceId}}\",\"type\":\"{{rule.type}}\"},\"kibana\":{\"alert\":{\"id\":\"{{alert.id}}\",\"context\":{\"{{rule__type}}\":\"{{context}}\"},\"actionGroup\":\"{{alert.actionGroup}}\",\"actionGroupName\":\"{{alert.actionGroupName}}\"}},\"event\":{\"kind\":\"alert\"}}"]}}],"alertTypeId":".es-query","apiKey":null,"apiKeyOwner":null,"consumer":"alerts","createdAt":"2023-11-20T09:12:19.237Z","createdBy":"elastic","enabled":false,"executionStatus":{"error":null,"lastExecutionDate":"2024-02-07T08:28:08.400Z","status":"pending"},"legacyId":null,"meta":{"versionApiKeyLastmodified":"7.17.0"},"muteAll":false,"mutedInstanceIds":[],"name":"Successful SSH login","notifyWhen":"onActionGroupChange","params":{"esQuery":"{ \"query\": { \"bool\": { \"must\": [ \n { \"term\": { \"log.file.path\": \"/var/log/auth.log\" } },\n { \"term\": { \"event.outcome\": \"success\" }}\n ] } } }","index":["filebeat-*"],"size":100,"threshold":[1],"thresholdComparator":">=","timeField":"@timestamp","timeWindowSize":1,"timeWindowUnit":"m"},"schedule":{"interval":"1m"},"scheduledTaskId":null,"tags":[],"throttle":null,"updatedAt":"2024-02-07T02:27:19.537Z","updatedBy":"elastic"},"coreMigrationVersion":"7.17.0","id":"e79aaa90-8784-11ee-b9ba-89bbe73df7ff","migrationVersion":{"alert":"7.16.0"},"references":[],"sort":[1707294457367,232778],"type":"alert","updated_at":"2024-02-07T08:27:37.367Z","version":"WzQ5NTQ0MCwxOV0="} From a45a9c14853fbdac92cfbc1aace7962d5b6123cf Mon Sep 17 00:00:00 2001 From: "Al hel md. shahriar zaman" Date: Mon, 23 Dec 2024 16:29:09 +0600 Subject: [PATCH 2/3] new alert rule added --- .../elastalert/rules/log-error-alert.yaml | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 infrastructure/monitoring/elastalert/rules/log-error-alert.yaml diff --git a/infrastructure/monitoring/elastalert/rules/log-error-alert.yaml b/infrastructure/monitoring/elastalert/rules/log-error-alert.yaml new file mode 100644 index 000000000..e2443ee71 --- /dev/null +++ b/infrastructure/monitoring/elastalert/rules/log-error-alert.yaml @@ -0,0 +1,38 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. +# +# OpenCRVS is also distributed under the terms of the Civil Registration +# & Healthcare Disclaimer located at http://opencrvs.org/license. +# +# Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. +name: 'log-alert' +type: 'frequency' +index: 'kibana-alert-history-default' +is_enabled: true +num_events: 1 +realert: + minutes: 5 +terms_size: 50 +timeframe: + minutes: 5 +timestamp_field: '@timestamp' +timestamp_type: 'iso' +use_strftime_index: false +# https://github.com/Yelp/elastalert/issues/1260 +raw_count_keys: false +filter: + - bool: + should: + - term: + rule.name.keyword: + value: 'Error while backup ' + minimum_should_match: 1 + +alert: post2 +http_post2_url: 'http://countryconfig:3040/email' +http_post2_payload: + subject: '{% raw %}{{DOMAIN}}{% endraw %}: {{rule.name}} 🚨' + html: 'Reason: {{kibana.alert.context.logs__alert__document__count.conditions}}. Login to https://kibana.{% raw %}{{DOMAIN}}{% endraw %} to view the alert.' + from: '{% raw %}{{SENDER_EMAIL_ADDRESS}}{% endraw %}' + to: '{% raw %}{{ALERT_EMAIL}}{% endraw %}' From 32cafa15610466deec5d8692a20e78915ed76536 Mon Sep 17 00:00:00 2001 From: "Al hel md. shahriar zaman" Date: Tue, 24 Dec 2024 14:10:10 +0600 Subject: [PATCH 3/3] refactored the alert error rule --- .../monitoring/elastalert/rules/log-error-alert.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/monitoring/elastalert/rules/log-error-alert.yaml b/infrastructure/monitoring/elastalert/rules/log-error-alert.yaml index e2443ee71..fe10b84c8 100644 --- a/infrastructure/monitoring/elastalert/rules/log-error-alert.yaml +++ b/infrastructure/monitoring/elastalert/rules/log-error-alert.yaml @@ -6,7 +6,7 @@ # & Healthcare Disclaimer located at http://opencrvs.org/license. # # Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. -name: 'log-alert' +name: 'log-error-alert' type: 'frequency' index: 'kibana-alert-history-default' is_enabled: true @@ -27,7 +27,7 @@ filter: - term: rule.name.keyword: value: 'Error while backup ' - minimum_should_match: 1 + alert: post2 http_post2_url: 'http://countryconfig:3040/email'