From 0fa3522db76e173533d4ca1b9a83317a71a08d18 Mon Sep 17 00:00:00 2001 From: Matias Bordese Date: Wed, 4 Sep 2024 14:47:09 -0300 Subject: [PATCH] Add doc references to `regex_search` jinja filter (#4973) This filter wasn't listed in cheatsheet or docs, but it is usually more flexible/simpler than `regex_match` if trying to search for a substring in payload. Also setup a [timeout](https://github.com/mrabarnett/mrab-regex?tab=readme-ov-file#timeout) when regex searching/matching/replacing. Related to [this issue](https://docs.google.com/document/d/1gESMLdbJSnLnSnK7Nhp7DvJ7f10qZXsEm5GrgBc5RqQ/edit) --- .../advanced-templates/index.md | 2 + engine/common/jinja_templater/filters.py | 16 ++--- .../common/tests/test_apply_jinja_template.py | 60 ++++++++++++++++++- engine/pyproject.toml | 1 + engine/requirements.in | 2 +- engine/requirements.txt | 2 +- .../CheatSheet/CheatSheet.config.ts | 6 +- 7 files changed, 75 insertions(+), 14 deletions(-) diff --git a/docs/sources/configure/jinja2-templating/advanced-templates/index.md b/docs/sources/configure/jinja2-templating/advanced-templates/index.md index 492170b1d5..a92f43afce 100644 --- a/docs/sources/configure/jinja2-templating/advanced-templates/index.md +++ b/docs/sources/configure/jinja2-templating/advanced-templates/index.md @@ -95,6 +95,8 @@ Grafana OnCall enhances Jinja with additional functions: - `regex_replace`: Performs a regex find and replace - `regex_match`: Performs a regex match, returns `True` or `False` - Usage example: `{{ payload.ruleName | regex_match(".*") }}` +- `regex_search`: Performs a regex search, returns `True` or `False` + - Usage example: `{{ payload.message | regex_search("Severity: (High|Critical)") }}` - `b64decode`: Performs a base64 string decode - Usage example: `{{ payload.data | b64decode }}` - `parse_json`:Parses a JSON string to an object diff --git a/engine/common/jinja_templater/filters.py b/engine/common/jinja_templater/filters.py index c8ffc12af0..430967309b 100644 --- a/engine/common/jinja_templater/filters.py +++ b/engine/common/jinja_templater/filters.py @@ -1,11 +1,13 @@ import base64 import json -import re from datetime import datetime +import regex from django.utils.dateparse import parse_datetime from pytz import timezone +REGEX_TIMEOUT = 2 + def datetimeparse(value, format="%H:%M / %d-%m-%Y"): try: @@ -52,22 +54,22 @@ def json_dumps(value): def regex_replace(value, find, replace): try: - return re.sub(find, replace, value) - except (ValueError, AttributeError, TypeError): + return regex.sub(find, replace, value, timeout=REGEX_TIMEOUT) + except (ValueError, AttributeError, TypeError, TimeoutError): return None def regex_match(pattern, value): try: - return bool(re.match(value, pattern)) - except (ValueError, AttributeError, TypeError): + return bool(regex.match(value, pattern, timeout=REGEX_TIMEOUT)) + except (ValueError, AttributeError, TypeError, TimeoutError): return None def regex_search(pattern, value): try: - return bool(re.search(value, pattern)) - except (ValueError, AttributeError, TypeError): + return bool(regex.search(value, pattern, timeout=REGEX_TIMEOUT)) + except (ValueError, AttributeError, TypeError, TimeoutError): return None diff --git a/engine/common/tests/test_apply_jinja_template.py b/engine/common/tests/test_apply_jinja_template.py index ca947236bb..3694ffa953 100644 --- a/engine/common/tests/test_apply_jinja_template.py +++ b/engine/common/tests/test_apply_jinja_template.py @@ -15,6 +15,38 @@ templated_value_is_truthy, ) +EMAIL_SAMPLE_PAYLOAD = { + "subject": "[Reminder] Review GKE getServerConfig API permission changes", + "message": "Hello Google Kubernetes Customer,\r\n" + "\r\n" + "We’re writing to remind you that starting October 22, 2024, " + "the \r\n" + "getServerConfig API for Google Kubernetes Engine (GKE) will " + "enforce \r\n" + "Identity and Access Management (IAM) container.clusters.list " + "checks. This \r\n" + "change follows a series of security improvements as IAM \r\n" + "container.clusters.list permissions are being enforced across " + "the \r\n" + "getServerConfig API.\r\n" + "\r\n" + "We’ve provided additional information below to guide you through " + "this \r\n" + "change.\r\n" + "\r\n" + "What you need to know\r\n" + "\r\n" + "The current implementation doesn’t apply a specific permissions " + "check via \r\n" + "getServerConfig API. After this change goes into effect for the " + "Google \r\n" + "Kubernetes Engine API getServerConfig, only authorized users with " + "the \r\n" + "container.clusters.list permissions will be able to call the \r\n" + "GetServerConfig.\r\n", + "sender": "someone@somewhere.dev", +} + def test_apply_jinja_template(): payload = {"name": "test"} @@ -127,25 +159,49 @@ def test_apply_jinja_template_json_dumps(): assert result == expected +@pytest.mark.filterwarnings("ignore:::jinja2.*") # ignore regex escape sequence warning def test_apply_jinja_template_regex_match(): - payload = {"name": "test"} + payload = { + "name": "test", + "message": json.dumps(EMAIL_SAMPLE_PAYLOAD), + } assert apply_jinja_template("{{ payload.name | regex_match('.*') }}", payload) == "True" assert apply_jinja_template("{{ payload.name | regex_match('tes') }}", payload) == "True" assert apply_jinja_template("{{ payload.name | regex_match('test1') }}", payload) == "False" + # check for timeouts + with patch("common.jinja_templater.filters.REGEX_TIMEOUT", 1): + assert ( + apply_jinja_template( + "{{ payload.message | regex_match('(.|\\s)+Severity(.|\\s){2}High(.|\\s)+') }}", payload + ) + == "False" + ) # Check that exception is raised when regex is invalid with pytest.raises(JinjaTemplateError): apply_jinja_template("{{ payload.name | regex_match('*') }}", payload) +@pytest.mark.filterwarnings("ignore:::jinja2.*") # ignore regex escape sequence warning def test_apply_jinja_template_regex_search(): - payload = {"name": "test"} + payload = { + "name": "test", + "message": json.dumps(EMAIL_SAMPLE_PAYLOAD), + } assert apply_jinja_template("{{ payload.name | regex_search('.*') }}", payload) == "True" assert apply_jinja_template("{{ payload.name | regex_search('tes') }}", payload) == "True" assert apply_jinja_template("{{ payload.name | regex_search('est') }}", payload) == "True" assert apply_jinja_template("{{ payload.name | regex_search('test1') }}", payload) == "False" + # check for timeouts + with patch("common.jinja_templater.filters.REGEX_TIMEOUT", 1): + assert ( + apply_jinja_template( + "{{ payload.message | regex_search('(.|\\s)+Severity(.|\\s){2}High(.|\\s)+') }}", payload + ) + == "False" + ) # Check that exception is raised when regex is invalid with pytest.raises(JinjaTemplateError): diff --git a/engine/pyproject.toml b/engine/pyproject.toml index e6708a8e30..acb325e2a4 100644 --- a/engine/pyproject.toml +++ b/engine/pyproject.toml @@ -71,6 +71,7 @@ module = [ "polymorphic.*", "pyroscope.*", "ratelimit.*", + "regex.*", "recurring_ical_events.*", "rest_polymorphic.*", "slackclient.*", diff --git a/engine/requirements.in b/engine/requirements.in index e172c8deeb..323847ab4f 100644 --- a/engine/requirements.in +++ b/engine/requirements.in @@ -52,7 +52,7 @@ PyMySQL==1.1.1 python-telegram-bot==13.13 recurring-ical-events==2.1.0 redis==5.0.1 -regex==2021.11.2 +regex==2024.7.24 requests==2.32.3 slack-export-viewer==1.1.4 slack_sdk==3.21.3 diff --git a/engine/requirements.txt b/engine/requirements.txt index 858a0ced91..ce4d1e6b99 100644 --- a/engine/requirements.txt +++ b/engine/requirements.txt @@ -393,7 +393,7 @@ referencing==0.33.0 # via # jsonschema # jsonschema-specifications -regex==2021.11.2 +regex==2024.7.24 # via -r engine/requirements.in requests==2.32.3 # via diff --git a/grafana-plugin/src/components/CheatSheet/CheatSheet.config.ts b/grafana-plugin/src/components/CheatSheet/CheatSheet.config.ts index 523b1dadac..66846368c1 100644 --- a/grafana-plugin/src/components/CheatSheet/CheatSheet.config.ts +++ b/grafana-plugin/src/components/CheatSheet/CheatSheet.config.ts @@ -21,7 +21,7 @@ export const groupingTemplateCheatSheet: CheatSheetInterface = { name: 'Additional variables and functions', listItems: [ { listItemName: 'time(), datetimeformat, iso8601_to_time' }, - { listItemName: 'regex_replace, regex_match' }, + { listItemName: 'regex_replace, regex_match, regex_search' }, ], }, { @@ -86,7 +86,7 @@ export const genericTemplateCheatSheet: CheatSheetInterface = { { listItemName: 'payload, grafana_oncall_link, grafana_oncall_incident_id, integration_name, source_link' }, { listItemName: 'time(), datetimeformat, datetimeformat_as_timezone, datetimeparse, iso8601_to_time' }, { listItemName: 'to_pretty_json' }, - { listItemName: 'regex_replace, regex_match' }, + { listItemName: 'regex_replace, regex_match, regex_search' }, { listItemName: 'b64decode' }, ], }, @@ -143,7 +143,7 @@ export const slackMessageTemplateCheatSheet: CheatSheetInterface = { { listItemName: 'payload, grafana_oncall_link, grafana_oncall_incident_id, integration_name, source_link' }, { listItemName: 'time(), datetimeformat, iso8601_to_time' }, { listItemName: 'to_pretty_json' }, - { listItemName: 'regex_replace, regex_match' }, + { listItemName: 'regex_replace, regex_match, regex_search' }, ], }, {