Skip to content

Commit

Permalink
Add doc references to regex_search jinja filter (#4973)
Browse files Browse the repository at this point in the history
This filter wasn't listed in cheatsheet or docs, but it is usually more
flexible/simpler than `regex_match` if trying to search for a substring
in payload.

Also setup a
[timeout](https://github.com/mrabarnett/mrab-regex?tab=readme-ov-file#timeout)
when regex searching/matching/replacing.

Related to [this
issue](https://docs.google.com/document/d/1gESMLdbJSnLnSnK7Nhp7DvJ7f10qZXsEm5GrgBc5RqQ/edit)
  • Loading branch information
matiasb authored Sep 4, 2024
1 parent de40bbb commit 0fa3522
Show file tree
Hide file tree
Showing 7 changed files with 75 additions and 14 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,8 @@ Grafana OnCall enhances Jinja with additional functions:
- `regex_replace`: Performs a regex find and replace
- `regex_match`: Performs a regex match, returns `True` or `False`
- Usage example: `{{ payload.ruleName | regex_match(".*") }}`
- `regex_search`: Performs a regex search, returns `True` or `False`
- Usage example: `{{ payload.message | regex_search("Severity: (High|Critical)") }}`
- `b64decode`: Performs a base64 string decode
- Usage example: `{{ payload.data | b64decode }}`
- `parse_json`:Parses a JSON string to an object
Expand Down
16 changes: 9 additions & 7 deletions engine/common/jinja_templater/filters.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
import base64
import json
import re
from datetime import datetime

import regex
from django.utils.dateparse import parse_datetime
from pytz import timezone

REGEX_TIMEOUT = 2


def datetimeparse(value, format="%H:%M / %d-%m-%Y"):
try:
Expand Down Expand Up @@ -52,22 +54,22 @@ def json_dumps(value):

def regex_replace(value, find, replace):
try:
return re.sub(find, replace, value)
except (ValueError, AttributeError, TypeError):
return regex.sub(find, replace, value, timeout=REGEX_TIMEOUT)
except (ValueError, AttributeError, TypeError, TimeoutError):
return None


def regex_match(pattern, value):
try:
return bool(re.match(value, pattern))
except (ValueError, AttributeError, TypeError):
return bool(regex.match(value, pattern, timeout=REGEX_TIMEOUT))
except (ValueError, AttributeError, TypeError, TimeoutError):
return None


def regex_search(pattern, value):
try:
return bool(re.search(value, pattern))
except (ValueError, AttributeError, TypeError):
return bool(regex.search(value, pattern, timeout=REGEX_TIMEOUT))
except (ValueError, AttributeError, TypeError, TimeoutError):
return None


Expand Down
60 changes: 58 additions & 2 deletions engine/common/tests/test_apply_jinja_template.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,38 @@
templated_value_is_truthy,
)

EMAIL_SAMPLE_PAYLOAD = {
"subject": "[Reminder] Review GKE getServerConfig API permission changes",
"message": "Hello Google Kubernetes Customer,\r\n"
"\r\n"
"We’re writing to remind you that starting October 22, 2024, "
"the \r\n"
"getServerConfig API for Google Kubernetes Engine (GKE) will "
"enforce \r\n"
"Identity and Access Management (IAM) container.clusters.list "
"checks. This \r\n"
"change follows a series of security improvements as IAM \r\n"
"container.clusters.list permissions are being enforced across "
"the \r\n"
"getServerConfig API.\r\n"
"\r\n"
"We’ve provided additional information below to guide you through "
"this \r\n"
"change.\r\n"
"\r\n"
"What you need to know\r\n"
"\r\n"
"The current implementation doesn’t apply a specific permissions "
"check via \r\n"
"getServerConfig API. After this change goes into effect for the "
"Google \r\n"
"Kubernetes Engine API getServerConfig, only authorized users with "
"the \r\n"
"container.clusters.list permissions will be able to call the \r\n"
"GetServerConfig.\r\n",
"sender": "[email protected]",
}


def test_apply_jinja_template():
payload = {"name": "test"}
Expand Down Expand Up @@ -127,25 +159,49 @@ def test_apply_jinja_template_json_dumps():
assert result == expected


@pytest.mark.filterwarnings("ignore:::jinja2.*") # ignore regex escape sequence warning
def test_apply_jinja_template_regex_match():
payload = {"name": "test"}
payload = {
"name": "test",
"message": json.dumps(EMAIL_SAMPLE_PAYLOAD),
}

assert apply_jinja_template("{{ payload.name | regex_match('.*') }}", payload) == "True"
assert apply_jinja_template("{{ payload.name | regex_match('tes') }}", payload) == "True"
assert apply_jinja_template("{{ payload.name | regex_match('test1') }}", payload) == "False"
# check for timeouts
with patch("common.jinja_templater.filters.REGEX_TIMEOUT", 1):
assert (
apply_jinja_template(
"{{ payload.message | regex_match('(.|\\s)+Severity(.|\\s){2}High(.|\\s)+') }}", payload
)
== "False"
)

# Check that exception is raised when regex is invalid
with pytest.raises(JinjaTemplateError):
apply_jinja_template("{{ payload.name | regex_match('*') }}", payload)


@pytest.mark.filterwarnings("ignore:::jinja2.*") # ignore regex escape sequence warning
def test_apply_jinja_template_regex_search():
payload = {"name": "test"}
payload = {
"name": "test",
"message": json.dumps(EMAIL_SAMPLE_PAYLOAD),
}

assert apply_jinja_template("{{ payload.name | regex_search('.*') }}", payload) == "True"
assert apply_jinja_template("{{ payload.name | regex_search('tes') }}", payload) == "True"
assert apply_jinja_template("{{ payload.name | regex_search('est') }}", payload) == "True"
assert apply_jinja_template("{{ payload.name | regex_search('test1') }}", payload) == "False"
# check for timeouts
with patch("common.jinja_templater.filters.REGEX_TIMEOUT", 1):
assert (
apply_jinja_template(
"{{ payload.message | regex_search('(.|\\s)+Severity(.|\\s){2}High(.|\\s)+') }}", payload
)
== "False"
)

# Check that exception is raised when regex is invalid
with pytest.raises(JinjaTemplateError):
Expand Down
1 change: 1 addition & 0 deletions engine/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ module = [
"polymorphic.*",
"pyroscope.*",
"ratelimit.*",
"regex.*",
"recurring_ical_events.*",
"rest_polymorphic.*",
"slackclient.*",
Expand Down
2 changes: 1 addition & 1 deletion engine/requirements.in
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ PyMySQL==1.1.1
python-telegram-bot==13.13
recurring-ical-events==2.1.0
redis==5.0.1
regex==2021.11.2
regex==2024.7.24
requests==2.32.3
slack-export-viewer==1.1.4
slack_sdk==3.21.3
Expand Down
2 changes: 1 addition & 1 deletion engine/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,7 @@ referencing==0.33.0
# via
# jsonschema
# jsonschema-specifications
regex==2021.11.2
regex==2024.7.24
# via -r engine/requirements.in
requests==2.32.3
# via
Expand Down
6 changes: 3 additions & 3 deletions grafana-plugin/src/components/CheatSheet/CheatSheet.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ export const groupingTemplateCheatSheet: CheatSheetInterface = {
name: 'Additional variables and functions',
listItems: [
{ listItemName: 'time(), datetimeformat, iso8601_to_time' },
{ listItemName: 'regex_replace, regex_match' },
{ listItemName: 'regex_replace, regex_match, regex_search' },
],
},
{
Expand Down Expand Up @@ -86,7 +86,7 @@ export const genericTemplateCheatSheet: CheatSheetInterface = {
{ listItemName: 'payload, grafana_oncall_link, grafana_oncall_incident_id, integration_name, source_link' },
{ listItemName: 'time(), datetimeformat, datetimeformat_as_timezone, datetimeparse, iso8601_to_time' },
{ listItemName: 'to_pretty_json' },
{ listItemName: 'regex_replace, regex_match' },
{ listItemName: 'regex_replace, regex_match, regex_search' },
{ listItemName: 'b64decode' },
],
},
Expand Down Expand Up @@ -143,7 +143,7 @@ export const slackMessageTemplateCheatSheet: CheatSheetInterface = {
{ listItemName: 'payload, grafana_oncall_link, grafana_oncall_incident_id, integration_name, source_link' },
{ listItemName: 'time(), datetimeformat, iso8601_to_time' },
{ listItemName: 'to_pretty_json' },
{ listItemName: 'regex_replace, regex_match' },
{ listItemName: 'regex_replace, regex_match, regex_search' },
],
},
{
Expand Down

0 comments on commit 0fa3522

Please sign in to comment.