diff --git a/.github/workflows/linting-and-tests.yml b/.github/workflows/linting-and-tests.yml index d94506638b..7a01bbe061 100644 --- a/.github/workflows/linting-and-tests.yml +++ b/.github/workflows/linting-and-tests.yml @@ -190,9 +190,15 @@ jobs: uses: actions/checkout@v4 - name: Setup Python uses: ./.github/actions/setup-python + - name: Wait for MySQL to be ready + working-directory: engine + run: ./wait_for_test_mysql_start.sh + - name: Test Django migrations work from blank slate + working-directory: engine + run: python manage.py migrate - name: Unit Test Backend working-directory: engine - run: ./wait_for_test_mysql_start.sh && pytest -x + run: pytest -x unit-test-backend-postgresql-rabbitmq: name: "Backend Tests: PostgreSQL + RabbitMQ (RBAC enabled: ${{ matrix.rbac_enabled }})" @@ -229,6 +235,9 @@ jobs: uses: actions/checkout@v4 - name: Setup Python uses: ./.github/actions/setup-python + - name: Test Django migrations work from blank slate + working-directory: engine + run: python manage.py migrate - name: Unit Test Backend working-directory: engine run: pytest -x @@ -259,6 +268,9 @@ jobs: uses: actions/checkout@v4 - name: Setup Python uses: ./.github/actions/setup-python + - name: Test Django migrations work from blank slate + working-directory: engine + run: python manage.py migrate - name: Unit Test Backend working-directory: engine run: pytest -x diff --git a/docs/sources/configure/integrations/labels/index.md b/docs/sources/configure/integrations/labels/index.md index 07da1730a0..9f29a7fba2 100644 --- a/docs/sources/configure/integrations/labels/index.md +++ b/docs/sources/configure/integrations/labels/index.md @@ -37,8 +37,8 @@ To assign labels to an integration: 1. Go to the **Integrations** tab and select an integration from the list. 2. Click the **three dots** next to the integration name and select **Integration settings**. -3. Define a Key and Value pair for the label, either by selecting from an existing list or typing new ones in the fields. Press enter/return to accept. -4. To add more labels, click on the **Add** button. You can remove a label using the X button next to the key-value pair. +3. Click **Add** button in the **Integration labels** section. You can remove a label using the X button next to the key-value pair. +4. Define a Key and Value pair for the label, either by selecting from an existing list or typing new ones in the fields. Press enter/return to accept. 5. Click **Save** when finished. To filter integrations by labels: @@ -47,12 +47,7 @@ To filter integrations by labels: 2. Locate the **Search or filter results…** dropdown and select **Label**. 3. Start typing to find suggestions and select the key-value pair you’d like to filter by. -### Pass down integration labels - Labels are automatically assigned to each alert group based on the labels assigned to the integration. -You can choose to pass down specific labels in the Alert Group Labeling tab. - -To do this, navigate to the Integration Labels section in the Alert Group Labeling tab and enable/disable specific labels using the toggler. ## Alert Group labels @@ -70,23 +65,18 @@ Alert Group labeling can be configured for each integration. To find the Alert G 1. Navigate to the **Integrations** tab. 2. Select an integration from the list of enabled integrations. 3. Click the three dots next to the integration name. -4. Choose **Alert Group Labeling**. +4. Choose **Integration settings**. You can configure alert group labels mapping in the **Mapping** section. A maximum of 15 labels can be assigned to an alert group. If there are more than 15 labels, only the first 15 will be assigned. -### Dynamic & Static Labels +### Dynamic Labels -Dynamic and Static labels allow you to assign arbitrary labels to alert groups. +Dynamic labels allow you to assign arbitrary labels to alert groups. Dynamic labels have values extracted from the alert payload using Jinja, with keys remaining static. -Static labels have both key and value as static and are not derived from the payload. These labels will not be attached to the integration. - -1. In the **Alert Group Labeling** tab, navigate to **Dynamic & Static Labels**. -2. Press the **Add Label** button and choose between dynamic or static. - -#### Add Static Labels +These labels will not be attached to the integration. -1. Select or create key and value from the dropdown list. -2. These labels will be assigned to all alert groups received by this integration. +1. In the **Integration settings** tab, navigate to **Dynamic Labels**. +2. Press the **Add Label** button. #### Add Dynamic Labels diff --git a/docs/sources/configure/jinja2-templating/advanced-templates/index.md b/docs/sources/configure/jinja2-templating/advanced-templates/index.md index d98f642253..c349aec1e0 100644 --- a/docs/sources/configure/jinja2-templating/advanced-templates/index.md +++ b/docs/sources/configure/jinja2-templating/advanced-templates/index.md @@ -94,6 +94,7 @@ Grafana OnCall enhances Jinja with additional functions: - `datetimeparse`: Converts string to datetime according to strftime format codes (`%H:%M / %d-%m-%Y` by default) - `timedeltaparse`: Converts a time range (e.g., `5s`, `2m`, `6h`, `3d`) to a timedelta that can be added to or subtracted from a datetime - Usage example: `{% set delta = alert.window | timedeltaparse %}{{ alert.startsAt | iso8601_to_time - delta | datetimeformat }}` +- `timestamp_to_datetime`: Converts a Unix/Epoch time to a datetime object - `regex_replace`: Performs a regex find and replace - `regex_match`: Performs a regex match, returns `True` or `False` - Usage example: `{{ payload.ruleName | regex_match(".*") }}` diff --git a/engine/apps/alerts/migrations/0070_remove_resolutionnoteslackmessage__slack_channel_id_db.py b/engine/apps/alerts/migrations/0070_remove_resolutionnoteslackmessage__slack_channel_id_db.py index 70355474d3..f2eb3d5a3d 100644 --- a/engine/apps/alerts/migrations/0070_remove_resolutionnoteslackmessage__slack_channel_id_db.py +++ b/engine/apps/alerts/migrations/0070_remove_resolutionnoteslackmessage__slack_channel_id_db.py @@ -1,7 +1,7 @@ # Generated by Django 4.2.16 on 2024-11-20 20:23 -import common.migrations.remove_field -import django_migration_linter as linter +# import common.migrations.remove_field +# import django_migration_linter as linter from django.db import migrations @@ -12,10 +12,14 @@ class Migration(migrations.Migration): ] operations = [ - linter.IgnoreMigration(), - common.migrations.remove_field.RemoveFieldDB( - model_name='resolutionnoteslackmessage', - name='_slack_channel_id', - remove_state_migration=('alerts', '0068_remove_resolutionnoteslackmessage__slack_channel_id_state'), - ), + # NOTE: commented out due to some issues this was causing w/ SQLite: + # https://github.com/grafana/oncall/issues/5306 + # https://github.com/grafana/oncall/issues/5244#issuecomment-2503999986 + # + # linter.IgnoreMigration(), + # common.migrations.remove_field.RemoveFieldDB( + # model_name='resolutionnoteslackmessage', + # name='_slack_channel_id', + # remove_state_migration=('alerts', '0068_remove_resolutionnoteslackmessage__slack_channel_id_state'), + # ), ] diff --git a/engine/apps/alerts/migrations/0071_migrate_labels.py b/engine/apps/alerts/migrations/0071_migrate_labels.py new file mode 100644 index 0000000000..787b5fdbc8 --- /dev/null +++ b/engine/apps/alerts/migrations/0071_migrate_labels.py @@ -0,0 +1,59 @@ +# Generated by Django 4.2.15 on 2024-11-12 09:33 +import logging + +from django.db import migrations +import django_migration_linter as linter + +logger = logging.getLogger(__name__) + + +def migrate_static_labels(apps, schema_editor): + AlertReceiveChannelAssociatedLabel = apps.get_model("labels", "AlertReceiveChannelAssociatedLabel") + AlertReceiveChannel = apps.get_model("alerts", "AlertReceiveChannel") + + logging.info("Start migrating alert group static labels to integration labels") + + labels_associations_to_create = [] + alert_receive_channels_to_update = [] + + alert_receive_channels = AlertReceiveChannel.objects.filter(alert_group_labels_custom__isnull=False) + logging.info(f"Found {alert_receive_channels.count()} integrations with custom alert groups labels") + for alert_receive_channel in alert_receive_channels: + update_labels = False + labels = alert_receive_channel.alert_group_labels_custom[:] + for label in labels: + if label[1] is not None: + labels_associations_to_create.append( + AlertReceiveChannelAssociatedLabel( + key_id=label[0], + value_id=label[1], + organization=alert_receive_channel.organization, + alert_receive_channel=alert_receive_channel + ) + ) + alert_receive_channel.alert_group_labels_custom.remove(label) + update_labels = True + if update_labels: + alert_receive_channels_to_update.append(alert_receive_channel) + + AlertReceiveChannelAssociatedLabel.objects.bulk_create( + labels_associations_to_create, ignore_conflicts=True, batch_size=5000 + ) + logging.info("Bulk created label associations") + AlertReceiveChannel.objects.bulk_update(alert_receive_channels_to_update, fields=["alert_group_labels_custom"], batch_size=5000) + logging.info("Bulk updated integrations") + logging.info("Finished migrating static labels to integration labels") + + +class Migration(migrations.Migration): + + dependencies = [ + ('alerts', '0070_remove_resolutionnoteslackmessage__slack_channel_id_db'), + ('labels', '0005_labelkeycache_prescribed_labelvaluecache_prescribed'), + ] + + operations = [ + # migrate static alert group labels to integration labels + linter.IgnoreMigration(), + migrations.RunPython(migrate_static_labels, migrations.RunPython.noop), + ] diff --git a/engine/apps/api/serializers/alert_receive_channel.py b/engine/apps/api/serializers/alert_receive_channel.py index 33bf240faa..9065ca6801 100644 --- a/engine/apps/api/serializers/alert_receive_channel.py +++ b/engine/apps/api/serializers/alert_receive_channel.py @@ -3,7 +3,6 @@ from django.conf import settings from django.core.exceptions import ValidationError as DjangoValidationError -from django.db.models import Q from drf_spectacular.utils import PolymorphicProxySerializer, extend_schema_field from jinja2 import TemplateSyntaxError from rest_framework import serializers @@ -14,7 +13,7 @@ from apps.alerts.models import AlertReceiveChannel from apps.base.messaging import get_messaging_backends from apps.integrations.legacy_prefix import has_legacy_prefix -from apps.labels.models import LabelKeyCache, LabelValueCache +from apps.labels.models import AlertReceiveChannelAssociatedLabel, LabelKeyCache, LabelValueCache from apps.labels.types import LabelKey from apps.user_management.models import Organization from common.api_helpers.custom_fields import TeamPrimaryKeyRelatedField @@ -55,7 +54,7 @@ class AlertGroupCustomLabelAPI(typing.TypedDict): class IntegrationAlertGroupLabels(typing.TypedDict): - inheritable: dict[str, bool] + inheritable: dict[str, bool] | None # Deprecated custom: AlertGroupCustomLabelsAPI template: str | None @@ -99,7 +98,8 @@ class CustomLabelValueSerializer(serializers.Serializer): class IntegrationAlertGroupLabelsSerializer(serializers.Serializer): """Alert group labels configuration for the integration. See AlertReceiveChannel.alert_group_labels for details.""" - inheritable = serializers.DictField(child=serializers.BooleanField()) + # todo: inheritable field is deprecated. Remove in a future release + inheritable = serializers.DictField(child=serializers.BooleanField(), required=False) custom = CustomLabelSerializer(many=True) template = serializers.CharField(allow_null=True) @@ -107,12 +107,13 @@ class IntegrationAlertGroupLabelsSerializer(serializers.Serializer): def pop_alert_group_labels(validated_data: dict) -> IntegrationAlertGroupLabels | None: """Get alert group labels from validated data.""" - # the "alert_group_labels" field is optional, so either all 3 fields are present or none - if "inheritable" not in validated_data: + # the "alert_group_labels" field is optional, so either all 2 fields are present or none + # "inheritable" field is deprecated + if "custom" not in validated_data: return None return { - "inheritable": validated_data.pop("inheritable"), + "inheritable": validated_data.pop("inheritable", None), # deprecated "custom": validated_data.pop("custom"), "template": validated_data.pop("template"), } @@ -124,15 +125,11 @@ def update( if alert_group_labels is None: return instance - # update inheritable labels - inheritable_key_ids = [ - key_id for key_id, inheritable in alert_group_labels["inheritable"].items() if inheritable - ] - instance.labels.filter(key_id__in=inheritable_key_ids).update(inheritable=True) - instance.labels.filter(~Q(key_id__in=inheritable_key_ids)).update(inheritable=False) - # update DB cache for custom labels cls._create_custom_labels(instance.organization, alert_group_labels["custom"]) + # save static labels as integration labels + # todo: it's needed to cover delay between backend and frontend rollout, and can be removed later + cls._save_static_labels_as_integration_labels(instance, alert_group_labels["custom"]) # update custom labels instance.alert_group_labels_custom = cls._custom_labels_to_internal_value(alert_group_labels["custom"]) @@ -170,18 +167,38 @@ def _create_custom_labels(organization: Organization, labels: AlertGroupCustomLa LabelKeyCache.objects.bulk_create(label_keys, ignore_conflicts=True, batch_size=5000) LabelValueCache.objects.bulk_create(label_values, ignore_conflicts=True, batch_size=5000) + @staticmethod + def _save_static_labels_as_integration_labels(instance: AlertReceiveChannel, labels: AlertGroupCustomLabelsAPI): + labels_associations_to_create = [] + labels_copy = labels[:] + for label in labels_copy: + if label["value"]["id"] is not None: + labels_associations_to_create.append( + AlertReceiveChannelAssociatedLabel( + key_id=label["key"]["id"], + value_id=label["value"]["id"], + organization=instance.organization, + alert_receive_channel=instance, + ) + ) + labels.remove(label) + AlertReceiveChannelAssociatedLabel.objects.bulk_create( + labels_associations_to_create, ignore_conflicts=True, batch_size=5000 + ) + @classmethod def to_representation(cls, instance: AlertReceiveChannel) -> IntegrationAlertGroupLabels: """ The API representation of alert group labels is very different from the underlying model. - "inheritable" is based on AlertReceiveChannelAssociatedLabel.inheritable, a property of another model. + "inheritable" field is deprecated. Kept for api-backward compatibility. Will be removed in a future release "custom" is based on AlertReceiveChannel.alert_group_labels_custom, a JSONField with a different schema. "template" is based on AlertReceiveChannel.alert_group_labels_template, this one is straightforward. """ return { - "inheritable": {label.key_id: label.inheritable for label in instance.labels.all()}, + # todo: "inheritable" field is deprecated, remove in a future release. + "inheritable": {label.key_id: True for label in instance.labels.all()}, "custom": cls._custom_labels_to_representation(instance.alert_group_labels_custom), "template": instance.alert_group_labels_template, } diff --git a/engine/apps/api/tests/test_alert_receive_channel.py b/engine/apps/api/tests/test_alert_receive_channel.py index 4569cc2ec8..ac040cde1c 100644 --- a/engine/apps/api/tests/test_alert_receive_channel.py +++ b/engine/apps/api/tests/test_alert_receive_channel.py @@ -1674,8 +1674,8 @@ def test_alert_group_labels_put( organization, user, token = make_organization_and_user_with_plugin_token() alert_receive_channel = make_alert_receive_channel(organization) label_1 = make_integration_label_association(organization, alert_receive_channel) - label_2 = make_integration_label_association(organization, alert_receive_channel, inheritable=False) - label_3 = make_integration_label_association(organization, alert_receive_channel, inheritable=False) + label_2 = make_integration_label_association(organization, alert_receive_channel) + label_3 = make_integration_label_association(organization, alert_receive_channel) custom = [ # plain label @@ -1712,19 +1712,26 @@ def test_alert_group_labels_put( response = client.put(url, data, format="json", **make_user_auth_headers(user, token)) assert response.status_code == status.HTTP_200_OK + # check static labels were saved as integration labels assert response.json()["alert_group_labels"] == { - "inheritable": {label_1.key_id: False, label_2.key_id: True, label_3.key_id: False}, - "custom": custom, + "inheritable": {label_1.key_id: True, label_2.key_id: True, label_3.key_id: True, "hello": True}, + "custom": [ + { + "key": {"id": label_3.key.id, "name": label_3.key.name, "prescribed": False}, + "value": {"id": None, "name": "{{ payload.foo }}", "prescribed": False}, + } + ], "template": template, } alert_receive_channel.refresh_from_db() + # check static labels are not in the custom labels list assert alert_receive_channel.alert_group_labels_custom == [ - [label_2.key_id, label_2.value_id, None], - ["hello", "foo", None], [label_3.key_id, None, "{{ payload.foo }}"], ] assert alert_receive_channel.alert_group_labels_template == template + # check static labels were assigned to integration + assert alert_receive_channel.labels.filter(key_id__in=[label_2.key_id, "hello"]).count() == 2 # check label keys & values are created key = LabelKeyCache.objects.filter(id="hello", name="world", organization=organization).first() @@ -1766,6 +1773,20 @@ def test_alert_group_labels_post(alert_receive_channel_internal_api_setup, make_ { "key": {"id": "test", "name": "test", "prescribed": False}, "value": {"id": "123", "name": "123", "prescribed": False}, + }, + { + "key": {"id": "test2", "name": "test2", "prescribed": False}, + "value": {"id": None, "name": "{{ payload.foo }}", "prescribed": False}, + }, + ], + "template": "{{ payload.labels | tojson }}", + } + expected_alert_group_labels = { + "inheritable": {"test": True}, + "custom": [ + { + "key": {"id": "test2", "name": "test2", "prescribed": False}, + "value": {"id": None, "name": "{{ payload.foo }}", "prescribed": False}, } ], "template": "{{ payload.labels | tojson }}", @@ -1783,10 +1804,10 @@ def test_alert_group_labels_post(alert_receive_channel_internal_api_setup, make_ assert response.status_code == status.HTTP_201_CREATED assert response.json()["labels"] == labels - assert response.json()["alert_group_labels"] == alert_group_labels + assert response.json()["alert_group_labels"] == expected_alert_group_labels alert_receive_channel = AlertReceiveChannel.objects.get(public_primary_key=response.json()["id"]) - assert alert_receive_channel.alert_group_labels_custom == [["test", "123", None]] + assert alert_receive_channel.alert_group_labels_custom == [["test2", None, "{{ payload.foo }}"]] assert alert_receive_channel.alert_group_labels_template == "{{ payload.labels | tojson }}" diff --git a/engine/apps/labels/0007_remove_alertreceivechannelassociatedlabel_inheritable_db.py b/engine/apps/labels/0007_remove_alertreceivechannelassociatedlabel_inheritable_db.py new file mode 100644 index 0000000000..91504bd9a6 --- /dev/null +++ b/engine/apps/labels/0007_remove_alertreceivechannelassociatedlabel_inheritable_db.py @@ -0,0 +1,21 @@ +# TODO: MOVE IT TO /migrations DIRECTORY IN FUTURE RELEASE + +# Generated by Django 4.2.15 on 2024-11-26 13:37 + +from django.db import migrations + +import common.migrations.remove_field + + +class Migration(migrations.Migration): + dependencies = [ + ("labels", "0006_remove_alertreceivechannelassociatedlabel_inheritable_state"), + ] + + operations = [ + common.migrations.remove_field.RemoveFieldDB( + model_name="AlertReceiveChannelAssociatedLabel", + name="inheritable", + remove_state_migration=("labels", "0007_remove_alertreceivechannelassociatedlabel_inheritable_state"), + ), + ] diff --git a/engine/apps/labels/alert_group_labels.py b/engine/apps/labels/alert_group_labels.py index f63e60cdff..60bfad7c18 100644 --- a/engine/apps/labels/alert_group_labels.py +++ b/engine/apps/labels/alert_group_labels.py @@ -29,8 +29,7 @@ def gather_labels_from_alert_receive_channel_and_raw_request_data( # inherit labels from the integration labels = { - label.key.name: label.value.name - for label in alert_receive_channel.labels.filter(inheritable=True).select_related("key", "value") + label.key.name: label.value.name for label in alert_receive_channel.labels.all().select_related("key", "value") } # apply custom labels diff --git a/engine/apps/labels/migrations/0006_remove_alertreceivechannelassociatedlabel_inheritable_state.py b/engine/apps/labels/migrations/0006_remove_alertreceivechannelassociatedlabel_inheritable_state.py new file mode 100644 index 0000000000..8e71cc998b --- /dev/null +++ b/engine/apps/labels/migrations/0006_remove_alertreceivechannelassociatedlabel_inheritable_state.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.15 on 2024-11-26 13:37 + +import common.migrations.remove_field +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('labels', '0005_labelkeycache_prescribed_labelvaluecache_prescribed'), + ] + + operations = [ + common.migrations.remove_field.RemoveFieldState( + model_name='AlertReceiveChannelAssociatedLabel', + name='inheritable', + ), + ] diff --git a/engine/apps/labels/models.py b/engine/apps/labels/models.py index 8a4a626d6f..ecd06c268d 100644 --- a/engine/apps/labels/models.py +++ b/engine/apps/labels/models.py @@ -118,9 +118,6 @@ class AlertReceiveChannelAssociatedLabel(AssociatedLabel): "alerts.AlertReceiveChannel", on_delete=models.CASCADE, related_name="labels" ) - # If inheritable is True, then the label will be passed down to alert groups - inheritable = models.BooleanField(default=True, null=True) - class Meta: unique_together = ["key_id", "value_id", "alert_receive_channel_id"] diff --git a/engine/common/jinja_templater/filters.py b/engine/common/jinja_templater/filters.py index 742931d725..0dba8be0c9 100644 --- a/engine/common/jinja_templater/filters.py +++ b/engine/common/jinja_templater/filters.py @@ -30,6 +30,13 @@ def datetimeformat_as_timezone(value, format="%H:%M / %d-%m-%Y", tz="UTC"): return None +def timestamp_to_datetime(value): + try: + return datetime.fromtimestamp(value) + except (ValueError, AttributeError, TypeError): + return None + + def iso8601_to_time(value): try: return parse_datetime(value) diff --git a/engine/common/jinja_templater/jinja_template_env.py b/engine/common/jinja_templater/jinja_template_env.py index 910c287daf..34e9ccc77a 100644 --- a/engine/common/jinja_templater/jinja_template_env.py +++ b/engine/common/jinja_templater/jinja_template_env.py @@ -15,6 +15,7 @@ regex_replace, regex_search, timedeltaparse, + timestamp_to_datetime, to_pretty_json, ) @@ -39,3 +40,4 @@ def raise_security_exception(name): jinja_template_env.filters["json_dumps"] = json_dumps jinja_template_env.filters["b64decode"] = b64decode jinja_template_env.filters["parse_json"] = parse_json +jinja_template_env.filters["timestamp_to_datetime"] = timestamp_to_datetime diff --git a/engine/common/tests/test_apply_jinja_template.py b/engine/common/tests/test_apply_jinja_template.py index 03fcec1c20..5611431704 100644 --- a/engine/common/tests/test_apply_jinja_template.py +++ b/engine/common/tests/test_apply_jinja_template.py @@ -66,6 +66,17 @@ def test_apply_jinja_template_iso8601_to_time(): assert result == expected +def test_apply_jinja_template_timestamp_to_datetime(): + payload = {"sometime": 1730893740} + + result = apply_jinja_template( + "{{ payload.sometime | timestamp_to_datetime }}", + payload, + ) + expected = str(datetime.fromtimestamp(payload["sometime"])) + assert result == expected + + def test_apply_jinja_template_datetimeformat(): payload = {"aware": "2023-05-28 23:11:12+0000", "naive": "2023-05-28 23:11:12"}