Merge pull request #5376 from grafana/dev

dev -> main

engine/apps/alerts/models/alert_receive_channel.py

@@ -301,16 +301,21 @@ class AlertReceiveChannel(IntegrationOptionsMixin, MaintainableObject):
     rate_limited_in_slack_at = models.DateTimeField(null=True, default=None)
     rate_limit_message_task_id = models.CharField(max_length=100, null=True, default=None)
 
-    AlertGroupCustomLabelsDB = list[tuple[str, str | None, str | None]] | None
-    alert_group_labels_custom: AlertGroupCustomLabelsDB = models.JSONField(null=True, default=None)
+    DynamicLabelsEntryDB = tuple[str, str | None, str | None]
+    DynamicLabelsConfigDB = list[DynamicLabelsEntryDB] | None
+    alert_group_labels_custom: DynamicLabelsConfigDB = models.JSONField(null=True, default=None)
     """
-    Stores "custom labels" for alert group labels. Custom labels can be either "plain" or "templated".
-    For plain labels, the format is: [<LABEL_KEY_ID>, <LABEL_VALUE_ID>, None]
-    For templated labels, the format is: [<LABEL_KEY_ID>, None, <JINJA2_TEMPLATE>]
+    alert_group_labels_custom stores config of dynamic labels. It's stored as a list of tuples.
+    Format of tuple is: [<LABEL_KEY_ID>, None, <JINJA2_TEMPLATE>].
+    The second element is deprecated, so it's always None. It was used for static labels.
+    // TODO: refactor to use just regular DB fields for dynamic label config.
     """
 
     alert_group_labels_template: str | None = models.TextField(null=True, default=None)
-    """Stores a Jinja2 template for "advanced label templating" for alert group labels."""
+    """
+    alert_group_labels_template is a Jinja2 template for "multi-label extraction template".
+    It extracts multiple labels from incoming alert payload.
+    """
 
     additional_settings: dict | None = models.JSONField(null=True, default=None)
 

engine/apps/api/serializers/alert_receive_channel.py

@@ -33,6 +33,7 @@ def _additional_settings_serializer_from_type(integration_type: str) -> serializ
     return cls
 
 
+# TODO: refactor this types as w no longer support storing static labels in this field.
 # AlertGroupCustomLabelValue represents custom alert group label value for API requests
 # It handles two types of label's value:
 # 1. Just Label Value from a label repo for a static label
@@ -206,7 +207,7 @@ def to_representation(cls, instance: AlertReceiveChannel) -> IntegrationAlertGro
     @staticmethod
     def _custom_labels_to_internal_value(
         custom_labels: AlertGroupCustomLabelsAPI,
-    ) -> AlertReceiveChannel.AlertGroupCustomLabelsDB:
+    ) -> AlertReceiveChannel.DynamicLabelsConfigDB:
         """Convert custom labels from API representation to the schema used by the JSONField on the model."""
 
         return [
@@ -216,7 +217,7 @@ def _custom_labels_to_internal_value(
 
     @staticmethod
     def _custom_labels_to_representation(
-        custom_labels: AlertReceiveChannel.AlertGroupCustomLabelsDB,
+        custom_labels: AlertReceiveChannel.DynamicLabelsConfigDB,
     ) -> AlertGroupCustomLabelsAPI:
         """
         Inverse of the _custom_labels_to_internal_value method above.

engine/apps/api/tests/test_alert_receive_channel.py

@@ -1486,14 +1486,14 @@ def test_alert_receive_channel_contact_points_wrong_integration(
 def test_integration_filter_by_labels(
     make_organization_and_user_with_plugin_token,
     make_alert_receive_channel,
-    make_integration_label_association,
+    make_static_label_config,
     make_user_auth_headers,
 ):
     organization, user, token = make_organization_and_user_with_plugin_token()
     alert_receive_channel_1 = make_alert_receive_channel(organization)
     alert_receive_channel_2 = make_alert_receive_channel(organization)
-    associated_label_1 = make_integration_label_association(organization, alert_receive_channel_1)
-    associated_label_2 = make_integration_label_association(organization, alert_receive_channel_1)
+    associated_label_1 = make_static_label_config(organization, alert_receive_channel_1)
+    associated_label_2 = make_static_label_config(organization, alert_receive_channel_1)
     alert_receive_channel_2.labels.create(
         key=associated_label_1.key, value=associated_label_1.value, organization=organization
     )
@@ -1659,7 +1659,7 @@ def test_alert_group_labels_get(
     make_organization_and_user_with_plugin_token,
     make_alert_receive_channel,
     make_label_key_and_value,
-    make_integration_label_association,
+    make_static_label_config,
     make_user_auth_headers,
 ):
     organization, user, token = make_organization_and_user_with_plugin_token()
@@ -1674,7 +1674,7 @@ def test_alert_group_labels_get(
     assert response.status_code == status.HTTP_200_OK
     assert response.json()["alert_group_labels"] == {"inheritable": {}, "custom": [], "template": None}
 
-    label = make_integration_label_association(organization, alert_receive_channel)
+    label = make_static_label_config(organization, alert_receive_channel)
 
     template = "{{ payload.labels | tojson }}"
     alert_receive_channel.alert_group_labels_template = template
@@ -1707,14 +1707,14 @@ def test_alert_group_labels_get(
 def test_alert_group_labels_put(
     make_organization_and_user_with_plugin_token,
     make_alert_receive_channel,
-    make_integration_label_association,
+    make_static_label_config,
     make_user_auth_headers,
 ):
     organization, user, token = make_organization_and_user_with_plugin_token()
     alert_receive_channel = make_alert_receive_channel(organization)
-    label_1 = make_integration_label_association(organization, alert_receive_channel)
-    label_2 = make_integration_label_association(organization, alert_receive_channel)
-    label_3 = make_integration_label_association(organization, alert_receive_channel)
+    label_1 = make_static_label_config(organization, alert_receive_channel)
+    label_2 = make_static_label_config(organization, alert_receive_channel)
+    label_3 = make_static_label_config(organization, alert_receive_channel)
 
     custom = [
         # plain label

engine/apps/email/inbound.py

@@ -7,6 +7,8 @@
 from anymail.inbound import AnymailInboundMessage
 from anymail.signals import AnymailInboundEvent
 from anymail.webhooks import amazon_ses, mailgun, mailjet, mandrill, postal, postmark, sendgrid, sparkpost
+from bs4 import BeautifulSoup
+from django.conf import settings
 from django.http import HttpResponse, HttpResponseNotAllowed
 from django.utils import timezone
 from rest_framework import status
@@ -25,6 +27,15 @@ class AmazonSESValidatedInboundWebhookView(amazon_ses.AmazonSESInboundWebhookVie
     # disable "Your Anymail webhooks are insecure and open to anyone on the web." warning
     warn_if_no_basic_auth = False
 
+    def __init__(self):
+        super().__init__(
+            session_params={
+                "aws_access_key_id": settings.INBOUND_EMAIL_AWS_ACCESS_KEY_ID,
+                "aws_secret_access_key": settings.INBOUND_EMAIL_AWS_SECRET_ACCESS_KEY,
+                "region_name": settings.INBOUND_EMAIL_AWS_REGION,
+            },
+        )
+
     def validate_request(self, request):
         """Add SNS message validation to Amazon SES inbound webhook view, which is not implemented in Anymail."""
         if not validate_amazon_sns_message(self._parse_sns_message(request)):
@@ -74,11 +85,10 @@ def dispatch(self, request):
         if request.method.lower() == "head":
             return HttpResponse(status=status.HTTP_200_OK)
 
-        integration_token = self.get_integration_token_from_request(request)
-        if integration_token is None:
+        if self.integration_token is None:
             return HttpResponse(status=status.HTTP_400_BAD_REQUEST)
-        request.inbound_email_integration_token = integration_token  # used in RequestTimeLoggingMiddleware
-        return super().dispatch(request, alert_channel_key=integration_token)
+        request.inbound_email_integration_token = self.integration_token  # used in RequestTimeLoggingMiddleware
+        return super().dispatch(request, alert_channel_key=self.integration_token)
 
     def post(self, request):
         payload = self.get_alert_payload_from_email_message(self.message)
@@ -94,7 +104,8 @@ def post(self, request):
         )
         return Response("OK", status=status.HTTP_200_OK)
 
-    def get_integration_token_from_request(self, request) -> Optional[str]:
+    @cached_property
+    def integration_token(self) -> Optional[str]:
         if not self.message:
             return None
         # First try envelope_recipient field.
@@ -151,7 +162,8 @@ def message(self) -> AnymailInboundMessage | None:
         logger.error("Failed to parse inbound email message")
         return None
 
-    def check_inbound_email_settings_set(self):
+    @staticmethod
+    def check_inbound_email_settings_set():
         """
         Guard method to checks if INBOUND_EMAIL settings present.
         Returns InternalServerError if not.
@@ -167,16 +179,105 @@ def check_inbound_email_settings_set(self):
             logger.error("InboundEmailWebhookView: INBOUND_EMAIL_DOMAIN env variable must be set.")
             return HttpResponse(status=status.HTTP_500_INTERNAL_SERVER_ERROR)
 
-    def get_alert_payload_from_email_message(self, email: AnymailInboundMessage) -> EmailAlertPayload:
-        subject = email.subject or ""
-        subject = subject.strip()
-        message = email.text or ""
-        message = message.strip()
-        sender = self.get_sender_from_email_message(email)
+    @classmethod
+    def get_alert_payload_from_email_message(cls, email: AnymailInboundMessage) -> EmailAlertPayload:
+        if email.text:
+            message = email.text.strip()
+        elif email.html:
+            message = cls.html_to_plaintext(email.html)
+        else:
+            message = ""
+
+        return {
+            "subject": email.subject.strip() if email.subject else "",
+            "message": message,
+            "sender": cls.get_sender_from_email_message(email),
+        }
+
+    @staticmethod
+    def html_to_plaintext(html: str) -> str:
+        """
+        Converts HTML to plain text. Renders links as "text (href)" and removes any empty lines.
+        Converting HTML to plaintext is a non-trivial task, so this method may not work perfectly for all cases.
+        """
+        soup = BeautifulSoup(html, "html.parser")
+
+        # Browsers typically render these elements on their own line.
+        # There is no single official HTML5 list for this, so we go with HTML tags that render as
+        # display: block, display: list-item, display: table, display: table-row by default according to the HTML standard:
+        # https://html.spec.whatwg.org/multipage/rendering.html
+        newline_tags = [
+            "address",
+            "article",
+            "aside",
+            "blockquote",
+            "body",
+            "center",
+            "dd",
+            "details",
+            "dialog",
+            "dir",
+            "div",
+            "dl",
+            "dt",
+            "fieldset",
+            "figcaption",
+            "figure",
+            "footer",
+            "form",
+            "h1",
+            "h2",
+            "h3",
+            "h4",
+            "h5",
+            "h6",
+            "header",
+            "hgroup",
+            "hr",
+            "html",
+            "legend",
+            "li",
+            "listing",
+            "main",
+            "menu",
+            "nav",
+            "ol",
+            "p",
+            "plaintext",
+            "pre",
+            "search",
+            "section",
+            "summary",
+            "table",
+            "tr",
+            "ul",
+            "xmp",
+        ]
+        # Insert a newline after each block-level element
+        for tag in soup.find_all(newline_tags):
+            tag.insert_before("\n")
+            tag.insert_after("\n")
+
+        # <br> tags are also typically rendered as newlines
+        for br in soup.find_all("br"):
+            br.replace_with("\n")
+
+        # example: "<a href="https://example.com">example</a>" -> "example (https://example.com)"
+        for a in soup.find_all("a"):
+            if href := a.get("href"):
+                a.append(f" ({href})")
+
+        for li in soup.find_all("li"):
+            li.insert_before("* ")
+
+        for hr in soup.find_all("hr"):
+            hr.replace_with("-" * 32)
 
-        return {"subject": subject, "message": message, "sender": sender}
+        # remove empty lines
+        return "\n".join(line.strip() for line in soup.get_text().splitlines() if line.strip())
 
-    def get_sender_from_email_message(self, email: AnymailInboundMessage) -> str:
+    @staticmethod
+    def get_sender_from_email_message(email: AnymailInboundMessage) -> str:
         try:
             if isinstance(email.from_email, list):
                 sender = email.from_email[0].addr_spec