Add method to send notification bundle by SMS (#4624)

# What this PR does
Adds method to render and send notification bundle by sms.

Example of SMS message:
```
Grafana OnCall: Alert groups #1, #2, #3 and 2 more 
from stack: TestOrganization, 
integrations: Grafana Alerting and 1 more.
```

Should be merged with #4457

## Which issue(s) this PR closes
grafana/oncall-private#2713

## Checklist

- [x] Unit, integration, and e2e (if applicable) tests updated
- [x] Documentation added (or `pr:no public docs` PR label added if not
required)
- [x] Added the relevant release notes label (see labels prefixed w/
`release:`). These labels dictate how your PR will
    show up in the autogenerated release notes.

engine/apps/alerts/incident_appearance/renderers/base_renderer.py

@@ -1,10 +1,11 @@
 import typing
 from abc import ABC, abstractmethod
 
+from django.db.models import QuerySet
 from django.utils.functional import cached_property
 
 if typing.TYPE_CHECKING:
-    from apps.alerts.models import Alert, AlertGroup
+    from apps.alerts.models import Alert, AlertGroup, BundledNotification
 
 
 class AlertBaseRenderer(ABC):
@@ -33,3 +34,11 @@ def __init__(self, alert_group: "AlertGroup", alert: typing.Optional["Alert"] =
     @abstractmethod
     def alert_renderer_class(self):
         raise NotImplementedError
+
+
+class AlertGroupBundleBaseRenderer:
+    MAX_ALERT_GROUPS_TO_RENDER = 3
+    MAX_CHANNELS_TO_RENDER = 1
+
+    def __init__(self, notifications: "QuerySet[BundledNotification]"):
+        self.notifications = notifications

engine/apps/alerts/incident_appearance/renderers/sms_renderer.py

@@ -1,4 +1,10 @@
-from apps.alerts.incident_appearance.renderers.base_renderer import AlertBaseRenderer, AlertGroupBaseRenderer
+from django.db.models import Count
+
+from apps.alerts.incident_appearance.renderers.base_renderer import (
+    AlertBaseRenderer,
+    AlertGroupBaseRenderer,
+    AlertGroupBundleBaseRenderer,
+)
 from apps.alerts.incident_appearance.renderers.constants import DEFAULT_BACKUP_TITLE
 from apps.alerts.incident_appearance.templaters import AlertSmsTemplater
 from common.utils import str_or_backup
@@ -24,3 +30,53 @@ def render(self):
             f"integration: {self.alert_group.channel.short_name}, "
             f"alerts registered: {self.alert_group.alerts.count()}."
         )
+
+
+class AlertGroupSMSBundleRenderer(AlertGroupBundleBaseRenderer):
+    def render(self) -> str:
+        """
+        Renders SMS message for notification bundle: gets total count of unique alert groups and alert receive channels
+        in the bundle, renders text with `inside_organization_number` of 3 alert groups (MAX_ALERT_GROUPS_TO_RENDER) and
+        `short_name` of 1 alert receive channel (MAX_CHANNELS_TO_RENDER). If there are more unique alert groups / alert
+        receive channels to notify about, adds "and X more" to the SMS message
+        """
+
+        channels_and_alert_groups_count = self.notifications.aggregate(
+            channels_count=Count("alert_receive_channel", distinct=True),
+            alert_groups_count=Count("alert_group", distinct=True),
+        )
+        alert_groups_count = channels_and_alert_groups_count["alert_groups_count"]
+        channels_count = channels_and_alert_groups_count["channels_count"]
+
+        # get 3 unique alert groups from notifications
+        alert_groups_to_render = []
+        for notification in self.notifications:
+            if notification.alert_group not in alert_groups_to_render:
+                alert_groups_to_render.append(notification.alert_group)
+                if len(alert_groups_to_render) == self.MAX_ALERT_GROUPS_TO_RENDER:
+                    break
+        # render text for alert groups
+        alert_group_inside_organization_numbers = [
+            alert_group.inside_organization_number for alert_group in alert_groups_to_render
+        ]
+        numbers_str = ", ".join(f"#{x}" for x in alert_group_inside_organization_numbers)
+        alert_groups_text = "Alert groups " if alert_groups_count > 1 else "Alert group "
+        alert_groups_text += numbers_str
+
+        if alert_groups_count > self.MAX_ALERT_GROUPS_TO_RENDER:
+            alert_groups_text += f" and {alert_groups_count - self.MAX_ALERT_GROUPS_TO_RENDER} more"
+
+        # render text for alert receive channels
+        channels_to_render = [alert_groups_to_render[i].channel for i in range(self.MAX_CHANNELS_TO_RENDER)]
+        channel_names = ", ".join([channel.short_name for channel in channels_to_render])
+        channels_text = "integrations: " if channels_count > 1 else "integration: "
+        channels_text += channel_names
+
+        if channels_count > self.MAX_CHANNELS_TO_RENDER:
+            channels_text += f" and {channels_count - self.MAX_CHANNELS_TO_RENDER} more"
+
+        return (
+            f"Grafana OnCall: {alert_groups_text} "
+            f"from stack: {channels_to_render[0].organization.stack_slug}, "
+            f"{channels_text}."
+        )

engine/apps/alerts/tests/test_alert_group_renderer.py

@@ -1,7 +1,9 @@
 import pytest
 
+from apps.alerts.incident_appearance.renderers.sms_renderer import AlertGroupSMSBundleRenderer
 from apps.alerts.incident_appearance.templaters import AlertSlackTemplater, AlertWebTemplater
 from apps.alerts.models import AlertGroup
+from apps.base.models import UserNotificationPolicy
 from config_integrations import grafana
 
 
@@ -163,3 +165,58 @@ def test_get_resolved_text(
     alert_group.resolve(resolved_by=source, resolved_by_user=user)
 
     assert alert_group.get_resolve_text() == expected_text.format(username=user.get_username_with_slack_verbal())
+
+
+@pytest.mark.django_db
+def test_alert_group_sms_bundle_renderer(
+    make_organization_and_user,
+    make_alert_receive_channel,
+    make_alert_group,
+    make_user_notification_bundle,
+):
+    organization, user = make_organization_and_user()
+    alert_receive_channel_1 = make_alert_receive_channel(
+        organization,
+    )
+    alert_receive_channel_2 = make_alert_receive_channel(
+        organization,
+    )
+    alert_group_1 = make_alert_group(alert_receive_channel_1)
+    alert_group_2 = make_alert_group(alert_receive_channel_1)
+    alert_group_3 = make_alert_group(alert_receive_channel_1)
+    alert_group_4 = make_alert_group(alert_receive_channel_2)
+
+    notification_bundle = make_user_notification_bundle(user, UserNotificationPolicy.NotificationChannel.SMS)
+
+    # render 1 alert group and 1 channel
+    notification_bundle.append_notification(alert_group_1, None)
+    renderer = AlertGroupSMSBundleRenderer(notification_bundle.notifications.all())
+    message = renderer.render()
+    assert message == (
+        f"Grafana OnCall: Alert group #{alert_group_1.inside_organization_number} "
+        f"from stack: {organization.stack_slug}, "
+        f"integration: {alert_receive_channel_1.short_name}."
+    )
+
+    # render 3 alert groups and 1 channel
+    notification_bundle.append_notification(alert_group_2, None)
+    notification_bundle.append_notification(alert_group_3, None)
+    renderer = AlertGroupSMSBundleRenderer(notification_bundle.notifications.all())
+    message = renderer.render()
+    assert message == (
+        f"Grafana OnCall: Alert groups #{alert_group_1.inside_organization_number}, "
+        f"#{alert_group_2.inside_organization_number}, #{alert_group_3.inside_organization_number} "
+        f"from stack: {organization.stack_slug}, "
+        f"integration: {alert_receive_channel_1.short_name}."
+    )
+
+    # render 4 alert groups and 2 channels
+    notification_bundle.append_notification(alert_group_4, None)
+    renderer = AlertGroupSMSBundleRenderer(notification_bundle.notifications.all())
+    message = renderer.render()
+    assert message == (
+        f"Grafana OnCall: Alert groups #{alert_group_1.inside_organization_number}, "
+        f"#{alert_group_2.inside_organization_number}, #{alert_group_3.inside_organization_number} and 1 more "
+        f"from stack: {organization.stack_slug}, "
+        f"integrations: {alert_receive_channel_1.short_name} and 1 more."
+    )

engine/apps/exotel/status_callback.py

@@ -15,7 +15,7 @@ def get_call_status_callback_url():
 
 
 def update_exotel_call_status(call_id: str, call_status: str, user_choice: Optional[str] = None):
-    from apps.base.models import UserNotificationPolicyLogRecord
+    from apps.base.models import UserNotificationPolicy, UserNotificationPolicyLogRecord
 
     status_code = ExotelCallStatuses.DETERMINANT.get(call_status)
     if status_code is None:
@@ -62,12 +62,8 @@ def update_exotel_call_status(call_id: str, call_status: str, user_choice: Optio
             author=phone_call_record.receiver,
             notification_policy=phone_call_record.notification_policy,
             alert_group=phone_call_record.represents_alert_group,
-            notification_step=phone_call_record.notification_policy.step
-            if phone_call_record.notification_policy
-            else None,
-            notification_channel=phone_call_record.notification_policy.notify_by
-            if phone_call_record.notification_policy
-            else None,
+            notification_step=UserNotificationPolicy.Step.NOTIFY,
+            notification_channel=UserNotificationPolicy.NotificationChannel.PHONE_CALL,
         )
         log_record.save()
         logger.info(

engine/apps/phone_notifications/migrations/0003_smsrecord_represents_bundle_uuid.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.10 on 2024-07-03 08:15
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('phone_notifications', '0002_bannedphonenumber'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='smsrecord',
+            name='represents_bundle_uuid',
+            field=models.CharField(db_index=True, default=None, max_length=100, null=True),
+        ),
+    ]

engine/apps/phone_notifications/models/sms.py

@@ -45,7 +45,7 @@ class Meta:
     notification_policy = models.ForeignKey(
         "base.UserNotificationPolicy", on_delete=models.SET_NULL, null=True, default=None
     )
-
+    represents_bundle_uuid = models.CharField(max_length=100, null=True, default=None, db_index=True)
     receiver = models.ForeignKey("user_management.User", on_delete=models.CASCADE, null=True, default=None)
     grafana_cloud_notification = models.BooleanField(default=False)
     created_at = models.DateTimeField(auto_now_add=True)

engine/apps/phone_notifications/phone_backend.py

@@ -1,14 +1,15 @@
 import logging
-from typing import Optional
+from typing import Optional, Tuple
 
 import requests
 from django.conf import settings
 
 from apps.alerts.incident_appearance.renderers.phone_call_renderer import AlertGroupPhoneCallRenderer
-from apps.alerts.incident_appearance.renderers.sms_renderer import AlertGroupSmsRenderer
+from apps.alerts.incident_appearance.renderers.sms_renderer import AlertGroupSMSBundleRenderer, AlertGroupSmsRenderer
 from apps.alerts.signals import user_notification_action_triggered_signal
 from apps.base.utils import live_settings
 from common.api_helpers.utils import create_engine_url
+from common.custom_celery_tasks import shared_dedicated_queue_retry_task
 from common.utils import clean_markup
 
 from .exceptions import (
@@ -27,6 +28,19 @@
 logger = logging.getLogger(__name__)
 
 
+@shared_dedicated_queue_retry_task(
+    autoretry_for=(Exception,), retry_backoff=True, max_retries=0 if settings.DEBUG else None
+)
+def notify_by_sms_bundle_async_task(user_id, bundle_uuid):
+    from apps.user_management.models import User
+
+    user = User.objects.filter(id=user_id).first()
+    if not user:
+        return
+    phone_backend = PhoneBackend()
+    phone_backend.notify_by_sms_bundle(user, bundle_uuid)
+
+
 class PhoneBackend:
     def __init__(self):
         self.phone_provider: PhoneProvider = self._get_phone_provider()
@@ -148,16 +162,90 @@ def notify_by_sms(self, user, alert_group, notification_policy):
 
         from apps.base.models import UserNotificationPolicyLogRecord
 
-        log_record_error_code = None
-
         renderer = AlertGroupSmsRenderer(alert_group)
         message = renderer.render()
+        _, log_record_error_code = self._send_sms(
+            user=user,
+            alert_group=alert_group,
+            notification_policy=notification_policy,
+            message=message,
+        )
+
+        if log_record_error_code is not None:
+            log_record = UserNotificationPolicyLogRecord(
+                author=user,
+                type=UserNotificationPolicyLogRecord.TYPE_PERSONAL_NOTIFICATION_FAILED,
+                notification_policy=notification_policy,
+                alert_group=alert_group,
+                notification_error_code=log_record_error_code,
+                notification_step=notification_policy.step if notification_policy else None,
+                notification_channel=notification_policy.notify_by if notification_policy else None,
+            )
+            log_record.save()
+            user_notification_action_triggered_signal.send(sender=PhoneBackend.notify_by_sms, log_record=log_record)
+
+    @staticmethod
+    def notify_by_sms_bundle_async(user, bundle_uuid):
+        notify_by_sms_bundle_async_task.apply_async((user.id, bundle_uuid))
+
+    def notify_by_sms_bundle(self, user, bundle_uuid):
+        """
+        notify_by_sms_bundle sends an sms notification bundle to a user using configured phone provider.
+        It handles business logic - limits, cloud notifications and UserNotificationPolicyLogRecord creation.
+        It creates UserNotificationPolicyLogRecord for every notification in bundle, but only one SMSRecord.
+        SMS itself is handled by phone provider.
+        """
+
+        from apps.alerts.models import BundledNotification
+        from apps.base.models import UserNotificationPolicy, UserNotificationPolicyLogRecord
+
+        notifications = BundledNotification.objects.filter(bundle_uuid=bundle_uuid).select_related("alert_group")
+
+        if not notifications:
+            logger.info("Notification bundle is empty, related alert groups might have been deleted")
+            return
+        renderer = AlertGroupSMSBundleRenderer(notifications)
+        message = renderer.render()
+
+        _, log_record_error_code = self._send_sms(user=user, message=message, bundle_uuid=bundle_uuid)
 
+        if log_record_error_code is not None:
+            log_records_to_create = []
+            for notification in notifications:
+                log_record = UserNotificationPolicyLogRecord(
+                    author=user,
+                    type=UserNotificationPolicyLogRecord.TYPE_PERSONAL_NOTIFICATION_FAILED,
+                    notification_policy=notification.notification_policy,
+                    alert_group=notification.alert_group,
+                    notification_error_code=log_record_error_code,
+                    notification_step=UserNotificationPolicy.Step.NOTIFY,
+                    notification_channel=UserNotificationPolicy.NotificationChannel.SMS,
+                )
+                log_records_to_create.append(log_record)
+            if log_records_to_create:
+                if log_record_error_code in UserNotificationPolicyLogRecord.ERRORS_TO_SEND_IN_SLACK_CHANNEL:
+                    # create last log record outside of the bulk_create to get it as an object to send
+                    # the user_notification_action_triggered_signal
+                    log_record = log_records_to_create.pop()
+                    log_record.save()
+                    user_notification_action_triggered_signal.send(
+                        sender=PhoneBackend.notify_by_sms_bundle, log_record=log_record
+                    )
+
+                UserNotificationPolicyLogRecord.objects.bulk_create(log_records_to_create, batch_size=5000)
+
+    def _send_sms(
+        self, user, message, alert_group=None, notification_policy=None, bundle_uuid=None
+    ) -> Tuple[bool, Optional[int]]:
+        from apps.base.models import UserNotificationPolicyLogRecord
+
+        log_record_error_code = None
         record = SMSRecord(
             represents_alert_group=alert_group,
             receiver=user,
             notification_policy=notification_policy,
             exceeded_limit=False,
+            represents_bundle_uuid=bundle_uuid,
         )
 
         try:
@@ -180,22 +268,7 @@ def notify_by_sms(self, user, alert_group, notification_policy):
         except NumberNotVerified:
             log_record_error_code = UserNotificationPolicyLogRecord.ERROR_NOTIFICATION_PHONE_NUMBER_IS_NOT_VERIFIED
 
-        if log_record_error_code is not None:
-            log_record = UserNotificationPolicyLogRecord(
-                author=user,
-                type=UserNotificationPolicyLogRecord.TYPE_PERSONAL_NOTIFICATION_FAILED,
-                notification_policy=notification_policy,
-                alert_group=alert_group,
-                notification_error_code=log_record_error_code,
-                notification_step=notification_policy.step if notification_policy else None,
-                notification_channel=notification_policy.notify_by if notification_policy else None,
-            )
-            log_record.save()
-            user_notification_action_triggered_signal.send(sender=PhoneBackend.notify_by_sms, log_record=log_record)
-
-    @staticmethod
-    def notify_by_sms_bundle_async(user, bundle_uuid):
-        pass  # todo: will be added in a separate PR
+        return log_record_error_code is None, log_record_error_code
 
     def _notify_by_provider_sms(self, user, message) -> Optional[ProviderSMS]:
         """