New OnCall plugin initialization process (#4657)

# What this PR does

New OnCall plugin initialization process

## Checklist

- [x] Unit, integration, and e2e (if applicable) tests updated
- [x] Documentation added (or `pr:no public docs` PR label added if not
required)
- [x] Added the relevant release notes label (see labels prefixed w/
`release:`). These labels dictate how your PR will
    show up in the autogenerated release notes.

---------

Co-authored-by: Michael Derynck <[email protected]>
Co-authored-by: Matias Bordese <[email protected]>

.github/CODEOWNERS

@@ -1,5 +1,6 @@
 * @grafana/grafana-oncall-backend
 /grafana-plugin @grafana/grafana-oncall-frontend
+/grafana-plugin/pkg @grafana/grafana-oncall-backend
 /docs @grafana/docs-gops @grafana/grafana-oncall
 
 # `make docs` procedure is owned by @jdbaldry of @grafana/docs-squad.

.github/workflows/e2e-tests.yml

@@ -109,13 +109,11 @@ jobs:
 
       # ---------- Expensive e2e tests steps start -----------
       - name: Install Go
-        if: inputs.run-expensive-tests
         uses: actions/setup-go@v4
         with:
           go-version: "1.21.5"
 
       - name: Install Mage
-        if: inputs.run-expensive-tests
         run: go install github.com/magefile/[email protected]
 
       - name: Get Vault secrets

.github/workflows/linting-and-tests.yml

@@ -13,7 +13,7 @@ env:
 jobs:
   lint-entire-project:
     name: "Lint entire project"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-16-cores
     steps:
       - name: Checkout project
         uses: actions/checkout@v4
@@ -27,7 +27,7 @@ jobs:
 
   lint-test-and-build-frontend:
     name: "Lint, test, and build frontend"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-16-cores
     steps:
       - name: Checkout project
         uses: actions/checkout@v4
@@ -39,7 +39,7 @@ jobs:
 
   test-technical-documentation:
     name: "Test technical documentation"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-16-cores
     steps:
       - name: "Check out code"
         uses: "actions/checkout@v4"
@@ -56,7 +56,7 @@ jobs:
 
   lint-migrations-backend-mysql-rabbitmq:
     name: "Lint database migrations"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-16-cores
     services:
       rabbit_test:
         image: rabbitmq:3.12.0
@@ -87,7 +87,7 @@ jobs:
 
   unit-test-helm-chart:
     name: "Helm Chart Unit Tests"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-16-cores
     steps:
       - name: Checkout project
         uses: actions/checkout@v4
@@ -99,6 +99,16 @@ jobs:
       - name: Run tests
         run: helm unittest ./helm/oncall
 
+  unit-test-backend-plugin:
+    name: "Backend Tests: Plugin"
+    runs-on: ubuntu-latest-16-cores
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.21.5"
+      - run: cd grafana-plugin && go test ./pkg/...
+
   unit-test-backend-mysql-rabbitmq:
     name: "Backend Tests: MySQL + RabbitMQ (RBAC enabled: ${{ matrix.rbac_enabled }})"
     runs-on: ubuntu-latest-16-cores
@@ -202,7 +212,7 @@ jobs:
 
   unit-test-migrators:
     name: "Unit tests - Migrators"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-16-cores
     steps:
       - name: Checkout project
         uses: actions/checkout@v4
@@ -216,7 +226,7 @@ jobs:
 
   mypy:
     name: "mypy"
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-16-cores
     steps:
       - name: Checkout project
         uses: actions/checkout@v4

.tilt/plugin/Tiltfile

@@ -23,4 +23,21 @@ if not is_ci:
         serve_dir=grafana_plugin_dir,
         serve_cmd="yarn watch",
         allow_parallel=True,
-    )
+    )
+
+local_resource(
+    'build-oncall-plugin-backend',
+    labels=[label],
+    dir="../../grafana-plugin",
+    cmd="mage buildAll",
+    deps=['../../grafana-plugin/pkg/plugin']
+)
+
+local_resource(
+    'restart-oncall-plugin-backend',
+    labels=[label],
+    dir="../../dev/scripts",
+    cmd="chmod +x ./restart_backend_plugin.sh && ./restart_backend_plugin.sh",
+    resource_deps=["grafana", "build-oncall-plugin-backend"],
+    deps=['../../grafana-plugin/pkg/plugin']
+)

.tilt/tests/Tiltfile

@@ -11,7 +11,7 @@ local_resource(
     cmd=e2e_tests_cmd,
     trigger_mode=TRIGGER_MODE_MANUAL,
     auto_init=is_ci,
-    resource_deps=["build-ui", "grafana", "grafana-oncall-app-provisioning-configmap", "engine", "celery"]
+    resource_deps=["build-ui", "grafana", "grafana-oncall-app-provisioning-configmap", "engine", "celery", "build-oncall-plugin-backend"]
 )
 
 cmd_button(

Tiltfile

@@ -71,7 +71,7 @@ if not running_under_parent_tiltfile:
     # Load the custom Grafana extensions
     v1alpha1.extension_repo(
         name="grafana-tilt-extensions",
-        ref="v1.2.0",
+        ref="v1.4.2",
         url="https://github.com/grafana/tilt-extensions",
     )
 v1alpha1.extension(
@@ -83,6 +83,7 @@ def load_grafana():
     # The user/pass that you will login to Grafana with
     grafana_admin_user_pass = os.getenv("GRAFANA_ADMIN_USER_PASS", "oncall")
     grafana_version = os.getenv("GRAFANA_VERSION", "latest")
+    grafana_url = os.getenv("GRAFANA_URL", "http://grafana:3000")
 
 
     if 'plugin' in profiles:
@@ -100,11 +101,15 @@ def load_grafana():
             context="grafana-plugin",
             plugin_files=["grafana-plugin/src/plugin.json"],
             namespace="default",
-            deps=["grafana-oncall-app-provisioning-configmap", "build-ui"],
+            deps=["grafana-oncall-app-provisioning-configmap", "build-ui", "build-oncall-plugin-backend"],
             extra_env={
                 "GF_SECURITY_ADMIN_PASSWORD": "oncall",
                 "GF_SECURITY_ADMIN_USER": "oncall",
                 "GF_AUTH_ANONYMOUS_ENABLED": "false",
+                "GF_APP_URL": grafana_url,  # older versions of grafana need this
+                "GF_SERVER_ROOT_URL": grafana_url,
+                "GF_FEATURE_TOGGLES_ENABLE": "externalServiceAccounts",
+                "ONCALL_API_URL": "http://oncall-dev-engine:8080"
             },
         )
 # --- GRAFANA END ----

dev/grafana/provisioning/plugins/grafana-oncall-app-provisioning.yaml

@@ -5,4 +5,4 @@ apps:
     jsonData:
       stackId: 5
       orgId: 100
-      onCallApiUrl: http://oncall-dev-engine:8080
+      onCallApiUrl: $ONCALL_API_URL

dev/helm-local.yml

@@ -70,7 +70,7 @@ grafana:
         - name: DATABASE_PASSWORD
           value: oncallpassword
   env:
-    GF_FEATURE_TOGGLES_ENABLE: topnav
+    GF_FEATURE_TOGGLES_ENABLE: topnav,externalServiceAccounts
     GF_SECURITY_ADMIN_PASSWORD: oncall
     GF_SECURITY_ADMIN_USER: oncall
     GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS: grafana-oncall-app

dev/scripts/restart_backend_plugin.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Find a grafana pod
+pod=$(kubectl get pods -l app.kubernetes.io/name=grafana -o=jsonpath='{.items[0].metadata.name}')
+
+if [ -z "$pod" ]; then
+    echo "No pod found with the specified label."
+    exit 1
+fi
+
+# Exec into the pod
+kubectl exec -it "$pod" -- /bin/bash <<'EOF'
+
+# Find and kill the process containing "gpx_grafana" (plugin backend process)
+process_id=$(ps aux | grep gpx_grafana | grep -v grep | awk '{print $1}')
+echo $process_id
+if [ -n "$process_id" ]; then
+    echo "Killing process $process_id"
+    kill $process_id
+else
+    echo "No process containing 'gpx_grafana' in COMMAND found."
+fi
+EOF

docker-compose-developer.yml

@@ -52,8 +52,6 @@ services:
       context: ./grafana-plugin
       dockerfile: Dockerfile.dev
       labels: *oncall-labels
-    environment:
-      ONCALL_API_URL: http://host.docker.internal:8080
     volumes:
       - ./grafana-plugin:/etc/app
       - node_modules_dev:/etc/app/node_modules
@@ -324,6 +322,8 @@ services:
       GF_SECURITY_ADMIN_USER: oncall
       GF_SECURITY_ADMIN_PASSWORD: oncall
       GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS: grafana-oncall-app
+      GF_FEATURE_TOGGLES_ENABLE: externalServiceAccounts
+      ONCALL_API_URL: http://host.docker.internal:8080
     env_file:
       - ./dev/.env.${DB}.dev
     ports:

engine/apps/alerts/grafana_alerting_sync_manager/grafana_alerting_sync.py

@@ -370,16 +370,20 @@ def _get_connected_contact_points_from_config(self, alertmanager_config: dict, i
         return contact_points
 
     def _recursive_check_contact_point_is_in_routes(self, route_config: dict, receiver_name: str) -> bool:
-        if route_config.get("receiver") == receiver_name:
-            return True
-        routes = route_config.get("routes", [])
-        for route in routes:
-            if route.get("receiver") == receiver_name:
-                return True
-            if route.get("routes"):
-                if self._recursive_check_contact_point_is_in_routes(route, receiver_name):
-                    return True
-        return False
+        # TODO: Relaxing this condition due to API limitations when requesting config with external service account
+        # instead of Admin response does not contain child routes.  We are currently considering the integration
+        # connected as long as the contact point exists.
+        return True
+        # if route_config.get("receiver") == receiver_name:
+        #     return True
+        # routes = route_config.get("routes", [])
+        # for route in routes:
+        #     if route.get("receiver") == receiver_name:
+        #         return True
+        #     if route.get("routes"):
+        #         if self._recursive_check_contact_point_is_in_routes(route, receiver_name):
+        #             return True
+        # return False
 
     def _get_oncall_config_and_config_field_for_datasource_type(
         self, contact_point_name: str, is_grafana_datasource: bool, is_oncall_type_available: bool

engine/apps/alerts/tests/test_grafana_alerting_sync.py

@@ -334,7 +334,7 @@ def test_get_connected_contact_points_from_config(
             },
             {
                 "name": ALERTMANAGER_INACTIVE_RECEIVER_CONNECTED,
-                "notification_connected": False,
+                "notification_connected": True,
             },
         ]
         if alertmanager_config

engine/apps/grafana_plugin/helpers/gcom.py

@@ -54,10 +54,10 @@ def check_gcom_permission(token_string: str, context) -> GcomToken:
         if allow_signup:
             # Get org from db or create a new one
             organization, _ = Organization.objects.get_or_create(
-                stack_id=str(instance_info["id"]),
+                stack_id=instance_info["id"],
                 stack_slug=instance_info["slug"],
                 grafana_url=instance_info["url"],
-                org_id=str(instance_info["orgId"]),
+                org_id=instance_info["orgId"],
                 org_slug=instance_info["orgSlug"],
                 org_title=instance_info["orgName"],
                 region_slug=instance_info["regionSlug"],

engine/apps/grafana_plugin/serializers/sync_data.py

@@ -95,7 +95,13 @@ def to_internal_value(self, data):
         data = super().to_internal_value(data)
         users = data.get("users")
         if users:
-            data["users"] = [SyncUser(**user) for user in users]
+
+            def create_user(user):
+                permissions_data = user.pop("permissions", [])
+                permissions = [SyncPermission(**perm) for perm in permissions_data] if permissions_data else []
+                return SyncUser(permissions=permissions, **user)
+
+            data["users"] = [create_user(user) for user in users]
         teams = data.get("teams")
         if teams:
             data["teams"] = [SyncTeam(**team) for team in teams]

engine/apps/grafana_plugin/tests/test_sync_v2.py

@@ -14,6 +14,7 @@ def test_auth_success(make_organization_and_user_with_plugin_token, make_user_au
     client = APIClient()
 
     auth_headers = make_user_auth_headers(user, token)
+    del auth_headers["HTTP_X-Grafana-Context"]
 
     with patch("apps.grafana_plugin.views.sync_v2.SyncV2View.do_sync", return_value=organization) as mock_sync:
         response = client.post(reverse("grafana-plugin:sync-v2"), format="json", **auth_headers)
@@ -35,9 +36,11 @@ def test_invalid_auth(make_organization_and_user_with_plugin_token, make_user_au
     assert response.status_code == status.HTTP_401_UNAUTHORIZED
     assert not mock_sync.called
 
-    auth_headers = make_user_auth_headers(user, token)
+    auth_headers = make_user_auth_headers(None, token, organization=organization)
+    del auth_headers["HTTP_X-Instance-Context"]
+
     with patch("apps.grafana_plugin.views.sync_v2.SyncV2View.do_sync", return_value=organization) as mock_sync:
         response = client.post(reverse("grafana-plugin:sync-v2"), format="json", **auth_headers)
 
-    assert response.status_code == status.HTTP_403_FORBIDDEN
+    assert response.status_code == status.HTTP_401_UNAUTHORIZED
     assert not mock_sync.called

engine/apps/grafana_plugin/views/install_v2.py

@@ -1,4 +1,5 @@
 import logging
+from dataclasses import asdict
 
 from django.conf import settings
 from rest_framework import status
@@ -17,7 +18,7 @@ class InstallV2View(SyncV2View):
 
     def post(self, request: Request) -> Response:
         if settings.LICENSE != settings.OPEN_SOURCE_LICENSE_NAME:
-            return Response(data=SELF_HOSTED_ONLY_FEATURE_ERROR, status=status.HTTP_403_FORBIDDEN)
+            return Response(data=asdict(SELF_HOSTED_ONLY_FEATURE_ERROR), status=status.HTTP_403_FORBIDDEN)
 
         try:
             organization = self.do_sync(request)

engine/apps/grafana_plugin/views/sync_v2.py

@@ -1,14 +1,13 @@
 import logging
+from dataclasses import asdict
 
 from django.conf import settings
 from rest_framework import status
-from rest_framework.permissions import IsAuthenticated
 from rest_framework.request import Request
 from rest_framework.response import Response
 from rest_framework.views import APIView
 
-from apps.api.permissions import RBACPermission
-from apps.auth_token.auth import PluginAuthentication
+from apps.auth_token.auth import BasePluginAuthentication
 from apps.grafana_plugin.serializers.sync_data import SyncDataSerializer
 from apps.user_management.models import Organization
 from apps.user_management.sync import apply_sync_data, get_or_create_organization
@@ -23,11 +22,7 @@ def __init__(self, error_data):
 
 
 class SyncV2View(APIView):
-    authentication_classes = (PluginAuthentication,)
-    permission_classes = [IsAuthenticated, RBACPermission]
-    rbac_permissions = {
-        "post": [RBACPermission.Permissions.USER_SETTINGS_ADMIN],
-    }
+    authentication_classes = (BasePluginAuthentication,)
 
     def do_sync(self, request: Request) -> Organization:
         serializer = SyncDataSerializer(data=request.data)
@@ -40,7 +35,7 @@ def do_sync(self, request: Request) -> Organization:
             stack_id = settings.SELF_HOSTED_SETTINGS["STACK_ID"]
             org_id = settings.SELF_HOSTED_SETTINGS["ORG_ID"]
         else:
-            org_id = request.auth.organization
+            org_id = request.auth.organization.org_id
             stack_id = request.auth.organization.stack_id
 
         if sync_data.settings.org_id != org_id or sync_data.settings.stack_id != stack_id:
@@ -54,6 +49,6 @@ def post(self, request: Request) -> Response:
         try:
             self.do_sync(request)
         except SyncException as e:
-            return Response(data=e.error_data, status=status.HTTP_400_BAD_REQUEST)
+            return Response(data=asdict(e.error_data), status=status.HTTP_400_BAD_REQUEST)
 
         return Response(status=status.HTTP_200_OK)

engine/apps/user_management/sync.py

@@ -419,7 +419,10 @@ def _sync_teams_members_data(organization: Organization, team_members: dict[int,
     # set team members
     for team_id, members_ids in team_members.items():
         team = organization.teams.get(team_id=team_id)
-        team.users.set(organization.users.filter(user_id__in=members_ids))
+        if members_ids:
+            team.users.set(organization.users.filter(user_id__in=members_ids))
+        else:
+            team.users.clear()
 
 
 def apply_sync_data(organization: Organization, sync_data: SyncData):