Make sure organization token is valid before sync (#4904)

Since we will be triggering sync for orgs without a `last_time_synced`
set, we need to make sure the token is valid (previously both,
`last_time_synced` and the token, were updated from the frontend plugin)

engine/apps/user_management/sync.py

@@ -37,6 +37,14 @@ def _sync_organization(organization: Organization) -> None:
     grafana_api_client = GrafanaAPIClient(api_url=organization.grafana_url, api_token=organization.api_token)
     gcom_client = GcomAPIClient(settings.GRAFANA_COM_ADMIN_API_TOKEN)
 
+    # check organization API token is valid
+    _, check_token_call_status = grafana_api_client.check_token()
+    if not check_token_call_status["connected"]:
+        organization.api_token_status = Organization.API_TOKEN_STATUS_FAILED
+        organization.save(update_fields=["api_token_status"])
+        logger.warning(f"Sync not successful org={organization.pk} token_status=FAILED")
+        return
+
     rbac_is_enabled = organization.is_rbac_permissions_enabled
     # Update organization's RBAC status if it's an open-source instance, or it's an active cloud instance.
     # Don't update non-active cloud instances (e.g. paused) as they can return 200 OK but not have RBAC enabled.

engine/apps/user_management/tests/test_sync.py

@@ -10,7 +10,7 @@
 from apps.alerts.models import AlertReceiveChannel
 from apps.api.permissions import LegacyAccessControlRole
 from apps.grafana_plugin.sync_data import SyncData, SyncSettings, SyncUser
-from apps.user_management.models import User
+from apps.user_management.models import Organization, User
 from apps.user_management.sync import (
     apply_sync_data,
     cleanup_organization,
@@ -269,6 +269,18 @@ def test_sync_organization(make_organization):
     assert organization.is_grafana_labels_enabled is True
 
 
+@pytest.mark.django_db
+def test_sync_organization_invalid_api_token(make_organization):
+    organization = make_organization()
+
+    with patch("apps.user_management.sync.GrafanaAPIClient") as mock_grafana_api_client:
+        mock_grafana_api_client.return_value.check_token.return_value = (None, {"connected": False})
+        sync_organization(organization)
+
+    organization.refresh_from_db()
+    organization.api_token_status = Organization.API_TOKEN_STATUS_FAILED
+
+
 @pytest.mark.parametrize(
     "is_rbac_enabled_for_organization,expected",
     [