fix: 🐛 Provide retry logic for GoogleAPI execute (#323)

* fix: 🐛 Provide retry logic for GoogleAPI execute

* Add change doc

* Update changes/323.fixed

Co-authored-by: Gary Snider <[email protected]>

* style: 🎨 Adhere to ruff format

---------

Co-authored-by: Gary Snider <[email protected]>

changes/323.fixed

@@ -0,0 +1 @@
+Added retry logic to GoogleAPI execute.

nautobot_circuit_maintenance/filters.py

@@ -41,7 +41,7 @@ class Meta:
         """Meta class attributes for CircuitMaintenanceFilterSet."""
 
         model = CircuitMaintenance
-        fields = '__all__'
+        fields = "__all__"
 
     def search(self, queryset, name, value):  # pylint: disable=unused-argument
         """Perform the filtered search."""
@@ -71,7 +71,7 @@ class Meta:
         """Meta class attributes for CircuitImpactFilterSet."""
 
         model = CircuitImpact
-        fields = '__all__'
+        fields = "__all__"
 
 
 class NoteFilterSet(NautobotFilterSet):
@@ -88,7 +88,7 @@ class Meta:
         """Meta class attributes for NoteFilterSet."""
 
         model = Note
-        fields = '__all__'
+        fields = "__all__"
 
     def search(self, queryset, name, value):  # pylint: disable=unused-argument
         """Perform the filtered search."""
@@ -153,7 +153,7 @@ class Meta:
         """Meta class attributes for ParsedNotificationFilterSet."""
 
         model = ParsedNotification
-        fields = '__all__'
+        fields = "__all__"
 
     def search(self, queryset, name, value):  # pylint: disable=unused-argument
         """Perform the filtered search."""

nautobot_circuit_maintenance/handle_notifications/sources.py

@@ -8,6 +8,7 @@
 import logging
 import os
 import re
+import time
 from typing import Dict, Iterable, List, Optional, Tuple, Type, TypeVar, Union
 from urllib.parse import urlparse
 
@@ -580,18 +581,60 @@ def extract_raw_payload(self, body: Dict, msg_id: str) -> bytes:
 
         return b""
 
+    @staticmethod
+    def _execute_with_retries(request, job, retries=5, delay=1, backoff=2):
+        """
+        Executes a Google API request with retries and exponential backoff.
+
+        Args:
+            request: The Google API request object.
+            job: Job object to log.
+            retries (int): Maximum number of retries.
+            delay (int/float): Initial delay between retries.
+            backoff (int/float): Backoff multiplier to increase delay.
+
+        Returns:
+            The result of the successful execution of the request.
+
+        Raises:
+            HttpError: If all retries fail, it raises the last HttpError.
+        """
+        attempt = 0
+        last_response = None
+        while attempt < retries:
+            try:
+                return request.execute()
+            except HttpError as http_error:
+                # Check if the error is retryable (e.g., 500, 503, rate limit)
+                last_response = http_error.resp
+                if http_error.resp.status in [500, 502, 503, 504]:
+                    attempt += 1
+                    job.logger.warning(
+                        f"Google API attempt {attempt} failed: {http_error}. Retrying in {delay} seconds..."
+                    )
+                    time.sleep(delay)
+                    delay *= backoff
+                else:
+                    # Raise the error if it's not retryable (e.g., 400, 403)
+                    raise
+
+        if last_response:
+            raise HttpError(resp=last_response, content=last_response.reason)
+        return None
+
     def fetch_email(self, job: Job, msg_id: str) -> Optional[MaintenanceNotification]:
         """Fetch an specific email ID.
 
         See data format:  https://developers.google.com/gmail/api/reference/rest/v1/users.messages#Message
         """
-        received_email = (
+        request = (
             self.service.users()  # pylint: disable=no-member
             .messages()
             .get(userId=self.account, id=msg_id, format="raw")
-            .execute()
         )
 
+        received_email = self._execute_with_retries(request, job)
+
         raw_email_string = base64.urlsafe_b64decode(received_email["raw"].encode("utf8"))
         email_message = email.message_from_bytes(raw_email_string)
         return self.process_email(job, email_message, msg_id)

nautobot_circuit_maintenance/tests/test_sources.py

@@ -10,6 +10,8 @@
 import exchangelib
 from django.conf import settings
 from django.test import TestCase
+from googleapiclient.errors import HttpError
+from httplib2 import Response
 from nautobot.circuits.models import Provider
 from parameterized import parameterized
 from pydantic import ValidationError
@@ -659,6 +661,22 @@ def test_get_search_criteria(
 
         self.assertEqual(result, source._get_search_criteria(since_timestamp))  # pylint: disable=protected-access
 
+    @patch("time.sleep", return_value=None)
+    def test_execute_retry_logic(self, mock_sleep):
+        """Test the googleapi execute retry logic."""
+        mock_request = MagicMock()
+
+        # Mock the execute method to raise an HttpError the first time and succeed the second time
+        mock_request.execute.side_effect = [
+            HttpError(resp=Response({"status": 503}), content=b"Service Unavailable"),  # First attempt fails
+            "Success!",  # Second attempt succeeds
+        ]
+
+        result = self.source._execute_with_retries(mock_request, self.job)  # pylint: disable=protected-access
+        self.assertEqual(result, "Success!")
+        self.assertEqual(mock_request.execute.call_count, 2)
+        mock_sleep.assert_called_once_with(1)  # Initial delay of 1 second
+
 
 class TestExchangeWebService(TestCase):
     """Test methods of the ExchangeWebService EmailSource class."""