Error handling Raw Completions with no sheet named Sheet1

gdrive/export_api.py

@@ -5,6 +5,7 @@
 import io
 import json
 import logging
+import sys
 
 import fastapi
 from pydantic import BaseModel, Field
@@ -20,12 +21,14 @@
 
 @router.post("/export")
 async def upload_file(interactionId):
+    log.info(f"Export interaction {interactionId}")
     export_data = export_client.export(interactionId)
     export_bytes = io.BytesIO(
         export_client.codename(json.dumps(export_data, indent=2)).encode()
     )
     parent = drive_client.create_folder(interactionId, settings.ROOT_DIRECTORY)
     drive_client.upload_basic("analytics.json", parent, export_bytes)
+    log.info(f"Uploading {sys.getsizeof(export_bytes)} bytes to drive folder {parent}")
 
 
 class ParticipantModel(BaseModel):
@@ -67,17 +70,19 @@ async def survey_upload_response_task(request):
     """
     Background task that handles qualtrics response fetching and exporting
     """
+    log.info(f"Gathering response {request.responseId}")
     try:
         response = export_client.get_qualtrics_response(
             request.surveyId, request.responseId
         )
 
-        log.info("Response found, beginning export.")
+        log.info(f"{request.responseId} response found, beginning export.")
 
-        if response["status"] != "Complete":
-            raise error.ExportError(
-                f"Cannot upload incomplete survery response to raw completions spreadsheet: {request.responseId}"
-            )
+        # DELETE?
+        # if response["status"] != "Complete":
+        #     raise error.ExportError(
+        #         f"Cannot upload incomplete survery response to raw completions spreadsheet: {request.responseId}"
+        #     )
 
         # By the time we get here, we can count on the response containing the demographic data
         # as it is included in the Completed flow responses. Responses without complete status
@@ -86,7 +91,7 @@ async def survey_upload_response_task(request):
 
         if request.participant:
             participant = request.participant
-            sheets_client.upload_participant(
+            upload_result = sheets_client.upload_participant(
                 participant.first,
                 participant.last,
                 participant.email,
@@ -103,6 +108,9 @@ async def survey_upload_response_task(request):
                 survey_resp["skin_tone"],
             )
 
+            if upload_result:
+                log.info(f"Uploaded {request.responseId} to completions spreadsheet")
+
             crud.create_participant(
                 models.ParticipantModel(
                     survey_id=request.surveyId,
@@ -120,15 +128,18 @@ async def survey_upload_response_task(request):
                     skin_tone=survey_resp["skin_tone"],
                 )
             )
+            log.info(f"Wrote {request.responseId} to database")
 
         # call function that queries ES for all analytics entries (flow interactionId) with responseId
         interactionIds = export_client.export_response(request.responseId, response)
-
-        log.info("Analytics updated, beginning gdrive export.")
+        log.info(
+            f"Elastic Search returned {len(interactionIds)} interaction ids for this response"
+        )
 
         # export list of interactionIds to gdrive
         for id in interactionIds:
             await upload_file(id)
+            log.info(f"Exported {id} to gdrive")
     except error.ExportError as e:
         log.error(e.args)
 

gdrive/export_client.py

@@ -165,6 +165,7 @@ def get_qualtrics_response(surveyId: str, responseId: str):
         json={"surveyId": surveyId, "responseId": responseId},
         timeout=30,  # qualtrics microservice retries as it waits for response to become available
     )
+
     if r.status_code != 200:
         raise error.ExportError(
             f"No survey response found for responseId: {responseId}"

gdrive/settings.py

@@ -33,8 +33,8 @@
 ES_HOST = os.getenv("ES_HOST")
 ES_PORT = os.getenv("ES_PORT")
 
-QUALTRICS_APP_URL = os.getenv("QUALTRICS_APP_URL")
-QUALTRICS_APP_PORT = os.getenv("QUALTRICS_APP_PORT")
+QUALTRICS_APP_URL = os.getenv("QUALTRIX_APP_HOST")
+QUALTRICS_APP_PORT = os.getenv("QUALTRIX_APP_PORT")
 
 DB_URI = os.getenv("IDVA_DB_CONN_STR")
 SCHEMA = "idva"

gdrive/sheets_client.py

@@ -4,6 +4,7 @@
 
 from google.oauth2 import service_account
 from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
 
 from gdrive import settings, error
 
@@ -119,7 +120,7 @@ def add_pivot_tables(
 
 
 def add_new_pages(
-    page_names: [str], sheets_id: str, row_count: int = 1000, column_count: int = 26
+    page_names: List[str], sheets_id: str, row_count: int = 1000, column_count: int = 26
 ):
     new_sheets_reqs = []
     for label in page_names:
@@ -221,17 +222,23 @@ def upload_participant(
     ]
 
     body = {"values": values}
-    result = (
-        sheets_service.spreadsheets()
-        .values()
-        .append(
-            spreadsheetId=settings.SHEETS_ID,
-            range="Sheet1!A1",
-            valueInputOption="RAW",
-            body=body,
+
+    try:
+        result = (
+            sheets_service.spreadsheets()
+            .values()
+            .append(
+                spreadsheetId=settings.SHEETS_ID,
+                range="Sheet1!A1",
+                valueInputOption="RAW",
+                body=body,
+            )
+            .execute()
         )
-        .execute()
-    )
-    if "error" in result:
-        raise error.ExportError(result["error"]["message"])
-    return result
+
+        if "error" in result:
+            raise error.ExportError(result["error"]["message"])
+
+        return result
+    except HttpError as e:
+        raise error.ExportError(e)