From 186d5b1f584f88e468eef644dfcf0286f1187774 Mon Sep 17 00:00:00 2001 From: nathan-moore-97 Date: Tue, 5 Mar 2024 14:42:59 -0500 Subject: [PATCH] Error handling Raw Completions with no sheet named Sheet1 --- gdrive/export_api.py | 27 +++++++++++++++++++-------- gdrive/export_client.py | 1 + gdrive/settings.py | 4 ++-- gdrive/sheets_client.py | 35 +++++++++++++++++++++-------------- 4 files changed, 43 insertions(+), 24 deletions(-) diff --git a/gdrive/export_api.py b/gdrive/export_api.py index 6c9ddd3..9096f4f 100644 --- a/gdrive/export_api.py +++ b/gdrive/export_api.py @@ -5,6 +5,7 @@ import io import json import logging +import sys import fastapi from pydantic import BaseModel, Field @@ -20,12 +21,14 @@ @router.post("/export") async def upload_file(interactionId): + log.info(f"Export interaction {interactionId}") export_data = export_client.export(interactionId) export_bytes = io.BytesIO( export_client.codename(json.dumps(export_data, indent=2)).encode() ) parent = drive_client.create_folder(interactionId, settings.ROOT_DIRECTORY) drive_client.upload_basic("analytics.json", parent, export_bytes) + log.info(f"Uploading {sys.getsizeof(export_bytes)} bytes to drive folder {parent}") class ParticipantModel(BaseModel): @@ -67,17 +70,19 @@ async def survey_upload_response_task(request): """ Background task that handles qualtrics response fetching and exporting """ + log.info(f"Gathering response {request.responseId}") try: response = export_client.get_qualtrics_response( request.surveyId, request.responseId ) - log.info("Response found, beginning export.") + log.info(f"{request.responseId} response found, beginning export.") - if response["status"] != "Complete": - raise error.ExportError( - f"Cannot upload incomplete survery response to raw completions spreadsheet: {request.responseId}" - ) + # DELETE? + # if response["status"] != "Complete": + # raise error.ExportError( + # f"Cannot upload incomplete survery response to raw completions spreadsheet: {request.responseId}" + # ) # By the time we get here, we can count on the response containing the demographic data # as it is included in the Completed flow responses. Responses without complete status @@ -86,7 +91,7 @@ async def survey_upload_response_task(request): if request.participant: participant = request.participant - sheets_client.upload_participant( + upload_result = sheets_client.upload_participant( participant.first, participant.last, participant.email, @@ -103,6 +108,9 @@ async def survey_upload_response_task(request): survey_resp["skin_tone"], ) + if upload_result: + log.info(f"Uploaded {request.responseId} to completions spreadsheet") + crud.create_participant( models.ParticipantModel( survey_id=request.surveyId, @@ -120,15 +128,18 @@ async def survey_upload_response_task(request): skin_tone=survey_resp["skin_tone"], ) ) + log.info(f"Wrote {request.responseId} to database") # call function that queries ES for all analytics entries (flow interactionId) with responseId interactionIds = export_client.export_response(request.responseId, response) - - log.info("Analytics updated, beginning gdrive export.") + log.info( + f"Elastic Search returned {len(interactionIds)} interaction ids for this response" + ) # export list of interactionIds to gdrive for id in interactionIds: await upload_file(id) + log.info(f"Exported {id} to gdrive") except error.ExportError as e: log.error(e.args) diff --git a/gdrive/export_client.py b/gdrive/export_client.py index 79fbaec..03df509 100644 --- a/gdrive/export_client.py +++ b/gdrive/export_client.py @@ -165,6 +165,7 @@ def get_qualtrics_response(surveyId: str, responseId: str): json={"surveyId": surveyId, "responseId": responseId}, timeout=30, # qualtrics microservice retries as it waits for response to become available ) + if r.status_code != 200: raise error.ExportError( f"No survey response found for responseId: {responseId}" diff --git a/gdrive/settings.py b/gdrive/settings.py index 10271ad..e67678c 100644 --- a/gdrive/settings.py +++ b/gdrive/settings.py @@ -33,8 +33,8 @@ ES_HOST = os.getenv("ES_HOST") ES_PORT = os.getenv("ES_PORT") -QUALTRICS_APP_URL = os.getenv("QUALTRICS_APP_URL") -QUALTRICS_APP_PORT = os.getenv("QUALTRICS_APP_PORT") +QUALTRICS_APP_URL = os.getenv("QUALTRIX_APP_HOST") +QUALTRICS_APP_PORT = os.getenv("QUALTRIX_APP_PORT") DB_URI = os.getenv("IDVA_DB_CONN_STR") SCHEMA = "idva" diff --git a/gdrive/sheets_client.py b/gdrive/sheets_client.py index 72fa2f8..41dcbfd 100644 --- a/gdrive/sheets_client.py +++ b/gdrive/sheets_client.py @@ -4,6 +4,7 @@ from google.oauth2 import service_account from googleapiclient.discovery import build +from googleapiclient.errors import HttpError from gdrive import settings, error @@ -119,7 +120,7 @@ def add_pivot_tables( def add_new_pages( - page_names: [str], sheets_id: str, row_count: int = 1000, column_count: int = 26 + page_names: List[str], sheets_id: str, row_count: int = 1000, column_count: int = 26 ): new_sheets_reqs = [] for label in page_names: @@ -221,17 +222,23 @@ def upload_participant( ] body = {"values": values} - result = ( - sheets_service.spreadsheets() - .values() - .append( - spreadsheetId=settings.SHEETS_ID, - range="Sheet1!A1", - valueInputOption="RAW", - body=body, + + try: + result = ( + sheets_service.spreadsheets() + .values() + .append( + spreadsheetId=settings.SHEETS_ID, + range="Sheet1!A1", + valueInputOption="RAW", + body=body, + ) + .execute() ) - .execute() - ) - if "error" in result: - raise error.ExportError(result["error"]["message"]) - return result + + if "error" in result: + raise error.ExportError(result["error"]["message"]) + + return result + except HttpError as e: + raise error.ExportError(e)