From 324c2903ae04749fdd539c44b7c39e53454e140b Mon Sep 17 00:00:00 2001 From: Nathan Moore Date: Mon, 25 Mar 2024 14:36:31 -0400 Subject: [PATCH] fix (logging): Error handling Raw Completions with no sheet named Sheet1 (#182) * Error handling Raw Completions with no sheet named Sheet1 * Fixing linting issues * More lint * CR comments * CR Comments * Reverting setting change * Changing X to CS --- .../323012e80841_create_gdrive_table.py | 1 + .../b5c8e1cfcb42_create_participant_table.py | 2 ++ gdrive/database/database.py | 3 ++ gdrive/export_api.py | 29 +++++++++++---- gdrive/export_client.py | 1 + gdrive/main.py | 2 ++ gdrive/settings.py | 4 +++ gdrive/sheets_client.py | 35 +++++++++++-------- 8 files changed, 56 insertions(+), 21 deletions(-) diff --git a/alembic/versions/323012e80841_create_gdrive_table.py b/alembic/versions/323012e80841_create_gdrive_table.py index ecca318..d81c9fa 100644 --- a/alembic/versions/323012e80841_create_gdrive_table.py +++ b/alembic/versions/323012e80841_create_gdrive_table.py @@ -5,6 +5,7 @@ Create Date: 2023-10-25 22:19:12.231204 """ + from typing import Sequence, Union from alembic import op diff --git a/alembic/versions/b5c8e1cfcb42_create_participant_table.py b/alembic/versions/b5c8e1cfcb42_create_participant_table.py index 46eb40c..043d7b7 100644 --- a/alembic/versions/b5c8e1cfcb42_create_participant_table.py +++ b/alembic/versions/b5c8e1cfcb42_create_participant_table.py @@ -5,6 +5,7 @@ Create Date: 2023-10-25 22:22:37.857141 """ + from typing import Sequence, Union from alembic import op @@ -22,6 +23,7 @@ def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column("participant", sa.Column("survey_id", sa.String(), nullable=True)) op.add_column("participant", sa.Column("response_id", sa.String(), nullable=True)) + op.add_column( "participant", sa.Column("rules_consent_id", sa.String(), nullable=True) ) diff --git a/gdrive/database/database.py b/gdrive/database/database.py index eb93007..226ad54 100644 --- a/gdrive/database/database.py +++ b/gdrive/database/database.py @@ -1,6 +1,7 @@ """ DB Connection for Gdrive """ + import logging import sqlalchemy from sqlalchemy import orm @@ -17,6 +18,8 @@ settings.DB_URI, connect_args={"options": "-csearch_path=%s" % (settings.SCHEMA)}, ) + + else: log.info("No database configuration found. Creating in memory DB.") engine = sqlalchemy.create_engine("sqlite+pysqlite:///:memory:") diff --git a/gdrive/export_api.py b/gdrive/export_api.py index 6c9ddd3..85ad192 100644 --- a/gdrive/export_api.py +++ b/gdrive/export_api.py @@ -5,6 +5,7 @@ import io import json import logging +import sys import fastapi from pydantic import BaseModel, Field @@ -20,12 +21,14 @@ @router.post("/export") async def upload_file(interactionId): + log.info(f"Export interaction {interactionId}") export_data = export_client.export(interactionId) export_bytes = io.BytesIO( export_client.codename(json.dumps(export_data, indent=2)).encode() ) parent = drive_client.create_folder(interactionId, settings.ROOT_DIRECTORY) drive_client.upload_basic("analytics.json", parent, export_bytes) + log.info(f"Uploading {sys.getsizeof(export_bytes)} bytes to drive folder {parent}") class ParticipantModel(BaseModel): @@ -67,16 +70,17 @@ async def survey_upload_response_task(request): """ Background task that handles qualtrics response fetching and exporting """ + log.info(f"Gathering response {request.responseId}") try: response = export_client.get_qualtrics_response( request.surveyId, request.responseId ) - log.info("Response found, beginning export.") + log.info(f"{request.responseId} response found, beginning export.") if response["status"] != "Complete": - raise error.ExportError( - f"Cannot upload incomplete survery response to raw completions spreadsheet: {request.responseId}" + log.warn( + f"Incomplete survery response to raw completions spreadsheet: {request.responseId}" ) # By the time we get here, we can count on the response containing the demographic data @@ -86,7 +90,7 @@ async def survey_upload_response_task(request): if request.participant: participant = request.participant - sheets_client.upload_participant( + upload_result = sheets_client.upload_participant( participant.first, participant.last, participant.email, @@ -103,6 +107,12 @@ async def survey_upload_response_task(request): survey_resp["skin_tone"], ) + result_sheet_id = upload_result["spreadsheetId"] + if upload_result: + log.info( + f"Uploaded response: {request.responseId} to completions spreadsheet {result_sheet_id}" + ) + crud.create_participant( models.ParticipantModel( survey_id=request.surveyId, @@ -120,17 +130,22 @@ async def survey_upload_response_task(request): skin_tone=survey_resp["skin_tone"], ) ) + log.info(f"Wrote {request.responseId} to database") # call function that queries ES for all analytics entries (flow interactionId) with responseId interactionIds = export_client.export_response(request.responseId, response) - - log.info("Analytics updated, beginning gdrive export.") + log.info( + f"Elastic Search returned {len(interactionIds)} interaction ids for response: {request.responseId}" + ) # export list of interactionIds to gdrive for id in interactionIds: await upload_file(id) + log.info( + f"Exported response: {request.responseId} interaction: {id} to gdrive" + ) except error.ExportError as e: - log.error(e.args) + log.error(f"Response: {request.responseId} encountered an error: {e.args}") class FindModel(BaseModel): diff --git a/gdrive/export_client.py b/gdrive/export_client.py index 79fbaec..03df509 100644 --- a/gdrive/export_client.py +++ b/gdrive/export_client.py @@ -165,6 +165,7 @@ def get_qualtrics_response(surveyId: str, responseId: str): json={"surveyId": surveyId, "responseId": responseId}, timeout=30, # qualtrics microservice retries as it waits for response to become available ) + if r.status_code != 200: raise error.ExportError( f"No survey response found for responseId: {responseId}" diff --git a/gdrive/main.py b/gdrive/main.py index 0a438c7..b116022 100644 --- a/gdrive/main.py +++ b/gdrive/main.py @@ -1,9 +1,11 @@ """ GDrive Microservice FastAPI Web App. """ + import fastapi import starlette_prometheus + from . import api, export_api, analytics_api, settings app = fastapi.FastAPI() diff --git a/gdrive/settings.py b/gdrive/settings.py index 10271ad..e45d42e 100644 --- a/gdrive/settings.py +++ b/gdrive/settings.py @@ -2,6 +2,7 @@ Configuration for the gdrive microservice settings. Context is switched based on if the app is in debug mode. """ + import json import logging import os @@ -36,6 +37,8 @@ QUALTRICS_APP_URL = os.getenv("QUALTRICS_APP_URL") QUALTRICS_APP_PORT = os.getenv("QUALTRICS_APP_PORT") +RAW_COMPLETIONS_SHEET_NAME = os.getenv("GDRIVE_RAW_COMPLETIONS_SHEET_NAME", "Sheet1") + DB_URI = os.getenv("IDVA_DB_CONN_STR") SCHEMA = "idva" @@ -52,6 +55,7 @@ if service["name"] == "gdrive": log.info("Loading credentials from env var") config = service["credentials"] + break else: with open(SERVICE_ACCOUNT_FILE) as file: diff --git a/gdrive/sheets_client.py b/gdrive/sheets_client.py index 72fa2f8..8475a9b 100644 --- a/gdrive/sheets_client.py +++ b/gdrive/sheets_client.py @@ -4,6 +4,7 @@ from google.oauth2 import service_account from googleapiclient.discovery import build +from googleapiclient.errors import HttpError from gdrive import settings, error @@ -119,7 +120,7 @@ def add_pivot_tables( def add_new_pages( - page_names: [str], sheets_id: str, row_count: int = 1000, column_count: int = 26 + page_names: List[str], sheets_id: str, row_count: int = 1000, column_count: int = 26 ): new_sheets_reqs = [] for label in page_names: @@ -221,17 +222,23 @@ def upload_participant( ] body = {"values": values} - result = ( - sheets_service.spreadsheets() - .values() - .append( - spreadsheetId=settings.SHEETS_ID, - range="Sheet1!A1", - valueInputOption="RAW", - body=body, + + try: + result = ( + sheets_service.spreadsheets() + .values() + .append( + spreadsheetId=settings.SHEETS_ID, + range=f"{settings.RAW_COMPLETIONS_SHEET_NAME}!A1", + valueInputOption="RAW", + body=body, + ) + .execute() ) - .execute() - ) - if "error" in result: - raise error.ExportError(result["error"]["message"]) - return result + + if "error" in result: + raise error.ExportError(result["error"]["message"]) + + return result + except HttpError as e: + raise error.ExportError(e)