Merge pull request #30 from seattleflu/redcap-etl-updates

REDCap DET etl updates

lib/seattleflu/id3c/cli/command/etl/redcap_det_kiosk.py

@@ -38,7 +38,7 @@
 # REDCap DET records lacking this revision number in their log.  If a
 # change to the ETL routine necessitates re-processing all REDCap DET records,
 # this revision number should be incremented.
-REVISION = 1
+REVISION = 2
 
 
 @redcap_det.command_for_project(
@@ -165,6 +165,9 @@ def participant_zipcode(redcap_record: dict) -> str:
     """
     Extract the home zipcode for the participant from the given
     *redcap_record*.
+
+    If no zipcode could be found for the participant, then it returns
+    «{PROJECT_ID}-{record_id}»
     """
     if redcap_record.get('home_zipcode'):
         return redcap_record['home_zipcode']
@@ -180,8 +183,8 @@ def participant_zipcode(redcap_record: dict) -> str:
         address = determine_dorm_address(redcap_record['uw_dorm'])
         return address['zipcode']
 
-    else:
-        return None
+    LOG.warning(f"Could not extract zipcode from redcap record, using 'project_id-record_id' «{PROJECT_ID}-{redcap_record['record_id']}» instead")
+    return str(PROJECT_ID) + '-' + redcap_record['record_id']
 
 
 def determine_vaccine_date(vaccine_year: str, vaccine_month: str) -> Optional[str]:
@@ -806,6 +809,9 @@ def determine_all_questionnaire_items(redcap_record: dict) -> List[dict]:
         items['age'] = [{ 'valueInteger': age_ceiling(int(redcap_record['age'])) }]
         items['age_months'] = [{ 'valueInteger': int(age_ceiling(float(redcap_record['age_months']) / 12) * 12) }]
 
+    if redcap_record['acute_symptom_onset']:
+        items['acute_symptom_onset'] = [{ 'valueString': redcap_record['acute_symptom_onset']}]
+
     # Participant can select multiple insurance types, so create
     # a separate answer for each selection
     insurance_responses = find_selected_options('insurance___', redcap_record)

lib/seattleflu/id3c/cli/command/etl/redcap_det_swab_n_send.py

@@ -25,7 +25,7 @@
 LOG = logging.getLogger(__name__)
 
 
-REVISION = 1
+REVISION = 2
 
 REDCAP_URL = 'https://redcap.iths.org/'
 INTERNAL_SYSTEM = "https://seattleflu.org"
@@ -52,7 +52,17 @@
 def redcap_det_swab_n_send(*, db: DatabaseSession, cache: TTLCache, det: dict, redcap_record: dict) -> Optional[dict]:
     location_resource_entries = locations(db, cache, redcap_record)
     patient_entry, patient_reference = create_patient(redcap_record)
+
+    if not patient_entry:
+        LOG.warning("Skipping enrollment with insufficient information to construct patient")
+        return None
+
     encounter_entry, encounter_reference = create_encounter(redcap_record, patient_reference, location_resource_entries)
+
+    if not encounter_entry:
+        LOG.warning("Skipping enrollment with insufficient information to construct an encounter")
+        return None
+
     questionnaire_entry = create_questionnaire_response(redcap_record, patient_reference, encounter_reference)
     specimen_entry, specimen_reference = create_specimen(redcap_record, patient_reference)
 
@@ -194,6 +204,9 @@ def create_patient(record: dict) -> tuple:
         birth_date  = record['birthday'],
         postal_code = record['home_zipcode_2'])
 
+    if not patient_id:
+        return None, None
+
     patient_identifier = create_identifier(f"{INTERNAL_SYSTEM}/individual", patient_id)
     patient_resource = create_patient_resource([patient_identifier], gender)
 
@@ -352,18 +365,11 @@ def specimen_barcode(record: Any) -> str:
         value = barcode
     )
 
-    if not record.get('collection_date'):
-        LOG.warning("Could not create Specimen Resource due to lack of collection date.")
-        return None, None
-
     # YYYY-MM-DD in REDCap
-    collected_time = record['collection_date']
+    collected_time = record['collection_date'] or None
 
     # YYYY-MM-DD HH:MM:SS in REDCap
-    received_time = record['samp_process_date'].split()[0]
-    if not received_time:
-        LOG.warning("No sample process date found. Using collection date instead.")
-        received_time = collected_time
+    received_time = record['samp_process_date'].split()[0] if record['samp_process_date'] else None
 
     specimen_type = 'NSECR'  # Nasal swab.  TODO we may want shared mapping function
     specimen_resource = create_specimen_resource(