[ETL-655] Add Device field to fitbitdevices INDEX_FIELD_MAP (#117)

* Add `Device` field to fitbitdevices INDEX_FIELD_MAP

* Specify the number of integration test exports as env variable

.github/workflows/upload-and-deploy.yaml

@@ -11,6 +11,7 @@ env:
   DEV_INPUT_BUCKET: recover-dev-input-data
   DEV_PROCESSED_BUCKET: recover-dev-processed-data
   PROD_INPUT_BUCKET: recover-input-data
+  INTEGRATION_TEST_NUM_EXPORTS: 28
 
 jobs:
 
@@ -287,7 +288,7 @@ jobs:
             aws s3api list-objects-v2 \
             --bucket $DEV_INPUT_BUCKET \
             --prefix $EXPORT_S3_KEY_PREFIX \
-            --query '((sort_by(Contents[? !ends_with(Key, `/`) && !contains(Key, `owner.txt`)], &LastModified)[::-1])[:28])[*].Key' |
+            --query '((sort_by(Contents[? !ends_with(Key, `/`) && !contains(Key, `owner.txt`)], &LastModified)[::-1])[:${{ env.INTEGRATION_TEST_NUM_EXPORTS }}])[*].Key' |
             jq -c
           )" >> "$GITHUB_OUTPUT"
 
@@ -363,7 +364,7 @@ jobs:
             aws s3api list-objects-v2 \
             --bucket $PROD_INPUT_BUCKET \
             --prefix "$EXPORT_S3_KEY_PREFIX/" \
-            --query '((sort_by(Contents[? !ends_with(Key, `/`) && !contains(Key, `owner.txt`)], &LastModified)[::-1])[:28])[*].Key' |
+            --query '((sort_by(Contents[? !ends_with(Key, `/`) && !contains(Key, `owner.txt`)], &LastModified)[::-1])[:${{ env.INTEGRATION_TEST_NUM_EXPORTS }}])[*].Key' |
             jq -c
           )" >> "$GITHUB_OUTPUT"
 

src/glue/jobs/json_to_parquet.py

@@ -39,7 +39,7 @@
 INDEX_FIELD_MAP = {
     "enrolledparticipants": ["ParticipantIdentifier"],
     "fitbitprofiles": ["ParticipantIdentifier", "ModifiedDate"],
-    "fitbitdevices": ["ParticipantIdentifier", "Date"],
+    "fitbitdevices": ["ParticipantIdentifier", "Date", "Device"],
     "fitbitactivitylogs": ["ParticipantIdentifier", "LogId"],
     "fitbitdailydata": ["ParticipantIdentifier", "Date"],
     "fitbitecg": ["ParticipantIdentifier", "FitbitEcgKey"],