From 40dfd8d87cd684864e2b2485d684ed3dea6c6213 Mon Sep 17 00:00:00 2001 From: Phil Snyder Date: Tue, 12 Nov 2024 16:49:27 -0800 Subject: [PATCH 1/4] Add additional expectations for fitbit data types --- .../resources/data_values_expectations.json | 235 +++++++++++++++++- 1 file changed, 233 insertions(+), 2 deletions(-) diff --git a/src/glue/resources/data_values_expectations.json b/src/glue/resources/data_values_expectations.json index 05473c0..bff132c 100644 --- a/src/glue/resources/data_values_expectations.json +++ b/src/glue/resources/data_values_expectations.json @@ -1,6 +1,18 @@ { + "fitbitactivitylogs": { + "expectations": [ + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "AverageHeartRate", + "min_value": 25, + "max_value": 300 + } + } + ], + "expectation_suite_name": "fitbitactivitylogs_expectations" + }, "fitbitdailydata": { - "expectation_suite_name": "fitbitdailydata_expectations", "expectations": [ { "expectation_type": "expect_column_values_to_be_between", @@ -25,8 +37,227 @@ "min_value": 4, "max_value": 40 } + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "MinutesFairlyActive", + "min_value": 0, + "max_value": 1440 + } + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "MinutesLightlyActive", + "min_value": 0, + "max_value": 1440 + } + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "MinutesSedentary", + "min_value": 0, + "max_value": 1440 + } + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "MinutesVeryActive", + "min_value": 0, + "max_value": 1440 + } + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "RestingHeartRate", + "min_value": 25, + "max_value": 300 + } + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "Hrv_DailyRmssd", + "min_value": 3, + "max_value": 210 + } + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "SpO2_Avg", + "min_value": 50, + "max_value": 100 + } + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "SpO2_Min", + "min_value": 50, + "max_value": 100 + } + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "SpO2_Max", + "min_value": 50, + "max_value": 100 + } + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "TempSkin", + "min_value": -15, + "max_value": 15 + } } - ] + ], + "expectation_suite_name": "fitbitdailydata_expectations" + }, + "fitbitintradaycombined": { + "expectations": [ + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "DeepSleepSummaryBreathRate", + "min_value": 4, + "max_value": 40 + } + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "RemSleepSummaryBreathRate", + "min_value": 4, + "max_value": 40 + } + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "FullSleepSummaryBreathRate", + "min_value": 4, + "max_value": 40 + } + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "LightSleepSummaryBreathRate", + "min_value": 4, + "max_value": 40 + } + } + ], + "expectation_suite_name": "fitbitintradaycombined_expectations" + }, + "fitbitsleeplogs": { + "expectations": [ + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "MinutesAfterWakeup", + "min_value": 0, + "max_value": 1440 + } + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "MinutesAsleep", + "min_value": 0, + "max_value": 1440 + } + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "MinutesAwake", + "min_value": 0, + "max_value": 1440 + } + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "MinutesToFallAsleep", + "min_value": 0, + "max_value": 1440 + } + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "TimeInBed", + "min_value": 0, + "max_value": 1440 + } + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "SleepLevelAwake", + "min_value": 0, + "max_value": 1440 + } + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "SleepLevelAsleep", + "min_value": 0, + "max_value": 1440 + } + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "SleepLevelDeep", + "min_value": 0, + "max_value": 1440 + } + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "SleepLevelLight", + "min_value": 0, + "max_value": 1440 + } + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "SleepLevelRem", + "min_value": 0, + "max_value": 1440 + } + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "SleepLevelRestless", + "min_value": 0, + "max_value": 1440 + } + }, + { + "expectation_type": "expect_column_values_to_be_between", + "kwargs": { + "column": "SleepLevelWake", + "min_value": 0, + "max_value": 1440 + } + } + ], + "expectation_suite_name": "fitbitsleeplogs_expectations" }, "healthkitv2workouts": { "expectation_suite_name": "healthkitv2workouts_expectations", From bcfef0ee63990bdc71bd0e68f980a17423d20b60 Mon Sep 17 00:00:00 2001 From: Phil Snyder Date: Thu, 14 Nov 2024 11:40:22 -0800 Subject: [PATCH 2/4] upgrade to v2 of ecr login action --- .github/workflows/upload-and-deploy.yaml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/upload-and-deploy.yaml b/.github/workflows/upload-and-deploy.yaml index ea8b877..cb3a8b9 100755 --- a/.github/workflows/upload-and-deploy.yaml +++ b/.github/workflows/upload-and-deploy.yaml @@ -193,14 +193,9 @@ jobs: - name: Login to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v1 + with: + mask-password: 'false' - - name: Get ECR secret names - id: ecr - run: | - usernameKey=docker_username_$(echo ${{ steps.login-ecr.outputs.registry }} | tr '.-' _) - echo "username-key=$usernameKey" >> $GITHUB_OUTPUT - passwordKey=docker_password_$(echo ${{ steps.login-ecr.outputs.registry }} | tr '.-' _) - echo "password-key=$passwordKey" >> $GITHUB_OUTPUT - uses: actions/checkout@v3 - name: Set up Docker Buildx @@ -215,10 +210,11 @@ jobs: file: ${{ matrix.dockerfile }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache + outputs: ecr-registry: ${{ steps.login-ecr.outputs.registry }} - ecr-username: ${{ steps.login-ecr.outputs[steps.ecr.outputs.username-key] }} - ecr-password: ${{ steps.login-ecr.outputs[steps.ecr.outputs.password-key] }} + ecr-username: ${{ steps.login-ecr.outputs['docker_username_914833433684_dkr_ecr_us_east_1_amazonaws_com'] }} + ecr-password: ${{ steps.login-ecr.outputs['docker_password_914833433684_dkr_ecr_us_east_1_amazonaws_com'] }} glue-unit-tests: name: Run Pytest unit tests for AWS glue From c724bdbdcc7de31875ee4e7b9062ad8d5b970058 Mon Sep 17 00:00:00 2001 From: Phil Snyder Date: Thu, 14 Nov 2024 16:27:15 -0800 Subject: [PATCH 3/4] Revert "upgrade to v2 of ecr login action" This reverts commit bcfef0ee63990bdc71bd0e68f980a17423d20b60. --- .github/workflows/upload-and-deploy.yaml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/upload-and-deploy.yaml b/.github/workflows/upload-and-deploy.yaml index cb3a8b9..ea8b877 100755 --- a/.github/workflows/upload-and-deploy.yaml +++ b/.github/workflows/upload-and-deploy.yaml @@ -193,9 +193,14 @@ jobs: - name: Login to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v1 - with: - mask-password: 'false' + - name: Get ECR secret names + id: ecr + run: | + usernameKey=docker_username_$(echo ${{ steps.login-ecr.outputs.registry }} | tr '.-' _) + echo "username-key=$usernameKey" >> $GITHUB_OUTPUT + passwordKey=docker_password_$(echo ${{ steps.login-ecr.outputs.registry }} | tr '.-' _) + echo "password-key=$passwordKey" >> $GITHUB_OUTPUT - uses: actions/checkout@v3 - name: Set up Docker Buildx @@ -210,11 +215,10 @@ jobs: file: ${{ matrix.dockerfile }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache - outputs: ecr-registry: ${{ steps.login-ecr.outputs.registry }} - ecr-username: ${{ steps.login-ecr.outputs['docker_username_914833433684_dkr_ecr_us_east_1_amazonaws_com'] }} - ecr-password: ${{ steps.login-ecr.outputs['docker_password_914833433684_dkr_ecr_us_east_1_amazonaws_com'] }} + ecr-username: ${{ steps.login-ecr.outputs[steps.ecr.outputs.username-key] }} + ecr-password: ${{ steps.login-ecr.outputs[steps.ecr.outputs.password-key] }} glue-unit-tests: name: Run Pytest unit tests for AWS glue From a311def334bdb84843968cfea1c73b81128cbd4e Mon Sep 17 00:00:00 2001 From: Phil Snyder Date: Thu, 14 Nov 2024 16:40:47 -0800 Subject: [PATCH 4/4] Fix remove artifacts command in cleanup workflow --- .github/workflows/cleanup.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cleanup.yaml b/.github/workflows/cleanup.yaml index 5f49620..d209af9 100644 --- a/.github/workflows/cleanup.yaml +++ b/.github/workflows/cleanup.yaml @@ -89,7 +89,7 @@ jobs: run: pipenv run sceptre --debug --var namespace=${{ github.event.ref }} delete develop/namespaced --yes - name: Remove artifacts - run: | + run: > pipenv run python src/scripts/manage_artifacts/artifacts.py --remove --namespace ${{ github.event.ref }}