upload-and-deploy.yaml

name: upload-and-deploy

on:
  push:
    branches: "*"
    tags-ignore: "*"

env:
  NAMESPACE: main
  PYTHON_VERSION: 3.9
  DEV_INPUT_BUCKET: recover-dev-input-data
  DEV_RAW_BUCKET: recover-dev-raw-data
  DEV_INTERMEDIATE_BUCKET: recover-dev-intermediate-data
  DEV_PROCESSED_BUCKET: recover-dev-processed-data
  PROD_INPUT_BUCKET: recover-input-data
  PROD_RAW_BUCKET: recover-raw-data
  PROD_INTERMEDIATE_BUCKET: recover-intermediate-data
  INTEGRATION_TEST_NUM_EXPORTS: 28
  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true

jobs:

  pre-commit:
    name: Run pre-commit hooks against all files
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
      - uses: pre-commit/action@v3.0.0

  deploy-snowflake:
    name: Deploy Snowflake resources
    needs: pre-commit
    runs-on: ubuntu-latest
    env:
      PRIVATE_KEY_PASSPHRASE: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_PASSPHRASE }}
      SNOWFLAKE_ENVIRONMENT: ${{ github.ref_name == 'main' && 'staging' || github.ref_name }}
    steps:
      - uses: actions/checkout@v3

      - name: Configure Snowflake connection
        run: |
          # Create temporary files for config.toml and our private key
          config_file=$(mktemp)
          private_key_file=$(mktemp)

          # Write to the private key file
          echo "${{ secrets.SNOWFLAKE_PRIVATE_KEY }}" > $private_key_file

          # Write to config.toml file
          echo 'default_connection_name = "recover"' >> $config_file
          echo '[connections.recover]' >> $config_file
          echo "account = \"${{ vars.SNOWFLAKE_ACCOUNT }}\"" >> $config_file
          echo "user = \"${{ vars.SNOWFLAKE_USER }}\"" >> $config_file
          echo "role = \"${{ vars.SNOWFLAKE_ROLE }}\"" >> $config_file
          echo 'warehouse = "RECOVER_XSMALL"' >> $config_file
          echo 'authenticator = "SNOWFLAKE_JWT"' >> $config_file
          echo "private_key_path = \"$private_key_file\"" >> $config_file

          # Write config.toml path to global environment
          echo "SNOWFLAKE_CONFIG_PATH=$config_file" >> $GITHUB_ENV

      - name: Configuration file information
        run: |
          echo "Snowflake configuration is located at $SNOWFLAKE_CONFIG_PATH"
          cat $SNOWFLAKE_CONFIG_PATH

      - name: Install Snowflake CLI
        uses: Snowflake-Labs/snowflake-cli-action@v1.5
        with:
          default-config-file-path: ${{ env.SNOWFLAKE_CONFIG_PATH }}

      - name: Test Snowflake connection
        run: |
          snow --version
          snow connection test

      - name: Deploy Snowflake objects
        run: |
          snow sql \
            -D "git_branch=$GITHUB_REF_NAME" \
            -D "environment=$SNOWFLAKE_ENVIRONMENT" \
            -f snowflake/objects/deploy.sql

  upload-files:
    name: Upload files to S3 bucket in development
    runs-on: ubuntu-latest
    needs: pre-commit
    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
    permissions:
      id-token: write
      contents: read
    environment: develop
    steps:

      - name: Setup code, pipenv, aws
        uses: Sage-Bionetworks/action-pipenv-aws-setup@v3
        with:
          role_to_assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }}
          role_session_name: GitHubActions-${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}
          python_version: ${{ env.PYTHON_VERSION }}

      - name: Setup sam
        uses: aws-actions/setup-sam@v2

      - name: Set namespace for non-default branch
        if: github.ref_name != 'main'
        run: echo "NAMESPACE=$GITHUB_REF_NAME" >> $GITHUB_ENV

      - name: Copy files to templates bucket, use dev cloudformation bucket
        run: >
          python src/scripts/manage_artifacts/artifacts.py
          --upload
          --namespace $NAMESPACE
          --cfn_bucket ${{ vars.CFN_BUCKET }}

  nonglue-unit-tests:
    name: Runs unit tests that are not dependent on aws-glue package resources
    runs-on: ubuntu-latest
    needs: pre-commit
    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
    permissions:
      id-token: write
      contents: read
    environment: develop
    steps:
      - name: Setup code, pipenv, aws
        uses: Sage-Bionetworks/action-pipenv-aws-setup@v3
        with:
          role_to_assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }}
          role_session_name: ${{ github.event.repository.name }}-${{ github.run_id }}-nonglue-unit-tests
          python_version: ${{ env.PYTHON_VERSION }}

      - name: Install additional python dependency
        run: |
          pipenv install ecs_logging~=2.0
          pipenv install pytest-datadir

      - name: Test scripts with pytest (lambda, etc.)
        run: |
          pipenv run python -m pytest -v \
            tests/test_s3_event_config_lambda.py \
            tests/test_s3_to_glue_lambda.py \
            tests/test_lambda_dispatch.py \
            tests/test_consume_logs.py \
            tests/test_lambda_raw.py \
            tests/test_lambda_raw_sync.py

      - name: Test dev synapse folders for STS access with pytest
        run: >
          pipenv run python -m pytest tests/test_setup_external_storage.py
          --test-bucket $DEV_INPUT_BUCKET
          --test-synapse-folder-id syn51758510
          --namespace $NAMESPACE
          --test-sts-permission read_only
          -v

          pipenv run python -m pytest tests/test_setup_external_storage.py
          --test-bucket $DEV_PROCESSED_BUCKET
          --test-synapse-folder-id syn51084525
          --namespace $NAMESPACE/parquet
          --test-sts-permission read_write
          -v

  pytest-docker:
    name: Build and push testing docker images to the pytest ECR repository.
    needs: pre-commit
    runs-on: ubuntu-latest
    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix:
        include:
          - tag_name: aws_glue_3
            dockerfile: tests/Dockerfile.aws_glue_3
          - tag_name: aws_glue_4
            dockerfile: tests/Dockerfile.aws_glue_4
    environment: develop
    steps:
      - name: Assume AWS role
        uses: aws-actions/configure-aws-credentials@v2
        with:
          role-to-assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }}
          aws-region: "us-east-1"
          # unmasking of the AWS account ID allows the acct id to pass through outputs
          mask-aws-account-id: "no"

      - name: Login to Amazon ECR
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@v1

      - name: Get ECR secret names
        id: ecr
        run: |
          usernameKey=docker_username_$(echo ${{ steps.login-ecr.outputs.registry }} | tr '.-' _)
          echo "username-key=$usernameKey" >> $GITHUB_OUTPUT
          passwordKey=docker_password_$(echo ${{ steps.login-ecr.outputs.registry }} | tr '.-' _)
          echo "password-key=$passwordKey" >> $GITHUB_OUTPUT
      - uses: actions/checkout@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Build and push to ECR
        id: docker-build-push
        uses: docker/build-push-action@v4
        with:
          push: true
          tags: ${{ steps.login-ecr.outputs.registry }}/pytest:${{ github.ref_name }}_${{ matrix.tag_name }}
          file: ${{ matrix.dockerfile }}
          cache-from: type=local,src=/tmp/.buildx-cache
          cache-to: type=local,dest=/tmp/.buildx-cache
    outputs:
      ecr-registry: ${{ steps.login-ecr.outputs.registry }}
      ecr-username: ${{ steps.login-ecr.outputs[steps.ecr.outputs.username-key] }}
      ecr-password: ${{ steps.login-ecr.outputs[steps.ecr.outputs.password-key] }}

  glue-unit-tests:
    name: Run Pytest unit tests for AWS glue
    needs: pytest-docker
    environment: develop
    runs-on: ubuntu-latest
    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix:
          tag_name:
            ["aws_glue_3", "aws_glue_4"]
    container:
      image: ${{ needs.pytest-docker.outputs.ecr-registry }}/pytest:${{ github.ref_name }}_${{ matrix.tag_name }}
      credentials:
        username: ${{ needs.pytest-docker.outputs.ecr-username }}
        password: ${{ needs.pytest-docker.outputs.ecr-password }}
      env:
        DISABLE_SSL: true
      options: "--user root"
    steps:
      - name: Assume AWS role
        uses: aws-actions/configure-aws-credentials@v2
        with:
          role-to-assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }}
          aws-region: "us-east-1"
      - uses: actions/checkout@v3
      - run: chown -R glue_user $GITHUB_WORKSPACE
      - run: su - glue_user --command "aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID"
      - run: su - glue_user --command "aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY"
      - run: su - glue_user --command "aws configure set aws_session_token $AWS_SESSION_TOKEN"
      - run: su - glue_user --command "aws configure set region $AWS_REGION"

      - name: Set namespace for non-default branch or for tag
        if: github.ref_name != 'main'
        run: echo "NAMESPACE=$GITHUB_REF_NAME" >> $GITHUB_ENV

      - name: Run Pytest unit tests under AWS Glue 3.0
        if: matrix.tag_name == 'aws_glue_3'
        run: |
          su - glue_user --command "cd $GITHUB_WORKSPACE && python3 -m pytest \
            tests/test_s3_to_json.py \
            tests/test_compare_parquet_datasets.py -v"

      - name: Run unit tests for JSON to Parquet under AWS Glue 4.0
        if: matrix.tag_name == 'aws_glue_4'
        run: >
          su - glue_user --command "cd $GITHUB_WORKSPACE &&
          python3 -m pytest tests/test_json_to_parquet.py --namespace $NAMESPACE -v"

      - name: Run unit tests for Great Expectations on Parquet under AWS Glue 4.0
        if: matrix.tag_name == 'aws_glue_4'
        run: >
          su - glue_user --command "cd $GITHUB_WORKSPACE &&
          python3 -m pytest tests/test_run_great_expectations_on_parquet.py -v"

  sceptre-deploy-develop:
    name: Deploys branch using sceptre
    runs-on: ubuntu-latest
    needs: [upload-files, nonglue-unit-tests, glue-unit-tests]
    environment: develop
    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
    permissions:
      id-token: write
      contents: read

    steps:
      - name: Setup code, pipenv, aws
        uses: Sage-Bionetworks/action-pipenv-aws-setup@v3
        with:
          role_to_assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }}
          role_session_name: GitHubActions-${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}
          python_version: ${{ env.PYTHON_VERSION }}

      - name: Create directory for remote sceptre templates
        run: mkdir -p templates/remote/

      - name: Set namespace for non-default branch
        if: github.ref_name != 'main'
        run: echo "NAMESPACE=$GITHUB_REF_NAME" >> $GITHUB_ENV

      - name: "Deploy sceptre stacks to dev"
        run: pipenv run sceptre --debug --var "namespace=${{ env.NAMESPACE }}" launch develop --yes

      - name: Delete preexisting S3 event notification for this namespace
        uses: gagoar/invoke-aws-lambda@v3
        with:
          AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
          AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }}
          REGION: ${{ env.AWS_REGION }}
          FunctionName: ${{ env.NAMESPACE }}-S3EventConfig
          Payload: '{"RequestType": "Delete"}'
          LogType: Tail

      - name: Create S3 event notification for this namespace
        uses: gagoar/invoke-aws-lambda@v3
        with:
          AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
          AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }}
          REGION: ${{ env.AWS_REGION }}
          FunctionName: ${{ env.NAMESPACE }}-S3EventConfig
          Payload: '{"RequestType": "Create"}'
          LogType: Tail

  integration-test-develop-cleanup:
    name: Cleanup non-main branch data before integration tests
    runs-on: ubuntu-latest
    needs: sceptre-deploy-develop
    environment: develop
    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
    permissions:
      id-token: write
      contents: read
    steps:
      - name: Setup code, pipenv, aws
        uses: Sage-Bionetworks/action-pipenv-aws-setup@v3
        with:
          role_to_assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }}
          role_session_name: GitHubActions-${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}
          python_version: ${{ env.PYTHON_VERSION }}

      - name: Set namespace for non-default branch
        if: github.ref_name != 'main'
        run: echo "NAMESPACE=$GITHUB_REF_NAME" >> $GITHUB_ENV

      - name: Clean input data bucket
        run: >
          pipenv run python src/scripts/manage_artifacts/clean_for_integration_test.py
          --bucket $DEV_INPUT_BUCKET
          --bucket_prefix "${{ env.NAMESPACE }}/"

      - name: Clean raw data bucket
        run: >
          pipenv run python src/scripts/manage_artifacts/clean_for_integration_test.py
          --bucket $DEV_RAW_BUCKET
          --bucket_prefix "${{ env.NAMESPACE }}/json/"

      - name: Clean intermediate data bucket
        run: >
          pipenv run python src/scripts/manage_artifacts/clean_for_integration_test.py
          --bucket $DEV_INTERMEDIATE_BUCKET
          --bucket_prefix "${{ env.NAMESPACE }}/json/"

  integration-test-develop:
    name: Triggers ETL workflow with S3 test files
    runs-on: ubuntu-latest
    needs: [sceptre-deploy-develop, integration-test-develop-cleanup]
    environment: develop
    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
    permissions:
      id-token: write
      contents: read
    env:
      EXPORT_S3_KEY_PREFIX: pilot-data
    steps:
      - name: Setup code, pipenv, aws
        uses: Sage-Bionetworks/action-pipenv-aws-setup@v3
        with:
          role_to_assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }}
          role_session_name: integration-test-${{ github.run_id }}
          python_version: ${{ env.PYTHON_VERSION }}

      - name: Set namespace for non-default branch or for tag
        if: github.ref_name != 'main'
        run: echo "NAMESPACE=$GITHUB_REF_NAME" >> $GITHUB_ENV

      - name: Fetch the most recent exports.
        id: recent-exports
        run: |
          # Retrieve the last ~2 weeks of exports from each cohort
          # Ignore keys which end with "/" and which match "owner.txt"
          echo "KEYS=$(
            aws s3api list-objects-v2 \
            --bucket $DEV_INPUT_BUCKET \
            --prefix $EXPORT_S3_KEY_PREFIX \
            --query '((sort_by(Contents[? !ends_with(Key, `/`) && !contains(Key, `owner.txt`)], &LastModified)[::-1])[:${{ env.INTEGRATION_TEST_NUM_EXPORTS }}])[*].Key' |
            jq -c
          )" >> "$GITHUB_OUTPUT"

      - name: Copy most recent exports to this namespace
        run: >
          echo '${{ steps.recent-exports.outputs.KEYS }}' | jq -r '.[]' | while read -r key; do
            aws s3 cp "s3://$DEV_INPUT_BUCKET/$key" "s3://$DEV_INPUT_BUCKET/$NAMESPACE/${key#"$EXPORT_S3_KEY_PREFIX"/}";
          done

      - name: Write most recent exports to S3 cloudformation bucket
        run: >
          echo '${{ steps.recent-exports.outputs.KEYS }}' |
          jq --arg bucket "s3://$DEV_INPUT_BUCKET/" '.[] |= $bucket + .' |
          aws s3 cp - "s3://${{ vars.CFN_BUCKET }}/$NAMESPACE/integration_test_exports.json"

  sceptre-deploy-staging:
    name: Deploys to staging of prod using sceptre
    runs-on: ubuntu-latest
    needs: integration-test-develop
    if: github.ref_name == 'main'
    environment: prod
    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
    permissions:
      id-token: write
      contents: read
    steps:
      - name: Setup code, pipenv, aws
        uses: Sage-Bionetworks/action-pipenv-aws-setup@v3
        with:
          role_to_assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }}
          role_session_name: GitHubActions-${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}
          python_version: ${{ env.PYTHON_VERSION }}

      - name: Copy files to templates bucket
        run: >
          python src/scripts/manage_artifacts/artifacts.py
          --upload
          --namespace staging
          --cfn_bucket ${{ vars.CFN_BUCKET }}

      - name: Create directory for remote sceptre templates
        run: mkdir -p templates/remote/

      - name: Deploy sceptre stacks to staging on prod
        run: pipenv run sceptre --var "namespace=staging" launch prod --yes

  integration-test-staging-cleanup:
    name: Cleanup main branch staging data before integration tests
    runs-on: ubuntu-latest
    needs: sceptre-deploy-staging
    if: github.ref_name == 'main'
    environment: prod
    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
    permissions:
      id-token: write
      contents: read
    steps:
      - name: Setup code, pipenv, aws
        uses: Sage-Bionetworks/action-pipenv-aws-setup@v3
        with:
          role_to_assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }}
          role_session_name: GitHubActions-${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}
          python_version: ${{ env.PYTHON_VERSION }}

      - name: Clean input data bucket
        run: >
          pipenv run python src/scripts/manage_artifacts/clean_for_integration_test.py
          --bucket $PROD_INPUT_BUCKET
          --bucket_prefix "staging/"

      - name: Clean raw data bucket
        run: >
          pipenv run python src/scripts/manage_artifacts/clean_for_integration_test.py
          --bucket $PROD_RAW_BUCKET
          --bucket_prefix "staging/json/"

      - name: Clean intermediate data bucket
        run: >
          pipenv run python src/scripts/manage_artifacts/clean_for_integration_test.py
          --bucket $PROD_INTERMEDIATE_BUCKET
          --bucket_prefix "staging/json/"

  integration-test-staging:
    name: Triggers staging workflow with production data
    runs-on: ubuntu-latest
    needs: [sceptre-deploy-staging, integration-test-staging-cleanup]
    environment: prod
    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
    permissions:
      id-token: write
      contents: read
    env:
      EXPORT_S3_KEY_PREFIX: main
    steps:
      - name: Setup code, pipenv, aws
        uses: Sage-Bionetworks/action-pipenv-aws-setup@v3
        with:
          role_to_assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }}
          role_session_name: integration-test-${{ github.run_id }}
          python_version: ${{ env.PYTHON_VERSION }}

      - name: Fetch the most recent exports.
        id: recent-exports
        run: |
          # Retrieve the last ~2 weeks of exports from each cohort
          # Ignore keys which end with "/" and which match "owner.txt"
          echo "KEYS=$(
            aws s3api list-objects-v2 \
            --bucket $PROD_INPUT_BUCKET \
            --prefix "$EXPORT_S3_KEY_PREFIX/" \
            --query '((sort_by(Contents[? !ends_with(Key, `/`) && !contains(Key, `owner.txt`)], &LastModified)[::-1])[:${{ env.INTEGRATION_TEST_NUM_EXPORTS }}])[*].Key' |
            jq -c
          )" >> "$GITHUB_OUTPUT"

      - name: Copy most recent exports to this namespace
        run: >
          echo '${{ steps.recent-exports.outputs.KEYS }}' | jq -r '.[]' | while read -r key; do
            aws s3 cp "s3://$PROD_INPUT_BUCKET/$key" "s3://$PROD_INPUT_BUCKET/staging/${key#"$EXPORT_S3_KEY_PREFIX"/}";
          done

      - name: Write most recent exports to S3 cloudformation bucket
        run: >
          echo '${{ steps.recent-exports.outputs.KEYS }}' |
          jq --arg bucket "s3://$PROD_INPUT_BUCKET/" '.[] |= $bucket + .' |
          aws s3 cp - "s3://${{ vars.CFN_BUCKET }}/staging/integration_test_exports.json"