test-github-action.yml

name: Test elementary GitHub action
on:
  schedule:
    - cron: "0 0 * * *"
  workflow_dispatch:
    inputs:
      elementary-ref:
        type: string
        required: false
        description: Branch or tag to checkout for 'elementary' repository
      dbt-data-reliability-ref:
        type: string
        required: false
        description: Branch or tag to checkout for 'dbt-data-reliability' repository
      dbt-version:
        type: string
        required: false
        description: dbt's version to test with

env:
  DBT_PKG_INTEG_TESTS_DIR: ${{ github.workspace }}/dbt-data-reliability/integration_tests/deprecated_tests

jobs:
  test:
    runs-on: ubuntu-20.04
    defaults:
      run:
        working-directory: elementary
    steps:
      - name: Checkout Elementary
        uses: actions/checkout@v3
        with:
          path: elementary
          ref: ${{ inputs.elementary-ref }}

      - name: Checkout dbt package
        uses: actions/checkout@v3
        with:
          repository: elementary-data/dbt-data-reliability
          path: dbt-data-reliability
          ref: ${{ inputs.dbt-data-reliability-ref }}

      - name: Write dbt profiles
        id: profiles
        env:
          PROFILES_YML: ${{ secrets.TEST_GITHUB_ACTION_PROFILES_YML }}
        run: |
          mkdir -p ~/.dbt
          echo "$PROFILES_YML" > ~/.dbt/profiles.yml

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.8"

      - name: Install dbt
        run: pip install --pre
          "dbt-core${{ inputs.dbt-version && format('=={0}', inputs.dbt-version) }}"
          "dbt-snowflake${{ inputs.dbt-version && format('<={0}', inputs.dbt-version) }}"

      - name: Install Elementary
        run: |
          pip install -r dev-requirements.txt
          pip install ".[snowflake]"

      - name: Install dbt package
        run: |
          ELEMENTARY_PKG_LOCATION=$(pip show elementary-data | grep -i location | awk '{print $2}')
          DBT_PROJECT_PATH="$ELEMENTARY_PKG_LOCATION/elementary/monitor/dbt_project"
          DBT_PKGS_PATH="$DBT_PROJECT_PATH/dbt_packages"
          dbt deps --project-dir "$DBT_PROJECT_PATH"
          rm -rf "$DBT_PKGS_PATH/elementary"
          ln -vs "$GITHUB_WORKSPACE/dbt-data-reliability" "$DBT_PKGS_PATH/elementary"

      - name: Run dbt package integration tests
        working-directory: ${{ env.DBT_PKG_INTEG_TESTS_DIR }}
        run: |
          dbt deps
          python run_e2e_tests.py -t "snowflake" --clear-tests "True" -e "regular"

      - name: Run Elementary
        uses: elementary-data/run-elementary-action@v1.8
        with:
          warehouse-type: snowflake
          profiles-yml: ${{ secrets.TEST_GITHUB_ACTION_PROFILES_YML }}
          edr-command:
            edr monitor -t "snowflake" --slack-token "${{ secrets.CI_SLACK_TOKEN }}" --slack-channel-name data-ops
            &&
            edr report
            --file-path "report.html"

      - name: Upload report
        uses: actions/upload-artifact@v3
        with:
          name: report.html
          path: edr_target/report.html

      - name: Upload log
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: edr.log
          path: edr_target/edr.log

  notify_failures:
    name: Notify Slack
    needs: test
    if: |
      always() &&
      ! cancelled() &&
      ! contains(needs.test.result, 'success') &&
      ! contains(needs.test.result, 'cancelled')
      ${{ github.event_name == 'schedule' }}
    uses: ./.github/workflows/notify_slack.yml
    with:
      result: "failure"
      run_id: ${{ github.run_id }}
      workflow_name: "Test Elementary GitHub action"