diff --git a/.github/workflows/commit_manifest_temp.yml b/.github/workflows/commit_manifest_temp.yml new file mode 100644 index 00000000000..830a76c8570 --- /dev/null +++ b/.github/workflows/commit_manifest_temp.yml @@ -0,0 +1,51 @@ +name: Commit Manifest from PR (temp) + +on: + workflow_dispatch: + pull_request: + paths: + - hourly_spellbook/** + - .github/workflows/hourly_spellbook.yml + - .github/workflows/dbt_run.yml + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +jobs: + commit_manifest: + runs-on: [ self-hosted, linux, spellbook-trino-ci ] + + steps: + - uses: actions/setup-python@v3 + - name: Checkout main branch + uses: actions/checkout@v2 + with: + ref: main + + - name: Add git_sha to schema + run: "/runner/change_schema.sh wizard" + + - name: Setup variables + run: | + echo "GIT_SHA=$(echo ${{ github.sha }} | tr - _ | cut -c1-7)" >> $GITHUB_ENV + echo "PROFILE=--profiles-dir $HOME/.dbt --profile dunesql" >> $GITHUB_ENV + echo "S3_LOCATION=s3://manifest-spellbook-dunesql/hourly_spellbook" >> $GITHUB_ENV + PROJECT_DIR=hourly_spellbook + if [[ "$PROJECT_DIR" == "spellbook" ]]; then + PROJECT_DIR="." + fi + echo "PROJECT_DIR=$PROJECT_DIR" >> $GITHUB_ENV + + - name: dbt dependencies + working-directory: ${{env.PROJECT_DIR}} + run: "dbt deps" + + - name: Run dbt debug on Trino if dunesql engine is used, and retry until cluster is available + run: "./scripts/ensure_cluster.sh" + + - name: dbt compile to create prod manifest from main + run: "dbt compile --target-path . $PROFILE --project-dir ${PROJECT_DIR}" + + - name: upload manifest + run: "aws s3 cp $PROJECT_DIR/manifest.json $S3_LOCATION/manifest.json" diff --git a/.github/workflows/hourly_spellbook.yml b/.github/workflows/hourly_spellbook.yml index 79024ac0231..7ea3ecdd80d 100644 --- a/.github/workflows/hourly_spellbook.yml +++ b/.github/workflows/hourly_spellbook.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: pull_request: paths: - - daily_spellbook/** + - hourly_spellbook/** - .github/workflows/hourly_spellbook.yml - .github/workflows/dbt_run.yml