upload manifest through CI

.github/workflows/commit_manifest_temp.yml

@@ -0,0 +1,51 @@
+name: Commit Manifest from PR (temp)
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - hourly_spellbook/**
+      - .github/workflows/hourly_spellbook.yml
+      - .github/workflows/dbt_run.yml
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  commit_manifest:
+    runs-on: [ self-hosted, linux, spellbook-trino-ci ]
+
+    steps:
+    - uses: actions/setup-python@v3
+    - name: Checkout main branch
+      uses: actions/checkout@v2
+      with:
+        ref: main
+
+    - name: Add git_sha to schema
+      run: "/runner/change_schema.sh wizard"
+
+    - name: Setup variables
+      run: |
+        echo "GIT_SHA=$(echo ${{ github.sha }} | tr - _ | cut -c1-7)" >> $GITHUB_ENV
+        echo "PROFILE=--profiles-dir $HOME/.dbt --profile dunesql" >> $GITHUB_ENV
+        echo "S3_LOCATION=s3://manifest-spellbook-dunesql/hourly_spellbook" >> $GITHUB_ENV
+          PROJECT_DIR=hourly_spellbook
+          if [[ "$PROJECT_DIR" == "spellbook" ]]; then
+            PROJECT_DIR="."
+          fi
+          echo "PROJECT_DIR=$PROJECT_DIR" >> $GITHUB_ENV
+
+    - name: dbt dependencies
+      working-directory: ${{env.PROJECT_DIR}}
+      run: "dbt deps"
+
+    - name: Run dbt debug on Trino if dunesql engine is used, and retry until cluster is available
+      run: "./scripts/ensure_cluster.sh"
+
+    - name: dbt compile to create prod manifest from main
+      run: "dbt compile --target-path . $PROFILE --project-dir ${PROJECT_DIR}"
+
+    - name: upload manifest
+      run: "aws s3 cp $PROJECT_DIR/manifest.json $S3_LOCATION/manifest.json"

.github/workflows/hourly_spellbook.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
   pull_request:
     paths:
-      - daily_spellbook/**
+      - hourly_spellbook/**
       - .github/workflows/hourly_spellbook.yml
       - .github/workflows/dbt_run.yml