Add workflow to calculate schema changes

.github/workflows/update_dbt_marts_schema_changelog.yml

@@ -0,0 +1,73 @@
+name: Update changelog for DBT marts
+
+on:
+  schedule:
+    - cron: "0 23 * * 1-5" # This will run at 11 PM UTC, which is 5 PM CST
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }}-{{ github.event_name }}
+  cancel-in-progress: true
+
+env:
+  IS_RECENCY_AIRFLOW_TASK: "false"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v2
+
+      - name: Authenticate to crypto-stellar GCP
+        uses: "google-github-actions/auth@v2"
+        with:
+          project_id: hubble-261722
+          credentials_json: "${{ secrets.CREDS_PROD_HUBBLE }}"
+
+      - name: Set up Google Cloud SDK
+        run: |
+          echo "Installing Google Cloud SDK..."
+          echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+          curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
+          sudo apt-get update && sudo apt-get install -y google-cloud-sdk
+
+      - name: Create new branch
+        id: create_branch
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "GitHub Action"
+          BRANCH_NAME="update-data-schema-changelog-${{ github.run_id }}"
+          git checkout -b $BRANCH_NAME
+          echo "::set-output name=branch::$BRANCH_NAME"
+
+      - name: Run Bash Script
+        run: |
+          cd $GITHUB_WORKSPACE
+          output=$(. scripts/update_dbt_marts_schema_changelog.sh)
+          echo "$output" > changelog/dbt_marts.md
+
+      - name: Commit changes
+        id: commit_changes
+        run: |
+          git add changelog/dbt_marts.md
+          if git commit -m "Update changelog for DBT marts"; then
+            echo "Changes committed."
+            echo "changes_committed=true" >> $GITHUB_OUTPUT
+          else
+            echo "No changes to commit."
+            echo "changes_committed=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Push branch
+        if: steps.commit_changes.outputs.changes_committed == 'true'
+        run: |
+          git push origin ${{ steps.create_branch.outputs.branch }}
+
+      - name: Create Pull Request
+        if: steps.commit_changes.outputs.changes_committed == 'true'
+        run: |
+          gh pr create -B master -H ${{ steps.create_branch.outputs.branch }} \
+          --title 'Merge ${{ steps.create_branch.outputs.branch }} into master' \
+          --body 'Created by GitHub action'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.prettierignore

@@ -0,0 +1 @@
+**/*.md

.sqlfluffignore

@@ -1 +1,2 @@
 dags/ddls/queries
+changelog/dbt_marts.md

changelog/dbt_marts.md

@@ -0,0 +1,19 @@
+# Changes in DBT marts schema
+
+| Date       |       Table Name                | Operation     | Columns                  |
+|------------|---------------------------------|---------------|--------------------------|
+| 2024-09-12 | ASSET_STATS_AGG | column_added | airflow_start_ts |
+| 2024-09-12 | DIM_DATES | column_added | airflow_start_ts |
+| 2024-09-12 | DIM_MGI_WALLETS | column_added | airflow_start_ts |
+| 2024-09-12 | FCT_MGI_CASHFLOW | column_added | airflow_start_ts |
+| 2024-09-12 | LIQUIDITY_POOLS_VALUE | column_added | airflow_start_ts |
+| 2024-09-12 | LIQUIDITY_POOLS_VALUE_HISTORY | column_added | airflow_start_ts |
+| 2024-09-12 | LIQUIDITY_POOL_TRADE_VOLUME | column_added | airflow_start_ts |
+| 2024-09-12 | LIQUIDITY_PROVIDERS | column_added | airflow_start_ts |
+| 2024-09-12 | MGI_MONTHLY_USD_BALANCE | column_added | airflow_start_ts |
+| 2024-09-12 | MGI_NETWORK_STATS_AGG | column_added | airflow_start_ts |
+| 2024-09-12 | NETWORK_STATS_AGG | column_added | airflow_start_ts |
+| 2024-09-12 | OHLC_EXCHANGE_FACT | column_added | airflow_start_ts |
+| 2024-09-12 | PARTNERSHIP_ASSETS__ACCOUNT_HOLDERS_ACTIVITY_FACT | column_added | airflow_start_ts |
+| 2024-09-12 | PARTNERSHIP_ASSETS__ASSET_ACTIVITY_FACT | column_added | airflow_start_ts |
+| 2024-09-12 | PARTNERSHIP_ASSETS__MOST_ACTIVE_FACT | column_added | airflow_start_ts |

scripts/update_dbt_marts_schema_changelog.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+result=$(bq query --format=prettyjson --nouse_legacy_sql \
+    'SELECT
+        date(detected_at) as date
+        , table_name
+        , sub_type as operation
+        , ARRAY_AGG(column_name) as columns
+ FROM
+   `hubble-261722`.elementary.alerts_schema_changes
+    GROUP BY
+    1, 2, 3
+    ORDER BY 1 DESC, 2 ASC
+')
+
+echo "# Changes in DBT marts schema"
+
+echo ""
+
+echo "| Date       |       Table Name                | Operation     | Columns                  |"
+echo "|------------|---------------------------------|---------------|--------------------------|"
+
+echo "$result" | jq -r '.[] | "| \(.date) | \(.table_name ) | \(.operation) | \(.columns | join(", ")) |"'
+echo ""