test #1

.github/workflows/lint.yml

@@ -0,0 +1,82 @@
+name: CI test package
+
+on:
+    push:
+        branches:
+            - fix-lint
+env:
+    DBT_DEFAULT_PROFILE_TARGET: stellar_dbt_public
+    DBT_PROFILES_DIR: ${{ github.workspace }}
+    DBT_TARGET: dev
+    DBT_DATASET: crypto_stellar
+    DBT_PROJECT: crypto-stellar
+    DBT_MAX_BYTES_BILLED: 1000000000000
+    DBT_JOB_TIMEOUT: 300
+    DBT_THREADS: 1
+    DBT_JOB_RETRIES: 1
+
+jobs:
+    sqlfluff-lint-models:
+        name: Lint dbt models using SQLFluff
+        runs-on: ubuntu-latest
+        environment:
+            name: Integration Tests
+
+        steps:
+          - name: checkout
+            uses: actions/checkout@v3
+
+          - name: Setup Python
+            uses: actions/setup-python@v4
+            with:
+              python-version: 3.8
+
+          - name: install requirements
+            run: pip install -r requirements.txt
+
+          - name: Authenticate to GCP
+            uses: 'google-github-actions/auth@v2'
+            with:
+                project_id: crypto-stellar
+                credentials_json: "${{ secrets.CREDS_PROD_HUBBLE }}" 
+
+          - name: Get changed files
+            id: get_file_changes
+            uses: trilom/[email protected]
+            with:
+              output: " "
+
+          - name: Get new and changed .sql files in /models to lint
+            id: get_files_to_lint
+            shell: bash -l {0}
+            run: |
+              # Set the command in the $() brackets as an output to use in later steps
+              echo "::set-output name=lintees::$(
+              # Issue where grep regular expressions don't work as expected on the
+              # Github Actions shell, check dbt/models/ folder
+              echo \
+              $(echo ${{ steps.get_file_changes.outputs.files_modified }} |
+              tr -s ' ' '\n' |
+              grep -E '^models.*[.]sql$' |
+              tr -s '\n' ' ') \
+              $(echo ${{ steps.get_file_changes.outputs.files_added }} |
+              tr -s ' ' '\n' |
+              grep -E '^models.*[.]sql$' |
+              tr -s '\n' ' ')
+              )"
+          - name: Lint dbt models
+            id: sqlfluff_json
+            if: steps.get_files_to_lint.outputs.lintees != ''
+            shell: bash -l {0}
+            run: |
+              python -m pip install --upgrade pip
+              dbt deps
+              sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json
+              sed -i '/^\[/!d' annotations.json # see https://github.com/sqlfluff/sqlfluff/issues/2244
+          
+          - name: Annotate
+            uses: yuzutech/[email protected]
+            with:
+              repo-token: "${{ secrets.GITHUB_TOKEN }}"
+              title: "SQLFluff Lint"
+              input: "./annotations.json"

.gitignore

@@ -8,6 +8,7 @@ logs/
 
 ### virtual environment
 venv/
+env/
 
 ### gcp credentials
 credentials.json

.sqlfluff

@@ -80,10 +80,10 @@ project_dir = ./
 # If your project works only with .env files you need to:
 ### Change ~/.dbt/ for ./ on profiles_dir variable here
 ### Remove steps on the pipeline that create and copy the profiles.yml to ~/.dbt/ folder.
-profiles_dir = ~/.dbt/
+profiles_dir = ./
 # Name of the profiles.
 # CHANGE THIS TO THE PROFILE NAME ON YOUR PROFILES.YML FILE
-profile = stellar_dbt
+profile = stellar_dbt_public
 
 [sqlfluff:templater:jinja]
 load_macros_from_path = macros/percentile_iteration.sql,dbt_packages/dbt_utils/macros/

models/marts/enriched_history_operations.sql

@@ -32,7 +32,7 @@ with
             , batch_run_date
             , batch_insert_ts
         from {{ ref('stg_history_ledgers') }}
-         where
+        where
             cast(batch_run_date as date) < date_add(date('{{ dbt_airflow_macros.ds() }}'), interval 2 day)
             and date(closed_at) < date_add(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day)
             {% if is_incremental() %}
@@ -195,7 +195,7 @@ with
             , asset_balance_changes
             , parameters
             , parameters_decoded
-            , `function`
+            , function
             , address
             , soroban_operation_type
             , extend_to

profiles.yml

@@ -0,0 +1,28 @@
+stellar_dbt_public:
+  target: "{{ env_var('DBT_TARGET') }}"
+  outputs:
+    prod:
+      dataset: "{{ env_var('DBT_DATASET') }}"
+      maximum_bytes_billed: "{{ env_var('DBT_MAX_BYTES_BILLED') | as_number }}"
+      job_execution_timeout_seconds: "{{ env_var('DBT_JOB_TIMEOUT') | int }}"
+      threads: "{{ env_var('DBT_THREADS') | int }}"
+      job_retries: "{{ env_var('DBT_JOB_RETRIES') | int }}"
+      priority: interactive
+      project: crypto-stellar
+      location: us
+      method: oauth
+      type: bigquery
+    test:
+      <<: *default
+    jenkins:
+      dataset: "{{ env_var('PUBLIC_SOURCE_SCHEMA') }}"
+      maximum_bytes_billed: "{{ env_var('DBT_MAX_BYTES_BILLED') | as_number }}"
+      job_execution_timeout_seconds: "{{ env_var('DBT_JOB_TIMEOUT') | int }}"
+      threads: "{{ env_var('DBT_THREADS') | int }}"
+      job_retries: "{{ env_var('DBT_JOB_RETRIES') | int }}"
+      priority: interactive
+      location: us
+      type: bigquery
+      project: test-hubble-319619
+      keyfile: /stellar-dbt/service_account.json
+      method: service-account