From f0a46381ef61ba10596cd28a067c57b132a172bf Mon Sep 17 00:00:00 2001 From: laysabit Date: Wed, 28 Feb 2024 16:11:25 -0300 Subject: [PATCH] test #1 --- .github/workflows/lint.yml | 82 ++++++++++++++++++++ .gitignore | 1 + .sqlfluff | 4 +- models/marts/enriched_history_operations.sql | 4 +- profiles.yml | 28 +++++++ 5 files changed, 115 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/lint.yml create mode 100644 profiles.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..a5f690f --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,82 @@ +name: CI test package + +on: + push: + branches: + - fix-lint +env: + DBT_DEFAULT_PROFILE_TARGET: stellar_dbt_public + DBT_PROFILES_DIR: ${{ github.workspace }} + DBT_TARGET: dev + DBT_DATASET: crypto_stellar + DBT_PROJECT: crypto-stellar + DBT_MAX_BYTES_BILLED: 1000000000000 + DBT_JOB_TIMEOUT: 300 + DBT_THREADS: 1 + DBT_JOB_RETRIES: 1 + +jobs: + sqlfluff-lint-models: + name: Lint dbt models using SQLFluff + runs-on: ubuntu-latest + environment: + name: Integration Tests + + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + + - name: install requirements + run: pip install -r requirements.txt + + - name: Authenticate to GCP + uses: 'google-github-actions/auth@v2' + with: + project_id: crypto-stellar + credentials_json: "${{ secrets.CREDS_PROD_HUBBLE }}" + + - name: Get changed files + id: get_file_changes + uses: trilom/file-changes-action@v1.2.4 + with: + output: " " + + - name: Get new and changed .sql files in /models to lint + id: get_files_to_lint + shell: bash -l {0} + run: | + # Set the command in the $() brackets as an output to use in later steps + echo "::set-output name=lintees::$( + # Issue where grep regular expressions don't work as expected on the + # Github Actions shell, check dbt/models/ folder + echo \ + $(echo ${{ steps.get_file_changes.outputs.files_modified }} | + tr -s ' ' '\n' | + grep -E '^models.*[.]sql$' | + tr -s '\n' ' ') \ + $(echo ${{ steps.get_file_changes.outputs.files_added }} | + tr -s ' ' '\n' | + grep -E '^models.*[.]sql$' | + tr -s '\n' ' ') + )" + - name: Lint dbt models + id: sqlfluff_json + if: steps.get_files_to_lint.outputs.lintees != '' + shell: bash -l {0} + run: | + python -m pip install --upgrade pip + dbt deps + sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json + sed -i '/^\[/!d' annotations.json # see https://github.com/sqlfluff/sqlfluff/issues/2244 + + - name: Annotate + uses: yuzutech/annotations-action@v0.4.0 + with: + repo-token: "${{ secrets.GITHUB_TOKEN }}" + title: "SQLFluff Lint" + input: "./annotations.json" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 73b88cf..12d77d6 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ logs/ ### virtual environment venv/ +env/ ### gcp credentials credentials.json diff --git a/.sqlfluff b/.sqlfluff index cde1416..3bebeac 100644 --- a/.sqlfluff +++ b/.sqlfluff @@ -80,10 +80,10 @@ project_dir = ./ # If your project works only with .env files you need to: ### Change ~/.dbt/ for ./ on profiles_dir variable here ### Remove steps on the pipeline that create and copy the profiles.yml to ~/.dbt/ folder. -profiles_dir = ~/.dbt/ +profiles_dir = ./ # Name of the profiles. # CHANGE THIS TO THE PROFILE NAME ON YOUR PROFILES.YML FILE -profile = stellar_dbt +profile = stellar_dbt_public [sqlfluff:templater:jinja] load_macros_from_path = macros/percentile_iteration.sql,dbt_packages/dbt_utils/macros/ diff --git a/models/marts/enriched_history_operations.sql b/models/marts/enriched_history_operations.sql index ddefa85..fad6e2c 100644 --- a/models/marts/enriched_history_operations.sql +++ b/models/marts/enriched_history_operations.sql @@ -32,7 +32,7 @@ with , batch_run_date , batch_insert_ts from {{ ref('stg_history_ledgers') }} - where + where cast(batch_run_date as date) < date_add(date('{{ dbt_airflow_macros.ds() }}'), interval 2 day) and date(closed_at) < date_add(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day) {% if is_incremental() %} @@ -195,7 +195,7 @@ with , asset_balance_changes , parameters , parameters_decoded - , `function` + , function , address , soroban_operation_type , extend_to diff --git a/profiles.yml b/profiles.yml new file mode 100644 index 0000000..14ce9a6 --- /dev/null +++ b/profiles.yml @@ -0,0 +1,28 @@ +stellar_dbt_public: + target: "{{ env_var('DBT_TARGET') }}" + outputs: + prod: + dataset: "{{ env_var('DBT_DATASET') }}" + maximum_bytes_billed: "{{ env_var('DBT_MAX_BYTES_BILLED') | as_number }}" + job_execution_timeout_seconds: "{{ env_var('DBT_JOB_TIMEOUT') | int }}" + threads: "{{ env_var('DBT_THREADS') | int }}" + job_retries: "{{ env_var('DBT_JOB_RETRIES') | int }}" + priority: interactive + project: crypto-stellar + location: us + method: oauth + type: bigquery + test: + <<: *default + jenkins: + dataset: "{{ env_var('PUBLIC_SOURCE_SCHEMA') }}" + maximum_bytes_billed: "{{ env_var('DBT_MAX_BYTES_BILLED') | as_number }}" + job_execution_timeout_seconds: "{{ env_var('DBT_JOB_TIMEOUT') | int }}" + threads: "{{ env_var('DBT_THREADS') | int }}" + job_retries: "{{ env_var('DBT_JOB_RETRIES') | int }}" + priority: interactive + location: us + type: bigquery + project: test-hubble-319619 + keyfile: /stellar-dbt/service_account.json + method: service-account \ No newline at end of file