Add tests for staging (#100)

* Add tests for staging * lint * update col names * update tests * Remove recency test fot config setting * error when staging model fails for recency tests * rework some tests based on feedback * Run test incrementally remove quote * Support quote * Remove recency test for contract code * update update update update update update lint okay? update update why does it not ignore update update identify identify check was bool a prob cheeck? check update Revert update udpate check update update update update * conditional interval stringify * Remove acceptable value test for stg_contract_data * update sqlfluff update update update deps * ignore specific test * exclude specific test * revert all changes made for sqlfluff
stellar · Oct 9, 2024 · b36e80f · b36e80f
1 parent 3ee8ad5
commit b36e80f
Show file tree

Hide file tree

Showing 18 changed files with 2,978 additions and 4 deletions.
diff --git a/.github/workflows/diff-quality.yml b/.github/workflows/diff-quality.yml
@@ -68,4 +68,4 @@ jobs:
       - name: Checkout source branch and diff quality
         run: |
           git checkout ${{ github.head_ref }}
-          diff-quality --violations=sqlfluff --compare-branch=origin/master --fail-under=95
+          diff-quality --violations=sqlfluff --exclude incremental_accepted_values.sql --compare-branch=origin/master --fail-under=95
diff --git a/models/staging/stg_account_signers.yml b/models/staging/stg_account_signers.yml
@@ -0,0 +1,92 @@
+version: 2
+
+models:
+  - name: stg_account_signers
+    tests:
+      - dbt_utils.recency:
+          datepart: hour
+          field: cast(closed_at as timestamp)
+          interval: '{{ 1 if target.name == "prod" else 6 }}'
+          meta:
+            description: "Monitors the freshness of your table over time, as the expected time between data updates."
+      - incremental_unique_combination_of_columns:
+          date_column_name: "closed_at"
+          greater_than_equal_to: "2 day"
+          combination_of_columns:
+            - account_id
+            - signer
+            - ledger_sequence
+            - last_modified_ledger
+          meta:
+            description: "Tests the uniqueness combination of: account_id, signer, ledger_sequence and last_modified_ledger"
+    columns:
+      - name: account_id
+        description: '{{ doc("account_id") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: signer
+        description: '{{ doc("signer") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: weight
+        description: '{{ doc("weight") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: sponsor
+        description: '{{ doc("sponsor") }}'
+      - name: last_modified_ledger
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: ledger_entry_change
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+          - incremental_accepted_values:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+              values: [0, 1, 2]
+
+      - name: deleted
+        description: '{{ doc("deleted") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: batch_id
+        description: '{{ doc("batch_id") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: batch_run_date
+        description: '{{ doc("batch_run_date") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: closed_at
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: ledger_sequence
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: batch_insert_ts
+        description: '{{ doc("batch_insert_ts") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: airflow_start_ts
diff --git a/models/staging/stg_accounts.yml b/models/staging/stg_accounts.yml
@@ -0,0 +1,172 @@
+version: 2
+
+models:
+  - name: stg_accounts
+    tests:
+      - dbt_utils.recency:
+          datepart: hour
+          field: cast(closed_at as timestamp)
+          interval: '{{ 1 if target.name == "prod" else 6 }}'
+          meta:
+            description: "Monitors the freshness of your table over time, as the expected time between data updates."
+      - incremental_unique_combination_of_columns:
+          date_column_name: "closed_at"
+          greater_than_equal_to: "2 day"
+          combination_of_columns:
+            - account_id
+            - sequence_number
+            - ledger_entry_change
+            - last_modified_ledger
+          meta:
+            description: "Tests the uniqueness combination of: account_id, sequence_number, ledger_entry_change and last_modified_ledger."
+    columns:
+      - name: account_id
+        description: '{{ doc("account_id") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: balance
+        description: '{{ doc("balance") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: buying_liabilities
+        description: '{{ doc("buying_liabilities") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: selling_liabilities
+        description: '{{ doc("selling_liabilities") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: sequence_number
+        description: '{{ doc("sequence_number") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: num_subentries
+        description: '{{ doc("num_subentries") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: inflation_destination
+        description: '{{ doc("inflation_destination") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: flags
+        description: '{{ doc("flags_accounts_balances") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: home_domain
+        description: '{{ doc("home_domain") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: master_weight
+        description: '{{ doc("master_weight") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: threshold_low
+        description: '{{ doc("threshold_low") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: threshold_medium
+        description: '{{ doc("threshold_medium") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: threshold_high
+        description: '{{ doc("threshold_high") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: last_modified_ledger
+        description: '{{ doc("last_modified_ledger") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: ledger_entry_change
+        description: '{{ doc("ledger_entry_change") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+          - incremental_accepted_values:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+              values: [0, 1, 2]
+
+      - name: deleted
+        description: '{{ doc("deleted") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: sponsor
+        description: '{{ doc("sponsor") }}'
+      - name: num_sponsored
+        description: '{{ doc("num_sponsored") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: num_sponsoring
+        description: '{{ doc("num_sponsoring") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: sequence_ledger
+        description: '{{ doc("sequence_ledger") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: batch_id
+        description: '{{ doc("batch_id") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: batch_run_date
+        description: '{{ doc("batch_run_date") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: closed_at
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: ledger_sequence
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: batch_insert_ts
+        description: '{{ doc("batch_insert_ts") }}'
+        tests:
+          - incremental_not_null:
+              date_column_name: "closed_at"
+              greater_than_equal_to: "2 day"
+      - name: airflow_start_ts