Skip to content

Commit

Permalink
Tests updates with volume tests, timestamp column fix using the parti…
Browse files Browse the repository at this point in the history 
…tion column and added description to the tests
  • Loading branch information
@enercolini
enercolini committed Dec 28, 2023
1 parent b0fc578 commit 3d019b7
Show file tree
Hide file tree
Showing 16 changed files with 303 additions and 108 deletions.
6 changes: 5 additions & 1 deletion dbt_project.yml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

# Name your project! Project names should contain only lowercase characters
# and underscores. A good package name should reflect your organization's
# name or the intended use of these models
Expand Down Expand Up @@ -39,3 +38,8 @@ models:
+materialized: view
marts:
materialized: table

elementary:
+schema: elementary
+docs:
show: false
2 changes: 2 additions & 0 deletions models/marts/enriched_history_operations.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ with
{% if is_incremental() %}
and cast(batch_run_date as date) >= date('{{ dbt_airflow_macros.ds() }}') -- batch run is the min bound of a batch
and date(closed_at) >= date_sub(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day)
{% endif %}
)

, history_transactions as (
Expand Down Expand Up @@ -194,6 +195,7 @@ with
{% if is_incremental() %}
and cast(batch_run_date as date) >= date('{{ dbt_airflow_macros.ds() }}') -- batch run is the min bound of a batch
and date(closed_at) >= date_sub(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day)
{% endif %}

)

Expand Down
98 changes: 56 additions & 42 deletions models/marts/enriched_history_operations.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,20 @@ models:
time_bucket:
period: hour
count: 12
meta:
description:
"Monitors the freshness of your table over time, as the expected time between data updates."
- elementary.dimension_anomalies:
timestamp_column: closed_at
backfill_days: 90
dimensions:
- "type"
time_bucket:
period: day
count: 1
meta:
description:
"Monitors the frequency of values in the type column over time."
columns:
- name: op_id
description: '{{ doc("operation_id") }}'
Expand All @@ -20,21 +34,14 @@ models:
and closed_at < timestamp_trunc(current_timestamp(), day)
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
- relationships:
to: ref('stg_history_operations')
field: op_id
#- all_source_table_values_in_destination_table:
# to: ref('stg_history_operations')
# field: op_id
where: closed_at > current_timestamp - interval 2 day

- name: op_source_account
description: '{{ doc("source_account") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day

where: closed_at > current_timestamp - interval 2 day

- name: op_source_account_muxed
description: '{{ doc("source_account_muxed") }}'
Expand All @@ -44,24 +51,28 @@ models:
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
- relationships:
to: ref('stg_history_transactions')
field: transaction_id
where: closed_at > current_timestamp - interval 2 day

- name: type
description: '{{ doc("type") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day
- dbt_utils.expression_is_true:
expression: '<= 26'
config:
where: closed_at > current_timestamp - interval 90 day
meta:
description:
"Test to check if the type of operation added is not greater than 26."

- name: type_string
description: '{{ doc("type_string") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: account
description: '{{ doc("details_account") }}'
Expand Down Expand Up @@ -368,49 +379,49 @@ models:
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: ledger_sequence
description: '{{ doc("ledger_sequence") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: account_sequence
description: '{{ doc("account_sequence") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: max_fee
description: '{{ doc("max_fee") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: txn_operation_count
description: '{{ doc("transaction_operation_count") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: txn_created_at
description: '{{ doc("created_at") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: memo_type
description: '{{ doc("memo_type") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: memo
description: '{{ doc("memo") }}'
Expand All @@ -423,14 +434,14 @@ models:
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: fee_charged
description: '{{ doc("fee_charged") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: fee_account
description: '{{ doc("fee_account") }}'
Expand Down Expand Up @@ -476,118 +487,121 @@ models:
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: previous_ledger_hash
description: '{{ doc("previous_ledger_hash") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: transaction_count
description: '{{ doc("transaction_count") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: ledger_operation_count
description: '{{ doc("operation_count") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: closed_at
description: '{{ doc("closed_at") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: ledger_id
description: '{{ doc("ledger_id") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: total_coins
description: '{{ doc("total_coins") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: fee_pool
description: '{{ doc("fee_pool") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: base_fee
description: '{{ doc("base_fee") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: base_reserve
description: '{{ doc("base_reserve") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: max_tx_set_size
description: '{{ doc("max_tx_set_size") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: protocol_version
description: '{{ doc("protocol_version") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day
- dbt_utils.expression_is_true:
expression: '<= 19'
meta:
description:
"Test to check if the protocol version is not greater than 19."

- name: succesful_transaction_count
- name: successful_transaction_count
description: '{{ doc("successful_transaction_count") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: failed_transaction_count
description: '{{ doc("failed_transaction_count") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: batch_id
description: '{{ doc("batch_id") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: batch_run_date
description: '{{ doc("batch_run_date") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day

- name: batch_insert_ts
description: '{{ doc("batch_insert_ts") }}'
tests:
- not_null:
config:
where: batch_run_date > current_datetime - interval 2 day
where: closed_at > current_timestamp - interval 2 day
6 changes: 5 additions & 1 deletion models/marts/fee_stats_agg.sql
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
{{ config(
materialized='incremental',
unique_key=["day_agg"]
unique_key=["day_agg"],
partition_by={
"field": "day_agg"
, "data_type": "date"
, "granularity": "month"}
)
}}

Expand Down
Loading

0 comments on commit 3d019b7

Please sign in to comment.