From 4ae7a3bf8d730b79ed9d94c2af5b7330617cfd8d Mon Sep 17 00:00:00 2001 From: laysabit Date: Thu, 14 Mar 2024 19:31:19 -0300 Subject: [PATCH] test #10 --- models/marts/enriched_history_operations.sql | 8 +- tests/anomaly_detection_trade_count.sql | 33 ++++---- tests/anomaly_detection_trade_volume.sql | 80 ++++++++++---------- 3 files changed, 64 insertions(+), 57 deletions(-) diff --git a/models/marts/enriched_history_operations.sql b/models/marts/enriched_history_operations.sql index dc3d0cd..3768f8a 100644 --- a/models/marts/enriched_history_operations.sql +++ b/models/marts/enriched_history_operations.sql @@ -35,7 +35,7 @@ with where cast(batch_run_date as date) < date_add(date('{{ dbt_airflow_macros.ds() }}'), interval 2 day) and date(closed_at) < date_add(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day) - {% if is_incremental() %} + {% if is_incremental() %} and cast(batch_run_date as date) >= date_sub(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day) and date(closed_at) >= date_sub(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day) {% endif %} @@ -76,7 +76,7 @@ with where cast(batch_run_date as date) < date_add(date('{{ dbt_airflow_macros.ds() }}'), interval 2 day) and date(closed_at) < date_add(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day) - {% if is_incremental() %} + {% if is_incremental() %} and cast(batch_run_date as date) >= date_sub(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day) and date(closed_at) >= date_sub(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day) {% endif %} @@ -205,8 +205,8 @@ with where cast(batch_run_date as date) < date_add(date('{{ dbt_airflow_macros.ds() }}'), interval 2 day) and date(closed_at) < date_add(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day) - {% if is_incremental() %} - and cast(batch_run_date as date) >= date_sub(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day) + {% if is_incremental() %} + and cast(batch_run_date as date) >= date_sub(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day) and date(closed_at) >= date_sub(date('{{ dbt_airflow_macros.ds() }}'), interval 1 day) {% endif %} diff --git a/tests/anomaly_detection_trade_count.sql b/tests/anomaly_detection_trade_count.sql index e165fec..90d26fc 100644 --- a/tests/anomaly_detection_trade_count.sql +++ b/tests/anomaly_detection_trade_count.sql @@ -1,26 +1,29 @@ {{ config(severity="warn") }} -with trade_counts as ( - select - date(ledger_closed_at) as close_date - , count(*) as trade_count - from {{ ref('stg_history_trades') }} - where date(ledger_closed_at) >= date_sub(current_date(), interval 90 day) - and date(ledger_closed_at) < current_date() - group by close_date +with + trade_counts as ( + select + date(ledger_closed_at) as close_date + , count(*) as trade_count + from {{ ref('stg_history_trades') }} + where + date(ledger_closed_at) >= date_sub(current_date(), interval 90 day) + and date(ledger_closed_at) < current_date() + group by close_date ) -, bounds as ( - select - (avg(trade_count) - (3 * stddev(trade_count))) as lower_bound - , (avg(trade_count) + (3 * stddev(trade_count))) as upper_bound - from trade_counts + , bounds as ( + select + (avg(trade_count) - (3 * stddev(trade_count))) as lower_bound + , (avg(trade_count) + (3 * stddev(trade_count))) as upper_bound + from trade_counts ) select close_date , trade_count from trade_counts, bounds -where close_date = date_sub(current_date(), interval 1 day) +where + close_date = date_sub(current_date(), interval 1 day) and trade_count >= upper_bound - or trade_count <= lower_bound + or trade_count <= lower_bound diff --git a/tests/anomaly_detection_trade_volume.sql b/tests/anomaly_detection_trade_volume.sql index 57c56ff..ad13982 100644 --- a/tests/anomaly_detection_trade_volume.sql +++ b/tests/anomaly_detection_trade_volume.sql @@ -1,50 +1,54 @@ {{ config(severity="warn") }} -with base_trades as ( - select - date(ledger_closed_at) as close_date - , sum(buying_amount) as amount - from {{ ref('stg_history_trades') }} - where date(ledger_closed_at) >= date_sub(current_date(), interval 90 day) - and date(ledger_closed_at) <= current_date() - group by close_date -) +with + base_trades as ( + select + date(ledger_closed_at) as close_date + , sum(buying_amount) as amount + from {{ ref('stg_history_trades') }} + where + date(ledger_closed_at) >= date_sub(current_date(), interval 90 day) + and date(ledger_closed_at) <= current_date() + group by close_date + ) -, counter_trades as ( - select - date(ledger_closed_at) as close_date - , sum(selling_amount) as amount - from {{ ref('stg_history_trades') }} - where date(ledger_closed_at) >= date_sub(current_date(), interval 90 day) - and date(ledger_closed_at) < current_date() - group by close_date -) + , counter_trades as ( + select + date(ledger_closed_at) as close_date + , sum(selling_amount) as amount + from {{ ref('stg_history_trades') }} + where + date(ledger_closed_at) >= date_sub(current_date(), interval 90 day) + and date(ledger_closed_at) < current_date() + group by close_date + ) -, data_table as ( - select - close_date - , sum(amount) as amount - from ( - select * - from base_trades - union all - select * - from counter_trades + , data_table as ( + select + close_date + , sum(amount) as amount + from ( + select * + from base_trades + union all + select * + from counter_trades + ) + group by close_date ) - group by close_date -) -, bounds as ( - select - (avg(amount) - (3 * stddev(amount))) as lower_bound - , (avg(amount) + (3 * stddev(amount))) as upper_bound - from data_table -) + , bounds as ( + select + (avg(amount) - (3 * stddev(amount))) as lower_bound + , (avg(amount) + (3 * stddev(amount))) as upper_bound + from data_table + ) select close_date , amount from data_table, bounds -where close_date = date_sub(current_date(), interval 1 day) +where + close_date = date_sub(current_date(), interval 1 day) and amount >= upper_bound - or amount <= lower_bound + or amount <= lower_bound