Merge pull request #25 from epoch8/development

Multi sources now available
scalio · Jun 29, 2023 · 89e1eb5 · 89e1eb5
2 parents aa99be8 + 93a0c7d
commit 89e1eb5
Show file tree

Hide file tree

Showing 79 changed files with 1,222 additions and 1,094 deletions.
diff --git a/dbt_project.yml b/dbt_project.yml
@@ -25,12 +25,13 @@ clean-targets:         # directories to be removed by `dbt clean`
   - "dbt_packages"
 
 # vars:
-#   DBT_PACKAGE_GA4__PROJECT: "holybricks"
-#   DBT_PACKAGE_GA4__DATASET: "analytics_202294062"
+#   DBT_PACKAGE_GA4__PROJECT: "commonwealth-356813"
+#   DBT_PACKAGE_GA4__DATASET: "analytics_375681484"
+#   DBT_PACKAGE_GA4__DATASETS: "analytics_375681484,analytics_247024976"
 #   DBT_PACKAGE_GA4__INTERVAL: 14
 #   DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL: 4
-#   DBT_PACKAGE_GA4__STREAM_ID: ['2059071580', '2771133919']
-#   DBT_PACKAGE_GA4__TIME_ZONE: 'America/Los_Angeles'
+#   DBT_PACKAGE_GA4__STREAM_ID: "['2059071580', '5200866205', '5200058214']"
+#   DBT_PACKAGE_GA4__TIME_ZONE: "America/Los_Angeles"
 
 
 # Configuring models
@@ -43,5 +44,11 @@ clean-targets:         # directories to be removed by `dbt clean`
 models:
   dbt_package_ga4:
 
+    integration:
+      +dataset: integration
+
     anchor:
       +dataset: anchor
+
+    bi:
+      +dataset: bi
diff --git a/models/anchor/ga4_event/anchor__ga4_event.sql b/models/anchor/ga4_event/anchor__ga4_event.sql
@@ -1,4 +1,4 @@
-{{
+{{-
     config(
         enabled = env_var('DBT_PACKAGE_GA4__ENABLE__ANCHOR', 'true') == 'true',
         tags = ['dbt_package_ga4', 'anchor'],
@@ -11,12 +11,12 @@
         },
         cluster_by = 'ga4_event_id'
     )
-}}
+-}}
 
 
 WITH t1 AS (
     SELECT DISTINCT
-        PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) AS ga4_date_partition,
+        PARSE_DATE('%Y%m%d', TABLE_SUFFIX) AS ga4_date_partition,
         TO_HEX(
             MD5(
                 CONCAT(
@@ -27,16 +27,17 @@ WITH t1 AS (
                 )
             ) AS ga4_event_id
     FROM
-        {{ source('dbt_package_ga4', 'events') }} AS events
+        {{ ref('src_ga4__events') }} AS events
     WHERE
-        _TABLE_SUFFIX NOT LIKE '%intraday%'
-        AND PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) > DATE_SUB(DATE(CURRENT_DATE()), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL') }} DAY)
-        AND events.stream_id IN UNNEST({{ env_var('DBT_PACKAGE_GA4__STREAM_ID') }})
+        TABLE_SUFFIX NOT LIKE '%intraday%'
+    {%- if not is_incremental() %}
+        AND PARSE_DATE('%Y%m%d', TABLE_SUFFIX) > DATE_SUB(DATE(CURRENT_DATE()), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL') }} DAY)
+    {%- endif %}
 
-    {% if is_incremental() %}
-    {% set max_partition_date = macro__get_max_partition_date(this.schema, this.table) %}
-        AND PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
-    {% endif %}
+    {%- if is_incremental() %}
+    {%- set max_partition_date = macro__get_max_partition_date(this.schema, this.table) %}
+        AND PARSE_DATE('%Y%m%d', TABLE_SUFFIX) > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
+    {%- endif %}
 ),
 
 t2 AS (
@@ -60,7 +61,7 @@ final AS (
 
 SELECT * FROM final
 
-    {% if is_incremental() %}
+    {%- if is_incremental() %}
     WHERE
         final.ga4_date_partition > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
-    {% endif %}
+    {%- endif %}
diff --git a/models/anchor/ga4_event/attr__ga4_event__ga4_event_bundle_sequence_id.sql b/models/anchor/ga4_event/attr__ga4_event__ga4_event_bundle_sequence_id.sql
@@ -1,4 +1,4 @@
-{{
+{{-
     config(
         enabled = env_var('DBT_PACKAGE_GA4__ENABLE__ANCHOR', 'true') == 'true',
         tags = ['dbt_package_ga4', 'anchor'],
@@ -11,12 +11,12 @@
         },
         cluster_by = 'ga4_event_id'
     )
-}}
+-}}
 
 
 WITH t1 AS (
     SELECT DISTINCT
-        PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) AS ga4_date_partition,
+        PARSE_DATE('%Y%m%d', TABLE_SUFFIX) AS ga4_date_partition,
         TO_HEX(
             MD5(
                 CONCAT(
@@ -28,16 +28,17 @@ WITH t1 AS (
             ) AS ga4_event_id,
         events.event_bundle_sequence_id AS ga4_event_bundle_sequence_id
     FROM
-        {{ source('dbt_package_ga4', 'events') }} AS events
+        {{ ref('src_ga4__events') }} AS events
     WHERE
-        _TABLE_SUFFIX NOT LIKE '%intraday%'
-        AND PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) > DATE_SUB(DATE(CURRENT_DATE()), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL') }} DAY)
-        AND events.stream_id IN UNNEST({{ env_var('DBT_PACKAGE_GA4__STREAM_ID') }})
-
-    {% if is_incremental() %}
-    {% set max_partition_date = macro__get_max_partition_date(this.schema, this.table) %}
-        AND PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
-    {% endif %}
+        TABLE_SUFFIX NOT LIKE '%intraday%'
+    {%- if not is_incremental() %}
+        AND PARSE_DATE('%Y%m%d', TABLE_SUFFIX) > DATE_SUB(DATE(CURRENT_DATE()), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL') }} DAY)
+    {%- endif %}
+
+    {%- if is_incremental() %}
+    {%- set max_partition_date = macro__get_max_partition_date(this.schema, this.table) %}
+        AND PARSE_DATE('%Y%m%d', TABLE_SUFFIX) > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
+    {%- endif %}
 ),
 
 t2 AS (
@@ -76,7 +77,7 @@ final AS (
 
 SELECT * FROM final
 
-    {% if is_incremental() %}
+    {%- if is_incremental() %}
     WHERE
         final.ga4_date_partition > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
-    {% endif %}
+    {%- endif %}
diff --git a/models/anchor/ga4_event/attr__ga4_event__ga4_event_date.sql b/models/anchor/ga4_event/attr__ga4_event__ga4_event_date.sql
@@ -1,4 +1,4 @@
-{{
+{{-
     config(
         enabled = env_var('DBT_PACKAGE_GA4__ENABLE__ANCHOR', 'true') == 'true',
         tags = ['dbt_package_ga4', 'anchor'],
@@ -11,12 +11,12 @@
         },
         cluster_by = 'ga4_event_id'
     )
-}}
+-}}
 
 
 WITH t1 AS (
     SELECT DISTINCT
-        PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) AS ga4_date_partition,
+        PARSE_DATE('%Y%m%d', TABLE_SUFFIX) AS ga4_date_partition,
         TO_HEX(
             MD5(
                 CONCAT(
@@ -28,16 +28,17 @@ WITH t1 AS (
             ) AS ga4_event_id,
         SAFE_CAST(events.event_date AS DATE FORMAT 'YYYYMMDD') AS ga4_event_date
     FROM
-        {{ source('dbt_package_ga4', 'events') }} AS events
+        {{ ref('src_ga4__events') }} AS events
     WHERE
-        _TABLE_SUFFIX NOT LIKE '%intraday%'
-        AND PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) > DATE_SUB(DATE(CURRENT_DATE()), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL') }} DAY)
-        AND events.stream_id IN UNNEST({{ env_var('DBT_PACKAGE_GA4__STREAM_ID') }})
-
-    {% if is_incremental() %}
-    {% set max_partition_date = macro__get_max_partition_date(this.schema, this.table) %}
-        AND PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
-    {% endif %}
+        TABLE_SUFFIX NOT LIKE '%intraday%'
+    {%- if not is_incremental() %}
+        AND PARSE_DATE('%Y%m%d', TABLE_SUFFIX) > DATE_SUB(DATE(CURRENT_DATE()), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL') }} DAY)
+    {%- endif %}
+
+    {%- if is_incremental() %}
+    {%- set max_partition_date = macro__get_max_partition_date(this.schema, this.table) %}
+        AND PARSE_DATE('%Y%m%d', TABLE_SUFFIX) > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
+    {%- endif %}
 ),
 
 t2 AS (
@@ -64,7 +65,7 @@ final AS (
 
 SELECT * FROM final
 
-    {% if is_incremental() %}
+    {%- if is_incremental() %}
     WHERE
         final.ga4_date_partition > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
-    {% endif %}
+    {%- endif %}
diff --git a/models/anchor/ga4_event/attr__ga4_event__ga4_event_ecommerce_purchase_revenue.sql b/models/anchor/ga4_event/attr__ga4_event__ga4_event_ecommerce_purchase_revenue.sql
@@ -1,4 +1,4 @@
-{{
+{{-
     config(
         enabled = env_var('DBT_PACKAGE_GA4__ENABLE__ANCHOR', 'true') == 'true',
         tags = ['dbt_package_ga4', 'anchor'],
@@ -11,12 +11,12 @@
         },
         cluster_by = 'ga4_event_id'
     )
-}}
+-}}
 
 
 WITH t1 AS (
     SELECT DISTINCT
-        PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) AS ga4_date_partition,
+        PARSE_DATE('%Y%m%d', TABLE_SUFFIX) AS ga4_date_partition,
         TO_HEX(
             MD5(
                 CONCAT(
@@ -28,16 +28,17 @@ WITH t1 AS (
             ) AS ga4_event_id,
         events.ecommerce.purchase_revenue AS ga4_event_ecommerce_purchase_revenue
     FROM
-        {{ source('dbt_package_ga4', 'events') }} AS events
+        {{ ref('src_ga4__events') }} AS events
     WHERE
-        _TABLE_SUFFIX NOT LIKE '%intraday%'
-        AND PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) > DATE_SUB(DATE(CURRENT_DATE()), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL') }} DAY)
-        AND events.stream_id IN UNNEST({{ env_var('DBT_PACKAGE_GA4__STREAM_ID') }})
-
-    {% if is_incremental() %}
-    {% set max_partition_date = macro__get_max_partition_date(this.schema, this.table) %}
-        AND PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
-    {% endif %}
+        TABLE_SUFFIX NOT LIKE '%intraday%'
+    {%- if not is_incremental() %}
+        AND PARSE_DATE('%Y%m%d', TABLE_SUFFIX) > DATE_SUB(DATE(CURRENT_DATE()), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL') }} DAY)
+    {%- endif %}
+
+    {%- if is_incremental() %}
+    {%- set max_partition_date = macro__get_max_partition_date(this.schema, this.table) %}
+        AND PARSE_DATE('%Y%m%d', TABLE_SUFFIX) > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
+    {%- endif %}
 ),
 
 t2 AS (
@@ -64,7 +65,7 @@ final AS (
 
 SELECT * FROM final
 
-    {% if is_incremental() %}
+    {%- if is_incremental() %}
     WHERE
         final.ga4_date_partition > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
-    {% endif %}
+    {%- endif %}
diff --git a/models/anchor/ga4_event/attr__ga4_event__ga4_event_ecommerce_purchase_revenue_in_usd.sql b/models/anchor/ga4_event/attr__ga4_event__ga4_event_ecommerce_purchase_revenue_in_usd.sql
@@ -1,4 +1,4 @@
-{{
+{{-
     config(
         enabled = env_var('DBT_PACKAGE_GA4__ENABLE__ANCHOR', 'true') == 'true',
         tags = ['dbt_package_ga4', 'anchor'],
@@ -11,12 +11,12 @@
         },
         cluster_by = 'ga4_event_id'
     )
-}}
+-}}
 
 
 WITH t1 AS (
     SELECT DISTINCT
-        PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) AS ga4_date_partition,
+        PARSE_DATE('%Y%m%d', TABLE_SUFFIX) AS ga4_date_partition,
         TO_HEX(
             MD5(
                 CONCAT(
@@ -28,16 +28,17 @@ WITH t1 AS (
             ) AS ga4_event_id,
         events.ecommerce.purchase_revenue_in_usd AS ga4_event_ecommerce_purchase_revenue_in_usd
     FROM
-        {{ source('dbt_package_ga4', 'events') }} AS events
+        {{ ref('src_ga4__events') }} AS events
     WHERE
-        _TABLE_SUFFIX NOT LIKE '%intraday%'
-        AND PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) > DATE_SUB(DATE(CURRENT_DATE()), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL') }} DAY)
-        AND events.stream_id IN UNNEST({{ env_var('DBT_PACKAGE_GA4__STREAM_ID') }})
-
-    {% if is_incremental() %}
-    {% set max_partition_date = macro__get_max_partition_date(this.schema, this.table) %}
-        AND PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
-    {% endif %}
+        TABLE_SUFFIX NOT LIKE '%intraday%'
+    {%- if not is_incremental() %}
+        AND PARSE_DATE('%Y%m%d', TABLE_SUFFIX) > DATE_SUB(DATE(CURRENT_DATE()), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL') }} DAY)
+    {%- endif %}
+
+    {%- if is_incremental() %}
+    {%- set max_partition_date = macro__get_max_partition_date(this.schema, this.table) %}
+        AND PARSE_DATE('%Y%m%d', TABLE_SUFFIX) > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
+    {%- endif %}
 ),
 
 t2 AS (
@@ -64,7 +65,7 @@ final AS (
 
 SELECT * FROM final
 
-    {% if is_incremental() %}
+    {%- if is_incremental() %}
     WHERE
         final.ga4_date_partition > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
-    {% endif %}
+    {%- endif %}
diff --git a/models/anchor/ga4_event/attr__ga4_event__ga4_event_ecommerce_refund_value.sql b/models/anchor/ga4_event/attr__ga4_event__ga4_event_ecommerce_refund_value.sql
@@ -1,4 +1,4 @@
-{{
+{{-
     config(
         enabled = env_var('DBT_PACKAGE_GA4__ENABLE__ANCHOR', 'true') == 'true',
         tags = ['dbt_package_ga4', 'anchor'],
@@ -11,12 +11,12 @@
         },
         cluster_by = 'ga4_event_id'
     )
-}}
+-}}
 
 
 WITH t1 AS (
     SELECT DISTINCT
-        PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) AS ga4_date_partition,
+        PARSE_DATE('%Y%m%d', TABLE_SUFFIX) AS ga4_date_partition,
         TO_HEX(
             MD5(
                 CONCAT(
@@ -28,16 +28,17 @@ WITH t1 AS (
             ) AS ga4_event_id,
         events.ecommerce.refund_value AS ga4_event_ecommerce_refund_value
     FROM
-        {{ source('dbt_package_ga4', 'events') }} AS events
+        {{ ref('src_ga4__events') }} AS events
     WHERE
-        _TABLE_SUFFIX NOT LIKE '%intraday%'
-        AND PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) > DATE_SUB(DATE(CURRENT_DATE()), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL') }} DAY)
-        AND events.stream_id IN UNNEST({{ env_var('DBT_PACKAGE_GA4__STREAM_ID') }})
-
-    {% if is_incremental() %}
-    {% set max_partition_date = macro__get_max_partition_date(this.schema, this.table) %}
-        AND PARSE_DATE('%Y%m%d', _TABLE_SUFFIX) > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
-    {% endif %}
+        TABLE_SUFFIX NOT LIKE '%intraday%'
+    {%- if not is_incremental() %}
+        AND PARSE_DATE('%Y%m%d', TABLE_SUFFIX) > DATE_SUB(DATE(CURRENT_DATE()), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL') }} DAY)
+    {%- endif %}
+
+    {%- if is_incremental() %}
+    {%- set max_partition_date = macro__get_max_partition_date(this.schema, this.table) %}
+        AND PARSE_DATE('%Y%m%d', TABLE_SUFFIX) > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
+    {%- endif %}
 ),
 
 t2 AS (
@@ -64,7 +65,7 @@ final AS (
 
 SELECT * FROM final
 
-    {% if is_incremental() %}
+    {%- if is_incremental() %}
     WHERE
         final.ga4_date_partition > DATE_SUB(DATE('{{ max_partition_date }}'), INTERVAL {{ env_var('DBT_PACKAGE_GA4__INTERVAL_INCREMENTAL') }} DAY)
-    {% endif %}
+    {%- endif %}