Merge pull request #48 from fivetran/feature/all-passthru

Feature/all passthru

README.md

@@ -1,3 +1,4 @@
+[![Apache License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) ![dbt logo and version](https://img.shields.io/static/v1?logo=dbt&label=dbt-version&message=0.20.x&color=orange)
 # Hubspot (Source)
 
 This package models Hubspot data from [Fivetran's connector](https://fivetran.com/docs/applications/hubspot). It uses data in the format described by the [marketing](https://fivetran.com/docs/applications/hubspot#schemainformation), [sales](https://fivetran.com/docs/applications/hubspot#crmandsaleshubschema) and [service](https://fivetran.com/docs/applications/hubspot#servicehubschema) ERDs.
@@ -130,6 +131,16 @@ vars:
 
 ```
 
+**Alternatively**, if you would like to simply pass through **all columns** in the above four tables, add the following configuration to your dbt_project.yml. Note that this will override any `hubspot__[table_name]_pass_through_columns` variables.
+
+```yml
+# dbt_project.yml
+
+...
+vars:
+  hubspot__pass_through_all_columns: true # default is false
+```
+
 ### Changing the Build Schema
 By default this package will build the HubSpot staging models within a schema titled (<target_schema> + `_stg_hubspot`). If this is not where you would like your HubSpot staging models to be written to, add the following configuration to your `dbt_project.yml` file:
 

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'hubspot_source'
-version: '0.4.0'
+version: '0.4.1'
 config-version: 2
 require-dbt-version: ">=0.20.0"
 
@@ -66,3 +66,6 @@ vars:
     hubspot__contact_pass_through_columns: []
     hubspot__company_pass_through_columns: []
     hubspot__ticket_pass_through_columns: []
+
+    #OVERRIDE any ^ passthrough columns by just passing all columns
+    hubspot__pass_through_all_columns: false

integration_tests/dbt_project.yml

@@ -1,6 +1,6 @@
 
 name: 'hubspot_source_integration_tests'
-version: '0.4.0'
+version: '0.4.1'
 profile: 'integration_tests'
 config-version: 2
 

models/stg_hubspot__company.sql

@@ -20,7 +20,21 @@ with base as (
 ), fields as (
 
     select
-        id as company_id, 
+        id as company_id,
+
+{% if var('hubspot__pass_through_all_columns', false) %}
+        -- just pass everything through
+        {{ 
+            fivetran_utils.remove_prefix_from_columns(
+                columns=adapter.get_columns_in_relation(ref('stg_hubspot__company_tmp')), 
+                prefix='property_', exclude=['id']
+            ) 
+        }}
+
+    from base
+
+{% else %}
+        -- just default columns + explicitly configured passthrough columns
         _fivetran_synced,
         property_name as company_name,
         property_description as description,
@@ -37,7 +51,8 @@ with base as (
         {{ fivetran_utils.fill_pass_through_columns('hubspot__company_pass_through_columns') }}
 
     from macro
-
+
+{% endif %}
 )
 
 select *

models/stg_hubspot__company.yml

@@ -34,12 +34,6 @@ models:
         tests:
           - unique
           - not_null
-
-      - name: portal_id
-        description: '{{ doc("portal_id") }}'
-
-      - name: is_deleted
-        description: '{{ doc("is_deleted") }}'
 
       - name: company_name
         description: The name of the company.

models/stg_hubspot__contact.sql

@@ -20,21 +20,34 @@ with base as (
 ), fields as (
 
     select
-        _fivetran_deleted,
-        _fivetran_synced,
-        property_email as email,
         id as contact_id,
+
+{% if var('hubspot__pass_through_all_columns', false) %}
+        -- just pass everything through
+        {{ 
+            fivetran_utils.remove_prefix_from_columns(
+                columns=adapter.get_columns_in_relation(ref('stg_hubspot__contact_tmp')), 
+                prefix='property_', exclude=['id', 'property_contact_id']) 
+        }}
+    from base
+
+{% else %}
+        -- just default columns + explicitly configured passthrough columns
+        property_email as email,
         property_company as contact_company,
         property_firstname as first_name,
         property_lastname as last_name,
         property_createdate as created_at,
         property_jobtitle as job_title,
-        property_annualrevenue as company_annual_revenue
+        property_annualrevenue as company_annual_revenue,
+        _fivetran_deleted,
+        _fivetran_synced
 
         --The below macro adds the fields defined within your hubspot__contact_pass_through_columns variable into the staging model
         {{ fivetran_utils.fill_pass_through_columns('hubspot__contact_pass_through_columns') }}
 
     from macro
+{% endif %}    
 
 )
 

models/stg_hubspot__deal.sql

@@ -20,6 +20,18 @@ with base as (
 ), fields as (
 
     select
+
+{% if var('hubspot__pass_through_all_columns', false) %}
+        -- just pass everything through
+        {{ 
+            fivetran_utils.remove_prefix_from_columns(
+                columns=adapter.get_columns_in_relation(ref('stg_hubspot__deal_tmp')), 
+                prefix='property_') 
+        }}
+    from base
+
+{% else %}
+        -- just default columns + explicitly configured passthrough columns
         _fivetran_synced,
         deal_id,
         deal_pipeline_id,
@@ -36,6 +48,7 @@ with base as (
         {{ fivetran_utils.fill_pass_through_columns('hubspot__deal_pass_through_columns') }}
 
     from macro
+{% endif %}
 
 )
 

models/stg_hubspot__deal.yml

@@ -68,9 +68,6 @@ models:
           - not_null
           - unique
 
-      - name: is_deleted
-        description: Whether the record was deleted.
-
       - name: portal_id
         description: '{{ doc("portal_id") }}'
 

models/stg_hubspot__ticket.sql

@@ -21,6 +21,18 @@ with base as (
 
     select
         id as ticket_id,
+
+{% if var('hubspot__pass_through_all_columns', false) %}
+        -- just pass everything through
+        {{ 
+            fivetran_utils.remove_prefix_from_columns(
+                columns=adapter.get_columns_in_relation(ref('stg_hubspot__ticket_tmp')), 
+                prefix='property_', exclude=['id']) 
+        }}
+    from base
+
+{% else %}
+        -- just default columns + explicitly configured passthrough columns
         _fivetran_synced,
         is_deleted,
         property_closed_date as closed_at,
@@ -38,7 +50,8 @@ with base as (
         {{ fivetran_utils.fill_pass_through_columns('hubspot__ticket_pass_through_columns') }}
 
     from macro
-
+{% endif %}
+
 )
 
 select *

models/stg_hubspot__ticket.yml

@@ -86,15 +86,9 @@ models:
           - unique
           - not_null
 
-      - name: _fivetran_synced
-        description: '{{ doc("_fivetran_synced") }}'
-
       - name: is_deleted
         description: Whether the record was deleted.
 
-      - name: portal_id
-        description: '{{ doc("portal_id") }}'
-
       - name: closed_at
         description: The date the ticket was closed.