Merge pull request #115 from fivetran/bugfix/duplicate-columns

Bugfix/duplicate columns

CHANGELOG.md

@@ -1,3 +1,27 @@
+# dbt_hubspot_source v0.12.0
+## 🚨 Breaking Changes 🚨
+- The following models now use a custom macro to remove the property_hs_ prefix in staging columns, while also preventing duplicates. If de-prefixed columns match existing ones (e.g., `property_hs_meeting_outcome` vs. `meeting_outcome`), the macro favors the `property_hs_`field, aligning with the latest HubSpot API update. ([PR #115](https://github.com/fivetran/dbt_hubspot_source/pull/115))
+  - `stg_hubspot__engagement_call`
+  - `stg_hubspot__engagement_company`
+  - `stg_hubspot__engagement_contact`
+  - `stg_hubspot__engagement_deal`
+  - `stg_hubspot__engagement_email`
+  - `stg_hubspot__engagement_meeting`
+  - `stg_hubspot__engagement_note`
+  - `stg_hubspot__engagement_task`
+  - `stg_hubspot__ticket`
+  - `stg_hubspot__ticket_company`
+  - `stg_hubspot__ticket_contact`
+  - `stg_hubspot__ticket_deal`
+  - `stg_hubspot__ticket_engagement`
+  - `stg_hubspot__ticket_property_history`
+
+## Feature Updates
+- A new macro `remove_duplicate_and_prefix_from_columns` has been included which expands off the `fivetran_utils.remove_prefix_columns` macro by removing any duplicate columns that result from the prefix removal. ([PR #115](https://github.com/fivetran/dbt_hubspot_source/pull/115) and [PR #114](https://github.com/fivetran/dbt_hubspot_source/pull/114))
+
+## Contributors
+- [@greg-finley](https://github.com/greg-finley) ([PR #114](https://github.com/fivetran/dbt_hubspot_source/pull/114))
+
 # dbt_hubspot_source v0.11.0
 [PR #112](https://github.com/fivetran/dbt_hubspot_source/pull/112) includes the following updates:
 ## 🚨 Breaking Changes 🚨

README.md

@@ -44,7 +44,7 @@ Include the following hubspot_source package version in your `packages.yml` file
 ```yaml
 packages:
   - package: fivetran/hubspot_source
-    version: [">=0.11.0", "<0.12.0"]
+    version: [">=0.12.0", "<0.13.0"]
 ```
 ## Step 3: Define database and schema variables
 By default, this package runs using your destination and the `hubspot` schema. If this is not where your HubSpot data is (for example, if your HubSpot schema is named `hubspot_fivetran`), add the following configuration to your root `dbt_project.yml` file:

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'hubspot_source'
-version: '0.11.0'
+version: '0.12.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

integration_tests/.gitignore

@@ -2,3 +2,4 @@
 target/
 dbt_modules/
 logs/
+env/

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'hubspot_source_integration_tests'
-version: '0.11.0'
+version: '0.12.0'
 profile: 'integration_tests'
 config-version: 2
 models:

integration_tests/seeds/engagement_task_data.csv

@@ -1,2 +1,3 @@
-engagement_id,_fivetran_deleted,_fivetran_synced,property_hs_createdate,property_hs_object_id,property_hs_task_type,property_hs_timestamp,property_hubspot_owner_id,property_hubspot_team_id,type
-32034933897,false,2023-06-08 23:22:51.021000,2023-03-03 10:40:09.556000,32034933897,TODO,2023-03-08 02:30:00.000000,32779820,9747038,TASK
+engagement_id,_fivetran_deleted,_fivetran_synced,property_hs_createdate,property_hs_object_id,property_hs_task_type,property_hs_timestamp,property_hubspot_owner_id,property_hubspot_team_id,type,task_type,property_hs_engagement_source
+32034933897,false,2023-06-08 23:22:51.021000,2023-03-03 10:40:09.556000,32034933897,TODO,2023-03-08 02:30:00.000000,32779820,9747038,TASK,,Hello
+32034933899,false,2023-06-08 23:22:51.021000,2023-03-03 10:40:09.556000,32034933897,,2023-03-08 02:30:00.000000,32779820,,,TODO,

macros/remove_duplicate_and_prefix_from_columns.sql

@@ -0,0 +1,24 @@
+{% macro remove_duplicate_and_prefix_from_columns(columns, prefix='', exclude=[]) %}
+
+        {# Loop once to find duplicates #}
+        {%- set duplicate_exclude = [] -%}
+        {% for col in columns if col.name not in exclude %}
+        {%- for dupe in columns if col.name[prefix|length:]|lower == dupe.name|lower -%}
+        {%- do duplicate_exclude.append(col.name) -%}
+        {%- do duplicate_exclude.append(dupe.name) -%}
+        , {{ col.name }} as {{ col.name[prefix|length:] }}
+        {%- endfor %}
+        {% endfor %}
+
+        {# Loop again to find non-duplicates #}
+        {% for col in columns if col.name not in exclude %}
+        {%- if col.name|lower not in duplicate_exclude|lower -%}
+        {% if col.name[:prefix|length]|lower == prefix %}
+        , {{ col.name }} as {{ col.name[prefix|length:] }}
+        {%- else %}
+        , {{ col.name }}
+        {%- endif -%}
+        {%- endif -%}
+        {% endfor %}
+
+{% endmacro %}

models/stg_hubspot__engagement_call.sql

@@ -9,15 +9,15 @@ with base as (
 
     select
         {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__engagement_call_tmp')) %}
-        {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+        {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
             prefix='property_hs_',exclude=get_macro_columns(get_engagement_call_columns())) %}
         {{
             fivetran_utils.fill_staging_columns(source_columns=default_cols,
                 staging_columns=get_engagement_call_columns()
             )
         }}
         {% if new_cols | length > 0 %} 
-            ,{{ new_cols }} 
+            {{ new_cols }} 
         {% endif %}
     from base
 

models/stg_hubspot__engagement_company.sql

@@ -9,15 +9,15 @@ with base as (
 
     select 
         {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__engagement_company_tmp')) %}
-        {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+        {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
             prefix='property_hs_',exclude=get_macro_columns(get_engagement_company_columns())) %}
         {{
             fivetran_utils.fill_staging_columns(source_columns=default_cols,
                 staging_columns=get_engagement_company_columns()
             )
         }}
         {% if new_cols | length > 0 %} 
-            ,{{ new_cols }} 
+            {{ new_cols }} 
         {% endif %}
     from base
 )

models/stg_hubspot__engagement_contact.sql

@@ -9,15 +9,15 @@ with base as (
 
     select
         {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__engagement_contact_tmp')) %}
-        {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+        {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
             prefix='property_hs_',exclude=get_macro_columns(get_engagement_contact_columns())) %}
         {{
             fivetran_utils.fill_staging_columns(source_columns=default_cols,
                 staging_columns=get_engagement_contact_columns()
             )
         }}
         {% if new_cols | length > 0 %} 
-            ,{{ new_cols }} 
+            {{ new_cols }} 
         {% endif %}
     from base
 

models/stg_hubspot__engagement_deal.sql

@@ -9,15 +9,15 @@ with base as (
 
     select
         {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__engagement_deal_tmp')) %}
-        {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+        {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
             prefix='property_hs_',exclude=get_macro_columns(get_engagement_deal_columns())) %}
         {{
             fivetran_utils.fill_staging_columns(source_columns=default_cols,
                 staging_columns=get_engagement_deal_columns()
             )
         }}
         {% if new_cols | length > 0 %} 
-            ,{{ new_cols }} 
+            {{ new_cols }} 
         {% endif %}
     from base
 

models/stg_hubspot__engagement_email.sql

@@ -9,15 +9,15 @@ with base as (
 
     select
         {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__engagement_email_tmp')) %}
-        {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+        {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
             prefix='property_hs_',exclude=get_macro_columns(get_engagement_email_columns())) %}
         {{
             fivetran_utils.fill_staging_columns(source_columns=default_cols,
                 staging_columns=get_engagement_email_columns()
             )
         }}
         {% if new_cols | length > 0 %} 
-            ,{{ new_cols }} 
+            {{ new_cols }} 
         {% endif %}
     from base
 

models/stg_hubspot__engagement_meeting.sql

@@ -9,15 +9,15 @@ with base as (
 
     select
         {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__engagement_meeting_tmp')) %}
-        {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+        {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
             prefix='property_hs_',exclude=get_macro_columns(get_engagement_meeting_columns())) %}
         {{
             fivetran_utils.fill_staging_columns(source_columns=default_cols,
                 staging_columns=get_engagement_meeting_columns()
             )
         }}
         {% if new_cols | length > 0 %} 
-            ,{{ new_cols }} 
+            {{ new_cols }} 
         {% endif %}
     from base
 

models/stg_hubspot__engagement_note.sql

@@ -9,15 +9,15 @@ with base as (
 
     select
         {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__engagement_note_tmp')) %}
-        {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+        {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
             prefix='property_hs_',exclude=get_macro_columns(get_engagement_note_columns())) %}
         {{
             fivetran_utils.fill_staging_columns(source_columns=default_cols,
                 staging_columns=get_engagement_note_columns()
             )
         }}
         {% if new_cols | length > 0 %} 
-            ,{{ new_cols }} 
+            {{ new_cols }} 
         {% endif %}
     from base
 

models/stg_hubspot__engagement_task.sql

@@ -9,15 +9,15 @@ with base as (
 
     select
         {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__engagement_task_tmp')) %}
-        {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+        {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
             prefix='property_hs_',exclude=get_macro_columns(get_engagement_task_columns())) %}
         {{
             fivetran_utils.fill_staging_columns(source_columns=default_cols,
                 staging_columns=get_engagement_task_columns()
             )
         }}
         {% if new_cols | length > 0 %} 
-            ,{{ new_cols }} 
+            {{ new_cols }} 
         {% endif %}
     from base
 )

models/stg_hubspot__ticket.sql

@@ -30,7 +30,7 @@ with base as (
         {% if all_passthrough_column_check('stg_hubspot__ticket_tmp',get_ticket_columns()) > 0 %}
         -- just pass everything through if extra columns are present, but ensure required columns are present.
         ,{{  
-            fivetran_utils.remove_prefix_from_columns(
+            remove_duplicate_and_prefix_from_columns(
                 columns=adapter.get_columns_in_relation(ref('stg_hubspot__ticket_tmp')), 
                 prefix='property_', exclude=get_macro_columns(get_ticket_columns())) 
         }}

models/stg_hubspot__ticket_company.sql

@@ -9,15 +9,15 @@ with base as (
 
     select
         {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__ticket_company_tmp')) %}
-        {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+        {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
             prefix='property_hs_',exclude=get_macro_columns(get_ticket_company_columns())) %}
         {{
             fivetran_utils.fill_staging_columns(source_columns=default_cols,
                 staging_columns=get_ticket_company_columns()
             )
         }}
         {% if new_cols | length > 0 %} 
-            ,{{ new_cols }} 
+            {{ new_cols }} 
         {% endif %}
     from base
 

models/stg_hubspot__ticket_contact.sql

@@ -9,15 +9,15 @@ with base as (
 
     select
         {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__ticket_contact_tmp')) %}
-        {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+        {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
             prefix='property_hs_',exclude=get_macro_columns(get_ticket_contact_columns())) %}
         {{
             fivetran_utils.fill_staging_columns(source_columns=default_cols,
                 staging_columns=get_ticket_contact_columns()
             )
         }}
         {% if new_cols | length > 0 %} 
-            ,{{ new_cols }} 
+            {{ new_cols }} 
         {% endif %}
     from base
 

models/stg_hubspot__ticket_deal.sql

@@ -9,15 +9,15 @@ with base as (
 
     select
         {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__ticket_deal_tmp')) %}
-        {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+        {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
             prefix='property_hs_',exclude=get_macro_columns(get_ticket_deal_columns())) %}
         {{
             fivetran_utils.fill_staging_columns(source_columns=default_cols,
                 staging_columns=get_ticket_deal_columns()
             )
         }}
         {% if new_cols | length > 0 %} 
-            ,{{ new_cols }} 
+            {{ new_cols }} 
         {% endif %}
     from base
 

models/stg_hubspot__ticket_engagement.sql

@@ -9,15 +9,15 @@ with base as (
 
     select
         {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__ticket_engagement_tmp')) %}
-        {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+        {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
             prefix='property_hs_',exclude=get_macro_columns(get_ticket_engagement_columns())) %}
         {{
             fivetran_utils.fill_staging_columns(source_columns=default_cols,
                 staging_columns=get_ticket_engagement_columns()
             )
         }}
         {% if new_cols | length > 0 %} 
-            ,{{ new_cols }} 
+            {{ new_cols }} 
         {% endif %}
     from base
 

models/stg_hubspot__ticket_property_history.sql

@@ -9,15 +9,15 @@ with base as (
 
     select
         {% set default_cols = adapter.get_columns_in_relation(ref('stg_hubspot__ticket_property_history_tmp')) %}
-        {% set new_cols = fivetran_utils.remove_prefix_from_columns(columns=default_cols, 
+        {% set new_cols = remove_duplicate_and_prefix_from_columns(columns=default_cols, 
             prefix='property_hs_',exclude=get_macro_columns(get_ticket_property_history_columns())) %}
         {{
             fivetran_utils.fill_staging_columns(source_columns=default_cols,
                 staging_columns=get_ticket_property_history_columns()
             )
         }}
         {% if new_cols | length > 0 %} 
-            ,{{ new_cols }} 
+            {{ new_cols }} 
         {% endif %}
         from base