Merge pull request #135 from fivetran/merged_deals

Merged deals

.buildkite/scripts/run_models.sh

@@ -24,10 +24,10 @@ dbt run --target "$db" --vars '{hubspot_service_enabled: true}' --full-refresh
 dbt run --target "$db" --vars '{hubspot_service_enabled: true}'
 dbt test --target "$db"
 dbt run --vars '{hubspot_service_enabled: true, hubspot_marketing_enabled: true, hubspot_sales_enabled: false}' --target "$db" --full-refresh
-dbt run --vars '{hubspot_service_enabled: true, hubspot_marketing_enabled: true, hubspot_sales_enabled: false}' --target "$db"
+dbt run --vars '{hubspot_service_enabled: true, hubspot_marketing_enabled: true, hubspot_sales_enabled: false, hubspot_merged_deal_enabled: true}' --target "$db"
 dbt test --target "$db"
 dbt run --vars '{hubspot_marketing_enabled: true, hubspot_contact_merge_audit_enabled: true, hubspot_sales_enabled: false}' --target "$db" --full-refresh
-dbt run --vars '{hubspot_marketing_enabled: false, hubspot_sales_enabled: true, hubspot__pass_through_all_columns: true, hubspot_using_all_email_events: false, hubspot_owner_enabled: false}' --target "$db" --full-refresh
+dbt run --vars '{hubspot_marketing_enabled: false, hubspot_sales_enabled: true, hubspot_merged_deal_enabled: true, hubspot__pass_through_all_columns: true, hubspot_using_all_email_events: false, hubspot_owner_enabled: false}' --target "$db" --full-refresh
 dbt test --target "$db"
 
 dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"

.quickstart/quickstart.yml

@@ -0,0 +1,100 @@
+database_key: hubspot_database
+schema_key: hubspot_schema
+
+dbt_versions: ">=1.3.0 <2.0.0"
+
+table_variables:
+  hubspot_contact_enabled:
+    - contact
+  hubspot_contact_list_enabled:
+    - contact_list
+  hubspot_contact_list_member_enabled:
+    - contact_list_member
+  hubspot_contact_property_enabled:
+    - contact_property_history
+  hubspot_contact_property_history_enabled:
+    - contact_property_history
+  hubspot_email_event_enabled:
+    - email_campaign
+    - email_event
+  hubspot_email_event_bounce_enabled:
+    - email_event_bounce
+  hubspot_email_event_click_enabled:
+    - email_event_click
+  hubspot_email_event_deferred_enabled:
+    - email_event_deferred
+  hubspot_email_event_delivered_enabled:
+    - email_event_delivered
+  hubspot_email_event_dropped_enabled:
+    - email_event_dropped
+  hubspot_email_event_forward_enabled:
+    - email_event_forward
+  hubspot_email_event_open_enabled:
+    - email_event_open
+  hubspot_email_event_print_enabled:
+    - email_event_print
+  hubspot_email_event_sent_enabled:
+    - email_event_sent
+  hubspot_email_event_spam_report_enabled:
+    - email_event_spam_report
+  hubspot_email_event_status_change_enabled:
+    - email_event_status_change
+  hubspot_contact_merge_audit_enabled:
+    - contact_merge_audit
+  hubspot_company_enabled:
+    - company
+  hubspot_company_property_history_enabled:
+    - company_property_history
+  hubspot_deal_enabled:
+    - deal
+    - deal_stage
+    - deal_pipeline
+    - deal_pipeline_stage
+  hubspot_merged_deal_enabled:
+    - merged_deal
+  hubspot_deal_company_enabled:
+    - deal_company
+  hubspot_deal_contact_enabled:
+    - deal_contact
+  hubspot_deal_property_history_enabled:
+    - deal_property_history
+  hubspot_engagement_enabled:
+    - engagement
+  hubspot_engagement_contact_enabled:
+    - engagement_contact
+  hubspot_engagement_company_enabled:
+    - engagement_company
+  hubspot_engagement_deal_enabled:
+    - engagement_deal
+  hubspot_engagement_call_enabled:
+    - engagement_call
+  hubspot_engagement_email_enabled:
+    - engagement_email
+  hubspot_engagement_meeting_enabled:
+    - engagement_meeting
+  hubspot_engagement_note_enabled:
+    - engagement_note
+  hubspot_engagement_task_enabled:
+    - engagement_task
+  hubspot_owner_enabled:
+    - owner
+  hubspot_property_enabled:
+    - property
+  hubspot_property_enabled:
+    - property_option
+  hubspot_service_enabled: 
+    - ticket_company
+    - ticket_contact
+    - ticket_engagement
+    - ticket_pipeline_stage
+    - ticket_pipeline
+    - ticket_property_history
+    - ticket
+  hubspot_ticket_deal_enabled:
+    - ticket_deal
+
+destination_configurations:
+  databricks:
+    dispatch:
+      - macro_namespace: dbt_utils
+        search_order: [ 'spark_utils', 'dbt_utils' ]

CHANGELOG.md

@@ -1,3 +1,24 @@
+# dbt_hubspot v0.16.0
+[PR #135](https://github.com/fivetran/dbt_hubspot/pull/135) includes the following updates:
+
+## 🚨 Breaking Changes 🚨
+- Added unique tests on `event_id` for event-based models, `contact_id` for `hubspot__contacts`, `contact_list_id` for `hubspot__contact_lists`, `deal_id` for `hubspot__deals`, `deal_stage_id` for `hubspot__deal_stages`, and `company_id` for `hubspot__companies`, on the condition the titular fields have not been deleted. We would advise running through these to ensure they work successfully.
+
+## Features
+- Updates the `hubspot__deal_stages` and `hubspot__deals` models to remove stale deals that have been merged since into other deals. In addition we've introduced a new field `merged_deal_ids` that lists all historic merged deals for each deal. 
+   - **Please note** you must have the underlying `merged_deals` table to take advantage of the above mentioned merged deal update. This may be enabled by setting `hubspot_merged_deal_enabled` to True (by default this will be False). Also, `hubspot_sales_enabled` and `hubspot_deal_enabled` must not be set to False (by default these are True). See [Step 4 of the README](https://github.com/fivetran/dbt_hubspot#step-4-disable-models-for-non-existent-sources) for more details. 
+
+## Variable Bug Fixes
+- The following adjustments have been made concerning the `hubspot_contact_enabled` variable:
+  - The `email_events_joined()` macro now includes conditional logic to include contact information only if the `hubspot_contact_enabled` variable is defined as `true`.
+  - The `int_hubspot__email_metrics__by_contact_list` model is now dependent on the `hubspot_contact_enabled` variable being defined as `true`.
+  - The `hubspot__contact_lists` model now takes into consideration the `hubspot_contact_enabled` variable being defined as `true` before joining in email metric information to the contact lists data.
+- Modified the config enabled logic within the `hubspot__email_sends` model to be dependent on the `hubspot_email_event_sent_enabled` variable being `true` in addition to the `hubspot_marketing_enabled` and `hubspot_email_event_enabled` variables.
+
+## Under the Hood
+- Added quickstart.yml for Quickstart customers.
+- Added `not_null` tests that were previously missing to `deal_id` for `hubspot__deals` and `deal_stage_id` for `hubspot__deal_stages`.
+
 # dbt_hubspot v0.15.1
 [PR #129](https://github.com/fivetran/dbt_hubspot/pull/129) includes the following updates:
 

README.md

@@ -73,7 +73,7 @@ Include the following hubspot package version in your `packages.yml` file:
 ```yaml
 packages:
   - package: fivetran/hubspot
-    version: [">=0.15.0", "<0.16.0"] # we recommend using ranges to capture non-breaking changes automatically
+    version: [">=0.16.0", "<0.17.0"] # we recommend using ranges to capture non-breaking changes automatically
 
 ```
 Do **NOT** include the `hubspot_source` package in this file. The transformation package itself has a dependency on it and will install the source package as well.
@@ -96,7 +96,7 @@ vars:
 ```
 
 ## Step 4: Disable models for non-existent sources
-When setting up your Hubspot connection in Fivetran, it is possible that not every table this package expects will be synced. This can occur because you either don't use that functionality in Hubspot or have actively decided to not sync some tables. In order to disable the relevant functionality in the package, you will need to add the relevant variables. By default, all variables are assumed to be `true` (with exception of `hubspot_service_enabled`, `hubspot_ticket_deal_enabled`, and `hubspot_contact_merge_audit_enabled`). You only need to add variables within your root `dbt_project.yml` for the tables you would like to disable or enable respectively:
+When setting up your Hubspot connection in Fivetran, it is possible that not every table this package expects will be synced. This can occur because you either don't use that functionality in Hubspot or have actively decided to not sync some tables. In order to disable the relevant functionality in the package, you will need to add the relevant variables. By default, all variables are assumed to be `true` (with exception of `hubspot_service_enabled`, `hubspot_ticket_deal_enabled`, `hubspot_contact_merge_audit_enabled`, and `hubspot_merged_deal_enabled`). You only need to add variables within your root `dbt_project.yml` for the tables you would like to disable or enable respectively:
 
 ```yml
 vars:
@@ -131,6 +131,7 @@ vars:
   hubspot_company_enabled: false
   hubspot_company_property_history_enabled: false         # Disables the company property history models
   hubspot_deal_enabled: false
+  hubspot_merged_deal_enabled: true                       # Enables the merged_deal table, which will be used to filter out merged deals from the final deal models. False by default. Note that `hubspot_sales_enabled` and `hubspot_deal_enabled` must not be set to False.
   hubspot_deal_company_enabled: false
   hubspot_deal_contact_enabled: false
   hubspot_deal_property_history_enabled: false            # Disables the deal property history models

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'hubspot'
-version: '0.15.1'
+version: '0.16.0'
 
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
@@ -13,6 +13,7 @@ vars:
     company: "{{ ref('stg_hubspot__company') }}"
     company_property_history: "{{ ref('stg_hubspot__company_property_history') }}"
     deal: "{{ ref('stg_hubspot__deal') }}"
+    merged_deal: "{{ ref('stg_hubspot__merged_deal') }}"
     deal_stage: "{{ ref('stg_hubspot__deal_stage') }}"
     deal_company: "{{ ref('stg_hubspot__deal_company') }}"
     deal_pipeline: "{{ ref('stg_hubspot__deal_pipeline') }}"

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: hubspot_integration_tests_8
+      schema: hubspot_integration_tests_11
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: hubspot_integration_tests_8
+      schema: hubspot_integration_tests_11
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: hubspot_integration_tests_8
+      schema: hubspot_integration_tests_11
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: hubspot_integration_tests_8
+      schema: hubspot_integration_tests_11
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: hubspot_integration_tests_8
+      schema: hubspot_integration_tests_11
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,16 +1,17 @@
 name: 'hubspot_integration_tests'
-version: '0.15.1'
+version: '0.16.0'
 
 profile: 'integration_tests'
 config-version: 2
 vars:
-  hubspot_schema: hubspot_integration_tests_8
+  hubspot_schema: hubspot_integration_tests_11
   hubspot_source:
     hubspot_service_enabled: true
     # hubspot_sales_enabled: true # enable when generating docs
     # hubspot_marketing_enabled: true # enable when generating docs
     # hubspot_contact_merge_audit_enabled: true # enable when generating docs
     # hubspot_using_all_email_events: true # enable when generating docs
+    # hubspot_merged_deal_enabled: true # enable when generating docs  
     hubspot_company_property_history_identifier: "company_property_history_data"
     hubspot_company_identifier: "company_data"
     hubspot_contact_identifier: "contact_data"
@@ -58,6 +59,7 @@ vars:
     hubspot_contact_list_identifier: "contact_list_data"
     hubspot_email_event_sent_identifier: "email_event_sent_data"
     hubspot_email_event_dropped_identifier: "email_event_dropped_data"
+    hubspot_merged_deal_identifier: "merged_deal_data"
 
 seeds:
   hubspot_integration_tests:
@@ -173,6 +175,10 @@ seeds:
     ticket_property_history_data:
       +column_types:
         timestamp_instant: timestamp
+    merged_deal_data:
+      +column_types:
+        deal_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        merged_deal_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
 
 dispatch:
   - macro_namespace: dbt_utils

integration_tests/seeds/merged_deal_data.csv

@@ -0,0 +1,15 @@
+deal_id,merged_deal_id,_fivetran_synced
+9089908390,9014070905,2023-06-22 11:37:07.210000
+9089908390,9077310858,2023-06-22 11:37:07.210000
+9089908390,9077311778,2023-06-22 11:37:07.210000
+9089908390,9014071335,2023-06-22 11:37:07.210000
+1364146343,10829010820,2023-06-22 11:37:07.220000
+1364146343,10828874842,2023-06-22 11:37:07.220000
+1368273549,10829051678,2023-06-22 11:37:07.227000
+1368273549,10829028256,2023-06-22 11:37:07.227000
+1373595330,10879517198,2023-06-22 11:37:07.227000
+1373595330,10828874855,2023-06-22 11:37:07.227000
+1006463008,10064581040,2023-06-22 11:37:07.213000
+1006463008,9351825260,2023-06-22 11:37:07.213000
+1389291069,12351788138,2023-06-28 11:18:32.940000
+1389291069,13817889862,2023-06-28 11:18:32.939000

macros/email_events_joined.sql

@@ -10,10 +10,12 @@ with base as (
     select *
     from {{ var('email_event') }}
 
+{% if var('hubspot_contact_enabled', true) %}
 ), contacts as (
 
     select *
     from {{ ref('int_hubspot__contact_merge_adjust') }} 
+{% endif %}
 
 ), events_joined as (
 
@@ -28,6 +30,7 @@ with base as (
     left join events
         using (event_id)
 
+{% if var('hubspot_contact_enabled', true) %}
 ), contacts_joined as (
 
     select 
@@ -43,4 +46,11 @@ with base as (
 select *
 from contacts_joined
 
+{% else %}
+)
+
+select *
+from events_joined
+
+{% endif %}
 {% endmacro %}