Merge pull request #96 from fivetran/feature/include-deleted-records

updating models for deleted records

CHANGELOG.md

@@ -1,3 +1,26 @@
+# dbt_hubspot_source v0.8.0
+
+## 🚨 Breaking Changes 🚨:
+[PR #96](https://github.com/fivetran/dbt_hubspot_source/pull/96) incorporates the following updates: 
+- The `is_deleted` field has been renamed within the below models:
+  - `stg_hubspot__company` ( `is_company_deleted`)
+  - `stg_hubspot__deal` (`is_deal_deleted`)
+  - `stg_hubspot__ticket` (`is_ticket_deleted`)
+
+- The `_fivetran_deleted` field has been renamed within the below models:
+  - `stg_husbpot__contact_list_member` (`is_contact_list_member_deleted`)
+  - `stg_hubspot__contact_list` (`is_contact_list_deleted`)
+  - `stg_hubspot__contact` (`is_contact_deleted`)
+  - `stg_hubspot__deal_pipeline_stage` (`is_deal_pipeline_stage_deleted`)
+  - `stg_hubspot__deal_pipeline` (`is_deal_pipeline_deleted`)
+  - `stg_hubspot__ticket_pipeline_stage` (`is_ticket_pipeline_stage_deleted`)
+  - `stg_hubspot__ticket_pipeline` (`is_ticket_pipeline_deleted`)
+
+- Filtering deleted records with the methods `where not coalesce(is_deleted, false)` or `where not coalesce(_fivetran_deleted, false)` has been removed from the above models. Instead, the new `is_<model>_deleted` flags will now offer customers flexibility to filter models for deleted records as necessary.
+
+## Under the Hood
+- `stg*.yml` documentation has been updated such that all `is_<model>_deleted` fields point to the `is_deleted` definition rather than respective `is_deleted` and `_fivetran_deleted` since the two fields are equivalent and in order to maintain consistency. ([PR #96](https://github.com/fivetran/dbt_hubspot_source/pull/96)).
+
 # dbt_hubspot_source v0.7.0
 ## 🚨 Breaking Changes 🚨:
 [PR #89](https://github.com/fivetran/dbt_hubspot_source/pull/89/files) includes the following breaking changes:

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'hubspot_source'
-version: '0.7.0'
+version: '0.8.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: hubspot_source_integration_tests
+      schema: hubspot_source_integration_tests_1
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: hubspot_source_integration_tests
+      schema: hubspot_source_integration_tests_1
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: hubspot_source_integration_tests
+      schema: hubspot_source_integration_tests_1
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: hubspot_source_integration_tests
+      schema: hubspot_source_integration_tests_1
       threads: 8
     databricks:
       catalog: null
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: hubspot_source_integration_tests
+      schema: hubspot_source_integration_tests_1
       threads: 2
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,12 +1,12 @@
 name: 'hubspot_source_integration_tests'
-version: '0.7.0'
+version: '0.8.0'
 profile: 'integration_tests'
 config-version: 2
 models:
   hubspot_source:
     +schema:
 vars:
-  hubspot_schema: hubspot_source_integration_tests
+  hubspot_schema: hubspot_source_integration_tests_1
   hubspot_source:
     hubspot_service_enabled: true
     hubspot_company_property_history_identifier: "company_property_history_data"

models/docs.md

@@ -3,15 +3,15 @@ Timestamp of when Fivetran synced a record.
 {% enddocs %}
 
 {% docs _fivetran_deleted %}
-Timestamp of when Fivetran marked a record as deleted.
+Boolean indicating whether a record has been deleted in Hubspot and/or inferred deleted in Hubspot by Fivetran; _fivetran_deleted and is_deleted fields are equivalent. 
 {% enddocs %}
 
 {% docs portal_id %}
 The hub ID.
 {% enddocs %}
 
 {% docs is_deleted %}
-Whether the record has been deleted in Hubspot.
+Boolean indicating whether a record has been deleted in Hubspot and/or inferred deleted in Hubspot by Fivetran; _fivetran_deleted and is_deleted fields are equivalent.
 {% enddocs %}
 
 {% docs history_name %}

models/stg_hubspot__company.sql

@@ -20,13 +20,14 @@ with base as (
 
     select
         id as company_id,
+        is_deleted as is_company_deleted,
 
 {% if var('hubspot__pass_through_all_columns', false) %}
         -- just pass everything through
         {{ 
             fivetran_utils.remove_prefix_from_columns(
                 columns=adapter.get_columns_in_relation(ref('stg_hubspot__company_tmp')), 
-                prefix='property_', exclude=['id']
+                prefix='property_', exclude=['id', 'is_deleted']
             ) 
         }}
 
@@ -35,7 +36,6 @@ with base as (
 {% else %}
         -- just default columns + explicitly configured passthrough columns
         _fivetran_synced,
-        is_deleted,
         property_name as company_name,
         property_description as description,
         property_createdate as created_at,
@@ -59,5 +59,4 @@ with base as (
 )
 
 select *
-from fields
-where not coalesce(is_deleted, false) 
+from fields

models/stg_hubspot__company.yml

@@ -32,7 +32,7 @@ models:
       - name: _fivetran_synced
         description: '{{ doc("_fivetran_synced") }}'
 
-      - name: is_deleted
+      - name: is_company_deleted
         description: '{{ doc("is_deleted") }}'
 
       - name: company_id

models/stg_hubspot__contact.sql

@@ -20,13 +20,14 @@ with base as (
 
     select
         id as contact_id,
+        _fivetran_deleted as is_contact_deleted,
 
 {% if var('hubspot__pass_through_all_columns', false) %}
         -- just pass everything through
         {{ 
             fivetran_utils.remove_prefix_from_columns(
                 columns=adapter.get_columns_in_relation(ref('stg_hubspot__contact_tmp')), 
-                prefix='property_', exclude=['id', 'property_contact_id']) 
+                prefix='property_', exclude=['id', 'property_contact_id','_fivetran_deleted']) 
         }}
     from base
 
@@ -39,7 +40,6 @@ with base as (
         property_createdate as created_at,
         property_jobtitle as job_title,
         property_annualrevenue as company_annual_revenue,
-        _fivetran_deleted,
         _fivetran_synced
 
         --The below macro adds the fields defined within your hubspot__contact_pass_through_columns variable into the staging model
@@ -54,5 +54,4 @@ with base as (
 )
 
 select *
-from fields
-where not coalesce(_fivetran_deleted, false) 
+from fields

models/stg_hubspot__contact.yml

@@ -4,8 +4,8 @@ models:
   - name: stg_hubspot__contact_list_member
     description: Each record represents a 'link' between a contact and a contact list.
     columns:
-      - name: _fivetran_deleted
-        description: '{{ doc("_fivetran_deleted") }}'
+      - name: is_contact_list_member_deleted
+        description: '{{ doc("is_deleted") }}'
 
       - name: _fivetran_synced
         description: '{{ doc("_fivetran_synced") }}'
@@ -22,11 +22,11 @@ models:
   - name: stg_hubspot__contact_list
     description: Each record represents a contact list in Hubspot.
     columns:
-      - name: _fivetran_deleted
-        description: '{{ doc("_fivetran_deleted") }}'
+      - name: is_contact_list_deleted
+        description: '{{ doc("is_deleted") }}'
 
       - name: _fivetran_synced
-        description: '{{ doc("_fivetran_deleted") }}'
+        description: '{{ doc("_fivetran_synced") }}'
 
       - name: contact_list_id
         description: The ID of the contact list.
@@ -120,8 +120,8 @@ models:
   - name: stg_hubspot__contact
     description: Each record represents a contact in Hubspot.
     columns:
-      - name: _fivetran_deleted
-        description: '{{ doc("_fivetran_deleted") }}'
+      - name: is_contact_deleted
+        description: '{{ doc("is_deleted") }}'
 
       - name: _fivetran_synced
         description: '{{ doc("_fivetran_synced") }}'

models/stg_hubspot__contact_list.sql

@@ -19,7 +19,7 @@ with base as (
 ), fields as (
 
     select
-        _fivetran_deleted,
+        _fivetran_deleted as is_contact_list_deleted,
         _fivetran_synced,
         created_at as created_timestamp,
         deleteable as is_deletable,
@@ -38,5 +38,4 @@ with base as (
 )
 
 select *
-from fields
-where not coalesce(_fivetran_deleted, false) 
+from fields

models/stg_hubspot__contact_list_member.sql

@@ -19,7 +19,7 @@ with base as (
 ), fields as (
 
     select
-        _fivetran_deleted,
+        _fivetran_deleted as is_contact_list_member_deleted,
         _fivetran_synced,
         added_at as added_timestamp,
         contact_id,
@@ -29,5 +29,4 @@ with base as (
 )
 
 select *
-from fields
-where not coalesce(_fivetran_deleted, false)
+from fields

models/stg_hubspot__deal.sql

@@ -22,21 +22,21 @@ with base as (
         property_dealname as deal_name,
         property_closedate as closed_at,
         property_createdate as created_at,
+        is_deleted as is_deal_deleted,
 
 {% if var('hubspot__pass_through_all_columns', false) %}
         -- just pass everything else through
         {{ 
             fivetran_utils.remove_prefix_from_columns(
                 columns=adapter.get_columns_in_relation(ref('stg_hubspot__deal_tmp')), 
                 prefix='property_',
-                exclude=['property_dealname','property_closedate','property_createdate']) 
+                exclude=['property_dealname','property_closedate','property_createdate', 'is_deleted']) 
         }}
     from base
 
 {% else %}
         -- just default columns + explicitly configured passthrough columns
         _fivetran_synced,
-        is_deleted,
         deal_id,
         cast(deal_pipeline_id as {{ dbt.type_string() }}) as deal_pipeline_id,
         cast(deal_pipeline_stage_id as {{ dbt.type_string() }}) as deal_pipeline_stage_id,
@@ -57,5 +57,4 @@ with base as (
 )
 
 select *
-from fields
-where not coalesce(is_deleted, false)
+from fields

models/stg_hubspot__deal.yml

@@ -5,8 +5,8 @@ models:
   - name: stg_hubspot__deal_pipeline_stage
     description: Each record represents a pipeline stage in Hubspot.
     columns:
-      - name: _fivetran_deleted
-        description: '{{ doc("_fivetran_deleted") }}'
+      - name: is_deal_pipeline_stage_deleted
+        description: '{{ doc("is_deleted") }}'
 
       - name: _fivetran_synced
         description: '{{ doc("_fivetran_synced") }}'
@@ -38,8 +38,8 @@ models:
   - name: stg_hubspot__deal_pipeline
     description: Each record represents a pipeline in Hubspot.
     columns:
-      - name: _fivetran_deleted
-        description: '{{ doc("_fivetran_deleted") }}'
+      - name: is_deal_pipeline_deleted
+        description: '{{ doc("is_deleted") }}'
 
       - name: _fivetran_synced
         description: '{{ doc("_fivetran_synced") }}'
@@ -62,8 +62,8 @@ models:
   - name: stg_hubspot__deal
     description: Each record represents a deal in Hubspot.
     columns:
-      - name: _fivetran_deleted
-        description: '{{ doc("_fivetran_deleted") }}'
+      - name: is_deal_deleted
+        description: '{{ doc("is_deleted") }}'
 
       - name: _fivetran_synced
         description: '{{ doc("_fivetran_synced") }}'

models/stg_hubspot__deal_pipeline.sql

@@ -19,7 +19,7 @@ with base as (
 ), fields as (
 
     select
-        _fivetran_deleted,
+        _fivetran_deleted as is_deal_pipeline_deleted,
         _fivetran_synced,
         active as is_active,
         display_order,
@@ -31,5 +31,3 @@ with base as (
 
 select *
 from fields
-where not coalesce(_fivetran_deleted, false) 
-

models/stg_hubspot__deal_pipeline_stage.sql

@@ -19,7 +19,7 @@ with base as (
 ), fields as (
 
     select
-        _fivetran_deleted,
+        _fivetran_deleted as is_deal_pipeline_stage_deleted,
         _fivetran_synced,
         active as is_active,
         closed_won as is_closed_won,
@@ -33,5 +33,4 @@ with base as (
 )
 
 select *
-from fields
-where not coalesce(_fivetran_deleted, false) 
+from fields

models/stg_hubspot__ticket.sql

@@ -20,20 +20,20 @@ with base as (
 
     select
         id as ticket_id,
+        is_deleted as is_ticket_deleted,
 
 {% if var('hubspot__pass_through_all_columns', false) %}
         -- just pass everything through
         {{ 
             fivetran_utils.remove_prefix_from_columns(
                 columns=adapter.get_columns_in_relation(ref('stg_hubspot__ticket_tmp')), 
-                prefix='property_', exclude=['id']) 
+                prefix='property_', exclude=['id', 'is_deleted']) 
         }}
     from base
 
 {% else %}
         -- just default columns + explicitly configured passthrough columns
         _fivetran_synced,
-        is_deleted,
         property_closed_date as closed_at,
         property_createdate as created_at,
         property_first_agent_reply_date as first_agent_reply_at,
@@ -57,5 +57,4 @@ with base as (
 )
 
 select *
-from fields
-where not coalesce(is_deleted, false) 
+from fields