Merge pull request #109 from fivetran/feature/add-typeid

Feature/add typeid

.buildkite/pipeline.yml

@@ -58,7 +58,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_models.sh redshift
 
-  - label: ":bricks: Run Tests - Databricks"
+  - label: ":databricks: Run Tests - Databricks"
     key: "run_dbt_databricks"
     plugins:
       - docker#v3.13.0:

CHANGELOG.md

@@ -1,3 +1,20 @@
+# dbt_hubspot_source v0.10.0
+## 🚨 Breaking Changes 🚨
+- In the [May 2023 connector update](https://fivetran.com/docs/applications/hubspot/changelog#may2023) `type_id` was added to sources `DEAL_COMPANY` and `DEAL_CONTACT` as a part of the composite primary key for these tables. This column has been adding to the corresponding staging models. ([PR #109](https://github.com/fivetran/dbt_hubspot_source/pull/109))
+- Updated tests for these tables with `type_id` as part of the primary key. ([PR #109](https://github.com/fivetran/dbt_hubspot_source/pull/109))
+- Also resulting from the connector update, columns `updated_at` and `created_at` have been added to the following sources and their corresponding staging models: ([PR #109](https://github.com/fivetran/dbt_hubspot_source/pull/109))
+  - `DEAL_PIPELINE`
+  - `DEAL_PIPELINE_STAGE`
+  - `TICKET_PIPELINE`
+  - `TICKET_PIPELINE_STAGE`
+- Updated docs with these changes and a little housekeeping. ([PR #109](https://github.com/fivetran/dbt_hubspot_source/pull/109))
+
+## Feature Updates
+- Updated README to include the variable `hubspot_owner_enabled`. ([PR #109](https://github.com/fivetran/dbt_hubspot_source/pull/109))
+
+## 🚘 Under the Hood
+- Updated seed data for testing newly added columns. ([PR #109](https://github.com/fivetran/dbt_hubspot_source/pull/109))
+
 # dbt_hubspot_source v0.9.1
 ## Feature Updates
 - A new variable was added `hubspot_using_all_email_events` to allow package users to remove filtered email events from the `stg_hubspot__email_event` staging model as well as the relevant downstream reporting models. This is crucial for HubSpot users who greatly take advantage of marking events as filtered in order to provide accurate reporting. ([PR #104](https://github.com/fivetran/dbt_hubspot_source/pull/104))

README.md

@@ -44,7 +44,7 @@ Include the following hubspot_source package version in your `packages.yml` file
 ```yaml
 packages:
   - package: fivetran/hubspot_source
-    version: [">=0.9.0", "<0.10.0"]
+    version: [">=0.10.0", "<0.11.0"]
 ```
 ## Step 3: Define database and schema variables
 By default, this package runs using your destination and the `hubspot` schema. If this is not where your HubSpot data is (for example, if your HubSpot schema is named `hubspot_fivetran`), add the following configuration to your root `dbt_project.yml` file:
@@ -100,6 +100,7 @@ vars:
   hubspot_engagement_meeting_enabled: false
   hubspot_engagement_note_enabled: false
   hubspot_engagement_task_enabled: false
+  hubspot_owner_enabled: false
 
   # Service
   hubspot_service_enabled: true                           # Enables all service models

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: hubspot_source_integration_tests_2
+      schema: hubspot_source_integration_tests_4
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: hubspot_source_integration_tests_2
+      schema: hubspot_source_integration_tests_4
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: hubspot_source_integration_tests_2
+      schema: hubspot_source_integration_tests_4
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: hubspot_source_integration_tests_2
+      schema: hubspot_source_integration_tests_4
       threads: 8
     databricks:
       catalog: null
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: hubspot_source_integration_tests_2
+      schema: hubspot_source_integration_tests_4
       threads: 2
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,12 +1,12 @@
 name: 'hubspot_source_integration_tests'
-version: '0.9.1'
+version: '0.10.0'
 profile: 'integration_tests'
 config-version: 2
 models:
   hubspot_source:
     +schema:
 vars:
-  hubspot_schema: hubspot_source_integration_tests_2
+  hubspot_schema: hubspot_source_integration_tests_4
   hubspot_source:
     hubspot_service_enabled: true
     hubspot_company_property_history_identifier: "company_property_history_data"

integration_tests/seeds/deal_company_data.csv

@@ -1,3 +1,3 @@
-company_id,deal_id
-12345,6789
-56789,1234
+company_id,deal_id,type_id
+12345,6789,101112
+56789,1234,101112

integration_tests/seeds/deal_contact_data.csv

@@ -1,12 +1,12 @@
-contact_id,deal_id,_fivetran_synced
-1499466,86133127,2022-04-14 4:30:42
-1499466,86134193,2022-04-14 4:45:43
-1499368,86014324,2022-04-14 5:30:45
-1499468,86136486,2022-04-14 6:30:45
-1499469,86139887,2022-04-14 7:45:44
-1499469,86142279,2022-04-14 8:30:44
-1499596,86324169,2022-04-16 5:00:42
-1499595,86318106,2022-04-16 4:45:44
-1499597,86328114,2022-04-16 6:45:43
-1499597,86325643,2022-04-16 6:45:43
-1499598,86326974,2022-04-16 8:00:44
+contact_id,deal_id,_fivetran_synced,type_id
+1499466,86133127,2022-04-14 4:30:42,101112
+1499466,86134193,2022-04-14 4:45:43,123456
+1499368,86014324,2022-04-14 5:30:45,567891
+1499468,86136486,2022-04-14 6:30:45,101112
+1499469,86139887,2022-04-14 7:45:44,123456
+1499469,86142279,2022-04-14 8:30:44,101112
+1499596,86324169,2022-04-16 5:00:42,123456
+1499595,86318106,2022-04-16 4:45:44,567891
+1499597,86328114,2022-04-16 6:45:43,101112
+1499597,86325643,2022-04-16 6:45:43,567891
+1499598,86326974,2022-04-16 8:00:44,123456

integration_tests/seeds/deal_pipeline_data.csv

@@ -1,7 +1,7 @@
-pipeline_id,_fivetran_deleted,_fivetran_synced,active,display_order,label
-mp_pipeline_monitoring,false,2020-07-09 15:25:37.051,true,0,83t7Vz1pOW/Fx81JI4o+4g==
-default,false,2020-07-09 15:25:37.053,true,0,FioJAMNdGX8U65mUGCkDwA==
-mp_pipeline_service,false,2020-07-09 15:25:37.024,true,0,eDLdK7glKfd0VRV0G5OcLg==
-mp_pipeline_archived,false,2020-07-09 15:25:37.054,true,3,pHm8C3GMK7LnsWuMcT6sgw==
-mp_pipeline_sales,false,2020-07-09 15:25:37.049,true,4,4sZLZi4xR6kmTh+A8waTCw==
-mp_pipeline_operations,false,2020-07-09 15:25:37.052,true,5,r2LpcB5mggWrgSrGkJERpg==
+pipeline_id,_fivetran_deleted,_fivetran_synced,active,display_order,label,created_at,updated_at
+mp_pipeline_monitoring,FALSE,2020-07-09 15:25:37.051,TRUE,0,83t7Vz1pOW/Fx81JI4o+4g==,2020-07-09 15:25:37.051,2020-07-09 15:25:37.051
+default,FALSE,2020-07-09 15:25:37.053,TRUE,0,FioJAMNdGX8U65mUGCkDwA==,2020-07-09 15:25:37.053,2020-07-09 15:25:37.053
+mp_pipeline_service,FALSE,2020-07-09 15:25:37.024,TRUE,0,eDLdK7glKfd0VRV0G5OcLg==,2020-07-09 15:25:37.024,2020-07-09 15:25:37.024
+mp_pipeline_archived,FALSE,2020-07-09 15:25:37.054,TRUE,3,pHm8C3GMK7LnsWuMcT6sgw==,2020-07-09 15:25:37.054,2020-07-09 15:25:37.054
+mp_pipeline_sales,FALSE,2020-07-09 15:25:37.049,TRUE,4,4sZLZi4xR6kmTh+A8waTCw==,2020-07-09 15:25:37.049,2020-07-09 15:25:37.049
+mp_pipeline_operations,FALSE,2020-07-09 15:25:37.052,TRUE,5,r2LpcB5mggWrgSrGkJERpg==,2020-07-09 15:25:37.052,2020-07-09 15:25:37.052