Merge pull request #52 from fivetran/release/v0.4.3

Release/v0.4.3

README.md

@@ -103,7 +103,7 @@ vars:
 
 
 ### Passthrough Columns
-Additionally, this package includes all source columns defined in the macros folder. We highly recommend including custom fields in this package as models now only bring in a few fields for the `company`, `contact`, `deal`, and `ticket` tables. You can add more columns using our pass-through column variables. These variables allow for the pass-through fields to be aliased (`alias`) and casted (`transform_sql`) if desired, but not required. Datatype casting is configured via a sql snippet within the `transform_sql` key. You may add the desired sql while omitting the `as field_name` at the end and your custom pass-though fields will be casted accordingly. Use the below format for declaring the respective pass-through variables.
+Additionally, this package includes all source columns defined in the macros folder. Models by default only bring in a few fields for the `company`, `contact`, `deal`, and `ticket` tables. You can add more columns using our pass-through column variables. These variables allow for the pass-through fields to be aliased (`alias`) and casted (`transform_sql`) if desired, but not required. Datatype casting is configured via a sql snippet within the `transform_sql` key. You may add the desired sql while omitting the `as field_name` at the end and your custom pass-though fields will be casted accordingly. Use the below format for declaring the respective pass-through variables.
 
 ```yml
 # dbt_project.yml
@@ -140,6 +140,24 @@ vars:
   hubspot__pass_through_all_columns: true # default is false
 ```
 
+### Calculated Fields
+This package also provides the ability to pass calculated fields through to the `company`, `contact`, `deal`, and `ticket` staging models. If you would like to add a calculated field to any of the mentioned staging models, you may configure the respective `hubspot__[table_name]_calculated_fields` variables with the `name` of the field you would like to create, and the `transform_sql` which will be the actual calculation that will make up the calculated field.
+```yml
+vars:
+  hubspot__deal_calculated_fields:
+    - name:          "deal_calculated_field"
+      transform_sql: "existing_field * other_field"
+  hubspot__company_calculated_fields:
+    - name:          "company_calculated_field"
+      transform_sql: "concat(name_field, '_company_name')"
+  hubspot__contact_calculated_fields:
+    - name:          "contact_calculated_field"
+      transform_sql: "contact_revenue - contact_expense"
+  hubspot__ticket_calculated_fields:
+    - name:          "ticket_calculated_field"
+      transform_sql: "total_field / other_total_field"
+```
+
 ### Changing the Build Schema
 By default this package will build the HubSpot staging models within a schema titled (<target_schema> + `_stg_hubspot`). If this is not where you would like your HubSpot staging models to be written to, add the following configuration to your `dbt_project.yml` file:
 

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'hubspot_source'
-version: '0.4.2'
+version: '0.4.3'
 config-version: 2
 require-dbt-version: ">=0.20.0"
 

integration_tests/dbt_project.yml

@@ -1,6 +1,6 @@
 
 name: 'hubspot_source_integration_tests'
-version: '0.4.2'
+version: '0.4.3'
 profile: 'integration_tests'
 config-version: 2
 

models/stg_hubspot__company.sql

@@ -49,6 +49,9 @@ with base as (
 
         --The below macro adds the fields defined within your hubspot__ticket_pass_through_columns variable into the staging model
         {{ fivetran_utils.fill_pass_through_columns('hubspot__company_pass_through_columns') }}
+
+        -- The below macro add the ability to create calculated fields using the hubspot__company_calculated_fields variable.
+        {{ fivetran_utils.calculated_fields('hubspot__company_calculated_fields') }}
 
     from macro
 

models/stg_hubspot__contact.sql

@@ -46,6 +46,9 @@ with base as (
         --The below macro adds the fields defined within your hubspot__contact_pass_through_columns variable into the staging model
         {{ fivetran_utils.fill_pass_through_columns('hubspot__contact_pass_through_columns') }}
 
+        -- The below macro add the ability to create calculated fields using the hubspot__contact_calculated_fields variable.
+        {{ fivetran_utils.calculated_fields('hubspot__contact_calculated_fields') }}
+
     from macro
 {% endif %}    
 

models/stg_hubspot__deal.sql

@@ -47,6 +47,9 @@ with base as (
         --The below macro adds the fields defined within your hubspot__deal_pass_through_columns variable into the staging model
         {{ fivetran_utils.fill_pass_through_columns('hubspot__deal_pass_through_columns') }}
 
+        -- The below macro add the ability to create calculated fields using the hubspot__deal_calculated_fields variable.
+        {{ fivetran_utils.calculated_fields('hubspot__deal_calculated_fields') }}
+
     from macro
 {% endif %}
 

models/stg_hubspot__engagement_task.sql

@@ -16,12 +16,31 @@ with base as (
         }}
     from base
 
+/*
+Some users have experienced the `completion_date` field being synced as a string rather than a timestamp.
+To address this, we use the below run_query command to query a sinlge record from the engagement_task tmp table
+and then assess in a conditional within the fields cte if the engagement_task field is indeed a UTC timestamp or not.
+
+If the field is not a timestamp, then we safe_cast so downstream models do not fail. Otherwise, we do nothing to the 
+field.
+*/
+{% if execute -%}
+    {% set results = run_query('select completion_date from ' ~ ref('stg_hubspot__engagement_task_tmp') ~ ' where completion_date is not null limit 1') %}
+    {% set results_list = results.columns[0].values() | string %}
+{% endif -%}
+
 ), fields as (
 
     select
         _fivetran_synced,
         body as task_note,
-        {{ dbt_utils.safe_cast('completion_date', 'timestamp') }} as completion_timestamp,
+
+        {% if 'tzinfo=<UTC>' not in results_list %}
+            {{ dbt_utils.safe_cast('completion_date', 'timestamp') }} as completion_timestamp,
+        {% else %}
+            completion_date as completion_timestamp,
+        {% endif %}
+
         engagement_id,
         for_object_type,
         is_all_day,
@@ -36,5 +55,3 @@ with base as (
 
 select *
 from fields
-
-

models/stg_hubspot__ticket.sql

@@ -48,6 +48,9 @@ with base as (
 
         --The below macro adds the fields defined within your hubspot__ticket_pass_through_columns variable into the staging model
         {{ fivetran_utils.fill_pass_through_columns('hubspot__ticket_pass_through_columns') }}
+
+        -- The below macro add the ability to create calculated fields using the hubspot__ticket_calculated_fields variable.
+        {{ fivetran_utils.calculated_fields('hubspot__ticket_calculated_fields') }}
 
     from macro
 {% endif %}

packages.yml

@@ -1,3 +1,3 @@
 packages:
-- package: fivetran/fivetran_utils
-  version: [">=0.2.0", "<0.3.0"]
+  - package: fivetran/fivetran_utils
+    version: [">=0.2.0", "<0.3.0"]