update transformation to add is_record variable and tests

airbytehq · Nov 15, 2024 · b179961 · b179961
1 parent 06646bd
commit b179961
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 18 deletions.
diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/components.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/components.py
@@ -31,8 +31,10 @@ def transform(
         """
         Transform a record in place by adding fields directly to the record by manipulating the injected fields into a legacy field to avoid breaking syncs.
 
-        :param record: The input record to be transformed
+        :param record_or_schema: The input record or schema to be transformed.
         """
+        is_record = record_or_schema.get("properties") is not None
+
         for field, value in list(record_or_schema.get("properties", record_or_schema).items()):
             for legacy_field, new_field in self._field_mapping.items():
                 if new_field in field:
@@ -41,8 +43,9 @@ def transform(
                     if legacy_field == "hs_lifecyclestage_" and not transformed_field.endswith("_date"):
                         transformed_field += "_date"
 
-                    if record_or_schema.get("properties") is not None:
+                    if is_record:
                         if record_or_schema["properties"].get(transformed_field) is None:
                             record_or_schema["properties"][transformed_field] = value
-                    elif record_or_schema.get(transformed_field) is None:
-                        record_or_schema[transformed_field] = value
+                    else:
+                        if record_or_schema.get(transformed_field) is None:
+                            record_or_schema[transformed_field] = value
diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py
@@ -213,11 +213,13 @@ def test_stream_read_with_legacy_field_transformation(stream, endpoint, cursor_v
                     {
                         "id": "test_id",
                         "created": "2022-02-25T16:43:11Z",
-                        "hs_v2_date_entered_prospect": "2024-01-01T00:00:00Z",
-                        "hs_v2_date_exited_prospect": "2024-02-01T00:00:00Z",
-                        "hs_v2_latest_time_in_prospect": "1 month",
-                        "hs_v2_cumulative_time_in_prsopect": "1 month",
-                        "hs_v2_some_other_property_in_prospect": "Great property"
+                        "properties": {
+                            "hs_v2_latest_time_in_prospect": "1 month",
+                            "hs_v2_date_entered_prospect": "2024-01-01T00:00:00Z",
+                            "hs_v2_date_exited_prospect": "2024-02-01T00:00:00Z",
+                            "hs_v2_cumulative_time_in_prsopect": "1 month",
+                            "hs_v2_some_other_property_in_prospect": "Great property",
+                        }
                     }
                     | cursor_value
                 ],
@@ -244,18 +246,29 @@ def test_stream_read_with_legacy_field_transformation(stream, endpoint, cursor_v
     expected_record = {
             "id": "test_id",
             "created": "2022-02-25T16:43:11Z",
-            "hs_v2_date_entered_prospect": "2024-01-01T00:00:00Z",
-            "hs_v2_date_exited_prospect": "2024-02-01T00:00:00Z",
-            "hs_date_exited_prospect": "2024-02-01T00:00:00Z",
-            "hs_time_in_prospect": "1 month",
-            "hs_v2_latest_time_in_prospect": "1 month",
-            "hs_v2_cumulative_time_in_prsopect": "1 month",
-            "hs_v2_some_other_property_in_prospect": "Great property"
+            "properties": {
+                "hs_v2_date_entered_prospect": "2024-01-01T00:00:00Z",
+                "hs_v2_date_exited_prospect": "2024-02-01T00:00:00Z",
+                "hs_v2_latest_time_in_prospect": "1 month",
+                "hs_v2_cumulative_time_in_prsopect": "1 month",
+                "hs_v2_some_other_property_in_prospect": "Great property",
+                "hs_time_in_prospect": "1 month",
+                "hs_date_exited_prospect": "2024-02-01T00:00:00Z",
+            },
+            "properties_hs_v2_date_entered_prospect": "2024-01-01T00:00:00Z",
+            "properties_hs_v2_date_exited_prospect": "2024-02-01T00:00:00Z",
+            "properties_hs_v2_latest_time_in_prospect": "1 month",
+            "properties_hs_v2_cumulative_time_in_prsopect": "1 month",
+            "properties_hs_v2_some_other_property_in_prospect": "Great property",
+            "properties_hs_time_in_prospect": "1 month",
+            "properties_hs_date_exited_prospect": "2024-02-01T00:00:00Z",
         } | cursor_value
     if isinstance(stream, Contacts):
-        expected_record = expected_record | {"hs_lifecyclestage_prospect_date": "2024-01-01T00:00:00Z"}
+        expected_record = expected_record | {"properties_hs_lifecyclestage_prospect_date": "2024-01-01T00:00:00Z"}
+        expected_record["properties"] = expected_record["properties"] | {"hs_lifecyclestage_prospect_date": "2024-01-01T00:00:00Z"}
     else:
-        expected_record = expected_record | {"hs_date_entered_prospect": "2024-01-01T00:00:00Z" }
+        expected_record = expected_record | {"properties_hs_date_entered_prospect": "2024-01-01T00:00:00Z" }
+        expected_record["properties"] = expected_record["properties"] | {"hs_date_entered_prospect": "2024-01-01T00:00:00Z" }
     assert records[0] == expected_record