Merge pull request #121 from CloudFormations/develop

Develop
CloudFormations · Dec 11, 2024 · df09a7f · df09a7f
2 parents 0408694 + 07830e7
commit df09a7f
Show file tree

Hide file tree

Showing 9 changed files with 10 additions and 763 deletions.
diff --git a/.azuredevops/pipelines/database.build.yml b/.azuredevops/pipelines/database.build.yml
@@ -4,7 +4,7 @@ trigger:
 - none
 variables:
   - name:  SqlProjects
-    value: metadata.control, metadata.ingest, metadata.transform
+    value: metadata.common, metadata.control, metadata.ingest, metadata.transform
 
 stages:
   - stage: BuildAndPublish

diff --git a/.azuredevops/pipelines/datafactory.build.yml b/.azuredevops/pipelines/datafactory.build.yml
diff --git a/.azuredevops/pipelines/datafactory.cicd.yml b/.azuredevops/pipelines/datafactory.cicd.yml
diff --git a/.azuredevops/pipelines/datafactory.deploy.yml b/.azuredevops/pipelines/datafactory.deploy.yml
diff --git a/src/azure.databricks/python/notebooks/utils/WriteToDelta.py b/src/azure.databricks/python/notebooks/utils/WriteToDelta.py
@@ -67,7 +67,7 @@ def overwriteDelta(df: DataFrame, schemaName: str, tableName: str) -> None:
         tableName (str): Name of the target table for the dataset.
 
     """
-    df.write.format("delta").mode("overwrite").insertInto(f"{schemaName}.{tableName}")
+    df.write.format("delta").mode("overwrite").option("mergeSchema", "true").saveAsTable(f"{schemaName}.{tableName}")
 
     return
 

diff --git a/src/azure.datafactory/dataset/Ingest_DS_Salesforce_OAuth.json b/src/azure.datafactory/dataset/Ingest_DS_Salesforce_OAuth.json
diff --git a/src/azure.datafactory/linkedService/Ingest_LS_Salesforce_OAuth.json b/src/azure.datafactory/linkedService/Ingest_LS_Salesforce_OAuth.json
-Original file line number
+Diff line change
@@ Expand Up @@
             tableName (str): Name of the target table for the dataset.
         """
-        df.write.format("delta").mode("overwrite").insertInto(f"{schemaName}.{tableName}")
+        df.write.format("delta").mode("overwrite").option("mergeSchema", "true").saveAsTable(f"{schemaName}.{tableName}")
         return
@@ Expand Down @@