From 5beac7b4fcd20f0513abbb1cf5b56b4d35b40ce5 Mon Sep 17 00:00:00 2001
From: dleadbetter <derek@performantsoftware.com>
Date: Fri, 13 Dec 2024 07:31:19 -0500
Subject: [PATCH 1/3] CDC #321 - Updating import process to not overwrite
 user-defined field values not included in the import CSVs

---
 app/services/core_data_connector/import/base.rb          | 2 +-
 app/services/core_data_connector/import/events.rb        | 3 ++-
 app/services/core_data_connector/import/instances.rb     | 3 ++-
 app/services/core_data_connector/import/items.rb         | 3 ++-
 app/services/core_data_connector/import/organizations.rb | 3 ++-
 app/services/core_data_connector/import/people.rb        | 3 ++-
 app/services/core_data_connector/import/places.rb        | 3 ++-
 app/services/core_data_connector/import/relationships.rb | 6 ++++--
 app/services/core_data_connector/import/taxonomies.rb    | 3 ++-
 app/services/core_data_connector/import/works.rb         | 3 ++-
 10 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/app/services/core_data_connector/import/base.rb b/app/services/core_data_connector/import/base.rb
index 50f768a..ac98bdc 100644
--- a/app/services/core_data_connector/import/base.rb
+++ b/app/services/core_data_connector/import/base.rb
@@ -76,7 +76,7 @@ def transform
 
         execute <<-SQL.squish
           UPDATE #{table_name}
-             SET user_defined = json_strip_nulls(json_build_object(#{expression}))
+             SET user_defined = user_defined || json_strip_nulls(json_build_object(#{expression}))::jsonb
         SQL
 
         # Sets the "Select" and "FuzzyDate" user-defined types to JSONB
diff --git a/app/services/core_data_connector/import/events.rb b/app/services/core_data_connector/import/events.rb
index 285c181..7123976 100644
--- a/app/services/core_data_connector/import/events.rb
+++ b/app/services/core_data_connector/import/events.rb
@@ -156,7 +156,8 @@ def transform
 
         execute <<-SQL.squish
           UPDATE #{table_name} z_events
-             SET event_id = events.id
+             SET event_id = events.id,
+                 user_defined = events.user_defined
             FROM core_data_connector_events events
            WHERE events.uuid = z_events.uuid
         SQL
diff --git a/app/services/core_data_connector/import/instances.rb b/app/services/core_data_connector/import/instances.rb
index e224728..06e8e1c 100644
--- a/app/services/core_data_connector/import/instances.rb
+++ b/app/services/core_data_connector/import/instances.rb
@@ -87,7 +87,8 @@ def transform
 
         execute <<-SQL.squish
           UPDATE #{table_name} z_instances
-             SET instance_id = instances.id
+             SET instance_id = instances.id,
+                 user_defined = instances.user_defined
             FROM core_data_connector_instances instances
            WHERE instances.uuid = z_instances.uuid
         SQL
diff --git a/app/services/core_data_connector/import/items.rb b/app/services/core_data_connector/import/items.rb
index b6b4da6..53dcef2 100644
--- a/app/services/core_data_connector/import/items.rb
+++ b/app/services/core_data_connector/import/items.rb
@@ -87,7 +87,8 @@ def transform
 
         execute <<-SQL.squish
           UPDATE #{table_name} z_items
-             SET item_id = items.id
+             SET item_id = items.id,
+                 user_defined = items.user_defined
             FROM core_data_connector_items items
            WHERE items.uuid = z_items.uuid
         SQL
diff --git a/app/services/core_data_connector/import/organizations.rb b/app/services/core_data_connector/import/organizations.rb
index 98f9f70..fb7aa49 100644
--- a/app/services/core_data_connector/import/organizations.rb
+++ b/app/services/core_data_connector/import/organizations.rb
@@ -87,7 +87,8 @@ def transform
 
         execute <<-SQL.squish
           UPDATE #{table_name} z_organizations
-             SET organization_id = organizations.id
+             SET organization_id = organizations.id,
+                 user_defined = organizations.user_defined
             FROM core_data_connector_organizations organizations
            WHERE organizations.uuid = z_organizations.uuid
         SQL
diff --git a/app/services/core_data_connector/import/people.rb b/app/services/core_data_connector/import/people.rb
index c3d38d2..42b8a0d 100644
--- a/app/services/core_data_connector/import/people.rb
+++ b/app/services/core_data_connector/import/people.rb
@@ -90,7 +90,8 @@ def transform
 
         execute <<-SQL.squish
           UPDATE #{table_name} z_people
-             SET person_id = people.id
+             SET person_id = people.id,
+                 user_defined = people.user_defined
             FROM core_data_connector_people people
            WHERE people.uuid = z_people.uuid
         SQL
diff --git a/app/services/core_data_connector/import/places.rb b/app/services/core_data_connector/import/places.rb
index dfbb8dd..be7e87a 100644
--- a/app/services/core_data_connector/import/places.rb
+++ b/app/services/core_data_connector/import/places.rb
@@ -105,7 +105,8 @@ def transform
 
         execute <<-SQL.squish
           UPDATE #{table_name} z_places
-             SET place_id = places.id
+             SET place_id = places.id,
+                 user_defined = places.user_defined
             FROM core_data_connector_places places
            WHERE places.uuid = z_places.uuid
         SQL
diff --git a/app/services/core_data_connector/import/relationships.rb b/app/services/core_data_connector/import/relationships.rb
index 7f9677c..48dbf14 100644
--- a/app/services/core_data_connector/import/relationships.rb
+++ b/app/services/core_data_connector/import/relationships.rb
@@ -121,7 +121,8 @@ def transform
 
         execute <<-SQL.squish
           UPDATE #{table_name} z_relationships
-             SET relationship_id = relationships.id
+             SET relationship_id = relationships.id,
+                 user_defined = relationships.user_defined
             FROM core_data_connector_relationships relationships
            WHERE relationships.uuid = z_relationships.uuid
              AND z_relationships.relationship_id IS NULL
@@ -129,7 +130,8 @@ def transform
 
         execute <<-SQL.squish
           UPDATE #{table_name} z_relationships
-             SET relationship_id = relationships.id
+             SET relationship_id = relationships.id,
+                 user_defined = relationships.user_defined
             FROM core_data_connector_relationships relationships
            WHERE relationships.primary_record_id = z_relationships.primary_record_id
              AND relationships.primary_record_type = z_relationships.primary_record_type
diff --git a/app/services/core_data_connector/import/taxonomies.rb b/app/services/core_data_connector/import/taxonomies.rb
index d30ad4d..6918acf 100644
--- a/app/services/core_data_connector/import/taxonomies.rb
+++ b/app/services/core_data_connector/import/taxonomies.rb
@@ -62,7 +62,8 @@ def transform
 
         execute <<-SQL.squish
           UPDATE #{table_name} z_taxonomies
-             SET taxonomy_id = taxonomies.id
+             SET taxonomy_id = taxonomies.id,
+                 user_defined = taxonomies.user_defined
             FROM core_data_connector_taxonomies taxonomies
            WHERE taxonomies.uuid = z_taxonomies.uuid
         SQL
diff --git a/app/services/core_data_connector/import/works.rb b/app/services/core_data_connector/import/works.rb
index 34893b5..5b672cf 100644
--- a/app/services/core_data_connector/import/works.rb
+++ b/app/services/core_data_connector/import/works.rb
@@ -87,7 +87,8 @@ def transform
 
         execute <<-SQL.squish
           UPDATE #{table_name} z_works
-             SET work_id = works.id
+             SET work_id = works.id,
+                 user_defined = works.user_defined
             FROM core_data_connector_works works
            WHERE works.uuid = z_works.uuid
         SQL

From ee7df71ae0626e81eff02d1beec24c2463c624dc Mon Sep 17 00:00:00 2001
From: dleadbetter <derek@performantsoftware.com>
Date: Fri, 13 Dec 2024 07:32:10 -0500
Subject: [PATCH 2/3] CDC #321 - Fixing a bug that occurs in the export process
 when a user_defined value is nil

---
 app/services/core_data_connector/export/base.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/services/core_data_connector/export/base.rb b/app/services/core_data_connector/export/base.rb
index a507bdd..2571c27 100644
--- a/app/services/core_data_connector/export/base.rb
+++ b/app/services/core_data_connector/export/base.rb
@@ -44,7 +44,7 @@ def to_export_csv(user_defined_fields)
         private
 
         def add_user_defined_fields(hash, user_defined_fields)
-          return unless user_defined_fields.present? && self.respond_to?(:user_defined)
+          return unless user_defined_fields.present? && self.respond_to?(:user_defined) && self.user_defined.present?
 
           user_defined_fields.each do |user_defined_field|
             key = ImportAnalyze::Helper.uuid_to_column_name(user_defined_field.uuid)

From 1fb6a0784f22133281c63fa03b5b623cf9efb37a Mon Sep 17 00:00:00 2001
From: dleadbetter <derek@performantsoftware.com>
Date: Mon, 16 Dec 2024 12:09:44 -0500
Subject: [PATCH 3/3] CDC #321 - Removing "user_defined" column from taxonomies
 import

---
 app/services/core_data_connector/import/taxonomies.rb | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/app/services/core_data_connector/import/taxonomies.rb b/app/services/core_data_connector/import/taxonomies.rb
index 6918acf..d30ad4d 100644
--- a/app/services/core_data_connector/import/taxonomies.rb
+++ b/app/services/core_data_connector/import/taxonomies.rb
@@ -62,8 +62,7 @@ def transform
 
         execute <<-SQL.squish
           UPDATE #{table_name} z_taxonomies
-             SET taxonomy_id = taxonomies.id,
-                 user_defined = taxonomies.user_defined
+             SET taxonomy_id = taxonomies.id
             FROM core_data_connector_taxonomies taxonomies
            WHERE taxonomies.uuid = z_taxonomies.uuid
         SQL