Grouped fields with date details

CHANGELOG.md

@@ -13,6 +13,9 @@ This project bumps the version number for any changes (including documentation u
 
 ## [Unreleased] - i.e. pushed to main branch but not yet tagged as a release
 
+## [5.0.5] - 2024-02-02
+- For `date details` batch mode: add support for ingesting grouped fields at the same level as the structured date group. To find the group level of the structured date group, find the `date_field_group` value in the relevant CSV ingest template. If that field has a `REPEATING FIELD GROUP` value in the template, any other field values with the same `REPEATING FIELD GROUP` value can be ingested together.
+
 ## [5.0.4] - 2024-02-01
 - Ensure `shortid` field is populated when mapping date details for authorities
 

Gemfile.lock

@@ -12,7 +12,7 @@ GIT
 PATH
   remote: .
   specs:
-    collectionspace-mapper (5.0.4)
+    collectionspace-mapper (5.0.5)
       activesupport (= 7.0.4.3)
       chronic
       collectionspace-client (~> 0.15.0)

lib/collectionspace/mapper/authority.rb

@@ -4,11 +4,21 @@ module CollectionSpace
   module Mapper
     # special behavior for authority mapping
     module Authority
-      def special_mappings
+      module_function
+
+      def extended(mod)
+        if mod.respond_to?(:add_mapping)
+          special_mappings(mod).each do |mapping|
+            mod.add_mapping(mapping)
+          end
+        end
+      end
+
+      def special_mappings(mod)
         [
           {
             fieldname: "shortIdentifier",
-            namespace: handler.record.common_namespace,
+            namespace: mod.handler.record.common_namespace,
             data_type: "string",
             xpath: [],
             required: "not in input data",

lib/collectionspace/mapper/column_mappings.rb

@@ -10,20 +10,23 @@ class ColumnMappings
       include Enumerable
       extend Forwardable
 
+      attr_reader :handler
+
       def_delegators :@all, :each, :reject!
 
       # @param mappings [Array<Hash>] from record mapper JSON file
       # @param hander [CollectionSpace::Mapper::DataHandler]
       def initialize(mappings:, handler:)
         @handler = handler
-        handler.record.extensions.each { |ext| extend ext }
         @transforms = handler.batch.transforms
 
         @all = []
         @lkup = {}
+        handler.record.extensions.each { |ext| extend ext }
         mappings.each { |mapping| add_mapping(mapping) }
 
-        special_mappings.each { |mapping| add_mapping(mapping) }
+        # binding.pry
+        # special_mappings.each { |mapping| add_mapping(mapping) }
       end
 
       def <<(mapping)
@@ -47,21 +50,21 @@ def required_columns
         all.select(&:required?)
       end
 
-      private
-
-      attr_reader :handler, :transforms, :all, :lkup
-
       def add_mapping(mapping)
         mapobj = CollectionSpace::Mapper::ColumnMapping.new(
           mapping: mapping
         )
-        all << mapobj
-        lkup[mapobj.datacolumn] = mapobj
+        @all << mapobj
+        @lkup[mapobj.datacolumn] = mapobj
       end
 
-      def special_mappings
-        []
-      end
+      private
+
+      attr_reader :transforms, :all, :lkup
+
+      # def special_mappings
+      #   []
+      # end
     end
   end
 end

lib/collectionspace/mapper/data_mapper.rb

@@ -38,6 +38,7 @@ def set_identifier_value
           end
           thexpath = "//#{mapping.namespace}/#{mapping.fieldname}"
           value = doc.xpath(thexpath).first
+
           value = value.text
           response.add_identifier(value)
         end
@@ -53,22 +54,23 @@ def set_relation_id
       end
 
       def add_short_id
-        ns = handler.record.common_namespace
-        targetnode = doc.xpath("/document/#{ns}").first
-        child = Nokogiri::XML::Node.new("shortIdentifier", doc)
-
         shortid =
           if response.transformed_data.key?("shortidentifier")
-            response.transformed_data["shortidentifier"]
+            response.transformed_data["shortidentifier"][0]
           else
             term = response.split_data["termdisplayname"][0]
             CollectionSpace::Mapper::Identifiers::AuthorityShortIdentifier.call(
               term
             )
           end
+        response.add_identifier(shortid)
+        return if response.transformed_data.key?("shortidentifier")
+
+        ns = handler.record.common_namespace
+        targetnode = doc.xpath("/document/#{ns}").first
+        child = Nokogiri::XML::Node.new("shortIdentifier", doc)
         child.content = shortid
         targetnode.add_child(child)
-        response.add_identifier(shortid)
       end
 
       def map(xpath)

lib/collectionspace/mapper/date_details.rb

@@ -3,12 +3,21 @@
 module CollectionSpace
   module Mapper
     module DateDetails
-      # Methods used in ColumnMappings
-      def special_mappings
+      module_function
+
+      def extended(mod)
+        if mod.respond_to?(:add_mapping)
+          special_mappings(mod).each do |mapping|
+            mod.add_mapping(mapping)
+          end
+        end
+      end
+
+      def special_mappings(mod)
         base = [
           {
             fieldname: "date_field_group",
-            namespace: handler.record.common_namespace,
+            namespace: mod.handler.record.common_namespace,
             data_type: "string",
             xpath: [],
             required: "y",
@@ -19,7 +28,7 @@ def special_mappings
           },
           {
             fieldname: "scalarValuesComputed",
-            namespace: handler.record.common_namespace,
+            namespace: mod.handler.record.common_namespace,
             data_type: "string",
             xpath: [],
             required: "y",
@@ -29,10 +38,10 @@ def special_mappings
             transforms: {special: ["boolean"]}
           }
         ]
-        [base, vocab_mappings, optionlist_mappings].flatten
+        [base, vocab_mappings(mod), optionlist_mappings(mod)].flatten
       end
 
-      def vocab_mappings
+      def vocab_mappings(mod)
         {
           "dateLatestQualifierUnit" => "datequalifier",
           "dateLatestEra" => "dateera",
@@ -43,7 +52,7 @@ def vocab_mappings
         }.map do |fieldname, vocab|
           {
             fieldname: fieldname,
-            namespace: handler.record.common_namespace,
+            namespace: mod.handler.record.common_namespace,
             data_type: "string",
             xpath: [],
             required: "n",
@@ -57,14 +66,14 @@ def vocab_mappings
         end
       end
 
-      def optionlist_mappings
+      def optionlist_mappings(mod)
         {
           "dateLatestQualifier" => "dateQualifiers",
           "dateEarliestSingleQualifier" => "dateQualifiers"
         }.map do |fieldname, vocab|
           {
             fieldname: fieldname,
-            namespace: handler.record.common_namespace,
+            namespace: mod.handler.record.common_namespace,
             data_type: "string",
             xpath: [],
             required: "n",
@@ -93,7 +102,7 @@ def ensure_target_field_exists(response)
           "date_field_group value `#{val}` is not a known structured date "\
             "field group in a #{handler.record.recordtype} record. You must "\
             "enter a field that appears as a column header in the CSV "\
-            "template for this record type."
+            "template for this record type. Case sensitive!"
         )
       end
 

lib/collectionspace/mapper/date_details/data_prepper.rb

@@ -18,23 +18,39 @@ def initialize(data, handler)
             CollectionSpace::Mapper.structured_date_detailed_fields
               .map { |field| [field.downcase, field] }
               .to_h
+          @authority_data = nil
+          @grouped_data = nil
         end
 
         def prep
+          unless handler.grouped_fields
+            handler.check_fields(response.merged_data)
+          end
+          if handler.authority_handler
+            extract_authority_data
+            auth_prepped = handler.authority_handler
+              .prep(authority_data)
+          end
+          if handler.grouped_handler
+            extract_grouped_data
+            grouped_prepped = handler.grouped_handler.prep(grouped_data)
+          end
           split_data
           transform_data
           handle_term_fields
           extract_date_fields
           clean_transformed
           readd_id
           combine_data_fields
+          merge_authority_data(auth_prepped) if authority_data
+          merge_grouped_data(grouped_prepped) if grouped_data
           response
         end
 
         private
 
         attr_reader :id_field, :target, :target_mapping, :date_fields,
-          :date_field_lookup
+          :date_field_lookup, :grouped_data, :authority_data
 
         def date_data
           @date_data ||= response.merged_data
@@ -46,6 +62,33 @@ def non_date_data
             .reject { |field, _value| date_fields.any?(field) }
         end
 
+        def extract_authority_data
+          @authority_data = {}
+          copy_id_field_to(authority_data)
+        end
+
+        def extract_grouped_data
+          @grouped_data = {}
+          copy_id_field_to(grouped_data)
+          move_grouped_fields_to_grouped_data
+        end
+
+        def copy_id_field_to(data_var)
+          if response.merged_data.key?(id_field)
+            data_var[id_field] = response.merged_data[id_field]
+          elsif response.merged_data.key?("termdisplayname")
+            data_var["termdisplayname"] =
+              response.merged_data["termdisplayname"]
+          end
+        end
+
+        def move_grouped_fields_to_grouped_data
+          handler.grouped_fields.each do |field|
+            grouped_data[field] = response.merged_data[field]
+            response.merged_data.delete(field)
+          end
+        end
+
         def split_data
           response.merged_data.each do |field, val|
             splitval = if identifier?(field)
@@ -139,19 +182,42 @@ def clean_transformed
         def readd_id
           id = case id_field
           when "shortidentifier"
-            readd_authority_id
+            [authority_short_id]
           else
             response.split_data[id_field]
           end
           response.transformed_data[id_field] = id
         end
 
-        def readd_authority_id
+        def authority_short_id
           term = response.split_data["termdisplayname"][0]
           CollectionSpace::Mapper::Identifiers::AuthorityShortIdentifier.call(
             term
           )
         end
+
+        def merge_authority_data(auth_prepped)
+          added_paths = auth_prepped.xpaths
+            .reject { |path, xpath| response.xpaths.key?(path) }
+
+          added_paths.each do |path, xpath|
+            response.xpaths[path] = xpath
+            response.combined_data[path] = auth_prepped.combined_data[path]
+          end
+        end
+
+        def merge_grouped_data(grouped_prepped)
+          path = handler.target_path
+
+          response.combined_data[path].merge!(
+            grouped_prepped.combined_data[path]
+          )
+          grouped_prepped.errors.each { |err| response.add_error(err) }
+          grouped_prepped.terms.each { |term| response.add_term(term) }
+          grouped_prepped.warnings.each do |warning|
+            response.add_warning(warning)
+          end
+        end
       end
     end
   end