[WIP] add language processing to ldpath service

.circleci/config.yml

@@ -27,7 +27,7 @@ jobs:
           project: qa
 
       - samvera/engine_cart_generate:
-          cache_key: v7-internal-test-app-{{ checksum "qa.gemspec" }}-{{ checksum "spec/test_app_templates/lib/generators/test_app_generator.rb" }}-{{ checksum "lib/generators/qa/install/install_generator.rb" }}-<< parameters.rails_version >>-<< parameters.ruby_version >>
+          cache_key: v10-internal-test-app-{{ checksum "qa.gemspec" }}-{{ checksum "spec/test_app_templates/lib/generators/test_app_generator.rb" }}-{{ checksum "lib/generators/qa/install/install_generator.rb" }}-<< parameters.rails_version >>-<< parameters.ruby_version >>
 
       - samvera/bundle_for_gem:
           ruby_version: << parameters.ruby_version >>

Gemfile

@@ -7,6 +7,8 @@ group :development, :test do
   gem 'simplecov', require: false
 end
 
+gem 'ldpath', github: 'samvera-labs/ldpath', branch: 'maintain_literals'
+
 # BEGIN ENGINE_CART BLOCK
 # engine_cart: 0.10.0
 # engine_cart stanza: 0.10.0

app/services/qa/linked_data/ldpath_service.rb

@@ -4,39 +4,85 @@
 module Qa
   module LinkedData
     class LdpathService
-      VALUE_ON_ERROR = [].freeze
+      LANGUAGE_PATTERN = "*LANG*".freeze
+      PROPERTY_NAME = "property".freeze
 
       class_attribute :predefined_prefixes
       self.predefined_prefixes = Ldpath::Transform.default_prefixes.with_indifferent_access
 
-      # Create the ldpath program for a given ldpath.
-      # @param ldpath [String] ldpath to follow to get a value from a graph (documation: http://marmotta.apache.org/ldpath/language.html)
-      # @param prefixes [Hash] shortcut names for URI prefixes with key = part of predicate that is the same for all terms (e.g. { "madsrdf": "http://www.loc.gov/mads/rdf/v1#" })
-      # @return [Ldpath::Program] an executable program that will extract a value from a graph
-      def self.ldpath_program(ldpath:, prefixes: {})
-        program_code = ""
-        prefixes.each { |key, url| program_code << "@prefix #{key} : <#{url}> \;\n" }
-        program_code << "property = #{ldpath} \;"
-        Ldpath::Program.parse program_code
-      rescue => e
-        Rails.logger.warn("WARNING: #{I18n.t('qa.linked_data.ldpath.parse_logger_error')}... cause: #{e.message}\n   ldpath_program=\n#{program_code}")
-        raise StandardError, I18n.t("qa.linked_data.ldpath.parse_error") + "... cause: #{e.message}"
-      end
+      class << self
+        # Create the ldpath program for a given ldpath.
+        # @param ldpath [String] ldpath to follow to get a value from a graph (documation: http://marmotta.apache.org/ldpath/language.html)
+        # @param prefixes [Hash] shortcut names for URI prefixes with key = part of predicate that is the same for all terms (e.g. { "madsrdf": "http://www.loc.gov/mads/rdf/v1#" })
+        # @param languages [Array<Symbol>] limit results to these languages and anything not tagged (applies to ldpaths with *LANG* marker)
+        # @return [Ldpath::Program] an executable program that will extract a value from a graph
+        def ldpath_program(ldpath:, prefixes: {}, languages: [])
+          program_code = ldpath_program_code(ldpath: ldpath, prefixes: prefixes, languages: languages)
+          Ldpath::Program.parse program_code
+        rescue => e
+          Rails.logger.warn("WARNING: #{I18n.t('qa.linked_data.ldpath.parse_logger_error')}... cause: #{e.message}\n   ldpath_program=\n#{program_code}")
+          raise StandardError, I18n.t("qa.linked_data.ldpath.parse_error") + "... cause: #{e.message}"
+        end
+
+        # Create the program code for a given ldpath.
+        # @param ldpath [String] ldpath to follow to get a value from a graph (documation: http://marmotta.apache.org/ldpath/language.html)
+        # @param prefixes [Hash] shortcut names for URI prefixes with key = part of predicate that is the same for all terms (e.g. { "madsrdf": "http://www.loc.gov/mads/rdf/v1#" })
+        # @param languages [Array<Symbol>] limit results to these languages and anything not tagged (applies to ldpaths with *LANG* marker)
+        # @return [String] the program code string used with Ldpath::Program.parse
+        def ldpath_program_code(ldpath:, prefixes: {}, languages: [])
+          program_code = ""
+          prefixes.each { |key, url| program_code << "@prefix #{key} : <#{url}> \;\n" }
+          property_explode(program_code, ldpath, languages)
+        end
+
+        # Evaluate an ldpath for a specific subject uri in the context of a graph and return the extracted values.
+        # @param program [Ldpath::Program] an executable program that will extract a value from a graph
+        # @param graph [RDF::Graph] the graph from which the values will be extracted
+        # @param subject_uri [RDF::URI] retrieved values will be limited to those with the subject uri
+        # @param limit_to_context [Boolean] if true, the evaluation process will not make any outside network calls.
+        #        It will limit results to those found in the context graph.
+        ## @return [Array<RDF::Literal>] the extracted values based on the ldpath
+        def ldpath_evaluate(program:, graph:, subject_uri:, limit_to_context: Qa.config.limit_ldpath_to_context?, maintain_literals: false)
+          raise ArgumentError, "You must specify a program when calling ldpath_evaluate" if program.blank?
+          output = program.evaluate(subject_uri, context: graph, limit_to_context: limit_to_context, maintain_literals: maintain_literals)
+          maintain_literals ? property_implode(output) : output.values.flatten.uniq
+        rescue ParseError => e
+          Rails.logger.warn("WARNING: #{I18n.t('qa.linked_data.ldpath.evaluate_logger_error')} (cause: #{e.message}")
+          raise ParseError, I18n.t("qa.linked_data.ldpath.evaluate_error") + "... cause: #{e.message}"
+        end
+
+        private
+
+          # create program code with a property per language + untagged
+          def property_explode(program_code, ldpath, languages)
+            return program_code << "#{PROPERTY_NAME} = #{ldpath} \;\n" unless ldpath.index(LANGUAGE_PATTERN)
+            return program_code << "#{PROPERTY_NAME} = #{ldpath.gsub(LANGUAGE_PATTERN, '')} \;\n" unless languages.present?
+            languages.map { |language| program_code << "#{property_name_for(language)} = #{ldpath.gsub(LANGUAGE_PATTERN, "[@#{language}]")} \;\n" }
+            program_code << "#{PROPERTY_NAME} = #{ldpath.gsub(LANGUAGE_PATTERN, '[@none]')} \;\n"
+          end
+
+          # flatten all properties and turn into RDF::Literals with language tagging if appropriate
+          def property_implode(output)
+            return nil if output.blank?
+            output.each do |property_name, values|
+              output[property_name] = remap_string_values(property_name, values) if values.first.is_a? String
+            end
+            output.values.flatten.uniq
+          end
+
+          def property_name_for(language)
+            "#{language}_#{PROPERTY_NAME}"
+          end
+
+          def language_from(property_name)
+            return nil if property_name.casecmp?(PROPERTY_NAME)
+            property_name.chomp("_#{PROPERTY_NAME}")
+          end
 
-      # Evaluate an ldpath for a specific subject uri in the context of a graph and return the extracted values.
-      # @param program [Ldpath::Program] an executable program that will extract a value from a graph
-      # @param graph [RDF::Graph] the graph from which the values will be extracted
-      # @param subject_uri [RDF::URI] retrieved values will be limited to those with the subject uri
-      # @param limit_to_context [Boolean] if true, the evaluation process will not make any outside network calls.
-      #        It will limit results to those found in the context graph.
-      ## @return [Array<String>] the extracted values based on the ldpath
-      def self.ldpath_evaluate(program:, graph:, subject_uri:, limit_to_context: Qa.config.limit_ldpath_to_context?)
-        return VALUE_ON_ERROR if program.blank?
-        output = program.evaluate(subject_uri, context: graph, limit_to_context: limit_to_context)
-        output.present? ? output['property'].uniq : nil
-      rescue => e
-        Rails.logger.warn("WARNING: #{I18n.t('qa.linked_data.ldpath.evaluate_logger_error')} (cause: #{e.message}")
-        raise StandardError, I18n.t("qa.linked_data.ldpath.evaluate_error") + "... cause: #{e.message}"
+          def remap_string_values(property_name, values)
+            language = language_from(property_name)
+            values.map { |v| RDF::Literal.new(v, language: language) }
+          end
       end
     end
   end

app/services/qa/linked_data/mapper/graph_ldpath_mapper_service.rb

@@ -19,9 +19,10 @@ class GraphLdpathMapperService
         # @example ldpath map
         #   {
         #     uri: :subject_uri,
-        #     id: 'locid:lccn :: xsd::string',
+        #     id: 'locid:lccn',
         #     label: 'skos:prefLabel :: xsd::string',
         #     altlabel: 'skos:altLabel :: xsd::string',
+        #     sameas: 'skos:sameAs :: xsd::anyURI',
         #     sort: 'vivo:rank :: xsd::integer'
         #   }
         # @param subject_uri [RDF::URI] the subject within the graph for which the values are being extracted
@@ -37,7 +38,7 @@ def self.map_values(graph:, ldpath_map:, subject_uri:, prefixes: {})
           ldpath_map.each do |key, ldpath|
             next value_map[key] = [subject_uri] if ldpath == :subject_uri
             ldpath_program = ldpath_service.ldpath_program(ldpath: ldpath, prefixes: prefixes)
-            values = ldpath_service.ldpath_evaluate(program: ldpath_program, graph: graph, subject_uri: subject_uri)
+            values = ldpath_service.ldpath_evaluate(program: ldpath_program, graph: graph, subject_uri: subject_uri, maintain_literals: true)
             value_map[key] = values
           end
           value_map = yield value_map if block_given?

qa.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'activerecord-import'
   s.add_dependency 'deprecation'
   s.add_dependency 'faraday'
-  s.add_dependency 'ldpath'
+  # s.add_dependency 'ldpath'
   s.add_dependency 'nokogiri', '~> 1.6'
   s.add_dependency 'rails', '~> 5.0'
   s.add_dependency 'rdf'

spec/fixtures/authorities/linked_data/lod_lang_defaults.json

@@ -1,5 +1,8 @@
 {
   "QA_CONFIG_VERSION": "2.0",
+  "prefixes": {
+    "dcterms": "http://purl.org/dc/terms/"
+  },
   "term": {
     "url": {
       "@context": "http://www.w3.org/ns/hydra/context.jsonld",
@@ -21,9 +24,9 @@
     "term_id": "URI",
     "language": [ "fr" ],
     "results": {
-      "id_predicate":       "http://id.loc.gov/vocabulary/identifiers/lccn",
-      "label_predicate":    "http://www.w3.org/2004/02/skos/core#prefLabel",
-      "altlabel_predicate": "http://www.w3.org/2004/02/skos/core#altLabel"
+      "id_ldpath":       "loc:lccn",
+      "label_ldpath":    "skos:prefLabel",
+      "altlabel_ldpath": "skos:altLabel"
     }
   },
   "search": {
@@ -46,10 +49,10 @@
     },
     "language": [ "fr" ],
     "results": {
-      "id_predicate":       "http://purl.org/dc/terms/identifier",
-      "label_predicate":    "http://www.w3.org/2004/02/skos/core#prefLabel",
-      "altlabel_predicate": "http://www.w3.org/2004/02/skos/core#altLabel",
-      "sort_predicate":     "http://www.w3.org/2004/02/skos/core#prefLabel"
+      "id_ldpath":       "dcterms:identifier",
+      "label_ldpath":    "skos:prefLabel",
+      "altlabel_ldpath": "skos:altLabel",
+      "sort_ldpath":     "skos:prefLabel"
     }
   }
 }

spec/fixtures/authorities/linked_data/lod_lang_multi_defaults.json

@@ -1,5 +1,8 @@
 {
   "QA_CONFIG_VERSION": "2.0",
+  "prefixes": {
+    "dcterms": "http://purl.org/dc/terms/"
+  },
   "term": {
     "url": {
       "@context": "http://www.w3.org/ns/hydra/context.jsonld",
@@ -21,9 +24,9 @@
     "term_id": "URI",
     "language": [ "en", "fr" ],
     "results": {
-      "id_predicate":       "http://purl.org/dc/terms/identifier",
-      "label_predicate":    "http://www.w3.org/2004/02/skos/core#prefLabel",
-      "altlabel_predicate": "http://www.w3.org/2004/02/skos/core#altLabel"
+      "id_ldpath":       "dcterms:identifier",
+      "label_ldpath":    "skos:prefLabel",
+      "altlabel_ldpath": "skos:altLabel"
     }
   },
   "search": {
@@ -46,10 +49,10 @@
     },
     "language": [ "en", "fr" ],
     "results": {
-      "id_predicate":       "http://purl.org/dc/terms/identifier",
-      "label_predicate":    "http://www.w3.org/2004/02/skos/core#prefLabel",
-      "altlabel_predicate": "http://www.w3.org/2004/02/skos/core#altLabel",
-      "sort_predicate":     "http://www.w3.org/2004/02/skos/core#prefLabel"
+      "id_ldpath":       "dcterms:identifier",
+      "label_ldpath":    "skos:prefLabel",
+      "altlabel_ldpath": "skos:altLabel",
+      "sort_ldpath":     "skos:prefLabel"
     }
   }
 }

spec/fixtures/authorities/linked_data/lod_lang_no_defaults.json

@@ -1,5 +1,8 @@
 {
   "QA_CONFIG_VERSION": "2.0",
+  "prefixes": {
+    "dcterms": "http://purl.org/dc/terms/"
+  },
   "term": {
     "url": {
       "@context": "http://www.w3.org/ns/hydra/context.jsonld",
@@ -20,8 +23,8 @@
     },
     "term_id": "URI",
     "results": {
-      "label_predicate":    "http://www.w3.org/2004/02/skos/core#prefLabel",
-      "altlabel_predicate": "http://www.w3.org/2004/02/skos/core#altLabel"
+      "label_ldpath":    "skos:prefLabel",
+      "altlabel_ldpath": "skos:altLabel"
     }
   },
   "search": {
@@ -43,10 +46,10 @@
       "query": "query"
     },
     "results": {
-      "id_predicate":       "http://purl.org/dc/terms/identifier",
-      "label_predicate":    "http://www.w3.org/2004/02/skos/core#prefLabel",
-      "altlabel_predicate": "http://www.w3.org/2004/02/skos/core#altLabel",
-      "sort_predicate":     "http://www.w3.org/2004/02/skos/core#prefLabel"
+      "id_ldpath":       "dcterms:identifier",
+      "label_ldpath":    "skos:prefLabel",
+      "altlabel_ldpath": "skos:altLabel",
+      "sort_ldpath":     "skos:prefLabel"
     }
   }
 }

spec/fixtures/authorities/linked_data/lod_lang_param.json

@@ -1,5 +1,9 @@
 {
   "QA_CONFIG_VERSION": "2.0",
+  "prefixes": {
+    "dcterms": "http://purl.org/dc/terms/",
+    "loc":     "http://id.loc.gov/vocabulary/identifiers/"
+  },
   "term": {
     "url": {
       "@context": "http://www.w3.org/ns/hydra/context.jsonld",
@@ -27,9 +31,9 @@
     },
     "term_id": "URI",
     "results": {
-      "id_predicate":       "http://id.loc.gov/vocabulary/identifiers/lccn",
-      "label_predicate":    "http://www.w3.org/2004/02/skos/core#prefLabel",
-      "altlabel_predicate": "http://www.w3.org/2004/02/skos/core#altLabel"
+      "id_ldpath":       "loc:lccn",
+      "label_ldpath":    "skos:prefLabel",
+      "altlabel_ldpath": "skos:altLabel"
     }
   },
   "search": {
@@ -58,10 +62,10 @@
       "query": "query"
     },
     "results": {
-      "id_predicate":       "http://purl.org/dc/terms/identifier",
-      "label_predicate":    "http://www.w3.org/2004/02/skos/core#prefLabel",
-      "altlabel_predicate": "http://www.w3.org/2004/02/skos/core#altLabel",
-      "sort_predicate":     "http://www.w3.org/2004/02/skos/core#prefLabel"
+      "id_ldpath":       "dcterms:identifier",
+      "label_ldpath":    "skos:prefLabel",
+      "altlabel_ldpath": "skos:altLabel",
+      "sort_ldpath":     "skos:prefLabel"
     }
   }
 }

spec/fixtures/authorities/linked_data/lod_min_config.json

@@ -1,5 +1,8 @@
 {
   "QA_CONFIG_VERSION": "2.0",
+  "prefixes": {
+    "loc": "http://id.loc.gov/vocabulary/identifiers/"
+  },
   "term": {
     "url": {
       "@context": "http://www.w3.org/ns/hydra/context.jsonld",
@@ -20,8 +23,8 @@
     },
     "term_id": "URI",
     "results": {
-      "id_predicate":       "http://id.loc.gov/vocabulary/identifiers/lccn",
-      "label_predicate":    "http://www.w3.org/2004/02/skos/core#prefLabel"
+      "id_ldpath":    "loc:lccn",
+      "label_ldpath": "skos:prefLabel"
     }
   },
   "search": {
@@ -43,7 +46,7 @@
       "query": "query"
     },
     "results": {
-      "label_predicate":    "http://www.w3.org/2004/02/skos/core#prefLabel"
+      "label_ldpath": "skos:prefLabel"
     }
   }
 }

spec/models/linked_data/config/context_property_map_spec.rb

@@ -277,12 +277,12 @@
     let(:expanded_id) { '123' }
 
     before do
-      allow(Ldpath::Program).to receive(:parse).with('property = madsrdf:identifiesRWO/madsrdf:birthDate/schema:label ;').and_return(basic_program)
-      allow(Ldpath::Program).to receive(:parse).with('property = skos:prefLabel ::xsd:string ;').and_return(expanded_label_program)
-      allow(Ldpath::Program).to receive(:parse).with('property = loc:lccn ::xsd:string ;').and_return(expanded_id_program)
-      allow(basic_program).to receive(:evaluate).with(subject_uri, context: graph, limit_to_context: true).and_return('property' => [expanded_uri])
-      allow(expanded_label_program).to receive(:evaluate).with(RDF::URI.new(subject_uri), context: graph, limit_to_context: true).and_return('property' => [expanded_label])
-      allow(expanded_id_program).to receive(:evaluate).with(RDF::URI.new(subject_uri), context: graph, limit_to_context: true).and_return('property' => [expanded_id])
+      allow(Ldpath::Program).to receive(:parse).with("property = madsrdf:identifiesRWO/madsrdf:birthDate/schema:label ;\n").and_return(basic_program)
+      allow(Ldpath::Program).to receive(:parse).with("property = skos:prefLabel ::xsd:string ;\n").and_return(expanded_label_program)
+      allow(Ldpath::Program).to receive(:parse).with("property = loc:lccn ::xsd:string ;\n").and_return(expanded_id_program)
+      allow(basic_program).to receive(:evaluate).with(subject_uri, context: graph, limit_to_context: true, maintain_literals: false).and_return('property' => [expanded_uri])
+      allow(expanded_label_program).to receive(:evaluate).with(RDF::URI.new(subject_uri), context: graph, limit_to_context: true, maintain_literals: false).and_return('property' => [expanded_label])
+      allow(expanded_id_program).to receive(:evaluate).with(RDF::URI.new(subject_uri), context: graph, limit_to_context: true, maintain_literals: false).and_return('property' => [expanded_id])
     end
     it 'returns the uri, id, label for the expanded uri value' do
       expanded_values = subject.expanded_values(graph, subject_uri).first