WIP GobiertoData add dcat catalog endpoint

app/controllers/gobierto_data/api/v1/datasets_controller.rb

@@ -11,6 +11,14 @@ class DatasetsController < BaseController
         skip_before_action :authenticate_in_site, only: [:new, :create, :update, :destroy]
         skip_before_action :set_admin_with_token, except: [:new, :create, :update, :destroy]
 
+        # GET /api/v1/data/catalog.xml
+        def catalog
+          @catalog = DatasetPresenter.new(current_site).build_catalog
+          respond_to do |format|
+            format.xml
+          end
+        end
+
         # GET /api/v1/data/datasets
         # GET /api/v1/data/datasets.json
         # GET /api/v1/data/datasets.csv

app/models/gobierto_data/dataset.rb

@@ -11,6 +11,7 @@ class Dataset < ApplicationRecord
     include GobiertoAttachments::Attachable
     include GobiertoCommon::Collectionable
     include GobiertoCommon::Searchable
+    include GobiertoCommon::HasCustomFieldRecords
 
     multisearchable(
       against: [:name_translations ,:name_en, :name_es],
@@ -30,6 +31,7 @@ class Dataset < ApplicationRecord
     has_many :visualizations, dependent: :destroy, class_name: "GobiertoData::Visualization"
 
     scope :sorted, -> { order(data_updated_at: :desc) }
+    scope :visibles, -> { where(visibility_level: "active") }
 
     translates :name
 

app/presenters/gobierto_data/dataset_presenter.rb

@@ -0,0 +1,72 @@
+module GobiertoData
+  class DatasetPresenter
+    include ActionView::Helpers::UrlHelper
+    include ActionView::Helpers::TranslationHelper
+
+    attr_reader :site
+
+    def initialize(site)
+      @site = site
+    end
+
+    def build_catalog
+      catalog = {
+        identifier_uri: url_helpers.gobierto_data_root_url(host: site.domain),
+        title:          I18n.t('presenters.gobierto_data.catalog.title',site: site),
+        description:    I18n.t('presenters.gobierto_data.catalog.description',site: site, publisher_url: url_helpers.gobierto_data_root_url(host: site.domain)),
+        issued:         site.created_at,
+        modified:       site.updated_at,
+        language:       site_locale,
+        homepage:       url_helpers.gobierto_data_root_url(host: site.domain),
+        datasets:       build_datasets_for_catalog
+      }
+    end
+
+    private
+
+    def build_datasets_for_catalog
+      site.datasets.visibles.map do |dataset|
+        build_dataset_for_catalog(dataset)
+      end
+    end
+
+    def build_dataset_for_catalog(dataset)
+      {
+        url:            url_helpers.gobierto_data_datasets_url(host: site.domain, id: dataset.slug),
+        title:          dataset.name,
+        description:    description_custom_field_record(dataset),
+        issued:         dataset.created_at,
+        modified:       dataset.updated_at,
+        languages:      [site_locale],
+        publisher:      site.name,
+        publisher_mbox: site.reply_to_email,
+        distributions:  build_distribution_for_catalog(dataset)
+      }
+    end
+
+    def build_distribution_for_catalog(dataset)
+      [
+        {
+          format:       'application/csv',
+          download_url: url_helpers.download_gobierto_data_api_v1_dataset_url(slug: dataset.slug, host: site.domain)
+        }
+      ]
+    end
+
+    def url_helpers
+      Rails.application.routes.url_helpers
+    end
+
+    def site_locale
+      site.configuration.default_locale
+    end
+
+    def description_custom_field_record(dataset)
+      if dataset.custom_field_record_with_uid("description")
+        dataset.custom_field_record_with_uid("description").payload["description"][site_locale]
+      else
+        ""
+      end
+    end
+  end
+end

app/views/gobierto_data/api/v1/datasets/catalog.xml.erb

@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<rdf:RDF
+ xmlns:time="http://www.w3.org/2006/time#"
+ xmlns:dct="http://purl.org/dc/terms/"
+ xmlns:dcat="http://www.w3.org/ns/dcat#"
+ xmlns:foaf="http://xmlns.com/foaf/0.1/"
+ xmlns:xsd="http://www.w3.org/2001/XMLSchema#"
+ xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
+ xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+ <dcat:Catalog rdf:about="https://data.some.org/catalog">
+   <dct:identifier><%= @catalog[:identifier_uri] %></dct:identifier>
+   <dct:title><%= @catalog[:title] %></dct:title>
+   <dct:description><%= @catalog[:description] %></dct:description>
+   <dct:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime"><%= @catalog[:issued] %></dct:issued>
+   <dct:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime"><%= @catalog[:modified] %></dct:modified>
+   <dct:language><%= @catalog[:language] %></dct:language>
+   <foaf:homepage rdf:resource="<%= @catalog[:homepage] %>"/>
+   <% @catalog[:datasets].each do |dataset| %><dcat:dataset>
+     <dcat:Dataset rdf:about="<%= dataset[:url]%>">
+       <dct:identifier><%= dataset[:url] %></dct:identifier>
+       <dct:title><%= dataset[:title] %></dct:title>
+       <dct:description><%= dataset[:description] %></dct:description>
+       <dct:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime"><%= dataset[:issued] %></dct:issued>
+       <dct:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime"><%= dataset[:modified] %></dct:modified>
+       <%= dataset[:languages].map { |lang| "<dct:language>#{lang}</dct:language>" }.join.html_safe %>
+       <dct:license rdf:resource="<%= dataset[:title] %>"/>
+       <dct:publisher>
+         <foaf:Organization>
+             <foaf:name><%= dataset[:publisher] %></foaf:name>
+             <foaf:mbox><%= dataset[:publisher_mbox] %></foaf:mbox>
+         </foaf:Organization>
+       </dct:publisher>
+       <% dataset[:distributions].each do |distribution| %>
+       <dcat:distribution>
+         <dcat:Distribution>
+           <dct:identifier><%= @identifier_uri %></dct:identifier>
+           <dct:title><%= dataset[:title] %></dct:title>
+           <dct:description><%= dataset[:description] %></dct:description>
+           <dcat:downloadURL rdf:datatype="http://www.w3.org/2001/XMLSchema#anyURI"><%= distribution[:download_url] %></dcat:downloadURL>
+           <dcat:mediaType><%= distribution[:format] %></dcat:mediaType>
+         </dcat:Distribution>
+       </dcat:distribution>
+       <% end %>
+     </dcat:Dataset>
+   </dcat:dataset><% end %>
+ </dcat:Catalog>
+ </rdf:RDF>

config/locales/gobierto_data/presenters/ca.yml

@@ -0,0 +1,8 @@
+---
+ca:
+  presenters:
+    gobierto_data:
+      catalog:
+        description: Catàleg public de dades dels conjunts de dades publicades per%
+          {site}, a través de la URL %{publisher_url}
+        title: Catàleg DCAT de conjunts de dades de %{site} en format rdf/xml dcat

config/locales/gobierto_data/presenters/en.yml

@@ -0,0 +1,8 @@
+---
+en:
+  presenters:
+    gobierto_data:
+      catalog:
+        description: Public catalog with datasets published by %{site}, through URL
+          %{publisher_url}
+        title: Catalog for datasets of %{site} into format rdf/xml dcat

config/locales/gobierto_data/presenters/es.yml

@@ -0,0 +1,9 @@
+---
+es:
+  presenters:
+    gobierto_data:
+      catalog:
+        description: Catalogo publico de datos los conjuntos de datos publicados por
+          %{site}, a través de la URL %{publisher_url}
+        title: Catalogo DCAT para los conjuntos de datos de %{site} en formato rdf/xml
+          dcat

config/routes.rb

@@ -641,6 +641,7 @@
               resources :favorites, only: [:index]
               collection do
                 get :meta
+                get :catalog
               end
               member do
                 get "meta" => "datasets#dataset_meta"

test/controllers/gobierto_data/api/v1/datasets_controller_test.rb

@@ -67,6 +67,10 @@ def datasets_md_with_translations
           @datasets_md_with_translations ||= gobierto_common_custom_fields(:madrid_data_datasets_custom_field_md_with_translations)
         end
 
+        def datasets_descriptions
+          @datasets_descriptions ||= gobierto_common_custom_fields(:madrid_data_datasets_custom_field_description)
+        end
+
         def other_site_dataset
           @other_site_dataset ||= gobierto_data_datasets(:santander_dataset)
         end
@@ -93,7 +97,8 @@ def array_data(dataset)
             dataset.rails_model&.columns_hash&.transform_values(&:type)&.to_s,
             GobiertoCommon::CustomFieldRecord.find_by(item: dataset, custom_field: datasets_category)&.value_string,
             GobiertoCommon::CustomFieldRecord.find_by(item: dataset, custom_field: datasets_md_without_translations)&.value_string,
-            GobiertoCommon::CustomFieldRecord.find_by(item: dataset, custom_field: datasets_md_with_translations)&.value_string
+            GobiertoCommon::CustomFieldRecord.find_by(item: dataset, custom_field: datasets_md_with_translations)&.value_string || "",
+            GobiertoCommon::CustomFieldRecord.find_by(item: dataset, custom_field: datasets_descriptions)&.value_string
           ]
         end
 
@@ -168,8 +173,9 @@ def test_index_as_csv
             parsed_csv = CSV.parse(response_data).map { |row| row.map(&:to_s) }
 
             assert_equal active_datasets_count + 1, parsed_csv.count
-            assert_equal %w(id name slug table_name data_updated_at columns category md-without-translations md-with-translations), parsed_csv.first
-            assert_includes parsed_csv, array_data(dataset)
+            assert_equal %w(id name slug table_name data_updated_at columns category md-without-translations md-with-translations description-datasets), parsed_csv.first
+
+            assert_includes parsed_csv.drop(1).take(1), array_data(dataset)
             refute_includes parsed_csv, array_data(other_site_dataset)
           end
         end
@@ -202,7 +208,7 @@ def test_index_xlsx_format
             assert_equal 1, parsed_xlsx.worksheets.count
             sheet = parsed_xlsx.worksheets.first
             assert_nil sheet[active_datasets_count + 1]
-            assert_equal %w(id name slug table_name data_updated_at columns category md-without-translations md-with-translations), sheet[0].cells.map(&:value)
+            assert_equal %w(id name slug table_name data_updated_at columns category md-without-translations md-with-translations description-datasets), sheet[0].cells.map(&:value)
             values = (1..active_datasets_count).map do |row_number|
               sheet[row_number].cells.map { |cell| cell.value.to_s }
             end
@@ -315,12 +321,34 @@ def test_index_when_md_custom_field_changes_translations_availability
             parsed_csv = CSV.parse(response_data).map { |row| row.map(&:to_s) }
 
             assert_equal active_datasets_count + 1, parsed_csv.count
-            assert_equal %w(id name slug table_name data_updated_at columns category md-without-translations md-with-translations), parsed_csv.first
+            assert_equal %w(id name slug table_name data_updated_at columns category md-without-translations md-with-translations description-datasets), parsed_csv.first
             assert_includes parsed_csv, array_data(dataset)
             refute_includes parsed_csv, array_data(other_site_dataset)
           end
         end
 
+        def test_catalog
+          with(site: site) do
+            get catalog_gobierto_data_api_v1_datasets_path(format: :xml), as: :xml
+            assert_response :success
+
+            response_xml = response.parsed_body
+            expected =  File.read("test/fixtures/gobierto_data/catalog.xml")
+            assert_equal response_xml, expected
+          end
+        end
+
+        def test_catalog_dont_show_draft_dataset
+          with(site: site) do
+            get catalog_gobierto_data_api_v1_datasets_path(format: :xml), as: :xml
+            assert_response :success
+            response_xml = response.parsed_body
+
+            refute_includes response_xml, 'Interest Groups'
+            refute_includes response_xml, 'Grupos de Interés'
+          end
+        end
+
       end
     end
   end

test/fixtures/gobierto_common/custom_field_records.yml

@@ -391,6 +391,11 @@ users_dataset_md_with_translations_custom_field_record:
     "md-with-translations" => { en: "Paragraph with translations!", es: "¡Párrafo con traducciones!" }
    }.to_json %>
 
+users_dataset_md_with_translations_custom_field_record:
+  item: users_dataset (GobiertoData::Dataset)
+  custom_field: madrid_data_datasets_custom_field_description
+  payload: <%= {"description": {"en": "This dataset contains the council' persons related.", "es": "Este dataset contiene las personas relaccionadas con el ayuntamiento."}}.to_json %>
+
 ## Users
 
 peter_custom_field_issue:

test/fixtures/gobierto_common/custom_fields.yml

@@ -344,6 +344,14 @@ madrid_data_datasets_custom_field_md_with_translations:
   field_type: <%= GobiertoCommon::CustomField.field_types[:localized_paragraph] %>
   uid: md-with-translations
 
+madrid_data_datasets_custom_field_description:
+  site: madrid
+  class_name: GobiertoData::Dataset
+  position: 4
+  name_translations: <%= {"en": "Description", "es": "Descripción"}.to_json %>
+  field_type: <%= GobiertoCommon::CustomField.field_types[:localized_paragraph] %>
+  uid: description-datasets
+
 madrid_custom_field_human_resources_table_plugin:
   site: madrid
   class_name: GobiertoPlans::Node

test/fixtures/gobierto_data/catalog.xml

@@ -0,0 +1,95 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<rdf:RDF
+ xmlns:time="http://www.w3.org/2006/time#"
+ xmlns:dct="http://purl.org/dc/terms/"
+ xmlns:dcat="http://www.w3.org/ns/dcat#"
+ xmlns:foaf="http://xmlns.com/foaf/0.1/"
+ xmlns:xsd="http://www.w3.org/2001/XMLSchema#"
+ xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
+ xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+ <dcat:Catalog rdf:about="https://data.some.org/catalog">
+   <dct:identifier>http://madrid.gobierto.test/datos</dct:identifier>
+   <dct:title>Catalog for datasets of Ayuntamiento de Madrid into format rdf/xml dcat</dct:title>
+   <dct:description>Public catalog with datasets published by Ayuntamiento de Madrid, through URL http://madrid.gobierto.test/datos</dct:description>
+   <dct:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2019-01-01 08:00:00 +0100</dct:issued>
+   <dct:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2019-01-01 08:00:00 +0100</dct:modified>
+   <dct:language>en</dct:language>
+   <foaf:homepage rdf:resource="http://madrid.gobierto.test/datos"/>
+   <dcat:dataset>
+     <dcat:Dataset rdf:about="http://madrid.gobierto.test/datos/users-dataset">
+       <dct:identifier>http://madrid.gobierto.test/datos/users-dataset</dct:identifier>
+       <dct:title>Users</dct:title>
+       <dct:description></dct:description>
+       <dct:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2020-01-02 08:00:00 +0100</dct:issued>
+       <dct:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2020-01-02 08:00:00 +0100</dct:modified>
+       <dct:language>en</dct:language>
+       <dct:license rdf:resource="Users"/>
+       <dct:publisher>
+         <foaf:Organization>
+             <foaf:name>Ayuntamiento de Madrid</foaf:name>
+             <foaf:mbox>[email protected]</foaf:mbox>
+         </foaf:Organization>
+       </dct:publisher>
+       <dcat:distribution>
+         <dcat:Distribution>
+           <dct:identifier></dct:identifier>
+           <dct:title>Users</dct:title>
+           <dct:description></dct:description>
+           <dcat:downloadURL rdf:datatype="http://www.w3.org/2001/XMLSchema#anyURI">http://madrid.gobierto.test/api/v1/data/datasets/users-dataset/download.csv</dcat:downloadURL>
+           <dcat:mediaType>application/csv</dcat:mediaType>
+         </dcat:Distribution>
+       </dcat:distribution>
+     </dcat:Dataset>
+   </dcat:dataset><dcat:dataset>
+     <dcat:Dataset rdf:about="http://madrid.gobierto.test/datos/events-dataset">
+       <dct:identifier>http://madrid.gobierto.test/datos/events-dataset</dct:identifier>
+       <dct:title>Events</dct:title>
+       <dct:description></dct:description>
+       <dct:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2020-01-03 08:00:00 +0100</dct:issued>
+       <dct:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2020-01-03 08:00:00 +0100</dct:modified>
+       <dct:language>en</dct:language>
+       <dct:license rdf:resource="Events"/>
+       <dct:publisher>
+         <foaf:Organization>
+             <foaf:name>Ayuntamiento de Madrid</foaf:name>
+             <foaf:mbox>[email protected]</foaf:mbox>
+         </foaf:Organization>
+       </dct:publisher>
+       <dcat:distribution>
+         <dcat:Distribution>
+           <dct:identifier></dct:identifier>
+           <dct:title>Events</dct:title>
+           <dct:description></dct:description>
+           <dcat:downloadURL rdf:datatype="http://www.w3.org/2001/XMLSchema#anyURI">http://madrid.gobierto.test/api/v1/data/datasets/events-dataset/download.csv</dcat:downloadURL>
+           <dcat:mediaType>application/csv</dcat:mediaType>
+         </dcat:Distribution>
+       </dcat:distribution>
+     </dcat:Dataset>
+   </dcat:dataset><dcat:dataset>
+     <dcat:Dataset rdf:about="http://madrid.gobierto.test/datos/no-size">
+       <dct:identifier>http://madrid.gobierto.test/datos/no-size</dct:identifier>
+       <dct:title>No size</dct:title>
+       <dct:description></dct:description>
+       <dct:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2020-01-06 08:00:00 +0100</dct:issued>
+       <dct:modified rdf:datatype="http://www.w3.org/2001/XMLSchema#dateTime">2020-01-06 08:00:00 +0100</dct:modified>
+       <dct:language>en</dct:language>
+       <dct:license rdf:resource="No size"/>
+       <dct:publisher>
+         <foaf:Organization>
+             <foaf:name>Ayuntamiento de Madrid</foaf:name>
+             <foaf:mbox>[email protected]</foaf:mbox>
+         </foaf:Organization>
+       </dct:publisher>
+       <dcat:distribution>
+         <dcat:Distribution>
+           <dct:identifier></dct:identifier>
+           <dct:title>No size</dct:title>
+           <dct:description></dct:description>
+           <dcat:downloadURL rdf:datatype="http://www.w3.org/2001/XMLSchema#anyURI">http://madrid.gobierto.test/api/v1/data/datasets/no-size/download.csv</dcat:downloadURL>
+           <dcat:mediaType>application/csv</dcat:mediaType>
+         </dcat:Distribution>
+       </dcat:distribution>
+     </dcat:Dataset>
+   </dcat:dataset>
+ </dcat:Catalog>
+ </rdf:RDF>