From e4201cfb9786b11d2c23dda0638bfac1f9e4b800 Mon Sep 17 00:00:00 2001 From: stbn Date: Tue, 27 Apr 2021 15:03:58 +0200 Subject: [PATCH 1/8] WIP ci skip --- .../api/v1/datasets_controller.rb | 36 +++++++++++++ app/models/gobierto_data/dataset.rb | 1 + .../api/v1/datasets/catalog.xml.erb | 50 +++++++++++++++++++ config/routes.rb | 1 + 4 files changed, 88 insertions(+) create mode 100644 app/views/gobierto_data/api/v1/datasets/catalog.xml.erb diff --git a/app/controllers/gobierto_data/api/v1/datasets_controller.rb b/app/controllers/gobierto_data/api/v1/datasets_controller.rb index e17832b8cf..4006582875 100644 --- a/app/controllers/gobierto_data/api/v1/datasets_controller.rb +++ b/app/controllers/gobierto_data/api/v1/datasets_controller.rb @@ -11,6 +11,42 @@ class DatasetsController < BaseController skip_before_action :authenticate_in_site, only: [:new, :create, :update, :destroy] skip_before_action :set_admin_with_token, except: [:new, :create, :update, :destroy] + def catalog + @catalog_identifier_uri = gobierto_data_root_url + @catalog_title = "dcat catalog for #{current_site}" + @catalog_description = "this is catalog published by #{@catalog_identifier_uri} which contains datasets" + @catalog_issued = current_site.created_at + @catalog_modified = GobiertoData::Dataset.where(site_id: current_site.id).maximum(:created_at) || current_site.created_at + @catalog_languages = current_site.configuration.available_locales + @catalog_homepage = gobierto_data_root_url + @catalog_license_url = "https://opendatacommons.org/licenses/odbl/" + + @catalog_datasets = [] + GobiertoData::Dataset.where(site_id: current_site.id).each do |dataset| + @catalog_datasets << { + dataset_url: gobierto_data_datasets_url(id: dataset.slug), + dataset_title: dataset.name, + dataset_description: dataset.custom_field_record_with_uid("description").payload["description"][preferred_locale], + dataset_keywords: [], + dataset_issued: dataset.created_at, + dataset_modified: dataset.updated_at, + dataset_languages: [preferred_locale], + dataset_license_url: "https://opendatacommons.org/licenses/odbl/", # FIX license belongs to datase + dataset_publisher: current_site.name, + dataset_publisher_mbox: current_site.reply_to_email, + dataset_distribution: [ { + format: 'application/csv', + download_url: download_gobierto_data_api_v1_dataset_url(dataset) # FIX link don't work directly + } ] + } + end + + respond_to do |format| + format.xml + end + end + + # GET /api/v1/data/datasets # GET /api/v1/data/datasets.json # GET /api/v1/data/datasets.csv diff --git a/app/models/gobierto_data/dataset.rb b/app/models/gobierto_data/dataset.rb index 613b9d435f..09f29ff29e 100644 --- a/app/models/gobierto_data/dataset.rb +++ b/app/models/gobierto_data/dataset.rb @@ -10,6 +10,7 @@ class Dataset < ApplicationRecord include GobiertoData::Favoriteable include GobiertoAttachments::Attachable include GobiertoCommon::Collectionable + include GobiertoCommon::HasCustomFieldRecords belongs_to :site has_many :queries, dependent: :destroy, class_name: "GobiertoData::Query" diff --git a/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb b/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb new file mode 100644 index 0000000000..130a84bf23 --- /dev/null +++ b/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb @@ -0,0 +1,50 @@ + + + + <%= @identifier_uri %> + <%= @catalog_title %> + <%= @catalog_description %> + <%= @catalog_issued %> + <%= @catalog_modified %> + <%= @catalog_languages.map { |locale| "#{locale}" }.join.html_safe %> + + + <% @catalog_datasets.each do |dataset| %> + + <%= dataset[:dataset_url] %> + <%= dataset[:dataset_title] %> + <%= dataset[:dataset_description] %> + <%= dataset[:dataset_keywords].map { |keyword| "#{locale}" }.join.html_safe %> + <%= dataset[:dataset_issued] %> + <%= dataset[:dataset_modified] %> + <%= dataset[:dataset_languages].map { |lang| "#{lang}" }.join.html_safe %> + + + + <%= dataset[:dataset_publisher] %> + <%= dataset[:dataset_publisher_mbox] %> + + + <% dataset[:dataset_distribution].each do |distribution| %> + + + <%= @identifier_uri %> + <%= dataset[:dataset_title] %> in CSV format + <%= dataset[:dataset_description] %> + <%= distribution[:download_url] %> + <%= distribution[:format] %> + + + + <% end %> + + <% end %> + + diff --git a/config/routes.rb b/config/routes.rb index 4fa394b01a..fa04efc454 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -664,6 +664,7 @@ resource :favorite, only: [:create, :destroy] resources :favorites, only: [:index] end + get "catalog" => "datasets#catalog" end end end From 4efaad6e8e3f8c20144fa15cf76aeb4be006cb90 Mon Sep 17 00:00:00 2001 From: stbn Date: Wed, 28 Apr 2021 16:37:47 +0200 Subject: [PATCH 2/8] mv catalog generation to presenter & test ci skip --- .../api/v1/datasets_controller.rb | 36 +-------- app/models/gobierto_data/dataset.rb | 2 + .../gobierto_data/dataset_presenter.rb | 76 +++++++++++++++++++ .../api/v1/datasets/catalog.xml.erb | 52 ++++++------- .../gobierto_common/custom_field_records.yml | 5 ++ .../gobierto_common/custom_fields.yml | 8 ++ test/fixtures/sites.yml | 2 + .../gobierto_data/dataset_presenter_test.rb | 57 ++++++++++++++ 8 files changed, 180 insertions(+), 58 deletions(-) create mode 100644 app/presenters/gobierto_data/dataset_presenter.rb create mode 100644 test/presenters/gobierto_data/dataset_presenter_test.rb diff --git a/app/controllers/gobierto_data/api/v1/datasets_controller.rb b/app/controllers/gobierto_data/api/v1/datasets_controller.rb index 4006582875..85c6aedff6 100644 --- a/app/controllers/gobierto_data/api/v1/datasets_controller.rb +++ b/app/controllers/gobierto_data/api/v1/datasets_controller.rb @@ -11,42 +11,14 @@ class DatasetsController < BaseController skip_before_action :authenticate_in_site, only: [:new, :create, :update, :destroy] skip_before_action :set_admin_with_token, except: [:new, :create, :update, :destroy] + # GET /api/v1/data/catalog.xml def catalog - @catalog_identifier_uri = gobierto_data_root_url - @catalog_title = "dcat catalog for #{current_site}" - @catalog_description = "this is catalog published by #{@catalog_identifier_uri} which contains datasets" - @catalog_issued = current_site.created_at - @catalog_modified = GobiertoData::Dataset.where(site_id: current_site.id).maximum(:created_at) || current_site.created_at - @catalog_languages = current_site.configuration.available_locales - @catalog_homepage = gobierto_data_root_url - @catalog_license_url = "https://opendatacommons.org/licenses/odbl/" - - @catalog_datasets = [] - GobiertoData::Dataset.where(site_id: current_site.id).each do |dataset| - @catalog_datasets << { - dataset_url: gobierto_data_datasets_url(id: dataset.slug), - dataset_title: dataset.name, - dataset_description: dataset.custom_field_record_with_uid("description").payload["description"][preferred_locale], - dataset_keywords: [], - dataset_issued: dataset.created_at, - dataset_modified: dataset.updated_at, - dataset_languages: [preferred_locale], - dataset_license_url: "https://opendatacommons.org/licenses/odbl/", # FIX license belongs to datase - dataset_publisher: current_site.name, - dataset_publisher_mbox: current_site.reply_to_email, - dataset_distribution: [ { - format: 'application/csv', - download_url: download_gobierto_data_api_v1_dataset_url(dataset) # FIX link don't work directly - } ] - } - end - + @catalog = DatasetPresenter.new(current_site).build_catalog respond_to do |format| - format.xml - end + format.xml + end end - # GET /api/v1/data/datasets # GET /api/v1/data/datasets.json # GET /api/v1/data/datasets.csv diff --git a/app/models/gobierto_data/dataset.rb b/app/models/gobierto_data/dataset.rb index 09f29ff29e..65a137c7fa 100644 --- a/app/models/gobierto_data/dataset.rb +++ b/app/models/gobierto_data/dataset.rb @@ -17,6 +17,8 @@ class Dataset < ApplicationRecord has_many :visualizations, dependent: :destroy, class_name: "GobiertoData::Visualization" scope :sorted, -> { order(data_updated_at: :desc) } + scope :visibles, -> { where(visibility_level: "active") } + scope :by_site, ->(site_id) { where(site_id: site_id) } translates :name diff --git a/app/presenters/gobierto_data/dataset_presenter.rb b/app/presenters/gobierto_data/dataset_presenter.rb new file mode 100644 index 0000000000..73e0153893 --- /dev/null +++ b/app/presenters/gobierto_data/dataset_presenter.rb @@ -0,0 +1,76 @@ +module GobiertoData + class DatasetPresenter + include ActionView::Helpers::UrlHelper + + attr_reader :site + + def initialize(site) + @site = site + end + + def build_catalog + catalog = { + identifier_uri: url_helpers.gobierto_data_root_url(host: site.domain), + title: "dcat catalog for #{site}", + description: "this is catalog published by #{@catalog_identifier_uri} which contains datasets", + issued: site.created_at, + modified: GobiertoData::Dataset.where(site_id: site.id).maximum(:created_at) || site.created_at, + languages: site["configuration_data"]["available_locales"], + homepage: url_helpers.gobierto_data_root_url(host: site.domain), + license_url: "https://opendatacommons.org/licenses/odbl/", + datasets: build_datasets_for_catalog + } + end + + private + + def build_datasets_for_catalog + datasets = [] + Dataset.where(site_id: site.id).visibles.each do |dataset| + datasets << build_dataset_for_catalog(dataset) + end + datasets + end + + def build_dataset_for_catalog(dataset) + { + url: url_helpers.gobierto_data_datasets_url(host: site.domain, id: dataset.slug), + title: dataset.name, + description: description_custom_field_record(dataset), + keywords: [], + issued: dataset.created_at, + modified: dataset.updated_at, + languages: [site_locale], + license_url: "https://opendatacommons.org/licenses/odbl/", + publisher: site.name, + publisher_mbox: site.reply_to_email, + distributions: build_distribution_for_catalog(dataset) + } + end + + def build_distribution_for_catalog(dataset) + [ + { + format: 'application/csv', + download_url: url_helpers.download_gobierto_data_api_v1_dataset_url(dataset, host: site.domain) + } + ] + end + + def url_helpers + Rails.application.routes.url_helpers + end + + def site_locale + site.configuration_data["default_locale"] + end + + def description_custom_field_record(dataset) + if dataset.custom_field_record_with_uid("description") + dataset.custom_field_record_with_uid("description").payload["description"][site_locale] + else + "" + end + end + end +end diff --git a/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb b/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb index 130a84bf23..2e84ea7a4d 100644 --- a/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb +++ b/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb @@ -6,45 +6,45 @@ xmlns:foaf="http://xmlns.com/foaf/0.1/" xmlns:xsd="http://www.w3.org/2001/XMLSchema#" xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#" - xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"><% cache @catalog do %> - <%= @identifier_uri %> - <%= @catalog_title %> - <%= @catalog_description %> - <%= @catalog_issued %> - <%= @catalog_modified %> - <%= @catalog_languages.map { |locale| "#{locale}" }.join.html_safe %> - - - <% @catalog_datasets.each do |dataset| %> - - <%= dataset[:dataset_url] %> - <%= dataset[:dataset_title] %> - <%= dataset[:dataset_description] %> - <%= dataset[:dataset_keywords].map { |keyword| "#{locale}" }.join.html_safe %> - <%= dataset[:dataset_issued] %> - <%= dataset[:dataset_modified] %> - <%= dataset[:dataset_languages].map { |lang| "#{lang}" }.join.html_safe %> - + <%= @catalog[:identifier_uri] %> + <%= @catalog[:title] %> + <%= @catalog[:description] %> + <%= @catalog[:issued] %> + <%= @catalog[:modified] %> + <%= @catalog[:languages].map { |locale| "#{locale}" }.join.html_safe %> + + + <% @catalog[:datasets].each do |dataset| %> + + <%= dataset[:url] %> + <%= dataset[:title] %> + <%= dataset[:description] %> + <%= dataset[:keywords].map { |keyword| "#{locale}" }.join.html_safe %> + <%= dataset[:issued] %> + <%= dataset[:modified] %> + <%= dataset[:languages].map { |lang| "#{lang}" }.join.html_safe %> + - <%= dataset[:dataset_publisher] %> - <%= dataset[:dataset_publisher_mbox] %> + <%= dataset[:publisher] %> + <%= dataset[:publisher_mbox] %> - <% dataset[:dataset_distribution].each do |distribution| %> + <% dataset[:distributions].each do |distribution| %> <%= @identifier_uri %> - <%= dataset[:dataset_title] %> in CSV format - <%= dataset[:dataset_description] %> + <%= dataset[:title] %> in CSV format + <%= dataset[:description] %> <%= distribution[:download_url] %> <%= distribution[:format] %> - + <% end %> <% end %> - + <% end %> diff --git a/test/fixtures/gobierto_common/custom_field_records.yml b/test/fixtures/gobierto_common/custom_field_records.yml index e652223dfb..a1ee44a82a 100644 --- a/test/fixtures/gobierto_common/custom_field_records.yml +++ b/test/fixtures/gobierto_common/custom_field_records.yml @@ -391,6 +391,11 @@ users_dataset_md_with_translations_custom_field_record: "md-with-translations" => { en: "Paragraph with translations!", es: "¡Párrafo con traducciones!" } }.to_json %> +users_dataset_md_with_translations_custom_field_record: + item: users_dataset (GobiertoData::Dataset) + custom_field: madrid_data_datasets_custom_field_description + payload: <%= {"description": {"en": "This dataset contains the council' persons related.", "es": "Este dataset contiene las personas relaccionadas con el ayuntamiento."}}.to_json %> + ## Users peter_custom_field_issue: diff --git a/test/fixtures/gobierto_common/custom_fields.yml b/test/fixtures/gobierto_common/custom_fields.yml index e007fd5fcd..580212c638 100644 --- a/test/fixtures/gobierto_common/custom_fields.yml +++ b/test/fixtures/gobierto_common/custom_fields.yml @@ -344,6 +344,14 @@ madrid_data_datasets_custom_field_md_with_translations: field_type: <%= GobiertoCommon::CustomField.field_types[:localized_paragraph] %> uid: md-with-translations +madrid_data_datasets_custom_field_description: + site: madrid + class_name: GobiertoData::Dataset + position: 4 + name_translations: <%= {"en": "Description", "es": "Descripción"}.to_json %> + field_type: <%= GobiertoCommon::CustomField.field_types[:localized_paragraph] %> + uid: description + madrid_custom_field_human_resources_table_plugin: site: madrid class_name: GobiertoPlans::Node diff --git a/test/fixtures/sites.yml b/test/fixtures/sites.yml index 09e2c52358..c2b6e097b9 100644 --- a/test/fixtures/sites.yml +++ b/test/fixtures/sites.yml @@ -34,6 +34,8 @@ madrid: organization_address: Fake St., 123 organization_document_number: 0123456789A visibility_level: <%= Site.visibility_levels["active"] %> + created_at: <%= 1.year.ago %> + updated_at: <%= 1.week.ago %> santander: title_translations: <%= { 'en' => 'Transparencia Ciudadana', 'es' => 'Transparencia Ciudadana' }.to_json %> diff --git a/test/presenters/gobierto_data/dataset_presenter_test.rb b/test/presenters/gobierto_data/dataset_presenter_test.rb new file mode 100644 index 0000000000..2dcdaca88d --- /dev/null +++ b/test/presenters/gobierto_data/dataset_presenter_test.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require "test_helper" + +module GobiertoData + class DatasetPresenterTest < ActiveSupport::TestCase + + def setup + super + @site = sites(:madrid) + @subject = DatasetPresenter.new(@site) + @other_site = sites(:huesca) + @other_subject = DatasetPresenter.new(@other_site) + end + + def test_structure_catalog_building_do_not_show_draft_datasets + catalog = @subject.build_catalog + datasets_published = GobiertoData::Dataset.by_site(@site.id).visibles.size + assert_equal datasets_published, catalog[:datasets].size + end + + def test_structure_catalog_building_using_site_with_datasets + catalog = @subject.build_catalog + assert catalog.has_key? :identifier_uri + assert catalog.has_key? :title + assert catalog.has_key? :description + assert catalog.has_key? :issued + assert catalog.has_key? :modified + assert catalog.has_key? :languages + assert catalog.has_key? :homepage + assert catalog.has_key? :license_url + assert catalog.has_key? :datasets + catalog[:datasets].each do |dataset| + assert dataset.has_key? :url + assert dataset.has_key? :title + assert dataset.has_key? :description + assert dataset.has_key? :keywords + assert dataset.has_key? :issued + assert dataset.has_key? :modified + assert dataset.has_key? :languages + assert dataset.has_key? :license_url + assert dataset.has_key? :publisher + assert dataset.has_key? :publisher_mbox + assert dataset.has_key? :distributions + dataset[:distributions].each do |distribution| + assert distribution.has_key? :format + assert distribution.has_key? :download_url + end + end + end + + def test_structure_catalog_building_using_site_without_datasets + catalog = @other_subject.build_catalog + assert_equal 0, catalog[:datasets].size + end + end +end From 57d273c7be49abae3203da7dd79a0020fbf874c2 Mon Sep 17 00:00:00 2001 From: stbn Date: Thu, 29 Apr 2021 11:30:50 +0200 Subject: [PATCH 3/8] fixes for review --- .../api/v1/datasets_controller.rb | 4 +- .../gobierto_data/dataset_presenter.rb | 38 +++++++++---------- .../api/v1/datasets/catalog.xml.erb | 5 +-- .../locales/gobierto_data/presenters/ca.yml | 8 ++++ .../locales/gobierto_data/presenters/en.yml | 8 ++++ .../locales/gobierto_data/presenters/es.yml | 9 +++++ config/routes.rb | 3 +- .../gobierto_data/dataset_presenter_test.rb | 5 +-- 8 files changed, 48 insertions(+), 32 deletions(-) create mode 100644 config/locales/gobierto_data/presenters/ca.yml create mode 100644 config/locales/gobierto_data/presenters/en.yml create mode 100644 config/locales/gobierto_data/presenters/es.yml diff --git a/app/controllers/gobierto_data/api/v1/datasets_controller.rb b/app/controllers/gobierto_data/api/v1/datasets_controller.rb index 85c6aedff6..4e52fe9297 100644 --- a/app/controllers/gobierto_data/api/v1/datasets_controller.rb +++ b/app/controllers/gobierto_data/api/v1/datasets_controller.rb @@ -15,8 +15,8 @@ class DatasetsController < BaseController def catalog @catalog = DatasetPresenter.new(current_site).build_catalog respond_to do |format| - format.xml - end + format.xml + end end # GET /api/v1/data/datasets diff --git a/app/presenters/gobierto_data/dataset_presenter.rb b/app/presenters/gobierto_data/dataset_presenter.rb index 73e0153893..733d995a5a 100644 --- a/app/presenters/gobierto_data/dataset_presenter.rb +++ b/app/presenters/gobierto_data/dataset_presenter.rb @@ -1,6 +1,7 @@ module GobiertoData class DatasetPresenter include ActionView::Helpers::UrlHelper + include ActionView::Helpers::TranslationHelper attr_reader :site @@ -11,13 +12,12 @@ def initialize(site) def build_catalog catalog = { identifier_uri: url_helpers.gobierto_data_root_url(host: site.domain), - title: "dcat catalog for #{site}", - description: "this is catalog published by #{@catalog_identifier_uri} which contains datasets", + title: I18n.t('presenters.gobierto_data.catalog.title',site: site), + description: I18n.t('presenters.gobierto_data.catalog.description',site: site, publisher_url: url_helpers.gobierto_data_root_url(host: site.domain)), issued: site.created_at, - modified: GobiertoData::Dataset.where(site_id: site.id).maximum(:created_at) || site.created_at, - languages: site["configuration_data"]["available_locales"], + modified: site.updated_at, + language: site_locale, homepage: url_helpers.gobierto_data_root_url(host: site.domain), - license_url: "https://opendatacommons.org/licenses/odbl/", datasets: build_datasets_for_catalog } end @@ -25,34 +25,30 @@ def build_catalog private def build_datasets_for_catalog - datasets = [] - Dataset.where(site_id: site.id).visibles.each do |dataset| - datasets << build_dataset_for_catalog(dataset) + site.datasets.visibles.map do |dataset| + build_dataset_for_catalog(dataset) end - datasets end def build_dataset_for_catalog(dataset) { - url: url_helpers.gobierto_data_datasets_url(host: site.domain, id: dataset.slug), - title: dataset.name, - description: description_custom_field_record(dataset), - keywords: [], - issued: dataset.created_at, - modified: dataset.updated_at, - languages: [site_locale], - license_url: "https://opendatacommons.org/licenses/odbl/", - publisher: site.name, + url: url_helpers.gobierto_data_datasets_url(host: site.domain, id: dataset.slug), + title: dataset.name, + description: description_custom_field_record(dataset), + issued: dataset.created_at, + modified: dataset.updated_at, + languages: [site_locale], + publisher: site.name, publisher_mbox: site.reply_to_email, - distributions: build_distribution_for_catalog(dataset) + distributions: build_distribution_for_catalog(dataset) } end def build_distribution_for_catalog(dataset) [ { - format: 'application/csv', - download_url: url_helpers.download_gobierto_data_api_v1_dataset_url(dataset, host: site.domain) + format: 'application/csv', + download_url: url_helpers.download_gobierto_data_api_v1_dataset_url(slug: dataset.slug, host: site.domain) } ] end diff --git a/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb b/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb index 2e84ea7a4d..f79c345384 100644 --- a/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb +++ b/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb @@ -13,15 +13,13 @@ <%= @catalog[:description] %> <%= @catalog[:issued] %> <%= @catalog[:modified] %> - <%= @catalog[:languages].map { |locale| "#{locale}" }.join.html_safe %> + <%= @catalog[:language] %> - <% @catalog[:datasets].each do |dataset| %> <%= dataset[:url] %> <%= dataset[:title] %> <%= dataset[:description] %> - <%= dataset[:keywords].map { |keyword| "#{locale}" }.join.html_safe %> <%= dataset[:issued] %> <%= dataset[:modified] %> <%= dataset[:languages].map { |lang| "#{lang}" }.join.html_safe %> @@ -40,7 +38,6 @@ <%= dataset[:description] %> <%= distribution[:download_url] %> <%= distribution[:format] %> - <% end %> diff --git a/config/locales/gobierto_data/presenters/ca.yml b/config/locales/gobierto_data/presenters/ca.yml new file mode 100644 index 0000000000..4b18ad0796 --- /dev/null +++ b/config/locales/gobierto_data/presenters/ca.yml @@ -0,0 +1,8 @@ +--- +ca: + presenters: + gobierto_data: + catalog: + description: Catàleg public de dades dels conjunts de dades publicades per% + {site}, a través de la URL% {publisher_url} + title: Catàleg DCAT de conjunts de dades de %{site} en format rdf/xml dcat diff --git a/config/locales/gobierto_data/presenters/en.yml b/config/locales/gobierto_data/presenters/en.yml new file mode 100644 index 0000000000..54874ba590 --- /dev/null +++ b/config/locales/gobierto_data/presenters/en.yml @@ -0,0 +1,8 @@ +--- +en: + presenters: + gobierto_data: + catalog: + description: Public catalog with datasets published by %{site}, through URL + %{publisher_url} + title: Catalog for datasets of %{site} into format rdf/xml dcat diff --git a/config/locales/gobierto_data/presenters/es.yml b/config/locales/gobierto_data/presenters/es.yml new file mode 100644 index 0000000000..e928281ffc --- /dev/null +++ b/config/locales/gobierto_data/presenters/es.yml @@ -0,0 +1,9 @@ +--- +es: + presenters: + gobierto_data: + catalog: + description: Catalogo publico de datos los conjuntos de datos publicados por + %{site}, a través de la URL %{publisher_url} + title: Catalogo DCAT para los conjuntos de datos de %{site} en formato rdf/xml + dcat diff --git a/config/routes.rb b/config/routes.rb index fa04efc454..6cc04b6007 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -641,6 +641,8 @@ resources :favorites, only: [:index] collection do get :meta + # get :catalog + get "catalog" => "datasets#catalog" end member do get "meta" => "datasets#dataset_meta" @@ -664,7 +666,6 @@ resource :favorite, only: [:create, :destroy] resources :favorites, only: [:index] end - get "catalog" => "datasets#catalog" end end end diff --git a/test/presenters/gobierto_data/dataset_presenter_test.rb b/test/presenters/gobierto_data/dataset_presenter_test.rb index 2dcdaca88d..76e9cb9389 100644 --- a/test/presenters/gobierto_data/dataset_presenter_test.rb +++ b/test/presenters/gobierto_data/dataset_presenter_test.rb @@ -26,19 +26,16 @@ def test_structure_catalog_building_using_site_with_datasets assert catalog.has_key? :description assert catalog.has_key? :issued assert catalog.has_key? :modified - assert catalog.has_key? :languages + assert catalog.has_key? :language assert catalog.has_key? :homepage - assert catalog.has_key? :license_url assert catalog.has_key? :datasets catalog[:datasets].each do |dataset| assert dataset.has_key? :url assert dataset.has_key? :title assert dataset.has_key? :description - assert dataset.has_key? :keywords assert dataset.has_key? :issued assert dataset.has_key? :modified assert dataset.has_key? :languages - assert dataset.has_key? :license_url assert dataset.has_key? :publisher assert dataset.has_key? :publisher_mbox assert dataset.has_key? :distributions From 55d3919afcb4de2e9029c83637d63333c75fbc00 Mon Sep 17 00:00:00 2001 From: stbn Date: Thu, 29 Apr 2021 15:59:07 +0200 Subject: [PATCH 4/8] add&fix test --- app/models/gobierto_data/dataset.rb | 1 - .../gobierto_data/dataset_presenter.rb | 2 +- .../api/v1/datasets/catalog.xml.erb | 6 +- .../locales/gobierto_data/presenters/ca.yml | 2 +- config/routes.rb | 3 +- .../api/v1/datasets_controller_test.rb | 38 +++++++- .../gobierto_common/custom_fields.yml | 2 +- test/fixtures/gobierto_data/catalog.xml | 95 +++++++++++++++++++ test/fixtures/gobierto_data/datasets.yml | 21 ++-- test/fixtures/sites.yml | 4 +- .../gobierto_data/dataset_presenter_test.rb | 2 +- 11 files changed, 148 insertions(+), 28 deletions(-) create mode 100644 test/fixtures/gobierto_data/catalog.xml diff --git a/app/models/gobierto_data/dataset.rb b/app/models/gobierto_data/dataset.rb index 65a137c7fa..0e9ac80a20 100644 --- a/app/models/gobierto_data/dataset.rb +++ b/app/models/gobierto_data/dataset.rb @@ -18,7 +18,6 @@ class Dataset < ApplicationRecord scope :sorted, -> { order(data_updated_at: :desc) } scope :visibles, -> { where(visibility_level: "active") } - scope :by_site, ->(site_id) { where(site_id: site_id) } translates :name diff --git a/app/presenters/gobierto_data/dataset_presenter.rb b/app/presenters/gobierto_data/dataset_presenter.rb index 733d995a5a..1d6df40df2 100644 --- a/app/presenters/gobierto_data/dataset_presenter.rb +++ b/app/presenters/gobierto_data/dataset_presenter.rb @@ -58,7 +58,7 @@ def url_helpers end def site_locale - site.configuration_data["default_locale"] + site.configuration.default_locale end def description_custom_field_record(dataset) diff --git a/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb b/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb index f79c345384..f075ac34d1 100644 --- a/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb +++ b/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb @@ -6,7 +6,7 @@ xmlns:foaf="http://xmlns.com/foaf/0.1/" xmlns:xsd="http://www.w3.org/2001/XMLSchema#" xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#" - xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"><% cache @catalog do %> + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> <%= @catalog[:identifier_uri] %> <%= @catalog[:title] %> @@ -34,7 +34,7 @@ <%= @identifier_uri %> - <%= dataset[:title] %> in CSV format + <%= dataset[:title] %> <%= dataset[:description] %> <%= distribution[:download_url] %> <%= distribution[:format] %> @@ -43,5 +43,5 @@ <% end %> <% end %> - <% end %> + diff --git a/config/locales/gobierto_data/presenters/ca.yml b/config/locales/gobierto_data/presenters/ca.yml index 4b18ad0796..92169988fc 100644 --- a/config/locales/gobierto_data/presenters/ca.yml +++ b/config/locales/gobierto_data/presenters/ca.yml @@ -4,5 +4,5 @@ ca: gobierto_data: catalog: description: Catàleg public de dades dels conjunts de dades publicades per% - {site}, a través de la URL% {publisher_url} + {site}, a través de la URL %{publisher_url} title: Catàleg DCAT de conjunts de dades de %{site} en format rdf/xml dcat diff --git a/config/routes.rb b/config/routes.rb index 6cc04b6007..300ab5d1f4 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -641,8 +641,7 @@ resources :favorites, only: [:index] collection do get :meta - # get :catalog - get "catalog" => "datasets#catalog" + get :catalog end member do get "meta" => "datasets#dataset_meta" diff --git a/test/controllers/gobierto_data/api/v1/datasets_controller_test.rb b/test/controllers/gobierto_data/api/v1/datasets_controller_test.rb index a72cf5eda3..1a58d99db0 100644 --- a/test/controllers/gobierto_data/api/v1/datasets_controller_test.rb +++ b/test/controllers/gobierto_data/api/v1/datasets_controller_test.rb @@ -67,6 +67,10 @@ def datasets_md_with_translations @datasets_md_with_translations ||= gobierto_common_custom_fields(:madrid_data_datasets_custom_field_md_with_translations) end + def datasets_descriptions + @datasets_descriptions ||= gobierto_common_custom_fields(:madrid_data_datasets_custom_field_description) + end + def other_site_dataset @other_site_dataset ||= gobierto_data_datasets(:santander_dataset) end @@ -93,7 +97,8 @@ def array_data(dataset) dataset.rails_model&.columns_hash&.transform_values(&:type)&.to_s, GobiertoCommon::CustomFieldRecord.find_by(item: dataset, custom_field: datasets_category)&.value_string, GobiertoCommon::CustomFieldRecord.find_by(item: dataset, custom_field: datasets_md_without_translations)&.value_string, - GobiertoCommon::CustomFieldRecord.find_by(item: dataset, custom_field: datasets_md_with_translations)&.value_string + GobiertoCommon::CustomFieldRecord.find_by(item: dataset, custom_field: datasets_md_with_translations)&.value_string || "", + GobiertoCommon::CustomFieldRecord.find_by(item: dataset, custom_field: datasets_descriptions)&.value_string ] end @@ -168,8 +173,9 @@ def test_index_as_csv parsed_csv = CSV.parse(response_data).map { |row| row.map(&:to_s) } assert_equal active_datasets_count + 1, parsed_csv.count - assert_equal %w(id name slug table_name data_updated_at columns category md-without-translations md-with-translations), parsed_csv.first - assert_includes parsed_csv, array_data(dataset) + assert_equal %w(id name slug table_name data_updated_at columns category md-without-translations md-with-translations description-datasets), parsed_csv.first + + assert_includes parsed_csv.drop(1).take(1), array_data(dataset) refute_includes parsed_csv, array_data(other_site_dataset) end end @@ -202,7 +208,7 @@ def test_index_xlsx_format assert_equal 1, parsed_xlsx.worksheets.count sheet = parsed_xlsx.worksheets.first assert_nil sheet[active_datasets_count + 1] - assert_equal %w(id name slug table_name data_updated_at columns category md-without-translations md-with-translations), sheet[0].cells.map(&:value) + assert_equal %w(id name slug table_name data_updated_at columns category md-without-translations md-with-translations description-datasets), sheet[0].cells.map(&:value) values = (1..active_datasets_count).map do |row_number| sheet[row_number].cells.map { |cell| cell.value.to_s } end @@ -315,12 +321,34 @@ def test_index_when_md_custom_field_changes_translations_availability parsed_csv = CSV.parse(response_data).map { |row| row.map(&:to_s) } assert_equal active_datasets_count + 1, parsed_csv.count - assert_equal %w(id name slug table_name data_updated_at columns category md-without-translations md-with-translations), parsed_csv.first + assert_equal %w(id name slug table_name data_updated_at columns category md-without-translations md-with-translations description-datasets), parsed_csv.first assert_includes parsed_csv, array_data(dataset) refute_includes parsed_csv, array_data(other_site_dataset) end end + def test_catalog + with(site: site) do + get catalog_gobierto_data_api_v1_datasets_path(format: :xml), as: :xml + assert_response :success + + response_xml = response.parsed_body + expected = File.read("test/fixtures/gobierto_data/catalog.xml") + assert_equal response_xml, expected + end + end + + def test_catalog_dont_show_draft_dataset + with(site: site) do + get catalog_gobierto_data_api_v1_datasets_path(format: :xml), as: :xml + assert_response :success + response_xml = response.parsed_body + + refute_includes response_xml, 'Interest Groups' + refute_includes response_xml, 'Grupos de Interés' + end + end + end end end diff --git a/test/fixtures/gobierto_common/custom_fields.yml b/test/fixtures/gobierto_common/custom_fields.yml index 580212c638..ea675fd8f1 100644 --- a/test/fixtures/gobierto_common/custom_fields.yml +++ b/test/fixtures/gobierto_common/custom_fields.yml @@ -350,7 +350,7 @@ madrid_data_datasets_custom_field_description: position: 4 name_translations: <%= {"en": "Description", "es": "Descripción"}.to_json %> field_type: <%= GobiertoCommon::CustomField.field_types[:localized_paragraph] %> - uid: description + uid: description-datasets madrid_custom_field_human_resources_table_plugin: site: madrid diff --git a/test/fixtures/gobierto_data/catalog.xml b/test/fixtures/gobierto_data/catalog.xml new file mode 100644 index 0000000000..50fa6a5746 --- /dev/null +++ b/test/fixtures/gobierto_data/catalog.xml @@ -0,0 +1,95 @@ + + + + http://madrid.gobierto.test/datos + Catalog for datasets of Ayuntamiento de Madrid into format rdf/xml dcat + Public catalog with datasets published by Ayuntamiento de Madrid, through URL http://madrid.gobierto.test/datos + 2019-01-01 08:00:00 +0100 + 2019-01-01 08:00:00 +0100 + en + + + + http://madrid.gobierto.test/datos/users-dataset + Users + + 2020-01-02 08:00:00 +0100 + 2020-01-02 08:00:00 +0100 + en + + + + Ayuntamiento de Madrid + contact@madrid.es + + + + + + Users + + http://madrid.gobierto.test/api/v1/data/datasets/users-dataset/download.csv + application/csv + + + + + + http://madrid.gobierto.test/datos/events-dataset + Events + + 2020-01-03 08:00:00 +0100 + 2020-01-03 08:00:00 +0100 + en + + + + Ayuntamiento de Madrid + contact@madrid.es + + + + + + Events + + http://madrid.gobierto.test/api/v1/data/datasets/events-dataset/download.csv + application/csv + + + + + + http://madrid.gobierto.test/datos/no-size + No size + + 2020-01-06 08:00:00 +0100 + 2020-01-06 08:00:00 +0100 + en + + + + Ayuntamiento de Madrid + contact@madrid.es + + + + + + No size + + http://madrid.gobierto.test/api/v1/data/datasets/no-size/download.csv + application/csv + + + + + + diff --git a/test/fixtures/gobierto_data/datasets.yml b/test/fixtures/gobierto_data/datasets.yml index d5d841a76f..678a1f0f51 100644 --- a/test/fixtures/gobierto_data/datasets.yml +++ b/test/fixtures/gobierto_data/datasets.yml @@ -3,8 +3,8 @@ users_dataset: name_translations: <%= { en: "Users", es: "Usuarios" }.to_json %> table_name: users slug: users-dataset - created_at: <%= Time.current %> - updated_at: <%= Time.current %> + created_at: <%= Time.zone.parse('2020-01-02 8:00') %> + updated_at: <%= Time.zone.parse('2020-01-02 8:00') %> visibility_level: <%= GobiertoData::Dataset.visibility_levels["active"] %> size: <%= { csv: 15.megabytes, json: 25.megabytes }.to_json %> @@ -13,8 +13,8 @@ events_dataset: name_translations: <%= { en: "Events", es: "Eventos" }.to_json %> table_name: gc_events slug: events-dataset - created_at: <%= Time.current %> - updated_at: <%= Time.current %> + created_at: <%= Time.zone.parse('2020-01-03 8:00') %> + updated_at: <%= Time.zone.parse('2020-01-03 8:00') %> visibility_level: <%= GobiertoData::Dataset.visibility_levels["active"] %> size: <%= { csv: 3.megabytes, json: 4.megabytes }.to_json %> @@ -23,8 +23,8 @@ draft_dataset: name_translations: <%= { en: "Interest Groups", es: "Grupos de Interés" }.to_json %> table_name: gp_interest_groups slug: interest-groups-dataset - created_at: <%= Time.current %> - updated_at: <%= Time.current %> + created_at: <%= Time.zone.parse('2020-01-04 8:00') %> + updated_at: <%= Time.zone.parse('2020-01-04 8:00') %> visibility_level: <%= GobiertoData::Dataset.visibility_levels["draft"] %> santander_dataset: @@ -32,8 +32,8 @@ santander_dataset: name_translations: <%= { en: "Santander dataset", es: "Dataset de Santander" }.to_json %> table_name: activities slug: santander-dataset - created_at: <%= Time.current %> - updated_at: <%= Time.current %> + created_at: <%= Time.zone.parse('2020-01-05 8:00') %> + updated_at: <%= Time.zone.parse('2020-01-05 8:00') %> visibility_level: <%= GobiertoData::Dataset.visibility_levels["active"] %> no_size_dataset: @@ -41,7 +41,6 @@ no_size_dataset: name_translations: <%= { en: "No size", es: "No se el size" }.to_json %> table_name: sites slug: no-size - created_at: <%= Time.current %> - updated_at: <%= Time.current %> + created_at: <%= Time.zone.parse('2020-01-06 8:00') %> + updated_at: <%= Time.zone.parse('2020-01-06 8:00') %> visibility_level: <%= GobiertoData::Dataset.visibility_levels["active"] %> - diff --git a/test/fixtures/sites.yml b/test/fixtures/sites.yml index c2b6e097b9..f63f7229d1 100644 --- a/test/fixtures/sites.yml +++ b/test/fixtures/sites.yml @@ -34,8 +34,8 @@ madrid: organization_address: Fake St., 123 organization_document_number: 0123456789A visibility_level: <%= Site.visibility_levels["active"] %> - created_at: <%= 1.year.ago %> - updated_at: <%= 1.week.ago %> + created_at: <%= Time.zone.parse('2019-01-01 8:00') %> + updated_at: <%= Time.zone.parse('2019-01-01 8:00') %> santander: title_translations: <%= { 'en' => 'Transparencia Ciudadana', 'es' => 'Transparencia Ciudadana' }.to_json %> diff --git a/test/presenters/gobierto_data/dataset_presenter_test.rb b/test/presenters/gobierto_data/dataset_presenter_test.rb index 76e9cb9389..0cc96743fd 100644 --- a/test/presenters/gobierto_data/dataset_presenter_test.rb +++ b/test/presenters/gobierto_data/dataset_presenter_test.rb @@ -15,7 +15,7 @@ def setup def test_structure_catalog_building_do_not_show_draft_datasets catalog = @subject.build_catalog - datasets_published = GobiertoData::Dataset.by_site(@site.id).visibles.size + datasets_published = @site.datasets.visibles.size assert_equal datasets_published, catalog[:datasets].size end From 034e24fe65b2825980dd5d4ed3cf2b77ab7ea447 Mon Sep 17 00:00:00 2001 From: stbn Date: Tue, 4 May 2021 15:54:51 +0200 Subject: [PATCH 5/8] add description & license & category into dcat --- .../gobierto_data/dataset_presenter.rb | 25 ++- .../api/v1/datasets/catalog.xml.erb | 76 ++++---- test/fixtures/gobierto_data/catalog.xml | 166 +++++++++--------- 3 files changed, 141 insertions(+), 126 deletions(-) diff --git a/app/presenters/gobierto_data/dataset_presenter.rb b/app/presenters/gobierto_data/dataset_presenter.rb index 1d6df40df2..71cd879ba0 100644 --- a/app/presenters/gobierto_data/dataset_presenter.rb +++ b/app/presenters/gobierto_data/dataset_presenter.rb @@ -34,10 +34,12 @@ def build_dataset_for_catalog(dataset) { url: url_helpers.gobierto_data_datasets_url(host: site.domain, id: dataset.slug), title: dataset.name, - description: description_custom_field_record(dataset), + description: description_from_custom_field_record(dataset), + keyword: category_from_custom_field_record(dataset), issued: dataset.created_at, modified: dataset.updated_at, languages: [site_locale], + license_url: license_url_from_custom_field_record(dataset), publisher: site.name, publisher_mbox: site.reply_to_email, distributions: build_distribution_for_catalog(dataset) @@ -61,12 +63,31 @@ def site_locale site.configuration.default_locale end - def description_custom_field_record(dataset) + def description_from_custom_field_record(dataset) if dataset.custom_field_record_with_uid("description") dataset.custom_field_record_with_uid("description").payload["description"][site_locale] else "" end end + + def license_url_from_custom_field_record(dataset) + custom_field_record = dataset.custom_field_record_with_uid('dataset-license') + if !custom_field_record.nil? && !custom_field_record.payload["dataset-license"].blank? + site.terms.find_by(id: custom_field_record.payload["dataset-license"])[:description_translations][site_locale] + else + "" + end + end + + def category_from_custom_field_record(dataset) + custom_field_record = dataset.custom_field_record_with_uid("category") + if !custom_field_record.nil? && !custom_field_record.payload["category"].blank? + site.terms.find_by(id: custom_field_record.payload["category"])[:name_translations][site_locale] + else + "" + end + end + end end diff --git a/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb b/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb index f075ac34d1..5dd6732de9 100644 --- a/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb +++ b/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb @@ -7,41 +7,41 @@ xmlns:xsd="http://www.w3.org/2001/XMLSchema#" xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> - - <%= @catalog[:identifier_uri] %> - <%= @catalog[:title] %> - <%= @catalog[:description] %> - <%= @catalog[:issued] %> - <%= @catalog[:modified] %> - <%= @catalog[:language] %> - - <% @catalog[:datasets].each do |dataset| %> - - <%= dataset[:url] %> - <%= dataset[:title] %> - <%= dataset[:description] %> - <%= dataset[:issued] %> - <%= dataset[:modified] %> - <%= dataset[:languages].map { |lang| "#{lang}" }.join.html_safe %> - - - - <%= dataset[:publisher] %> - <%= dataset[:publisher_mbox] %> - - - <% dataset[:distributions].each do |distribution| %> - - - <%= @identifier_uri %> - <%= dataset[:title] %> - <%= dataset[:description] %> - <%= distribution[:download_url] %> - <%= distribution[:format] %> - - - <% end %> - - <% end %> - - + + <%= @catalog[:identifier_uri] %> + <%= @catalog[:title] %> + <%= @catalog[:description] %> + <%= @catalog[:issued].iso8601 %> + <%= @catalog[:modified].iso8601 %> + <%= @catalog[:language] %> + + + <% @catalog[:datasets].each do |dataset| %> + + + <%= dataset[:url] %> + <%= dataset[:title] %> + <%= dataset[:description] %> + <% unless dataset[:keyword].blank? %><%= dataset[:keyword] %><% end %> + <%= dataset[:issued].iso8601 %> + <%= dataset[:modified].iso8601 %> + <%= dataset[:languages].map { |lang| "#{lang}" }.join.html_safe %> + + + <% dataset[:distributions].each do |distribution| %> + + + <%= dataset[:url] %> + <%= dataset[:title] %> + <%= dataset[:description] %> + <%= distribution[:download_url] %> + <%= distribution[:format] %> + + + + <% end %> + + + <% end %> + + diff --git a/test/fixtures/gobierto_data/catalog.xml b/test/fixtures/gobierto_data/catalog.xml index 50fa6a5746..73d676d901 100644 --- a/test/fixtures/gobierto_data/catalog.xml +++ b/test/fixtures/gobierto_data/catalog.xml @@ -7,89 +7,83 @@ xmlns:xsd="http://www.w3.org/2001/XMLSchema#" xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> - - http://madrid.gobierto.test/datos - Catalog for datasets of Ayuntamiento de Madrid into format rdf/xml dcat - Public catalog with datasets published by Ayuntamiento de Madrid, through URL http://madrid.gobierto.test/datos - 2019-01-01 08:00:00 +0100 - 2019-01-01 08:00:00 +0100 - en - - - - http://madrid.gobierto.test/datos/users-dataset - Users - - 2020-01-02 08:00:00 +0100 - 2020-01-02 08:00:00 +0100 - en - - - - Ayuntamiento de Madrid - contact@madrid.es - - - - - - Users - - http://madrid.gobierto.test/api/v1/data/datasets/users-dataset/download.csv - application/csv - - - - - - http://madrid.gobierto.test/datos/events-dataset - Events - - 2020-01-03 08:00:00 +0100 - 2020-01-03 08:00:00 +0100 - en - - - - Ayuntamiento de Madrid - contact@madrid.es - - - - - - Events - - http://madrid.gobierto.test/api/v1/data/datasets/events-dataset/download.csv - application/csv - - - - - - http://madrid.gobierto.test/datos/no-size - No size - - 2020-01-06 08:00:00 +0100 - 2020-01-06 08:00:00 +0100 - en - - - - Ayuntamiento de Madrid - contact@madrid.es - - - - - - No size - - http://madrid.gobierto.test/api/v1/data/datasets/no-size/download.csv - application/csv - - - - - - + + http://madrid.gobierto.test/datos + Catalog for datasets of Ayuntamiento de Madrid into format rdf/xml dcat + Public catalog with datasets published by Ayuntamiento de Madrid, through URL http://madrid.gobierto.test/datos + 2019-01-01T08:00:00+01:00 + 2019-01-01T08:00:00+01:00 + en + + + + + http://madrid.gobierto.test/datos/users-dataset + Users + + Culture + 2020-01-02T08:00:00+01:00 + 2020-01-02T08:00:00+01:00 + en + + + + + http://madrid.gobierto.test/datos/users-dataset + Users + + http://madrid.gobierto.test/api/v1/data/datasets/users-dataset/download.csv + application/csv + + + + + + + + http://madrid.gobierto.test/datos/events-dataset + Events + + + 2020-01-03T08:00:00+01:00 + 2020-01-03T08:00:00+01:00 + en + + + + + http://madrid.gobierto.test/datos/events-dataset + Events + + http://madrid.gobierto.test/api/v1/data/datasets/events-dataset/download.csv + application/csv + + + + + + + + http://madrid.gobierto.test/datos/no-size + No size + + + 2020-01-06T08:00:00+01:00 + 2020-01-06T08:00:00+01:00 + en + + + + + http://madrid.gobierto.test/datos/no-size + No size + + http://madrid.gobierto.test/api/v1/data/datasets/no-size/download.csv + application/csv + + + + + + + From dd8abc933512519b5492268015f56a70012dc8a4 Mon Sep 17 00:00:00 2001 From: stbn Date: Thu, 6 May 2021 10:13:40 +0200 Subject: [PATCH 6/8] hard refactor to gob.es dcat template --- app/models/gobierto_data/dataset.rb | 1 + .../gobierto_data/dataset_presenter.rb | 92 ++++++--- .../api/v1/datasets/catalog.xml.erb | 105 ++++++---- .../locales/gobierto_data/presenters/ca.yml | 1 + .../locales/gobierto_data/presenters/en.yml | 1 + .../locales/gobierto_data/presenters/es.yml | 1 + .../gobierto_seeds/gobierto_data/recipe.rb | 59 +++++- test/fixtures/gobierto_data/catalog.xml | 180 +++++++++++------- .../gobierto_data/dataset_presenter_test.rb | 27 ++- 9 files changed, 316 insertions(+), 151 deletions(-) diff --git a/app/models/gobierto_data/dataset.rb b/app/models/gobierto_data/dataset.rb index cb4ecac24b..a0bdeecbad 100644 --- a/app/models/gobierto_data/dataset.rb +++ b/app/models/gobierto_data/dataset.rb @@ -31,6 +31,7 @@ class Dataset < ApplicationRecord has_many :visualizations, dependent: :destroy, class_name: "GobiertoData::Visualization" scope :sorted, -> { order(data_updated_at: :desc) } + scope :sorted_by_creation, -> { order(created_at: :desc) } scope :visibles, -> { where(visibility_level: "active") } translates :name diff --git a/app/presenters/gobierto_data/dataset_presenter.rb b/app/presenters/gobierto_data/dataset_presenter.rb index 71cd879ba0..8f5d702052 100644 --- a/app/presenters/gobierto_data/dataset_presenter.rb +++ b/app/presenters/gobierto_data/dataset_presenter.rb @@ -10,48 +10,58 @@ def initialize(site) end def build_catalog - catalog = { - identifier_uri: url_helpers.gobierto_data_root_url(host: site.domain), - title: I18n.t('presenters.gobierto_data.catalog.title',site: site), - description: I18n.t('presenters.gobierto_data.catalog.description',site: site, publisher_url: url_helpers.gobierto_data_root_url(host: site.domain)), - issued: site.created_at, - modified: site.updated_at, - language: site_locale, - homepage: url_helpers.gobierto_data_root_url(host: site.domain), - datasets: build_datasets_for_catalog - } + Hash.new.tap do |catalog| + catalog[:languages] = site.configuration.available_locales + catalog[:identifier_uri] = url_helpers.gobierto_data_root_url(host: site.domain) + site.configuration.available_locales.each do |locale| + catalog["title_#{locale}".to_sym] = I18n.t('presenters.gobierto_data.catalog.title',site: site, locale: locale) + catalog["description_#{locale}".to_sym] = I18n.t('presenters.gobierto_data.catalog.description',site: site, publisher_url: url_helpers.gobierto_data_root_url(host: site.domain), locale: locale) + end + catalog[:issued] = site.created_at.iso8601 + catalog[:modified] = site.updated_at.iso8601 + catalog[:homepage] = url_helpers.gobierto_data_root_url(host: site.domain) + catalog[:publisher] = catalog_record_from_custom_field("catalog-publisher") + catalog[:spatials] = catalog_record_from_custom_field("catalog-spatials") + catalog[:theme] = catalog_record_from_custom_field("catalog-theme-taxonomy") + catalog[:datasets] = build_datasets_for_catalog + end end private def build_datasets_for_catalog - site.datasets.visibles.map do |dataset| + site.datasets.visibles.sorted_by_creation.map do |dataset| build_dataset_for_catalog(dataset) end end def build_dataset_for_catalog(dataset) - { - url: url_helpers.gobierto_data_datasets_url(host: site.domain, id: dataset.slug), - title: dataset.name, - description: description_from_custom_field_record(dataset), - keyword: category_from_custom_field_record(dataset), - issued: dataset.created_at, - modified: dataset.updated_at, - languages: [site_locale], - license_url: license_url_from_custom_field_record(dataset), - publisher: site.name, - publisher_mbox: site.reply_to_email, - distributions: build_distribution_for_catalog(dataset) - } + Hash.new.tap do |dataset_presenter| + dataset_presenter[:url] = url_helpers.gobierto_data_datasets_url(host: site.domain, id: dataset.slug) + site.configuration.available_locales.each do |locale| + dataset_presenter["title_#{locale}".to_sym] = dataset.name(locale: locale) + dataset_presenter["description_#{locale}".to_sym] = description_from_custom_field_record(dataset)[locale] + end + dataset_presenter[:keywords] = category_from_custom_field_record(dataset) + dataset_presenter[:issued] = dataset.created_at.iso8601 + dataset_presenter[:modified] = dataset.updated_at.iso8601 + dataset_presenter[:update_frequency] = frecuency_from_custom_field_record(dataset).first + dataset_presenter[:update_frequency_in_days] = frecuency_from_custom_field_record(dataset).last + dataset_presenter[:license_url] = license_url_from_custom_field_record(dataset) + dataset_presenter[:distributions] = build_distribution_for_catalog(dataset) + end end def build_distribution_for_catalog(dataset) [ - { - format: 'application/csv', - download_url: url_helpers.download_gobierto_data_api_v1_dataset_url(slug: dataset.slug, host: site.domain) - } + Hash.new.tap do |distribution| + site.configuration.available_locales.each do |locale| + distribution["title_#{locale}_extended".to_sym] = "#{dataset.name(locale: locale)} #{I18n.t('presenters.gobierto_data.catalog.at_format')} application/csv" + end + distribution[:format] = 'application/csv' + distribution[:download_url] = url_helpers.download_gobierto_data_api_v1_dataset_url(slug: dataset.slug, host: site.domain) + distribution[:csv_size] = dataset.size&.[](:csv) + end ] end @@ -65,7 +75,7 @@ def site_locale def description_from_custom_field_record(dataset) if dataset.custom_field_record_with_uid("description") - dataset.custom_field_record_with_uid("description").payload["description"][site_locale] + dataset.custom_field_record_with_uid("description").payload["description"] else "" end @@ -83,11 +93,33 @@ def license_url_from_custom_field_record(dataset) def category_from_custom_field_record(dataset) custom_field_record = dataset.custom_field_record_with_uid("category") if !custom_field_record.nil? && !custom_field_record.payload["category"].blank? - site.terms.find_by(id: custom_field_record.payload["category"])[:name_translations][site_locale] + site.terms.find_by(id: custom_field_record.payload["category"])[:name_translations] else "" end end + def frecuency_from_custom_field_record(dataset) + custom_field_record = dataset.custom_field_record_with_uid("frequency") + if !custom_field_record.nil? && !custom_field_record.payload["frequency"].blank? + term = site.terms.find_by(id: custom_field_record.payload["frequency"]) + [ + term[:name_translations]["en"], + term[:description_translations]["en"] + ] + else + ["", ""] + end + end + + def catalog_record_from_custom_field(uid) + custom_field = site.custom_fields.find_by(uid: uid) + if custom_field + site.custom_field_records.find_by(custom_field_id: custom_field.id)&.payload["no-translatable"] + else + [] + end + end + end end diff --git a/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb b/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb index 5dd6732de9..6332b2f479 100644 --- a/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb +++ b/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb @@ -1,47 +1,78 @@ - + - + xmlns:time="http://www.w3.org/2006/time#" + xmlns:dct="http://purl.org/dc/terms/" + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:dcat="http://www.w3.org/ns/dcat#" + xmlns:foaf="http://xmlns.com/foaf/0.1/" + xmlns:xsd="http://www.w3.org/2001/XMLSchema#" + xmlns:tema="http://datos.gob.es/kos/sector-publico/sector/" + xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> + <%= @catalog[:identifier_uri] %> - <%= @catalog[:title] %> - <%= @catalog[:description] %> - <%= @catalog[:issued].iso8601 %> - <%= @catalog[:modified].iso8601 %> - <%= @catalog[:language] %> - - + <% @catalog[:languages].each do |locale| %> + <%= @catalog["title_#{locale}".to_sym] %> + <%= @catalog["description_#{locale}".to_sym] %> + <% end %> + + <%= @catalog[:issued] %> + <%= @catalog[:modified] %> + <% @catalog[:languages].each do |locale| %><%= locale %><% end %> + <% @catalog[:spatials].each do |spatial| %><% end %> + + <% @catalog[:datasets].each do |dataset| %> - <%= dataset[:url] %> - <%= dataset[:title] %> - <%= dataset[:description] %> - <% unless dataset[:keyword].blank? %><%= dataset[:keyword] %><% end %> - <%= dataset[:issued].iso8601 %> - <%= dataset[:modified].iso8601 %> - <%= dataset[:languages].map { |lang| "#{lang}" }.join.html_safe %> - - - <% dataset[:distributions].each do |distribution| %> - - - <%= dataset[:url] %> - <%= dataset[:title] %> - <%= dataset[:description] %> - <%= distribution[:download_url] %> - <%= distribution[:format] %> - - - + <%= dataset[:url]%> + <% @catalog[:languages].each do |locale| %> + <%= dataset["title_#{locale}".to_sym] %> + <%= dataset["description_#{locale}".to_sym] %> + <% end %> + <% @catalog[:languages].each do |locale| %> + <%= dataset[:keywords][locale] %> + <% end %> + <%= dataset[:issued] %> + <%= dataset[:modified] %> + <% if !dataset[:update_frequency].blank? && !dataset[:update_in_days].blank? %> + + + + + <%= dataset[:update_frequency] %> + <%= dataset[:update_frequency_in_days] %> + + + + + <% end %> + <% @catalog[:languages].each do |locale| %> + <%= locale %> + <% end %> + + + <% @catalog[:spatials].each do |spatial| %><% end %> + <% dataset[:distributions].each do |distribution| %> + + + <%= dataset[:url] %> + <% @catalog[:languages].each do |locale| %> + <%= dataset["title_#{locale}_extended"] %> <% end %> + <%= distribution[:download_url] %> + <% unless distribution[:csv_size].blank? %><%= distribution[:csv_size] %><% end %> + + + <%= distribution[:format] %> + Comma-separated values (CSV) + + + + + <% end %> <% end %> - + diff --git a/config/locales/gobierto_data/presenters/ca.yml b/config/locales/gobierto_data/presenters/ca.yml index 92169988fc..f4d5bdf13f 100644 --- a/config/locales/gobierto_data/presenters/ca.yml +++ b/config/locales/gobierto_data/presenters/ca.yml @@ -3,6 +3,7 @@ ca: presenters: gobierto_data: catalog: + at_format: en format description: Catàleg public de dades dels conjunts de dades publicades per% {site}, a través de la URL %{publisher_url} title: Catàleg DCAT de conjunts de dades de %{site} en format rdf/xml dcat diff --git a/config/locales/gobierto_data/presenters/en.yml b/config/locales/gobierto_data/presenters/en.yml index 54874ba590..820f5af79f 100644 --- a/config/locales/gobierto_data/presenters/en.yml +++ b/config/locales/gobierto_data/presenters/en.yml @@ -3,6 +3,7 @@ en: presenters: gobierto_data: catalog: + at_format: at format description: Public catalog with datasets published by %{site}, through URL %{publisher_url} title: Catalog for datasets of %{site} into format rdf/xml dcat diff --git a/config/locales/gobierto_data/presenters/es.yml b/config/locales/gobierto_data/presenters/es.yml index e928281ffc..9ab463e299 100644 --- a/config/locales/gobierto_data/presenters/es.yml +++ b/config/locales/gobierto_data/presenters/es.yml @@ -3,6 +3,7 @@ es: presenters: gobierto_data: catalog: + at_format: en formato description: Catalogo publico de datos los conjuntos de datos publicados por %{site}, a través de la URL %{publisher_url} title: Catalogo DCAT para los conjuntos de datos de %{site} en formato rdf/xml diff --git a/db/seeds/gobierto_seeds/gobierto_data/recipe.rb b/db/seeds/gobierto_seeds/gobierto_data/recipe.rb index b59cbe2be1..750137626e 100644 --- a/db/seeds/gobierto_seeds/gobierto_data/recipe.rb +++ b/db/seeds/gobierto_seeds/gobierto_data/recipe.rb @@ -46,10 +46,10 @@ def self.run(site) if vocabulary.new_record? vocabulary.name_translations = { ca: "Freqüència de conjunt de dades", en: "Dataset frequency", es: "Frecuencia de conjunto de datos" } vocabulary.save - vocabulary.terms.create(name_translations: { ca: "Anual", en: "Annual", es: "Anual" }, position: 1) - vocabulary.terms.create(name_translations: { ca: "Trimestral", en: "Quarterly", es: "Trimestral" }, position: 2) - vocabulary.terms.create(name_translations: { ca: "Mensual", en: "Monthly", es: "Mensual" }, position: 3) - vocabulary.terms.create(name_translations: { ca: "Diària", en: "Daily", es: "Diaria" }, position: 4) + vocabulary.terms.create(name_translations: { ca: "Anual", en: "Annual", es: "Anual" }, description_translations: { ca: 365, en: 365, es: 365 }, position: 1) + vocabulary.terms.create(name_translations: { ca: "Trimestral", en: "Quarterly", es: "Trimestral" }, description_translations: { ca: 120, en: 120, es: 120 }, position: 2) + vocabulary.terms.create(name_translations: { ca: "Mensual", en: "Monthly", es: "Mensual" }, description_translations: { ca: 30, en: 30, es: 30 }, position: 3) + vocabulary.terms.create(name_translations: { ca: "Diària", en: "Daily", es: "Diaria" }, description_translations: { ca: 1, en: 1, es: 1 }, position: 4) end frequency.name_translations = { ca: "Freqüència", en: "Frequency", es: "Frecuencia" } frequency.position = 2 @@ -122,7 +122,58 @@ def self.run(site) } license.save end + + publisher_organism = site.custom_fields.string.where(class_name: "Site").find_or_initialize_by(uid: "catalog-publisher") + if publisher_organism.new_record? + publisher_organism.name_translations = { ca: "URL Organisme publicador", en: "Organism publisher URL", es: "URL Organisme publicador" } + publisher_organism.position = 1 + publisher_organism.options = { configuration: {} } + publisher_organism.save + + custom_field_record = GobiertoCommon::CustomFieldRecord.new + custom_field_record.item_type = "Site" + custom_field_record.item_id = site.id + custom_field_record.custom_field_id = publisher_organism.id + custom_field_record.payload = { "no-translatable": "http://datos.gob.es/recurso/sector-publico/org/Organismo/L01280650" } # https://datos.gob.es/recurso/sector-publico/org/Organismo#search + custom_field_record.save + end + + publisher_spatial = site.custom_fields.string.where(class_name: "Site").find_or_initialize_by(uid: "catalog-spatials") + if publisher_spatial.new_record? + publisher_spatial.name_translations = { ca: "Organisme publicador spatials", en: "Organism publisher spatials", es: "URL Organisme spatials" } + publisher_spatial.position = 2 + publisher_spatial.options = { configuration: {} } + publisher_spatial.save + + custom_field_record = GobiertoCommon::CustomFieldRecord.new + custom_field_record.item_type = "Site" + custom_field_record.item_id = site.id + custom_field_record.custom_field_id = publisher_spatial.id + custom_field_record.payload = {"no-translatable": [ + "http://datos.gob.es/recurso/sector-publico/territorio/Pais/España", + "http://datos.gob.es/recurso/sector-publico/territorio/Autonomia/Madrid", + "http://datos.gob.es/recurso/sector-publico/territorio/Provincia/Madrid", + "https://datos.gob.es/recurso/sector-publico/territorio/Ciudad/Getafe" #FIXME + ]} + custom_field_record.save + end + + publisher_taxonomy = site.custom_fields.string.where(class_name: "Site").find_or_initialize_by(uid: "catalog-theme-taxonomy") + if publisher_taxonomy.new_record? + publisher_taxonomy.name_translations = { ca: "Tema (taxonomia gob.es)", en: "Theme (taxonomia gob.es)", es: "Tema (taxonomia gob.es)" } + publisher_taxonomy.position = 3 + publisher_taxonomy.options = { configuration: {} } + publisher_taxonomy.save + + custom_field_record = GobiertoCommon::CustomFieldRecord.new + custom_field_record.item_type = "Site" + custom_field_record.item_id = site.id + custom_field_record.custom_field_id = publisher_taxonomy.id + custom_field_record.payload = { "no-translatable": "http://datos.gob.es/kos/sector-publico/sector/sector-publico" } + custom_field_record.save + end end + end end end diff --git a/test/fixtures/gobierto_data/catalog.xml b/test/fixtures/gobierto_data/catalog.xml index 73d676d901..51b1edea61 100644 --- a/test/fixtures/gobierto_data/catalog.xml +++ b/test/fixtures/gobierto_data/catalog.xml @@ -1,89 +1,125 @@ - + - + xmlns:time="http://www.w3.org/2006/time#" + xmlns:dct="http://purl.org/dc/terms/" + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:dcat="http://www.w3.org/ns/dcat#" + xmlns:foaf="http://xmlns.com/foaf/0.1/" + xmlns:xsd="http://www.w3.org/2001/XMLSchema#" + xmlns:tema="http://datos.gob.es/kos/sector-publico/sector/" + xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> + http://madrid.gobierto.test/datos - Catalog for datasets of Ayuntamiento de Madrid into format rdf/xml dcat - Public catalog with datasets published by Ayuntamiento de Madrid, through URL http://madrid.gobierto.test/datos + Catalog for datasets of Ayuntamiento de Madrid into format rdf/xml dcat + Public catalog with datasets published by Ayuntamiento de Madrid, through URL http://madrid.gobierto.test/datos + Catalogo DCAT para los conjuntos de datos de Ayuntamiento de Madrid en formato rdf/xml dcat + Catalogo publico de datos los conjuntos de datos publicados por Ayuntamiento de Madrid, a través de la URL http://madrid.gobierto.test/datos + 2019-01-01T08:00:00+01:00 2019-01-01T08:00:00+01:00 - en - - + enes + + + - - http://madrid.gobierto.test/datos/users-dataset - Users - - Culture - 2020-01-02T08:00:00+01:00 - 2020-01-02T08:00:00+01:00 - en - - - - - http://madrid.gobierto.test/datos/users-dataset - Users - - http://madrid.gobierto.test/api/v1/data/datasets/users-dataset/download.csv - application/csv - - - + + http://madrid.gobierto.test/datos/no-size + No size + + No se el size + + + + 2020-01-06T08:00:00+01:00 + 2020-01-06T08:00:00+01:00 + en + es + + + + + + http://madrid.gobierto.test/datos/no-size + + + http://madrid.gobierto.test/api/v1/data/datasets/no-size/download.csv + + + + application/csv + Comma-separated values (CSV) + + + + - http://madrid.gobierto.test/datos/events-dataset - Events - + http://madrid.gobierto.test/datos/events-dataset + Events + + Eventos + + + + 2020-01-03T08:00:00+01:00 + 2020-01-03T08:00:00+01:00 + en + es + + + + + + http://madrid.gobierto.test/datos/events-dataset + + + http://madrid.gobierto.test/api/v1/data/datasets/events-dataset/download.csv - 2020-01-03T08:00:00+01:00 - 2020-01-03T08:00:00+01:00 - en - - - - - http://madrid.gobierto.test/datos/events-dataset - Events - - http://madrid.gobierto.test/api/v1/data/datasets/events-dataset/download.csv - application/csv - - - + + + application/csv + Comma-separated values (CSV) + + + + - - http://madrid.gobierto.test/datos/no-size - No size - + + http://madrid.gobierto.test/datos/users-dataset + Users + + Usuarios + + Culture + Cultura + 2020-01-02T08:00:00+01:00 + 2020-01-02T08:00:00+01:00 + en + es + + + + + + http://madrid.gobierto.test/datos/users-dataset + + + http://madrid.gobierto.test/api/v1/data/datasets/users-dataset/download.csv - 2020-01-06T08:00:00+01:00 - 2020-01-06T08:00:00+01:00 - en - - - - - http://madrid.gobierto.test/datos/no-size - No size - - http://madrid.gobierto.test/api/v1/data/datasets/no-size/download.csv - application/csv - - - + + + application/csv + Comma-separated values (CSV) + + + + - + diff --git a/test/presenters/gobierto_data/dataset_presenter_test.rb b/test/presenters/gobierto_data/dataset_presenter_test.rb index 0cc96743fd..807ab0e648 100644 --- a/test/presenters/gobierto_data/dataset_presenter_test.rb +++ b/test/presenters/gobierto_data/dataset_presenter_test.rb @@ -22,26 +22,37 @@ def test_structure_catalog_building_do_not_show_draft_datasets def test_structure_catalog_building_using_site_with_datasets catalog = @subject.build_catalog assert catalog.has_key? :identifier_uri - assert catalog.has_key? :title - assert catalog.has_key? :description + @site.configuration.available_locales.each do |locale| + assert catalog.has_key? "title_#{locale}".to_sym + assert catalog.has_key? "description_#{locale}".to_sym + end assert catalog.has_key? :issued assert catalog.has_key? :modified - assert catalog.has_key? :language assert catalog.has_key? :homepage + assert catalog.has_key? :publisher + assert catalog.has_key? :spatials + assert catalog.has_key? :theme assert catalog.has_key? :datasets catalog[:datasets].each do |dataset| assert dataset.has_key? :url - assert dataset.has_key? :title - assert dataset.has_key? :description + @site.configuration.available_locales.each do |locale| + assert dataset.has_key? "title_#{locale}".to_sym + assert dataset.has_key? "description_#{locale}".to_sym + end + assert dataset.has_key? :keywords assert dataset.has_key? :issued assert dataset.has_key? :modified - assert dataset.has_key? :languages - assert dataset.has_key? :publisher - assert dataset.has_key? :publisher_mbox + assert dataset.has_key? :update_frequency + assert dataset.has_key? :update_frequency_in_days + assert dataset.has_key? :license_url assert dataset.has_key? :distributions dataset[:distributions].each do |distribution| + @site.configuration.available_locales.each do |locale| + assert distribution.has_key? "title_#{locale}_extended".to_sym + end assert distribution.has_key? :format assert distribution.has_key? :download_url + assert distribution.has_key? :csv_size end end end From c3b7307f734c910171682295c7f54f9874eb3cc4 Mon Sep 17 00:00:00 2001 From: stbn Date: Thu, 6 May 2021 14:14:12 +0200 Subject: [PATCH 7/8] mv dcat config from custom_fields_records to site.config --- .../gobierto_data/dataset_presenter.rb | 16 +- .../api/v1/datasets/catalog.xml.erb | 122 ++++++----- .../gobierto_seeds/gobierto_data/recipe.rb | 58 +---- test/fixtures/gobierto_data/catalog.xml | 206 +++++++++--------- 4 files changed, 179 insertions(+), 223 deletions(-) diff --git a/app/presenters/gobierto_data/dataset_presenter.rb b/app/presenters/gobierto_data/dataset_presenter.rb index 8f5d702052..9229918593 100644 --- a/app/presenters/gobierto_data/dataset_presenter.rb +++ b/app/presenters/gobierto_data/dataset_presenter.rb @@ -20,9 +20,9 @@ def build_catalog catalog[:issued] = site.created_at.iso8601 catalog[:modified] = site.updated_at.iso8601 catalog[:homepage] = url_helpers.gobierto_data_root_url(host: site.domain) - catalog[:publisher] = catalog_record_from_custom_field("catalog-publisher") - catalog[:spatials] = catalog_record_from_custom_field("catalog-spatials") - catalog[:theme] = catalog_record_from_custom_field("catalog-theme-taxonomy") + catalog[:publisher] = site.configuration.configuration_variables["gobierto_data_catalog_organism"] || "" + catalog[:spatials] = site.configuration.configuration_variables["gobierto_data_spatials"] || [] + catalog[:theme] = site.configuration.configuration_variables["gobierto_data_theme_taxonomy"] || "" catalog[:datasets] = build_datasets_for_catalog end end @@ -111,15 +111,5 @@ def frecuency_from_custom_field_record(dataset) ["", ""] end end - - def catalog_record_from_custom_field(uid) - custom_field = site.custom_fields.find_by(uid: uid) - if custom_field - site.custom_field_records.find_by(custom_field_id: custom_field.id)&.payload["no-translatable"] - else - [] - end - end - end end diff --git a/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb b/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb index 6332b2f479..1374c96ead 100644 --- a/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb +++ b/app/views/gobierto_data/api/v1/datasets/catalog.xml.erb @@ -11,68 +11,74 @@ xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> <%= @catalog[:identifier_uri] %> - <% @catalog[:languages].each do |locale| %> - <%= @catalog["title_#{locale}".to_sym] %> - <%= @catalog["description_#{locale}".to_sym] %> - <% end %> + <% @catalog[:languages].each do |locale| -%> + <%= @catalog["title_#{locale}".to_sym] %> + <%= @catalog["description_#{locale}".to_sym] %> + <% end -%> <%= @catalog[:issued] %> <%= @catalog[:modified] %> - <% @catalog[:languages].each do |locale| %><%= locale %><% end %> - <% @catalog[:spatials].each do |spatial| %><% end %> + <% @catalog[:languages].each do |locale| -%> + <%= locale %> + <% end -%> + <% @catalog[:spatials].each do |spatial| -%> + + <% end -%> - <% @catalog[:datasets].each do |dataset| %> - - - <%= dataset[:url]%> - <% @catalog[:languages].each do |locale| %> - <%= dataset["title_#{locale}".to_sym] %> - <%= dataset["description_#{locale}".to_sym] %> - <% end %> - <% @catalog[:languages].each do |locale| %> - <%= dataset[:keywords][locale] %> - <% end %> - <%= dataset[:issued] %> - <%= dataset[:modified] %> - <% if !dataset[:update_frequency].blank? && !dataset[:update_in_days].blank? %> - - - - - <%= dataset[:update_frequency] %> - <%= dataset[:update_frequency_in_days] %> - - - - - <% end %> - <% @catalog[:languages].each do |locale| %> - <%= locale %> - <% end %> - - - <% @catalog[:spatials].each do |spatial| %><% end %> - <% dataset[:distributions].each do |distribution| %> - - - <%= dataset[:url] %> - <% @catalog[:languages].each do |locale| %> - <%= dataset["title_#{locale}_extended"] %> - <% end %> - <%= distribution[:download_url] %> - <% unless distribution[:csv_size].blank? %><%= distribution[:csv_size] %><% end %> - - - <%= distribution[:format] %> - Comma-separated values (CSV) - - - - - <% end %> - - - <% end %> + <% @catalog[:datasets].each do |dataset| -%> + + + <%= dataset[:url]%> + <% @catalog[:languages].each do |locale| -%> + <%= dataset["title_#{locale}".to_sym] %> + <%= dataset["description_#{locale}".to_sym] %> + <%= dataset[:keywords][locale] %> + <% end -%> + <%= dataset[:issued] %> + <%= dataset[:modified] %> + <% if !dataset[:update_frequency].blank? && !dataset[:update_in_days].blank? -%> + + + + + <%= dataset[:update_frequency] %> + <%= dataset[:update_frequency_in_days] %> + + + + + <% end -%> + <% @catalog[:languages].each do |locale| -%> + <%= locale %> + <% end -%> + + + <% @catalog[:spatials].each do |spatial| -%> + + <% end -%> + <% dataset[:distributions].each do |distribution| -%> + + + <%= dataset[:url] %> + <% @catalog[:languages].each do |locale| -%> + <%= dataset["title_#{locale}_extended"] %> + <% end -%> + <%= distribution[:download_url] %> + <% unless distribution[:csv_size].blank? -%> + <%= distribution[:csv_size] %> + <% end -%> + + + <%= distribution[:format] %> + Comma-separated values (CSV) + + + + + <% end -%> + + + <% end -%> diff --git a/db/seeds/gobierto_seeds/gobierto_data/recipe.rb b/db/seeds/gobierto_seeds/gobierto_data/recipe.rb index 750137626e..8eb005b31e 100644 --- a/db/seeds/gobierto_seeds/gobierto_data/recipe.rb +++ b/db/seeds/gobierto_seeds/gobierto_data/recipe.rb @@ -123,54 +123,20 @@ def self.run(site) license.save end - publisher_organism = site.custom_fields.string.where(class_name: "Site").find_or_initialize_by(uid: "catalog-publisher") - if publisher_organism.new_record? - publisher_organism.name_translations = { ca: "URL Organisme publicador", en: "Organism publisher URL", es: "URL Organisme publicador" } - publisher_organism.position = 1 - publisher_organism.options = { configuration: {} } - publisher_organism.save - - custom_field_record = GobiertoCommon::CustomFieldRecord.new - custom_field_record.item_type = "Site" - custom_field_record.item_id = site.id - custom_field_record.custom_field_id = publisher_organism.id - custom_field_record.payload = { "no-translatable": "http://datos.gob.es/recurso/sector-publico/org/Organismo/L01280650" } # https://datos.gob.es/recurso/sector-publico/org/Organismo#search - custom_field_record.save + unless site.configuration.configuration_variables["gobierto_data_catalog_organism"].present? + extra_conf = {"gobierto_data_catalog_organism" => "https://datos.gob.es/es/recurso/sector-publico/org/Organismo/"} + site.configuration.raw_configuration_variables = YAML.load(site.configuration.raw_configuration_variables).merge(extra_conf).to_yaml + site.save end - - publisher_spatial = site.custom_fields.string.where(class_name: "Site").find_or_initialize_by(uid: "catalog-spatials") - if publisher_spatial.new_record? - publisher_spatial.name_translations = { ca: "Organisme publicador spatials", en: "Organism publisher spatials", es: "URL Organisme spatials" } - publisher_spatial.position = 2 - publisher_spatial.options = { configuration: {} } - publisher_spatial.save - - custom_field_record = GobiertoCommon::CustomFieldRecord.new - custom_field_record.item_type = "Site" - custom_field_record.item_id = site.id - custom_field_record.custom_field_id = publisher_spatial.id - custom_field_record.payload = {"no-translatable": [ - "http://datos.gob.es/recurso/sector-publico/territorio/Pais/España", - "http://datos.gob.es/recurso/sector-publico/territorio/Autonomia/Madrid", - "http://datos.gob.es/recurso/sector-publico/territorio/Provincia/Madrid", - "https://datos.gob.es/recurso/sector-publico/territorio/Ciudad/Getafe" #FIXME - ]} - custom_field_record.save + unless site.configuration.configuration_variables["gobierto_data_spatials"].present? + extra_conf = { "gobierto_data_spatials" => [ "http://datos.gob.es/recurso/sector-publico/territorio/Pais/España", "http://datos.gob.es/recurso/sector-publico/territorio/Autonomia/Madrid", "http://datos.gob.es/recurso/sector-publico/territorio/Provincia/Madrid", "https://datos.gob.es/recurso/sector-publico/territorio/Ciudad/Getafe" ] } + site.configuration.raw_configuration_variables = YAML.load(site.configuration.raw_configuration_variables).merge(extra_conf).to_yaml + site.save end - - publisher_taxonomy = site.custom_fields.string.where(class_name: "Site").find_or_initialize_by(uid: "catalog-theme-taxonomy") - if publisher_taxonomy.new_record? - publisher_taxonomy.name_translations = { ca: "Tema (taxonomia gob.es)", en: "Theme (taxonomia gob.es)", es: "Tema (taxonomia gob.es)" } - publisher_taxonomy.position = 3 - publisher_taxonomy.options = { configuration: {} } - publisher_taxonomy.save - - custom_field_record = GobiertoCommon::CustomFieldRecord.new - custom_field_record.item_type = "Site" - custom_field_record.item_id = site.id - custom_field_record.custom_field_id = publisher_taxonomy.id - custom_field_record.payload = { "no-translatable": "http://datos.gob.es/kos/sector-publico/sector/sector-publico" } - custom_field_record.save + unless site.configuration.configuration_variables["gobierto_data_theme_taxonomy"].present? + extra_conf = { "gobierto_data_theme_taxonomy" => "http://datos.gob.es/kos/sector-publico/sector/sector-publico"} + site.configuration.raw_configuration_variables = YAML.load(site.configuration.raw_configuration_variables).merge(extra_conf).to_yaml + site.save end end diff --git a/test/fixtures/gobierto_data/catalog.xml b/test/fixtures/gobierto_data/catalog.xml index 51b1edea61..f237f6c598 100644 --- a/test/fixtures/gobierto_data/catalog.xml +++ b/test/fixtures/gobierto_data/catalog.xml @@ -11,115 +11,109 @@ xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"> http://madrid.gobierto.test/datos - Catalog for datasets of Ayuntamiento de Madrid into format rdf/xml dcat - Public catalog with datasets published by Ayuntamiento de Madrid, through URL http://madrid.gobierto.test/datos - Catalogo DCAT para los conjuntos de datos de Ayuntamiento de Madrid en formato rdf/xml dcat - Catalogo publico de datos los conjuntos de datos publicados por Ayuntamiento de Madrid, a través de la URL http://madrid.gobierto.test/datos - + Catalog for datasets of Ayuntamiento de Madrid into format rdf/xml dcat + Public catalog with datasets published by Ayuntamiento de Madrid, through URL http://madrid.gobierto.test/datos + Catalogo DCAT para los conjuntos de datos de Ayuntamiento de Madrid en formato rdf/xml dcat + Catalogo publico de datos los conjuntos de datos publicados por Ayuntamiento de Madrid, a través de la URL http://madrid.gobierto.test/datos + 2019-01-01T08:00:00+01:00 2019-01-01T08:00:00+01:00 - enes - - - - - - http://madrid.gobierto.test/datos/no-size - No size - - No se el size - - - - 2020-01-06T08:00:00+01:00 - 2020-01-06T08:00:00+01:00 - en - es - - - - - - http://madrid.gobierto.test/datos/no-size - - - http://madrid.gobierto.test/api/v1/data/datasets/no-size/download.csv - - - - application/csv - Comma-separated values (CSV) - - - - - - - - - http://madrid.gobierto.test/datos/events-dataset - Events - - Eventos - - - - 2020-01-03T08:00:00+01:00 - 2020-01-03T08:00:00+01:00 en es - - - - - - http://madrid.gobierto.test/datos/events-dataset - - - http://madrid.gobierto.test/api/v1/data/datasets/events-dataset/download.csv - - - - application/csv - Comma-separated values (CSV) - - - - - - - - - http://madrid.gobierto.test/datos/users-dataset - Users - - Usuarios - - Culture - Cultura - 2020-01-02T08:00:00+01:00 - 2020-01-02T08:00:00+01:00 - en - es - - - - - - http://madrid.gobierto.test/datos/users-dataset - - - http://madrid.gobierto.test/api/v1/data/datasets/users-dataset/download.csv - - - - application/csv - Comma-separated values (CSV) - - - - - - + + + + + http://madrid.gobierto.test/datos/no-size + No size + + + No se el size + + + 2020-01-06T08:00:00+01:00 + 2020-01-06T08:00:00+01:00 + en + es + + + + + http://madrid.gobierto.test/datos/no-size + + + http://madrid.gobierto.test/api/v1/data/datasets/no-size/download.csv + + + application/csv + Comma-separated values (CSV) + + + + + + + + + http://madrid.gobierto.test/datos/events-dataset + Events + + + Eventos + + + 2020-01-03T08:00:00+01:00 + 2020-01-03T08:00:00+01:00 + en + es + + + + + http://madrid.gobierto.test/datos/events-dataset + + + http://madrid.gobierto.test/api/v1/data/datasets/events-dataset/download.csv + + + application/csv + Comma-separated values (CSV) + + + + + + + + + http://madrid.gobierto.test/datos/users-dataset + Users + + Culture + Usuarios + + Cultura + 2020-01-02T08:00:00+01:00 + 2020-01-02T08:00:00+01:00 + en + es + + + + + http://madrid.gobierto.test/datos/users-dataset + + + http://madrid.gobierto.test/api/v1/data/datasets/users-dataset/download.csv + + + application/csv + Comma-separated values (CSV) + + + + + + From dcd63d822c768c316d3d3f3e6b3c7875ab52e2dd Mon Sep 17 00:00:00 2001 From: stbn Date: Mon, 10 May 2021 09:31:01 +0200 Subject: [PATCH 8/8] fix if existing config was nil --- db/seeds/gobierto_seeds/gobierto_data/recipe.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/db/seeds/gobierto_seeds/gobierto_data/recipe.rb b/db/seeds/gobierto_seeds/gobierto_data/recipe.rb index 8eb005b31e..6827ca1dd4 100644 --- a/db/seeds/gobierto_seeds/gobierto_data/recipe.rb +++ b/db/seeds/gobierto_seeds/gobierto_data/recipe.rb @@ -125,17 +125,20 @@ def self.run(site) unless site.configuration.configuration_variables["gobierto_data_catalog_organism"].present? extra_conf = {"gobierto_data_catalog_organism" => "https://datos.gob.es/es/recurso/sector-publico/org/Organismo/"} - site.configuration.raw_configuration_variables = YAML.load(site.configuration.raw_configuration_variables).merge(extra_conf).to_yaml + existing_conf = YAML.load(site.configuration.raw_configuration_variables) || {} + site.configuration.raw_configuration_variables = existing_conf.merge(extra_conf).to_yaml site.save end unless site.configuration.configuration_variables["gobierto_data_spatials"].present? extra_conf = { "gobierto_data_spatials" => [ "http://datos.gob.es/recurso/sector-publico/territorio/Pais/España", "http://datos.gob.es/recurso/sector-publico/territorio/Autonomia/Madrid", "http://datos.gob.es/recurso/sector-publico/territorio/Provincia/Madrid", "https://datos.gob.es/recurso/sector-publico/territorio/Ciudad/Getafe" ] } - site.configuration.raw_configuration_variables = YAML.load(site.configuration.raw_configuration_variables).merge(extra_conf).to_yaml + existing_conf = YAML.load(site.configuration.raw_configuration_variables) || {} + site.configuration.raw_configuration_variables = existing_conf.merge(extra_conf).to_yaml site.save end unless site.configuration.configuration_variables["gobierto_data_theme_taxonomy"].present? extra_conf = { "gobierto_data_theme_taxonomy" => "http://datos.gob.es/kos/sector-publico/sector/sector-publico"} - site.configuration.raw_configuration_variables = YAML.load(site.configuration.raw_configuration_variables).merge(extra_conf).to_yaml + existing_conf = YAML.load(site.configuration.raw_configuration_variables) || {} + site.configuration.raw_configuration_variables = existing_conf.merge(extra_conf).to_yaml site.save end end