From f033f6b10c2f5b88205928b6a9df93327f9f4611 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Wed, 26 Jun 2024 10:17:28 +0200 Subject: [PATCH 1/5] Remove unused harvest patch - Now the custom harvest panel is derived to ckanext-schemingdcat. Update .env.example and remove patch. - Remove unused recline_view --- .env.example | 4 +- .../00_improve_harvest_panel.patch | 144 ------------------ 2 files changed, 2 insertions(+), 146 deletions(-) delete mode 100644 ckan/patches/ckanext-harvest/00_improve_harvest_panel.patch diff --git a/.env.example b/.env.example index f9ef19ea..cdbcd21c 100644 --- a/.env.example +++ b/.env.example @@ -119,7 +119,7 @@ CKAN__RESOURCE_PROXY__MAX_FILE_SIZE=50048576 CKAN__RESOURCE_PROXY__CHUNK_SIZE=4096 ## Default timeout for fetching proxied items CKAN__RESOURCE_PROXY__TIMEOUT=10 -CKAN__VIEWS__DEFAULT_VIEWS="image_view webpage_view text_view recline_view wmts_view geojson_view geo_view shp_view pdf_view" +CKAN__VIEWS__DEFAULT_VIEWS="image_view webpage_view text_view datatables_view wmts_view geojson_view geo_view shp_view pdf_view" # Localization CKAN__LOCALE_DEFAULT="en" @@ -127,7 +127,7 @@ CKAN__LOCALE_ORDER="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru CKAN__LOCALES_OFFERED="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv" # Extensions -CKAN__PLUGINS="envvars stats image_view text_view datatables_view webpage_view resourcedictionary datastore xloader harvest spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat schemingdcat_ckan_harvester schemingdcat_xls_harvester pdf_view pages fluent" +CKAN__PLUGINS="envvars stats image_view text_view datatables_view webpage_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent" # ckanext-harvest CKAN__HARVEST__MQ__TYPE=redis diff --git a/ckan/patches/ckanext-harvest/00_improve_harvest_panel.patch b/ckan/patches/ckanext-harvest/00_improve_harvest_panel.patch deleted file mode 100644 index da31dde9..00000000 --- a/ckan/patches/ckanext-harvest/00_improve_harvest_panel.patch +++ /dev/null @@ -1,144 +0,0 @@ -diff --git a/ckanext/harvest/logic/action/update.py b/ckanext/harvest/logic/action/update.py -index 7dce1f5..ac4db4f 100644 ---- a/ckanext/harvest/logic/action/update.py -+++ b/ckanext/harvest/logic/action/update.py -@@ -706,7 +706,7 @@ def harvest_jobs_run(context, data_dict): - notify_all = toolkit.asbool(config.get('ckan.harvest.status_mail.all')) - notify_errors = toolkit.asbool(config.get('ckan.harvest.status_mail.errored')) - last_job_errors = status['last_job']['stats'].get('errored', 0) -- log.debug('Notifications: All:{} On error:{} Errors:{}'.format(notify_all, notify_errors, last_job_errors)) -+ log.debug('Notifications: All:{} On error:{} Errors:{} Errordict:{}'.format(notify_all, notify_errors, last_job_errors, status['last_job']['stats'])) - - if last_job_errors > 0 and (notify_all or notify_errors): - send_error_email(context, job_obj.source.id, status) -diff --git a/ckanext/harvest/templates/snippets/source_item.html b/ckanext/harvest/templates/snippets/source_item.html -index a8ba26f..3f50bdb 100644 ---- a/ckanext/harvest/templates/snippets/source_item.html -+++ b/ckanext/harvest/templates/snippets/source_item.html -@@ -48,6 +48,7 @@ Example: - {% endif %} - {% if not within_organization and source.organization %} - — {{ _('Organization') }}: {{ h.link_to(source.organization.title or source.organization.name, h.url_for('organization.read', id=source.organization.name)) }} -+ — {{ _('Datasets') }}: {{ h.link_to(h.schemingdcat_package_count_for_source(source.id), url) }} - {% endif %} -

- -diff --git a/ckanext/harvest/templates/source/new.html b/ckanext/harvest/templates/source/new.html -index b7feb3d..b1fe472 100644 ---- a/ckanext/harvest/templates/source/new.html -+++ b/ckanext/harvest/templates/source/new.html -@@ -24,12 +24,18 @@ -
-

- {% trans %} -- Harvest sources allow importing remote metadata into this catalog. -- Remote sources can be other catalogs such as other CKAN instances, CSW -- servers or Web Accessible Folders (WAF) (depending on the actual -- harvesters enabled for this instance). -+ Harvest sources allow importing remote metadata into this catalog. Remote sources can be other catalogs such as other CKAN instances, CSW servers, XML metadata files, XLSX with metadata records or Web Accessible Folder (WAF). - {% endtrans %} -

-+ -+

-+ {{ _('Depending on the actual harvesters enabled for this instance. eg: ') }} -+

-+

-
- - {% endblock %} -diff --git a/ckanext/harvest/templates/source/new_source_form.html b/ckanext/harvest/templates/source/new_source_form.html -index 324d012..37358fc 100644 ---- a/ckanext/harvest/templates/source/new_source_form.html -+++ b/ckanext/harvest/templates/source/new_source_form.html -@@ -8,7 +8,7 @@ - - {% call form.input('url', id='field-url', label=_('URL'), value=data.url, error=errors.url, classes=['control-full', 'control-large']) %} - -- {{ _('This should include the http:// part of the URL') }} -+ {{ _('This should include the http:// part of the URL') }} - - {% endcall %} - -@@ -26,7 +26,7 @@ - {{ form.markdown('notes', id='field-notes', label=_('Description'), value=data.notes, error=errors.notes) }} - -
-- -+ -
- {% for harvester in h.harvesters_info() %} - {% set checked = False %} -@@ -46,7 +46,11 @@ - {{ form.select('frequency', id='field-frequency', label=_('Update frequency'), options=h.harvest_frequencies(), selected=data.frequency, error=errors.frequency) }} - - {% block extra_config %} -- {{ form.textarea('config', id='field-config', label=_('Configuration'), value=data.config, error=errors.config) }} -+ {% call form.textarea('config', id='field-config', label=_('Configuration'), value=data.config, error=errors.config) %} -+ -+ {{ _('You can validate the JSON at: ') }} {{ _('JSONLint') }} -+ -+ {% endcall %} - {% endblock extra_config %} - - {# if we have a default group then this wants remembering #} -diff --git a/ckanext/harvest/templates/source/read.html b/ckanext/harvest/templates/source/read.html -index e016e55..e60a18c 100644 ---- a/ckanext/harvest/templates/source/read.html -+++ b/ckanext/harvest/templates/source/read.html -@@ -3,6 +3,6 @@ - {% block primary_content_inner %} -
-

{{ _('Datasets') }}

-- {{ h.package_list_for_source(harvest_source.id) }} -+ {{ h.schemingdcat_package_list_for_source(harvest_source.id) }} -
- {% endblock %} -diff --git a/ckanext/harvest/templates/source/read_base.html b/ckanext/harvest/templates/source/read_base.html -index 5126e25..3191c38 100644 ---- a/ckanext/harvest/templates/source/read_base.html -+++ b/ckanext/harvest/templates/source/read_base.html -@@ -15,7 +15,7 @@ -
-
-
{{ _('Datasets') }}
--
{{ h.package_count_for_source(harvest_source.id) }}
-+
{{ h.schemingdcat_package_count_for_source(harvest_source.id) }}
-
-
- -diff --git a/ckanext/harvest/templates/source/search.html b/ckanext/harvest/templates/source/search.html -index d9ceeea..f44a0cb 100644 ---- a/ckanext/harvest/templates/source/search.html -+++ b/ckanext/harvest/templates/source/search.html -@@ -44,7 +44,26 @@ - - - --{% block secondary_content %} -+ {% block secondary_content %} -+
-+

{{ _('Harvest sources') }}

-+
-+

-+ {% trans %} -+ Harvest sources allow importing remote metadata into this catalog. Remote sources can be other catalogs such as other CKAN instances, CSW servers, XML metadata files, XLSX with metadata records or Web Accessible Folder (WAF). -+ {% endtrans %} -+

-+ -+

-+ {{ _('Depending on the actual harvesters enabled for this instance. eg: ') }} -+

-+

-+
-+
- {% for facet in c.facet_titles %} - {{ h.snippet('snippets/facet_list.html', title=c.facet_titles[facet], name=facet, alternative_url=h.url_for('{0}.search'.format(c.dataset_type))) }} - {% endfor %} From 11d7b9ef1772dd24f790c94add05ac87a92248f3 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Wed, 26 Jun 2024 10:17:47 +0200 Subject: [PATCH 2/5] Fix dataset_scope field type --- solr/Dockerfile.spatial | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/Dockerfile.spatial b/solr/Dockerfile.spatial index a2643cb6..56f9348c 100644 --- a/solr/Dockerfile.spatial +++ b/solr/Dockerfile.spatial @@ -59,7 +59,7 @@ ENV SOLR_BBOX_FIELDS ' \ ' \ '' \ - '' + '' RUN sed -i "//a $SOLR_BBOX_FIELDS" $SOLR_SCHEMA_FILE From db84d2b17f5c491f7ff2a65027fe458c9383a037 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Wed, 26 Jun 2024 11:11:46 +0200 Subject: [PATCH 3/5] Update CKAN configuration to include metadata templates search identifier --- .env.example | 1 + ckan/docker-entrypoint.d/02_setup_scheming.sh | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index cdbcd21c..4c9bd9a9 100644 --- a/.env.example +++ b/.env.example @@ -176,6 +176,7 @@ CKANEXT__SCHEMINGDCAT_GROUP_CUSTOM_FACETS=True CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_ICON="theme" CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_SHOW_SPATIAL=True CKANEXT__SCHEMINGDCAT_SHOW_METADATA_TEMPLATES_TOOLBAR=False +CKANEXT__METADATA_TEMPLATES_SEARCH_IDENTIFIER="schemingdcat_xls-template" # ckanext-pages CKANEXT__PAGES__ALOW_HTML=False diff --git a/ckan/docker-entrypoint.d/02_setup_scheming.sh b/ckan/docker-entrypoint.d/02_setup_scheming.sh index 76cb13e8..83fc3842 100644 --- a/ckan/docker-entrypoint.d/02_setup_scheming.sh +++ b/ckan/docker-entrypoint.d/02_setup_scheming.sh @@ -16,7 +16,8 @@ ckan config-tool $CKAN_INI \ "schemingdcat.geometadata_base_uri=$CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI" \ "schemingdcat.default_package_item_icon=$CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_ICON" \ "schemingdcat.default_package_item_show_spatial=$CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_SHOW_SPATIAL" \ - "schemingdcat.show_metadata_templates_toolbar=$CKANEXT__SCHEMINGDCAT_SHOW_METADATA_TEMPLATES_TOOLBAR" + "schemingdcat.show_metadata_templates_toolbar=$CKANEXT__SCHEMINGDCAT_SHOW_METADATA_TEMPLATES_TOOLBAR" \ + "schemingdcat.metadata_templates_search_identifier=$CKANEXT__METADATA_TEMPLATES_SEARCH_IDENTIFIER" echo "[docker-entrypoint.02_setup_scheming] Rebuild index" ckan -c $CKAN_INI search-index rebuild \ No newline at end of file From dbc4f168ce4091cffc8ede36ecba159ce5850369 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Thu, 27 Jun 2024 10:37:23 +0200 Subject: [PATCH 4/5] Update CKAN configuration to include recline_view and metadata templates search identifier --- .env.example | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 4c9bd9a9..9c2399dc 100644 --- a/.env.example +++ b/.env.example @@ -119,7 +119,7 @@ CKAN__RESOURCE_PROXY__MAX_FILE_SIZE=50048576 CKAN__RESOURCE_PROXY__CHUNK_SIZE=4096 ## Default timeout for fetching proxied items CKAN__RESOURCE_PROXY__TIMEOUT=10 -CKAN__VIEWS__DEFAULT_VIEWS="image_view webpage_view text_view datatables_view wmts_view geojson_view geo_view shp_view pdf_view" +CKAN__VIEWS__DEFAULT_VIEWS="image_view webpage_view text_view recline_view wmts_view geojson_view geo_view shp_view pdf_view" # Localization CKAN__LOCALE_DEFAULT="en" @@ -127,7 +127,7 @@ CKAN__LOCALE_ORDER="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru CKAN__LOCALES_OFFERED="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv" # Extensions -CKAN__PLUGINS="envvars stats image_view text_view datatables_view webpage_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent" +CKAN__PLUGINS="envvars stats image_view text_view recline_view webpage_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent" # ckanext-harvest CKAN__HARVEST__MQ__TYPE=redis From 4dd374c2c5655269f68fc8f71c386cf37b86fc91 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Thu, 27 Jun 2024 10:00:09 +0000 Subject: [PATCH 5/5] Update ckanext-schemingdcat to v3.2.1 --- ckan/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/Dockerfile b/ckan/Dockerfile index 1851ed5f..893fa5af 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -22,7 +22,7 @@ COPY req_fixes req_fixes ## Pages - v0.5.2 ## ## PDFView - 0.0.8 ## ## Fluent - v1.0.1 (mjanez/Forked stable version) ## -## Scheming DCAT - v3.2.0 (mjanez/GeoDCAT-AP/NTI-RISP extended version) ## +## Scheming DCAT - v3.2.1 (mjanez/GeoDCAT-AP/NTI-RISP extended version) ## RUN echo ${TZ} > /etc/timezone && \ if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then cp /usr/share/zoneinfo/${TZ} /etc/localtime; fi && \ # Remove apk cache @@ -54,7 +54,7 @@ RUN echo ${TZ} > /etc/timezone && \ echo "mjanez/ckanext-fluent" && \ pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-fluent.git@v1.0.1#egg=ckanext-fluent && \ echo "mjanez/ckanext-schemingdcat" && \ - pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-schemingdcat.git@v3.2.0#egg=ckanext_schemingdcat && \ + pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-schemingdcat.git@v3.2.1#egg=ckanext_schemingdcat && \ pip3 install --no-cache-dir -r ${APP_DIR}/src/ckanext-schemingdcat/requirements.txt # Used to configure the container environment by setting environment variables, creating users, running initialization scripts, .etc