From 196dacbd99857f30b418cf36100e8cce41a8bc14 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Wed, 19 Apr 2023 13:56:53 +0000 Subject: [PATCH 1/6] Update solr Dockerfile - Fix lineage_process_steps - Add theme_es --- solr/Dockerfile.iepnb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/solr/Dockerfile.iepnb b/solr/Dockerfile.iepnb index 7cff6735..f6fcd5f6 100644 --- a/solr/Dockerfile.iepnb +++ b/solr/Dockerfile.iepnb @@ -51,9 +51,10 @@ ENV SOLR_BBOX_FIELDS ' \ \ \ - \ + \ \ \ + \ \ ' From 11099ff140de31147b29f881fe91117e2776617a Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Wed, 19 Apr 2023 13:57:17 +0000 Subject: [PATCH 2/6] Update ckanext-iepnb to 0.1.2 --- ckan/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/Dockerfile b/ckan/Dockerfile index 2837989a..082fb3d9 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -20,7 +20,7 @@ RUN if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then \ ### Resource dictionary - 1.0.1 ### ### Pages - v0.5.1 ### ### PDFView - v0.0.8 ### -### IEPNB Theme - v0.0.5 ### +### IEPNB Theme - 0.1.2 ### RUN echo "ckan/ckanext-xloader" && \ pip3 install -e 'git+https://github.com/ckan/ckanext-xloader.git@0.12.2#egg=ckanext-xloader' && \ pip3 install -r ${APP_DIR}/src/ckanext-xloader/requirements.txt && \ @@ -45,7 +45,7 @@ RUN echo "ckan/ckanext-xloader" && \ echo "ckan/ckanext-pdfview" && \ pip3 install -e git+https://github.com/ckan/ckanext-pdfview.git@0.0.8#egg=ckanext-pdfview && \ echo "OpenDataGIS/ckanext-iepnb" && \ - pip3 install -e git+https://github.com/OpenDataGIS/ckanext-iepnb.git@0.0.5#egg=ckanext-iepnb && \ + pip3 install -e git+https://github.com/OpenDataGIS/ckanext-iepnb.git@0.1.2#egg=ckanext-iepnb && \ pip3 install -r https://raw.githubusercontent.com/OpenDataGIS/ckanext-iepnb/main/requirements.txt # Used to configure the container environment by setting environment variables, creating users, running initialization scripts, .etc From 2420175cf90f94ad7dc3bca447cff14eb3908842 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Wed, 19 Apr 2023 13:58:17 +0000 Subject: [PATCH 3/6] Remove spatial_widget patches (Core) https://github.com/OpenDataGIS/ckan-docker-iepnb/pull/6 --- .../ckan/00_spatial_widget_dataset.patch | 14 ----- .../ckan/01_spatial_widget_search.patch | 10 ---- .../02_resourcedictionary_templates.patch | 60 +++++++++++++++++++ 3 files changed, 60 insertions(+), 24 deletions(-) delete mode 100644 ckan/patches/ckan/00_spatial_widget_dataset.patch delete mode 100644 ckan/patches/ckan/01_spatial_widget_search.patch create mode 100644 ckan/patches/ckan/02_resourcedictionary_templates.patch diff --git a/ckan/patches/ckan/00_spatial_widget_dataset.patch b/ckan/patches/ckan/00_spatial_widget_dataset.patch deleted file mode 100644 index 68854022..00000000 --- a/ckan/patches/ckan/00_spatial_widget_dataset.patch +++ /dev/null @@ -1,14 +0,0 @@ ---- /ckan/templates/package/read_base.html -+++ /ckan/templates/package/read_base.html -@@ -23,6 +23,12 @@ - {% endblock %} - - {% block secondary_content %} -+ {{ super() }} -+ -+ {% set dataset_extent = pkg.spatial %} -+ {% if dataset_extent %} -+ {% snippet "spatial/snippets/dataset_map_sidebar.html", extent=dataset_extent %} -+ {% endif %} - - {% block secondary_help_content %}{% endblock %} \ No newline at end of file diff --git a/ckan/patches/ckan/01_spatial_widget_search.patch b/ckan/patches/ckan/01_spatial_widget_search.patch deleted file mode 100644 index 6f660f02..00000000 --- a/ckan/patches/ckan/01_spatial_widget_search.patch +++ /dev/null @@ -1,10 +0,0 @@ ---- /ckan/templates/package/search.html -+++ /ckan/templates/package/search.html -@@ -70,6 +70,7 @@ - - - {% block secondary_content %} -+{% snippet "spatial/snippets/spatial_query.html", default_extent="[[26.78,-19.21], [44.08, 5.66]]" %} -
-
- {% for facet in facet_titles %} \ No newline at end of file diff --git a/ckan/patches/ckan/02_resourcedictionary_templates.patch b/ckan/patches/ckan/02_resourcedictionary_templates.patch new file mode 100644 index 00000000..900a0407 --- /dev/null +++ b/ckan/patches/ckan/02_resourcedictionary_templates.patch @@ -0,0 +1,60 @@ +diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html +index 7c8c6f9..6f2342a 100644 +--- a/ckan/templates/package/resource_read.html ++++ b/ckan/templates/package/resource_read.html +@@ -40,12 +40,14 @@ + {{ _('API Endpoint') }} + {% elif not res.has_views and not res.url_type == 'upload' %} + {{ _('Go to resource') }} ++ {% elif res.has_views and not res.url_type == 'upload' %} ++ {{ _('Go to resource') }} + {% else %} + {{ _('Download') }} + {% endif %} + + {% block download_resource_button %} +- {% if res.datastore_active %} ++ {% if res.datastore_active and res.url_type == 'datastore' %} + +@@ -224,4 +226,4 @@ + {% block resource_license %} + {% snippet "snippets/social.html" %} + {% endblock %} +-{% endblock %} ++{% endblock %} +\ No newline at end of file +diff --git a/ckanext/datastore/templates-bs2/package/resource_read.html b/ckanext/datastore/templates-bs2/package/resource_read.html +index 3c77876..1d856e0 100644 +--- a/ckanext/datastore/templates-bs2/package/resource_read.html ++++ b/ckanext/datastore/templates-bs2/package/resource_read.html +@@ -2,7 +2,7 @@ + + {% block resource_actions_inner %} + {{ super() }} +- {% if res.datastore_active %} ++ {% if res.datastore_active and res.url_type == 'datastore' %} +
  • {% snippet 'package/snippets/data_api_button.html', resource=res %}
    • + {% endif %} + {% endblock %} +@@ -34,4 +34,4 @@ +
    + {% endif %} + {{ super() }} +-{% endblock %} ++{% endblock %} +\ No newline at end of file +diff --git a/ckanext/datastore/templates/package/resource_read.html b/ckanext/datastore/templates/package/resource_read.html +index 2e1fdef..9caad53 100644 +--- a/ckanext/datastore/templates/package/resource_read.html ++++ b/ckanext/datastore/templates/package/resource_read.html +@@ -2,7 +2,7 @@ + + {% block resource_actions_inner %} + {{ super() }} +- {% if res.datastore_active %} ++ {% if res.datastore_active and res.url_type == 'datastore' %} +
  • {% snippet 'package/snippets/data_api_button.html', resource=res %}
    • + {% endif %} + {% endblock %} From 51f48ddf8006f439b73befdb262093aee50e1246 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Wed, 19 Apr 2023 13:59:26 +0000 Subject: [PATCH 4/6] Update docker-compose.yml - Comment ckan-iepnb image by default - TODO: Custom redis image --- docker-compose.yml | 10 +++++----- redis/Dockerfile | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8ac27832..35997a2f 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,7 +33,7 @@ services: dockerfile: Dockerfile args: - TZ=${TZ} - image: ghcr.io/opendatagis/ckan-iepnb:master + #image: ghcr.io/opendatagis/ckan-iepnb:master env_file: - .env logging: @@ -121,10 +121,10 @@ services: redis: container_name: ${REDIS_CONTAINER_NAME} - build: - context: redis/ - dockerfile: Dockerfile - #image: redis:${REDIS_VERSION} + # build: + # context: redis/ + # dockerfile: Dockerfile + image: redis:${REDIS_VERSION} logging: driver: "json-file" options: diff --git a/redis/Dockerfile b/redis/Dockerfile index 5f465ead..71d9251e 100644 --- a/redis/Dockerfile +++ b/redis/Dockerfile @@ -1,3 +1,4 @@ +#TODO: Test custom redis image FROM alpine:3.17 # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added From a0d54891579edbdf97762bc05e3144ce05927a6c Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Wed, 19 Apr 2023 14:07:49 +0000 Subject: [PATCH 5/6] Patch ckanext-iepnb Fix harvest icon --- ckan/patches/ckanext-iepnb/00_fix_harvest_ico.patch | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 ckan/patches/ckanext-iepnb/00_fix_harvest_ico.patch diff --git a/ckan/patches/ckanext-iepnb/00_fix_harvest_ico.patch b/ckan/patches/ckanext-iepnb/00_fix_harvest_ico.patch new file mode 100644 index 00000000..fd3ac79e --- /dev/null +++ b/ckan/patches/ckanext-iepnb/00_fix_harvest_ico.patch @@ -0,0 +1,13 @@ +diff --git a/ckanext/iepnb/templates/home/columna_izquierda.html b/ckanext/iepnb/templates/home/columna_izquierda.html +index 683403a..09d1186 100644 +--- a/ckanext/iepnb/templates/home/columna_izquierda.html ++++ b/ckanext/iepnb/templates/home/columna_izquierda.html +@@ -20,7 +20,7 @@ + + + +- Harvest ++ Harvest +

    {{ _('Harvest') }}

    +     + From d185b3a9e5990676702e013af955123d9c425126 Mon Sep 17 00:00:00 2001 From: mjanez <96422458+mjanez@users.noreply.github.com> Date: Wed, 19 Apr 2023 14:09:05 +0000 Subject: [PATCH 6/6] Update setup xloader and setup iepnb --- README.md | 17 +++++++++------- ckan/docker-entrypoint.d/01_setup_xloader.sh | 21 +++++++++++++++++--- ckan/docker-entrypoint.d/05_setup_iepnb.sh | 11 +++++----- 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 11a8816c..821ae414 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ Information about extensions installed in the `main` image. More info described | Extension | [ckanext-resourcedictionary](https://github.com/OpenDataGIS/ckanext-resourcedictionary) | main | Completed | ✔️ | ✔️ | Stable installation. This extension extends the default CKAN Data Dictionary functionality by adding possibility to create data dictionary before actual data is uploaded to datastore. | | Extension | [ckanext-pages](https://github.com/ckan/ckanext-pages) | 0.5.1 | Completed | ✔️ | ✔️ | Stable installation. This extension gives you an easy way to add simple pages to CKAN. | | Extension | [ckanext-pdfview](https://github.com/ckan/ckanext-pdfview) | 0.0.8 | Completed | ✔️ | ✔️ | Stable installation. This extension provides a view plugin for PDF files using an html object tag. | -| Extension | [ckanext-iepnb](https://github.com/OpenDataGIS/ckanext-iepnb) | 0.05 | Completed | ✔️ | ✔️ | Stable installation for 0.0.5 version, the modification of the Solr schema.xml with the new fields for faceted and failed python dependencies needs to be reviewed. | +| Extension | [ckanext-iepnb](https://github.com/OpenDataGIS/ckanext-iepnb) | 0.1.2 | Completed | ✔️ | ✔️ | Stable installation for 0.1.2 version, the modification of the Solr schema.xml with the new fields for faceted and failed python dependencies needs to be reviewed. | | Software | [ckan-pycsw](https://github.com/mjanez/ckan-pycsw) | main | Completed | ✔️ | ✔️ | Stable installation. PyCSW Endpoint of Open Data Portal with docker compose config. Harvest the CKAN catalogue in a CSW endpoint based on existing spatial datasets in the open data portal. | @@ -194,7 +194,9 @@ Use this if you are a maintainer and will not be making code changes to CKAN or > Please note that when accessing CKAN directly (via a browser) ie: not going through Apache/NGINX you will need to make sure you have "ckan" set up to be an alias to localhost in the local hosts file. Either that or you will need to change the `.env` entry for `CKAN_SITE_URL` >**Warning**:
    - > Using the default values on the `.env` file will get you a working CKAN instance. There is a sysadmin user created by default with the values defined in `CKAN_SYSADMIN_NAME` and `CKAN_SYSADMIN_PASSWORD`(`ckan_admin` and `test1234` by default). **This should be obviously changed before running this setup as a public CKAN instance.** + > Using the default values on the `.env` file will get you a working CKAN instance. There is a sysadmin user created by default with the values defined in `CKAN_SYSADMIN_NAME` and `CKAN_SYSADMIN_PASSWORD` (`ckan_admin` and `test1234` by default). All ennvars with `API_TOKEN` are automatically regenerated when CKAN is loaded, no editing is required. + > + >**This should be obviously changed before running this setup as a public CKAN instance.** 3. Build the images: ```bash @@ -322,12 +324,13 @@ The new extension files and directories are created in the `/srv/app/src_extensi ## CKAN images -![ckan images](https://user-images.githubusercontent.com/54408245/207079416-a01235af-2dea-4425-b6fd-f8c3687dd993.png) + +![ckan images](https://raw.githubusercontent.com/mjanez/ckan-docker/master/doc/img/ckan-docker-images-iepnb.png) The Docker image config files used to build your CKAN project are located in the `ckan/` folder. There are two Docker files: -* `Dockerfile`: this is based on `ckan/ckan-base:`, a base image located in the DockerHub repository, that has CKAN installed along with all its dependencies, properly configured and running on [uWSGI](https://uwsgi-docs.readthedocs.io/en/latest/) (production setup) -* `Dockerfile.dev`: this is based on `ckan/ckan-base:-dev` also located located in the DockerHub repository, and extends `ckan/ckan-base:` to include: +* `Dockerfile`: this is based on `ckan/ckan-base-spatial:`, a base image located in the DockerHub repository, that has CKAN installed along with all its dependencies, properly configured and running on [uWSGI](https://uwsgi-docs.readthedocs.io/en/latest/) (production setup) +* `Dockerfile.dev`: this is based on `ckan/ckan-base-spatial:-dev` also located located in the DockerHub repository, and extends `ckan/ckan-base-spatial:` to include: * Any extension cloned on the `src` folder will be installed in the CKAN container when booting up Docker Compose (`docker compose up`). This includes installing any requirements listed in a `requirements.txt` (or `pip-requirements.txt`) file and running `python setup.py develop`. * CKAN is started running this: `/usr/bin/ckan -c /srv/app/ckan.ini run -H 0.0.0.0`. @@ -336,7 +339,7 @@ The Docker image config files used to build your CKAN project are located in the ## CKAN images enhancement ### Extending the base images -You can modify the docker files to build your own customized image tailored to your project, installing any extensions and extra requirements needed. For example here is where you would update to use a different CKAN base image ie: `ckan/ckan-base:` +You can modify the docker files to build your own customized image tailored to your project, installing any extensions and extra requirements needed. For example here is where you would update to use a different CKAN base image ie: `ckan/ckan-base-spatial:` To perform extra initialization steps you can add scripts to your custom images and copy them to the `/docker-entrypoint.d` folder (The folder should be created for you when you build the image). Any `*.sh` and `*.py` file in that folder will be executed just after the main initialization script ([`prerun.py`](https://github.com/ckan/ckan-docker-base/blob/main/ckan-2.9/base/setup/prerun.py)) is executed and just before the web server and supervisor processes are started. @@ -362,7 +365,7 @@ ckan -c /srv/app/ckan.ini validation init-db And then in our `Dockerfile.dev` file we install the extension and copy the initialization scripts: ```Dockerfile -FROM ckan/ckan-base:2.9.8-dev +FROM ckan/ckan-base-spatial:2.9.8 RUN pip install -e git+https://github.com/frictionlessdata/ckanext-validation.git#egg=ckanext-validation && \ pip install -r https://raw.githubusercontent.com/frictionlessdata/ckanext-validation/master/requirements.txt diff --git a/ckan/docker-entrypoint.d/01_setup_xloader.sh b/ckan/docker-entrypoint.d/01_setup_xloader.sh index 8aa25696..4f5e3e58 100644 --- a/ckan/docker-entrypoint.d/01_setup_xloader.sh +++ b/ckan/docker-entrypoint.d/01_setup_xloader.sh @@ -1,14 +1,29 @@ #!/bin/bash +# Set the name of the token to revoke +TOKEN_NAME="xloader" + +# Get the list of tokens and extract the IDs for tokens with the specified name +TOKEN_IDS=$(ckan -c $CKAN_INI user token list ckan_admin | grep "$TOKEN_NAME" | awk -F'[][]' '{print $2}' | tr -d '[]') + +# Revoke each previous token of xloader +for TOKEN_ID in $TOKEN_IDS +do + ckan -c $CKAN_INI user token revoke $TOKEN_ID + if [ $? -eq 0 ]; then + echo "API Token $TOKEN_ID has been revoked" + fi +done + # Add ckanext.xloader.api_token to the CKAN config file echo "Loading ckanext-xloader settings in the CKAN config file" ckan config-tool $CKAN_INI \ - "ckanext.xloader.api_token = xxx" \ - "ckanext.xloader.jobs_db.uri = $CKANEXT__XLOADER__JOBS__DB_URI" + "ckanext.xloader.api_token=xxx" \ + "ckanext.xloader.jobs_db.uri=$CKANEXT__XLOADER__JOBS__DB_URI" # Create ckanext-xloader API_TOKEN echo "Set up ckanext.xloader.api_token in the CKAN config file" -ckan config-tool $CKAN_INI "ckanext.xloader.api_token = $(ckan -c $CKAN_INI user token add ckan_admin xloader | tail -n 1 | tr -d '\t')" +ckan config-tool $CKAN_INI "ckanext.xloader.api_token=$(ckan -c $CKAN_INI user token add ckan_admin xloader | tail -n 1 | tr -d '\t')" #TODO: Setup worker background #echo "Set up CKAN jobs worker" diff --git a/ckan/docker-entrypoint.d/05_setup_iepnb.sh b/ckan/docker-entrypoint.d/05_setup_iepnb.sh index 2a85a80a..b5f1233f 100644 --- a/ckan/docker-entrypoint.d/05_setup_iepnb.sh +++ b/ckan/docker-entrypoint.d/05_setup_iepnb.sh @@ -1,11 +1,10 @@ #!/bin/bash # Update ckanext-iepnb settings -echo "Set up ckanext-iepnb" - -#TODO: Version with solr -#ckan -c $CKAN_INI search-index clear +echo "Set up ckanext-iepnb. Clear index" +ckan -c $CKAN_INI search-index clear +echo "ckanext-iepnb. Load envvars" ckan config-tool $CKAN_INI \ "iepnb.server=$CKANEXT__IEPNB_SERVER" \ "iepnb.path_menu =$CKANEXT__IEPNB_PATH_MENU" \ @@ -13,5 +12,5 @@ ckan config-tool $CKAN_INI \ "iepnb.popular_tags=$CKANEXT__IEPNB_POPULAR_TAGS" \ "iepnb.facet_list=$CKANEXT__IEPNB_FACET_LIST" -#TODO: Version with solr -#ckan -c $CKAN_INI search-index rebuild \ No newline at end of file +echo "ckanext-iepnb. Rebuild index" +ckan -c $CKAN_INI search-index rebuild \ No newline at end of file