Merge pull request #124 from mjanez/ckan-2.9.11

Improve ckan-docker 2.9.11

.env.example

@@ -1,16 +1,5 @@
-# Base
 APP_DIR=/srv/app
 
-# Container names
-NGINX_CONTAINER_NAME=nginx
-REDIS_CONTAINER_NAME=redis
-POSTGRESQL_CONTAINER_NAME=db
-SOLR_CONTAINER_NAME=solr
-CKAN_CONTAINER_NAME=ckan
-WORKER_CONTAINER_NAME=ckan-worker
-APACHE_CONTAINER_NAME=apache
-PYCSW_CONTAINER_NAME=pycsw
-
 # Host Ports
 CKAN_PORT_HOST=5000
 NGINX_PORT_HOST=81
@@ -114,7 +103,10 @@ CKAN__PREVIEW__XML_FORMATS="xml rdf rdf+xml owl+xml atom rss turtle ttl n3 n-tri
 CKAN__PREVIEW__TEXT_FORMATS="text plain text/plain text/turtle csv tsv rss txt json"
 CKAN__PREVIEW__LOADABLE="html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json arcgis_rest"
 # ckanext-spatial: Allow Solr local params: https://github.com/ckan/ckanext-spatial/issues/328
-CKAN__SEARCH__SOLR_ALLOWED_QUERY_PARSERS = field
+CKAN__SEARCH__SOLR_ALLOWED_QUERY_PARSERS=field
+# CORS Settings. If True, all origins will be allowed (the response header Access-Control-Allow-Origin is set to ‘*’). If False, only origins from the ckan.cors.origin_whitelist setting will be allowed.
+CKAN__CORS__ORIGIN_ALLOW_ALL=False
+CKAN__CORS__ORIGIN_WHITELIST=""
 
 ## Resource Proxy settings
 ### Preview size limit, default: 1MB
@@ -131,7 +123,7 @@ CKAN__LOCALE_ORDER="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru
 CKAN__LOCALES_OFFERED="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv"
 
 # Extensions
-CKAN__PLUGINS="envvars stats text_view image_view webpage_view recline_view resourcedictionary datastore xloader harvest spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat schemingdcat_ckan_harvester schemingdcat_xls_harvester pdf_view pages fluent"
+CKAN__PLUGINS="envvars stats image_view text_view datatables_view webpage_view resourcedictionary datastore xloader harvest spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat schemingdcat_ckan_harvester schemingdcat_xls_harvester pdf_view pages fluent"
 
 # ckanext-harvest
 CKAN__HARVEST__MQ__TYPE=redis

README.md

@@ -203,7 +203,7 @@ To start the containers:
 
 	docker compose -f docker-compose.dev.yml up
 
-See [CKAN Images](#ckan-images) for more details of what happens when using development mode.
+See [CKAN images](#5-ckan-images) for more details of what happens when using development mode.
 
 #### Create an extension
 You can use the ckan [extension](https://docs.ckan.org/en/latest/extensions/tutorial.html#creating-a-new-extension) instructions to create a CKAN extension, only executing the command inside the CKAN container and setting the mounted `src/` folder as output:
@@ -215,15 +215,19 @@ You can use the ckan [extension](https://docs.ckan.org/en/latest/extensions/tuto
 
 The new extension files and directories are created in the `/srv/app/src_extensions/` folder in the running container. They will also exist in the local src/ directory as local `/src` directory is mounted as `/srv/app/src_extensions/` on the ckan container. You might need to change the owner of its folder to have the appropiate permissions.
 
-##### Running HTTPS on development mode
+#### Running HTTPS on development mode
 
 Sometimes is useful to run your local development instance under HTTPS, for instance if you are using authentication extensions like [ckanext-saml2auth](https://github.com/keitaroinc/ckanext-saml2auth). To enable it, set the following in your `.env` file:
 
+```
   USE_HTTPS_FOR_DEV=true
+```
 
 and update the site URL setting:
 
+```
   CKAN_SITE_URL=https://localhost:5000
+```
 
 After recreating the `ckan-dev` container, you should be able to access CKAN at https://localhost:5000
 
@@ -330,30 +334,52 @@ ckan
 
 ## ckan-docker addons
 ### Debugging
-#### VSCode dev containers
+#### Debugging CKAN Development Instance with VSCode Dev Containers and debugpy
+
 The [Visual Studio Code Dev Containers](https://code.visualstudio.com/docs/devcontainers/containers) extension is a powerful tool that enables developers to use a container as a complete development environment. With this extension, developers can open any folder inside a container and take advantage of the full range of features provided by Visual Studio Code. To do this, developers create a `devcontainer.json `file in their project that specifies how to access or create a development container with a predefined tool and runtime stack. This allows developers to work in an isolated environment, ensuring that the development environment is consistent across team members and that project dependencies are easy to manage.
 
 ![Developing inside a Container](https://code.visualstudio.com/assets/docs/devcontainers/containers/architecture-containers.png)
 
+To set this up:
+
 1. Install [VSCode](https://code.visualstudio.com/).
 
-1. Install the[ Remote Development extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.vscode-remote-extensionpack) for VSCode. 
+2. Install the [Remote Development extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.vscode-remote-extensionpack) for VSCode.
+
+3. In your project directory, create a `devcontainer.json` file. This file will contain the configuration for your development container.
 
-3. In your project directory, create a file named `devcontainer.json.` This file will contain the configuration for your `dev container`.
+4. In the `devcontainer.json` file, specify the Docker image for your development container and any additional configuration settings, such as environment variables, ports to expose, and startup commands.
 
-4. In the `devcontainer.json` file, specify the Docker image that you want to use for your `dev container`. 
+5. Enable `debugpy` for your development instance in your `.env` file:
 
-5. Specify any additional configuration settings for your `dev container`, such as environment variables, ports to expose, and startup commands.
+  ```ini
+  USE_DEBUGPY_FOR_DEV=true
+  ```
+
+6. Start the containers in [development mode](#development-mode) and launch VS Code.
+
+7. Install the "Dev Container" extension: press `CTRL+SHIFT+X`, type "dev container", click "install".
+
+8. Click the `Open a Remote Window` button in the bottom-left of the VS Code window.
 
-6. Open your project in a `dev container` by using the Remote Development extension in VSCode. You can do this by clicking the `Open Folder in Container` button in the command palette or by opening the folder using the `Remote-Containers: Open Folder in Container` command. Also you can attach to an active container `Attach to Running Container`.
+9. Click `Attach to Running Container...` and select your ckan-dev container, e.g. `ckan-docker-ckan-dev-1`.
 
-7. VSCode will start a new container based on the configuration settings in your `devcontainer.json` file. Once the container is started, you can work on your project just like you would on your local machine.
+10. Click the `Run and Debug` icon on the left panel then `create a launch.json`, select `Python Debugger`, `Remote Attach`, host `localhost` and port `5678`.
 
+11. Press `F5` or click the `Run` menu and `Start Debugging`.
+
+You can now set breakpoints and remote debug your CKAN development instance using VSCode Dev Containers and debugpy.
 
 #### pdb
 Add these lines to the `ckan-dev` service in the docker compose.dev.yml file
 
-![pdb](https://user-images.githubusercontent.com/54408245/179964232-9e98a451-5fe9-4842-ba9b-751bcc627730.png)
+```yaml
+ports:
+  - "0.0.0.0:${CKAN_PORT}:5000"
+
+stdin_open: true
+tty: true
+```
 
 Debug with pdb (example) - Interact with `docker attach $(docker container ls -qf name=ckan)`
 

ckan/docker-entrypoint.d/01_setup_xloader.sh

@@ -27,8 +27,4 @@ ckan config-tool $CKAN_INI \
 
 # Create ckanext-xloader API_TOKEN
 echo "[docker-entrypoint.01_setup_xloader] Set up ckanext.xloader.api_token in the CKAN config file"
-ckan config-tool $CKAN_INI "ckanext.xloader.api_token=$(ckan -c $CKAN_INI user token add ckan_admin xloader | tail -n 1 | tr -d '\t')"
-
-#TODO: Setup worker background
-#echo "[docker-entrypoint.01_setup_xloader] Set up CKAN jobs worker"
-#ckan -c $CKAN_INI jobs worker default
+ckan config-tool $CKAN_INI "ckanext.xloader.api_token=$(ckan -c $CKAN_INI user token add ckan_admin xloader | tail -n 1 | tr -d '\t')"

docker-compose.apache.yml

@@ -1,14 +1,11 @@
-version: "3"
-
 volumes:
   ckan_storage:
+  ckan_logs:
   pg_data:
   solr_data:
 
 services:
-
   apache:
-    container_name: ${APACHE_CONTAINER_NAME}
     build:
       context: apache/
       dockerfile: Dockerfile
@@ -23,14 +20,12 @@ services:
       - webnet
       - ckannet
     depends_on:
-      ckan:
-        condition: service_healthy
+      - ckan
     ports:
       - "0.0.0.0:${APACHE_PORT_HOST}:${APACHE_PORT}"
     restart: on-failure:3
 
   ckan:
-    container_name: ${CKAN_CONTAINER_NAME}
     build:
       context: ckan/
       dockerfile: Dockerfile
@@ -49,22 +44,20 @@ services:
           max-size: "100m"
           max-file: "10"
     depends_on:
-      db:
-        condition: service_healthy
-      solr:
-        condition: service_healthy
-      redis:
-        condition: service_healthy
+      - db
+      - solr
+      - redis
     ports:
       - "0.0.0.0:${CKAN_PORT_HOST}:${CKAN_PORT}" 
     volumes:
       - ckan_storage:/var/lib/ckan
+      - ckan_logs:/var/log
+      - /usr/lib/python3.9/site-packages
     restart: unless-stopped
     healthcheck:
       test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${CKAN_PORT}"]
 
   pycsw:
-    container_name: ${PYCSW_CONTAINER_NAME}
     build:
       context: ckan-pycsw/
       dockerfile: Dockerfile
@@ -79,8 +72,7 @@ services:
       - webnet
       - ckannet
     depends_on:
-      ckan:
-        condition: service_healthy
+      - ckan
     ports:
       - "0.0.0.0:${PYCSW_PORT_HOST}:${PYCSW_PORT}"
     volumes:
@@ -91,7 +83,6 @@ services:
       test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${PYCSW_PORT}"]
 
   db:
-    container_name: ${POSTGRESQL_CONTAINER_NAME}
     build:
       context: postgresql/
     networks:
@@ -118,7 +109,6 @@ services:
       test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"]
 
   solr:
-    container_name: ${SOLR_CONTAINER_NAME}
     build:
       context: solr/
       dockerfile: Dockerfile.spatial
@@ -138,7 +128,6 @@ services:
       test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${SOLR_PORT}/solr/"]
 
   redis:
-    container_name: ${REDIS_CONTAINER_NAME}
     image: redis:${REDIS_VERSION}
     logging:
       driver: "json-file"

docker-compose.dev.yml

@@ -1,29 +1,23 @@
-version: "3"
-
-
 volumes:
   ckan_storage:
+  ckan_logs:
   pg_data:
   solr_data:
 
 services:
-
   apache:
-    container_name: ${APACHE_CONTAINER_NAME}
     build:
       context: apache/
       dockerfile: Dockerfile
     env_file:
       - .env
     depends_on:
-      ckan-dev:
-        condition: service_healthy
+      - ckan-dev
     ports:
       - "0.0.0.0:${APACHE_PORT_HOST}:${APACHE_PORT}"
     restart: on-failure:3
 
   ckan-dev:
-    container_name: ${CKAN_CONTAINER_NAME}
     build:
       context: ckan/
       dockerfile: Dockerfile.dev
@@ -37,31 +31,29 @@ services:
           max-size: "100m"
           max-file: "10"
     depends_on:
-      db:
-        condition: service_healthy
-      solr:
-        condition: service_healthy
-      redis:
-        condition: service_healthy
+      - db
+      - solr
+      - redis
     ports:
       - "0.0.0.0:${CKAN_PORT_HOST}:${CKAN_PORT}"
     volumes:
       - ckan_storage:/var/lib/ckan
+      - ckan_logs:/var/log
       - ./src:/srv/app/src_extensions
+      - /usr/lib/python3.9/site-packages
+      - /root/.vscode-server
     restart: unless-stopped
     healthcheck:
       test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${CKAN_PORT}"]
 
   pycsw:
-    container_name: ${PYCSW_CONTAINER_NAME}
     build:
       context: ckan-pycsw/
       dockerfile: Dockerfile
     env_file:
       - .env
     depends_on:
-      ckan-dev:
-        condition: service_healthy
+      - ckan-dev
     ports:
       - "0.0.0.0:${PYCSW_PORT_HOST}:${PYCSW_PORT}"
     volumes:
@@ -72,7 +64,6 @@ services:
       test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${PYCSW_PORT}"]
 
   db:
-    container_name: ${POSTGRESQL_CONTAINER_NAME}
     build:
       context: postgresql/
     environment:
@@ -92,7 +83,6 @@ services:
       test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"]
 
   solr:
-    container_name: ${SOLR_CONTAINER_NAME}
     build:
       context: solr/
       dockerfile: Dockerfile.spatial
@@ -110,7 +100,6 @@ services:
       test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${SOLR_PORT}/solr/"]
 
   redis:
-    container_name: ${REDIS_CONTAINER_NAME}
     image: redis:${REDIS_VERSION}
     logging:
       driver: "json-file"