+ {% for harvester in h.harvesters_info() %}
+ {% set checked = False %}
+diff --git a/ckanext/harvest/templates/source/search.html b/ckanext/harvest/templates/source/search.html
+index a929943..06cb373 100644
+--- a/ckanext/harvest/templates/source/search.html
++++ b/ckanext/harvest/templates/source/search.html
+@@ -44,7 +44,26 @@
+
+
+
+-{% block secondary_content %}
++ {% block secondary_content %}
++
++ {{ _('Harvest sources') }}
++
++
++ {% trans %}
++ Harvest sources allow importing remote metadata into this catalog. Remote sources can be other catalogs such as other CKAN instances, CSW servers, XML metadata files, XLSX with metadata records or Web Accessible Folder (WAF).
++ {% endtrans %}
++
++
++
++ {{ _('Depending on the actual harvesters enabled for this instance. eg: ') }}
++
++
++
+ {% for facet in c.facet_titles %}
+ {{ h.snippet('snippets/facet_list.html', title=c.facet_titles[facet], name=facet, alternative_url=h.url_for('{0}.search'.format(c.dataset_type))) }}
+ {% endfor %}
diff --git a/ckan/setup/start_ckan.sh.override b/ckan/setup/start_ckan.sh.override
index ce6eebde..bc9ee01c 100644
--- a/ckan/setup/start_ckan.sh.override
+++ b/ckan/setup/start_ckan.sh.override
@@ -1,8 +1,5 @@
#!/bin/sh
-# Add ckan.datapusher.api_token to the CKAN config file (updated with corrected value later)
-ckan config-tool $CKAN_INI ckan.datapusher.api_token=xxx
-
# Set up the Secret key used by Beaker and Flask
# This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var
if grep -E "beaker.session.secret ?= ?$" ckan.ini
@@ -16,7 +13,7 @@ then
fi
# Run the prerun script to init CKAN and create the default admin user
-sudo -u ckan -EH python3 prerun.py
+python3 prerun.py
# Run any startup scripts provided by images extending this one
if [[ -d "/docker-entrypoint.d" ]]
@@ -31,6 +28,14 @@ then
done
fi
+# Create Harvester logs directory and change its ownership
+mkdir -p $CKAN_LOGS_PATH/harvester
+chown -R ckan:ckan $CKAN_LOGS_PATH/harvester
+
+# Create xloader logs directory and change its ownership
+mkdir -p $CKAN_LOGS_PATH/xloader
+chown -R ckan:ckan $CKAN_LOGS_PATH/xloader
+
# Set the common uwsgi options
UWSGI_OPTS="--plugins http,python \
--socket /tmp/uwsgi.sock \
@@ -48,7 +53,15 @@ then
# Start supervisord
supervisord --configuration /etc/supervisord.conf &
# Start uwsgi
- sudo -u ckan -EH uwsgi $UWSGI_OPTS
+ uwsgi $UWSGI_OPTS
else
echo "[prerun] failed...not starting CKAN."
-fi
\ No newline at end of file
+fi
+# Workers
+## Start the Harvester worker
+echo "[prerun.workers] Starting the CKAN Harvester worker"
+ckan harvester run
+## Add harvester to crontab
+crontab -l | { cat; echo "0 */2 * * * ckan harvester run"; } | crontab -
+## Clean-up mechanism for the harvest log table. 'ckan.harvest.log_timeframe'. The default time frame is 30 days
+crontab -l | { cat; echo "0 5 * * * ckan harvester clean-harvest-log"; } | crontab -
diff --git a/ckan/setup/start_ckan_development.sh.override b/ckan/setup/start_ckan_development.sh.override
index 8dcc7465..01efe80e 100644
--- a/ckan/setup/start_ckan_development.sh.override
+++ b/ckan/setup/start_ckan_development.sh.override
@@ -45,9 +45,6 @@ done
echo "Enabling debug mode"
ckan config-tool $CKAN_INI -s DEFAULT "debug = true"
-# Add ckan.datapusher.api_token to the CKAN config file (updated with corrected value later)
-ckan config-tool $CKAN_INI ckan.datapusher.api_token=xxx
-
# Set up the Secret key used by Beaker and Flask
# This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var
if grep -E "beaker.session.secret ?= ?$" ckan.ini
@@ -74,7 +71,7 @@ ckan config-tool $SRC_DIR/ckan/test-core.ini \
"ckan.redis.url = $TEST_CKAN_REDIS_URL"
# Run the prerun script to init CKAN and create the default admin user
-sudo -u ckan -EH python3 prerun.py
+python3 prerun.py
# Run any startup scripts provided by images extending this one
if [[ -d "/docker-entrypoint.d" ]]
@@ -89,8 +86,24 @@ then
done
fi
+# Create Harvester logs directory and change its ownership
+mkdir -p $CKAN_LOGS_PATH/harvester
+chown -R ckan:ckan $CKAN_LOGS_PATH/harvester
+
+# Create xloader logs directory and change its ownership
+mkdir -p $CKAN_LOGS_PATH/xloader
+chown -R ckan:ckan $CKAN_LOGS_PATH/xloader
+
# Start supervisord
supervisord --configuration /etc/supervisord.conf &
-# Start the development server with automatic reload
-sudo -u ckan -EH ckan -c $CKAN_INI run -H 0.0.0.0
\ No newline at end of file
+# Workers
+## Start the Harvester worker
+echo "[prerun.workers] Starting the CKAN Harvester worker"
+ckan harvester run
+
+## Clean-up mechanism for the harvest log table
+ckan harvester clean-harvest-log
+
+# Start the development server as the ckan user with automatic reload
+su ckan -c "/usr/bin/ckan -c $CKAN_INI run -H 0.0.0.0"
\ No newline at end of file
diff --git a/ckan/setup/workers/harvester.conf b/ckan/setup/workers/harvester.conf
new file mode 100644
index 00000000..69b1fa34
--- /dev/null
+++ b/ckan/setup/workers/harvester.conf
@@ -0,0 +1,19 @@
+[program:ckan_gather_consumer]
+command=ckan harvester gather-consumer
+user=ckan
+numprocs=1
+stdout_logfile=/var/log/harvester/gather_consumer.log
+stderr_logfile=/var/log/harvester/gather_consumer.err.log
+autostart=true
+autorestart=true
+startsecs=10
+
+[program:ckan_fetch_consumer]
+command=ckan harvester fetch-consumer
+user=ckan
+numprocs=1
+stdout_logfile=/var/log/harvester/fetch_consumer.log
+stderr_logfile=/var/log/harvester/fetch_consumer.err.log
+autostart=true
+autorestart=true
+startsecs=10
\ No newline at end of file
diff --git a/docker-compose.yml b/docker-compose.yml
index 33ffedf1..78e93a74 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -3,6 +3,7 @@ version: "3"
volumes:
ckan_storage:
+ ckan_logs:
pg_data:
solr_data:
@@ -57,6 +58,7 @@ services:
condition: service_healthy
volumes:
- ckan_storage:/var/lib/ckan
+ - ckan_logs:/var/log
restart: unless-stopped
healthcheck:
test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:${CKAN_PORT}"]
diff --git a/samples/.env.apache.example b/samples/.env.apache.example
index 8615b731..bc9de228 100644
--- a/samples/.env.apache.example
+++ b/samples/.env.apache.example
@@ -89,6 +89,7 @@ CKAN_SYSADMIN_NAME=ckan_admin
CKAN_SYSADMIN_PASSWORD=test1234
CKAN_SYSADMIN_EMAIL=your_email@example.com
CKAN_STORAGE_PATH=/var/lib/ckan
+CKAN_LOGS_PATH=/var/log
CKAN_SMTP_SERVER=smtp.corporateict.domain:25
CKAN_SMTP_STARTTLS=True
CKAN_SMTP_USER=user
diff --git a/samples/.env.localhost b/samples/.env.localhost
index 880f2a9a..25434e53 100644
--- a/samples/.env.localhost
+++ b/samples/.env.localhost
@@ -97,6 +97,7 @@ CKAN_SYSADMIN_NAME=ckan_admin
CKAN_SYSADMIN_PASSWORD=test1234
CKAN_SYSADMIN_EMAIL=your_email@example.com
CKAN_STORAGE_PATH=/var/lib/ckan
+CKAN_LOGS_PATH=/var/log
CKAN_SMTP_SERVER=smtp.corporateict.domain:25
CKAN_SMTP_STARTTLS=True
CKAN_SMTP_USER=user
diff --git a/samples/.env.nginx.example b/samples/.env.nginx.example
index c80c8e8e..fcd5ad36 100644
--- a/samples/.env.nginx.example
+++ b/samples/.env.nginx.example
@@ -89,6 +89,7 @@ CKAN_SYSADMIN_NAME=ckan_admin
CKAN_SYSADMIN_PASSWORD=test1234
CKAN_SYSADMIN_EMAIL=your_email@example.com
CKAN_STORAGE_PATH=/var/lib/ckan
+CKAN_LOGS_PATH=/var/log
CKAN_SMTP_SERVER=smtp.corporateict.domain:25
CKAN_SMTP_STARTTLS=True
CKAN_SMTP_USER=user
diff --git a/samples/custom/.env.es.example b/samples/custom/.env.es.example
index 1ae16c2b..10c9efc3 100644
--- a/samples/custom/.env.es.example
+++ b/samples/custom/.env.es.example
@@ -97,6 +97,7 @@ CKAN_SYSADMIN_NAME=ckan_admin
CKAN_SYSADMIN_PASSWORD=test1234
CKAN_SYSADMIN_EMAIL=your_email@example.com
CKAN_STORAGE_PATH=/var/lib/ckan
+CKAN_LOGS_PATH=/var/log
CKAN_SMTP_SERVER=smtp.corporateict.domain:25
CKAN_SMTP_STARTTLS=True
CKAN_SMTP_USER=user
From 96a4c394a74bd7c62031f52f4c9902055c03414e Mon Sep 17 00:00:00 2001
From: mjanez <96422458+mjanez@users.noreply.github.com>
Date: Tue, 5 Mar 2024 07:40:19 +0100
Subject: [PATCH 3/7] Fix token revocation in 01_setup_xloader.sh
---
ckan/docker-entrypoint.d/01_setup_xloader.sh | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/ckan/docker-entrypoint.d/01_setup_xloader.sh b/ckan/docker-entrypoint.d/01_setup_xloader.sh
index 3630c939..9d0e8946 100644
--- a/ckan/docker-entrypoint.d/01_setup_xloader.sh
+++ b/ckan/docker-entrypoint.d/01_setup_xloader.sh
@@ -9,7 +9,11 @@ TOKEN_IDS=$(ckan -c $CKAN_INI user token list ckan_admin | grep "$TOKEN_NAME" |
# Revoke each previous token of xloader
for TOKEN_ID in $TOKEN_IDS
do
- ckan -c $CKAN_INI user token revoke $TOKEN_ID
+ if [ -z "$TOKEN_ID" ]; then
+ echo "[docker-entrypoint.01_setup_xloader] No API Token to revoke"
+ continue
+ fi
+ ckan -c $CKAN_INI user token revoke -- $TOKEN_ID
if [ $? -eq 0 ]; then
echo "[docker-entrypoint.01_setup_xloader] API Token $TOKEN_ID has been revoked"
fi
From c2c1ef60040d56c665f3862afb7524d3304c68c5 Mon Sep 17 00:00:00 2001
From: mjanez <96422458+mjanez@users.noreply.github.com>
Date: Tue, 5 Mar 2024 07:40:34 +0100
Subject: [PATCH 4/7] Fix supervisord command in
start_ckan_development.sh.override
---
ckan/setup/start_ckan_development.sh.override | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/ckan/setup/start_ckan_development.sh.override b/ckan/setup/start_ckan_development.sh.override
index 01efe80e..dd065061 100644
--- a/ckan/setup/start_ckan_development.sh.override
+++ b/ckan/setup/start_ckan_development.sh.override
@@ -95,7 +95,7 @@ mkdir -p $CKAN_LOGS_PATH/xloader
chown -R ckan:ckan $CKAN_LOGS_PATH/xloader
# Start supervisord
-supervisord --configuration /etc/supervisord.conf &
+supervisord --configuration /etc/supervisord.conf
# Workers
## Start the Harvester worker
From 3b9e94b121886fd62a55478489ee05e508c6c381 Mon Sep 17 00:00:00 2001
From: mjanez <96422458+mjanez@users.noreply.github.com>
Date: Tue, 5 Mar 2024 07:40:46 +0100
Subject: [PATCH 5/7] Update ckanext-harvest patch
---
.../ckanext-harvest/00_translates.patch | 26 +++++++++++++++++--
1 file changed, 24 insertions(+), 2 deletions(-)
diff --git a/ckan/patches/ckanext-harvest/00_translates.patch b/ckan/patches/ckanext-harvest/00_translates.patch
index cec207a8..976ed463 100644
--- a/ckan/patches/ckanext-harvest/00_translates.patch
+++ b/ckan/patches/ckanext-harvest/00_translates.patch
@@ -26,9 +26,18 @@ index b7feb3d..b773a44 100644
{% endblock %}
diff --git a/ckanext/harvest/templates/source/new_source_form.html b/ckanext/harvest/templates/source/new_source_form.html
-index 324d012..8a500b9 100644
+index 324d012..37358fc 100644
--- a/ckanext/harvest/templates/source/new_source_form.html
+++ b/ckanext/harvest/templates/source/new_source_form.html
+@@ -8,7 +8,7 @@
+
+ {% call form.input('url', id='field-url', label=_('URL'), value=data.url, error=errors.url, classes=['control-full', 'control-large']) %}
+
+- {{ _('This should include the http:// part of the URL') }}
++ {{ _('This should include the http:// part of the URL') }}
+
+ {% endcall %}
+
@@ -26,7 +26,7 @@
{{ form.markdown('notes', id='field-notes', label=_('Description'), value=data.notes, error=errors.notes) }}
@@ -38,8 +47,21 @@ index 324d012..8a500b9 100644
{% for harvester in h.harvesters_info() %}
{% set checked = False %}
+@@ -46,7 +46,11 @@
+ {{ form.select('frequency', id='field-frequency', label=_('Update frequency'), options=h.harvest_frequencies(), selected=data.frequency, error=errors.frequency) }}
+
+ {% block extra_config %}
+- {{ form.textarea('config', id='field-config', label=_('Configuration'), value=data.config, error=errors.config) }}
++ {% call form.textarea('config', id='field-config', label=_('Configuration'), value=data.config, error=errors.config) %}
++
++ {{ _('You can validate the JSON at: ') }} {{ _('JSONLint') }}
++
++ {% endcall %}
+ {% endblock extra_config %}
+
+ {# if we have a default group then this wants remembering #}
diff --git a/ckanext/harvest/templates/source/search.html b/ckanext/harvest/templates/source/search.html
-index a929943..06cb373 100644
+index d9ceeea..44d118b 100644
--- a/ckanext/harvest/templates/source/search.html
+++ b/ckanext/harvest/templates/source/search.html
@@ -44,7 +44,26 @@
From 0ed33f2af50dff0ffa074ed9ee2cadc1015b4d6b Mon Sep 17 00:00:00 2001
From: mjanez <96422458+mjanez@users.noreply.github.com>
Date: Tue, 5 Mar 2024 07:41:05 +0100
Subject: [PATCH 6/7] Update CKAN user management instructions
---
README.md | 17 ++++++++++-------
1 file changed, 10 insertions(+), 7 deletions(-)
diff --git a/README.md b/README.md
index 19e52dc6..0b4b7879 100644
--- a/README.md
+++ b/README.md
@@ -536,27 +536,30 @@ If need to use a backup, restore it:
### CKAN. Manage new users
-1. Create a new user from the Docker host, for example to create a new user called 'admin'
+1. Create a new user from the Docker host, for example to create a new user called `user_example`
```bash
- docker exec -it
ckan -c ckan.ini user add admin email=admin@localhost
+ docker exec -it ckan -c ckan.ini user add user_example email=user_example@localhost
+
+ # Admin user
+ docker exec -it ckan -c ckan.ini sysadmin add admin_example email=admin_example@localhost name=admin_example
```
- To delete the 'admin' user
+ To delete the 'user_example' user
```bash
- docker exec -it ckan -c ckan.ini user remove admin`
+ docker exec -it ckan -c ckan.ini user remove user_example`
```
1. Create a new user from within the ckan container. You will need to get a session on the running container
```bash
- ckan -c ckan.ini user add admin email=admin@localhost`
+ ckan -c ckan.ini user add user_example email=user_example@localhost`
```
- To delete the 'admin' user
+ To delete the 'user_example' user
```bash
- ckan -c ckan.ini user remove admin`
+ ckan -c ckan.ini user remove user_example`
```
From 68a9ce9eccde43ec130e0148cf3282441bd91a03 Mon Sep 17 00:00:00 2001
From: mjanez <96422458+mjanez@users.noreply.github.com>
Date: Fri, 8 Mar 2024 21:31:08 +0100
Subject: [PATCH 7/7] Add harvest & xloader worker configuration
- Update Dockerfiles
- Update start_ckan.sh with custom workers
- Update start_ckan_development.sh.override to remove automatic workers
- Add supervisor harvester.conf
- Add supervisor xloader.conf
- Update Scheming DCAT version to 2.1.0
---
.env.example | 5 ++-
ckan/Dockerfile | 10 ++---
ckan/Dockerfile.dev | 4 +-
ckan/Dockerfile.ghcr | 9 +++++
ckan/setup/start_ckan.sh.override | 18 ++++-----
ckan/setup/start_ckan_development.sh.override | 19 ++++------
ckan/setup/workers/harvester.conf | 38 +++++++++++++++++--
ckan/setup/workers/xloader.conf | 12 ++++++
8 files changed, 83 insertions(+), 32 deletions(-)
create mode 100644 ckan/setup/workers/xloader.conf
diff --git a/.env.example b/.env.example
index 4b5164d3..137e42cb 100644
--- a/.env.example
+++ b/.env.example
@@ -89,6 +89,7 @@ CKAN_SITE_URL=http://localhost:81
CKAN__ROOT_PATH=/catalog/{{LANG}}
CKAN_PORT=5000
CKAN__FAVICON=/catalog/base/images/ckan.ico
+CKAN__SITE_LOGO=/images/default/ckan-logo.png
CKAN___BEAKER__SESSION__SECRET=CHANGE_ME
# See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings
CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME
@@ -125,7 +126,7 @@ CKAN__LOCALE_ORDER="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru
CKAN__LOCALES_OFFERED="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv"
# Extensions
-CKAN__PLUGINS="envvars stats text_view image_view webpage_view recline_view resourcedictionary datastore xloader harvest ckan_harvester spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface scheming_dcat_datasets scheming_dcat_groups scheming_dcat_organizations scheming_dcat scheming_dcat_ckan_harvester scheming_dcat_csw_harvester pdf_view pages fluent"
+CKAN__PLUGINS="envvars stats text_view image_view webpage_view recline_view resourcedictionary datastore xloader harvest spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface scheming_dcat_datasets scheming_dcat_groups scheming_dcat_organizations scheming_dcat scheming_dcat_ckan_harvester scheming_dcat_xls_harvester pdf_view pages fluent"
# ckanext-harvest
CKAN__HARVEST__MQ__TYPE=redis
@@ -137,7 +138,7 @@ CKAN__HARVEST__LOG_TIMEFRAME=40
# ckanext-xloader
CKANEXT__XLOADER__API_TOKEN=api_token
-CKANEXT__XLOADER__JOBS__DB_URI=postgresql://ckan:ckan@db/ckan
+CKANEXT__XLOADER__JOBS__DB_URI=postgresql://ckandbuser:ckandbpassword@db/ckandb
# ckanext-dcat
CKANEXT__DCAT__BASE_URI=${CKAN_URL}
diff --git a/ckan/Dockerfile b/ckan/Dockerfile
index eba653fe..1afab4a7 100644
--- a/ckan/Dockerfile
+++ b/ckan/Dockerfile
@@ -22,7 +22,7 @@ COPY req_fixes req_fixes
## Pages - v0.5.2 ##
## PDFView - 0.0.8 ##
## Fluent - v1.0.1 (Forked stable version) ##
-## Scheming DCAT - v2.0.0 (GeoDCAT-AP/NTI-RISP extended version) ##
+## Scheming DCAT - v2.1.0 (GeoDCAT-AP/NTI-RISP extended version) ##
RUN echo ${TZ} > /etc/timezone && \
if ! [ /usr/share/zoneinfo/${TZ} -ef /etc/localtime ]; then cp /usr/share/zoneinfo/${TZ} /etc/localtime; fi && \
# Remove apk cache
@@ -54,8 +54,8 @@ RUN echo ${TZ} > /etc/timezone && \
echo "mjanez/ckanext-fluent" && \
pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-fluent.git@v1.0.1#egg=ckanext-fluent && \
echo "mjanez/ckanext-scheming_dcat" && \
- pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-scheming_dcat.git@v2.0.0#egg=ckanext_scheming_dcat && \
- pip3 install --no-cache-dir -r https://raw.githubusercontent.com/mjanez/ckanext-scheming_dcat/v2.0.0/requirements.txt
+ pip3 install --no-cache-dir -e git+https://github.com/mjanez/ckanext-scheming_dcat.git@v2.1.0#egg=ckanext_scheming_dcat && \
+ pip3 install --no-cache-dir -r https://raw.githubusercontent.com/mjanez/ckanext-scheming_dcat/v2.1.0/requirements.txt
# Used to configure the container environment by setting environment variables, creating users, running initialization scripts, .etc
COPY docker-entrypoint.d/* /docker-entrypoint.d/
@@ -79,8 +79,8 @@ RUN for d in $APP_DIR/patches/*; do \
COPY setup/start_ckan.sh.override ${APP_DIR}/start_ckan.sh
RUN chmod +x ${APP_DIR}/start_ckan.sh
-## Harvester
-COPY setup/workers/harvester.conf /etc/supervisord.d/harvester.conf
+## Load workers supervisor configuration
+COPY setup/workers/* /etc/supervisord.d/
# Start CKAN
CMD ["/bin/sh", "-c", "$APP_DIR/start_ckan.sh"]
\ No newline at end of file
diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev
index 0431ba35..97bbbf8d 100644
--- a/ckan/Dockerfile.dev
+++ b/ckan/Dockerfile.dev
@@ -60,8 +60,8 @@ COPY setup/who.ini ./
COPY setup/start_ckan_development.sh.override ./start_ckan_development.sh
RUN chmod +x ./start_ckan_development.sh
-## Harvester
-COPY setup/workers/harvester.conf /etc/supervisord.d/harvester.conf
+## Load workers supervisor configuration
+COPY setup/workers/* /etc/supervisord.d/
# Apply any patches needed to CKAN core or any of the built extensions (not the
# runtime mounted ones)
diff --git a/ckan/Dockerfile.ghcr b/ckan/Dockerfile.ghcr
index 92614036..98430ee1 100644
--- a/ckan/Dockerfile.ghcr
+++ b/ckan/Dockerfile.ghcr
@@ -38,4 +38,13 @@ RUN for d in $APP_DIR/patches/*; do \
fi ; \
done
+# Workers
+## Update start_ckan.sh with custom workers
+COPY setup/start_ckan.sh.override ${APP_DIR}/start_ckan.sh
+RUN chmod +x ${APP_DIR}/start_ckan.sh
+
+## Load workers supervisor configuration
+COPY setup/workers/* /etc/supervisord.d/
+
+# Start CKAN
CMD ["/bin/sh", "-c", "$APP_DIR/start_ckan.sh"]
\ No newline at end of file
diff --git a/ckan/setup/start_ckan.sh.override b/ckan/setup/start_ckan.sh.override
index bc9ee01c..84656952 100644
--- a/ckan/setup/start_ckan.sh.override
+++ b/ckan/setup/start_ckan.sh.override
@@ -6,7 +6,6 @@ if grep -E "beaker.session.secret ?= ?$" ckan.ini
then
echo "Setting beaker.session.secret in ini file"
ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')"
- ckan config-tool $CKAN_INI "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')"
JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())')
ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}"
ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}"
@@ -51,17 +50,18 @@ UWSGI_OPTS="--plugins http,python \
if [ $? -eq 0 ]
then
# Start supervisord
+ echo "[prerun.workers] Loading the CKAN workers with supervisord..."
supervisord --configuration /etc/supervisord.conf &
+
+ # Workers
+ ## Add harvester background procces to crontab
+ echo "[prerun.workers] Add harvester background procceses to crontab"
+ crontab -l | { cat; echo "*/15 * * * * /usr/bin/supervisorctl start ckan_harvester_run"; } | crontab -
+ ## Clean-up mechanism for the harvest log table. 'ckan.harvest.log_timeframe'. The default time frame is 30 days
+ crontab -l | { cat; echo "0 5 */30 * * /usr/bin/supervisorctl start ckan_harvester_clean_log"; } | crontab -
+
# Start uwsgi
uwsgi $UWSGI_OPTS
else
echo "[prerun] failed...not starting CKAN."
fi
-# Workers
-## Start the Harvester worker
-echo "[prerun.workers] Starting the CKAN Harvester worker"
-ckan harvester run
-## Add harvester to crontab
-crontab -l | { cat; echo "0 */2 * * * ckan harvester run"; } | crontab -
-## Clean-up mechanism for the harvest log table. 'ckan.harvest.log_timeframe'. The default time frame is 30 days
-crontab -l | { cat; echo "0 5 * * * ckan harvester clean-harvest-log"; } | crontab -
diff --git a/ckan/setup/start_ckan_development.sh.override b/ckan/setup/start_ckan_development.sh.override
index dd065061..b481c807 100644
--- a/ckan/setup/start_ckan_development.sh.override
+++ b/ckan/setup/start_ckan_development.sh.override
@@ -51,7 +51,6 @@ if grep -E "beaker.session.secret ?= ?$" ckan.ini
then
echo "Setting beaker.session.secret in ini file"
ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')"
- ckan config-tool $CKAN_INI "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')"
JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())')
ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}"
ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}"
@@ -95,15 +94,13 @@ mkdir -p $CKAN_LOGS_PATH/xloader
chown -R ckan:ckan $CKAN_LOGS_PATH/xloader
# Start supervisord
-supervisord --configuration /etc/supervisord.conf
-
-# Workers
-## Start the Harvester worker
-echo "[prerun.workers] Starting the CKAN Harvester worker"
-ckan harvester run
-
-## Clean-up mechanism for the harvest log table
-ckan harvester clean-harvest-log
+#supervisord --configuration /etc/supervisord.conf &
# Start the development server as the ckan user with automatic reload
-su ckan -c "/usr/bin/ckan -c $CKAN_INI run -H 0.0.0.0"
\ No newline at end of file
+su ckan -c "/usr/bin/ckan -c $CKAN_INI run -H 0.0.0.0"
+
+# Workers
+# To start the Harvester worker
+# ckan harvester run
+# Clean-up mechanism for the harvest log table
+# ckan harvester clean-harvest-log
\ No newline at end of file
diff --git a/ckan/setup/workers/harvester.conf b/ckan/setup/workers/harvester.conf
index 69b1fa34..cb1c2e3d 100644
--- a/ckan/setup/workers/harvester.conf
+++ b/ckan/setup/workers/harvester.conf
@@ -3,17 +3,49 @@ command=ckan harvester gather-consumer
user=ckan
numprocs=1
stdout_logfile=/var/log/harvester/gather_consumer.log
-stderr_logfile=/var/log/harvester/gather_consumer.err.log
+stdout_logfile_maxbytes=50MB
+stderr_logfile=/var/log/harvester/gather_consumer.log
+stderr_logfile_maxbytes=50MB
autostart=true
autorestart=true
startsecs=10
+priority=1
[program:ckan_fetch_consumer]
command=ckan harvester fetch-consumer
user=ckan
numprocs=1
stdout_logfile=/var/log/harvester/fetch_consumer.log
-stderr_logfile=/var/log/harvester/fetch_consumer.err.log
+stdout_logfile_maxbytes=50MB
+stderr_logfile=/var/log/harvester/fetch_consumer.log
+stderr_logfile_maxbytes=50MB
autostart=true
autorestart=true
-startsecs=10
\ No newline at end of file
+startsecs=10
+priority=2
+
+[program:ckan_harvester_run]
+command=ckan harvester run
+user=ckan
+numprocs=1
+stdout_logfile=/var/log/harvester/ckan_harvester.log
+stdout_logfile_maxbytes=25MB
+stderr_logfile=/var/log/harvester/ckan_harvester.log
+stderr_logfile_maxbytes=25MB
+autostart=true
+autorestart=false
+startsecs=2
+priority=3
+
+[program:ckan_harvester_clean_log]
+command=ckan harvester clean-harvest-log
+user=ckan
+numprocs=1
+stdout_logfile=/var/log/harvester/ckan_harvester_clean_log.log
+stdout_logfile_maxbytes=25MB
+stderr_logfile=/var/log/harvester/ckan_harvester_clean_log.log
+stderr_logfile_maxbytes=25MB
+autostart=false
+autorestart=false
+startsecs=2
+priority=4
\ No newline at end of file
diff --git a/ckan/setup/workers/xloader.conf b/ckan/setup/workers/xloader.conf
new file mode 100644
index 00000000..c7749ddb
--- /dev/null
+++ b/ckan/setup/workers/xloader.conf
@@ -0,0 +1,12 @@
+[program:ckan_xloader]
+command=ckan jobs worker default
+user=ckan
+numprocs=1
+stdout_logfile=/var/log/harvester/ckan_xloader.log
+stdout_logfile_maxbytes=100MB
+stderr_logfile=/var/log/harvester/ckan_xloader.log
+stderr_logfile_maxbytes=100MB
+autostart=true
+autorestart=true
+startsecs=4
+priority=1
\ No newline at end of file