diff --git a/.docker-compose.vital-strategies-theme.yaml b/.docker-compose.vital-strategies-theme.yaml index 44910c2..45c9848 100644 --- a/.docker-compose.vital-strategies-theme.yaml +++ b/.docker-compose.vital-strategies-theme.yaml @@ -23,9 +23,9 @@ services: EXTRA_PACKAGES: cron EXTRA_FILESYSTEM: "./overrides/vital-strategies/filesystem/" POST_INSTALL: | - install_standard_ckan_extension_github -r datopian/ckanext-querytool -b dev/python3-upgrade &&\ + install_standard_ckan_extension_github -r datopian/ckanext-querytool -b cc6c8e6f19f59e6842d370bf7ac87d94e37a2831 &&\ install_standard_ckan_extension_github -r ckan/ckanext-geoview && \ - #install_standard_ckan_extension_github -r datopian/ckanext-sentry -b 2.10 && \ + install_standard_ckan_extension_github -r datopian/ckanext-sentry -b 2.10 && \ install_standard_ckan_extension_github -r datopian/ckanext-gtm && \ install_standard_ckan_extension_github -r datopian/ckanext-s3filestore -b ckan-2.10 && \ cd ~/venv/src/ckanext-querytool && ~/venv/bin/python setup.py compile_catalog -l en -f && \ @@ -47,7 +47,7 @@ services: CKAN_BRANCH: ckan-2.10.4 POST_INSTALL: | install_standard_ckan_extension_github -r datopian/ckanext-s3filestore -b ckan-2.10 &&\ - install_standard_ckan_extension_github -r datopian/ckanext-querytool -b dev/python3-upgrade &&\ + install_standard_ckan_extension_github -r datopian/ckanext-querytool -b cc6c8e6f19f59e6842d370bf7ac87d94e37a2831 &&\ install_standard_ckan_extension_github -r ckan/ckanext-geoview environment: - CKAN_CONFIG_TEMPLATE_PREFIX=vital-strategies-theme- diff --git a/ckan/Dockerfile b/ckan/Dockerfile index 6ed4ef5..56f4e19 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -101,7 +101,7 @@ RUN ckan-pip install --upgrade pip &&\ RUN ckan-pip install supervisor && \ ckan-pip install uwsgi==2.0.19.1 && \ - ckan-pip install gevent==21.12.0 + ckan-pip install gevent==24.2.1 RUN apt-get update && apt-get install -y \ uwsgi-plugins-all \ diff --git a/ckan/post_install_functions.sh b/ckan/post_install_functions.sh index ec2c17e..9cd2c7a 100644 --- a/ckan/post_install_functions.sh +++ b/ckan/post_install_functions.sh @@ -30,11 +30,15 @@ install_standard_ckan_extension_github() { # Check if the branch exists by examining the output directly BRANCH_EXISTS=$(git ls-remote --heads ${GITHUB_URL}/${REPO_NAME}.git ${BRANCH}) + if [ ! -n "$BRANCH_EXISTS" ]; then + # It might be a commit instead of a branch + BRANCH_EXISTS=$(git ls-remote --heads ${GITHUB_URL}/${REPO_NAME}.git | grep -o $BRANCH) + fi + if [ -z "$BRANCH_EXISTS" ]; then echo "#### BRANCH EXISTS: $BRANCH_EXISTS ####" if [ "$BRANCH" = "master" ]; then - # Attempt to switch to 'main' if 'master' does not exist BRANCH_EXISTS=$(git ls-remote --heads ${GITHUB_URL}/${REPO_NAME}.git main) if [ -n "$BRANCH_EXISTS" ]; then echo "Branch 'master' not found, switching to 'main'." @@ -44,7 +48,6 @@ install_standard_ckan_extension_github() { exit 1 fi else - # If a specific branch was provided and does not exist, print an error and exit echo "Branch '$BRANCH' not found. Please check the branch name." exit 1 fi @@ -62,14 +65,40 @@ install_standard_ckan_extension_github() { done &&\ ckan-pip install --no-use-pep517 --index-url ${PIP_INDEX_URL} -e ${TMPDIR} else - TEMPFILE=`mktemp` - for REQUIREMENTS_FILE_NAME in requirements pip-requirements - do - if wget -O $TEMPFILE https://raw.githubusercontent.com/${REPO_NAME}/$BRANCH/$REQUIREMENTS_FILE_NAME.txt - then ckan-pip install --index-url ${PIP_INDEX_URL} -r $TEMPFILE && break; + # Remove poetry files: ckan-cloud-docker currently has issues with poetry dependencies + if [ "${REPO_NAME}" = "datopian/ckanext-sentry" ]; then + TMPDIR=${CKAN_VENV}/src/${EGG} + git clone -b $BRANCH ${GITHUB_URL}/${REPO_NAME}.git ${TMPDIR} + + CURRENT_DIR=$(pwd) + + cd ${TMPDIR} + + if [ -f "poetry.lock" ] && [ -f "pyproject.toml" ]; then + rm -f "poetry.lock" "pyproject.toml" fi - done &&\ - ckan-pip install --no-use-pep517 --index-url ${PIP_INDEX_URL} -e git+${GITHUB_URL}/${REPO_NAME}.git@$BRANCH#egg=${EGG} + + for REQUIREMENTS_FILE_NAME in requirements pip-requirements + do + if [ -f ${TMPDIR}/$REQUIREMENTS_FILE_NAME.txt ]; then + ckan-pip install --index-url ${PIP_INDEX_URL} -r ${TMPDIR}/$REQUIREMENTS_FILE_NAME.txt && break; + fi + done + + ckan-pip install --no-use-pep517 --index-url ${PIP_INDEX_URL} -e ${TMPDIR} + + cd ${CURRENT_DIR} + + else + TEMPFILE=`mktemp` + for REQUIREMENTS_FILE_NAME in requirements pip-requirements + do + if wget -O $TEMPFILE https://raw.githubusercontent.com/${REPO_NAME}/$BRANCH/$REQUIREMENTS_FILE_NAME.txt + then ckan-pip install --index-url ${PIP_INDEX_URL} -r $TEMPFILE && break; + fi + done &&\ + ckan-pip install --no-use-pep517 --index-url ${PIP_INDEX_URL} -e git+${GITHUB_URL}/${REPO_NAME}.git@$BRANCH#egg=${EGG} + fi fi } diff --git a/ckan/templater.sh b/ckan/templater.sh old mode 100644 new mode 100755 index 80f92b5..459f8cd --- a/ckan/templater.sh +++ b/ckan/templater.sh @@ -79,5 +79,20 @@ for var in $vars; do value=$(echo "$value" | sed 's/\//\\\//g'); replaces="-e 's/{{$var}}/${value}/g' $replaces" done + +if [[ ! -z "$(var_value GTM_ID)" ]]; then + replaces="-e '/^ckan.plugins =/ s/$/ gtm/' $replaces" +fi + +if [[ ! -z "$(var_value SENTRY_DSN)" ]]; then + replaces="-e '/^ckan.plugins =/ s/$/ sentry/' $replaces" +fi + +if [[ ! -z "$(var_value "AWS_ACCESS_KEY_ID")" ]] && [[ ! -z "$(var_value "AWS_SECRET_ACCESS_KEY")" ]] && [[ "$CURRENT_PLUGINS" != *"s3filestore"* ]]; then + replaces="-e '/^ckan.plugins =/ s/$/ s3filestore/' $replaces" +fi + +replaces="$replaces -e '/^ckan.plugins =/ s/querytool//g' -e '/^ckan.plugins =/ s/$/ querytool/'" + escaped_template_path=$(echo $template | sed 's/ /\\ /g') eval sed $replaces "$escaped_template_path" diff --git a/docker-compose.yaml b/docker-compose.yaml index e43935e..d5b83ba 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -3,11 +3,15 @@ services: image: traefik:1.7.2-alpine restart: always volumes: - - ./traefik/traefik.toml:/traefik.toml # Comment out for development - #- ./traefik/traefik.dev.toml:/traefik.toml # Uncomment for development + - ./traefik/traefik.toml.template:/traefik.toml.template + #- ./traefik/traefik.dev.toml:/traefik.dev.toml # Uncomment for development - ./traefik/acme.json:/acme.json + - ./cca-operator/templater.sh:/templater.sh + - ./docker-compose/traefik-secrets.sh:/traefik-secrets.sh + - ./traefik/entrypoint.sh:/entrypoint.sh networks: - ckan-multi + entrypoint: ["/bin/sh", "-c", "/entrypoint.sh"] varnish: image: million12/varnish diff --git a/docker-compose/ckan-conf-templates/vital-strategies-theme-ckan.ini.template b/docker-compose/ckan-conf-templates/vital-strategies-theme-ckan.ini.template index 6b98ba6..ddd70cf 100644 --- a/docker-compose/ckan-conf-templates/vital-strategies-theme-ckan.ini.template +++ b/docker-compose/ckan-conf-templates/vital-strategies-theme-ckan.ini.template @@ -153,6 +153,10 @@ ckanext.querytool.allow_nav_bar = True ## Google Analytics +ckanext.gtm.gtm_id = {{GTM_ID}} + +# OR + googleanalytics.id = {{GA_ID}} googleanalytics.account = {{GA_ACCOUNT}} googleanalytics.username = {{GA_USERNAME}} @@ -189,11 +193,11 @@ ckanext.xloader.jobs_db.uri = postgresql://postgres:123456@jobs-db/postgres #email_to = errors@example.com #error_email_from = ckan-errors@example.com -# smtp.server = {{SMTP_SERVER}} -# smtp.starttls = True -# smtp.user = {{SMTP_USER}} -# smtp.password = {{SMTP_PASSWORD}} -# smtp.mail_from = vitalplatform@vitalstrategies.org +smtp.server = {{SMTP_SERVER}} +smtp.starttls = True +smtp.user = {{SMTP_USER}} +smtp.password = {{SMTP_PASSWORD}} +smtp.mail_from = {{SMTP_MAIL_FROM}} ### @@ -207,9 +211,9 @@ ckan.harvest.mq.hostname = redis ckan.harvest.mq.redis_db = 9 ## Sentry settings -# sentry_dsn = {{SENTRY_DSN}} -# ckan.sentry.configure_logging = True -# ckan.sentry.log_level = ERROR +ckan.sentry.dsn = {{SENTRY_DSN}} +ckan.sentry.configure_logging = True +ckan.sentry.log_level = ERROR ## Logging configuration diff --git a/docker-compose/ckan-secrets.dat b/docker-compose/ckan-secrets.dat index 27a5f4a..18b76a3 100644 --- a/docker-compose/ckan-secrets.dat +++ b/docker-compose/ckan-secrets.dat @@ -16,11 +16,14 @@ ckan required SMTP_USER info Enter SMTP server username ckan optional SMTP_PASSWORD empty Enter SMTP server password ckan optional SMTP_MAIL_FROM empty Enter SMTP mail from ckan required CKAN_SITE_URL http://ckan:5000 Enter Website URL (including https:// or http://) -ckan optional AWS_ACCESS_KEY_ID empty Enter AWS secret key (if any) -ckan optional AWS_SECRET_ACCESS_KEY empty AWS secret access key (if any) -ckan optional SENTRY_DSN https://@sentry.io/ Enter Sentry DSN URL with token and ID -ckan optional GA_ID empty Enter Google Analytics ID -ckan optional GA_ACCOUNT empty Enter Google Analytics account name -ckan optional GA_PASSWORD empty Enter Google Analytics password -ckan optional GA_USERNAME empty Enter Google Analytics username -datapusher optional TIMEZONE UTC Enter Datapusher timezone +ckan optional AWS_ACCESS_KEY_ID empty Enter AWS secret key [Skip if not using AWS] +ckan optional AWS_SECRET_ACCESS_KEY empty AWS secret access key [Skip if not using AWS] +ckan optional SENTRY_DSN https://@sentry.io/ Enter Sentry DSN URL with token and ID [Skip if not using Sentry] +ckan optional GTM_ID empty Enter Google Tag Manager ID [Skip if not using the new Google Analytics (GTM) or not using Google Analytics at all] +ckan optional GA_ID empty Enter Google Analytics ID [Skip if using GTM or not using Google Analytics at all] +ckan optional GA_ACCOUNT empty Enter Google Analytics account name [Skip if using GTM or not using Google Analytics at all] +ckan optional GA_PASSWORD empty Enter Google Analytics password [Skip if using GTM or not using Google Analytics at all] +ckan optional GA_USERNAME empty Enter Google Analytics username [Skip if using GTM or not using Google Analytics at all] +datapusher optional TIMEZONE UTC Enter Datapusher timezone +traefik optional CERTIFICATE_EMAIL Enter email address for Let's Encrypt certificate [Skip if using self-signed certificate] +traefik optional CERTIFICATE_DOMAIN Enter domain for Let's Encrypt certificate [Skip if using self-signed certificate] diff --git a/traefik/entrypoint.sh b/traefik/entrypoint.sh new file mode 100755 index 0000000..b34e2cb --- /dev/null +++ b/traefik/entrypoint.sh @@ -0,0 +1,42 @@ +#!/bin/sh +set -e + +# Update the Traefik configuration file with secrets if not development mode +if [ -f /traefik.dev.toml ]; then + echo "Using development configuration" + cp /traefik.dev.toml /traefik.toml +else + if [ ! -f /traefik.toml.template ]; then + echo "Traefik template file does not exist, exiting" + exit 1 + fi + if [ ! -f /traefik-secrets.sh ]; then + echo "Traefik secrets file does not exist. Please run 'make secret' to generate it before starting the container" + exit 1 + fi + if [ ! -f /templater.sh ]; then + echo "Templater script does not exist, exiting" + exit 1 + fi + + echo "Traefik configuration file does not exist, templating" + + chmod +x /templater.sh + ./templater.sh /traefik.toml.template -f /traefik-secrets.sh > traefik.toml +fi + +# Fix acme.json file permissions: set to 600 +chmod 600 /acme.json + +# first arg is `-f` or `--some-option` +if [ "${1#-}" != "$1" ]; then + set -- traefik "$@" +fi + +# if our command is a valid Traefik subcommand, let's invoke it through Traefik instead +# (this allows for "docker run traefik version", etc) +if traefik "$1" --help | grep -s -q "help"; then + set -- traefik "$@" +fi + +exec "$@" diff --git a/traefik/traefik.toml.template b/traefik/traefik.toml.template new file mode 100644 index 0000000..3d36de4 --- /dev/null +++ b/traefik/traefik.toml.template @@ -0,0 +1,49 @@ +debug = false +defaultEntryPoints = ["http", "https"] + +[entryPoints] + [entryPoints.http] + address = ":80" + + [entryPoints.https] + address = ":443" + [entryPoints.https.tls] + [entryPoints.api] + address = ":8081" + + [api] + entryPoint = "api" + + [ping] + entryPoint = "http" + + [acme] + email = "{{CERTIFICATE_EMAIL}}" + storage = "/acme.json" + entryPoint = "https" + onHostRule = true + + [[acme.domains]] + main = "{{CERTIFICATE_DOMAIN}}" + + [acme.httpChallenge] + entryPoint = "http" + + [accessLog] + + [file] + watch = true + + [backends] + [backends.ckan] + [backends.ckan.servers.server1] + url = "http://nginx:8080" + + [frontends] + [frontends.ckan] + backend="ckan" + passHostHeader = true + [frontends.ckan.headers] + SSLRedirect = true + [frontends.ckan.routes.route1] + rule = "Host:{{CERTIFICATE_DOMAIN}}"