Merge pull request conwetlab#58 from qld-gov-au/develop

Develop to master

.ahoy.yml

@@ -10,14 +10,19 @@ commands:
       ahoy title "Building project"
       ahoy pre-flight
       ahoy clean
-      (docker network prune -f > /dev/null && docker network inspect amazeeio-network > /dev/null || docker network create amazeeio-network)
+      ahoy build-network
       ahoy up -- --build --force-recreate
-      ahoy install-dev
       ahoy install-site
-      # ahoy title "Build complete"
+      ahoy title "Build complete"
       ahoy doctor
       ahoy info 1
 
+  build-network:
+    usage: Ensure that the amazeeio network exists.
+    cmd: |
+      docker network prune -f > /dev/null
+      docker network inspect amazeeio-network > /dev/null || docker network create amazeeio-network
+
   info:
     usage: Print information about this project.
     cmd: |
@@ -31,6 +36,8 @@ commands:
     usage: Build and start Docker containers.
     cmd: |
       docker-compose up -d "$@"
+      sleep 10
+      docker-compose logs
       ahoy cli "dockerize -wait tcp://ckan:3000 -timeout 1m"
       if docker-compose logs | grep -q "\[Error\]"; then docker-compose logs; exit 1; fi
       if docker-compose logs | grep -q "Exception"; then docker-compose logs; exit 1; fi
@@ -39,7 +46,7 @@ commands:
 
   down:
     usage: Stop Docker containers and remove container, images, volumes and networks.
-    cmd: "if [ -f \"docker-compose.yml\" ]; then docker-compose down --volumes; fi"
+    cmd: 'if [ -f "docker-compose.yml" ]; then docker-compose down --volumes; fi'
 
   start:
     usage: Start existing Docker containers.
@@ -55,26 +62,25 @@ commands:
 
   logs:
     usage: Show Docker logs.
-    cmd: docker-compose logs -f "$@"
+    cmd: docker-compose logs "$@"
 
   pull:
     usage: Pull latest docker images.
     cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep amazeeio/ | grep -v none | xargs -n1 docker pull | cat; fi
 
   cli:
     usage: Start a shell inside CLI container or run a command.
-    cmd: if \[ "${#}" -ne 0 \]; then docker exec -i $(docker-compose ps -q ckan) sh -c ". /app/ckan/default/bin/activate; $*"; else docker exec -it $(docker-compose ps -q ckan) sh -c ". /app/ckan/default/bin/activate && sh"; fi
+    cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c '. ${VENV_DIR}/bin/activate; cd $APP_DIR;'" $*"; else docker exec $(docker-compose ps -q ckan) sh -c '. ${VENV_DIR}/bin/activate && cd $APP_DIR && sh'; fi
 
   doctor:
     usage: Find problems with current project setup.
     cmd: .docker/scripts/doctor.sh "$@"
 
-
   install-site:
     usage: Install a site.
     cmd: |
       ahoy title "Installing a fresh site"
-      ahoy cli "/app/scripts/init.sh"
+      ahoy cli "./scripts/init.sh"
 
   clean:
     usage: Remove containers and all build files.
@@ -93,29 +99,20 @@ commands:
       git ls-files --others -i --exclude-from=.git/info/exclude | xargs rm -Rf
       find . -type d -not -path "./.git/*" -empty -delete
 
-  install-dev:
-    usage: Install dependencies.
-    cmd: |
-      docker cp -L requirements-dev.txt $(docker-compose ps -q ckan):/app/.
-      docker cp -L .flake8 $(docker-compose ps -q ckan):/app/.
-      docker cp -L test $(docker-compose ps -q ckan):/app/.
-      ahoy cli "pip install -r /app/requirements-dev.txt"
-    hide: true
-
   flush-redis:
     usage: Flush Redis cache.
     cmd: docker exec -i $(docker-compose ps -q redis) redis-cli flushall > /dev/null
 
   lint:
     usage: Lint code.
     cmd: |
-      ahoy cli "flake8 ${@:-/app/ckanext}" || \
+      ahoy cli "flake8 ${@:-ckanext}" || \
       [ "${ALLOW_LINT_FAIL:-0}" -eq 1 ]
 
   test-unit:
     usage: Run unit tests.
     cmd: |
-      ahoy cli "nosetests" || \
+      ahoy cli 'nosetests --with-pylons=${CKAN_INI}' || \
       [ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ]
 
   test-bdd:
@@ -124,17 +121,16 @@ commands:
       ahoy start-ckan-job-worker &
       ahoy start-mailmock &
       sleep 5 &&
-      ahoy cli "behave ${*:-/app/test/features}" || \
+      ahoy cli "behave ${*:-test/features}" || \
       [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
       ahoy stop-mailmock
       ahoy stop-ckan-job-worker
 
-
   start-mailmock:
     usage: Starts email mock server used for email BDD tests
     cmd: |
       ahoy title 'Starting mailmock'
-      ahoy cli "cd /app/ckan/default/bin/ && ./mailmock -p 8025 -o /app/test/emails --no-stdout" # for debugging mailmock email output remove --no-stdout
+      ahoy cli 'mailmock -p 8025 -o ${APP_DIR}/test/emails --no-stdout' # for debugging mailmock email output remove --no-stdout
 
   stop-mailmock:
     usage: Stops email mock server used for email BDD tests
@@ -146,10 +142,8 @@ commands:
     usage: Starts CKAN background job worker
     cmd: |
       ahoy title 'Starting CKAN background job worker'
-      ahoy cli "cd /usr/lib/ckan/default/src/ckan && \
-                export CKAN_INI=/app/ckan/default/production.ini &&
-                /app/scripts/ckan_cli jobs clear && \
-                /app/scripts/ckan_cli jobs worker default"
+      ahoy cli "ckan_cli jobs clear && \
+                ckan_cli jobs worker"
 
   stop-ckan-job-worker:
     usage: Stops CKAN background job worker
@@ -191,5 +185,5 @@ entrypoint:
     export LAGOON_LOCALDEV_URL=http://ckanext-datarequests.docker.amazee.io
     [ -f .env ] && [ -s .env ] && export $(grep -v '^#' .env | xargs) && if [ -f .env.local ] && [ -s .env.local ]; then export $(grep -v '^#' .env.local | xargs); fi
     bash -e -c "$0" "$@"
-  - '{{cmd}}'
-  - '{{name}}'
+  - "{{cmd}}"
+  - "{{name}}"

.circleci/process-artifacts.sh

@@ -6,8 +6,8 @@ set -e
 
 # Create screenshots directory in case it was not created before. This is to
 # avoid this script to fail when copying artifacts.
-ahoy cli "mkdir -p /app/test/screenshots"
+ahoy cli "mkdir -p test/screenshots"
 
 # Copy from the app container to the build host for storage.
-mkdir -p /tmp/artifacts/behave/screenshots
-docker cp "$(docker-compose ps -q ckan)":/app/test/screenshots /tmp/artifacts/behave
+mkdir -p /tmp/artifacts/behave
+docker cp "$(docker-compose ps -q ckan)":/app/test/screenshots /tmp/artifacts/behave/

.circleci/test.sh

@@ -11,4 +11,4 @@ echo "==> Run Unit tests"
 ahoy test-unit
 
 echo "==> Run BDD tests"
-ahoy test-bdd
+ahoy test-bdd || (ahoy logs; exit 1)

.docker/Dockerfile.ckan

@@ -1,44 +1,46 @@
-FROM amazeeio/python:2.7-ckan-v0.23.1
-
-WORKDIR /app
+FROM amazeeio/python:2.7-ckan-21.6.0
 
 ARG SITE_URL
 ENV SITE_URL="${SITE_URL}"
+ENV VENV_DIR=/app/ckan/default
+ENV APP_DIR=/app
+ENV CKAN_INI=/app/ckan/default/production.ini
+
+WORKDIR "${APP_DIR}"
 
 ENV DOCKERIZE_VERSION v0.6.1
-RUN apk add --no-cache curl && curl -s -L -O https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
+RUN apk add --no-cache curl build-base \
+    && curl -s -L -O https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
     && tar -C /usr/local/bin -xzvf dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
     && rm dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz
 
 # Install CKAN.
-ENV CKAN_VERSION 2.8.3
-RUN . /app/ckan/default/bin/activate \
-    && cd /app/ckan/default \
+ENV CKAN_VERSION 2.8.8
+
+RUN . ${VENV_DIR}/bin/activate \
     && pip install setuptools==36.1 \
     && pip install -e "git+https://github.com/ckan/ckan.git@ckan-${CKAN_VERSION}#egg=ckan" \
-    && sed -i "s/psycopg2==2.4.5/psycopg2==2.7.7/g" "/app/ckan/default/src/ckan/requirements.txt" \
-    && pip install -r "/app/ckan/default/src/ckan/requirements.txt" \
-    && ln -s "/app/ckan/default/src/ckan/who.ini" "/app/ckan/default/who.ini" \
+    && sed -i "s/psycopg2==2.4.5/psycopg2==2.7.7/g" "${VENV_DIR}/src/ckan/requirements.txt" \
+    && pip install -r "${VENV_DIR}/src/ckan/requirements.txt" \
+    && ln -s "${VENV_DIR}/src/ckan/who.ini" "${VENV_DIR}/who.ini" \
     && deactivate \
-    && ln -s /app/ckan /usr/lib/ckan
+    && ln -s ${APP_DIR}/ckan /usr/lib/ckan \
+    && fix-permissions ${APP_DIR}/ckan
 
-COPY .docker/test.ini /app/ckan/default/production.ini
+COPY .docker/test.ini $CKAN_INI
 
-COPY .docker/scripts /app/scripts
+# Add current extension and files.
+COPY . ${APP_DIR}/
 
-RUN fix-permissions /app/ckan \
-    && chmod +x /app/scripts/create-test-data.sh \
-    && chmod +x /app/scripts/init.sh \
-    && chmod +x /app/scripts/init-ext.sh \
-    && chmod +x /app/scripts/serve.sh \
-    && chmod +x /app/scripts/ckan_cli
+COPY .docker/scripts ${APP_DIR}/scripts
 
-# Add current extension and files.
-COPY ckanext /app/ckanext
-COPY requirements.txt requirements-dev.txt setup.cfg setup.py /app/
+COPY .docker/scripts/ckan_cli ${VENV_DIR}/bin/
+
+RUN chmod +x ${APP_DIR}/scripts/*.sh\
+    && chmod +x ${VENV_DIR}/bin/ckan_cli
 
 # Init current extension.
-RUN /app/scripts/init-ext.sh
+RUN ${APP_DIR}/scripts/init-ext.sh
 
 ENTRYPOINT ["/sbin/tini", "--", "/lagoon/entrypoints.sh"]
 CMD ["/app/scripts/serve.sh"]

.docker/scripts/ckan_cli

@@ -57,10 +57,10 @@ else
 fi
 
 if [ "$COMMAND" = "ckan" ]; then
-    echo "Using 'ckan' command from $ENV_DIR..." >&2
+    echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI}..." >&2
     exec $ENV_DIR/ckan -c ${CKAN_INI} "$@"
 elif [ "$COMMAND" = "paster" ]; then
-    echo "Using 'paster' command from $ENV_DIR..." >&2
+    echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI}..." >&2
     exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN "$@" -c ${CKAN_INI}
 else
     echo "Unable to locate 'ckan' or 'paster' command in $ENV_DIR" >&2