[QOL-6424] add Docker and CircleCI tests

.ahoy.yml

@@ -0,0 +1,163 @@
+---
+ahoyapi: v2
+
+commands:
+
+  # Docker commands.
+  build:
+    usage: Build or rebuild project.
+    cmd: |
+      ahoy title "Building project"
+      ahoy pre-flight
+      ahoy clean
+      (docker network prune -f > /dev/null && docker network inspect amazeeio-network > /dev/null || docker network create amazeeio-network)
+      ahoy up -- --build --force-recreate
+      ahoy install-dev
+      ahoy install-site
+      ahoy title "Build complete"
+      ahoy doctor
+      ahoy info 1
+
+  info:
+    usage: Print information about this project.
+    cmd: |
+      ahoy line "Project                  : " ${PROJECT}
+      ahoy line "Site local URL           : " ${LAGOON_LOCALDEV_URL}
+      ahoy line "DB port on host          : " $(docker port $(docker-compose ps -q postgres) 5432 | cut -d : -f 2)
+      ahoy line "Solr port on host        : " $(docker port $(docker-compose ps -q solr) 8983 | cut -d : -f 2)
+      ahoy line "Mailhog URL              : " http://mailhog.docker.amazee.io/
+
+  up:
+    usage: Build and start Docker containers.
+    cmd: |
+      docker-compose up -d "$@"
+      ahoy cli "dockerize -wait tcp://ckan:3000 -timeout 1m"
+      if docker-compose logs | grep -q "\[Error\]"; then docker-compose logs; exit 1; fi
+      if docker-compose logs | grep -q "Exception"; then docker-compose logs; exit 1; fi
+      docker ps -a --filter name=^/${COMPOSE_PROJECT_NAME}_
+      export DOCTOR_CHECK_CLI=0
+
+  down:
+    usage: Stop Docker containers and remove container, images, volumes and networks.
+    cmd: "if [ -f \"docker-compose.yml\" ]; then docker-compose down --volumes; fi"
+
+  start:
+    usage: Start existing Docker containers.
+    cmd: docker-compose start "$@"
+
+  stop:
+    usage: Stop running Docker containers.
+    cmd: docker-compose stop "$@"
+
+  restart:
+    usage: Restart all stopped and running Docker containers.
+    cmd: docker-compose restart
+
+  logs:
+    usage: Show Docker logs.
+    cmd: docker-compose logs "$@"
+
+  pull:
+    usage: Pull latest docker images.
+    cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep amazeeio/ | grep -v none | xargs -n1 docker pull | cat; fi
+
+  cli:
+    usage: Start a shell inside CLI container or run a command.
+    cmd: if \[ "${#}" -ne 0 \]; then docker exec -i $(docker-compose ps -q ckan) sh -c ". /app/ckan/default/bin/activate; $*"; else docker exec -it $(docker-compose ps -q ckan) sh -c ". /app/ckan/default/bin/activate && sh"; fi
+
+  doctor:
+    usage: Find problems with current project setup.
+    cmd: .docker/scripts/doctor.sh "$@"
+
+
+  install-site:
+    usage: Install a site.
+    cmd: |
+      ahoy title "Installing a fresh site"
+      docker cp -L .docker/test.ini $(docker-compose ps -q ckan):/app/ckan/default/production.ini
+      ahoy cli "/app/scripts/init.sh"
+
+  clean:
+    usage: Remove containers and all build files.
+    cmd: |
+      ahoy down
+      # Remove other directories.
+      # @todo: Add destinations below.
+      rm -rf \
+        ./ckan
+
+  reset:
+    usage: "Reset environment: remove containers, all build, manually created and Drupal-Dev files."
+    cmd: |
+      ahoy clean
+      git ls-files --others -i --exclude-from=.git/info/exclude | xargs chmod 777
+      git ls-files --others -i --exclude-from=.git/info/exclude | xargs rm -Rf
+      find . -type d -not -path "./.git/*" -empty -delete
+
+  install-dev:
+    usage: Install dependencies.
+    cmd: |
+      docker cp -L requirements-dev.txt $(docker-compose ps -q ckan):/app/.
+      docker cp -L .flake8 $(docker-compose ps -q ckan):/app/.
+      docker cp -L test $(docker-compose ps -q ckan):/app/.
+      ahoy cli "pip install -r /app/requirements-dev.txt"
+    hide: true
+
+  flush-redis:
+    usage: Flush Redis cache.
+    cmd: docker exec -i $(docker-compose ps -q redis) redis-cli flushall > /dev/null
+
+  lint:
+    usage: Lint code.
+    cmd: |
+      ahoy cli "flake8 ${@:-/app/ckanext}" || \
+      [ "${ALLOW_LINT_FAIL:-0}" -eq 1 ]
+
+  test-unit:
+    usage: Run unit tests.
+    cmd: |
+      ahoy cli "nosetests" || \
+      [ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ]
+
+  test-bdd:
+    usage: Run BDD tests.
+    cmd: |
+      ahoy cli "behave ${*:-/app/test/features}" || \
+      [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
+
+  # Utilities.
+  title:
+    cmd: printf "$(tput -Txterm setaf 4)==> ${1}$(tput -Txterm sgr0)\n"
+    hide: true
+
+  line:
+    cmd: printf "$(tput -Txterm setaf 2)${1}$(tput -Txterm sgr0)${2}\n"
+    hide: true
+
+  getvar:
+    cmd: eval echo "${@}"
+    hide: true
+
+  pre-flight:
+    cmd: |
+      export DOCTOR_CHECK_DB=${DOCTOR_CHECK_DB:-1}
+      export DOCTOR_CHECK_TOOLS=${DOCTOR_CHECK_TOOLS:-1}
+      export DOCTOR_CHECK_PORT=${DOCTOR_CHECK_PORT:-0}
+      export DOCTOR_CHECK_PYGMY=${DOCTOR_CHECK_PYGMY:-1}
+      export DOCTOR_CHECK_CLI=${DOCTOR_CHECK_CLI:-0}
+      export DOCTOR_CHECK_SSH=${DOCTOR_CHECK_SSH:-0}
+      export DOCTOR_CHECK_WEBSERVER=${DOCTOR_CHECK_WEBSERVER:-0}
+      export DOCTOR_CHECK_BOOTSTRAP=${DOCTOR_CHECK_BOOTSTRAP:-0}
+      ahoy doctor
+    hide: true
+
+entrypoint:
+  - bash
+  - "-c"
+  - "-e"
+  - |
+    export LAGOON_LOCALDEV_URL=http://ckanext-qgov.docker.amazee.io
+    [ -f .env ] && [ -s .env ] && export $(grep -v '^#' .env | xargs) && if [ -f .env.local ] && [ -s .env.local ]; then export $(grep -v '^#' .env.local | xargs); fi
+    bash -e -c "$0" "$@"
+  - '{{cmd}}'
+  - '{{name}}'

.circleci/build.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+##
+# Build site in CI.
+#
+set -e
+
+# Process Docker Compose configuration. This is used to avoid multiple
+# docker-compose.yml files.
+# Remove lines containing '###'.
+sed -i -e "/###/d" docker-compose.yml
+# Uncomment lines containing '##'.
+sed -i -e "s/##//" docker-compose.yml
+
+# Pull the latest images.
+ahoy pull
+
+# Disable checks used on host machine.
+export DOCTOR_CHECK_PYGMY=0
+export DOCTOR_CHECK_PORT=0
+export DOCTOR_CHECK_SSH=0
+export DOCTOR_CHECK_WEBSERVER=0
+export DOCTOR_CHECK_BOOTSTRAP=0
+
+ahoy build

.circleci/config.yml

@@ -0,0 +1,40 @@
+version: 2
+aliases:
+
+  # Shared configuration applied to each job.
+  - &container_config
+    working_directory: /app
+    docker:
+      #; Using "runner" container where each job will be executed. This container
+      #; has all necessary tools to run dockerized environment.
+      #; @see https://github.com/integratedexperts/ci-builder
+      - image: integratedexperts/ci-builder
+
+  # Step to setup remote docker.
+  - &step_setup_remote_docker
+      setup_remote_docker
+
+jobs:
+  build:
+    <<: *container_config
+    parallelism: 1
+    steps:
+      - attach_workspace:
+          at: /workspace
+      - checkout
+      - *step_setup_remote_docker
+      - run: .circleci/build.sh
+      - run: .circleci/test.sh
+      - run:
+          name: Process artifacts
+          command: .circleci/process-artifacts.sh
+          when: always
+      - store_artifacts:
+          path: /tmp/artifacts
+          when: always
+
+workflows:
+  version: 2
+  main:
+    jobs:
+      - build

.circleci/process-artifacts.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+##
+# Process test artifacts.
+#
+set -e
+
+# Create screenshots directory in case it was not created before. This is to
+# avoid this script to fail when copying artifacts.
+ahoy cli "mkdir -p /app/test/screenshots"
+
+# Copy from the app container to the build host for storage.
+mkdir -p /tmp/artifacts/behave/screenshots
+docker cp "$(docker-compose ps -q ckan)":/app/test/screenshots /tmp/artifacts/behave

.circleci/test.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+##
+# Run tests in CI.
+#
+set -e
+
+echo "==> Lint code"
+ahoy lint
+
+echo "==> Run Unit tests"
+ahoy test-unit
+
+echo "==> Run BDD tests"
+ahoy test-bdd

.docker/Dockerfile.ckan

@@ -0,0 +1,42 @@
+FROM amazeeio/python:2.7-ckan-v0.23.1
+
+WORKDIR /app
+
+ARG SITE_URL
+ENV SITE_URL="${SITE_URL}"
+
+ENV DOCKERIZE_VERSION v0.6.1
+RUN apk add --no-cache curl && curl -s -L -O http://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
+    && tar -C /usr/local/bin -xzvf dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
+    && rm dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz
+
+# Install CKAN.
+ENV CKAN_VERSION 2.8.3
+RUN . /app/ckan/default/bin/activate \
+    && cd /app/ckan/default \
+    && pip install setuptools==36.1 \
+    && pip install -e "git+https://github.com/ckan/ckan.git@ckan-${CKAN_VERSION}#egg=ckan" \
+    && sed -i "s/psycopg2==2.4.5/psycopg2==2.7.7/g" "/app/ckan/default/src/ckan/requirements.txt" \
+    && pip install -r "/app/ckan/default/src/ckan/requirements.txt" \
+    && ln -s "/app/ckan/default/src/ckan/who.ini" "/app/ckan/default/who.ini" \
+    && deactivate \
+    && ln -s /app/ckan /usr/lib/ckan
+
+COPY .docker/test.ini /app/ckan/default/production.ini
+
+COPY .docker/scripts /app/scripts
+
+RUN fix-permissions /app/ckan \
+    && chmod +x /app/scripts/init.sh \
+    && chmod +x /app/scripts/init-ext.sh \
+    && chmod +x /app/scripts/serve.sh
+
+# Add current extension and files.
+COPY ckanext /app/ckanext
+COPY requirements.txt requirements-dev.txt setup.py /app/
+
+# Init current extension.
+RUN /app/scripts/init-ext.sh
+
+ENTRYPOINT ["/sbin/tini", "--", "/lagoon/entrypoints.sh"]
+CMD ["/app/scripts/serve.sh"]

.docker/Dockerfile.solr

@@ -0,0 +1,3 @@
+FROM amazeeio/solr:6.6-ckan-v0.23.1
+
+# @todo: Support customisations to solr configuration.

.docker/config.json

@@ -0,0 +1,10 @@
+{ "proxies":
+    {
+      "default":
+        {
+          "httpProxy": "http://167.123.1.2:8008",
+          "httpsProxy": "http://167.123.1.2:8008"
+        }
+    }
+}
+

.docker/scripts/create-test-data.sh

@@ -0,0 +1,31 @@
+
+#!/usr/bin/env sh
+##
+# Create some example content for extension BDD tests.
+#
+set -e
+
+CKAN_ACTION_URL=http://ckan:3000/api/action
+CKAN_INI_FILE=/app/ckan/default/production.ini
+
+. /app/ckan/default/bin/activate \
+    && cd /app/ckan/default/src/ckan
+
+# We know the "admin" sysadmin account exists, so we'll use her API KEY to create further data
+API_KEY=$(paster --plugin=ckan user admin -c ${CKAN_INI_FILE} | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/')
+
+# Creating test data hierarchy which creates organisations assigend to datasets
+paster create-test-data hierarchy -c ${CKAN_INI_FILE}
+
+# Creating basic test data which has datasets with resources
+paster create-test-data -c ${CKAN_INI_FILE}
+
+echo "Updating annakarenina to use department-of-health Organisation:"
+package_owner_org_update=$( \
+    curl -L -s --header "Authorization: ${API_KEY}" \
+    --data "id=annakarenina&organization_id=department-of-health" \
+    ${CKAN_ACTION_URL}/package_owner_org_update
+)
+echo ${package_owner_org_update}
+
+deactivate