diff --git a/.ahoy.yml b/.ahoy.yml new file mode 100644 index 00000000..fa9e64e4 --- /dev/null +++ b/.ahoy.yml @@ -0,0 +1,163 @@ +--- +ahoyapi: v2 + +commands: + + # Docker commands. + build: + usage: Build or rebuild project. + cmd: | + ahoy title "Building project" + ahoy pre-flight + ahoy clean + (docker network prune -f > /dev/null && docker network inspect amazeeio-network > /dev/null || docker network create amazeeio-network) + ahoy up -- --build --force-recreate + ahoy install-dev + ahoy install-site + ahoy title "Build complete" + ahoy doctor + ahoy info 1 + + info: + usage: Print information about this project. + cmd: | + ahoy line "Project : " ${PROJECT} + ahoy line "Site local URL : " ${LAGOON_LOCALDEV_URL} + ahoy line "DB port on host : " $(docker port $(docker-compose ps -q postgres) 5432 | cut -d : -f 2) + ahoy line "Solr port on host : " $(docker port $(docker-compose ps -q solr) 8983 | cut -d : -f 2) + ahoy line "Mailhog URL : " http://mailhog.docker.amazee.io/ + + up: + usage: Build and start Docker containers. + cmd: | + docker-compose up -d "$@" + ahoy cli "dockerize -wait tcp://ckan:3000 -timeout 1m" + if docker-compose logs | grep -q "\[Error\]"; then docker-compose logs; exit 1; fi + if docker-compose logs | grep -q "Exception"; then docker-compose logs; exit 1; fi + docker ps -a --filter name=^/${COMPOSE_PROJECT_NAME}_ + export DOCTOR_CHECK_CLI=0 + + down: + usage: Stop Docker containers and remove container, images, volumes and networks. + cmd: "if [ -f \"docker-compose.yml\" ]; then docker-compose down --volumes; fi" + + start: + usage: Start existing Docker containers. + cmd: docker-compose start "$@" + + stop: + usage: Stop running Docker containers. + cmd: docker-compose stop "$@" + + restart: + usage: Restart all stopped and running Docker containers. + cmd: docker-compose restart + + logs: + usage: Show Docker logs. + cmd: docker-compose logs "$@" + + pull: + usage: Pull latest docker images. + cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep amazeeio/ | grep -v none | xargs -n1 docker pull | cat; fi + + cli: + usage: Start a shell inside CLI container or run a command. + cmd: if \[ "${#}" -ne 0 \]; then docker exec -i $(docker-compose ps -q ckan) sh -c ". /app/ckan/default/bin/activate; $*"; else docker exec -it $(docker-compose ps -q ckan) sh -c ". /app/ckan/default/bin/activate && sh"; fi + + doctor: + usage: Find problems with current project setup. + cmd: .docker/scripts/doctor.sh "$@" + + + install-site: + usage: Install a site. + cmd: | + ahoy title "Installing a fresh site" + docker cp -L .docker/test.ini $(docker-compose ps -q ckan):/app/ckan/default/production.ini + ahoy cli "/app/scripts/init.sh" + + clean: + usage: Remove containers and all build files. + cmd: | + ahoy down + # Remove other directories. + # @todo: Add destinations below. + rm -rf \ + ./ckan + + reset: + usage: "Reset environment: remove containers, all build, manually created and Drupal-Dev files." + cmd: | + ahoy clean + git ls-files --others -i --exclude-from=.git/info/exclude | xargs chmod 777 + git ls-files --others -i --exclude-from=.git/info/exclude | xargs rm -Rf + find . -type d -not -path "./.git/*" -empty -delete + + install-dev: + usage: Install dependencies. + cmd: | + docker cp -L requirements-dev.txt $(docker-compose ps -q ckan):/app/. + docker cp -L .flake8 $(docker-compose ps -q ckan):/app/. + docker cp -L test $(docker-compose ps -q ckan):/app/. + ahoy cli "pip install -r /app/requirements-dev.txt" + hide: true + + flush-redis: + usage: Flush Redis cache. + cmd: docker exec -i $(docker-compose ps -q redis) redis-cli flushall > /dev/null + + lint: + usage: Lint code. + cmd: | + ahoy cli "flake8 ${@:-/app/ckanext}" || \ + [ "${ALLOW_LINT_FAIL:-0}" -eq 1 ] + + test-unit: + usage: Run unit tests. + cmd: | + ahoy cli "nosetests" || \ + [ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ] + + test-bdd: + usage: Run BDD tests. + cmd: | + ahoy cli "behave ${*:-/app/test/features}" || \ + [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ] + + # Utilities. + title: + cmd: printf "$(tput -Txterm setaf 4)==> ${1}$(tput -Txterm sgr0)\n" + hide: true + + line: + cmd: printf "$(tput -Txterm setaf 2)${1}$(tput -Txterm sgr0)${2}\n" + hide: true + + getvar: + cmd: eval echo "${@}" + hide: true + + pre-flight: + cmd: | + export DOCTOR_CHECK_DB=${DOCTOR_CHECK_DB:-1} + export DOCTOR_CHECK_TOOLS=${DOCTOR_CHECK_TOOLS:-1} + export DOCTOR_CHECK_PORT=${DOCTOR_CHECK_PORT:-0} + export DOCTOR_CHECK_PYGMY=${DOCTOR_CHECK_PYGMY:-1} + export DOCTOR_CHECK_CLI=${DOCTOR_CHECK_CLI:-0} + export DOCTOR_CHECK_SSH=${DOCTOR_CHECK_SSH:-0} + export DOCTOR_CHECK_WEBSERVER=${DOCTOR_CHECK_WEBSERVER:-0} + export DOCTOR_CHECK_BOOTSTRAP=${DOCTOR_CHECK_BOOTSTRAP:-0} + ahoy doctor + hide: true + +entrypoint: + - bash + - "-c" + - "-e" + - | + export LAGOON_LOCALDEV_URL=http://ckanext-qgov.docker.amazee.io + [ -f .env ] && [ -s .env ] && export $(grep -v '^#' .env | xargs) && if [ -f .env.local ] && [ -s .env.local ]; then export $(grep -v '^#' .env.local | xargs); fi + bash -e -c "$0" "$@" + - '{{cmd}}' + - '{{name}}' diff --git a/.circleci/build.sh b/.circleci/build.sh new file mode 100755 index 00000000..611dca97 --- /dev/null +++ b/.circleci/build.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +## +# Build site in CI. +# +set -e + +# Process Docker Compose configuration. This is used to avoid multiple +# docker-compose.yml files. +# Remove lines containing '###'. +sed -i -e "/###/d" docker-compose.yml +# Uncomment lines containing '##'. +sed -i -e "s/##//" docker-compose.yml + +# Pull the latest images. +ahoy pull + +# Disable checks used on host machine. +export DOCTOR_CHECK_PYGMY=0 +export DOCTOR_CHECK_PORT=0 +export DOCTOR_CHECK_SSH=0 +export DOCTOR_CHECK_WEBSERVER=0 +export DOCTOR_CHECK_BOOTSTRAP=0 + +ahoy build diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..b70b58fc --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,40 @@ +version: 2 +aliases: + + # Shared configuration applied to each job. + - &container_config + working_directory: /app + docker: + #; Using "runner" container where each job will be executed. This container + #; has all necessary tools to run dockerized environment. + #; @see https://github.com/integratedexperts/ci-builder + - image: integratedexperts/ci-builder + + # Step to setup remote docker. + - &step_setup_remote_docker + setup_remote_docker + +jobs: + build: + <<: *container_config + parallelism: 1 + steps: + - attach_workspace: + at: /workspace + - checkout + - *step_setup_remote_docker + - run: .circleci/build.sh + - run: .circleci/test.sh + - run: + name: Process artifacts + command: .circleci/process-artifacts.sh + when: always + - store_artifacts: + path: /tmp/artifacts + when: always + +workflows: + version: 2 + main: + jobs: + - build diff --git a/.circleci/process-artifacts.sh b/.circleci/process-artifacts.sh new file mode 100755 index 00000000..8fbc3d20 --- /dev/null +++ b/.circleci/process-artifacts.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +## +# Process test artifacts. +# +set -e + +# Create screenshots directory in case it was not created before. This is to +# avoid this script to fail when copying artifacts. +ahoy cli "mkdir -p /app/test/screenshots" + +# Copy from the app container to the build host for storage. +mkdir -p /tmp/artifacts/behave/screenshots +docker cp "$(docker-compose ps -q ckan)":/app/test/screenshots /tmp/artifacts/behave diff --git a/.circleci/test.sh b/.circleci/test.sh new file mode 100755 index 00000000..e96dfb22 --- /dev/null +++ b/.circleci/test.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +## +# Run tests in CI. +# +set -e + +echo "==> Lint code" +ahoy lint + +echo "==> Run Unit tests" +ahoy test-unit + +echo "==> Run BDD tests" +ahoy test-bdd diff --git a/.docker/Dockerfile.ckan b/.docker/Dockerfile.ckan new file mode 100644 index 00000000..c22142cd --- /dev/null +++ b/.docker/Dockerfile.ckan @@ -0,0 +1,42 @@ +FROM amazeeio/python:2.7-ckan-v0.23.1 + +WORKDIR /app + +ARG SITE_URL +ENV SITE_URL="${SITE_URL}" + +ENV DOCKERIZE_VERSION v0.6.1 +RUN apk add --no-cache curl && curl -s -L -O http://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \ + && tar -C /usr/local/bin -xzvf dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \ + && rm dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz + +# Install CKAN. +ENV CKAN_VERSION 2.8.3 +RUN . /app/ckan/default/bin/activate \ + && cd /app/ckan/default \ + && pip install setuptools==36.1 \ + && pip install -e "git+https://github.com/ckan/ckan.git@ckan-${CKAN_VERSION}#egg=ckan" \ + && sed -i "s/psycopg2==2.4.5/psycopg2==2.7.7/g" "/app/ckan/default/src/ckan/requirements.txt" \ + && pip install -r "/app/ckan/default/src/ckan/requirements.txt" \ + && ln -s "/app/ckan/default/src/ckan/who.ini" "/app/ckan/default/who.ini" \ + && deactivate \ + && ln -s /app/ckan /usr/lib/ckan + +COPY .docker/test.ini /app/ckan/default/production.ini + +COPY .docker/scripts /app/scripts + +RUN fix-permissions /app/ckan \ + && chmod +x /app/scripts/init.sh \ + && chmod +x /app/scripts/init-ext.sh \ + && chmod +x /app/scripts/serve.sh + +# Add current extension and files. +COPY ckanext /app/ckanext +COPY requirements.txt requirements-dev.txt setup.py /app/ + +# Init current extension. +RUN /app/scripts/init-ext.sh + +ENTRYPOINT ["/sbin/tini", "--", "/lagoon/entrypoints.sh"] +CMD ["/app/scripts/serve.sh"] diff --git a/.docker/Dockerfile.solr b/.docker/Dockerfile.solr new file mode 100644 index 00000000..2c0ef416 --- /dev/null +++ b/.docker/Dockerfile.solr @@ -0,0 +1,3 @@ +FROM amazeeio/solr:6.6-ckan-v0.23.1 + +# @todo: Support customisations to solr configuration. diff --git a/.docker/config.json b/.docker/config.json new file mode 100644 index 00000000..5f76493a --- /dev/null +++ b/.docker/config.json @@ -0,0 +1,10 @@ +{ "proxies": + { + "default": + { + "httpProxy": "http://167.123.1.2:8008", + "httpsProxy": "http://167.123.1.2:8008" + } + } +} + diff --git a/.docker/scripts/create-test-data.sh b/.docker/scripts/create-test-data.sh new file mode 100644 index 00000000..95339e87 --- /dev/null +++ b/.docker/scripts/create-test-data.sh @@ -0,0 +1,31 @@ + +#!/usr/bin/env sh +## +# Create some example content for extension BDD tests. +# +set -e + +CKAN_ACTION_URL=http://ckan:3000/api/action +CKAN_INI_FILE=/app/ckan/default/production.ini + +. /app/ckan/default/bin/activate \ + && cd /app/ckan/default/src/ckan + +# We know the "admin" sysadmin account exists, so we'll use her API KEY to create further data +API_KEY=$(paster --plugin=ckan user admin -c ${CKAN_INI_FILE} | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/') + +# Creating test data hierarchy which creates organisations assigend to datasets +paster create-test-data hierarchy -c ${CKAN_INI_FILE} + +# Creating basic test data which has datasets with resources +paster create-test-data -c ${CKAN_INI_FILE} + +echo "Updating annakarenina to use department-of-health Organisation:" +package_owner_org_update=$( \ + curl -L -s --header "Authorization: ${API_KEY}" \ + --data "id=annakarenina&organization_id=department-of-health" \ + ${CKAN_ACTION_URL}/package_owner_org_update +) +echo ${package_owner_org_update} + +deactivate diff --git a/.docker/scripts/doctor.sh b/.docker/scripts/doctor.sh new file mode 100755 index 00000000..5bcf065c --- /dev/null +++ b/.docker/scripts/doctor.sh @@ -0,0 +1,202 @@ +#!/usr/bin/env bash +# +# Check Drupal-Dev project requirements. +# +set -e + +DOCTOR_CHECK_TOOLS="${DOCTOR_CHECK_TOOLS:-1}" +DOCTOR_CHECK_PORT="${DOCTOR_CHECK_PORT:-0}" +DOCTOR_CHECK_PYGMY="${DOCTOR_CHECK_PYGMY:-1}" +DOCTOR_CHECK_CLI="${DOCTOR_CHECK_CLI:-1}" +DOCTOR_CHECK_SSH="${DOCTOR_CHECK_SSH:-0}" +DOCTOR_CHECK_WEBSERVER="${DOCTOR_CHECK_WEBSERVER:-1}" +DOCTOR_CHECK_BOOTSTRAP="${DOCTOR_CHECK_BOOTSTRAP:-1}" + +APP_PORT="${APP_PORT:-80}" +CLI="${CLI:-cli}" +LAGOON_LOCALDEV_URL="${LAGOON_LOCALDEV_URL:-http://your-site.docker.amazee.io/}" +SSH_KEY_FILE="${SSH_KEY_FILE:-$HOME/.ssh/id_rsa}" +DATAROOT="${DATAROOT:-.data}" + +#------------------------------------------------------------------------------- +# DO NOT CHANGE ANYTHING BELOW THIS LINE +#------------------------------------------------------------------------------- + + +# +# Main entry point. +# +main() { + status "Checking project requirements" + + if [ "${DOCTOR_CHECK_TOOLS}" == "1" ]; then + [ "$(command_exists docker)" == "1" ] && error "Please install Docker (https://www.docker.com/get-started)" && exit 1 + [ "$(command_exists docker-compose)" == "1" ] && error "Please install docker-compose (https://docs.docker.com/compose/install/)" && exit 1 + [ "$(command_exists composer)" == "1" ] && error "Please install Composer (https://getcomposer.org/)" && exit 1 + [ "$(command_exists pygmy)" == "1" ] && error "Please install Pygmy (https://pygmy.readthedocs.io/)" && exit 1 + [ "$(command_exists ahoy)" == "1" ] && error "Please install Ahoy (https://ahoy-cli.readthedocs.io/)" && exit 1 + success "All required tools are present" + fi + + if [ "${DOCTOR_CHECK_PORT}" == "1" ]; then + if ! lsof -i :3000 | grep LISTEN | grep -q om.docke; then + error "Port 3000 is occupied by a service other than Docker. Stop this service and run 'pygmy up'" + fi + success "Port 3000 is available" + fi + + if [ "${DOCTOR_CHECK_PYGMY}" == "1" ]; then + if ! pygmy status > /dev/null 2>&1; then + error "pygmy is not running. Run 'pygmy up' to start pygmy." + exit 1 + fi + success "Pygmy is running" + fi + + # Check that the stack is running. + if [ "${DOCTOR_CHECK_CLI}" == "1" ]; then + if ! docker ps -q --no-trunc | grep "$(docker-compose ps -q ckan)" > /dev/null 2>&1; then + error "CLI container is not running. Run 'ahoy up'." + exit 1 + fi + success "CLI container is running" + fi + + if [ "${DOCTOR_CHECK_SSH}" == "1" ]; then + # SSH key injection is required to access Lagoon services from within + # containers. For example, to connect to production environment to run + # drush script. + # Pygmy makes this possible in the following way: + # 1. Pygmy starts `amazeeio/ssh-agent` container with a volume `/tmp/amazeeio_ssh-agent` + # 2. Pygmy adds a default SSH key from the host into this volume. + # 3. `docker-compose.yml` should have volume inclusion specified for CLI container: + # ``` + # volumes_from: + # - container:amazeeio-ssh-agent + # ``` + # 4. When CLI container starts, the volume is mounted and an entrypoint script + # adds SHH key into agent. + # @see https://github.com/amazeeio/lagoon/blob/master/images/php/cli/10-ssh-agent.sh + # + # Running `ssh-add -L` within CLI container should show that the SSH key + # is correctly loaded. + # + # As rule of a thumb, one must restart the CLI container after restarting + # Pygmy ONLY if SSH key was not loaded in pygmy before the stack starts. + # No need to restart CLI container if key was added, but pygmy was + # restarted - the volume mount will retain and the key will still be + # available in CLI container. + + # Check that the key is injected into pygmy ssh-agent container. + if ! pygmy status 2>&1 | grep -q "${SSH_KEY_FILE}"; then + error "SSH key is not added to pygmy. Run 'pygmy stop && pygmy start' and then 'ahoy up -- --build'." + exit 1 + fi + + # Check that the volume is mounted into CLI container. + if ! docker exec -i "$(docker-compose ps -q ckan)" sh -c "grep \"^/dev\" /etc/mtab|grep -q /tmp/amazeeio_ssh-agent"; then + error "SSH key is added to Pygmy, but the volume is not mounted into container. Make sure that your your \"docker-compose.yml\" has the following lines:" + error "volumes_from:" + error " - container:amazeeio-ssh-agent" + error "After adding these lines, run 'ahoy up -- --build'" + exit 1 + fi + + # Check that ssh key is available in the container. + if ! docker exec -i "$(docker-compose ps -q ckan)" bash -c "ssh-add -L | grep -q 'ssh-rsa'" ; then + error "SSH key was not added into container. Run 'ahoy up -- --build'." + exit 1 + fi + + success "SSH key is available within CLI container" + fi + + + if [ "${DOCTOR_CHECK_WEBSERVER}" == "1" ]; then + host_app_port="$(docker port $(docker-compose ps -q ckan) 3000 | cut -d : -f 2)" + if ! curl -L -s -o /dev/null -w "%{http_code}" "${LAGOON_LOCALDEV_URL}:${host_app_port}" | grep -q 200; then + error "Web server is not accessible at ${LAGOON_LOCALDEV_URL}:${host_app_port}" + exit 1 + fi + success "Web server is running and accessible at ${LAGOON_LOCALDEV_URL}:${host_app_port}" + fi + + if [ "${DOCTOR_CHECK_BOOTSTRAP}" == "1" ]; then + host_app_port="$(docker port $(docker-compose ps -q ckan) 3000 | cut -d : -f 2)" + if ! curl -L -s -N "${LAGOON_LOCALDEV_URL}:${host_app_port}" | grep -q -i "meta name=\"generator\" content=\"ckan"; then + error "Website is running, but cannot be bootstrapped. Try pulling latest container images with 'ahoy pull'" + exit 1 + fi + success "Successfully bootstrapped website at ${LAGOON_LOCALDEV_URL}:${host_app_port}" + fi + + status "All required checks have passed" +} + +# +# Check that command exists. +# +command_exists() { + local cmd=$1 + command -v "${cmd}" | grep -ohq "${cmd}" + local res=$? + + # Try homebrew lookup, if brew is available. + if command -v "brew" | grep -ohq "brew" && [ "$res" == "1" ] ; then + brew --prefix "${cmd}" > /dev/null + res=$? + fi + + echo ${res} +} + +# +# Status echo. +# +status() { + cecho blue "✚ $1"; +} + +# +# Success echo. +# +success() { + cecho green " ✓ $1"; +} + +# +# Error echo. +# +error() { + cecho red " ✘ $1"; + exit 1 +} + +# +# Colored echo. +# +cecho() { + local prefix="\033[" + local input_color=$1 + local message="$2" + + local color="" + case "$input_color" in + black | bk) color="${prefix}0;30m";; + red | r) color="${prefix}1;31m";; + green | g) color="${prefix}1;32m";; + yellow | y) color="${prefix}1;33m";; + blue | b) color="${prefix}1;34m";; + purple | p) color="${prefix}1;35m";; + cyan | c) color="${prefix}1;36m";; + gray | gr) color="${prefix}0;37m";; + *) message="$1" + esac + + # Format message with color codes, but only if a correct color was provided. + [ -n "$color" ] && message="${color}${message}${prefix}0m" + + echo -e "$message" +} + +main "$@" diff --git a/.docker/scripts/init-ext.sh b/.docker/scripts/init-ext.sh new file mode 100755 index 00000000..94fc3c7e --- /dev/null +++ b/.docker/scripts/init-ext.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh +## +# Install current extension. +# +set -e + +. /app/ckan/default/bin/activate + +pip install -r "/app/requirements.txt" +pip install -r "/app/requirements-dev.txt" +python setup.py develop + +# Validate that the extension was installed correctly. +if ! pip list | grep ckanext-qgov > /dev/null; then echo "Unable to find the extension in the list"; exit 1; fi + +deactivate diff --git a/.docker/scripts/init.sh b/.docker/scripts/init.sh new file mode 100755 index 00000000..09a98419 --- /dev/null +++ b/.docker/scripts/init.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env sh +## +# Initialise CKAN instance. +# +set -e + +CKAN_USER_NAME="${CKAN_USER_NAME:-admin}" +CKAN_USER_PASSWORD="${CKAN_USER_PASSWORD:-Password123!}" +CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}" + +. /app/ckan/default/bin/activate \ + && cd /app/ckan/default/src/ckan \ + && paster db clean -c /app/ckan/default/production.ini \ + && paster db init -c /app/ckan/default/production.ini \ + && paster --plugin=ckan user add "${CKAN_USER_NAME}" email="${CKAN_USER_EMAIL}" password="${CKAN_USER_PASSWORD}" -c /app/ckan/default/production.ini \ + && paster --plugin=ckan sysadmin add "${CKAN_USER_NAME}" -c /app/ckan/default/production.ini + +# Create some base test data +. /app/scripts/create-test-data.sh \ No newline at end of file diff --git a/.docker/scripts/serve.sh b/.docker/scripts/serve.sh new file mode 100755 index 00000000..6880ed6d --- /dev/null +++ b/.docker/scripts/serve.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env sh +set -e + +dockerize -wait tcp://postgres:5432 -timeout 1m +dockerize -wait tcp://solr:8983 -timeout 1m + +sed -i "s@SITE_URL@${SITE_URL}@g" /app/ckan/default/production.ini + +. /app/ckan/default/bin/activate \ + && paster serve /app/ckan/default/production.ini diff --git a/.docker/test.ini b/.docker/test.ini new file mode 100644 index 00000000..2b7f75dd --- /dev/null +++ b/.docker/test.ini @@ -0,0 +1,205 @@ +# +# CKAN - Pylons configuration +# +# These are some of the configuration options available for your CKAN +# instance. Check the documentation in 'doc/configuration.rst' or at the +# following URL for a description of what they do and the full list of +# available options: +# +# http://docs.ckan.org/en/latest/maintaining/configuration.html +# +# The %(here)s variable will be replaced with the parent directory of this file +# + +[DEFAULT] +debug = false +smtp_server = localhost +error_email_from = paste@localhost + +[server:main] +use = egg:Paste#http +host = 0.0.0.0 +port = 3000 + +[app:main] +use = egg:ckan +full_stack = true +cache_dir = /tmp/%(ckan.site_id)s/ + +# This is the secret token that the beaker library uses to hash the cookie sent +# to the client. `paster make-config` generates a unique value for this each +# time it generates a config file. +beaker.session.secret = bSmgPpaxg2M+ZRes3u1TXwIcE + +# `paster make-config` generates a unique value for this each time it generates +# a config file. +app_instance_uuid = 6e3daf8e-1c6b-443b-911f-c7ab4c5f9605 + +who.config_file = %(here)s/who.ini +who.log_level = warning +who.log_file = %(cache_dir)s/who_log.ini + +## Database Settings +sqlalchemy.url = postgresql://ckan:ckan@postgres/ckan?sslmode=disable + +ckan.datastore.write_url = postgresql://ckan:ckan@postgres-datastore/ckan?sslmode=disable +ckan.datastore.read_url = postgresql://ckan_datastore:ckan@postgres-datastore/ckan?sslmode=disable + +# PostgreSQL' full-text search parameters +ckan.datastore.default_fts_lang = english +ckan.datastore.default_fts_index_method = gist + +## Site Settings. +ckan.site_url = http://ckan:3000/ + +## Authorization Settings + +ckan.auth.anon_create_dataset = false +ckan.auth.create_unowned_dataset = false +ckan.auth.create_dataset_if_not_in_organization = false +ckan.auth.user_create_groups = false +ckan.auth.user_create_organizations = false +ckan.auth.user_delete_groups = true +ckan.auth.user_delete_organizations = true +ckan.auth.create_user_via_api = false +ckan.auth.create_user_via_web = true +ckan.auth.roles_that_cascade_to_sub_groups = admin + + +## Search Settings + +ckan.site_id = default +solr_url = http://solr:8983/solr/ckan + + +## Redis Settings + +# URL to your Redis instance, including the database to be used. +ckan.redis.url = redis://redis:6379 + + +## CORS Settings + +# If cors.origin_allow_all is true, all origins are allowed. +# If false, the cors.origin_whitelist is used. +# ckan.cors.origin_allow_all = true +# cors.origin_whitelist is a space separated list of allowed domains. +# ckan.cors.origin_whitelist = http://example1.com http://example2.com + + +## Plugins Settings + +# Note: Add ``datastore`` to enable the CKAN DataStore +# Add ``datapusher`` to enable DataPusher +# Add ``resource_proxy`` to enable resorce proxying and get around the +# same origin policy +# @todo:setup Cleanup the list to use only required plugins. +ckan.plugins = stats text_view image_view recline_view qgovext + +# Define which views should be created by default +# (plugins must be loaded in ckan.plugins) +ckan.views.default_views = image_view text_view recline_view + +# Customize which text formats the text_view plugin will show +#ckan.preview.json_formats = json +#ckan.preview.xml_formats = xml rdf rdf+xml owl+xml atom rss +#ckan.preview.text_formats = text plain text/plain + +# Customize which image formats the image_view plugin will show +#ckan.preview.image_formats = png jpeg jpg gif + +## Front-End Settings +ckan.favicon = https://www.qld.gov.au/favicon.ico + +## Internationalisation Settings +ckan.locale_default = en_AU +ckan.locale_order = en pt_BR ja it cs_CZ ca es fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv +ckan.locales_offered = +ckan.locales_filtered_out = en_AU + +## Feeds Settings + +ckan.feeds.authority_name = +ckan.feeds.date = +ckan.feeds.author_name = +ckan.feeds.author_link = + +## Storage Settings + +ckan.storage_path = /app/filestore +#ckan.max_resource_size = 10 +#ckan.max_image_size = 2 + +## Datapusher settings + +# Make sure you have set up the DataStore + +#ckan.datapusher.formats = csv xls xlsx tsv application/csv application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet +#ckan.datapusher.url = http://127.0.0.1:8800/ +#ckan.datapusher.assume_task_stale_after = 3600 + +# Resource Proxy settings +# Preview size limit, default: 1MB +#ckan.resource_proxy.max_file_size = 1048576 +# Size of chunks to read/write. +#ckan.resource_proxy.chunk_size = 4096 + +## Activity Streams Settings + +#ckan.activity_streams_enabled = true +#ckan.activity_list_limit = 31 +#ckan.activity_streams_email_notifications = true +#ckan.email_notifications_since = 2 days +ckan.hide_activity_from_users = %(ckan.site_id)s + + +## Email settings + +#email_to = errors@example.com +#error_email_from = ckan-errors@example.com +#smtp.server = localhost +#smtp.starttls = False +#smtp.user = username@example.com +#smtp.password = your_password +#smtp.mail_from = + +## Harvester settings +ckan.harvest.mq.type = redis +ckan.harvest.mq.hostname = redis +ckan.harvest.mq.port = 6379 +ckan.harvest.mq.redis_db = 0 + +## Logging configuration +[loggers] +keys = root, ckan, ckanext + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console + +[logger_ckan] +level = INFO +handlers = console +qualname = ckan +propagate = 0 + +[logger_ckanext] +level = DEBUG +handlers = console +qualname = ckanext +propagate = 0 + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s diff --git a/.env b/.env new file mode 100644 index 00000000..a3380acf --- /dev/null +++ b/.env @@ -0,0 +1,25 @@ +## +# Project environment variables. +# +# It is used by Ahoy and other scripts to read default values. +# +# The values must be scalar (cannot be another variable). +# +# You may also create .env.local file to override any values locally (it is +# excluded from git). +# + +# Project name. +PROJECT="ckanext-qgov" + +# Docker Compose project name. All containers will have this name. +COMPOSE_PROJECT_NAME="ckanext-qgov" + +# Flag to allow code linting failures. +ALLOW_LINT_FAIL=0 + +# Flag to allow unit tests failures. +ALLOW_UNIT_FAIL=0 + +# Flag to allow BDD tests failures. +ALLOW_BDD_FAIL=0 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..aee1b377 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,8 @@ +# Ignore files for distribution archives. +/.ahoy.yml export-ignore +/.circleci export-ignore +/.docker export-ignore +/.env export-ignore +/.gitatributes export-ignore +/docker-compose.yml export-ignore +/test export-ignore diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..b1e6bac9 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,100 @@ +version: '2.3' + +x-project: + &project ckanext-qgov + +x-volumes: + &default-volumes + volumes: + - /app/ckan ### Local overrides to mount host filesystem. Automatically removed in CI and PROD. + - ./ckanext:/app/ckanext:${VOLUME_FLAGS:-delegated} ### Local overrides to mount host filesystem. Automatically removed in CI and PROD. + - ./test:/app/test:${VOLUME_FLAGS:-delegated} ### Local overrides to mount host filesystem. Automatically removed in CI and PROD. + ##- /app/filestore # Override for environment without host mounts. Automatically uncommented in CI. + +x-environment: + &default-environment + AMAZEEIO: AMAZEEIO + no_proxy: "ckan,postgres,postgres-datastore,redis,solr,chrome,mailhog.docker.amazee.io" + +x-user: + &default-user + # The default user under which the containers should run. + # Change this if you are on linux and run with another user than id `1000`. + user: '1000' + +services: + + ckan: + build: + context: . + dockerfile: .docker/Dockerfile.ckan + args: + SITE_URL: http://ckanext-qgov.docker.amazee.io + depends_on: + - postgres + - solr + networks: + - amazeeio-network + - default + ports: + - "3000" + image: *project + <<: *default-volumes + environment: + <<: *default-environment + AMAZEEIO_HTTP_PORT: 3000 + LAGOON_LOCALDEV_URL: http://ckanext-qgov.docker.amazee.io + AMAZEEIO_URL: ckanext-qgov.docker.amazee.io + + postgres: + image: amazeeio/postgres-ckan + ports: + - "5432" + networks: + - amazeeio-network + - default + <<: *default-user + environment: + <<: *default-environment + + redis: + image: amazeeio/redis + <<: *default-user + environment: + <<: *default-environment + networks: + - amazeeio-network + - default + + solr: + build: + context: . + dockerfile: .docker/Dockerfile.solr + user: '8983' + ports: + - "8983" + environment: + <<: *default-environment + networks: + - amazeeio-network + - default + + chrome: + image: selenium/standalone-chrome:3.141.59-oxygen + shm_size: '1gb' + depends_on: + - ckan + <<: *default-volumes + <<: *default-user + environment: + <<: *default-environment + networks: + - amazeeio-network + - default + +volumes: + solr-data: {} + +networks: + amazeeio-network: + external: true diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..762a1311 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,6 @@ +behave==1.2.6 +behaving==1.5.6 +flake8==3.7.9 +nose==1.3.7 +splinter>=0.13.0 +-e git+https://github.com/ckan/ckanapi@ckanapi-4.3#egg=ckanapi diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..e69de29b diff --git a/test/features/environment.py b/test/features/environment.py new file mode 100644 index 00000000..9a3d0ee2 --- /dev/null +++ b/test/features/environment.py @@ -0,0 +1,64 @@ +import os +from behaving import environment as benv + +from behaving.web.steps.browser import named_browser + +# Path to the root of the project. +ROOT_PATH = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../')) + +# Base URL for relative paths resolution. +BASE_URL = 'http://ckan:3000/' + +# URL of remote Chrome instance. +REMOTE_CHROME_URL = 'http://chrome:4444/wd/hub' + +# @see .docker/scripts/init.sh for credentials. +PERSONAS = { + 'Admin': dict( + name=u'admin', + email=u'admin@localhost', + password=u'Password123!' + ) +} + + +def before_all(context): + # The path where screenshots will be saved. + context.screenshots_dir = os.path.join(ROOT_PATH, 'test/screenshots') + # The path where file attachments can be found. + context.attachment_dir = os.path.join(ROOT_PATH, 'test/fixtures') + + # Set base url for all relative links. + context.base_url = BASE_URL + + # Always use remote web driver. + context.remote_webdriver = 1 + context.default_browser = 'chrome' + context.browser_args = {'command_executor': REMOTE_CHROME_URL} + + # Set the rest of the settings to default Behaving's settings. + benv.before_all(context) + + +def after_all(context): + benv.after_all(context) + + +def before_feature(context, feature): + benv.before_feature(context, feature) + + +def after_feature(context, feature): + benv.after_feature(context, feature) + + +def before_scenario(context, scenario): + benv.before_scenario(context, scenario) + # Always use remote browser. + named_browser(context, 'remote') + # Set personas. + context.personas = PERSONAS + + +def after_scenario(context, scenario): + benv.after_scenario(context, scenario) diff --git a/test/features/homepage.feature b/test/features/homepage.feature new file mode 100644 index 00000000..964e3890 --- /dev/null +++ b/test/features/homepage.feature @@ -0,0 +1,7 @@ +@smoke +Feature: Homepage + + @homepage + Scenario: Smoke test to ensure Homepage is accessible + When I go to homepage + Then I take a screenshot \ No newline at end of file diff --git a/test/features/login.feature b/test/features/login.feature new file mode 100644 index 00000000..4e450114 --- /dev/null +++ b/test/features/login.feature @@ -0,0 +1,18 @@ +@smoke @login +Feature: Login + + Scenario: Smoke test to ensure Login process works + Given "Admin" as the persona + When I go to homepage + And I click the link with text that contains "Log in" + And I fill in "login" with "$name" + And I fill in "password" with "$password" + # Login is a button without "name" or "id". + And I press the element with xpath "//button[contains(string(), 'Login')]" + And I take a screenshot + Then I should see an element with xpath "//a[contains(string(), 'Log out')]" + + Scenario: Smoke test to ensure Login step works + Given "Admin" as the persona + When I log in + Then I take a screenshot \ No newline at end of file diff --git a/test/features/steps/steps.py b/test/features/steps/steps.py new file mode 100644 index 00000000..1070e327 --- /dev/null +++ b/test/features/steps/steps.py @@ -0,0 +1,38 @@ +from behave import step +from behaving.web.steps import * # noqa: F401, F403 +from behaving.personas.steps import * # noqa: F401, F403 +from behaving.web.steps.url import when_i_visit_url + + +@step('I go to homepage') +def go_to_home(context): + when_i_visit_url(context, '/') + + +@step('I log in') +def log_in(context): + + assert context.persona + context.execute_steps(u""" + When I go to homepage + And I click the link with text that contains "Log in" + And I fill in "login" with "$name" + And I fill in "password" with "$password" + And I press the element with xpath "//button[contains(string(), 'Login')]" + Then I should see an element with xpath "//a[contains(string(), 'Log out')]" + """) + + +@step('I go to dataset page') +def go_to_dataset_page(context): + when_i_visit_url(context, '/dataset') + + +@step('I go to organisation page') +def go_to_organisation_page(context): + when_i_visit_url(context, '/organization') + + +@step('I go to register page') +def go_to_register_page(context): + when_i_visit_url(context, '/user/register')