diff --git a/.ahoy.yml b/.ahoy.yml index aab8def3..5413ff7b 100644 --- a/.ahoy.yml +++ b/.ahoy.yml @@ -10,14 +10,19 @@ commands: ahoy title "Building project" ahoy pre-flight ahoy clean - (docker network prune -f > /dev/null && docker network inspect amazeeio-network > /dev/null || docker network create amazeeio-network) + ahoy build-network ahoy up -- --build --force-recreate - ahoy install-dev ahoy install-site - # ahoy title "Build complete" + ahoy title "Build complete" ahoy doctor ahoy info 1 + build-network: + usage: Ensure that the amazeeio network exists. + cmd: | + docker network prune -f > /dev/null + docker network inspect amazeeio-network > /dev/null || docker network create amazeeio-network + info: usage: Print information about this project. cmd: | @@ -31,6 +36,8 @@ commands: usage: Build and start Docker containers. cmd: | docker-compose up -d "$@" + sleep 10 + docker-compose logs ahoy cli "dockerize -wait tcp://ckan:3000 -timeout 1m" if docker-compose logs | grep -q "\[Error\]"; then docker-compose logs; exit 1; fi if docker-compose logs | grep -q "Exception"; then docker-compose logs; exit 1; fi @@ -39,7 +46,7 @@ commands: down: usage: Stop Docker containers and remove container, images, volumes and networks. - cmd: "if [ -f \"docker-compose.yml\" ]; then docker-compose down --volumes; fi" + cmd: 'if [ -f "docker-compose.yml" ]; then docker-compose down --volumes; fi' start: usage: Start existing Docker containers. @@ -55,7 +62,7 @@ commands: logs: usage: Show Docker logs. - cmd: docker-compose logs -f "$@" + cmd: docker-compose logs "$@" pull: usage: Pull latest docker images. @@ -63,18 +70,17 @@ commands: cli: usage: Start a shell inside CLI container or run a command. - cmd: if \[ "${#}" -ne 0 \]; then docker exec -i $(docker-compose ps -q ckan) sh -c ". /app/ckan/default/bin/activate; $*"; else docker exec -it $(docker-compose ps -q ckan) sh -c ". /app/ckan/default/bin/activate && sh"; fi + cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c '. ${VENV_DIR}/bin/activate; cd $APP_DIR;'" $*"; else docker exec $(docker-compose ps -q ckan) sh -c '. ${VENV_DIR}/bin/activate && cd $APP_DIR && sh'; fi doctor: usage: Find problems with current project setup. cmd: .docker/scripts/doctor.sh "$@" - install-site: usage: Install a site. cmd: | ahoy title "Installing a fresh site" - ahoy cli "/app/scripts/init.sh" + ahoy cli "./scripts/init.sh" clean: usage: Remove containers and all build files. @@ -93,15 +99,6 @@ commands: git ls-files --others -i --exclude-from=.git/info/exclude | xargs rm -Rf find . -type d -not -path "./.git/*" -empty -delete - install-dev: - usage: Install dependencies. - cmd: | - docker cp -L requirements-dev.txt $(docker-compose ps -q ckan):/app/. - docker cp -L .flake8 $(docker-compose ps -q ckan):/app/. - docker cp -L test $(docker-compose ps -q ckan):/app/. - ahoy cli "pip install -r /app/requirements-dev.txt" - hide: true - flush-redis: usage: Flush Redis cache. cmd: docker exec -i $(docker-compose ps -q redis) redis-cli flushall > /dev/null @@ -109,13 +106,13 @@ commands: lint: usage: Lint code. cmd: | - ahoy cli "flake8 ${@:-/app/ckanext}" || \ + ahoy cli "flake8 ${@:-ckanext}" || \ [ "${ALLOW_LINT_FAIL:-0}" -eq 1 ] test-unit: usage: Run unit tests. cmd: | - ahoy cli "nosetests" || \ + ahoy cli 'nosetests --with-pylons=${CKAN_INI}' || \ [ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ] test-bdd: @@ -124,17 +121,16 @@ commands: ahoy start-ckan-job-worker & ahoy start-mailmock & sleep 5 && - ahoy cli "behave ${*:-/app/test/features}" || \ + ahoy cli "behave ${*:-test/features}" || \ [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ] ahoy stop-mailmock ahoy stop-ckan-job-worker - start-mailmock: usage: Starts email mock server used for email BDD tests cmd: | ahoy title 'Starting mailmock' - ahoy cli "cd /app/ckan/default/bin/ && ./mailmock -p 8025 -o /app/test/emails --no-stdout" # for debugging mailmock email output remove --no-stdout + ahoy cli 'mailmock -p 8025 -o ${APP_DIR}/test/emails --no-stdout' # for debugging mailmock email output remove --no-stdout stop-mailmock: usage: Stops email mock server used for email BDD tests @@ -146,10 +142,8 @@ commands: usage: Starts CKAN background job worker cmd: | ahoy title 'Starting CKAN background job worker' - ahoy cli "cd /usr/lib/ckan/default/src/ckan && \ - export CKAN_INI=/app/ckan/default/production.ini && - /app/scripts/ckan_cli jobs clear && \ - /app/scripts/ckan_cli jobs worker default" + ahoy cli "ckan_cli jobs clear && \ + ckan_cli jobs worker" stop-ckan-job-worker: usage: Stops CKAN background job worker @@ -191,5 +185,5 @@ entrypoint: export LAGOON_LOCALDEV_URL=http://ckanext-datarequests.docker.amazee.io [ -f .env ] && [ -s .env ] && export $(grep -v '^#' .env | xargs) && if [ -f .env.local ] && [ -s .env.local ]; then export $(grep -v '^#' .env.local | xargs); fi bash -e -c "$0" "$@" - - '{{cmd}}' - - '{{name}}' + - "{{cmd}}" + - "{{name}}" diff --git a/.circleci/process-artifacts.sh b/.circleci/process-artifacts.sh index 8fbc3d20..55bdbef9 100755 --- a/.circleci/process-artifacts.sh +++ b/.circleci/process-artifacts.sh @@ -6,8 +6,8 @@ set -e # Create screenshots directory in case it was not created before. This is to # avoid this script to fail when copying artifacts. -ahoy cli "mkdir -p /app/test/screenshots" +ahoy cli "mkdir -p test/screenshots" # Copy from the app container to the build host for storage. -mkdir -p /tmp/artifacts/behave/screenshots -docker cp "$(docker-compose ps -q ckan)":/app/test/screenshots /tmp/artifacts/behave +mkdir -p /tmp/artifacts/behave +docker cp "$(docker-compose ps -q ckan)":/app/test/screenshots /tmp/artifacts/behave/ diff --git a/.circleci/test.sh b/.circleci/test.sh index e96dfb22..9c584d9b 100755 --- a/.circleci/test.sh +++ b/.circleci/test.sh @@ -11,4 +11,4 @@ echo "==> Run Unit tests" ahoy test-unit echo "==> Run BDD tests" -ahoy test-bdd +ahoy test-bdd || (ahoy logs; exit 1) diff --git a/.docker/Dockerfile.ckan b/.docker/Dockerfile.ckan index 5324f7e3..789bf9e1 100644 --- a/.docker/Dockerfile.ckan +++ b/.docker/Dockerfile.ckan @@ -1,44 +1,46 @@ -FROM amazeeio/python:2.7-ckan-v0.23.1 - -WORKDIR /app +FROM amazeeio/python:2.7-ckan-21.6.0 ARG SITE_URL ENV SITE_URL="${SITE_URL}" +ENV VENV_DIR=/app/ckan/default +ENV APP_DIR=/app +ENV CKAN_INI=/app/ckan/default/production.ini + +WORKDIR "${APP_DIR}" ENV DOCKERIZE_VERSION v0.6.1 -RUN apk add --no-cache curl && curl -s -L -O https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \ +RUN apk add --no-cache curl build-base \ + && curl -s -L -O https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \ && tar -C /usr/local/bin -xzvf dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \ && rm dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz # Install CKAN. -ENV CKAN_VERSION 2.8.3 -RUN . /app/ckan/default/bin/activate \ - && cd /app/ckan/default \ +ENV CKAN_VERSION 2.8.8 + +RUN . ${VENV_DIR}/bin/activate \ && pip install setuptools==36.1 \ && pip install -e "git+https://github.com/ckan/ckan.git@ckan-${CKAN_VERSION}#egg=ckan" \ - && sed -i "s/psycopg2==2.4.5/psycopg2==2.7.7/g" "/app/ckan/default/src/ckan/requirements.txt" \ - && pip install -r "/app/ckan/default/src/ckan/requirements.txt" \ - && ln -s "/app/ckan/default/src/ckan/who.ini" "/app/ckan/default/who.ini" \ + && sed -i "s/psycopg2==2.4.5/psycopg2==2.7.7/g" "${VENV_DIR}/src/ckan/requirements.txt" \ + && pip install -r "${VENV_DIR}/src/ckan/requirements.txt" \ + && ln -s "${VENV_DIR}/src/ckan/who.ini" "${VENV_DIR}/who.ini" \ && deactivate \ - && ln -s /app/ckan /usr/lib/ckan + && ln -s ${APP_DIR}/ckan /usr/lib/ckan \ + && fix-permissions ${APP_DIR}/ckan -COPY .docker/test.ini /app/ckan/default/production.ini +COPY .docker/test.ini $CKAN_INI -COPY .docker/scripts /app/scripts +# Add current extension and files. +COPY . ${APP_DIR}/ -RUN fix-permissions /app/ckan \ - && chmod +x /app/scripts/create-test-data.sh \ - && chmod +x /app/scripts/init.sh \ - && chmod +x /app/scripts/init-ext.sh \ - && chmod +x /app/scripts/serve.sh \ - && chmod +x /app/scripts/ckan_cli +COPY .docker/scripts ${APP_DIR}/scripts -# Add current extension and files. -COPY ckanext /app/ckanext -COPY requirements.txt requirements-dev.txt setup.cfg setup.py /app/ +COPY .docker/scripts/ckan_cli ${VENV_DIR}/bin/ + +RUN chmod +x ${APP_DIR}/scripts/*.sh\ + && chmod +x ${VENV_DIR}/bin/ckan_cli # Init current extension. -RUN /app/scripts/init-ext.sh +RUN ${APP_DIR}/scripts/init-ext.sh ENTRYPOINT ["/sbin/tini", "--", "/lagoon/entrypoints.sh"] CMD ["/app/scripts/serve.sh"] diff --git a/.docker/Dockerfile.solr b/.docker/Dockerfile.solr deleted file mode 100644 index 2c0ef416..00000000 --- a/.docker/Dockerfile.solr +++ /dev/null @@ -1,3 +0,0 @@ -FROM amazeeio/solr:6.6-ckan-v0.23.1 - -# @todo: Support customisations to solr configuration. diff --git a/.docker/scripts/ckan_cli b/.docker/scripts/ckan_cli index 88526ff9..fcc045fd 100644 --- a/.docker/scripts/ckan_cli +++ b/.docker/scripts/ckan_cli @@ -57,10 +57,10 @@ else fi if [ "$COMMAND" = "ckan" ]; then - echo "Using 'ckan' command from $ENV_DIR..." >&2 + echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI}..." >&2 exec $ENV_DIR/ckan -c ${CKAN_INI} "$@" elif [ "$COMMAND" = "paster" ]; then - echo "Using 'paster' command from $ENV_DIR..." >&2 + echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI}..." >&2 exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN "$@" -c ${CKAN_INI} else echo "Unable to locate 'ckan' or 'paster' command in $ENV_DIR" >&2 diff --git a/.docker/scripts/create-test-data.sh b/.docker/scripts/create-test-data.sh index 504f9ac4..8feb8026 100644 --- a/.docker/scripts/create-test-data.sh +++ b/.docker/scripts/create-test-data.sh @@ -1,4 +1,3 @@ - #!/usr/bin/env sh ## # Create some example content for extension BDD tests. @@ -6,12 +5,29 @@ set -e CKAN_ACTION_URL=http://ckan:3000/api/action -export CKAN_INI=/app/ckan/default/production.ini -. /app/ckan/default/bin/activate +if [ "$VENV_DIR" != "" ]; then + . ${VENV_DIR}/bin/activate +fi + +CKAN_USER_NAME="${CKAN_USER_NAME:-admin}" +CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}" +CKAN_USER_PASSWORD="${CKAN_USER_PASSWORD:-password}" +CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}" + +add_user_if_needed () { + echo "Adding user '$2' ($1) with email address [$3]" + ckan_cli user "$1" | grep "$1" || ckan_cli user add "$1"\ + fullname="$2"\ + email="$3"\ + password="$4" +} + +add_user_if_needed "$CKAN_USER_NAME" "$CKAN_DISPLAY_NAME" "$CKAN_USER_EMAIL" "$CKAN_USER_PASSWORD" +ckan_cli sysadmin add "${CKAN_USER_NAME}" # We know the "admin" sysadmin account exists, so we'll use her API KEY to create further data -API_KEY=$(/app/scripts/ckan_cli user admin | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/') +API_KEY=$(ckan_cli user admin | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/') # # ## @@ -20,7 +36,8 @@ API_KEY=$(/app/scripts/ckan_cli user admin | tr -d '\n' | sed -r 's/^(.*)apikey= # echo "Adding ckan.datarequests.closing_circumstances:" -curl -L -s --header "Authorization: ${API_KEY}" --header "Content-Type: application/json" \ +curl -LsH "Authorization: ${API_KEY}" \ + --header "Content-Type: application/json" \ --data '{"ckan.datarequests.closing_circumstances":"Released as open data|nominate_dataset\nOpen dataset already exists|nominate_dataset\nPartially released|nominate_dataset\nTo be released as open data at a later date|nominate_approximate_date\nData openly available elsewhere\nNot suitable for release as open data\nRequested data not available/cannot be compiled\nRequestor initiated closure"}' \ ${CKAN_ACTION_URL}/config_option_update @@ -36,15 +53,15 @@ TEST_ORG_TITLE="Test" echo "Creating test users for ${TEST_ORG_TITLE} Organisation:" -/app/scripts/ckan_cli user add ckan_user email=ckan_user@localhost password=password -/app/scripts/ckan_cli user add test_org_admin email=test_org_admin@localhost password=password -/app/scripts/ckan_cli user add test_org_editor email=test_org_editor@localhost password=password -/app/scripts/ckan_cli user add test_org_member email=test_org_member@localhost password=password +add_user_if_needed ckan_user "CKAN User" ckan_user@localhost password +add_user_if_needed test_org_admin "Test Admin" test_org_admin@localhost password +add_user_if_needed test_org_editor "Test Editor" test_org_editor@localhost password +add_user_if_needed test_org_member "Test Member" test_org_member@localhost password echo "Creating ${TEST_ORG_TITLE} Organisation:" TEST_ORG=$( \ - curl -L -s --header "Authorization: ${API_KEY}" \ + curl -LsH "Authorization: ${API_KEY}" \ --data "name=${TEST_ORG_NAME}&title=${TEST_ORG_TITLE}" \ ${CKAN_ACTION_URL}/organization_create ) @@ -53,15 +70,15 @@ TEST_ORG_ID=$(echo $TEST_ORG | sed -r 's/^(.*)"id": "(.*)",(.*)/\2/') echo "Assigning test users to ${TEST_ORG_TITLE} Organisation:" -curl -L -s --header "Authorization: ${API_KEY}" \ +curl -LsH "Authorization: ${API_KEY}" \ --data "id=${TEST_ORG_ID}&object=test_org_admin&object_type=user&capacity=admin" \ ${CKAN_ACTION_URL}/member_create -curl -L -s --header "Authorization: ${API_KEY}" \ +curl -LsH "Authorization: ${API_KEY}" \ --data "id=${TEST_ORG_ID}&object=test_org_editor&object_type=user&capacity=editor" \ ${CKAN_ACTION_URL}/member_create -curl -L -s --header "Authorization: ${API_KEY}" \ +curl -LsH "Authorization: ${API_KEY}" \ --data "id=${TEST_ORG_ID}&object=test_org_member&object_type=user&capacity=member" \ ${CKAN_ACTION_URL}/member_create ## @@ -77,14 +94,14 @@ DR_ORG_TITLE="Open Data Administration (data requests)" echo "Creating test users for ${DR_ORG_TITLE} Organisation:" -/app/scripts/ckan_cli user add dr_admin email=dr_admin@localhost password=password -/app/scripts/ckan_cli user add dr_editor email=dr_editor@localhost password=password -/app/scripts/ckan_cli user add dr_member email=dr_member@localhost password=password +add_user_if_needed dr_admin "Data Request Admin" dr_admin@localhost password +add_user_if_needed dr_editor "Data Request Editor" dr_editor@localhost password +add_user_if_needed dr_member "Data Request Member" dr_member@localhost password echo "Creating ${DR_ORG_TITLE} Organisation:" DR_ORG=$( \ - curl -L -s --header "Authorization: ${API_KEY}" \ + curl -LsH "Authorization: ${API_KEY}" \ --data "name=${DR_ORG_NAME}&title=${DR_ORG_TITLE}" \ ${CKAN_ACTION_URL}/organization_create ) @@ -93,30 +110,29 @@ DR_ORG_ID=$(echo $DR_ORG | sed -r 's/^(.*)"id": "(.*)",(.*)/\2/') echo "Assigning test users to ${DR_ORG_TITLE} Organisation:" -curl -L -s --header "Authorization: ${API_KEY}" \ +curl -LsH "Authorization: ${API_KEY}" \ --data "id=${DR_ORG_ID}&object=dr_admin&object_type=user&capacity=admin" \ ${CKAN_ACTION_URL}/member_create -curl -L -s --header "Authorization: ${API_KEY}" \ +curl -LsH "Authorization: ${API_KEY}" \ --data "id=${DR_ORG_ID}&object=dr_editor&object_type=user&capacity=editor" \ ${CKAN_ACTION_URL}/member_create -curl -L -s --header "Authorization: ${API_KEY}" \ +curl -LsH "Authorization: ${API_KEY}" \ --data "id=${DR_ORG_ID}&object=dr_member&object_type=user&capacity=member" \ ${CKAN_ACTION_URL}/member_create echo "Creating test Data Request:" -curl -L -s --header "Authorization: ${API_KEY}" \ +curl -LsH "Authorization: ${API_KEY}" \ --data "title=Test Request&description=This is an example&organization_id=${DR_ORG_ID}" \ ${CKAN_ACTION_URL}/create_datarequest echo "Creating closed Data Request:" Closed_DR=$( \ - curl -L -s \ - --header "Authorization: ${API_KEY}" \ + curl -LsH "Authorization: ${API_KEY}" \ --data "title=Closed Request&description=This is an example&organization_id=${DR_ORG_ID}" \ ${CKAN_ACTION_URL}/create_datarequest \ ) @@ -129,7 +145,7 @@ echo $CLOSE_DR_ID echo "Closing Data Request:" -curl -L -s --header "Authorization: ${API_KEY}" \ +curl -LsH "Authorization: ${API_KEY}" \ --data "id=${CLOSE_DR_ID}&close_circumstance=Requestor initiated closure" \ ${CKAN_ACTION_URL}/close_datarequest @@ -138,21 +154,24 @@ curl -L -s --header "Authorization: ${API_KEY}" \ # # Use CKAN's built-in commands for creating some test datasets... -/app/scripts/ckan_cli create-test-data +ckan_cli create-test-data # Datasets need to be assigned to an organisation echo "Assigning test Datasets to Organisation open-data-administration-data-requests..." -curl -L -s -q --header "Authorization: ${API_KEY}" \ +curl -LsH "Authorization: ${API_KEY}" \ --data "id=annakarenina&owner_org=${DR_ORG_ID}" \ ${CKAN_ACTION_URL}/package_patch >> /dev/null -curl -L -s -q --header "Authorization: ${API_KEY}" \ +curl -LsH "Authorization: ${API_KEY}" \ --data "id=warandpeace&owner_org=${DR_ORG_ID}" \ ${CKAN_ACTION_URL}/package_patch >> /dev/null ## # END. # -deactivate + +if [ "$VENV_DIR" != "" ]; then + deactivate +fi diff --git a/.docker/scripts/init-ext.sh b/.docker/scripts/init-ext.sh index 73ebd4ad..c6b00ec9 100755 --- a/.docker/scripts/init-ext.sh +++ b/.docker/scripts/init-ext.sh @@ -4,14 +4,17 @@ # set -e -. /app/ckan/default/bin/activate - -pip install -r "/app/requirements.txt" -pip install -r "/app/requirements-dev.txt" +if [ "$VENV_DIR" != "" ]; then + . ${VENV_DIR}/bin/activate +fi +pip install -r "requirements-dev.txt" +pip install -r "requirements.txt" python setup.py develop installed_name=$(grep '^\s*name=' setup.py |sed "s|[^']*'\([-a-zA-Z0-9]*\)'.*|\1|") # Validate that the extension was installed correctly. if ! pip list | grep "$installed_name" > /dev/null; then echo "Unable to find the extension in the list"; exit 1; fi -deactivate +if [ "$VENV_DIR" != "" ]; then + deactivate +fi diff --git a/.docker/scripts/init.sh b/.docker/scripts/init.sh index 2dca5136..681f3fe3 100755 --- a/.docker/scripts/init.sh +++ b/.docker/scripts/init.sh @@ -4,28 +4,25 @@ # set -e -CKAN_USER_NAME="${CKAN_USER_NAME:-admin}" -CKAN_USER_PASSWORD="${CKAN_USER_PASSWORD:-password}" -CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}" -export CKAN_INI=/app/ckan/default/production.ini - -. /app/ckan/default/bin/activate -/app/scripts/ckan_cli db clean -/app/scripts/ckan_cli db init -/app/scripts/ckan_cli user add "${CKAN_USER_NAME}" email="${CKAN_USER_EMAIL}" password="${CKAN_USER_PASSWORD}" -/app/scripts/ckan_cli sysadmin add "${CKAN_USER_NAME}" +if [ "$VENV_DIR" != "" ]; then + . ${VENV_DIR}/bin/activate +fi +ckan_cli db clean +ckan_cli db init # Initialise the Comments database tables -PASTER_PLUGIN=ckanext-ytp-comments /app/scripts/ckan_cli initdb +PASTER_PLUGIN=ckanext-ytp-comments ckan_cli initdb +PASTER_PLUGIN=ckanext-ytp-comments ckan_cli updatedb +PASTER_PLUGIN=ckanext-ytp-comments ckan_cli init_notifications_db # Initialise the archiver database tables -PASTER_PLUGIN=ckanext-archiver /app/scripts/ckan_cli archiver init +PASTER_PLUGIN=ckanext-archiver ckan_cli archiver init # Initialise the reporting database tables -PASTER_PLUGIN=ckanext-report /app/scripts/ckan_cli report initdb +PASTER_PLUGIN=ckanext-report ckan_cli report initdb # Initialise the QA database tables -PASTER_PLUGIN=ckanext-qa /app/scripts/ckan_cli qa init +PASTER_PLUGIN=ckanext-qa ckan_cli qa init # Create some base test data -. /app/scripts/create-test-data.sh +. $APP_DIR/scripts/create-test-data.sh diff --git a/.docker/scripts/serve.sh b/.docker/scripts/serve.sh index 6880ed6d..10f3dead 100755 --- a/.docker/scripts/serve.sh +++ b/.docker/scripts/serve.sh @@ -3,8 +3,15 @@ set -e dockerize -wait tcp://postgres:5432 -timeout 1m dockerize -wait tcp://solr:8983 -timeout 1m +dockerize -wait tcp://redis:6379 -timeout 1m -sed -i "s@SITE_URL@${SITE_URL}@g" /app/ckan/default/production.ini +sed -i "s@SITE_URL@${SITE_URL}@g" $CKAN_INI -. /app/ckan/default/bin/activate \ - && paster serve /app/ckan/default/production.ini +if [ "$VENV_DIR" != "" ]; then + . ${VENV_DIR}/bin/activate +fi +if (which ckan > /dev/null); then + ckan -c ${CKAN_INI} run +else + paster serve ${CKAN_INI} +fi diff --git a/.docker/test.ini b/.docker/test.ini index 8535bfd5..3a7332a8 100644 --- a/.docker/test.ini +++ b/.docker/test.ini @@ -64,6 +64,7 @@ ckan.auth.user_delete_organizations = true ckan.auth.create_user_via_api = false ckan.auth.create_user_via_web = true ckan.auth.roles_that_cascade_to_sub_groups = admin +ckan.auth.public_user_details = False ## Search Settings @@ -91,7 +92,7 @@ ckan.redis.url = redis://redis:6379 # Note: Add ``datastore`` to enable the CKAN DataStore # Add ``datapusher`` to enable DataPusher -# Add ``resource_proxy`` to enable resorce proxying and get around the +# Add ``resource_proxy`` to enable resource proxying and get around the # same origin policy # @todo:setup Cleanup the list to use only required plugins. ckan.plugins = stats text_view image_view recline_view datastore data_qld_theme datarequests data_qld_resources data_qld_integration ytp_comments qa archiver report diff --git a/.env b/.env index a209c7f7..8aeda729 100644 --- a/.env +++ b/.env @@ -13,7 +13,7 @@ PROJECT="ckanext-datarequests" # Docker Compose project name. All containers will have this name. -COMPOSE_PROJECT_NAME="ckanext-datarequests" +COMPOSE_PROJECT_NAME="$PROJECT" # Flag to allow code linting failures. ALLOW_LINT_FAIL=1 diff --git a/.flake8 b/.flake8 index 09a146e3..9653307b 100644 --- a/.flake8 +++ b/.flake8 @@ -19,3 +19,4 @@ ignore = E266 E501 C901 + W503 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..89fa60d7 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,40 @@ +--- +#based on https://raw.githubusercontent.com/ckan/ckanext-scheming/master/.github/workflows/test.yml +# alternative https://github.com/ckan/ckan/blob/master/contrib/cookiecutter/ckan_extension/%7B%7Bcookiecutter.project%7D%7D/.github/workflows/test.yml +name: Tests +on: + push: + +jobs: + test: + strategy: + fail-fast: false + + name: Continuous Integration build + runs-on: ubuntu-latest + container: integratedexperts/ci-builder + + steps: + - uses: actions/checkout@v2 + timeout-minutes: 2 + + - name: Build + run: .circleci/build.sh + timeout-minutes: 10 + + - name: Test + run: .circleci/test.sh + timeout-minutes: 15 + + - name: Retrieve screenshots + if: failure() + run: .circleci/process-artifacts.sh + timeout-minutes: 1 + + - name: Upload screenshots + if: failure() + uses: actions/upload-artifact@v2 + with: + name: screenshots + path: /tmp/artifacts/behave/screenshots + timeout-minutes: 1 diff --git a/.gitignore b/.gitignore index 12dfca6f..b834defd 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ nosetests.xml coverage.xml # Translations +*.mo *.pot # Django stuff: @@ -50,3 +51,4 @@ docs/_build/ .env.local /test/screenshots !/test/screenshots/.gitkeep +.idea diff --git a/ckanext/datarequests/actions.py b/ckanext/datarequests/actions.py index bf74d402..bf8c86d2 100644 --- a/ckanext/datarequests/actions.py +++ b/ckanext/datarequests/actions.py @@ -175,7 +175,7 @@ def _send_mail(user_ids, action_type, datarequest): mailer.mail_user(user_data, subject, body) except Exception: - logging.exception("Error sending notification to {0}".format(user_id)) + log.exception("Error sending notification to {0}".format(user_id)) def create_datarequest(context, data_dict): diff --git a/ckanext/datarequests/plugin.py b/ckanext/datarequests/plugin.py index 05232b5e..863f4fcd 100644 --- a/ckanext/datarequests/plugin.py +++ b/ckanext/datarequests/plugin.py @@ -153,70 +153,70 @@ def update_config_schema(self, schema): ############################## IROUTES ############################### ###################################################################### - def before_map(self, m): + def before_map(self, mapper): + from routes.mapper import SubMapper + controller_map = SubMapper( + mapper, controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI') + + m = SubMapper(controller_map, path_prefix="/" + constants.DATAREQUESTS_MAIN_PATH) + # Data Requests index - m.connect('datarequests_index', "/%s" % constants.DATAREQUESTS_MAIN_PATH, - controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', - action='index', conditions={'method': ['GET']}) + m.connect('datarequests_index', '', action='index', conditions={'method': ['GET']}) + m.connect('datarequest.index', '', action='index', conditions={'method': ['GET']}) # Create a Data Request - m.connect('/%s/new' % constants.DATAREQUESTS_MAIN_PATH, - controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', - action='new', conditions={'method': ['GET', 'POST']}) + m.connect('datarequest.new', '/new', action='new', conditions={'method': ['GET', 'POST']}) # Show a Data Request - m.connect('show_datarequest', '/%s/{id}' % constants.DATAREQUESTS_MAIN_PATH, - controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', + m.connect('show_datarequest', '/{id}', + action='show', conditions={'method': ['GET']}, ckan_icon=get_question_icon()) + m.connect('datarequest.show', '/{id}', action='show', conditions={'method': ['GET']}, ckan_icon=get_question_icon()) # Update a Data Request - m.connect('/%s/edit/{id}' % constants.DATAREQUESTS_MAIN_PATH, - controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', + m.connect('datarequest.update', '/edit/{id}', action='update', conditions={'method': ['GET', 'POST']}) # Delete a Data Request - m.connect('/%s/delete/{id}' % constants.DATAREQUESTS_MAIN_PATH, - controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', + m.connect('datarequest.delete', '/delete/{id}', action='delete', conditions={'method': ['POST']}) # Close a Data Request - m.connect('/%s/close/{id}' % constants.DATAREQUESTS_MAIN_PATH, - controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', + m.connect('datarequest.close', '/close/{id}', action='close', conditions={'method': ['GET', 'POST']}) - # Data Request that belongs to an organization - m.connect('organization_datarequests', '/organization/%s/{id}' % constants.DATAREQUESTS_MAIN_PATH, - controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', - action='organization_datarequests', conditions={'method': ['GET']}, - ckan_icon=get_question_icon()) - - # Data Request that belongs to an user - m.connect('user_datarequests', '/user/%s/{id}' % constants.DATAREQUESTS_MAIN_PATH, - controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', - action='user_datarequests', conditions={'method': ['GET']}, - ckan_icon=get_question_icon()) - # Follow & Unfollow - m.connect('/%s/follow/{id}' % constants.DATAREQUESTS_MAIN_PATH, - controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', + m.connect('datarequest.follow', '/follow/{id}', action='follow', conditions={'method': ['POST']}) - m.connect('/%s/unfollow/{id}' % constants.DATAREQUESTS_MAIN_PATH, - controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', + m.connect('datarequest.unfollow', '/unfollow/{id}', action='unfollow', conditions={'method': ['POST']}) if self.comments_enabled: # Comment, update and view comments (of) a Data Request - m.connect('comment_datarequest', '/%s/comment/{id}' % constants.DATAREQUESTS_MAIN_PATH, - controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', + m.connect('comment_datarequest', '/comment/{id}', + action='comment', conditions={'method': ['GET', 'POST']}, ckan_icon='comment') + m.connect('datarequest.comment', '/comment/{id}', action='comment', conditions={'method': ['GET', 'POST']}, ckan_icon='comment') # Delete data request - m.connect('/%s/comment/{datarequest_id}/delete/{comment_id}' % constants.DATAREQUESTS_MAIN_PATH, - controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', + m.connect('datarequest.delete_comment', '/comment/{datarequest_id}/delete/{comment_id}', action='delete_comment', conditions={'method': ['GET', 'POST']}) - return m + list_datarequests_map = SubMapper( + controller_map, conditions={'method': ['GET']}, ckan_icon=get_question_icon()) + + # Data Requests that belong to an organization + list_datarequests_map.connect( + 'organization_datarequests', '/organization/%s/{id}' % constants.DATAREQUESTS_MAIN_PATH, + action='organization_datarequests') + + # Data Requests that belong to a user + list_datarequests_map.connect( + 'user_datarequests', '/user/%s/{id}' % constants.DATAREQUESTS_MAIN_PATH, + action='user_datarequests') + + return mapper ###################################################################### ######################### ITEMPLATESHELPER ########################### diff --git a/ckanext/datarequests/templates/header.html b/ckanext/datarequests/templates/header.html index 73494271..1319faf2 100644 --- a/ckanext/datarequests/templates/header.html +++ b/ckanext/datarequests/templates/header.html @@ -5,7 +5,7 @@ ('search', _('Datasets')), ('organizations_index', _('Organizations')), ('group_index', _('Groups')), - ('datarequests_index', _('Data Requests') + h.get_open_datarequests_badge()), + ('datarequest.index', _('Data Requests') + h.get_open_datarequests_badge()), ('about', _('About')) ) }} {% endblock %} diff --git a/ckanext/datarequests/tests/test_plugin.py b/ckanext/datarequests/tests/test_plugin.py index 3fc67088..a3f4b78b 100644 --- a/ckanext/datarequests/tests/test_plugin.py +++ b/ckanext/datarequests/tests/test_plugin.py @@ -195,8 +195,8 @@ def test_update_config(self): ]) def test_before_map(self, comments_enabled): - urls_set = 12 - mapa_calls = urls_set if comments_enabled == 'True' else urls_set - 2 + urls_set = 15 + mapa_calls = urls_set if comments_enabled == 'True' else urls_set - 3 # Configure config and get instance plugin.config.get.return_value = comments_enabled @@ -219,7 +219,12 @@ def test_before_map(self, comments_enabled): action='index', conditions={'method': ['GET']}) mapa.connect.assert_any_call( - '/%s/new' % dr_basic_path, + 'datarequest.index', "/%s" % dr_basic_path, + controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', + action='index', conditions={'method': ['GET']}) + + mapa.connect.assert_any_call( + 'datarequest.new', '/%s/new' % dr_basic_path, controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', action='new', conditions={'method': ['GET', 'POST']}) @@ -229,17 +234,22 @@ def test_before_map(self, comments_enabled): action='show', conditions={'method': ['GET']}, ckan_icon=mock_icon) mapa.connect.assert_any_call( - '/%s/edit/{id}' % dr_basic_path, + 'datarequest.show', '/%s/{id}' % dr_basic_path, + controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', + action='show', conditions={'method': ['GET']}, ckan_icon=mock_icon) + + mapa.connect.assert_any_call( + 'datarequest.update', '/%s/edit/{id}' % dr_basic_path, controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', action='update', conditions={'method': ['GET', 'POST']}) mapa.connect.assert_any_call( - '/%s/delete/{id}' % dr_basic_path, + 'datarequest.delete', '/%s/delete/{id}' % dr_basic_path, controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', action='delete', conditions={'method': ['POST']}) mapa.connect.assert_any_call( - '/%s/close/{id}' % dr_basic_path, + 'datarequest.close', '/%s/close/{id}' % dr_basic_path, controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', action='close', conditions={'method': ['GET', 'POST']}) @@ -265,12 +275,12 @@ def test_before_map(self, comments_enabled): ckan_icon=mock_icon) mapa.connect.assert_any_call( - '/%s/follow/{id}' % dr_basic_path, + 'datarequest.follow', '/%s/follow/{id}' % dr_basic_path, controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', action='follow', conditions={'method': ['POST']}) mapa.connect.assert_any_call( - '/%s/unfollow/{id}' % dr_basic_path, + 'datarequest.unfollow', '/%s/unfollow/{id}' % dr_basic_path, controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', action='unfollow', conditions={'method': ['POST']}) @@ -281,7 +291,12 @@ def test_before_map(self, comments_enabled): action='comment', conditions={'method': ['GET', 'POST']}, ckan_icon='comment') mapa.connect.assert_any_call( - '/%s/comment/{datarequest_id}/delete/{comment_id}' % dr_basic_path, + 'datarequest.comment', '/%s/comment/{id}' % dr_basic_path, + controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', + action='comment', conditions={'method': ['GET', 'POST']}, ckan_icon='comment') + + mapa.connect.assert_any_call( + 'datarequest.delete_comment', '/%s/comment/{datarequest_id}/delete/{comment_id}' % dr_basic_path, controller='ckanext.datarequests.controllers.ui_controller:DataRequestsUI', action='delete_comment', conditions={'method': ['GET', 'POST']}) diff --git a/docker-compose.yml b/docker-compose.yml index 44824d17..15417d94 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '2.3' x-project: - &project ckanext-datarequests + &project "${PROJECT}" x-volumes: &default-volumes @@ -29,7 +29,7 @@ services: context: . dockerfile: .docker/Dockerfile.ckan args: - SITE_URL: http://ckanext-datarequests.docker.amazee.io + SITE_URL: "http://${PROJECT}.docker.amazee.io" depends_on: - postgres - solr @@ -43,8 +43,8 @@ services: environment: <<: *default-environment AMAZEEIO_HTTP_PORT: 3000 - LAGOON_LOCALDEV_URL: http://ckanext-datarequests.docker.amazee.io - AMAZEEIO_URL: ckanext-datarequests.docker.amazee.io + LAGOON_LOCALDEV_URL: "http://${PROJECT}.docker.amazee.io" + AMAZEEIO_URL: "${PROJECT}.docker.amazee.io" postgres: image: amazeeio/postgres-ckan @@ -69,7 +69,7 @@ services: <<: *default-environment redis: - image: amazeeio/redis + image: redis:6-alpine <<: *default-user environment: <<: *default-environment @@ -78,9 +78,7 @@ services: - default solr: - build: - context: . - dockerfile: .docker/Dockerfile.solr + image: ckan/ckan-solr-dev:2.8 user: '8983' ports: - "8983" diff --git a/requirements-dev.txt b/requirements-dev.txt index c6544865..1d2cec9c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,7 @@ beautifulsoup4==4.9.1 behave==1.2.6 -behaving==1.5.6 +behaving==2.0.0 +Appium-Python-Client<=0.52 flake8==3.8.3 nose==1.3.7 mock diff --git a/test/features/datarequest.feature b/test/features/datarequest.feature index 89dfb7fe..b2fedf9b 100644 --- a/test/features/datarequest.feature +++ b/test/features/datarequest.feature @@ -31,7 +31,7 @@ Feature: Datarequest And I should see "Description cannot be empty" within 1 seconds - Scenario Outline: Sysadmin or Admin users of the assigned organisation for a data request can see a "Re-open" button on the data request detail page for closed data requests + Scenario Outline: Sysadmin or Admin users of the assigned organisation for a data request can see a 'Re-open' button on the data request detail page for closed data requests Given "" as the persona When I log in and go to datarequest page And I press "Closed Request" @@ -43,7 +43,7 @@ Feature: Datarequest | DataRequestOrgAdmin | - Scenario Outline: Non-admin users should not see "Re-open" button on the data request detail page for closed data requests + Scenario Outline: Non-admin users should not see 'Re-open' button on the data request detail page for closed data requests Given "" as the persona When I log in and go to datarequest page And I press "Closed Request" @@ -59,7 +59,7 @@ Feature: Datarequest | TestOrgMember | - Scenario Outline: Data request creator, Sysadmin and Admin users of the assigned organisation for a data request can see a "Close" button on the data request detail page for opened data requests + Scenario Outline: Data request creator, Sysadmin and Admin users of the assigned organisation for a data request can see a 'Close' button on the data request detail page for opened data requests Given "" as the persona When I log in and go to datarequest page And I press "Test Request" @@ -71,7 +71,7 @@ Feature: Datarequest | DataRequestOrgAdmin | - Scenario Outline: Non admin users cannot not see a "Close" button on the data request detail page for opened data requests + Scenario Outline: Non admin users cannot see a 'Close' button on the data request detail page for opened data requests Given "" as the persona When I log in and go to datarequest page And I press "Test Request" @@ -91,9 +91,7 @@ Feature: Datarequest Given "TestOrgEditor" as the persona When I log in and create a datarequest When I wait for 3 seconds - Then I should receive an email at "dr_admin@localhost" with subject "Queensland Government Open Data - Data Request" - And I should receive a base64 email at "dr_admin@localhost" containing "A new data request has been added and assigned to your organisation." - And I should receive an email at "admin@localhost" with subject "Queensland Government Open Data - Data Request" + Then I should receive a base64 email at "dr_admin@localhost" containing "A new data request has been added and assigned to your organisation." And I should receive a base64 email at "admin@localhost" containing "A new data request has been added and assigned to your organisation." @@ -104,8 +102,7 @@ Feature: Datarequest And I select "Requestor initiated closure" from "close_circumstance" And I press the element with xpath "//button[contains(string(), 'Close data request')]" When I wait for 3 seconds - Then I should receive an email at "dr_admin@localhost" with subject "Queensland Government Open Data - Data Request" - And I should receive a base64 email at "dr_admin@localhost" containing "Your data request has been closed." + Then I should receive a base64 email at "dr_admin@localhost" containing "Your data request has been closed." Scenario: Re-Opening a data request will email the Admin users of the organisation and creator @@ -116,9 +113,7 @@ Feature: Datarequest And I press the element with xpath "//button[contains(string(), 'Close data request')]" And I press the element with xpath "//a[@class='btn btn-success' and contains(string(), ' Re-open')]" When I wait for 3 seconds - Then I should receive an email at "dr_admin@localhost" with subject "Queensland Government Open Data - Data Request" - And I should receive a base64 email at "dr_admin@localhost" containing "Your data request has been re-opened." - And I should receive an email at "admin@localhost" with subject "Queensland Government Open Data - Data Request" + Then I should receive a base64 email at "dr_admin@localhost" containing "Your data request has been re-opened." And I should receive a base64 email at "admin@localhost" containing "A data request assigned to your organisation has been re-opened." @@ -131,7 +126,5 @@ Feature: Datarequest Then I execute the script "document.getElementById('field-organizations').value = document.getElementById('field-organizations').options[1].value" And I press the element with xpath "//button[contains(string(), 'Update data request')]" When I wait for 3 seconds - Then I should receive an email at "admin@localhost" with subject "Queensland Government Open Data - Data Request" - And I should receive a base64 email at "admin@localhost" containing "A data request that was assigned to your organisation has been re-assigned to another organisation." - And I should receive an email at "test_org_admin@localhost" with subject "Queensland Government Open Data - Data Request" - And I should receive a base64 email at "test_org_admin@localhost" containing "A new data request has been added and assigned to your organisation." \ No newline at end of file + Then I should receive a base64 email at "admin@localhost" containing "A data request that was assigned to your organisation has been re-assigned to another organisation." + And I should receive a base64 email at "test_org_admin@localhost" containing "A new data request has been added and assigned to your organisation." diff --git a/test/features/environment.py b/test/features/environment.py index 7ffa0e80..0a66e80d 100644 --- a/test/features/environment.py +++ b/test/features/environment.py @@ -19,7 +19,7 @@ 'email': u'admin@localhost', 'password': u'password' }, - 'Unathenticated': { + 'Unauthenticated': { 'name': u'', 'email': u'', 'password': u'' @@ -70,6 +70,7 @@ def before_all(context): context.attachment_dir = os.path.join(ROOT_PATH, 'test/fixtures') # The path where emails can be found. context.mail_path = os.path.join(ROOT_PATH, 'test/emails') + # Set base url for all relative links. context.base_url = BASE_URL