Merge branch 'test' into 'stable'

Merge test to stable: v2.2.3-rc1

See merge request fairdata/fairdata-metax!10

.gitlab-ci.yml

@@ -0,0 +1,70 @@
+workflow:
+  rules:
+    - if: '$CI_COMMIT_BRANCH =~ /^(demo|stable|staging|test)$/'
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+
+stages:
+  - deploy
+  - test
+  - clean_test
+  - clean_env
+  - update
+  - clean_build
+
+deploy:
+  stage: deploy
+  environment: $CI_COMMIT_REF_NAME
+  script:
+    - ansible-playbook -i $ANSIBLE_INVENTORY $DEPLOY_PLAYBOOK -e "build_id=$CI_COMMIT_SHORT_SHA repo_version=$CI_COMMIT_REF_NAME"
+
+integration_test:
+  stage: test
+  environment: $CI_COMMIT_REF_NAME
+  script:
+    - ansible-playbook -i $ANSIBLE_INVENTORY $TEST_PLAYBOOK -e "build_id=$CI_COMMIT_SHORT_SHA"
+
+clean_test:
+  stage: clean_test
+  environment:
+    name: $CI_COMMIT_REF_NAME
+    on_stop: clean_gitlab_env
+  script:
+    - ansible-playbook -i $ANSIBLE_INVENTORY $DELETE_PLAYBOOK -e "build_id=$CI_COMMIT_SHORT_SHA"
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: always
+    - when: on_failure
+
+clean_gitlab_env:
+  stage: clean_env
+  variables:
+    GIT_STRATEGY: none
+  environment:
+    name: $CI_COMMIT_REF_NAME
+    action: stop
+  script:
+    - echo "Cleaning deleted branches from environments"
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: manual
+    - when: never
+
+update_proxy:
+  stage: update
+  environment: $CI_COMMIT_REF_NAME
+  script:
+    - ansible-playbook -i $ANSIBLE_INVENTORY $UPDATE_PROXY_PLAYBOOK -e "build_id=$CI_COMMIT_SHORT_SHA"
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: never
+    - when: always
+
+clean_previous_build:
+  stage: clean_build
+  environment: $CI_COMMIT_REF_NAME
+  script:
+    - ansible-playbook -i $ANSIBLE_INVENTORY $DELETE_PLAYBOOK -e "build_id=${CI_COMMIT_BEFORE_SHA:0:8}"
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: never
+    - when: always

.travis.yml

@@ -52,8 +52,8 @@ install:
 
 before_script:
 - psql -U postgres < resources/sql/init_test.sql
-- sudo mkdir -p src/log
-- sudo chown -R $USER:$USER src/log
+- sudo mkdir -p /var/log/metax-api/errors
+- sudo chown -R $USER:$USER /var/log/metax-api
 
 script:
 - cd src && python manage.py migrate metax_api

Dockerfile

@@ -1,8 +1,10 @@
-FROM python:3.6
+FROM python:3.8
 
 ENV PYTHONUNBUFFERED 1
 ENV PYTHONDONTWRITEBYTECODE 1
 
+RUN mkdir -p /var/log/metax-api/errors && touch /var/log/metax-api/metax-api.json.log
+
 WORKDIR /code
 
 COPY requirements.txt /code/
@@ -15,19 +17,4 @@ RUN pip install -r requirements.txt
 EXPOSE 8008
 EXPOSE 8006
 
-ARG METAX_DATABASE_HOST
-ARG REDIS_HOST
-ARG RABBITMQ_HOST
-ARG RABBIT_MQ_PASSWORD=guest
-ARG RABBIT_MQ_USER=guest
-ARG ELASTIC_SEARCH_HOST
-
-ENV METAX_DATABASE_HOST $METAX_DATABASE_HOST
-ENV REDIS_HOST $REDIS_HOST
-ENV RABBIT_MQ_HOSTS $RABBITMQ_HOST
-ENV RABBIT_MQ_PASSWORD $RABBIT_MQ_PASSWORD
-ENV RABBIT_MQ_USER $RABBIT_MQ_USER
-ENV ELASTIC_SEARCH_HOSTS $ELASTIC_SEARCH_HOST
-
-# CMD ["python", "/code/manage.py", "runserver", "0.0.0.0:8008"]
-CMD ["python", "manage.py", "runsslserver", "--certificate", ".certs/cert.pem","--key", ".certs/key.pem", "0.0.0.0:8008"]
+CMD ["python", "manage.py", "runserver", "0.0.0.0:8008"]

README.md

@@ -1,138 +1,53 @@
 This repository contains the code for Metax API service.
 
-# Build status
+## License
 
-## Test branch
-[![Build Status](https://travis-ci.com/CSCfi/metax-api.svg?branch=test)](https://travis-ci.com/CSCfi/metax-api)
-
-## Stable branch
-[![Build Status](https://travis-ci.com/CSCfi/metax-api.svg?branch=stable)](https://travis-ci.com/CSCfi/metax-api)
-
-License
--------
 Copyright (c) 2018-2020 Ministry of Education and Culture, Finland
 
 Licensed under [GNU GPLv2 License](LICENSE)
 
 
 ## Setting up local development environment
 
-### Prerequisites
-
-#### Docker-Engine
-
-Install Docker-Engine either following instructions below or looking up your platform specific instructions [from docs.docker.com][1] 
-
-##### Linux
-
-`$ curl -fsSL https://get.docker.com -o get-docker.sh`
-
-`$ sudo sh get-docker.sh`
-
-`$ sudo usermod -aG docker $USER`
-
-Log out and back in to activate non-sudo docker capabilities
-
-##### Mac
-
-https://docs.docker.com/docker-for-mac/install/
-
-#### Portainer (Optional)
-
-We will use portainer container management tool for monitoring various development dependencies. Command below will start portainer on every system startup.
-
-`$ docker volume create portainer_data` (optional for mac)
-
-`$ docker run -d -p 8000:8000 -p 9000:9000 --name=portainer --restart=always -v /var/run/docker.sock:/var/run/docker.sock -v portainer_data:/data portainer/portainer-ce`
-
-Finish the Portainer setup by logging in at http://localhost:9000, create a local endpoint from the Portainer interface. 
-
-#### Docker commands
-
-__NOTICE If you want to start the services everytime your computer boots, replace `--restart=unless-stopped` with `--restart=always`__
-
-Run the following docker commands to start services:
-
-##### Redis
-
-`docker run -d -p 6379:6379 --name metax-redis -v metax-redis:/data --restart=unless-stopped redis`
-
-##### Postgres
-
-`docker run -d -p 5432:5432 --name metax-postgres -v metax-postgres:/var/lib/postgresql96/data -e POSTGRES_USER=metax_db_user -e POSTGRES_PASSWORD=YMDLekQMqrVKcs3 -e POSTGRES_DB=metax_db --restart=unless-stopped  postgres:9`
-
-__NOTICE: copy values of `POSTGRES_USER`, `POSTGRES_PASSWORD` and `POSTGRES_DB` into your `.env` files as `METAX_DATABASE_USER`, `METAX_DATABASE_PASSWORD` and `METAX_DATABASE`__
-
-##### Elasticsearch
-
-`docker run -d -p 9200:9200 -p 9300:9300 -v metax-es:/usr/share/elasticsearch/data --name metax-es -e discovery.type=single-node --restart=unless-stopped elasticsearch:7.9.2`
-
-##### RabbitMQ
-
-`docker run -d -p 5671:5671 -p 5672:5672 -p 15672:15672 -v metax-rabbitmq:/var/lib/rabbitmq --name metax-rabbitmq --restart=unless-stopped rabbitmq:3-management`
-
-#### mkcerts
-
-Install [mkcerts][2] and run `mkcert -install` and after it the following command:
-`mkcert -cert-file cert.pem -key-file key.pem 0.0.0.0 localhost 127.0.0.1 ::1 metax.csc.local 20.20.20.20`
-Move the `cert.pem` and `key.pem` to `src/.certs` folder (create the folder if not present).
+You can also set up the development environment with [Docker-swarm setup](/docs/docker-stack.md) or with [standalone Docker-containers setup](/docs/single-docker-images.md).
 
 ### Required environmental variables
 
 copy `src/metax_api/settings/.env.template` as `src/metax_api/settings/.env` and fill required variables, you can find examples in ENV_VARS.md
 
 ### Create log directory 
 
-`mkdir -p src/log/errors`
-
-### Run Metax inside a container (Optional)
-
-Check the IP addresses of Redis, RabbitMQ, ElasticSearch and Postgres:9 either from Portainer container list (click the link in the container name to see all attributes) or by going to portainer network tab or by typing `docker container ps` followed by `docker network inspect bridge`
-
-Build new docker image from repository root with this command (change ip-addresses to real ones:
-
-`docker build -t metax-api:latest --build-arg METAX_DATABASE_HOST=xxx.xx.x.x --build-arg REDIS_HOST=xxx.xx.x.x --build-arg RABBITMQ_HOST=xxx.xx.x.x --build-arg ELASTIC_SEARCH_HOST=xxx.xx.x.x:xxxx .`
-
-Run the built container with command:
-
-`docker run -it --name metax-web --mount type=bind,source="$(pwd)"/src,target=/code -p 8008:8008 --rm metax-api:latest`
-
-You should see metax-server starting at port 8008 with hot reload enabled
+`mkdir -p /var/log/metax-api/errors`
 
 ### Initial setup commands
-docker network inspect bridge
-IF you configured metax-container, access the command line of the container with `docker exec -it metax-web bash`
 
-__NOTICE: Skip activating virtualenv and navigating to src folder if you have metax running on container__
+Activate your python 3.8 virtualenv, install requirements with `pip install -r requirements.txt`
 
-Activate your python 3.6 virtualenv, `cd` into `src` folder and run following commands:
-
-setup the database with migrate command:
+`cd` into `src` folder and run following commands:
 
 `python manage.py migrate`
 
-__NOTICE: Skip following steps if your running metax on container and have terminal open in the container__
-
 start the development server with:
-`python manage.py runsslserver --certificate .certs/cert.pem --key .certs/key.pem 8008`
 
-Open another terminal and `cd` into `src`, and load the initial data with following commands: 
+`python manage.py runserver 8008`
 
-__These commands must be run in both setups (container/not-container metax)__
+Open another terminal and `cd` into `src`, and load the initial data with following commands:
 
 `python manage.py index_refdata`
 
 `python manage.py reload_refdata_cache`
 
 `python manage.py loadinitialdata`
 
-`python manage.py loaddata metax_api/tests/testdata/test_data.json` 
+`python manage.py loaddata metax_api/tests/testdata/test_data.json`
+
+Metax api is available from your browser at http://localhost:8008
+
+## Running tests
+
+run the tests with command `DJANGO_ENV=test python manage.py test --parallel --failfast --keepdb -v 0`
 
-run the tests with command `DJANGO_ENV=test python manage.py test --failfast --keepdb -v 0`
 
-Metax api is available from your browser at https://localhost:8008
 
 
-[1]: https://docs.docker.com/engine/install/
-[2]: https://github.com/FiloSottile/mkcert
 

config-swap-stack.yml

@@ -0,0 +1,55 @@
+version: "3.8"
+
+services:
+  metax:
+    image: fairdata-docker.artifactory.ci.csc.fi/fairdata-metax-web
+    ports:
+      - 8008:8008
+      - 8000:8000
+    volumes:
+      - ./src:/code
+    environment:
+      DEBUG: 'true'
+      METAX_DATABASE: 'metax_db_test'
+      METAX_DATABASE_PASSWORD: 'YMDLekQMqrVKcs3'
+      METAX_DATABASE_USER: 'metax_test'
+      REDIS_HOST: metax-redis
+      RABBIT_MQ_HOSTS: metax-rabbitmq
+      ELASTIC_SEARCH_HOSTS: metax-elasticsearch
+      METAX_DATABASE_HOST: metax-db
+
+  metax-redis:
+    image: redis
+    volumes:
+      - metax-redis:/data
+  metax-db:
+    image: postgres:9
+    environment:
+      POSTGRES_USER: 'metax_test'
+      POSTGRES_PASSWORD: 'YMDLekQMqrVKcs3'
+      POSTGRES_DB: 'metax_db_test'
+    volumes:
+      - metax-postgres:/var/lib/postgresql/data
+  metax-elasticsearch:
+    image: elasticsearch:7.9.2
+    environment:
+      discovery.type: 'single-node'
+    volumes:
+      - metax-es:/usr/share/elasticsearch/data
+  metax-rabbitmq:
+    image: rabbitmq:3-management
+    volumes:
+      - metax-rabbitmq:/var/lib/rabbitmq
+
+volumes:
+  metax-rabbitmq:
+    external: true
+  metax-es:
+    external: true
+  metax-postgres:
+    external: true
+  metax-redis:
+    external: true
+
+
+

containers/nginx-docker.yml

@@ -0,0 +1,42 @@
+version: "3.8"
+
+services:
+  metax-nginx:
+    image: nginx:latest
+    configs:
+      - source: metax-nginx-config
+        target: '/etc/nginx/nginx.conf'
+      - source: fairdata-ssl-certificate
+        target: '/etc/nginx/ssl_certs/fd-dev.csc.fi.crt.pem'
+      - source: fairdata-ssl-certificate-key
+        target: '/etc/nginx/ssl_certs/fd-dev.csc.fi.key.pem'
+      - source: metax-nginx-elastic-headers-config
+        target: '/etc/nginx/elastic_headers.conf'
+      - source: metax-nginx-shared-headers-config
+        target: '/etc/nginx/shared_headers.conf'
+      - source: metax-nginx-api-response-headers-config
+        target: '/etc/nginx/api_response_headers.conf'
+      - source: metax-nginx-static-file-headers-config
+        target: '/etc/nginx/static_file_headers.conf'
+      - source: metax-nginx-dh-param-config
+        target: '/etc/nginx/ssl_certs/nginx_dhparam.pem'
+    ports:
+      - 443:443
+
+configs:
+  fairdata-ssl-certificate:
+    external: True
+  fairdata-ssl-certificate-key:
+    external: True
+  metax-nginx-config:
+    external: True
+  metax-nginx-elastic-headers-config:
+    external: True
+  metax-nginx-shared-headers-config:
+    external: True
+  metax-nginx-api-response-headers-config:
+    external: True
+  metax-nginx-static-file-headers-config:
+    external: True
+  metax-nginx-dh-param-config:
+    external: True