Merge branch 'refactor_ci' into 'test'

CSCFAIRADM-1271: Refactor CI/CD

See merge request fairdata/fairdata-metax!171

.gitlab-ci.yml

@@ -4,37 +4,55 @@ workflow:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME !~ /^(demo|master|stable|staging|test)$/
 
 stages:
-  - deploy
-  - sonarqube-check
+  - clean_build
   - test
-  - clean_test
-  - clean_env
+  - deploy
   - update
-  - clean_build
-
-deploy:
-  stage: deploy
-  environment: $CI_COMMIT_REF_NAME
-  script:
-    - ansible-playbook -i $ANSIBLE_INVENTORY $DEPLOY_PLAYBOOK --vault-id $ANSIBLE_VAULT_FILE -e "build_id=$CI_COMMIT_REF_SLUG repo_version=$CI_COMMIT_REF_NAME"
+  - clean_env
 
 integration_test:
   stage: test
+  tags:
+    - docker
+  image:
+    name: python:3.8
+  services:
+    - postgres:9.6
+    - rabbitmq:3
+    - redis:latest
+    - name: elasticsearch:7.9.2
+      alias: elasticsearch
+      command: [ "bin/elasticsearch", "-Expack.security.enabled=false", "-Ediscovery.type=single-node" ]
+  variables:
+    POSTGRES_USER: metax_user
+    POSTGRES_PASSWORD: password
+    POSTGRES_DB: metax_db
+    METAX_DATABASE: metax_db
+    METAX_DATABASE_PASSWORD: password
+    METAX_DATABASE_USER: metax_user
+    METAX_DATABASE_HOST: postgres
+    ELASTIC_SEARCH_HOSTS: elasticsearch
+    RABBIT_MQ_HOSTS: rabbitmq
+    REDIS_HOST: redis
+    REDIS_PORT: 6379
+    DJANGO_SECRET_KEY: django-insecure-22388&4#2_#u0e%$%!)5vo3mpys)#f7@vsk9az354!jgze--&e
+    DJANGO_ENV: unittests
   environment: $CI_COMMIT_REF_NAME
   script:
-    - ansible-playbook -i $ANSIBLE_INVENTORY $TEST_PLAYBOOK --vault-id $ANSIBLE_VAULT_FILE -e "build_id=$CI_COMMIT_REF_SLUG repo_version=$CI_COMMIT_REF_NAME"
-
-clean_test:
-  stage: clean_test
-  environment:
-    name: $CI_COMMIT_REF_NAME
-    on_stop: clean_gitlab_env
-  script:
-    - ansible-playbook -i $ANSIBLE_INVENTORY $DELETE_PLAYBOOK --vault-id $ANSIBLE_VAULT_FILE -e "build_id=$CI_COMMIT_REF_SLUG repo_version=$CI_COMMIT_REF_NAME"
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      when: always
-    - when: never
+    - apt-get update && apt-get install -y xqilla libxerces-c-dev build-essential libssl-dev libffi-dev python-dev libxqilla-dev
+    - mkdir -p /var/log/metax-api/errors
+    - pip install -r requirements.txt
+    - cd src
+    - python manage.py first_time_setup
+    - coverage run manage.py test --parallel
+    - coverage combine
+    - coverage report -m
+    - coverage xml
+  artifacts:
+    reports:
+      cobertura: src/coverage.xml
+    paths:
+      - src/coverage.xml
 
 clean_gitlab_env:
   stage: clean_env
@@ -46,10 +64,19 @@ clean_gitlab_env:
   script:
     - echo "Cleaning deleted branches from environments"
   rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME !~ /^(demo|master|stable|staging|test)$/
       when: never
     - when: manual
 
+deploy:
+  stage: deploy
+  environment: $CI_COMMIT_REF_NAME
+  script:
+    - ansible-playbook -i $ANSIBLE_INVENTORY $DEPLOY_PLAYBOOK --vault-id $ANSIBLE_VAULT_FILE -e "build_id=$CI_COMMIT_REF_SLUG repo_version=$CI_COMMIT_REF_NAME"
+  rules:
+    - if: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^(demo|master|stable|staging|test)$/
+      when: always
+
 update_metax:
   stage: update
   environment: $CI_COMMIT_REF_NAME
@@ -68,12 +95,12 @@ clean_previous_build:
     - ansible-playbook -i $ANSIBLE_INVENTORY $DELETE_PLAYBOOK --vault-id $ANSIBLE_VAULT_FILE -e "build_id=$CI_COMMIT_REF_SLUG repo_version=$CI_COMMIT_REF_NAME"
   rules:
     - if: $CI_COMMIT_BRANCH =~ /^(staging|test|stable)$/
-      when: never
-      when: manual
+      when: always
 
 sonarqube-check:
   tags: 
     - docker
+  stage: test
   image: 
     name: sonarsource/sonar-scanner-cli:latest
     entrypoint: [""]
@@ -87,6 +114,9 @@ sonarqube-check:
   script: 
     - sonar-scanner
   allow_failure: true
+  needs:
+    - job: integration_test
+      artifacts: true
   only:
     - merge_requests
     - master

README.md

@@ -1,3 +1,5 @@
+# Metax API
+
 This repository contains the code for Metax API service.
 
 ## License

sonar-project.properties

@@ -6,3 +6,4 @@ sonar.qualitygate.wait=true
 sonar.sources=src/metax_api/
 sonar.exclusions=src/metax_api/tests/**/*
 sonar.tests=src/metax_api/tests/
+sonar.python.coverage.reportPaths=src/coverage*.xml

src/.coveragerc

@@ -10,6 +10,8 @@ omit =
     */urls.py
     */router.py
     metax_api/onappstart.py
+    */site-packages/*
+
 branch = True
 concurrency = multiprocessing
 data_file = coverage_data/coverage.db
@@ -29,6 +31,8 @@ exclude_lines =
     raise NotImplementedError
     if 0:
     if __name__ == .__main__.:
+
+
 skip_covered = True
 omit =
     */migrations/*
@@ -40,4 +44,5 @@ omit =
     */manage.py
     */urls.py
     */router.py
-    metax_api/onappstart.py
+    metax_api/onappstart.py
+    */site-packages/*