From 0550668308ffb80a12fcd40f139bcc09026471dc Mon Sep 17 00:00:00 2001
From: Toni Nurmi <tonurmi@csc.fi>
Date: Wed, 30 Mar 2022 13:27:39 +0000
Subject: [PATCH] CSCFAIRADM-1271: Refactor CI/CD

---
 .gitlab-ci.yml           | 84 +++++++++++++++++++++++++++-------------
 README.md                |  2 +
 sonar-project.properties |  1 +
 src/.coveragerc          |  7 +++-
 4 files changed, 66 insertions(+), 28 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d4e4226f..4cae1f0e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,37 +4,55 @@ workflow:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME !~ /^(demo|master|stable|staging|test)$/
 
 stages:
-  - deploy
-  - sonarqube-check
+  - clean_build
   - test
-  - clean_test
-  - clean_env
+  - deploy
   - update
-  - clean_build
-
-deploy:
-  stage: deploy
-  environment: $CI_COMMIT_REF_NAME
-  script:
-    - ansible-playbook -i $ANSIBLE_INVENTORY $DEPLOY_PLAYBOOK --vault-id $ANSIBLE_VAULT_FILE -e "build_id=$CI_COMMIT_REF_SLUG repo_version=$CI_COMMIT_REF_NAME"
+  - clean_env
 
 integration_test:
   stage: test
+  tags:
+    - docker
+  image:
+    name: python:3.8
+  services:
+    - postgres:9.6
+    - rabbitmq:3
+    - redis:latest
+    - name: elasticsearch:7.9.2
+      alias: elasticsearch
+      command: [ "bin/elasticsearch", "-Expack.security.enabled=false", "-Ediscovery.type=single-node" ]
+  variables:
+    POSTGRES_USER: metax_user
+    POSTGRES_PASSWORD: password
+    POSTGRES_DB: metax_db
+    METAX_DATABASE: metax_db
+    METAX_DATABASE_PASSWORD: password
+    METAX_DATABASE_USER: metax_user
+    METAX_DATABASE_HOST: postgres
+    ELASTIC_SEARCH_HOSTS: elasticsearch
+    RABBIT_MQ_HOSTS: rabbitmq
+    REDIS_HOST: redis
+    REDIS_PORT: 6379
+    DJANGO_SECRET_KEY: django-insecure-22388&4#2_#u0e%$%!)5vo3mpys)#f7@vsk9az354!jgze--&e
+    DJANGO_ENV: unittests
   environment: $CI_COMMIT_REF_NAME
   script:
-    - ansible-playbook -i $ANSIBLE_INVENTORY $TEST_PLAYBOOK --vault-id $ANSIBLE_VAULT_FILE -e "build_id=$CI_COMMIT_REF_SLUG repo_version=$CI_COMMIT_REF_NAME"
-
-clean_test:
-  stage: clean_test
-  environment:
-    name: $CI_COMMIT_REF_NAME
-    on_stop: clean_gitlab_env
-  script:
-    - ansible-playbook -i $ANSIBLE_INVENTORY $DELETE_PLAYBOOK --vault-id $ANSIBLE_VAULT_FILE -e "build_id=$CI_COMMIT_REF_SLUG repo_version=$CI_COMMIT_REF_NAME"
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      when: always
-    - when: never
+    - apt-get update && apt-get install -y xqilla libxerces-c-dev build-essential libssl-dev libffi-dev python-dev libxqilla-dev
+    - mkdir -p /var/log/metax-api/errors
+    - pip install -r requirements.txt
+    - cd src
+    - python manage.py first_time_setup
+    - coverage run manage.py test --parallel
+    - coverage combine
+    - coverage report -m
+    - coverage xml
+  artifacts:
+    reports:
+      cobertura: src/coverage.xml
+    paths:
+      - src/coverage.xml
 
 clean_gitlab_env:
   stage: clean_env
@@ -46,10 +64,19 @@ clean_gitlab_env:
   script:
     - echo "Cleaning deleted branches from environments"
   rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME !~ /^(demo|master|stable|staging|test)$/
       when: never
     - when: manual
 
+deploy:
+  stage: deploy
+  environment: $CI_COMMIT_REF_NAME
+  script:
+    - ansible-playbook -i $ANSIBLE_INVENTORY $DEPLOY_PLAYBOOK --vault-id $ANSIBLE_VAULT_FILE -e "build_id=$CI_COMMIT_REF_SLUG repo_version=$CI_COMMIT_REF_NAME"
+  rules:
+    - if: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ /^(demo|master|stable|staging|test)$/
+      when: always
+
 update_metax:
   stage: update
   environment: $CI_COMMIT_REF_NAME
@@ -68,12 +95,12 @@ clean_previous_build:
     - ansible-playbook -i $ANSIBLE_INVENTORY $DELETE_PLAYBOOK --vault-id $ANSIBLE_VAULT_FILE -e "build_id=$CI_COMMIT_REF_SLUG repo_version=$CI_COMMIT_REF_NAME"
   rules:
     - if: $CI_COMMIT_BRANCH =~ /^(staging|test|stable)$/
-      when: never
-      when: manual
+      when: always
 
 sonarqube-check:
   tags: 
     - docker
+  stage: test
   image: 
     name: sonarsource/sonar-scanner-cli:latest
     entrypoint: [""]
@@ -87,6 +114,9 @@ sonarqube-check:
   script: 
     - sonar-scanner
   allow_failure: true
+  needs:
+    - job: integration_test
+      artifacts: true
   only:
     - merge_requests
     - master
diff --git a/README.md b/README.md
index b753029d..59210c65 100755
--- a/README.md
+++ b/README.md
@@ -1,3 +1,5 @@
+# Metax API
+
 This repository contains the code for Metax API service.
 
 ## License
diff --git a/sonar-project.properties b/sonar-project.properties
index eb26c71d..98d55388 100644
--- a/sonar-project.properties
+++ b/sonar-project.properties
@@ -6,3 +6,4 @@ sonar.qualitygate.wait=true
 sonar.sources=src/metax_api/
 sonar.exclusions=src/metax_api/tests/**/*
 sonar.tests=src/metax_api/tests/
+sonar.python.coverage.reportPaths=src/coverage*.xml
diff --git a/src/.coveragerc b/src/.coveragerc
index 63a369dc..3d5d6384 100755
--- a/src/.coveragerc
+++ b/src/.coveragerc
@@ -10,6 +10,8 @@ omit =
     */urls.py
     */router.py
     metax_api/onappstart.py
+    */site-packages/*
+
 branch = True
 concurrency = multiprocessing
 data_file = coverage_data/coverage.db
@@ -29,6 +31,8 @@ exclude_lines =
     raise NotImplementedError
     if 0:
     if __name__ == .__main__.:
+
+
 skip_covered = True
 omit =
     */migrations/*
@@ -40,4 +44,5 @@ omit =
     */manage.py
     */urls.py
     */router.py
-    metax_api/onappstart.py
\ No newline at end of file
+    metax_api/onappstart.py
+    */site-packages/*