From 4cb831b38e7df21f610e036caf99a255a502cbfa Mon Sep 17 00:00:00 2001 From: Toni Nurmi Date: Thu, 11 Feb 2021 16:17:20 +0200 Subject: [PATCH 01/34] rewrite load initial data management command (#758) * rewrite load initial data management command Co-authored-by: Toni Nurmi --- .../management/commands/loadinitialdata.py | 204 +++--------------- 1 file changed, 28 insertions(+), 176 deletions(-) diff --git a/src/metax_api/management/commands/loadinitialdata.py b/src/metax_api/management/commands/loadinitialdata.py index 83bcf43c..246a4b14 100755 --- a/src/metax_api/management/commands/loadinitialdata.py +++ b/src/metax_api/management/commands/loadinitialdata.py @@ -6,189 +6,41 @@ # :license: MIT import json +import datetime +import logging -import requests -import urllib3 -from django.conf import settings -from django.core.management.base import BaseCommand, CommandError -from icecream import ic +from django.core.management.base import BaseCommand +from django.db import IntegrityError -from metax_api.utils import executing_test_case -from django.conf import settings as django_settings - -urllib3.disable_warnings() +from metax_api.models import DataCatalog +from metax_api.models import FileStorage +logger = logging.getLogger(__name__) class Command(BaseCommand): help = "Load initial data for Metax: Data catalogs, file storages." - def add_arguments(self, parser): - # optional arguments to pass in special settings during test case execution - parser.add_argument( - "--test-settings", - type=str, - nargs="?", - help="Set url to use during test execution", - ) - def handle(self, *args, **options): - ic(executing_test_case()) - if executing_test_case(): - self._set_test_confs(options) - else: - self._set_real_confs() - - self._load_data_catalogs() - self._load_file_storages() - - def _set_test_confs(self, options): - """ - For test case execution, a testserver url and test metax credentials are expected - as parameters from the testrunner. - """ - if "test_settings" not in options: - raise CommandError("test case execution requires settings as a parameter") - - test_opts = options["test_settings"] - self._metax_api_root = test_opts["metax_url"] - self._metax_api_user = ( - test_opts["metax_credentials"]["username"], - test_opts["metax_credentials"]["password"], - ) - - if self._metax_api_root == "https://localhost": - # extra precaution... - raise CommandError("Test case tried to write into real db") - - def _set_real_confs(self): - """ - Set metax url to localhost, and get user credentials from app_config. - """ - self._metax_api_root = settings.METAX_API_ROOT - - for user in settings.API_USERS: - if user["username"] == "metax": - self._metax_api_user = (user["username"], user["password"]) - break - else: - raise CommandError("Could not find metax-user from app_config ?") - - def _error_is_already_exists(self, details): - try: - for field_name, errors in details.items(): - if field_name == "identifier" and "already exists" in errors[0]: - return True - except: - pass - return False - - def _load_data_catalogs(self): - try: - with open("metax_api/initialdata/datacatalogs.json", "r") as f: - data_catalogs = json.load(f) - except FileNotFoundError: # noqa - raise CommandError("File initialdata/datacatalogs.json does not exist?") - except json.decoder.JSONDecodeError as e: - raise CommandError("Error loading data catalog json: %s" % str(e)) - - self.stdout.write("Creating %d data catalogs..." % len(data_catalogs)) - - for dc in data_catalogs: - if dc['catalog_json']['research_dataset_schema'] == 'dft': - pass - else: - if 'v2' in django_settings.API_VERSIONS_ENABLED: - response = requests.post('%s/rest/v2/datacatalogs' % self._metax_api_root, - json=dc, auth=self._metax_api_user, verify=False) - else: - response = requests.post('%s/rest/datacatalogs' % self._metax_api_root, - json=dc, auth=self._metax_api_user, verify=False) - - if response.status_code == 201: - self.stdout.write('Created catalog: %s' % dc['catalog_json']['identifier']) - else: - # update instead - try: - errors = response.json() - except: - raise CommandError(response.content) - - if self._error_is_already_exists(errors.get('catalog_json', {})): - self.stdout.write('Catalog %s already exists, updating instead...' % - dc['catalog_json']['identifier']) - - if 'v2' in django_settings.API_VERSIONS_ENABLED: - response = requests.put('%s/rest/v2/datacatalogs/%s' % - (self._metax_api_root, dc['catalog_json']['identifier']), - json=dc, auth=self._metax_api_user, verify=False) - else: - response = requests.put('%s/rest/datacatalogs/%s' % - (self._metax_api_root, dc['catalog_json']['identifier']), - json=dc, auth=self._metax_api_user, verify=False) - - if response.status_code == 200: - self.stdout.write( - "Updated catalog: %s" % dc["catalog_json"]["identifier"] - ) - continue - - # create or update ended in error - raise CommandError('Failed to process catalog: %s. Reason: %s' % - (dc['catalog_json']['identifier'], errors)) - - def _load_file_storages(self): - try: - with open("metax_api/initialdata/filestorages.json", "r") as f: - storages = json.load(f) - except FileNotFoundError: # noqa - raise CommandError("File initialdata/filestorages.json does not exist?") - except json.decoder.JSONDecodeError as e: - raise CommandError("Error loading file storage json: %s" % str(e)) - - self.stdout.write("Creating %d file storages..." % len(storages)) - - for fs in storages: - response = requests.post( - "%s/rest/filestorages" % self._metax_api_root, - json=fs, - auth=self._metax_api_user, - verify=False, - ) - - if response.status_code == 201: - self.stdout.write( - "Created file storage: %s" % fs["file_storage_json"]["identifier"] - ) - else: - # update instead + with open("metax_api/initialdata/datacatalogs.json", "r") as f: + data_catalogs = json.load(f) + for json_dc in data_catalogs: try: - errors = response.json() - except: - raise CommandError(response.content) - if self._error_is_already_exists(errors.get("file_storage_json", {})): - self.stdout.write( - "File storage %s already exists, updating instead..." - % fs["file_storage_json"]["identifier"] - ) - - response = requests.put( - "%s/rest/filestorages/%s" - % (self._metax_api_root, fs["file_storage_json"]["identifier"]), - json=fs, - auth=self._metax_api_user, - verify=False, - ) - - if response.status_code == 200: - self.stdout.write( - "Updated file storage: %s" - % fs["file_storage_json"]["identifier"] - ) - continue - - # create or update ended in error - raise CommandError( - "Failed to process storage: %s. Reason: %s" - % (fs["file_storage_json"]["identifier"], errors) - ) + dc = DataCatalog(catalog_json=json_dc["catalog_json"], date_created=datetime.datetime.now()) + dc.catalog_record_services_create = json_dc["catalog_record_services_create"] + dc.catalog_record_services_edit = json_dc["catalog_record_services_edit"] + dc.catalog_record_services_read = json_dc["catalog_record_services_read"] + dc.save() + except IntegrityError as e: + logger.error("datacatalog already exists in the database") + logger.info("Successfully created missing datacatalogs") + + with open("metax_api/initialdata/filestorages.json", "r") as f: + storages = json.load(f) + for fs in storages: + try: + fs = FileStorage(file_storage_json=fs["file_storage_json"], date_created=datetime.datetime.now()) + fs.save() + except IntegrityError as e: + logger.error("filestorage already exists in the database") + logger.info("Successfully created missing filestorages") From f46e463b47fbbcb763a932728779a3f3baa50b68 Mon Sep 17 00:00:00 2001 From: Toni Nurmi Date: Mon, 15 Feb 2021 16:13:58 +0200 Subject: [PATCH 02/34] delete flake8 check from travis. (#760) * delete flake8 check from travis. * cd into src before calling manage.py Co-authored-by: Toni Nurmi --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 457c9c43..d070fca4 100755 --- a/.travis.yml +++ b/.travis.yml @@ -56,8 +56,7 @@ before_script: - sudo chown -R $USER:$USER src/log script: -- cd src && flake8 . -- python manage.py migrate metax_api +- cd src && python manage.py migrate metax_api - coverage run --source="." manage.py test metax_api - COVERALLS_REPO_TOKEN=$coveralls_token coveralls From d3a1e053d4042ab29d792cd9550cd8daa2916131 Mon Sep 17 00:00:00 2001 From: Toni Nurmi Date: Mon, 15 Feb 2021 16:14:43 +0200 Subject: [PATCH 03/34] fix service role issues (#761) * fix service role issues * add draft perms Co-authored-by: Toni Nurmi --- src/metax_api/settings/environments/remote.py | 4 ++++ src/metax_api/settings/environments/stable.py | 22 ++++++++++--------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/metax_api/settings/environments/remote.py b/src/metax_api/settings/environments/remote.py index c1eced5c..72ff9bc1 100644 --- a/src/metax_api/settings/environments/remote.py +++ b/src/metax_api/settings/environments/remote.py @@ -40,6 +40,10 @@ { "password": env("DOWNLOAD_USER_PASSWORD"), "username": "download" + }, + { + "password": env("JYU_USER_PASSWORD"), + "username": "jyu" } ] diff --git a/src/metax_api/settings/environments/stable.py b/src/metax_api/settings/environments/stable.py index 50a3c33f..7ce69b00 100644 --- a/src/metax_api/settings/environments/stable.py +++ b/src/metax_api/settings/environments/stable.py @@ -33,12 +33,12 @@ "metax", "etsin" ], + "delete": [ "metax", "etsin" ], "read": [ - "metax", "all" ], "update": [ @@ -54,6 +54,7 @@ "qvain-light", "etsin", "tpas", + "jyu", "endusers" ], "delete": [ @@ -63,8 +64,10 @@ "qvain-light", "etsin", "tpas", + "jyu", "endusers" ], + "read": [ "all" ], @@ -75,14 +78,15 @@ "qvain-light", "etsin", "tpas", + "jyu", "endusers" ] }, "directories": { "read": [ "metax", - "qvain", "ida", + "qvain", "qvain-light", "etsin", "tpas", @@ -101,6 +105,7 @@ "ida", "tpas" ], + "read": [ "metax", "ida", @@ -121,20 +126,16 @@ }, "filestorages": { "create": [ - "metax", - "ida" + "metax" ], "delete": [ - "metax", - "ida" + "metax" ], "read": [ - "metax", - "ida" + "metax" ], "update": [ - "metax", - "ida" + "metax" ] }, "schemas": { @@ -240,6 +241,7 @@ "all" ] }, + "deprecated_datasets_cumulative": { "use": [ "all" From ba12f0d59d28e7e7698e728631d04b99fa8a3423 Mon Sep 17 00:00:00 2001 From: Katri Tegel <17436908+katrite@users.noreply.github.com> Date: Mon, 15 Feb 2021 16:15:14 +0200 Subject: [PATCH 04/34] Add_metax_env_variable_to_components (#756) Add METAX_ENV to components via version control Co-authored-by: Katri Tegel Co-authored-by: Toni Nurmi --- src/metax_api/settings/components/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/metax_api/settings/components/common.py b/src/metax_api/settings/components/common.py index aa060a31..55de2c90 100755 --- a/src/metax_api/settings/components/common.py +++ b/src/metax_api/settings/components/common.py @@ -3,6 +3,7 @@ from metax_api.settings import env from metax_api.settings.components import BASE_DIR +METAX_ENV=(str, "local_development") DEBUG = env("DEBUG") SECRET_KEY = env("DJANGO_SECRET_KEY") METAX_API_ROOT = env("METAX_API_ROOT") From f27e2ba1daa4416652624c8eb6394819a98ac2b1 Mon Sep 17 00:00:00 2001 From: Toni Nurmi Date: Wed, 17 Feb 2021 11:02:07 +0200 Subject: [PATCH 05/34] fresh docker-compose.yml --- docker-compose.yml | 35 ++--------------------------------- 1 file changed, 2 insertions(+), 33 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 82b62f9f..7382799e 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,34 +1,3 @@ -version: "3.8" +version: "3.3" + -services: - db: - image: library/postgres:9 - environment: - - POSTGRES_DB=postgres - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=postgres - volumes: - - postgres_data:/var/lib/postgresql/data/ - web: - build: . - command: python manage.py runserver 0.0.0.0:8008 - volumes: - - src:/code - ports: - - "8008:8008" - depends_on: - - db - - redis - # env_file: - # - ./src/metax_api/settings/.env - environment: - - REDIS_HOST=redis - links: - - redis:redis - redis: - image: library/redis - ports: - - "6379:6379" -volumes: - postgres_data: - src: From 2a4263ac78622adbcb5f65b54eccafb08152fa0a Mon Sep 17 00:00:00 2001 From: Katri Tegel <17436908+katrite@users.noreply.github.com> Date: Thu, 18 Feb 2021 13:46:25 +0200 Subject: [PATCH 06/34] CSCFAIRMETA-912-recalculate-bytesize-filecount-for-cr (#765) - Recalculate byte_size and file_count for catalog record Co-authored-by: Katri Tegel --- .../management/commands/fix_file_counts_for_cr.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 src/metax_api/management/commands/fix_file_counts_for_cr.py diff --git a/src/metax_api/management/commands/fix_file_counts_for_cr.py b/src/metax_api/management/commands/fix_file_counts_for_cr.py new file mode 100644 index 00000000..2409ae14 --- /dev/null +++ b/src/metax_api/management/commands/fix_file_counts_for_cr.py @@ -0,0 +1,15 @@ +import logging + +from django.core.management.base import BaseCommand + +from metax_api.models import CatalogRecord + +logger = logging.getLogger(__name__) + +class Command(BaseCommand): + def handle(self, *args, **options): + CRS = CatalogRecord.objects.all() + for catalog_record in CRS: + catalog_record.calculate_directory_byte_sizes_and_file_counts() + logger.info(f"Calculating {catalog_record.identifier} ") + logger.info(f"fix_file_counts command executed successfully") \ No newline at end of file From 3e8bd01604a4c55eba09a6469bff9f8125998d4c Mon Sep 17 00:00:00 2001 From: Katri Tegel <17436908+katrite@users.noreply.github.com> Date: Mon, 22 Feb 2021 13:49:23 +0200 Subject: [PATCH 07/34] CSCFAIRMETA-954-publish-records-to-TTV (#767) - Fetch and publish all existing datasets into, and only to, TTV-operations queue Co-authored-by: Katri Tegel --- .../management/commands/load_data_to_TTV.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 src/metax_api/management/commands/load_data_to_TTV.py diff --git a/src/metax_api/management/commands/load_data_to_TTV.py b/src/metax_api/management/commands/load_data_to_TTV.py new file mode 100644 index 00000000..d5ad428c --- /dev/null +++ b/src/metax_api/management/commands/load_data_to_TTV.py @@ -0,0 +1,36 @@ +# This file is part of the Metax API service +# +# Copyright 2017-2018 Ministry of Education and Culture, Finland +# +# :author: CSC - IT Center for Science Ltd., Espoo Finland +# :license: MIT + +import logging +import json + +from django.core.management.base import BaseCommand +from metax_api.models import CatalogRecord +from metax_api.services import RabbitMQService +from metax_api.api.rest.base.serializers import CatalogRecordSerializer + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + + help = "Upload all existing data to TTV's RabbitMQ queue" + + def handle(self, *args, **options): + catalog_records = CatalogRecord.objects.all() + logger.info(f"found {catalog_records.count()} catalog records") + aff_rows = 0 + + for catalog_record in catalog_records: + cr = CatalogRecordSerializer(catalog_record).data + logger.info(f"Publishing catalog record {aff_rows} from TOTAL {catalog_records.count()}") + RabbitMQService.publish(cr, routing_key='', exchange="TTV-datasets") + aff_rows += 1 + logger.info(f"Published catalog record id={cr['identifier']}") + + logger.info(f"All catalog records published") + From 728a71bac06b1768078bfc06d1b4c5a09390c3eb Mon Sep 17 00:00:00 2001 From: Toni Nurmi Date: Tue, 23 Feb 2021 14:17:33 +0200 Subject: [PATCH 08/34] initial docker swarm file --- docker-compose.yml | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 7382799e..f4eca04c 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,48 @@ version: "3.3" +services: + metax: + image: metax-api + ports: + - 8008:8008 + - 8000:8000 + volumes: + - ./src:/code + environment: + DEBUG: 'true' + DJANGO_ENV: 'test' + METAX_DATABASE: 'metax_db_test' + METAX_DATABASE_PASSWORD: 'YMDLekQMqrVKcs3' + METAX_DATABASE_USER: 'metax_test' + OAI_BASE_URL: 'http://mockurl.com' + OAI_ETSIN_URL_TEMPLATE: 'http://mockurl.com/%s' + OAI_ADMIN_EMAIL: 'mocked@mail.com' + REMS_ENABLED: 'true' + REMS_API_KEY: 'not_set' + REMS_BASE_URL: 'http://mockurl.com' + REMS_ETSIN_URL_TEMPLATE: 'http://mockurl.com/%s' + REMS_METAX_USER: 'not_set' + REMS_REPORTER_USER: 'not_set' + REMS_AUTO_APPROVER: 'not_set' + REMS_FORM_ID: 99 + ENABLE_V1_ENDPOINTS: 'true' + ENABLE_V2_ENDPOINTS: 'true' + REDIS_HOST: redis + RABBIT_MQ_HOSTS: rabbitmq + ELASTIC_SEARCH_HOSTS: elasticsearch + METAX_DATABASE_HOST: db + redis: + image: redis + db: + image: postgres:9 + environment: + POSTGRES_USER: 'metax_test' + POSTGRES_PASSWORD: 'YMDLekQMqrVKcs3' + POSTGRES_DB: 'metax_db_test' + elasticsearch: + image: elasticsearch:7.9.2 + environment: + discovery.type: 'single-node' + rabbitmq: + image: rabbitmq:3-management From 59c8a16046fa7efa6518ae3b34505dfd4b13b296 Mon Sep 17 00:00:00 2001 From: Toni Nurmi Date: Wed, 24 Feb 2021 13:51:15 +0200 Subject: [PATCH 09/34] change DJANGO_ENV in docker-compose.yml --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index f4eca04c..5eccc2cb 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: - ./src:/code environment: DEBUG: 'true' - DJANGO_ENV: 'test' + DJANGO_ENV: 'local' METAX_DATABASE: 'metax_db_test' METAX_DATABASE_PASSWORD: 'YMDLekQMqrVKcs3' METAX_DATABASE_USER: 'metax_test' From e1f5786fd3af378c7392cd9372fd4888a867d0ad Mon Sep 17 00:00:00 2001 From: Toni Nurmi Date: Tue, 2 Mar 2021 16:27:50 +0200 Subject: [PATCH 10/34] change logging file defaults, rename docker-compose metax image, add logging folder generation to Dockerfile --- Dockerfile | 16 ++-------------- README.md | 6 +++++- docker-compose.yml | 2 +- src/metax_api/settings/__init__.py | 10 +++++----- 4 files changed, 13 insertions(+), 21 deletions(-) diff --git a/Dockerfile b/Dockerfile index 77fedea1..5baf63ed 100755 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,8 @@ FROM python:3.6 ENV PYTHONUNBUFFERED 1 ENV PYTHONDONTWRITEBYTECODE 1 +RUN mkdir -p /var/log/metax-api/errors && touch /var/log/metax-api/metax-api.json.log + WORKDIR /code COPY requirements.txt /code/ @@ -15,19 +17,5 @@ RUN pip install -r requirements.txt EXPOSE 8008 EXPOSE 8006 -ARG METAX_DATABASE_HOST -ARG REDIS_HOST -ARG RABBITMQ_HOST -ARG RABBIT_MQ_PASSWORD=guest -ARG RABBIT_MQ_USER=guest -ARG ELASTIC_SEARCH_HOST - -ENV METAX_DATABASE_HOST $METAX_DATABASE_HOST -ENV REDIS_HOST $REDIS_HOST -ENV RABBIT_MQ_HOSTS $RABBITMQ_HOST -ENV RABBIT_MQ_PASSWORD $RABBIT_MQ_PASSWORD -ENV RABBIT_MQ_USER $RABBIT_MQ_USER -ENV ELASTIC_SEARCH_HOSTS $ELASTIC_SEARCH_HOST - # CMD ["python", "/code/manage.py", "runserver", "0.0.0.0:8008"] CMD ["python", "manage.py", "runsslserver", "--certificate", ".certs/cert.pem","--key", ".certs/key.pem", "0.0.0.0:8008"] \ No newline at end of file diff --git a/README.md b/README.md index 03a0516b..668edf26 100755 --- a/README.md +++ b/README.md @@ -91,12 +91,16 @@ Check the IP addresses of Redis, RabbitMQ, ElasticSearch and Postgres:9 either f Build new docker image from repository root with this command (change ip-addresses to real ones: -`docker build -t metax-api:latest --build-arg METAX_DATABASE_HOST=xxx.xx.x.x --build-arg REDIS_HOST=xxx.xx.x.x --build-arg RABBITMQ_HOST=xxx.xx.x.x --build-arg ELASTIC_SEARCH_HOST=xxx.xx.x.x:xxxx .` +`docker build -t metax-api:latest .` Run the built container with command: `docker run -it --name metax-web --mount type=bind,source="$(pwd)"/src,target=/code -p 8008:8008 --rm metax-api:latest` +Run Metax with Docker Stack: +`docker stack deploy -c docker-compose.yml metax-dev` + + You should see metax-server starting at port 8008 with hot reload enabled ### Initial setup commands diff --git a/docker-compose.yml b/docker-compose.yml index 5eccc2cb..3e5dcf13 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.3" services: metax: - image: metax-api + image: fairdata-metax-web ports: - 8008:8008 - 8000:8000 diff --git a/src/metax_api/settings/__init__.py b/src/metax_api/settings/__init__.py index 42dccade..fd247302 100755 --- a/src/metax_api/settings/__init__.py +++ b/src/metax_api/settings/__init__.py @@ -27,12 +27,12 @@ ELASTIC_SEARCH_USE_SSL=(bool, False), ENABLE_V1_ENDPOINTS=(bool, True), ENABLE_V2_ENDPOINTS=(bool, True), - ERROR_FILES_PATH=(str, join(BASE_DIR, "log", "errors")), + ERROR_FILES_PATH=(str, join("/var", "log", "metax-api", "errors")), ES_CONFIG_DIR=(str, join(REFDATA_INDEXER_PATH, "resources", "es-config/")), LOCAL_REF_DATA_FOLDER=(str,join(REFDATA_INDEXER_PATH, "resources", "local-refdata/"),), - LOGGING_DEBUG_HANDLER_FILE=(str, join(BASE_DIR, "log", "metax_api.log")), - LOGGING_GENERAL_HANDLER_FILE=(str, join(BASE_DIR, "log", "metax_api.log")), - LOGGING_JSON_FILE_HANDLER_FILE=(str, join(BASE_DIR, "log", "metax_api.json.log")), + LOGGING_DEBUG_HANDLER_FILE=(str, join("/var", "log", "metax-api", "metax_api.log")), + LOGGING_GENERAL_HANDLER_FILE=(str, join("/var", "log", "metax-api", "metax_api.log")), + LOGGING_JSON_FILE_HANDLER_FILE=(str, join("/var", "log", "metax-api", "metax_api.json.log")), METAX_DATABASE_HOST=(str, "localhost"), METAX_DATABASE_PORT=(str, 5432), METAX_ENV=(str, "local_development"), @@ -73,6 +73,6 @@ # optional('environments/legacy.py'), ] ic(ENV) - +ic(env("LOGGING_GENERAL_HANDLER_FILE")) # Include settings: include(*base_settings) From 799871633ce4b2e11a7cfaa5929a46eb9e5b4a4f Mon Sep 17 00:00:00 2001 From: Toni Nurmi Date: Wed, 3 Mar 2021 15:02:19 +0200 Subject: [PATCH 11/34] remove redundant env vars, add docker config blocks, add volumes to docker-compose file --- docker-compose.yml | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3e5dcf13..cdc7d039 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.3" services: metax: - image: fairdata-metax-web + image: fairdata-docker.artifactory.ci.csc.fi/fairdata-metax-web ports: - 8008:8008 - 8000:8000 @@ -10,39 +10,47 @@ services: - ./src:/code environment: DEBUG: 'true' - DJANGO_ENV: 'local' METAX_DATABASE: 'metax_db_test' METAX_DATABASE_PASSWORD: 'YMDLekQMqrVKcs3' METAX_DATABASE_USER: 'metax_test' - OAI_BASE_URL: 'http://mockurl.com' - OAI_ETSIN_URL_TEMPLATE: 'http://mockurl.com/%s' - OAI_ADMIN_EMAIL: 'mocked@mail.com' - REMS_ENABLED: 'true' - REMS_API_KEY: 'not_set' - REMS_BASE_URL: 'http://mockurl.com' - REMS_ETSIN_URL_TEMPLATE: 'http://mockurl.com/%s' - REMS_METAX_USER: 'not_set' - REMS_REPORTER_USER: 'not_set' - REMS_AUTO_APPROVER: 'not_set' - REMS_FORM_ID: 99 - ENABLE_V1_ENDPOINTS: 'true' - ENABLE_V2_ENDPOINTS: 'true' REDIS_HOST: redis RABBIT_MQ_HOSTS: rabbitmq ELASTIC_SEARCH_HOSTS: elasticsearch METAX_DATABASE_HOST: db + configs: + - source: metax-config + target: '/code/metax_api/settings/.env' + redis: image: redis + volumes: + - metax-redis:/data db: image: postgres:9 environment: POSTGRES_USER: 'metax_test' POSTGRES_PASSWORD: 'YMDLekQMqrVKcs3' POSTGRES_DB: 'metax_db_test' + volumes: + - metax-postgres:/var/lib/postgresql96/data elasticsearch: image: elasticsearch:7.9.2 environment: discovery.type: 'single-node' + volumes: + - metax-es:/usr/share/elasticsearch/data rabbitmq: image: rabbitmq:3-management + volumes: + - metax-rabbitmq:/var/lib/rabbitmq +configs: + metax-config: + external: True +volumes: + metax-rabbitmq: + metax-es: + metax-postgres: + metax-redis: + + From fdb37d71ab340d084a6be6a0de84af7c2a96702e Mon Sep 17 00:00:00 2001 From: Toni Nurmi Date: Thu, 4 Mar 2021 10:04:54 +0200 Subject: [PATCH 12/34] fix config name on docker-compose.yml --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index cdc7d039..9b305710 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,7 +44,7 @@ services: volumes: - metax-rabbitmq:/var/lib/rabbitmq configs: - metax-config: + metax-web-config: external: True volumes: metax-rabbitmq: From 3e06e063f0a63752e5fa06c4ea8b4d2d848b2426 Mon Sep 17 00:00:00 2001 From: Toni Nurmi Date: Thu, 4 Mar 2021 10:06:00 +0200 Subject: [PATCH 13/34] fix config name on docker-compose.yml --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9b305710..71ee6a73 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,7 +18,7 @@ services: ELASTIC_SEARCH_HOSTS: elasticsearch METAX_DATABASE_HOST: db configs: - - source: metax-config + - source: metax-web-config target: '/code/metax_api/settings/.env' redis: From b59085bf4c3df31823706d4b8397a9f60028edf6 Mon Sep 17 00:00:00 2001 From: Toni Nurmi Date: Thu, 4 Mar 2021 10:35:29 +0200 Subject: [PATCH 14/34] try fix travis configuration --- .travis.yml | 2 +- src/metax_api/settings/__init__.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d070fca4..38c20ba3 100755 --- a/.travis.yml +++ b/.travis.yml @@ -52,7 +52,7 @@ install: before_script: - psql -U postgres < resources/sql/init_test.sql -- sudo mkdir -p src/log +- sudo mkdir -p var/log/metax-api/errors - sudo chown -R $USER:$USER src/log script: diff --git a/src/metax_api/settings/__init__.py b/src/metax_api/settings/__init__.py index fd247302..d20975b7 100755 --- a/src/metax_api/settings/__init__.py +++ b/src/metax_api/settings/__init__.py @@ -73,6 +73,5 @@ # optional('environments/legacy.py'), ] ic(ENV) -ic(env("LOGGING_GENERAL_HANDLER_FILE")) # Include settings: include(*base_settings) From 1e909576b159ebbfcd865198581966f5e6e636ab Mon Sep 17 00:00:00 2001 From: Toni Nurmi Date: Thu, 4 Mar 2021 10:55:02 +0200 Subject: [PATCH 15/34] try fix travis configuration --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 38c20ba3..0fe692cc 100755 --- a/.travis.yml +++ b/.travis.yml @@ -52,8 +52,8 @@ install: before_script: - psql -U postgres < resources/sql/init_test.sql -- sudo mkdir -p var/log/metax-api/errors -- sudo chown -R $USER:$USER src/log +- sudo mkdir -p /var/log/metax-api/errors +- sudo chown -R $USER:$USER /var/log/metax-api script: - cd src && python manage.py migrate metax_api From e1fae4f82d5b0e44fbc81081b142cfaf7143ae7f Mon Sep 17 00:00:00 2001 From: Toni Nurmi Date: Mon, 8 Mar 2021 12:39:58 +0200 Subject: [PATCH 16/34] remove templates.json --- containers/portainer/templates.json | 234 ---------------------------- 1 file changed, 234 deletions(-) delete mode 100755 containers/portainer/templates.json diff --git a/containers/portainer/templates.json b/containers/portainer/templates.json deleted file mode 100755 index 7d2d1f33..00000000 --- a/containers/portainer/templates.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "version": "2", - "templates": [ - { - "type": 1, - "title": "Nginx", - "description": "High performance web server", - "categories": [ - "webserver" - ], - "platform": "linux", - "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/nginx.png", - "image": "nginx:latest", - "ports": [ - "80/tcp", - "443/tcp" - ], - "volumes": [ - { - "container": "/etc/nginx" - }, - { - "container": "/usr/share/nginx/html" - } - ] - }, - { - "type": 1, - "title": "PostgreSQL", - "description": "The most advanced open-source database", - "categories": [ - "database" - ], - "platform": "linux", - "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/postgres.png", - "image": "postgres:latest", - "env": [ - { - "name": "POSTGRES_USER", - "label": "Superuser" - }, - { - "name": "POSTGRES_PASSWORD", - "label": "Superuser password" - } - ], - "ports": [ - "5432/tcp" - ], - "volumes": [ - { - "container": "/var/lib/postgresql/data" - } - ] - }, - { - "type": 1, - "title": "PostgreSQL 9", - "description": "The most advanced open-source database", - "categories": [ - "database" - ], - "platform": "linux", - "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/postgres.png", - "image": "postgres:9", - "env": [ - { - "name": "POSTGRES_USER", - "label": "postgres user", - "default": "metax_db_user" - }, - { - "name": "POSTGRES_PASSWORD", - "label": "postgres password", - "default": "YMDLekQMqrVKcs3" - } - ], - "ports": [ - "5432/tcp" - ], - "volumes": [ - { - "container": "/var/lib/postgresql96/data" - } - ] - }, - { - "type": 1, - "title": "Elasticsearch", - "description": "Open-source search and analytics engine", - "categories": [ - "database" - ], - "platform": "linux", - "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/elasticsearch.png", - "image": "elasticsearch:7.9.2", - "ports": [ - "9200/tcp", - "9300/tcp" - ], - "volumes": [ - { - "container": "/usr/share/elasticsearch/data" - } - ], - "env": [ - { - "name": "discovery.type", - "label": "Discovery Type", - "default": "single-node" - } - ] - }, - { - "type": 1, - "title": "Redis", - "description": "Open-source in-memory data structure store", - "categories": [ - "database" - ], - "platform": "linux", - "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/redis.png", - "image": "redis:latest", - "ports": [ - "6379/tcp" - ], - "volumes": [ - { - "container": "/data" - } - ] - }, - { - "type": 1, - "title": "Bitnami-Redis", - "description": "Open-source in-memory data structure store", - "categories": [ - "database" - ], - "platform": "linux", - "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/redis.png", - "image": "bitnami/redis:latest", - "ports": [ - "6379/tcp" - ], - "volumes": [ - { - "container": "/data" - } - ], - "env": [ - { - "name": "ALLOW_EMPTY_PASSWORD", - "label": "Allow empty password", - "default": "no" - }, - { - "name": "REDIS_PASSWORD", - "label": "Redis password", - "default": "NDE6oXov3hpoZV" - } - ] - }, - { - "type": 1, - "title": "Redis Sentinel by Bitnami ", - "description": "Open-source in-memory data structure store", - "categories": [ - "database" - ], - "platform": "linux", - "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/redis.png", - "image": "bitnami/redis-sentinel:latest", - "ports": [ - "26379/tcp" - ], - "volumes": [ - { - "container": "/data" - } - ], - "env": [ - { - "name": "REDIS_MASTER_HOST", - "label": "Host of the Redis master to monitor", - "default": "redis" - }, - { - "name": "REDIS_MASTER_PORT_NUMBER", - "label": "Port of the Redis master to monitor", - "default": "6379" - }, - { - "name": "REDIS_MASTER_SET", - "label": "Name of the set of Redis instances to monitor", - "default": "mymaster" - }, - { - "name": "REDIS_MASTER_PASSWORD", - "label": "Password to authenticate with the master", - "default": "7RewDzhRdXhoYc" - }, - { - "name": "REDIS_MASTER_USER", - "label": "Username to authenticate with when ACL is enabled for the master" - }, - { - "name": "REDIS_SENTINEL_PASSWORD", - "label": "Password to authenticate with this sentinel and to authenticate to other sentinels" - } - ] - }, - { - "type": 1, - "title": "RabbitMQ", - "description": "Highly reliable enterprise messaging system", - "categories": [ - "messaging" - ], - "platform": "linux", - "logo": "https://portainer-io-assets.sfo2.digitaloceanspaces.com/logos/rabbitmq.png", - "image": "rabbitmq:latest", - "ports": [ - "5671/tcp", - "5672/tcp" - ], - "volumes": [ - { - "container": "/var/lib/rabbitmq" - } - ] - } - ] - } \ No newline at end of file From e5ac32900e87d83a6c8d00bd81c0a42c1a475ad8 Mon Sep 17 00:00:00 2001 From: Toni Nurmi Date: Mon, 8 Mar 2021 16:55:21 +0200 Subject: [PATCH 17/34] add nginx service to docker-compose.yml --- docker-compose.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 71ee6a73..a1cd12c8 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,7 +20,6 @@ services: configs: - source: metax-web-config target: '/code/metax_api/settings/.env' - redis: image: redis volumes: @@ -43,9 +42,26 @@ services: image: rabbitmq:3-management volumes: - metax-rabbitmq:/var/lib/rabbitmq + nginx: + image: nginx:latest + configs: + - source: metax-nginx-config + target: '/etc/nginx/nginx.conf' + - source: fairdata-ssl-certificate + target: '/etc/nginx/ssl_certs/fd-dev.csc.fi.crt.pem' + - source: fairdata-ssl-certificate-key + target: '/etc/nginx/ssl_certs/fd-dev.csc.fi.key.pem' + ports: + - 4433:443 + configs: metax-web-config: external: True + fairdata-ssl-certificate: + external: True + fairdata-ssl-certificate-key: + external: True + volumes: metax-rabbitmq: metax-es: From 412ed554efd4f5d49a06b9e6f7ffc9732ef2c41f Mon Sep 17 00:00:00 2001 From: Toni Nurmi Date: Mon, 8 Mar 2021 16:58:20 +0200 Subject: [PATCH 18/34] add nginx config reference --- docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index a1cd12c8..88501d08 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -61,7 +61,8 @@ configs: external: True fairdata-ssl-certificate-key: external: True - + metax-nginx-config: + external: True volumes: metax-rabbitmq: metax-es: From 3a763d32f1a2aaba1b81ca049d5dadbe813ca0fc Mon Sep 17 00:00:00 2001 From: Toni Nurmi Date: Tue, 9 Mar 2021 11:55:07 +0200 Subject: [PATCH 19/34] disable nginx conf --- docker-compose.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 88501d08..a109283d 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: "3.3" +version: "3.8" services: metax: @@ -42,25 +42,25 @@ services: image: rabbitmq:3-management volumes: - metax-rabbitmq:/var/lib/rabbitmq - nginx: - image: nginx:latest - configs: - - source: metax-nginx-config - target: '/etc/nginx/nginx.conf' - - source: fairdata-ssl-certificate - target: '/etc/nginx/ssl_certs/fd-dev.csc.fi.crt.pem' - - source: fairdata-ssl-certificate-key - target: '/etc/nginx/ssl_certs/fd-dev.csc.fi.key.pem' - ports: - - 4433:443 +# nginx: +# image: nginx:latest +# configs: +# - source: metax-nginx-config +# target: '/etc/nginx/nginx.conf' + # - source: fairdata-ssl-certificate + # target: '/etc/nginx/ssl_certs/fd-dev.csc.fi.crt.pem' + # - source: fairdata-ssl-certificate-key + # target: '/etc/nginx/ssl_certs/fd-dev.csc.fi.key.pem' +# ports: +# - 4433:443 configs: metax-web-config: external: True - fairdata-ssl-certificate: - external: True - fairdata-ssl-certificate-key: - external: True + # fairdata-ssl-certificate: + # external: True + # fairdata-ssl-certificate-key: + # external: True metax-nginx-config: external: True volumes: From 68091b15e65e32bcfbb924c193624eeaecdf5deb Mon Sep 17 00:00:00 2001 From: Toni Nurmi Date: Tue, 9 Mar 2021 13:54:20 +0200 Subject: [PATCH 20/34] add additional nginx configs --- docker-compose.yml | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a109283d..e231590b 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -42,27 +42,36 @@ services: image: rabbitmq:3-management volumes: - metax-rabbitmq:/var/lib/rabbitmq -# nginx: -# image: nginx:latest -# configs: -# - source: metax-nginx-config -# target: '/etc/nginx/nginx.conf' - # - source: fairdata-ssl-certificate - # target: '/etc/nginx/ssl_certs/fd-dev.csc.fi.crt.pem' - # - source: fairdata-ssl-certificate-key - # target: '/etc/nginx/ssl_certs/fd-dev.csc.fi.key.pem' -# ports: -# - 4433:443 + nginx: + image: nginx:latest + configs: + - source: metax-nginx-config + target: '/etc/nginx/nginx.conf' + - source: fairdata-ssl-certificate + target: '/etc/nginx/ssl_certs/fd-dev.csc.fi.crt.pem' + - source: fairdata-ssl-certificate-key + target: '/etc/nginx/ssl_certs/fd-dev.csc.fi.key.pem' + ports: + - 4433:443 configs: metax-web-config: external: True - # fairdata-ssl-certificate: - # external: True - # fairdata-ssl-certificate-key: - # external: True + fairdata-ssl-certificate: + external: True + fairdata-ssl-certificate-key: + external: True metax-nginx-config: external: True + metax-nginx-elastic-headers-config: + external: True + metax-nginx-shared-headers-config: + external: True + metax-nginx-api-response-headers-config: + external: True + metax-nginx-static-file-headers-config: + external: True + volumes: metax-rabbitmq: metax-es: From fe62f15cea1864547a833479746dc073a9d64fea Mon Sep 17 00:00:00 2001 From: Toni Nurmi Date: Tue, 9 Mar 2021 14:26:00 +0200 Subject: [PATCH 21/34] add more nginx confs --- docker-compose.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index e231590b..51e538ec 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,6 +51,14 @@ services: target: '/etc/nginx/ssl_certs/fd-dev.csc.fi.crt.pem' - source: fairdata-ssl-certificate-key target: '/etc/nginx/ssl_certs/fd-dev.csc.fi.key.pem' + - source: metax-nginx-elastic-headers-config + target: '/etc/nginx/elastic_headers.conf' + - source: metax-nginx-shared-headers-config + target: '/etc/nginx/shared_headers.conf' + - source: metax-nginx-api-response-headers-config + target: '/etc/nginx/api_response_headers.conf' + - source: metax-nginx-static-file-headers-config + target: '/etc/nginx/static_file_headers.conf' ports: - 4433:443 From 06128723d910303b48ca4db19d2b43f780db0dd1 Mon Sep 17 00:00:00 2001 From: Toni Nurmi Date: Tue, 9 Mar 2021 15:07:53 +0200 Subject: [PATCH 22/34] nginx not crashing --- docker-compose.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 51e538ec..9e8a40df 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,9 +48,9 @@ services: - source: metax-nginx-config target: '/etc/nginx/nginx.conf' - source: fairdata-ssl-certificate - target: '/etc/nginx/ssl_certs/fd-dev.csc.fi.crt.pem' + target: '/etc/nginx/ssl_certs/fd-test.csc.fi.crt.pem' - source: fairdata-ssl-certificate-key - target: '/etc/nginx/ssl_certs/fd-dev.csc.fi.key.pem' + target: '/etc/nginx/ssl_certs/fd-test.csc.fi.key.pem' - source: metax-nginx-elastic-headers-config target: '/etc/nginx/elastic_headers.conf' - source: metax-nginx-shared-headers-config @@ -59,6 +59,8 @@ services: target: '/etc/nginx/api_response_headers.conf' - source: metax-nginx-static-file-headers-config target: '/etc/nginx/static_file_headers.conf' + - source: metax-nginx-dh-param-config + target: '/etc/nginx/ssl_certs/nginx_dhparam.pem' ports: - 4433:443 @@ -79,6 +81,8 @@ configs: external: True metax-nginx-static-file-headers-config: external: True + metax-nginx-dh-param-config: + external: True volumes: metax-rabbitmq: From dcf7c2563179dab67484eddd5d376f4fa8bbec6d Mon Sep 17 00:00:00 2001 From: Toni Nurmi Date: Wed, 10 Mar 2021 11:47:09 +0200 Subject: [PATCH 23/34] Fix auth logging (#727) * Bump django from 3.1.2 to 3.1.4 Bumps [django](https://github.com/django/django) from 3.1.2 to 3.1.4. - [Release notes](https://github.com/django/django/releases) - [Commits](https://github.com/django/django/compare/3.1.2...3.1.4) Signed-off-by: dependabot[bot] * include comparison values in logging when comparing auth password if debug is on * Update-test-db-with-migrations - Add migrations to correspond production db * Add-files-to-att-dataset (#732) Modify single dataset a0651f93-1826-4264-8fe4-df39f1ee72ac in prod Related to changing catalog Co-authored-by: Katri Tegel * Add-files-to-att-dataset (#734) Add dataset_version_set to datasets Co-authored-by: Katri Tegel * Att test dataset (#736) * Add-files-to-att-dataset -Add test dataset that belongs to IDA-project * att-test-dataset Add att-test-dataset i# with '#' will be ignored, and an empty message aborts the commit. Co-authored-by: Katri Tegel * Bump djangorestframework from 3.12.1 to 3.12.2 (#718) Bumps [djangorestframework](https://github.com/encode/django-rest-framework) from 3.12.1 to 3.12.2. - [Release notes](https://github.com/encode/django-rest-framework/releases) - [Commits](https://github.com/encode/django-rest-framework/compare/3.12.1...3.12.2) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Toni Nurmi * Bump lxml from 4.6.1 to 4.6.2 (#733) Bumps [lxml](https://github.com/lxml/lxml) from 4.6.1 to 4.6.2. - [Release notes](https://github.com/lxml/lxml/releases) - [Changelog](https://github.com/lxml/lxml/blob/master/CHANGES.txt) - [Commits](https://github.com/lxml/lxml/compare/lxml-4.6.1...lxml-4.6.2) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Toni Nurmi * Bump pytz from 2020.1 to 2020.5 (#730) Bumps [pytz](https://github.com/stub42/pytz) from 2020.1 to 2020.5. - [Release notes](https://github.com/stub42/pytz/releases) - [Commits](https://github.com/stub42/pytz/compare/release_2020.1...release_2020.5) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Toni Nurmi * Merge Test to Stable (#737) * add error handling to migration 27 in case of missing syke.csv file * split settings with django-split-settings, enable root logger, add .env support * replace redis client with new implementation, replace yml file with settings * integrate refdata loading as a task * remove unnecessary 403 error on flush_records * remove the yaml loading blocks * replace configs with env variables * fix some unnecessary asserts in tests * remove unnecessary request logging logic * update .gitignore, deps, and travis. Create .env.template * delete settings.py * update readme * fix ENV_VARS instructions * CSCFAIRADM-375: [ADD] instructions for mac and for db creation (#679) * add watchman system monitoring, bind elasticsearch hosts to env * disable tests testing file permissions, make elasticsearch take its host from env vars, add Docker as an option for local development setup * Increase logging on redis and rabbitmq connections, fix rabbitmq retry behaviour to be faster * optimize imports * CSCFAIRADM-375: [FIX] typos in variables and ignore log directory * CSCFAIRMETA-701: [FIX] crash on v2 dataset rpc api - Prevents creating a new dataset version if there exists a draft * CSCFAIRMETA-701: trigger travis * Update-test-db-with-migrations - Add migrations to correspond production db * Bump django from 3.1.2 to 3.1.4 Bumps [django](https://github.com/django/django) from 3.1.2 to 3.1.4. - [Release notes](https://github.com/django/django/releases) - [Commits](https://github.com/django/django/compare/3.1.2...3.1.4) Signed-off-by: dependabot[bot] * Modify-JYU-catalog-logo - Remove URL from JYU catalog logo name * Document env vars (#720) Co-authored-by: Toni Nurmi * add OAI env vars OAI_REPOSITORY_NAME, OAI_ETSIN_URL_TEMPLATE and OAI_ADMIN_EMAIL. Add new remote environment (#724) Co-authored-by: Toni Nurmi * Add env var to control if rabbitmq should use vhost. Change RabbitMQ service to use VHOST only if env var allows it (#726) Co-authored-by: Toni Nurmi * CSCFAIRMETA-799-RabbitMQ-TTV (#722) * CSCFAIRMETA-799-RabbitMQ-TTV - Add new connection with new vhost to TTV - Add new exchange and routing_key for TTV - Publish messages to 1 queue for TTV * CSCFAIRMETA-799-RabbitMQ-TTV - Add rabbitmq consumer for ttv * CSCFAIRMETA-799: [FIX] simplify publish process - Use the same virtual host for ttv and etsin - Adding exchanges should now only need to define it in settings - Define queues in settings - Move rabbitmq consumers to local environment since they are not used in other places Co-authored-by: Katri Tegel Co-authored-by: Tommi Pulli * CSCFAIRMETA-799-RabbitMQ-TTV-fix-settings (#728) -Rename django_settings to settings Co-authored-by: Katri Tegel * Add-files-to-att-dataset (#732) Modify single dataset a0651f93-1826-4264-8fe4-df39f1ee72ac in prod Related to changing catalog Co-authored-by: Katri Tegel * Add ENVs to travis.yml and fixed related settings - added SSL for elasticsearch in ENV - rabbitmq vhost is name is not needed if it's not used - travis environment settings are the same as in test environment * linting and removed unnecessary setting variable * fix flake8 problems and ones that are consequent - api_access settings were defined too permissive for unit tests - uncommented previously commented auth tests because of the api_access change - deleted a file permission related test that tests trivial issue and causes a headache in the new docker local env * add directory for logs during travis before_script * add rabbitmq hosts to default env * start using real redis in travis - Ignore console logging in Travis - test/stable deploys were removed since they aren't used anyways * Add-files-to-att-dataset (#734) Add dataset_version_set to datasets Co-authored-by: Katri Tegel * Att test dataset (#736) * Add-files-to-att-dataset -Add test dataset that belongs to IDA-project Co-authored-by: Katri Tegel * Cscfairmeta 847 relax draft validation (#740) * CSCFAIRMETA-847: [ADD] needed changes in draft json schema * CSCFAIRMETA-847: [FIX] draft datasets use dft schema - Add extra validation to publishing method to validate against the given data catalog * CSCFAIRMETA-847: [ADD|FIX] tests * CSCFAIRMETA-890-DOI-as-pref-identifier (#741) * CSCFAIRMETA-890-DOI-as-pref-identifier - Add a check for DOIs from remote datasets - Add a check that remote datasets with DOI are validated against datacite * CSCFAIRMETA-890-DOI-as-pref-identifier - Modify check for which DOIs are sent for registration Co-authored-by: Katri Tegel * CSCFAIRMETA-776: [ADD] Bulk delete (#725) * CSCFAIRMETA-776: [ADD] Bulk delete * CSCFAIRMETA-776-bulk-delete -Enable deleting of drafts Co-authored-by: genie9 Co-authored-by: Katri Tegel * CSCFAIRMETA-847: [FIX] bug where files could not be added to drafts * Api endpoints env (#743) Co-authored-by: Toni Nurmi * remove test comparing datetimes down to seconds as not needed (#744) Co-authored-by: Toni Nurmi * Bump djangorestframework from 3.12.1 to 3.12.2 (#718) Bumps [djangorestframework](https://github.com/encode/django-rest-framework) from 3.12.1 to 3.12.2. - [Release notes](https://github.com/encode/django-rest-framework/releases) - [Commits](https://github.com/encode/django-rest-framework/compare/3.12.1...3.12.2) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Toni Nurmi * Bump lxml from 4.6.1 to 4.6.2 (#733) Bumps [lxml](https://github.com/lxml/lxml) from 4.6.1 to 4.6.2. - [Release notes](https://github.com/lxml/lxml/releases) - [Changelog](https://github.com/lxml/lxml/blob/master/CHANGES.txt) - [Commits](https://github.com/lxml/lxml/compare/lxml-4.6.1...lxml-4.6.2) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Toni Nurmi * Bump pytz from 2020.1 to 2020.5 (#730) Bumps [pytz](https://github.com/stub42/pytz) from 2020.1 to 2020.5. - [Release notes](https://github.com/stub42/pytz/releases) - [Commits](https://github.com/stub42/pytz/compare/release_2020.1...release_2020.5) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Toni Nurmi Co-authored-by: Toni Nurmi Co-authored-by: Tommi Pulli <43133956+tompulli@users.noreply.github.com> Co-authored-by: Tommi Pulli Co-authored-by: Katri Tegel Co-authored-by: Toni Nurmi Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Evgenia Lyjina Co-authored-by: genie9 * fix near production api access scheme (#747) Co-authored-by: Toni Nurmi * add migration to fix incorrect file counts on directories (#752) Co-authored-by: Toni Nurmi * hotfixes to stable before production update (#748) * fix near production api access scheme (#747) Co-authored-by: Toni Nurmi * add migration to fix incorrect file counts on directories (#752) Co-authored-by: Toni Nurmi * make new management command to verify filecount for directories (#755) * make new management command to verify filecount for directories * fix/directory_file Fix typos s Co-authored-by: Toni Nurmi Co-authored-by: Katri Tegel * rewrite load initial data management command (#758) * rewrite load initial data management command Co-authored-by: Toni Nurmi * delete flake8 check from travis. (#760) * delete flake8 check from travis. * cd into src before calling manage.py Co-authored-by: Toni Nurmi * fix service role issues (#761) * fix service role issues * add draft perms Co-authored-by: Toni Nurmi * Add_metax_env_variable_to_components (#756) Add METAX_ENV to components via version control Co-authored-by: Katri Tegel Co-authored-by: Toni Nurmi * CSCFAIRMETA-912-recalculate-bytesize-filecount-for-cr (#765) - Recalculate byte_size and file_count for catalog record Co-authored-by: Katri Tegel * CSCFAIRMETA-954-publish-records-to-TTV (#767) - Fetch and publish all existing datasets into, and only to, TTV-operations queue Co-authored-by: Katri Tegel * include comparison values in logging when comparing auth password if debug is on * clear API_USERS setting if server is restarted Co-authored-by: Katri Tegel <17436908+katrite@users.noreply.github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Toni Nurmi Co-authored-by: Katri Tegel Co-authored-by: Tommi Pulli <43133956+tompulli@users.noreply.github.com> Co-authored-by: Tommi Pulli Co-authored-by: Evgenia Lyjina Co-authored-by: genie9 --- src/metax_api/middleware/identifyapicaller.py | 5 ++++- src/metax_api/settings/environments/remote.py | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/metax_api/middleware/identifyapicaller.py b/src/metax_api/middleware/identifyapicaller.py index 26454ef7..41539181 100755 --- a/src/metax_api/middleware/identifyapicaller.py +++ b/src/metax_api/middleware/identifyapicaller.py @@ -10,7 +10,7 @@ from base64 import b64decode import requests -from django.conf import settings as django_settings +from django.conf import settings as django_settings, settings from django.http import HttpResponseForbidden from metax_api.exceptions import Http403 @@ -162,6 +162,9 @@ def _auth_basic(self, request, auth_b64): raise Http403 if apikey != user["password"]: _logger.warning("Failed authnz for user %s: password mismatch" % username) + if settings.DEBUG: + _logger.warning(f"Auth password mismatch with apikey {apikey} != {user['password']}") + raise Http403 request.user.username = username diff --git a/src/metax_api/settings/environments/remote.py b/src/metax_api/settings/environments/remote.py index 72ff9bc1..fc3c528b 100644 --- a/src/metax_api/settings/environments/remote.py +++ b/src/metax_api/settings/environments/remote.py @@ -1,4 +1,7 @@ from metax_api.settings import env +from metax_api.settings.components.common import API_USERS + +API_USERS.clear() API_USERS = [ { From 5b868a8db07a1f92373007fdffc7083f2d02c7f9 Mon Sep 17 00:00:00 2001 From: Antero Vainio Date: Thu, 18 Feb 2021 15:02:26 +0200 Subject: [PATCH 24/34] Remove unnecessary /rest prefix from swagger endpoints --- swagger/v1/swagger.yaml | 56 ++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/swagger/v1/swagger.yaml b/swagger/v1/swagger.yaml index ec997116..e1a8aca9 100755 --- a/swagger/v1/swagger.yaml +++ b/swagger/v1/swagger.yaml @@ -11,7 +11,7 @@ produces: paths: # File Storage API - /rest/filestorages: + /filestorages: get: summary: List File Storages parameters: @@ -43,7 +43,7 @@ paths: description: Forbidden. Must have permission for resource tags: - File Storage API - /rest/filestorages/{PID}: + /filestorages/{PID}: get: summary: Get File Storage metadata parameters: @@ -125,7 +125,7 @@ paths: # File API - /rest/files: + /files: get: summary: Get list of files parameters: @@ -264,7 +264,7 @@ paths: description: None of the provided identifiers were found. tags: - File API - /rest/files/{PID}: + /files/{PID}: get: summary: Get file metadata parameters: @@ -363,7 +363,7 @@ paths: description: Object not found tags: - File API - /rest/files/{PID}/XML: + /files/{PID}/XML: get: summary: Get XML metadata from file produces: @@ -474,7 +474,7 @@ paths: description: File or namespace does not exist tags: - File API - /rest/files/datasets: + /files/datasets: post: summary: Get datasets where files belong to description: | @@ -521,7 +521,7 @@ paths: description: Forbidden. Must have permission for resource tags: - File API - /rest/files/restore: + /files/restore: post: summary: Restore removed files back to "not removed" state. description: | @@ -545,7 +545,7 @@ paths: description: Forbidden. Must have permission for resource tags: - File API - /rest/directories/{PID}: + /directories/{PID}: get: summary: Get details of a directory description: Does not contain the directory's files and sub-directories. For that, use /rest/directories/{PID}/files @@ -565,7 +565,7 @@ paths: description: Not found. Also when not authenticated request tags: - File API - /rest/directories/{PID}/files: + /directories/{PID}/files: get: summary: Get list of files and directories contained by a directory parameters: @@ -661,7 +661,7 @@ paths: description: Directory not found. Also when not authenticated request tags: - File API - /rest/directories/files: + /directories/files: get: summary: Get list of files and directories contained by a directory, queried by dir path and project description: | @@ -733,7 +733,7 @@ paths: description: Directory not found. Also when not authenticated request tags: - File API - /rest/directories/root: + /directories/root: get: summary: Return root directory for a project, and its files and directories description: Useful when starting to browse files for a project, when individual root-level directory identifier is not yet known. @@ -757,7 +757,7 @@ paths: # Data Catalog API - /rest/datacatalogs: + /datacatalogs: get: summary: List of data catalogs parameters: @@ -795,7 +795,7 @@ paths: description: Forbidden. Must have permission for resource tags: - Data Catalog API - /rest/datacatalogs/{PID}: + /datacatalogs/{PID}: get: summary: Get data catalog metadata parameters: @@ -895,7 +895,7 @@ paths: # Dataset API - /rest/datasets: + /datasets: # some of the parameters and returned fields are for TPAS usage only get: summary: List datasets @@ -1105,7 +1105,7 @@ paths: description: None of the provided identifiers were found. tags: - Dataset API - /rest/datasets/identifiers: + /datasets/identifiers: get: summary: List all dataset identifiers parameters: @@ -1139,7 +1139,7 @@ paths: description: Successful operation, returns a list of all dataset metadata version identifiers. tags: - Dataset API - /rest/datasets/unique_preferred_identifiers: + /datasets/unique_preferred_identifiers: get: summary: List all unique dataset preferred identifiers parameters: @@ -1173,7 +1173,7 @@ paths: description: Successful operation, returns a list of all unique dataset preferred identifiers. tags: - Dataset API - /rest/datasets/list: + /datasets/list: post: summary: List datasets corresponding IDs or identifiers given in body parameters: @@ -1193,7 +1193,7 @@ paths: description: Forbidden. Must have permission for resource tags: - Dataset API - /rest/datasets/{PID}: + /datasets/{PID}: get: summary: Get dataset metadata parameters: @@ -1341,7 +1341,7 @@ paths: description: Not found tags: - Dataset API - /rest/datasets/{PID}/files: + /datasets/{PID}/files: get: summary: Get list of files in a dataset parameters: @@ -1365,7 +1365,7 @@ paths: description: Successful operation, return list of full files metadata tags: - Dataset API - /rest/datasets/{PID}/metadata_versions: + /datasets/{PID}/metadata_versions: get: summary: List old research_dataset entries of a record parameters: @@ -1381,7 +1381,7 @@ paths: $ref: '#/definitions/StringList' tags: - Dataset API - /rest/datasets/{PID}/metadata_versions/{MVI}: + /datasets/{PID}/metadata_versions/{MVI}: get: summary: Get contents of a specific old research_dataset of a record. parameters: @@ -1405,7 +1405,7 @@ paths: # Contract API - /rest/contracts: + /contracts: get: summary: List contracts parameters: @@ -1452,7 +1452,7 @@ paths: description: Forbidden. Must have permission for resource tags: - Contract API - /rest/contracts/{PID}: + /contracts/{PID}: get: summary: Get contract metadata parameters: @@ -1529,7 +1529,7 @@ paths: # ApiErrors API - /rest/apierrors: + /apierrors: get: summary: List errors produced during api requests responses: @@ -1539,7 +1539,7 @@ paths: description: Forbidden. Must have permission for resource tags: - ApiErrors API - /rest/apierrors/{identifier}: + /apierrors/{identifier}: get: summary: Get details of a single error entry parameters: @@ -1574,7 +1574,7 @@ paths: description: Not found tags: - ApiErrors API - /rest/apierrors/flush: + /apierrors/flush: post: summary: "Delete all error entries" responses: @@ -1587,7 +1587,7 @@ paths: # Schema API - /rest/schemas: + /schemas: get: summary: "List schemas" responses: @@ -1595,7 +1595,7 @@ paths: description: Successful operation, return list of schema names tags: - Schema API - /rest/schemas/{name}: + /schemas/{name}: get: summary: Get schema content parameters: From 8bc8ecd053178601626562bd50a8e1e713fc4194 Mon Sep 17 00:00:00 2001 From: Tommi Pulli Date: Thu, 18 Mar 2021 11:52:13 +0200 Subject: [PATCH 25/34] add CI pipeline for static environments and merge requests --- .gitlab-ci.yml | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..0cfd40fb --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,54 @@ +workflow: + rules: + - if: '$CI_COMMIT_BRANCH =~ /^(demo|stable|staging|test)$/' + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + +stages: + - deploy + - test + - clean_test + - update + - clean_build + +deploy: + stage: deploy + environment: $CI_COMMIT_REF_NAME + script: + - ansible-playbook -i $ANSIBLE_INVENTORY $DEPLOY_PLAYBOOK -e "build_id=$CI_COMMIT_SHORT_SHA repo_version=$CI_COMMIT_REF_NAME" + +integration_test: + stage: test + environment: $CI_COMMIT_REF_NAME + script: + - ansible-playbook -i $ANSIBLE_INVENTORY $TEST_PLAYBOOK -e "build_id=$CI_COMMIT_SHORT_SHA" + +clean_test: + stage: clean_test + environment: + name: $CI_COMMIT_REF_NAME + script: + - ansible-playbook -i $ANSIBLE_INVENTORY $DELETE_PLAYBOOK -e "build_id=$CI_COMMIT_SHORT_SHA" + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: always + - when: on_failure + +update_proxy: + stage: update + environment: $CI_COMMIT_REF_NAME + script: + - ansible-playbook -i $ANSIBLE_INVENTORY $UPDATE_PROXY_PLAYBOOK -e "build_id=$CI_COMMIT_SHORT_SHA" + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: never + - when: always + +clean_previous_build: + stage: clean_build + environment: $CI_COMMIT_REF_NAME + script: + - ansible-playbook -i $ANSIBLE_INVENTORY $DELETE_PLAYBOOK -e "build_id=${CI_COMMIT_BEFORE_SHA:0:8}" + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: never + - when: always From 1bd39a9ddcb5b483d7b8a746fdda8961821bb4f2 Mon Sep 17 00:00:00 2001 From: Tommi Pulli Date: Thu, 18 Mar 2021 11:53:45 +0200 Subject: [PATCH 26/34] clean Gitlab envs on merge requests --- .gitlab-ci.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0cfd40fb..da855842 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,6 +7,7 @@ stages: - deploy - test - clean_test + - clean_env - update - clean_build @@ -26,6 +27,7 @@ clean_test: stage: clean_test environment: name: $CI_COMMIT_REF_NAME + on_stop: clean_gitlab_env script: - ansible-playbook -i $ANSIBLE_INVENTORY $DELETE_PLAYBOOK -e "build_id=$CI_COMMIT_SHORT_SHA" rules: @@ -33,6 +35,20 @@ clean_test: when: always - when: on_failure +clean_gitlab_env: + stage: clean_env + variables: + GIT_STRATEGY: none + environment: + name: $CI_COMMIT_REF_NAME + action: stop + script: + - echo "Cleaning deleted branches from environments" + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: manual + - when: never + update_proxy: stage: update environment: $CI_COMMIT_REF_NAME From 0752d8426b1f9c4abddbae30889a0e046545a7c6 Mon Sep 17 00:00:00 2001 From: Tommi Pulli Date: Fri, 19 Mar 2021 09:12:49 +0200 Subject: [PATCH 27/34] Revert "Remove unnecessary /rest prefix from swagger endpoints" This reverts commit 5b868a8db07a1f92373007fdffc7083f2d02c7f9. --- swagger/v1/swagger.yaml | 56 ++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/swagger/v1/swagger.yaml b/swagger/v1/swagger.yaml index e1a8aca9..ec997116 100755 --- a/swagger/v1/swagger.yaml +++ b/swagger/v1/swagger.yaml @@ -11,7 +11,7 @@ produces: paths: # File Storage API - /filestorages: + /rest/filestorages: get: summary: List File Storages parameters: @@ -43,7 +43,7 @@ paths: description: Forbidden. Must have permission for resource tags: - File Storage API - /filestorages/{PID}: + /rest/filestorages/{PID}: get: summary: Get File Storage metadata parameters: @@ -125,7 +125,7 @@ paths: # File API - /files: + /rest/files: get: summary: Get list of files parameters: @@ -264,7 +264,7 @@ paths: description: None of the provided identifiers were found. tags: - File API - /files/{PID}: + /rest/files/{PID}: get: summary: Get file metadata parameters: @@ -363,7 +363,7 @@ paths: description: Object not found tags: - File API - /files/{PID}/XML: + /rest/files/{PID}/XML: get: summary: Get XML metadata from file produces: @@ -474,7 +474,7 @@ paths: description: File or namespace does not exist tags: - File API - /files/datasets: + /rest/files/datasets: post: summary: Get datasets where files belong to description: | @@ -521,7 +521,7 @@ paths: description: Forbidden. Must have permission for resource tags: - File API - /files/restore: + /rest/files/restore: post: summary: Restore removed files back to "not removed" state. description: | @@ -545,7 +545,7 @@ paths: description: Forbidden. Must have permission for resource tags: - File API - /directories/{PID}: + /rest/directories/{PID}: get: summary: Get details of a directory description: Does not contain the directory's files and sub-directories. For that, use /rest/directories/{PID}/files @@ -565,7 +565,7 @@ paths: description: Not found. Also when not authenticated request tags: - File API - /directories/{PID}/files: + /rest/directories/{PID}/files: get: summary: Get list of files and directories contained by a directory parameters: @@ -661,7 +661,7 @@ paths: description: Directory not found. Also when not authenticated request tags: - File API - /directories/files: + /rest/directories/files: get: summary: Get list of files and directories contained by a directory, queried by dir path and project description: | @@ -733,7 +733,7 @@ paths: description: Directory not found. Also when not authenticated request tags: - File API - /directories/root: + /rest/directories/root: get: summary: Return root directory for a project, and its files and directories description: Useful when starting to browse files for a project, when individual root-level directory identifier is not yet known. @@ -757,7 +757,7 @@ paths: # Data Catalog API - /datacatalogs: + /rest/datacatalogs: get: summary: List of data catalogs parameters: @@ -795,7 +795,7 @@ paths: description: Forbidden. Must have permission for resource tags: - Data Catalog API - /datacatalogs/{PID}: + /rest/datacatalogs/{PID}: get: summary: Get data catalog metadata parameters: @@ -895,7 +895,7 @@ paths: # Dataset API - /datasets: + /rest/datasets: # some of the parameters and returned fields are for TPAS usage only get: summary: List datasets @@ -1105,7 +1105,7 @@ paths: description: None of the provided identifiers were found. tags: - Dataset API - /datasets/identifiers: + /rest/datasets/identifiers: get: summary: List all dataset identifiers parameters: @@ -1139,7 +1139,7 @@ paths: description: Successful operation, returns a list of all dataset metadata version identifiers. tags: - Dataset API - /datasets/unique_preferred_identifiers: + /rest/datasets/unique_preferred_identifiers: get: summary: List all unique dataset preferred identifiers parameters: @@ -1173,7 +1173,7 @@ paths: description: Successful operation, returns a list of all unique dataset preferred identifiers. tags: - Dataset API - /datasets/list: + /rest/datasets/list: post: summary: List datasets corresponding IDs or identifiers given in body parameters: @@ -1193,7 +1193,7 @@ paths: description: Forbidden. Must have permission for resource tags: - Dataset API - /datasets/{PID}: + /rest/datasets/{PID}: get: summary: Get dataset metadata parameters: @@ -1341,7 +1341,7 @@ paths: description: Not found tags: - Dataset API - /datasets/{PID}/files: + /rest/datasets/{PID}/files: get: summary: Get list of files in a dataset parameters: @@ -1365,7 +1365,7 @@ paths: description: Successful operation, return list of full files metadata tags: - Dataset API - /datasets/{PID}/metadata_versions: + /rest/datasets/{PID}/metadata_versions: get: summary: List old research_dataset entries of a record parameters: @@ -1381,7 +1381,7 @@ paths: $ref: '#/definitions/StringList' tags: - Dataset API - /datasets/{PID}/metadata_versions/{MVI}: + /rest/datasets/{PID}/metadata_versions/{MVI}: get: summary: Get contents of a specific old research_dataset of a record. parameters: @@ -1405,7 +1405,7 @@ paths: # Contract API - /contracts: + /rest/contracts: get: summary: List contracts parameters: @@ -1452,7 +1452,7 @@ paths: description: Forbidden. Must have permission for resource tags: - Contract API - /contracts/{PID}: + /rest/contracts/{PID}: get: summary: Get contract metadata parameters: @@ -1529,7 +1529,7 @@ paths: # ApiErrors API - /apierrors: + /rest/apierrors: get: summary: List errors produced during api requests responses: @@ -1539,7 +1539,7 @@ paths: description: Forbidden. Must have permission for resource tags: - ApiErrors API - /apierrors/{identifier}: + /rest/apierrors/{identifier}: get: summary: Get details of a single error entry parameters: @@ -1574,7 +1574,7 @@ paths: description: Not found tags: - ApiErrors API - /apierrors/flush: + /rest/apierrors/flush: post: summary: "Delete all error entries" responses: @@ -1587,7 +1587,7 @@ paths: # Schema API - /schemas: + /rest/schemas: get: summary: "List schemas" responses: @@ -1595,7 +1595,7 @@ paths: description: Successful operation, return list of schema names tags: - Schema API - /schemas/{name}: + /rest/schemas/{name}: get: summary: Get schema content parameters: From 2d32b934a1ea93d6f7d57be77c312a63cd2c4a3d Mon Sep 17 00:00:00 2001 From: Toni Nurmi Date: Tue, 30 Mar 2021 13:51:13 +0000 Subject: [PATCH 28/34] Dynamic docker config --- Dockerfile | 5 +- README.md | 117 +-------- config-swap-stack.yml | 55 ++++ containers/nginx-docker.yml | 42 +++ docker-compose.yml | 72 ++---- docs/docker-prerequisites.md | 30 +++ docs/docker-stack.md | 53 ++++ docs/local_ssl_setup.md | 13 + docs/manage-commands.md | 33 +++ docs/single-docker-images.md | 37 +++ requirements.in | 5 +- requirements.txt | 258 ++++++++++++++----- src/metax_api/settings/environments/local.py | 19 +- src/metax_api/urls.py | 5 +- 14 files changed, 512 insertions(+), 232 deletions(-) create mode 100644 config-swap-stack.yml create mode 100644 containers/nginx-docker.yml create mode 100644 docs/docker-prerequisites.md create mode 100644 docs/docker-stack.md create mode 100644 docs/local_ssl_setup.md create mode 100644 docs/manage-commands.md create mode 100644 docs/single-docker-images.md diff --git a/Dockerfile b/Dockerfile index 5baf63ed..6f23920d 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.6 +FROM python:3.8 ENV PYTHONUNBUFFERED 1 ENV PYTHONDONTWRITEBYTECODE 1 @@ -17,5 +17,4 @@ RUN pip install -r requirements.txt EXPOSE 8008 EXPOSE 8006 -# CMD ["python", "/code/manage.py", "runserver", "0.0.0.0:8008"] -CMD ["python", "manage.py", "runsslserver", "--certificate", ".certs/cert.pem","--key", ".certs/key.pem", "0.0.0.0:8008"] \ No newline at end of file +CMD ["python", "manage.py", "runserver", "0.0.0.0:8008"] \ No newline at end of file diff --git a/README.md b/README.md index 668edf26..cd781101 100755 --- a/README.md +++ b/README.md @@ -1,15 +1,7 @@ This repository contains the code for Metax API service. -# Build status +## License -## Test branch -[![Build Status](https://travis-ci.com/CSCfi/metax-api.svg?branch=test)](https://travis-ci.com/CSCfi/metax-api) - -## Stable branch -[![Build Status](https://travis-ci.com/CSCfi/metax-api.svg?branch=stable)](https://travis-ci.com/CSCfi/metax-api) - -License -------- Copyright (c) 2018-2020 Ministry of Education and Culture, Finland Licensed under [GNU GPLv2 License](LICENSE) @@ -17,65 +9,7 @@ Licensed under [GNU GPLv2 License](LICENSE) ## Setting up local development environment -### Prerequisites - -#### Docker-Engine - -Install Docker-Engine either following instructions below or looking up your platform specific instructions [from docs.docker.com][1] - -##### Linux - -`$ curl -fsSL https://get.docker.com -o get-docker.sh` - -`$ sudo sh get-docker.sh` - -`$ sudo usermod -aG docker $USER` - -Log out and back in to activate non-sudo docker capabilities - -##### Mac - -https://docs.docker.com/docker-for-mac/install/ - -#### Portainer (Optional) - -We will use portainer container management tool for monitoring various development dependencies. Command below will start portainer on every system startup. - -`$ docker volume create portainer_data` (optional for mac) - -`$ docker run -d -p 8000:8000 -p 9000:9000 --name=portainer --restart=always -v /var/run/docker.sock:/var/run/docker.sock -v portainer_data:/data portainer/portainer-ce` - -Finish the Portainer setup by logging in at http://localhost:9000, create a local endpoint from the Portainer interface. - -#### Docker commands - -__NOTICE If you want to start the services everytime your computer boots, replace `--restart=unless-stopped` with `--restart=always`__ - -Run the following docker commands to start services: - -##### Redis - -`docker run -d -p 6379:6379 --name metax-redis -v metax-redis:/data --restart=unless-stopped redis` - -##### Postgres - -`docker run -d -p 5432:5432 --name metax-postgres -v metax-postgres:/var/lib/postgresql96/data -e POSTGRES_USER=metax_db_user -e POSTGRES_PASSWORD=YMDLekQMqrVKcs3 -e POSTGRES_DB=metax_db --restart=unless-stopped postgres:9` - -__NOTICE: copy values of `POSTGRES_USER`, `POSTGRES_PASSWORD` and `POSTGRES_DB` into your `.env` files as `METAX_DATABASE_USER`, `METAX_DATABASE_PASSWORD` and `METAX_DATABASE`__ - -##### Elasticsearch - -`docker run -d -p 9200:9200 -p 9300:9300 -v metax-es:/usr/share/elasticsearch/data --name metax-es -e discovery.type=single-node --restart=unless-stopped elasticsearch:7.9.2` - -##### RabbitMQ - -`docker run -d -p 5671:5671 -p 5672:5672 -p 15672:15672 -v metax-rabbitmq:/var/lib/rabbitmq --name metax-rabbitmq --restart=unless-stopped rabbitmq:3-management` - -#### mkcerts - -Install [mkcerts][2] and run `mkcert -install` and after it the following command: -`mkcert -cert-file cert.pem -key-file key.pem 0.0.0.0 localhost 127.0.0.1 ::1 metax.csc.local 20.20.20.20` -Move the `cert.pem` and `key.pem` to `src/.certs` folder (create the folder if not present). +You can also set up the development environment with [Docker-swarm setup](/docs/docker-stack.md) or with [standalone Docker-containers setup](/docs/single-docker-images.md). ### Required environmental variables @@ -83,46 +17,21 @@ copy `src/metax_api/settings/.env.template` as `src/metax_api/settings/.env` and ### Create log directory -`mkdir -p src/log/errors` - -### Run Metax inside a container (Optional) - -Check the IP addresses of Redis, RabbitMQ, ElasticSearch and Postgres:9 either from Portainer container list (click the link in the container name to see all attributes) or by going to portainer network tab or by typing `docker container ps` followed by `docker network inspect bridge` - -Build new docker image from repository root with this command (change ip-addresses to real ones: - -`docker build -t metax-api:latest .` - -Run the built container with command: - -`docker run -it --name metax-web --mount type=bind,source="$(pwd)"/src,target=/code -p 8008:8008 --rm metax-api:latest` - -Run Metax with Docker Stack: -`docker stack deploy -c docker-compose.yml metax-dev` - - -You should see metax-server starting at port 8008 with hot reload enabled +`mkdir -p /var/log/metax-api/errors` ### Initial setup commands -docker network inspect bridge -IF you configured metax-container, access the command line of the container with `docker exec -it metax-web bash` -__NOTICE: Skip activating virtualenv and navigating to src folder if you have metax running on container__ +Activate your python 3.8 virtualenv, install requirements with `pip install -r requirements.txt` -Activate your python 3.6 virtualenv, `cd` into `src` folder and run following commands: - -setup the database with migrate command: +`cd` into `src` folder and run following commands: `python manage.py migrate` -__NOTICE: Skip following steps if your running metax on container and have terminal open in the container__ - start the development server with: -`python manage.py runsslserver --certificate .certs/cert.pem --key .certs/key.pem 8008` -Open another terminal and `cd` into `src`, and load the initial data with following commands: +`python manage.py runserver 8008` -__These commands must be run in both setups (container/not-container metax)__ +Open another terminal and `cd` into `src`, and load the initial data with following commands: `python manage.py index_refdata` @@ -130,13 +39,15 @@ __These commands must be run in both setups (container/not-container metax)__ `python manage.py loadinitialdata` -`python manage.py loaddata metax_api/tests/testdata/test_data.json` +`python manage.py loaddata metax_api/tests/testdata/test_data.json` + +Metax api is available from your browser at http://localhost:8008 + +## Running tests + +run the tests with command `DJANGO_ENV=test python manage.py test --parallel --failfast --keepdb -v 0` -run the tests with command `DJANGO_ENV=test python manage.py test --failfast --keepdb -v 0` -Metax api is available from your browser at https://localhost:8008 -[1]: https://docs.docker.com/engine/install/ -[2]: https://github.com/FiloSottile/mkcert diff --git a/config-swap-stack.yml b/config-swap-stack.yml new file mode 100644 index 00000000..590672c6 --- /dev/null +++ b/config-swap-stack.yml @@ -0,0 +1,55 @@ +version: "3.8" + +services: + metax: + image: fairdata-docker.artifactory.ci.csc.fi/fairdata-metax-web + ports: + - 8008:8008 + - 8000:8000 + volumes: + - ./src:/code + environment: + DEBUG: 'true' + METAX_DATABASE: 'metax_db_test' + METAX_DATABASE_PASSWORD: 'YMDLekQMqrVKcs3' + METAX_DATABASE_USER: 'metax_test' + REDIS_HOST: metax-redis + RABBIT_MQ_HOSTS: metax-rabbitmq + ELASTIC_SEARCH_HOSTS: metax-elasticsearch + METAX_DATABASE_HOST: metax-db + + metax-redis: + image: redis + volumes: + - metax-redis:/data + metax-db: + image: postgres:9 + environment: + POSTGRES_USER: 'metax_test' + POSTGRES_PASSWORD: 'YMDLekQMqrVKcs3' + POSTGRES_DB: 'metax_db_test' + volumes: + - metax-postgres:/var/lib/postgresql/data + metax-elasticsearch: + image: elasticsearch:7.9.2 + environment: + discovery.type: 'single-node' + volumes: + - metax-es:/usr/share/elasticsearch/data + metax-rabbitmq: + image: rabbitmq:3-management + volumes: + - metax-rabbitmq:/var/lib/rabbitmq + +volumes: + metax-rabbitmq: + external: true + metax-es: + external: true + metax-postgres: + external: true + metax-redis: + external: true + + + diff --git a/containers/nginx-docker.yml b/containers/nginx-docker.yml new file mode 100644 index 00000000..602fc888 --- /dev/null +++ b/containers/nginx-docker.yml @@ -0,0 +1,42 @@ +version: "3.8" + +services: + metax-nginx: + image: nginx:latest + configs: + - source: metax-nginx-config + target: '/etc/nginx/nginx.conf' + - source: fairdata-ssl-certificate + target: '/etc/nginx/ssl_certs/fd-dev.csc.fi.crt.pem' + - source: fairdata-ssl-certificate-key + target: '/etc/nginx/ssl_certs/fd-dev.csc.fi.key.pem' + - source: metax-nginx-elastic-headers-config + target: '/etc/nginx/elastic_headers.conf' + - source: metax-nginx-shared-headers-config + target: '/etc/nginx/shared_headers.conf' + - source: metax-nginx-api-response-headers-config + target: '/etc/nginx/api_response_headers.conf' + - source: metax-nginx-static-file-headers-config + target: '/etc/nginx/static_file_headers.conf' + - source: metax-nginx-dh-param-config + target: '/etc/nginx/ssl_certs/nginx_dhparam.pem' + ports: + - 443:443 + +configs: + fairdata-ssl-certificate: + external: True + fairdata-ssl-certificate-key: + external: True + metax-nginx-config: + external: True + metax-nginx-elastic-headers-config: + external: True + metax-nginx-shared-headers-config: + external: True + metax-nginx-api-response-headers-config: + external: True + metax-nginx-static-file-headers-config: + external: True + metax-nginx-dh-param-config: + external: True \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 9e8a40df..76fc8108 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,82 +13,50 @@ services: METAX_DATABASE: 'metax_db_test' METAX_DATABASE_PASSWORD: 'YMDLekQMqrVKcs3' METAX_DATABASE_USER: 'metax_test' - REDIS_HOST: redis - RABBIT_MQ_HOSTS: rabbitmq - ELASTIC_SEARCH_HOSTS: elasticsearch - METAX_DATABASE_HOST: db + REDIS_HOST: metax-redis + RABBIT_MQ_HOSTS: metax-rabbitmq + ELASTIC_SEARCH_HOSTS: metax-elasticsearch + METAX_DATABASE_HOST: metax-db configs: - source: metax-web-config target: '/code/metax_api/settings/.env' - redis: + + metax-redis: image: redis volumes: - metax-redis:/data - db: + + metax-db: image: postgres:9 environment: POSTGRES_USER: 'metax_test' POSTGRES_PASSWORD: 'YMDLekQMqrVKcs3' POSTGRES_DB: 'metax_db_test' volumes: - - metax-postgres:/var/lib/postgresql96/data - elasticsearch: + - metax-postgres:/var/lib/postgresql/data + + metax-elasticsearch: image: elasticsearch:7.9.2 environment: discovery.type: 'single-node' volumes: - metax-es:/usr/share/elasticsearch/data - rabbitmq: + + metax-rabbitmq: image: rabbitmq:3-management volumes: - metax-rabbitmq:/var/lib/rabbitmq - nginx: - image: nginx:latest - configs: - - source: metax-nginx-config - target: '/etc/nginx/nginx.conf' - - source: fairdata-ssl-certificate - target: '/etc/nginx/ssl_certs/fd-test.csc.fi.crt.pem' - - source: fairdata-ssl-certificate-key - target: '/etc/nginx/ssl_certs/fd-test.csc.fi.key.pem' - - source: metax-nginx-elastic-headers-config - target: '/etc/nginx/elastic_headers.conf' - - source: metax-nginx-shared-headers-config - target: '/etc/nginx/shared_headers.conf' - - source: metax-nginx-api-response-headers-config - target: '/etc/nginx/api_response_headers.conf' - - source: metax-nginx-static-file-headers-config - target: '/etc/nginx/static_file_headers.conf' - - source: metax-nginx-dh-param-config - target: '/etc/nginx/ssl_certs/nginx_dhparam.pem' - ports: - - 4433:443 - -configs: - metax-web-config: - external: True - fairdata-ssl-certificate: - external: True - fairdata-ssl-certificate-key: - external: True - metax-nginx-config: - external: True - metax-nginx-elastic-headers-config: - external: True - metax-nginx-shared-headers-config: - external: True - metax-nginx-api-response-headers-config: - external: True - metax-nginx-static-file-headers-config: - external: True - metax-nginx-dh-param-config: - external: True volumes: metax-rabbitmq: + external: true metax-es: + external: true metax-postgres: + external: true metax-redis: - - + external: true +configs: + metax-web-config: + external: True diff --git a/docs/docker-prerequisites.md b/docs/docker-prerequisites.md new file mode 100644 index 00000000..3340f706 --- /dev/null +++ b/docs/docker-prerequisites.md @@ -0,0 +1,30 @@ + +# Docker setup prerequisites + +## Docker-Engine + +Install Docker-Engine either following instructions below or looking up your platform specific instructions [from docs.docker.com][1] + +### Linux + +`$ curl -fsSL https://get.docker.com -o get-docker.sh` + +`$ sudo sh get-docker.sh` + +`$ sudo usermod -aG docker $USER` + +Log out and back in to activate non-sudo docker capabilities + +### Mac + +https://docs.docker.com/docker-for-mac/install/ + +## Portainer (Optional) + +You can use portainer container management tool for monitoring various development dependencies. Command below will start portainer on every system startup. + +`$ docker run -d -p 8000:8000 -p 9000:9000 --name=portainer --restart=always -v /var/run/docker.sock:/var/run/docker.sock -v portainer_data:/data portainer/portainer-ce` + +Finish the Portainer setup by logging in at http://localhost:9000, create a local endpoint from the Portainer interface. + +[1]: https://docs.docker.com/engine/install/ \ No newline at end of file diff --git a/docs/docker-stack.md b/docs/docker-stack.md new file mode 100644 index 00000000..889bc898 --- /dev/null +++ b/docs/docker-stack.md @@ -0,0 +1,53 @@ +# Local development with Docker-swarm + +## Building metax-image + +After installing [Docker prerequisites](docker-prerequisites.md), build the metax-web docker image with the following command: + +`docker build -t fairdata-docker.artifactory.ci.csc.fi/fairdata-metax-web .` + +## Building httpd-image + +`docker build -t fairdata-docker.artifactory.ci.csc.fi/fairdata-metax-httpd -f containers/apache-image.Dockerfile .` + +## Pushing metax-image to Artifactory + + `docker push fairdata-docker.artifactory.ci.csc.fi/fairdata-metax-web` + +## Pushing httpd-image to Artifactory + +`docker push fairdata-docker.artifactory.ci.csc.fi/fairdata-metax-httpd` + + +## Running the stack locally + +In the repository root, run + +`docker stack deploy -c docker-compose.yml --resolve-image=always --with-registry-auth metax-dev` + +## Running the stack without predefined docker-configs + +`docker stack deploy -c config-swap-stack.yml --resolve-image=always --with-registry-auth metax-dev` + +## Adding nginx to the stack + +`docker stack deploy -c docker-compose.yml -c containers/nginx-docker.yml --resolve-image=always --with-registry-auth metax-dev` + +## Running all services + +`docker stack deploy --resolve-image=always --with-registry-auth -c docker-compose.yml -c containers/nginx-docker.yml -c containers/apache-docker.yml metax-dev` + +## Running Metax management commands + +To run Metax management commands, locate the running metax-dev_metax container and open terminal inside it with: + +`docker exec -it bash` + +## Adding docker-config to the stack + +`docker service update --config-add source=metax-web-stable-config,target=/code/metax_api/settings/.env metax-dev_metax` + +## Swapping docker-config in the stack + +`docker service update --config-rm --config-add source=,target=/code/metax_api/settings/.env metax-dev_metax` + diff --git a/docs/local_ssl_setup.md b/docs/local_ssl_setup.md new file mode 100644 index 00000000..5ed07119 --- /dev/null +++ b/docs/local_ssl_setup.md @@ -0,0 +1,13 @@ +# Install ssl cert for local https testing + +# Setup + +Install [mkcerts][2] and run `mkcert -install` and after it the following command: +`mkcert -cert-file cert.pem -key-file key.pem 0.0.0.0 localhost 127.0.0.1 ::1 metax.csc.local 20.20.20.20` +Move the `cert.pem` and `key.pem` to `src/.certs` folder (create the folder if not present). + +## Run SSL enabled development server + +`python manage.py runsslserver --certificate .certs/cert.pem --key .certs/key.pem 8008` + +[2]: https://github.com/FiloSottile/mkcert \ No newline at end of file diff --git a/docs/manage-commands.md b/docs/manage-commands.md new file mode 100644 index 00000000..39ed3160 --- /dev/null +++ b/docs/manage-commands.md @@ -0,0 +1,33 @@ +# Metax management commands + +## Create and migrate database + +`python manage.py migrate` + +## Index the reference data + +`python manage.py index_refdata` + +## Reload reference data to redis cache + +`python manage.py reload_refdata_cache` + +## Add necessary initial data to database + +`python manage.py loadinitialdata` + +## Add some test datasets to database + +`python manage.py loaddata metax_api/tests/testdata/test_data.json` + +## Run all tests + +`DJANGO_ENV=test python manage.py test --parallel --failfast --keepdb -v 0` + +## Inspect current application settings + +`python manage.py diffsettings --output unified --force-color` + +## Execute management commands against docker swarm metax-api container + +`docker exec $(docker ps -q -f name=metax-dev_metax) python manage.py check` diff --git a/docs/single-docker-images.md b/docs/single-docker-images.md new file mode 100644 index 00000000..7d3b0f00 --- /dev/null +++ b/docs/single-docker-images.md @@ -0,0 +1,37 @@ +# Docker setup without swarm + +__NOTICE If you want to start the services everytime your computer boots, replace `--restart=unless-stopped` with `--restart=always`__ + +After installing [Docker prerequisites](docker-prerequisites.md),run the following docker commands to start services: + +## Redis + +`docker run -d -p 6379:6379 --name metax-redis -v metax-redis:/data --restart=unless-stopped redis` + +## Postgres + +`docker run -d -p 5432:5432 --name metax-postgres -v metax-postgres:/var/lib/postgresql96/data -e POSTGRES_USER=metax_db_user -e POSTGRES_PASSWORD=YMDLekQMqrVKcs3 -e POSTGRES_DB=metax_db --restart=unless-stopped postgres:9` + +__NOTICE: copy values of `POSTGRES_USER`, `POSTGRES_PASSWORD` and `POSTGRES_DB` into your `.env` files as `METAX_DATABASE_USER`, `METAX_DATABASE_PASSWORD` and `METAX_DATABASE`__ + +## Elasticsearch + +`docker run -d -p 9200:9200 -p 9300:9300 -v metax-es:/usr/share/elasticsearch/data --name metax-es -e discovery.type=single-node --restart=unless-stopped elasticsearch:7.9.2` + +## RabbitMQ + +`docker run -d -p 5671:5671 -p 5672:5672 -p 15672:15672 -v metax-rabbitmq:/var/lib/rabbitmq --name metax-rabbitmq --restart=unless-stopped rabbitmq:3-management` + +## Metax +Check the IP addresses of Redis, RabbitMQ, ElasticSearch and Postgres:9 either from Portainer container list (click the link in the container name to see all attributes) or by going to portainer network tab or by typing `docker container ps` followed by `docker network inspect bridge` + +Build new docker image from repository root with this command + +`docker build -t fairdata-metax-web:latest .` + +Run the built container with command: + +`docker run -it --name fairdata-metax-web --mount type=bind,source="$(pwd)"/src,target=/code -p 8008:8008 --rm -e METAX_DATABASE_USER= -e METAX_DATABASE_PASSWORD= -e METAX_DATABASE= -e REDIS_HOST= -e RABBIT_MQ_HOSTS= -e ELASTIC_SEARCH_HOSTS= -e METAX_DATABASE_HOST= fairdata-metax-web:latest` + +## Metax management commands +access the command line of the container with `docker exec -it metax-web bash` diff --git a/requirements.in b/requirements.in index 4326b1cb..006d948f 100755 --- a/requirements.in +++ b/requirements.in @@ -29,12 +29,13 @@ xmltodict # MIT-license pip-tools # keeping dependencies updated traitlets>=4.3.2,<5.0.0 # via ipython, version 5.0+ is Python 3.7+ only xmltodict # MIT-license -pip-tools # keeping dependencies updated django-environ django-split-settings icecream rdflib django-sslserver +tblib django-watchman - +django-debug-toolbar +pip-tools>6 diff --git a/requirements.txt b/requirements.txt index 235ab8ee..999147c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,76 +4,194 @@ # # pip-compile # -asgiref==3.2.10 # via django -asttokens==2.0.4 # via icecream -attrs==20.3.0 # via jsonschema -backcall==0.2.0 # via ipython -certifi==2020.6.20 # via elasticsearch, requests -chardet==3.0.4 # via requests -click==7.1.2 # via pip-tools -colorama==0.4.4 # via icecream -coverage==5.3 # via coveralls -coveralls==2.1.2 # via -r requirements.in -datacite==1.0.1 # via -r requirements.in -decorator==4.4.2 # via ipython, traitlets -django-environ==0.4.5 # via -r requirements.in -django-rainbowtests==0.6.0 # via -r requirements.in -django-split-settings==1.0.1 # via -r requirements.in -django-sslserver==0.22 # via -r requirements.in -django-watchman==1.2.0 # via -r requirements.in -djangorestframework==3.12.2 # via -r requirements.in -django==3.1.4 # via -r requirements.in, django-rainbowtests, django-sslserver, django-watchman, djangorestframework -docopt==0.6.2 # via coveralls -elasticsearch==7.10.0 # via -r requirements.in -executing==0.5.3 # via icecream -flake8==3.8.4 # via -r requirements.in -gevent==20.9.0 # via -r requirements.in -greenlet==0.4.17 # via gevent -gunicorn==20.0.4 # via -r requirements.in -hiredis==1.1.0 # via -r requirements.in -icecream==2.0.0 # via -r requirements.in -idna==2.10 # via requests -ipdb==0.13.4 # via -r requirements.in -ipython-genutils==0.2.0 # via traitlets -ipython==7.16.1 # via ipdb -isodate==0.6.0 # via rdflib -jedi==0.17.2 # via ipython -jsonschema==3.2.0 # via -r requirements.in, datacite -lxml==4.6.2 # via -r requirements.in, datacite, pyoai -mccabe==0.6.1 # via flake8 -parso==0.7.1 # via jedi -pexpect==4.8.0 # via ipython -pickleshare==0.7.5 # via ipython -pika==1.1.0 # via -r requirements.in -pip-tools==5.3.1 # via -r requirements.in -prompt-toolkit==3.0.8 # via ipython -psycopg2-binary==2.8.6 # via -r requirements.in -ptyprocess==0.6.0 # via pexpect -pycodestyle==2.6.0 # via flake8 -pyflakes==2.2.0 # via flake8 -pygments==2.7.2 # via icecream, ipython -pyjwt==1.7.1 # via -r requirements.in -pyoai==2.5.0 # via -r requirements.in -pyparsing==2.4.7 # via rdflib -pyrsistent==0.17.3 # via jsonschema -python-dateutil==2.8.1 # via -r requirements.in -python-simplexquery==1.0.5.3 # via -r requirements.in -pytz==2020.5 # via -r requirements.in, django -pyyaml==5.3.1 # via -r requirements.in -rdflib==5.0.0 # via -r requirements.in -redis==3.5.3 # via -r requirements.in -requests==2.25.0 # via -r requirements.in, coveralls, datacite, responses -responses==0.12.1 # via -r requirements.in -simplejson==3.17.2 # via -r requirements.in -six==1.15.0 # via asttokens, isodate, jsonschema, pip-tools, pyoai, python-dateutil, rdflib, responses, structlog, traitlets -sqlparse==0.4.1 # via django -structlog==20.1.0 # via -r requirements.in -traitlets==4.3.3 # via -r requirements.in, ipython -urllib3==1.25.11 # via -r requirements.in, elasticsearch, requests, responses -wcwidth==0.2.5 # via prompt-toolkit -xmltodict==0.12.0 # via -r requirements.in -zope.event==4.5.0 # via gevent -zope.interface==5.1.2 # via gevent +asgiref==3.2.10 + # via django +asttokens==2.0.4 + # via icecream +attrs==20.3.0 + # via jsonschema +backcall==0.2.0 + # via ipython +certifi==2020.6.20 + # via + # elasticsearch + # requests +chardet==3.0.4 + # via requests +click==7.1.2 + # via pip-tools +colorama==0.4.4 + # via icecream +coverage==5.3 + # via coveralls +coveralls==2.1.2 + # via -r requirements.in +datacite==1.0.1 + # via -r requirements.in +decorator==4.4.2 + # via + # ipython + # traitlets +django-debug-toolbar==3.2 + # via -r requirements.in +django-environ==0.4.5 + # via -r requirements.in +django-rainbowtests==0.6.0 + # via -r requirements.in +django-split-settings==1.0.1 + # via -r requirements.in +django-sslserver==0.22 + # via -r requirements.in +django-watchman==1.2.0 + # via -r requirements.in +django==3.1.4 + # via + # -r requirements.in + # django-debug-toolbar + # django-rainbowtests + # django-sslserver + # django-watchman + # djangorestframework +djangorestframework==3.12.2 + # via -r requirements.in +docopt==0.6.2 + # via coveralls +elasticsearch==7.10.0 + # via -r requirements.in +executing==0.5.3 + # via icecream +flake8==3.8.4 + # via -r requirements.in +gevent==20.9.0 + # via -r requirements.in +greenlet==0.4.17 + # via gevent +gunicorn==20.0.4 + # via -r requirements.in +hiredis==1.1.0 + # via -r requirements.in +icecream==2.0.0 + # via -r requirements.in +idna==2.10 + # via requests +ipdb==0.13.4 + # via -r requirements.in +ipython-genutils==0.2.0 + # via traitlets +ipython==7.16.1 + # via ipdb +isodate==0.6.0 + # via rdflib +jedi==0.17.2 + # via ipython +jsonschema==3.2.0 + # via + # -r requirements.in + # datacite +lxml==4.6.2 + # via + # -r requirements.in + # datacite + # pyoai +mccabe==0.6.1 + # via flake8 +parso==0.7.1 + # via jedi +pep517==0.10.0 + # via pip-tools +pexpect==4.8.0 + # via ipython +pickleshare==0.7.5 + # via ipython +pika==1.1.0 + # via -r requirements.in +pip-tools==6.0.1 + # via -r requirements.in +prompt-toolkit==3.0.8 + # via ipython +psycopg2-binary==2.8.6 + # via -r requirements.in +ptyprocess==0.6.0 + # via pexpect +pycodestyle==2.6.0 + # via flake8 +pyflakes==2.2.0 + # via flake8 +pygments==2.7.2 + # via + # icecream + # ipython +pyjwt==1.7.1 + # via -r requirements.in +pyoai==2.5.0 + # via -r requirements.in +pyparsing==2.4.7 + # via rdflib +pyrsistent==0.17.3 + # via jsonschema +python-dateutil==2.8.1 + # via -r requirements.in +python-simplexquery==1.0.5.3 + # via -r requirements.in +pytz==2020.5 + # via + # -r requirements.in + # django +pyyaml==5.3.1 + # via -r requirements.in +rdflib==5.0.0 + # via -r requirements.in +redis==3.5.3 + # via -r requirements.in +requests==2.25.0 + # via + # -r requirements.in + # coveralls + # datacite + # responses +responses==0.12.1 + # via -r requirements.in +simplejson==3.17.2 + # via -r requirements.in +six==1.15.0 + # via + # asttokens + # isodate + # jsonschema + # pyoai + # python-dateutil + # rdflib + # responses + # structlog + # traitlets +sqlparse==0.4.1 + # via + # django + # django-debug-toolbar +structlog==20.1.0 + # via -r requirements.in +tblib==1.7.0 + # via -r requirements.in +toml==0.10.2 + # via pep517 +traitlets==4.3.3 + # via + # -r requirements.in + # ipython +urllib3==1.25.11 + # via + # -r requirements.in + # elasticsearch + # requests + # responses +wcwidth==0.2.5 + # via prompt-toolkit +xmltodict==0.12.0 + # via -r requirements.in +zope.event==4.5.0 + # via gevent +zope.interface==5.1.2 + # via gevent # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/src/metax_api/settings/environments/local.py b/src/metax_api/settings/environments/local.py index 33d1d00f..ca3cf163 100755 --- a/src/metax_api/settings/environments/local.py +++ b/src/metax_api/settings/environments/local.py @@ -1,6 +1,6 @@ from watchman import constants as watchman_constants -from metax_api.settings.components.common import INSTALLED_APPS, ALLOWED_HOSTS +from metax_api.settings.components.common import INSTALLED_APPS, ALLOWED_HOSTS, MIDDLEWARE, DEBUG INSTALLED_APPS += ["watchman"] @@ -47,3 +47,20 @@ "vhost": "metax", }, ] +if 'debug_toolbar' not in INSTALLED_APPS: + INSTALLED_APPS += ['debug_toolbar'] +if 'debug_toolbar.middleware.DebugToolbarMiddleware' not in MIDDLEWARE: + MIDDLEWARE = ['debug_toolbar.middleware.DebugToolbarMiddleware'] + MIDDLEWARE +INTERNAL_IPS = [ + '127.0.0.1', + '0.0.0.0' +] +def show_toolbar(request): + if DEBUG: + return True + else: + return False +DEBUG_TOOLBAR_CONFIG = { + "SHOW_TOOLBAR_CALLBACK" : show_toolbar, +} + diff --git a/src/metax_api/urls.py b/src/metax_api/urls.py index 9fd05f99..30c1f749 100755 --- a/src/metax_api/urls.py +++ b/src/metax_api/urls.py @@ -22,7 +22,7 @@ """ from django.conf import settings as django_settings from django.conf.urls import url, include -from django.urls import re_path +from django.urls import re_path, path from metax_api.api.oaipmh.base.view import oaipmh_view as oaipmh from metax_api.api.rest.base.router import api_urlpatterns as rest_api_v1 @@ -30,6 +30,8 @@ from metax_api.api.rpc.base.router import api_urlpatterns as rpc_api_v1 from metax_api.api.rpc.v2.router import api_urlpatterns as rpc_api_v2 from metax_api.views.router import view_urlpatterns +import debug_toolbar + v1_urls = [ url('', include(view_urlpatterns)), @@ -55,4 +57,5 @@ urlpatterns += [ re_path(r'^watchman/', include('watchman.urls')), + path('__debug__/', include(debug_toolbar.urls)), ] From a189d64e7938ac132d099a381bd7c67bcf0eecbd Mon Sep 17 00:00:00 2001 From: Tommi Pulli Date: Mon, 29 Mar 2021 15:50:49 +0300 Subject: [PATCH 29/34] CSCFAIRMETA-988: [FIX] bug where drafts was published in RabbitMQ --- src/metax_api/models/catalog_record.py | 26 ++++++++++++----------- src/metax_api/models/catalog_record_v2.py | 10 +++------ src/metax_api/services/file_service.py | 2 +- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/metax_api/models/catalog_record.py b/src/metax_api/models/catalog_record.py index 36544d6f..3722c0ce 100755 --- a/src/metax_api/models/catalog_record.py +++ b/src/metax_api/models/catalog_record.py @@ -1267,8 +1267,7 @@ def _post_create_operations(self): super().save() - if self.state == self.STATE_PUBLISHED: - self.add_post_request_callable(RabbitMQPublishRecord(self, 'create')) + self.add_post_request_callable(RabbitMQPublishRecord(self, 'create')) _logger.info( 'Created a new Date: Wed, 31 Mar 2021 14:00:54 +0300 Subject: [PATCH 30/34] fix TTV load data script --- .../management/commands/load_data_to_TTV.py | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/src/metax_api/management/commands/load_data_to_TTV.py b/src/metax_api/management/commands/load_data_to_TTV.py index d5ad428c..874886aa 100644 --- a/src/metax_api/management/commands/load_data_to_TTV.py +++ b/src/metax_api/management/commands/load_data_to_TTV.py @@ -6,31 +6,42 @@ # :license: MIT import logging -import json from django.core.management.base import BaseCommand from metax_api.models import CatalogRecord from metax_api.services import RabbitMQService -from metax_api.api.rest.base.serializers import CatalogRecordSerializer logger = logging.getLogger(__name__) +# serializer needs these in context so give them. +# definitely not the way to do it but still, here it is.. +class User: + def __init__(self): + self.is_service = True + +class Request: + def __init__(self, user): + self.user = user + self.query_params = [] + self.method = 'POST' class Command(BaseCommand): - help = "Upload all existing data to TTV's RabbitMQ queue" + help = "Upload all existing data to TTV's RabbitMQ queue" - def handle(self, *args, **options): - catalog_records = CatalogRecord.objects.all() - logger.info(f"found {catalog_records.count()} catalog records") - aff_rows = 0 + def handle(self, *args, **options): + catalog_records = CatalogRecord.objects.filter(state='published') + aff_rows = 0 + user = User() + request = Request(user) + context = {'request': request} - for catalog_record in catalog_records: - cr = CatalogRecordSerializer(catalog_record).data - logger.info(f"Publishing catalog record {aff_rows} from TOTAL {catalog_records.count()}") - RabbitMQService.publish(cr, routing_key='', exchange="TTV-datasets") - aff_rows += 1 - logger.info(f"Published catalog record id={cr['identifier']}") + for catalog_record in catalog_records: + serializer = catalog_record.serializer_class + cr_json = serializer(catalog_record, context=context).data + cr_json['data_catalog'] = {'catalog_json': catalog_record.data_catalog.catalog_json} - logger.info(f"All catalog records published") + RabbitMQService.publish(cr_json, routing_key='create', exchange="TTV-datasets") + aff_rows += 1 + logger.info("All catalog records published to TTV exchange") From a4780f73897a0a7abdfcc96af7d270cb67154397 Mon Sep 17 00:00:00 2001 From: Tommi Pulli Date: Wed, 31 Mar 2021 16:27:41 +0300 Subject: [PATCH 31/34] fix swagger 'try it out' --- swagger/v1/swagger.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swagger/v1/swagger.yaml b/swagger/v1/swagger.yaml index ec997116..e91f7d3f 100755 --- a/swagger/v1/swagger.yaml +++ b/swagger/v1/swagger.yaml @@ -5,7 +5,7 @@ info: host: __METAX_ENV_DOMAIN__ schemes: - https -basePath: /rest/v1/ +basePath: '' produces: - application/json paths: From a179ca27fa92c06b6c6bff51e29475167ca4f267 Mon Sep 17 00:00:00 2001 From: Toni Nurmi Date: Thu, 18 Feb 2021 09:31:34 +0200 Subject: [PATCH 32/34] fix directory count commands --- .../management/commands/fix_file_counts.py | 14 ++++++++++---- src/metax_api/services/file_service.py | 1 - 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/metax_api/management/commands/fix_file_counts.py b/src/metax_api/management/commands/fix_file_counts.py index 121f968e..8bdc7a53 100644 --- a/src/metax_api/management/commands/fix_file_counts.py +++ b/src/metax_api/management/commands/fix_file_counts.py @@ -8,9 +8,15 @@ class Command(BaseCommand): def handle(self, *args, **options): - dirs_with_no_files = Directory.objects.filter(file_count=0, parent_directory=None) - logger.info(f"fix_file_counts command found {dirs_with_no_files.count()} directories with file_count=0") + dirs_with_no_files = Directory.objects_unfiltered.all() + dir_sum = dirs_with_no_files.count() + logger.info(f"fix_file_counts command found {dir_sum} directories") + i=0 for dir in dirs_with_no_files: - dir.calculate_byte_size_and_file_count() - logger.info(f"folder has {dir.file_count} files after recalculation") + i += 1 + try: + dir.calculate_byte_size_and_file_count() + except Exception as e: + logger.error(f"can't fix filecount for directory {i}/{dir_sum}") + logger.info(f"folder {i}/{dir_sum} has {dir.file_count} files after recalculation") logger.info(f"fix_file_counts command executed successfully") \ No newline at end of file diff --git a/src/metax_api/services/file_service.py b/src/metax_api/services/file_service.py index 4fc3ee32..8ecb1bf1 100755 --- a/src/metax_api/services/file_service.py +++ b/src/metax_api/services/file_service.py @@ -4,7 +4,6 @@ # # :author: CSC - IT Center for Science Ltd., Espoo Finland # :license: MIT - import logging from collections import defaultdict from os import getpid From 8c43b8cea92fdbcd1c56a2e4ea87a7db40253e0b Mon Sep 17 00:00:00 2001 From: Toni Nurmi Date: Thu, 18 Feb 2021 16:46:14 +0200 Subject: [PATCH 33/34] fix issues in fix_file_counts_cr command --- .../management/commands/fix_file_counts_cr.py | 19 +++++++++++++++++++ src/metax_api/models/catalog_record.py | 11 +++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 src/metax_api/management/commands/fix_file_counts_cr.py diff --git a/src/metax_api/management/commands/fix_file_counts_cr.py b/src/metax_api/management/commands/fix_file_counts_cr.py new file mode 100644 index 00000000..6768ffc7 --- /dev/null +++ b/src/metax_api/management/commands/fix_file_counts_cr.py @@ -0,0 +1,19 @@ +import logging + +from django.core.management.base import BaseCommand + +from metax_api.models import CatalogRecord + +logger = logging.getLogger(__name__) + +class Command(BaseCommand): + def handle(self, *args, **options): + CRS = CatalogRecord.objects.all() + crs_sum = CRS.count() + logger.info(f"fix_file_counts command found {crs_sum} catalog records with file_count=0 and byte_size=0") + i = 1 + for catalog_record in CRS: + logger.info(f"Calculating {i}/{crs_sum} {catalog_record.identifier} ") + catalog_record.calculate_directory_byte_sizes_and_file_counts() + i += 1 + logger.info(f"fix_file_counts command executed successfully") \ No newline at end of file diff --git a/src/metax_api/models/catalog_record.py b/src/metax_api/models/catalog_record.py index 3722c0ce..6d829952 100755 --- a/src/metax_api/models/catalog_record.py +++ b/src/metax_api/models/catalog_record.py @@ -2097,9 +2097,13 @@ def calculate_directory_byte_sizes_and_file_counts(self): if self.research_dataset.get('directories', None): dir_identifiers = [d['identifier'] for d in self.research_dataset['directories']] + file_dir_identifiers = [] if self.research_dataset.get('files', None): - file_dir_identifiers = [File.objects.get(identifier=f['identifier']).parent_directory.identifier - for f in self.research_dataset['files']] + try: + file_dir_identifiers = [File.objects.get(identifier=f['identifier']).parent_directory.identifier + for f in self.research_dataset['files']] + except Exception as e: + _logger.error(e) if not dir_identifiers and not file_dir_identifiers: return @@ -2108,6 +2112,9 @@ def calculate_directory_byte_sizes_and_file_counts(self): highest_level_dirs_by_project = self._get_top_level_parent_dirs_by_project(dir_identifiers) + if len(highest_level_dirs_by_project) == 0: + return + directory_data = {} for project_identifier, dir_paths in highest_level_dirs_by_project.items(): From 3e48565b94d40633f64e80744926dc95ca36db39 Mon Sep 17 00:00:00 2001 From: Tommi Pulli Date: Thu, 1 Apr 2021 10:16:07 +0300 Subject: [PATCH 34/34] Revert password logging This reverts commit 113b5b57143ef341f5dd024c80a048c0250d9223. --- src/metax_api/middleware/identifyapicaller.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/metax_api/middleware/identifyapicaller.py b/src/metax_api/middleware/identifyapicaller.py index 41539181..26454ef7 100755 --- a/src/metax_api/middleware/identifyapicaller.py +++ b/src/metax_api/middleware/identifyapicaller.py @@ -10,7 +10,7 @@ from base64 import b64decode import requests -from django.conf import settings as django_settings, settings +from django.conf import settings as django_settings from django.http import HttpResponseForbidden from metax_api.exceptions import Http403 @@ -162,9 +162,6 @@ def _auth_basic(self, request, auth_b64): raise Http403 if apikey != user["password"]: _logger.warning("Failed authnz for user %s: password mismatch" % username) - if settings.DEBUG: - _logger.warning(f"Auth password mismatch with apikey {apikey} != {user['password']}") - raise Http403 request.user.username = username