Merge branch 'stable'

.travis.yml

@@ -56,8 +56,7 @@ before_script:
 - sudo chown -R $USER:$USER src/log
 
 script:
-- cd src && flake8 .
-- python manage.py migrate metax_api
+- cd src && python manage.py migrate metax_api
 - coverage run --source="." manage.py test metax_api
 - COVERALLS_REPO_TOKEN=$coveralls_token coveralls
 

src/metax_api/management/commands/fix_file_counts.py

@@ -0,0 +1,16 @@
+import logging
+
+from django.core.management.base import BaseCommand
+
+from metax_api.models import Directory
+
+logger = logging.getLogger(__name__)
+
+class Command(BaseCommand):
+    def handle(self, *args, **options):
+        dirs_with_no_files = Directory.objects.filter(file_count=0, parent_directory=None)
+        logger.info(f"fix_file_counts command found {dirs_with_no_files.count()} directories with file_count=0")
+        for dir in dirs_with_no_files:
+            dir.calculate_byte_size_and_file_count()
+            logger.info(f"folder has {dir.file_count} files after recalculation")
+        logger.info(f"fix_file_counts command executed successfully")

src/metax_api/management/commands/loadinitialdata.py

@@ -6,189 +6,41 @@
 # :license: MIT
 
 import json
+import datetime
+import logging
 
-import requests
-import urllib3
-from django.conf import settings
-from django.core.management.base import BaseCommand, CommandError
-from icecream import ic
+from django.core.management.base import BaseCommand
+from django.db import IntegrityError
 
-from metax_api.utils import executing_test_case
-from django.conf import settings as django_settings
-
-urllib3.disable_warnings()
+from metax_api.models import DataCatalog
+from metax_api.models import FileStorage
 
+logger = logging.getLogger(__name__)
 
 class Command(BaseCommand):
 
     help = "Load initial data for Metax: Data catalogs, file storages."
 
-    def add_arguments(self, parser):
-        # optional arguments to pass in special settings during test case execution
-        parser.add_argument(
-            "--test-settings",
-            type=str,
-            nargs="?",
-            help="Set url to use during test execution",
-        )
-
     def handle(self, *args, **options):
-        ic(executing_test_case())
-        if executing_test_case():
-            self._set_test_confs(options)
-        else:
-            self._set_real_confs()
-
-        self._load_data_catalogs()
-        self._load_file_storages()
-
-    def _set_test_confs(self, options):
-        """
-        For test case execution, a testserver url and test metax credentials are expected
-        as parameters from the testrunner.
-        """
-        if "test_settings" not in options:
-            raise CommandError("test case execution requires settings as a parameter")
-
-        test_opts = options["test_settings"]
-        self._metax_api_root = test_opts["metax_url"]
-        self._metax_api_user = (
-            test_opts["metax_credentials"]["username"],
-            test_opts["metax_credentials"]["password"],
-        )
-
-        if self._metax_api_root == "https://localhost":
-            # extra precaution...
-            raise CommandError("Test case tried to write into real db")
-
-    def _set_real_confs(self):
-        """
-        Set metax url to localhost, and get user credentials from app_config.
-        """
-        self._metax_api_root = settings.METAX_API_ROOT
-
-        for user in settings.API_USERS:
-            if user["username"] == "metax":
-                self._metax_api_user = (user["username"], user["password"])
-                break
-        else:
-            raise CommandError("Could not find metax-user from app_config ?")
-
-    def _error_is_already_exists(self, details):
-        try:
-            for field_name, errors in details.items():
-                if field_name == "identifier" and "already exists" in errors[0]:
-                    return True
-        except:
-            pass
-        return False
-
-    def _load_data_catalogs(self):
-        try:
-            with open("metax_api/initialdata/datacatalogs.json", "r") as f:
-                data_catalogs = json.load(f)
-        except FileNotFoundError: # noqa
-            raise CommandError("File initialdata/datacatalogs.json does not exist?")
-        except json.decoder.JSONDecodeError as e:
-            raise CommandError("Error loading data catalog json: %s" % str(e))
-
-        self.stdout.write("Creating %d data catalogs..." % len(data_catalogs))
-
-        for dc in data_catalogs:
-            if dc['catalog_json']['research_dataset_schema'] == 'dft':
-                pass
-            else:
-                if 'v2' in django_settings.API_VERSIONS_ENABLED:
-                    response = requests.post('%s/rest/v2/datacatalogs' % self._metax_api_root,
-                        json=dc, auth=self._metax_api_user, verify=False)
-                else:
-                    response = requests.post('%s/rest/datacatalogs' % self._metax_api_root,
-                        json=dc, auth=self._metax_api_user, verify=False)
-
-                if response.status_code == 201:
-                    self.stdout.write('Created catalog: %s' % dc['catalog_json']['identifier'])
-                else:
-                    # update instead
-                    try:
-                        errors = response.json()
-                    except:
-                        raise CommandError(response.content)
-
-                    if self._error_is_already_exists(errors.get('catalog_json', {})):
-                        self.stdout.write('Catalog %s already exists, updating instead...' %
-                            dc['catalog_json']['identifier'])
-
-                        if 'v2' in django_settings.API_VERSIONS_ENABLED:
-                            response = requests.put('%s/rest/v2/datacatalogs/%s' %
-                                (self._metax_api_root, dc['catalog_json']['identifier']),
-                                json=dc, auth=self._metax_api_user, verify=False)
-                        else:
-                            response = requests.put('%s/rest/datacatalogs/%s' %
-                                (self._metax_api_root, dc['catalog_json']['identifier']),
-                                json=dc, auth=self._metax_api_user, verify=False)
-
-                    if response.status_code == 200:
-                        self.stdout.write(
-                            "Updated catalog: %s" % dc["catalog_json"]["identifier"]
-                        )
-                        continue
-
-                    # create or update ended in error
-                    raise CommandError('Failed to process catalog: %s. Reason: %s' %
-                        (dc['catalog_json']['identifier'], errors))
-
-    def _load_file_storages(self):
-        try:
-            with open("metax_api/initialdata/filestorages.json", "r") as f:
-                storages = json.load(f)
-        except FileNotFoundError: # noqa
-            raise CommandError("File initialdata/filestorages.json does not exist?")
-        except json.decoder.JSONDecodeError as e:
-            raise CommandError("Error loading file storage json: %s" % str(e))
-
-        self.stdout.write("Creating %d file storages..." % len(storages))
-
-        for fs in storages:
-            response = requests.post(
-                "%s/rest/filestorages" % self._metax_api_root,
-                json=fs,
-                auth=self._metax_api_user,
-                verify=False,
-            )
-
-            if response.status_code == 201:
-                self.stdout.write(
-                    "Created file storage: %s" % fs["file_storage_json"]["identifier"]
-                )
-            else:
-                # update instead
+        with open("metax_api/initialdata/datacatalogs.json", "r") as f:
+            data_catalogs = json.load(f)
+            for json_dc in data_catalogs:
                 try:
-                    errors = response.json()
-                except:
-                    raise CommandError(response.content)
-                if self._error_is_already_exists(errors.get("file_storage_json", {})):
-                    self.stdout.write(
-                        "File storage %s already exists, updating instead..."
-                        % fs["file_storage_json"]["identifier"]
-                    )
-
-                    response = requests.put(
-                        "%s/rest/filestorages/%s"
-                        % (self._metax_api_root, fs["file_storage_json"]["identifier"]),
-                        json=fs,
-                        auth=self._metax_api_user,
-                        verify=False,
-                    )
-
-                    if response.status_code == 200:
-                        self.stdout.write(
-                            "Updated file storage: %s"
-                            % fs["file_storage_json"]["identifier"]
-                        )
-                        continue
-
-                # create or update ended in error
-                raise CommandError(
-                    "Failed to process storage: %s. Reason: %s"
-                    % (fs["file_storage_json"]["identifier"], errors)
-                )
+                    dc = DataCatalog(catalog_json=json_dc["catalog_json"], date_created=datetime.datetime.now())
+                    dc.catalog_record_services_create = json_dc["catalog_record_services_create"]
+                    dc.catalog_record_services_edit = json_dc["catalog_record_services_edit"]
+                    dc.catalog_record_services_read = json_dc["catalog_record_services_read"]
+                    dc.save()
+                except IntegrityError as e:
+                    logger.error("datacatalog already exists in the database")
+        logger.info("Successfully created missing datacatalogs")
+
+        with open("metax_api/initialdata/filestorages.json", "r") as f:
+            storages = json.load(f)
+            for fs in storages:
+                try:
+                    fs = FileStorage(file_storage_json=fs["file_storage_json"], date_created=datetime.datetime.now())
+                    fs.save()
+                except IntegrityError as e:
+                    logger.error("filestorage already exists in the database")
+        logger.info("Successfully created missing filestorages")

src/metax_api/settings/components/common.py

@@ -3,6 +3,7 @@
 from metax_api.settings import env
 from metax_api.settings.components import BASE_DIR
 
+METAX_ENV=(str, "local_development")
 DEBUG = env("DEBUG")
 SECRET_KEY = env("DJANGO_SECRET_KEY")
 METAX_API_ROOT = env("METAX_API_ROOT")

src/metax_api/settings/environments/remote.py

@@ -40,6 +40,10 @@
     {
         "password": env("DOWNLOAD_USER_PASSWORD"),
         "username": "download"
+    },
+    {
+        "password": env("JYU_USER_PASSWORD"),
+        "username": "jyu"
     }
 ]
 

src/metax_api/settings/environments/stable.py

@@ -33,6 +33,7 @@
                 "metax",
                 "etsin"
             ],
+
             "delete": [
                 "metax",
                 "etsin"
@@ -54,6 +55,7 @@
                 "qvain-light",
                 "etsin",
                 "tpas",
+                "jyu",
                 "endusers"
             ],
             "delete": [
@@ -63,8 +65,10 @@
                 "qvain-light",
                 "etsin",
                 "tpas",
+                "jyu",
                 "endusers"
             ],
+
             "read": [
                 "all"
             ],
@@ -75,14 +79,15 @@
                 "qvain-light",
                 "etsin",
                 "tpas",
+                "jyu",
                 "endusers"
             ]
         },
         "directories": {
             "read": [
                 "metax",
-                "qvain",
                 "ida",
+                "qvain",
                 "qvain-light",
                 "etsin",
                 "tpas",
@@ -101,6 +106,7 @@
                 "ida",
                 "tpas"
             ],
+
             "read": [
                 "metax",
                 "ida",
@@ -121,6 +127,16 @@
         },
         "filestorages": {
             "create": [
+                "metax"
+            ],
+            "delete": [
+                "metax"
+            ],
+            "read": [
+                "metax"
+            ],
+            "update": [
+                "metax"
                 "metax",
                 "ida"
             ],
@@ -240,6 +256,7 @@
                     "all"
                 ]
             },
+
             "deprecated_datasets_cumulative": {
                 "use": [
                     "all"