Merge pull request #573 from CSCfi/stable

Merge stable to master: v1.7.0

src/metax_api/api/oaipmh/base/metax_oai_server.py

@@ -110,6 +110,8 @@ def _get_filtered_records_data(self, verb, metadata_prefix, set, cursor, batch_s
                     data_catalog__catalog_json__identifier__in=settings.OAI['SET_MAPPINGS'][set])
         else:
             query_set = query_set.filter(data_catalog__catalog_json__identifier__in=self._get_default_set_filter())
+            query_set = query_set.filter(state='published')
+
         data = []
         for record in query_set:
             if verb == 'ListRecords':
@@ -440,6 +442,8 @@ def getRecord(self, metadataPrefix, identifier):
             if metadataPrefix == OAI_DC_URNRESOLVER_MDPREFIX:
                 raise BadArgumentError('Invalid metadataPrefix value. It can be only used with ListRecords verb')
             record = CatalogRecord.objects.get(identifier__exact=identifier)
+            if record.state == 'draft':
+                raise IdDoesNotExistError("No record with identifier %s is available." % identifier)
         except CatalogRecord.DoesNotExist:
             try:
                 record = DataCatalog.objects.get(catalog_json__identifier__exact=identifier)
@@ -455,4 +459,4 @@ def getRecord(self, metadataPrefix, identifier):
             raise NoRecordsMatchError
 
         return (common.Header('', identifier, self._get_header_timestamp(record), ['metax'], False),
-                common.Metadata('', metadata), None)
+            common.Metadata('', metadata), None)

src/metax_api/api/rest/base/serializers/catalog_record_serializer.py

@@ -82,7 +82,8 @@ class Meta:
             'date_cumulation_started',
             'date_cumulation_ended',
             'date_last_cumulative_addition',
-            'rems_identifier'
+            'rems_identifier',
+            'access_granter'
         ) + CommonSerializer.Meta.fields
 
         extra_kwargs = {
@@ -124,6 +125,7 @@ def is_valid(self, raise_exception=False):
         self.initial_data.pop('preservation_dataset_version', None)
         self.initial_data.pop('preservation_dataset_origin_version', None)
         self.initial_data.pop('rems_identifier', None)
+        self.initial_data.pop('access_granter', None)
 
         if self._data_catalog_is_changed():
             # updating data catalog, but not necessarily research_dataset.
@@ -302,6 +304,7 @@ def _check_and_strip_sensitive_fields(self, instance, res):
                     CRS.remove_contact_info_metadata(res['research_dataset']))
 
                 res.pop('rems_identifier', None)
+                res.pop('access_granter', None)
 
         return res
 

src/metax_api/api/rest/base/views/common_view.py

@@ -34,7 +34,7 @@ class CommonViewSet(ModelViewSet):
 
     api_type = 'rest'
     authentication_classes = ()
-    permission_classes = (EndUserPermissions, ServicePermissions)
+    permission_classes = [EndUserPermissions, ServicePermissions]
 
     cache = RedisCacheService
 

src/metax_api/api/rest/base/views/directory_view.py

@@ -11,6 +11,8 @@
 from rest_framework.response import Response
 
 from metax_api.api.rest.base.serializers import DirectorySerializer
+from metax_api.services.pagination import DirectoryPagination
+from rest_framework.pagination import LimitOffsetPagination
 from metax_api.exceptions import Http400, Http403, Http501
 from metax_api.models import Directory
 from metax_api.services import CommonService, FileService
@@ -20,6 +22,7 @@
 class DirectoryViewSet(CommonViewSet):
 
     serializer_class = DirectorySerializer
+    pagination_class = DirectoryPagination
     object = Directory
     select_related = ['parent_directory']
     lookup_field_other = 'identifier'
@@ -56,6 +59,7 @@ def _get_directory_contents(self, request, identifier=None):
         A wrapper to call FS to collect and validate parameters from the request,
         and then call FS.get_directory_contents().
         """
+        paginate = CommonService.get_boolean_query_param(request, 'pagination')
         include_parent = CommonService.get_boolean_query_param(request, 'include_parent')
         dirs_only = CommonService.get_boolean_query_param(request, 'directories_only')
         recursive = CommonService.get_boolean_query_param(request, 'recursive')
@@ -86,6 +90,19 @@ def _get_directory_contents(self, request, identifier=None):
             request=request
         )
 
+        if paginate:
+            if isinstance(files_and_dirs, dict):
+                paginated = self.paginate_queryset(files_and_dirs)
+                if include_parent:
+                    for k, v in files_and_dirs.items():
+                        if k not in ['directories', 'files']:
+                            paginated[k] = v
+                return self.get_paginated_response(paginated)
+            else:
+                paginator = LimitOffsetPagination()
+                context = paginator.paginate_queryset(files_and_dirs, request)
+                return paginator.get_paginated_response(context)
+
         return Response(files_and_dirs)
 
     @detail_route(methods=['get'], url_path="files")

src/metax_api/migrations/0018_auto_20200330_1101.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.10 on 2020-03-30 08:01
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('metax_api', '0017_catalogrecord_rems_identifier'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='catalogrecord',
+            old_name='_access_granter',
+            new_name='access_granter',
+        ),
+    ]

src/metax_api/models/catalog_record.py

@@ -310,7 +310,7 @@ class CatalogRecord(Common):
     date_last_cumulative_addition = models.DateTimeField(null=True, default=None,
         help_text='Date of last file addition while actively cumulative.')
 
-    _access_granter = JSONField(null=True, default=None,
+    access_granter = JSONField(null=True, default=None,
         help_text='Stores data of REMS user who is currently granting access to this dataset')
 
     rems_identifier = models.CharField(max_length=200, null=True, default=None,
@@ -908,11 +908,8 @@ def delete(self, *args, **kwargs):
                                                              'delete'))
 
         if self._dataset_has_rems_managed_access() and settings.REMS['ENABLED']:
-            self.add_post_request_callable(
-                REMSUpdate(self, 'close', rems_id=self.rems_identifier, reason='dataset deletion')
-            )
-            self.rems_identifier = None
-            super().save(update_fields=['rems_identifier'])
+            self._pre_rems_deletion('dataset deletion')
+            super().save(update_fields=['rems_identifier', 'access_granter'])
 
         self.add_post_request_callable(RabbitMQPublishRecord(self, 'delete'))
 
@@ -940,11 +937,8 @@ def deprecate(self, timestamp=None):
         self.date_deprecated = self.date_modified = timestamp or get_tz_aware_now_without_micros()
 
         if self._dataset_has_rems_managed_access() and settings.REMS['ENABLED']:
-            self.add_post_request_callable(
-                REMSUpdate(self, 'close', rems_id=self.rems_identifier, reason='dataset deprecation')
-            )
-            self.rems_identifier = None
-            super().save(update_fields=['rems_identifier'])
+            self._pre_rems_deletion('dataset deprecation')
+            super().save(update_fields=['rems_identifier', 'access_granter'])
 
         super().save(update_fields=['deprecated', 'date_deprecated', 'date_modified'])
         self.add_post_request_callable(DelayedLog(
@@ -1159,8 +1153,8 @@ def _post_create_operations(self):
                                                                 'create'))
 
             if self._dataset_has_rems_managed_access() and settings.REMS['ENABLED']:
-                self._handle_rems_managed_access()
-                super().save(update_fields=['rems_identifier'])
+                self._pre_rems_creation()
+                super().save(update_fields=['rems_identifier', 'access_granter'])
 
             self.add_post_request_callable(RabbitMQPublishRecord(self, 'create'))
 
@@ -1256,12 +1250,9 @@ def _pre_update_operations(self):
             if self._dataset_rems_changed():
                 if self._dataset_rems_access_type_changed():
                     if self._dataset_has_rems_managed_access():
-                        self._handle_rems_managed_access()
+                        self._pre_rems_creation()
                     else:
-                        self.add_post_request_callable(
-                            REMSUpdate(self, 'close', rems_id=self.rems_identifier, reason='access type change')
-                        )
-                        self.rems_identifier = None
+                        self._pre_rems_deletion(reason='access type change')
 
                 elif self._dataset_license_changed() and self._dataset_has_rems_managed_access():
                     if self._dataset_has_license():
@@ -1271,10 +1262,7 @@ def _pre_update_operations(self):
                         self.rems_identifier = generate_uuid_identifier()
 
                     else:
-                        self.add_post_request_callable(
-                            REMSUpdate(self, 'close', rems_id=self.rems_identifier, reason='license deletion')
-                        )
-                        self.rems_identifier = None
+                        self._pre_rems_deletion(reason='license deletion')
 
         if self.field_changed('research_dataset'):
             if self.preservation_state in (
@@ -1482,17 +1470,28 @@ def _files_added_for_first_time(self):
         # creating a new dataset version already occurred once
         return not metadata_versions_with_files_exist
 
-    def _handle_rems_managed_access(self):
+    def _pre_rems_creation(self):
         """
         Ensure that all necessary information is avaliable for REMS access
         and save post request callable to create correspoding REMS entity.
         """
         self._validate_for_rems()
         user_info = self._get_user_info_for_rems()
-        self._access_granter = user_info
+        self.access_granter = user_info
         self.rems_identifier = generate_uuid_identifier()
         self.add_post_request_callable(REMSUpdate(self, 'create', user_info=user_info))
 
+    def _pre_rems_deletion(self, reason):
+        """
+        Delete rems information and save post request callable to close
+        corresponding REMS entity.
+        """
+        self.add_post_request_callable(
+            REMSUpdate(self, 'close', rems_id=self.rems_identifier, reason=reason)
+        )
+        self.rems_identifier = None
+        self.access_granter = None
+
     def _dataset_has_rems_managed_access(self):
         """
         Check if dataset uses REMS for managing access.

src/metax_api/models/data_catalog.py

@@ -35,6 +35,8 @@ class DataCatalog(Common):
 
     # END OF MODEL FIELD DEFINITIONS #
 
+    READ_METHODS = ('GET', 'HEAD', 'OPTIONS')
+
     def __init__(self, *args, **kwargs):
         super(DataCatalog, self).__init__(*args, **kwargs)
         self.track_fields('catalog_json.identifier')
@@ -60,3 +62,12 @@ def __repr__(self):
             self.catalog_json['research_dataset_schema'],
             self.catalog_json['dataset_versioning'],
         )
+
+    def user_has_access(self, request):
+        """
+        Overriding inherited operation to check permissions for datacatalogs
+        """
+
+        if request.method in self.READ_METHODS or request.user.is_service:
+            return True
+        return False

src/metax_api/permissions/permissions.py

@@ -112,16 +112,13 @@ def _check_rest_perms(self, request, api_name):
         execute specific operation type on given API endpoint.
         """
         if request.method in METHOD_MAP:
-
             operation_type = METHOD_MAP[request.method]
-
             if 'all' in self.perms['rest'][api_name].get(operation_type, []):
                 has_perm = True
             else:
                 has_perm = self._check_user_rest_perms(request, api_name, operation_type)
         else:
             raise MethodNotAllowed
-
         return has_perm
 
     def _check_rpc_perms(self, request, api_name):
@@ -147,7 +144,6 @@ def _check_rpc_perms(self, request, api_name):
 
     def has_object_permission(self, request, view, obj):
         has_perm = obj.user_has_access(request)
-
         if not has_perm:
             self.message = 'You are not permitted to access this resource.'
         return has_perm

src/metax_api/services/file_service.py

@@ -729,8 +729,12 @@ def _get_directory_contents(cls, directory_id, recursive=False, max_depth=1, dep
 
         if cr_id:
             try:
-                dirs, files = cls._get_directory_contents_for_catalog_record(directory_id, cr_id,
-                    dirs_only=dirs_only, directory_fields=directory_fields, file_fields=file_fields)
+                dirs, files = cls._get_directory_contents_for_catalog_record(
+                    directory_id,
+                    cr_id,
+                    dirs_only=dirs_only,
+                    directory_fields=directory_fields,
+                    file_fields=file_fields)
             except Http404:
                 if recursive:
                     return {'directories': []}

src/metax_api/services/pagination.py

@@ -0,0 +1,58 @@
+from rest_framework.pagination import LimitOffsetPagination
+
+
+class DirectoryPagination(LimitOffsetPagination):
+
+    page_size = 10
+    page_size_query_param = 'page_size'
+
+    def paginate_queryset(self, queryset, request, view=None):
+        self.count = self.get_count(queryset)
+        self.limit = self.get_limit(request)
+        if self.limit is None:
+            return None
+
+        self.offset = self.get_offset(request)
+        self.request = request
+        if self.count > self.limit and self.template is not None:
+            self.display_page_controls = True
+
+        if self.count == 0 or self.offset > self.count:
+            return []
+
+        # serves filters directories_only and files_only which returnes dictionaries
+        if len(queryset) == 1:
+            key = list(queryset.keys())[0]
+            return dict({key: queryset[key][self.offset:self.offset + self.limit]})
+
+        dirs = []
+        files = []
+        dir_len = len(queryset['directories'])
+
+        # if no directories left to show
+        if self.offset >= dir_len:
+            offset = self.offset - dir_len
+            files = queryset['files'][offset:offset + self.limit]
+
+        # if directories are not enough for one page limit
+        elif (self.offset + self.limit) >= dir_len:
+            dirs = queryset['directories'][self.offset:]
+            files_to_show = self.limit - (dir_len - self.offset)
+            if files_to_show > 0:
+                files = queryset['files'][0:files_to_show]
+
+        # if enough directories for page limit
+        else:
+            dirs = queryset['directories'][self.offset:self.offset + self.limit]
+
+        return dict({'directories': dirs, 'files': files})
+
+    def get_count(self, queryset):
+        """
+        Determine a count of directory dictionary.
+        """
+        count = 0
+        for q, v in queryset.items():
+            if q in ['directories', 'files']:
+                count = count + len(v)
+        return count