Merge pull request #246 from CSCfi/CSCMETAX-371-return-selected-fields

CSCMETAX-371: [ADD] Return specified fields only from various GET api's

src/metax_api/api/rest/base/router.py

@@ -29,11 +29,10 @@
 
 class CustomRouter(DefaultRouter):
 
-    """
-    Override default router to allow PUT and PATCH methods in resource list url
-    """
-
     def __init__(self, *args, **kwargs):
+        """
+        Override to allow PUT and PATCH methods in resource list url.
+        """
         self.routes.pop(0)
         self.routes.insert(0, Route(
             url=r'^{prefix}{trailing_slash}$',
@@ -49,6 +48,13 @@ def __init__(self, *args, **kwargs):
         ))
         super(CustomRouter, self).__init__(*args, **kwargs)
 
+    def get_default_base_name(self, viewset):
+        """
+        When a viewset has no queryset set, or base_name is not passed to a router as the
+        3rd parameter, automatically determine base name.
+        """
+        return viewset.__class__.__name__.split('View')[0]
+
 
 router = CustomRouter(trailing_slash=False)
 router.register(r'apierrors/?', ApiErrorViewSet)
@@ -58,7 +64,7 @@ def __init__(self, *args, **kwargs):
 router.register(r'directories/?', DirectoryViewSet)
 router.register(r'files/?', FileViewSet)
 router.register(r'filestorages/?', FileStorageViewSet)
-router.register(r'schemas/?', SchemaViewSet, 'schema')
+router.register(r'schemas/?', SchemaViewSet)
 
 # note: this somehow maps to list-api... but the end result works when
 # the presence of the parameters is inspected in the list-api method.

src/metax_api/api/rest/base/serializers/catalog_record_serializer.py

@@ -34,6 +34,9 @@ class Meta:
             'preservation_state_modified',
             'preservation_description',
             'preservation_reason_description',
+            'dataset_version_set',
+            'next_dataset_version',
+            'previous_dataset_version',
             'mets_object_identifier',
             'editor',
             'removed',
@@ -92,13 +95,14 @@ def create(self, validated_data):
     def to_representation(self, instance):
         res = super(CatalogRecordSerializer, self).to_representation(instance)
 
-        if self.expand_relation_requested('data_catalog'):
-            res['data_catalog'] = DataCatalogSerializer(instance.data_catalog).data
-        else:
-            res['data_catalog'] = {
-                'id': instance.data_catalog.id,
-                'identifier': instance.data_catalog.catalog_json['identifier'],
-            }
+        if 'data_catalog' in res:
+            if self.expand_relation_requested('data_catalog'):
+                res['data_catalog'] = DataCatalogSerializer(instance.data_catalog).data
+            else:
+                res['data_catalog'] = {
+                    'id': instance.data_catalog.id,
+                    'identifier': instance.data_catalog.catalog_json['identifier'],
+                }
 
         if 'contract' in res:
             if self.expand_relation_requested('contract'):
@@ -117,14 +121,14 @@ def to_representation(self, instance):
         if 'dataset_version_set' in res:
             res['dataset_version_set'] = instance.dataset_version_set.get_listing()
 
-        if instance.next_dataset_version_id:
+        if 'next_dataset_version' in res:
             res['next_dataset_version'] = {
                 'id': instance.next_dataset_version.id,
                 'identifier': instance.next_dataset_version.identifier,
                 'preferred_identifier': instance.next_dataset_version.preferred_identifier,
             }
 
-        if instance.previous_dataset_version_id:
+        if 'previous_dataset_version' in res:
             res['previous_dataset_version'] = {
                 'id': instance.previous_dataset_version.id,
                 'identifier': instance.previous_dataset_version.identifier,

src/metax_api/api/rest/base/serializers/common_serializer.py

@@ -14,6 +14,9 @@
 
 class CommonSerializer(ModelSerializer):
 
+    # when query parameter ?fields=x,y is used, will include a list of fields to return
+    requested_fields = None
+
     class Meta:
         model = Common
         fields = (
@@ -37,6 +40,24 @@ class Meta:
             'service_created':     { 'required': False },
         }
 
+    def __init__(self, *args, **kwargs):
+        """
+        For most usual GET requests, the fields to retrieve for an object can be
+        specified via the query param ?fields=x,y,z. Retrieve those fields from the
+        implicitly passed request object for processing in the to_representation() method.
+
+        The list of fields can also be explicitly passed to the serializer as a list
+        in the kw arg 'only_fields', when serializing objects outside of the common GET
+        api's.
+        """
+        if 'only_fields' in kwargs:
+            self.requested_fields = kwargs.pop('only_fields')
+
+        super(CommonSerializer, self).__init__(*args, **kwargs)
+
+        if not self.requested_fields and 'request' in self.context and 'fields' in self.context['request'].query_params:
+            self.requested_fields = self.context['request'].query_params['fields'].split(',')
+
     @transaction.atomic
     def save(self, *args, **kwargs):
         """
@@ -55,14 +76,20 @@ def save(self, *args, **kwargs):
 
     def to_representation(self, instance):
         """
-        Copy-pasta / overrided from rest_framework code. Only return fields which have a non-null value
+        Copy-pasta / overrided from rest_framework code with the following modifications:
+        - Only return fields which have a non-null value
+        - When only specific fields are requested, skip fields accordingly
 
         Object instance -> Dict of primitive datatypes.
         """
         ret = OrderedDict()
         fields = self._readable_fields
 
         for field in fields:
+
+            if self.requested_fields and field.field_name not in self.requested_fields:
+                continue
+
             try:
                 attribute = field.get_attribute(instance)
             except SkipField:
@@ -75,7 +102,7 @@ def to_representation(self, instance):
             # resolve the pk value.
             check_for_none = attribute.pk if isinstance(attribute, PKOnlyObject) else attribute
             if check_for_none is None:
-                # this is the overrided block. dont return nulls
+                # this is an overrided block. dont return nulls
                 # ret[field.field_name] = None
                 pass
             else:

src/metax_api/api/rest/base/serializers/directory_serializer.py

@@ -62,7 +62,7 @@ def validate_directory_path(self, value):
     def to_representation(self, instance):
         res = super(DirectorySerializer, self).to_representation(instance)
 
-        if instance.parent_directory:
+        if 'parent_directory' in res and instance.parent_directory:
             res['parent_directory'] = {
                 'id': instance.parent_directory.id,
                 'identifier': instance.parent_directory.identifier,

src/metax_api/api/rest/base/serializers/file_serializer.py

@@ -62,23 +62,26 @@ def is_valid(self, raise_exception=False):
     def to_representation(self, instance):
         res = super(FileSerializer, self).to_representation(instance)
 
-        if self.expand_relation_requested('file_storage'):
-            res['file_storage'] = FileStorageSerializer(instance.file_storage).data
-        else:
-            res['file_storage'] = {
-                'id': instance.file_storage.id,
-                'identifier': instance.file_storage.file_storage_json['identifier'],
-            }
-
-        if self.expand_relation_requested('parent_directory'):
-            res['parent_directory'] = DirectorySerializer(instance.parent_directory).data
-        else:
-            res['parent_directory'] = {
-                'id': instance.parent_directory.id,
-                'identifier': instance.parent_directory.identifier,
-            }
-
-        res['checksum'] = self._form_checksum(res)
+        if 'file_storage' in res:
+            if self.expand_relation_requested('file_storage'):
+                res['file_storage'] = FileStorageSerializer(instance.file_storage).data
+            else:
+                res['file_storage'] = {
+                    'id': instance.file_storage.id,
+                    'identifier': instance.file_storage.file_storage_json['identifier'],
+                }
+
+        if 'parent_directory' in res:
+            if self.expand_relation_requested('parent_directory'):
+                res['parent_directory'] = DirectorySerializer(instance.parent_directory).data
+            else:
+                res['parent_directory'] = {
+                    'id': instance.parent_directory.id,
+                    'identifier': instance.parent_directory.identifier,
+                }
+
+        if not self.requested_fields or 'checksum' in self.requested_fields:
+            res['checksum'] = self._form_checksum(res)
 
         return res
 

src/metax_api/api/rest/base/views/api_error_view.py

@@ -4,10 +4,8 @@
 from rest_framework.response import Response
 
 from metax_api.exceptions import Http403, Http501
-from metax_api.models import File
 from metax_api.services import ApiErrorService
 from .common_view import CommonViewSet
-from ..serializers import FileSerializer
 
 
 """
@@ -28,10 +26,6 @@ class ApiErrorViewSet(CommonViewSet):
     authentication_classes = ()
     permission_classes = ()
 
-    # required to make viewset work, dont serve a purpose...
-    queryset = File.objects.all()
-    serializer_class = FileSerializer
-
     def initial(self, request, *args, **kwargs):
         if request.user.username != 'metax':
             raise Http403

src/metax_api/api/rest/base/views/common_view.py

@@ -26,10 +26,25 @@ class CommonViewSet(ModelViewSet):
     lookup_field_internal = None
     cache = RedisSentinelCache()
 
+    # get_queryset() automatically includes these in .select_related(field1, field2...) when returning
+    # queryset to the caller
+    select_related = []
+
     # assigning the create_bulk method here allows for other views to assing their other,
     # customized method to be called instead instead of the generic one.
     create_bulk_method = CS.create_bulk
 
+    def __init__(self, *args, **kwargs):
+        super(CommonViewSet, self).__init__(*args, **kwargs)
+        if (hasattr(self, 'object') and self.object) and (not hasattr(self, 'queryset') or self.queryset is None):
+            # ^ must have attribute 'object' set, AND queryset not set.
+
+            # the primary location where a queryset is initialized for
+            # any inheriting viewset, in case not already specified in their ViewSet class.
+            # avoids having to specify queryset in each ViewSet separately.
+            self.queryset = self.object.objects.all()
+            self.queryset_unfiltered = self.object.objects_unfiltered.all()
+
     def handle_exception(self, exc):
         """
         Store request and response data to disk for later inspection
@@ -47,12 +62,25 @@ def paginate_queryset(self, queryset):
     def get_queryset(self):
         additional_filters = {}
 
-        removed = CS.get_boolean_query_param(self.request, 'removed')
-        if removed:
+        if CS.get_boolean_query_param(self.request, 'removed'):
             additional_filters.update({'removed': True})
             self.queryset = self.queryset_unfiltered
 
-        return super(CommonViewSet, self).get_queryset().filter(**additional_filters)
+        if self.request.query_params.get('fields', False):
+            # only specified fields are requested to be returned
+
+            fields = self.request.query_params['fields'].split(',')
+
+            # causes only requested fields to be loaded from the db
+            self.queryset = self.queryset.only(*fields)
+
+            # check if requested fields are relations, so that we know to include them in select_related.
+            # if no fields is relation, select_related will be made empty.
+            self.select_related = [ rel for rel in self.select_related if rel in fields ]
+
+        return super(CommonViewSet, self).get_queryset() \
+            .select_related(*self.select_related) \
+            .filter(**additional_filters)
 
     def get_object(self, search_params=None):
         """

src/metax_api/api/rest/base/views/contract_view.py

@@ -18,9 +18,6 @@ class ContractViewSet(CommonViewSet):
     authentication_classes = ()
     permission_classes = ()
 
-    queryset = Contract.objects.all()
-    queryset_unfiltered = Contract.objects_unfiltered.all()
-
     serializer_class = ContractSerializer
     object = Contract
 

src/metax_api/api/rest/base/views/data_catalog_view.py

@@ -15,10 +15,6 @@ class DataCatalogViewSet(CommonViewSet):
     authentication_classes = ()
     permission_classes = ()
 
-    # note: override get_queryset() to get more control
-    queryset = DataCatalog.objects.filter(active=True, removed=False)
-    queryset_unfiltered = DataCatalog.objects_unfiltered.all()
-
     serializer_class = DataCatalogSerializer
     object = DataCatalog
 

src/metax_api/api/rest/base/views/dataset_view.py

@@ -24,12 +24,9 @@ class DatasetViewSet(CommonViewSet):
     authentication_classes = ()
     permission_classes = ()
 
-    # note: override get_queryset() to get more control
-    queryset = CatalogRecord.objects.select_related('data_catalog', 'contract').all()
-    queryset_unfiltered = CatalogRecord.objects_unfiltered.select_related('data_catalog', 'contract').all()
-
     serializer_class = CatalogRecordSerializer
     object = CatalogRecord
+    select_related = ['data_catalog', 'contract']
 
     lookup_field = 'pk'
 
@@ -49,7 +46,6 @@ def get_object(self):
         return self._search_using_dataset_identifiers()
 
     def get_queryset(self):
-
         additional_filters = {}
         q_filters = []
 
@@ -74,7 +70,7 @@ def retrieve(self, request, *args, **kwargs):
             res.data = CRS.transform_datasets_to_format(res.data, request.query_params['dataset_format'])
             request.accepted_renderer = XMLRenderer()
         elif 'file_details' in request.query_params:
-            CRS.populate_file_details(res.data)
+            CRS.populate_file_details(res.data, request)
 
         return res
 

src/metax_api/api/rest/base/views/directory_view.py

@@ -19,15 +19,11 @@ class DirectoryViewSet(CommonViewSet):
     authentication_classes = ()
     permission_classes = ()
 
-    object = Directory
-
-    queryset = Directory.objects.select_related('parent_directory').all()
-    queryset_unfiltered = Directory.objects_unfiltered.select_related('parent_directory').all()
-
     serializer_class = DirectorySerializer
+    object = Directory
+    select_related = ['parent_directory']
 
     lookup_field_other = 'identifier'
-    create_bulk_method = FileService.create_bulk
 
     def list(self, request, *args, **kwargs):
         raise Http501()
@@ -77,7 +73,8 @@ def _get_directory_contents(self, request, identifier=None):
             max_depth=max_depth,
             dirs_only=dirs_only,
             include_parent=include_parent,
-            cr_identifier=cr_identifier
+            cr_identifier=cr_identifier,
+            request=request
         )
 
         return Response(files_and_dirs)

src/metax_api/api/rest/base/views/file_storage_view.py

@@ -13,9 +13,6 @@ class FileStorageViewSet(CommonViewSet):
     authentication_classes = ()
     permission_classes = ()
 
-    queryset = FileStorage.objects.filter(active=True, removed=False)
-    queryset_unfiltered = FileStorage.objects_unfiltered.all()
-
     serializer_class = FileStorageSerializer
     object = FileStorage
 

src/metax_api/api/rest/base/views/file_view.py

@@ -28,12 +28,9 @@ class FileViewSet(CommonViewSet):
     authentication_classes = ()
     permission_classes = ()
 
-    # note: override get_queryset() to get more control
-    queryset = File.objects.select_related('file_storage', 'parent_directory').all()
-    queryset_unfiltered = File.objects_unfiltered.select_related('file_storage', 'parent_directory').all()
-
     serializer_class = FileSerializer
     object = File
+    select_related = ['file_storage', 'parent_directory']
 
     lookup_field = 'pk'