Updates for public PurlDB

Makefile

@@ -107,7 +107,8 @@ process_scans:
 
 test:
 	@echo "-> Run the test suite"
-	${ACTIVATE} DJANGO_SETTINGS_MODULE=purldb.settings ${PYTHON_EXE} -m pytest -vvs --ignore matchcode-toolkit
+	${ACTIVATE} DJANGO_SETTINGS_MODULE=purldb_project.settings ${PYTHON_EXE} -m pytest -vvs --ignore matchcode-toolkit --ignore packagedb/tests/test_throttling.py
+	${ACTIVATE} DJANGO_SETTINGS_MODULE=purldb_project.settings ${PYTHON_EXE} -m pytest -vvs packagedb/tests/test_throttling.py
 	${ACTIVATE} ${PYTHON_EXE} -m pytest -vvs matchcode-toolkit --ignore matchcode-toolkit/src/matchcode_toolkit/pipelines
 
 shell:

manage.py

@@ -15,5 +15,5 @@
 if __name__ == '__main__':
     from django.core.management import execute_from_command_line
 
-    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'purldb.settings')
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'purldb_project.settings')
     execute_from_command_line(sys.argv)

packagedb/api.py

@@ -19,6 +19,7 @@
 from rest_framework import status, viewsets
 from rest_framework.decorators import action
 from rest_framework.response import Response
+from rest_framework.throttling import AnonRateThrottle
 from univers.version_constraint import InvalidConstraintsError
 from univers.version_range import RANGE_CLASS_BY_SCHEMES, VersionRange
 from univers.versions import InvalidVersion
@@ -39,6 +40,8 @@
                                    PackageAPISerializer,
                                    PackageSetAPISerializer, PartySerializer,
                                    ResourceAPISerializer)
+from packagedb.throttling import StaffUserRateThrottle
+
 
 logger = logging.getLogger(__name__)
 
@@ -87,6 +90,7 @@ class ResourceViewSet(viewsets.ReadOnlyModelViewSet):
     queryset = Resource.objects.select_related('package')
     serializer_class = ResourceAPISerializer
     filterset_class = ResourceFilterSet
+    throttle_classes = [StaffUserRateThrottle, AnonRateThrottle]
     lookup_field = 'sha1'
 
     @action(detail=False, methods=['post'])
@@ -102,7 +106,7 @@ def filter_by_checksums(self, request, *args, **kwargs):
         - sha1
 
         Example:
-        
+
             {
                 "sha1": [
                     "b55fd82f80cc1bd0bdabf9c6e3153788d35d7911",
@@ -248,11 +252,12 @@ class Meta:
         )
 
 
-class PackageViewSet(viewsets.ReadOnlyModelViewSet):
+class PackagePublicViewSet(viewsets.ReadOnlyModelViewSet):
     queryset = Package.objects.prefetch_related('dependencies', 'parties')
     serializer_class = PackageAPISerializer
     lookup_field = 'uuid'
     filterset_class = PackageFilterSet
+    throttle_classes = [StaffUserRateThrottle, AnonRateThrottle]
 
     @action(detail=True, methods=['get'])
     def latest_version(self, request, *args, **kwargs):
@@ -292,18 +297,6 @@ def get_enhanced_package_data(self, request, *args, **kwargs):
         package_data = get_enhanced_package(package)
         return Response(package_data)
 
-    @action(detail=True)
-    def reindex_package(self, request, *args, **kwargs):
-        """
-        Reindex this package instance
-        """
-        package = self.get_object()
-        package.rescan()
-        data = {
-            'status': f'{package.package_url} has been queued for reindexing'
-        }
-        return Response(data)
-
     @action(detail=False, methods=['post'])
     def filter_by_checksums(self, request, *args, **kwargs):
         """
@@ -328,7 +321,7 @@ def filter_by_checksums(self, request, *args, **kwargs):
             }
 
         Multiple checksums algorithms can be passed together:
-        
+
             {
                 "sha1": [
                     "b55fd82f80cc1bd0bdabf9c6e3153788d35d7911",
@@ -389,6 +382,20 @@ def filter_by_checksums(self, request, *args, **kwargs):
         return self.get_paginated_response(serialized_package_data)
 
 
+class PackageViewSet(PackagePublicViewSet):
+    @action(detail=True)
+    def reindex_package(self, request, *args, **kwargs):
+        """
+        Reindex this package instance
+        """
+        package = self.get_object()
+        package.rescan()
+        data = {
+            'status': f'{package.package_url} has been queued for reindexing'
+        }
+        return Response(data)
+
+
 UPDATEABLE_FIELDS = [
     'primary_language',
     'copyright',
@@ -516,7 +523,7 @@ class CollectViewSet(viewsets.ViewSet):
 
     If the package does not exist, we will fetch the Package data and return
     it in the same request.
-    
+
     **Note:** Use `Index packages` for bulk indexing of packages; use `Reindex packages`
     for bulk reindexing of existing packages.
     """
@@ -556,15 +563,15 @@ def list(self, request, format=None):
 
         serializer = PackageAPISerializer(packages, many=True, context={'request': request})
         return Response(serializer.data)
-    
+
     @action(detail=False, methods=['post'])
     def index_packages(self, request, *args, **kwargs):
         """
         Take a list of `packages` (where each item is a dictionary containing either PURL
         or versionless PURL along with vers range) and index it.
-        
+
         If `reindex` flag is True then existing package will be rescanned, if `reindex_set`
-        is True then all the package in the same set will be rescanned.  
+        is True then all the package in the same set will be rescanned.
         If reindex flag is set to true then all the non existing package will be indexed.
 
         **Note:** When a versionless PURL is supplied without a vers range, then all the versions
@@ -590,7 +597,7 @@ def index_packages(self, request, *args, **kwargs):
                     "reindex": true,
                     "reindex_set": false,
                 }
-        
+
         Then return a mapping containing:
 
         - queued_packages_count
@@ -602,15 +609,15 @@ def index_packages(self, request, *args, **kwargs):
         - requeued_packages
             - A list of existing package urls that were placed on the rescan queue.
         - unqueued_packages_count
-            - The number of package urls not placed on the index queue.  
+            - The number of package urls not placed on the index queue.
                 This is because the package url already exists on the index queue and has not
                 yet been processed.
         - unqueued_packages
             - A list of package urls that were not placed on the index queue.
         - unsupported_packages_count
             - The number of package urls that are not processable by the index queue.
         - unsupported_packages
-            - A list of package urls that are not processable by the index queue.  
+            - A list of package urls that are not processable by the index queue.
                 The package indexing queue can only handle npm and maven purls.
         - unsupported_vers_count
             - The number of vers range that are not supported by the univers or package_manager.
@@ -622,18 +629,18 @@ def _reindex_package(package, reindexed_packages):
                 return
             package.rescan()
             reindexed_packages.append(package)
-        
+
         packages = request.data.get('packages') or []
         reindex = request.data.get('reindex') or False
         reindex_set = request.data.get('reindex_set') or False
 
         queued_packages = []
         unqueued_packages = []
-        
+
         nonexistent_packages = []
         reindexed_packages = []
         requeued_packages = []
-    
+
         supported_ecosystems = ['maven', 'npm']
 
         unique_purls, unsupported_packages, unsupported_vers = get_resolved_purls(packages, supported_ecosystems)

packagedb/management/commands/create_api_user.py

@@ -0,0 +1,56 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# PurlDB is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from django.core import exceptions
+from django.core.management.base import BaseCommand
+from django.core.management.base import CommandError
+from django.core.validators import validate_email
+
+from packagedb.models import ApiUser
+
+"""
+Create a basic API-only user based on an email.
+"""
+
+
+class Command(BaseCommand):
+    help = "Create a basic passwordless user with an API key for sole API authentication usage."
+    requires_migrations_checks = True
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--email",
+            help="Specifies the email for the user.",
+        )
+        parser.add_argument(
+            "--first-name",
+            default="",
+            help="First name.",
+        )
+        parser.add_argument(
+            "--last-name",
+            default="",
+            help="Last name.",
+        )
+
+    def handle(self, *args, **options):
+
+        email = options["email"]
+        try:
+            validate_email(email)
+            user = ApiUser.objects.create_api_user(
+                username=email,
+                first_name=options["first_name"] or "",
+                last_name=options["last_name"] or "",
+            )
+        except exceptions.ValidationError as e:
+            raise CommandError(str(e))
+
+        msg = f"User {user.email} created with API key: {user.auth_token.key}"
+        self.stdout.write(msg, self.style.SUCCESS)

packagedb/migrations/0081_apiuser.py

@@ -0,0 +1,27 @@
+# Generated by Django 4.2.6 on 2023-12-06 01:02
+
+from django.db import migrations
+import packagedb.models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("auth", "0012_alter_user_first_name_max_length"),
+        ("packagedb", "0080_remove_package_packagedb_p_search__8d33bb_gin_and_more"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ApiUser",
+            fields=[],
+            options={
+                "proxy": True,
+                "indexes": [],
+                "constraints": [],
+            },
+            bases=("auth.user",),
+            managers=[
+                ("objects", packagedb.models.ApiUserManager()),
+            ],
+        ),
+    ]

packagedb/models.py

@@ -14,7 +14,10 @@
 import sys
 import uuid
 
+from django.contrib.auth import get_user_model
+from django.contrib.auth.models import UserManager
 from django.contrib.postgres.fields import ArrayField
+from django.core import exceptions
 from django.core.paginator import Paginator
 from django.db import models
 from django.db import transaction
@@ -27,6 +30,8 @@
 from packageurl import PackageURL
 from packageurl.contrib.django.models import PackageURLMixin
 from packageurl.contrib.django.models import PackageURLQuerySetMixin
+from rest_framework.authtoken.models import Token
+
 
 TRACE = False
 
@@ -1236,3 +1241,58 @@ def get_package_set_members(self):
         return self.packages.order_by(
             'package_content',
         )
+
+
+UserModel = get_user_model()
+
+
+class ApiUserManager(UserManager):
+    def create_api_user(self, username, first_name="", last_name="", **extra_fields):
+        """
+        Create and return an API-only user. Raise ValidationError.
+        """
+        username = self.normalize_email(username)
+        email = username
+        self._validate_username(email)
+
+        # note we use the email as username and we could instead override
+        # django.contrib.auth.models.AbstractUser.USERNAME_FIELD
+
+        user = self.create_user(
+            username=email,
+            email=email,
+            password=None,
+            first_name=first_name,
+            last_name=last_name,
+            **extra_fields,
+        )
+
+        # this ensure that this is not a valid password
+        user.set_unusable_password()
+        user.save()
+
+        Token._default_manager.get_or_create(user=user)
+
+        return user
+
+    def _validate_username(self, email):
+        """
+        Validate username. If invalid, raise a ValidationError
+        """
+        try:
+            self.get_by_natural_key(email)
+        except models.ObjectDoesNotExist:
+            pass
+        else:
+            raise exceptions.ValidationError(f"Error: This email already exists: {email}")
+
+
+class ApiUser(UserModel):
+    """
+    A User proxy model to facilitate simplified admin API user creation.
+    """
+
+    objects = ApiUserManager()
+
+    class Meta:
+        proxy = True

packagedb/templates/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: *