feat: Support Pagination of Results (#70)

* updated deps

* added offset and limit parameters to DatabaseClient.get_all_metadata

* requiring kwargs for optional params

* fix: updated default limit vault to 10

* added default value

* reduced DAILY_UPLOAD_LIMIT_BYTES by order of magnitude

* Added TODOs

* updated python deps

* updated get and search files endpoints

* updated pyproject.toml and set project version to 0.7.0

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "smolvault"
-version = "0.6.0"
+version = "0.7.0"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -53,7 +53,7 @@ omit = ["**/__init__.py"]
 line-length = 120
 indent-width = 4
 
-target-version = "py311"
+target-version = "py312"
 
 [tool.ruff.lint]
 select = ["E", "F", "W", "C90", "I", "N", "UP", "ASYNC", "S", "B", "ERA", "PLE", "PLW", "PLC", "PLW", "PERF", "RUF", "SIM", "PT", "T20", "PTH", "LOG", "G"]

scripts/test.sh

@@ -4,7 +4,7 @@ export SMOLVAULT_BUCKET="test-bucket"
 export SMOLVAULT_DB="test.db"
 export SMOLVAULT_CACHE="./uploads/"
 export AUTH_SECRET_KEY="09d25e094faa6ca2556c818166b7a9563b93f7099f6f0f4caa6cf63b88e8d3e7" # key from FastAPI docs to use in tests
-export DAILY_UPLOAD_LIMIT_BYTES="500000"
+export DAILY_UPLOAD_LIMIT_BYTES="50000"
 export USERS_LIMIT="20"
 export USER_WHITELIST="1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20"
 

src/smolvault/clients/database.py

@@ -1,6 +1,9 @@
 from collections.abc import Sequence
 from datetime import datetime
+from typing import Annotated
 
+from pydantic import Field as PydanticField
+from pydantic import validate_call
 from sqlmodel import Field, Session, SQLModel, create_engine, select
 
 from smolvault.auth.models import NewUserDTO
@@ -60,17 +63,24 @@ def add_metadata(self, file_upload: FileUploadDTO, key: str) -> None:
                 session.add(FileTag(tag_name=tag, file_id=file_metadata.id))
             session.commit()
 
+    @validate_call
     def get_all_metadata(
-        self, user_id: int, start_time: datetime | None = None, end_time: datetime | None = None
+        self,
+        user_id: int,
+        *,
+        start_time: datetime | None = None,
+        end_time: datetime | None = None,
+        offset: int | None = 0,
+        limit: Annotated[int | None, PydanticField(default=10, lt=100)] = 10,
     ) -> Sequence[FileMetadataRecord]:
         with Session(self.engine) as session:
             statement = select(FileMetadataRecord).where(FileMetadataRecord.user_id == user_id)
             if start_time:
                 statement = statement.where(FileMetadataRecord.upload_timestamp >= start_time.isoformat())
             if end_time:
                 statement = statement.where(FileMetadataRecord.upload_timestamp <= end_time.isoformat())
-            results = session.exec(statement)
-            return results.fetchall()
+            results = session.exec(statement.offset(offset).limit(limit)).all()
+            return results
 
     def get_metadata(self, filename: str, user_id: int) -> FileMetadataRecord | None:
         with Session(self.engine) as session:
@@ -81,15 +91,21 @@ def get_metadata(self, filename: str, user_id: int) -> FileMetadataRecord | None
             )
             return session.exec(statement).first()
 
-    def select_metadata_by_tag(self, tag: str, user_id: int) -> Sequence[FileMetadataRecord]:
+    def select_metadata_by_tag(
+        self,
+        tag: str,
+        user_id: int,
+        offset: int | None = 0,
+        limit: Annotated[int | None, PydanticField(default=10, lt=100)] = 10,
+    ) -> Sequence[FileMetadataRecord]:
         with Session(self.engine) as session:
             statement = (
                 select(FileMetadataRecord)
                 .where(FileTag.file_id == FileMetadataRecord.id)
                 .where(FileTag.tag_name == tag)
                 .where(FileMetadataRecord.user_id == user_id)
             )
-            results = session.exec(statement)
+            results = session.exec(statement.offset(offset).limit(limit))
             return results.fetchall()
 
     def update_metadata(self, record: FileMetadataRecord) -> None:

src/smolvault/main.py

@@ -179,9 +179,11 @@ async def get_file_metadata(
 async def get_files(
     current_user: Annotated[User, Depends(get_current_user)],
     db_client: Annotated[DatabaseClient, Depends(DatabaseClient)],
+    offset: int | None = None,
+    limit: int | None = None,
 ) -> list[FileMetadata]:
     logger.info("Retrieving all files for user %s", current_user.username)
-    raw_metadata = db_client.get_all_metadata(current_user.id)
+    raw_metadata = db_client.get_all_metadata(user_id=current_user.id, offset=offset, limit=limit)
     logger.info("Retrieved %d records from database", len(raw_metadata))
     results = [FileMetadata.model_validate(metadata.model_dump()) for metadata in raw_metadata]
     return results
@@ -192,9 +194,11 @@ async def search_files(
     current_user: Annotated[User, Depends(get_current_user)],
     db_client: Annotated[DatabaseClient, Depends(DatabaseClient)],
     tag: str,
+    offset: int | None = None,
+    limit: int | None = None,
 ) -> list[FileMetadata]:
     logger.info("Retrieving files with tag %s for user %s", tag, current_user.username)
-    raw_metadata = db_client.select_metadata_by_tag(tag, current_user.id)
+    raw_metadata = db_client.select_metadata_by_tag(tag, current_user.id, offset=offset, limit=limit)
     logger.info("Retrieved %d records from database with tag %s", len(raw_metadata), tag)
     results = [FileMetadata.model_validate(metadata.model_dump()) for metadata in raw_metadata]
     return results

tests/conftest.py

@@ -13,10 +13,7 @@
 from polyfactory.pytest_plugin import register_fixture
 from sqlmodel import SQLModel, create_engine
 
-from smolvault.clients.database import (
-    DatabaseClient,
-    FileMetadataRecord,
-)
+from smolvault.clients.database import DatabaseClient, FileMetadataRecord
 from smolvault.main import app
 from smolvault.models import FileMetadata
 
@@ -51,7 +48,7 @@ def user(user_factory: UserFactory, db_client: TestDatabaseClient) -> tuple[str,
 @pytest.fixture(scope="module")
 def client() -> AsyncClient:
     app.dependency_overrides[DatabaseClient] = TestDatabaseClient
-    return AsyncClient(transport=ASGITransport(app=app), base_url="http://testserver")  # type: ignore
+    return AsyncClient(transport=ASGITransport(app=app), base_url="http://testserver")
 
 
 @pytest.fixture