v0.3.0 (#40)

* updated deps

* pathlib, Background Tasks, Logging Updates (#38)

* updated CacheManager to use pathlib

* updated some ops to be background tasks

* fixed ruff lint errors

* finally fixed logging. logs writing to app.log again

* added test for delete file

* Updated Endpoint for Downloading File (#39)

* updated starter script to use 2 workers

* added log messages

* added :path to route

* added trailing slash to get_file path

* updated route and now passing filename through queryparam

* updated link generated in FileUploadDTO model to match new route

* added gzip middleware

* added invoke and helper tasks

* fixed delete_file test

* updated tests to use new endpoint

* updated test client and added logger to cachemanager

* updated deps

* moved rich to dev dependencies and updated project version to 0.3.0

logging.conf

@@ -2,14 +2,14 @@
 keys=root, hypercorn.error, hypercorn.access
 
 [handlers]
-keys=console, error_file, access_file
+keys=console, error_file, access_file, logfile
 
 [formatters]
-keys=generic, access
+keys=generic, access, logfile
 
 [logger_root]
 level=DEBUG
-handlers=console
+handlers=console, logfile
 
 [logger_hypercorn.error]
 level=INFO
@@ -38,11 +38,20 @@ class=logging.FileHandler
 formatter=access
 args=('hypercorn.access.log',)
 
+[handler_logfile]
+class=handlers.RotatingFileHandler
+level=INFO
+args=('app.log', 'a', 100000, 10)
+formatter=logfile
+
 [formatter_generic]
-format=%(asctime)s [%(process)d] [%(levelname)s] %(message)s
+format=%(asctime)s [%(levelname)s] %(name)s: %(message)s
 datefmt=%Y-%m-%d %H:%M:%S
 class=logging.Formatter
 
 [formatter_access]
 format=%(message)s
 class=logging.Formatter
+
+[formatter_logfile]
+format=%(asctime)s [%(levelname)s] %(name)s: %(message)s

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "smolvault"
-version = "0.2.0"
+version = "0.3.0"
 description = ""
 license = "MIT"
 authors = ["Zach Fuller <[email protected]>"]
@@ -9,23 +9,24 @@ packages = [{include = "smolvault", from = "src"}]
 
 [tool.poetry.dependencies]
 python = "^3.11"
-pydantic = "^2.7.4"
+pydantic = "^2.8.2"
 fastapi = "^0.111.0"
 sqlmodel = "^0.0.19"
-rich = "^13.7.1"
 python-multipart = "^0.0.9"
 python-dotenv = "^1.0.1"
 pydantic-settings = "^2.3.4"
 hypercorn = "^0.17.3"
 
 [tool.poetry.group.dev.dependencies]
-boto3-stubs = {extras = ["essential"], version = "^1.34.136"}
+boto3-stubs = {extras = ["essential"], version = "^1.34.144"}
 pre-commit = "^3.7.1"
 ruff = "^0.5.0"
 mypy = "^1.10.1"
 pytest = "^8.2.2"
 pytest-asyncio = "^0.23.7"
-moto = {extras = ["all"], version = "^5.0.10"}
+moto = {extras = ["all"], version = "^5.0.11"}
+invoke = "^2.2.0"
+rich = "^13.7.1"
 
 [tool.ruff]
 line-length = 120
@@ -34,7 +35,7 @@ indent-width = 4
 target-version = "py311"
 
 [tool.ruff.lint]
-select = ["E", "F", "W", "C90", "I", "N", "UP", "ASYNC", "S", "B", "ERA", "PLE", "PLW", "PERF", "RUF", "SIM", "PT", "T20"]
+select = ["E", "F", "W", "C90", "I", "N", "UP", "ASYNC", "S", "B", "ERA", "PLE", "PLW", "PLC", "PLW", "PERF", "RUF", "SIM", "PT", "T20", "PTH", "LOG", "G"]
 ignore = ["E501", "S101"]
 
 # Allow fix for all enabled rules (when `--fix`) is provided.

requirements.txt

@@ -445,9 +445,9 @@ typer==0.12.3 ; python_version >= "3.11" and python_version < "4.0" \
 typing-extensions==4.12.2 ; python_version >= "3.11" and python_version < "4.0" \
     --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \
     --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8
-uvicorn[standard]==0.30.1 ; python_version >= "3.11" and python_version < "4.0" \
-    --hash=sha256:cd17daa7f3b9d7a24de3617820e634d0933b69eed8e33a516071174427238c81 \
-    --hash=sha256:d46cd8e0fd80240baffbcd9ec1012a712938754afcf81bce56c024c1656aece8
+uvicorn[standard]==0.30.3 ; python_version >= "3.11" and python_version < "4.0" \
+    --hash=sha256:0d114d0831ff1adbf231d358cbf42f17333413042552a624ea6a9b4c33dcfd81 \
+    --hash=sha256:94a3608da0e530cea8f69683aa4126364ac18e3826b6630d1a65f4638aade503
 uvloop==0.19.0 ; (sys_platform != "win32" and sys_platform != "cygwin") and platform_python_implementation != "PyPy" and python_version >= "3.11" and python_version < "4.0" \
     --hash=sha256:0246f4fd1bf2bf702e06b0d45ee91677ee5c31242f39aab4ea6fe0c51aedd0fd \
     --hash=sha256:02506dc23a5d90e04d4f65c7791e65cf44bd91b37f24cfc3ef6cf2aff05dc7ec \

scripts/start_app.sh

@@ -1,3 +1,4 @@
 #!/bin/bash
 
-poetry run hypercorn src.smolvault.main:app -b 0.0.0.0 --debug --log-config=logging.conf --log-level=DEBUG --access-logfile=hypercorn.access.log --error-logfile=hypercorn.error.log --keep-alive=120 --workers=1
+poetry run hypercorn src.smolvault.main:app -b 0.0.0.0 --debug --log-config=logging.conf --log-level=DEBUG --access-logfile=hypercorn.access.log --error-logfile=hypercorn.error.log --keep-alive=120 --workers=2
+

scripts/test.sh

@@ -17,4 +17,4 @@ fi
 # create local cache dir
 mkdir uploads
 
-poetry run pytest -vv tests/
+poetry run pytest -vvv tests/ -x

src/smolvault/cache/cache_manager.py

@@ -1,15 +1,24 @@
-import os
+import logging
+import pathlib
+
+logger = logging.getLogger(__name__)
 
 
 class CacheManager:
     def __init__(self, cache_dir: str) -> None:
-        self.cache_dir = cache_dir
+        self.cache_dir = pathlib.Path(cache_dir)
+        logger.info("Created CacheManager with cache directory %s", self.cache_dir)
 
     def file_exists(self, filename: str) -> bool:
-        return os.path.exists(os.path.join(self.cache_dir, filename))
+        file_path = self.cache_dir / filename
+        return file_path.exists()
 
     def save_file(self, filename: str, data: bytes) -> str:
-        file_path = os.path.join(self.cache_dir, filename)
-        with open(file_path, "wb") as f:
+        file_path = self.cache_dir / filename
+        with file_path.open("wb") as f:
             f.write(data)
-        return file_path
+        return file_path.as_posix()
+
+    def delete_file(self, local_path: str) -> None:
+        file_path = pathlib.Path(local_path)
+        file_path.unlink(missing_ok=True)

src/smolvault/clients/aws.py

@@ -11,25 +11,24 @@
 
 class S3Client:
     def __init__(self, bucket_name: str) -> None:
-        logger.debug(f"Creating S3 client for bucket {bucket_name}")
+        logger.info("Creating S3 client for bucket %s", bucket_name)
         self.settings = get_settings()
         self.bucket_name = bucket_name
         self.session = boto3.Session()
         self.client = self.session.client("s3")
         self.bucket = self.session.resource("s3").Bucket(bucket_name)
 
     def upload(self, data: FileUploadDTO) -> str:
-        logger.debug(f"Uploading file {data.name} to S3")
         key = data.name
         self.bucket.put_object(Key=key, Body=data.content)
+        logger.info("File %s uploaded successfully", key)
         return key
 
     def download(self, key: str) -> Any:
-        logger.debug(f"Downloading file {key} from S3")
         response = self.client.get_object(Bucket=self.bucket_name, Key=key)
-        logger.info("File downloaded successfully")
+        logger.info("File %s downloaded successfully", key)
         return response["Body"].read()
 
     def delete(self, key: str) -> None:
         self.client.delete_object(Bucket=self.bucket_name, Key=key)
-        logger.info(f"Deleted file {key} from S3")
+        logger.info("Deleted file %s from S3", key)

src/smolvault/main.py

@@ -1,13 +1,14 @@
 import json
 import logging
-import os
+import pathlib
 import sys
 import urllib.parse
 from logging.handlers import RotatingFileHandler
 from typing import Annotated
 
-from fastapi import Depends, FastAPI, File, Form, UploadFile
+from fastapi import BackgroundTasks, Depends, FastAPI, File, Form, UploadFile
 from fastapi.middleware.cors import CORSMiddleware
+from fastapi.middleware.gzip import GZipMiddleware
 from fastapi.responses import FileResponse, Response
 
 from smolvault.cache.cache_manager import CacheManager
@@ -24,6 +25,7 @@
 logger = logging.getLogger(__name__)
 
 app = FastAPI(title="smolvault")
+app.add_middleware(GZipMiddleware, minimum_size=1000)
 app.add_middleware(
     CORSMiddleware,
     allow_origins=["*"],
@@ -50,8 +52,10 @@ async def upload_file(
 ) -> Response:
     contents = await file.read()
     if file.filename is None:
+        logger.error("Filename not received in request")
         raise ValueError("Filename is required")
     file_upload = FileUploadDTO(name=file.filename, size=len(contents), content=contents, tags=tags)
+    logger.info("Uploading file to S3 with name %s", file_upload.name)
     object_key = s3_client.upload(data=file_upload)
     db_client.add_metadata(file_upload, object_key)
     return Response(
@@ -61,17 +65,22 @@ async def upload_file(
     )
 
 
-@app.get("/file/{name}")
-async def get_file(db_client: Annotated[DatabaseClient, Depends(DatabaseClient)], name: str) -> Response:
-    record = db_client.get_metadata(urllib.parse.unquote(name))
+@app.get("/file/original")
+async def get_file(
+    db_client: Annotated[DatabaseClient, Depends(DatabaseClient)], filename: str, background_tasks: BackgroundTasks
+) -> Response:
+    record = db_client.get_metadata(filename)
     if record is None:
+        logger.info("File not found: %s", filename)
         return Response(content=json.dumps({"error": "File not found"}), status_code=404, media_type="application/json")
-    if record.local_path is None or cache.file_exists(record.local_path) is False:
+    if record.local_path is None or cache.file_exists(record.file_name) is False:
+        logger.info("File %s not found in cache, downloading from S3", filename)
         content = s3_client.download(record.object_key)
         record.local_path = cache.save_file(record.file_name, content)
-        record.cache_timestamp = int(os.path.getmtime(record.local_path))
-        db_client.update_metadata(record)
-
+        record.cache_timestamp = int(pathlib.Path(record.local_path).stat().st_mtime)
+        logger.info("Saved file %s at time %d", record.local_path, record.cache_timestamp)
+        background_tasks.add_task(db_client.update_metadata, record)
+    logger.info("Serving file %s from cache", record.file_name)
     return FileResponse(path=record.local_path, filename=record.file_name)
 
 
@@ -88,13 +97,15 @@ async def get_file_metadata(
 @app.get("/files")
 async def get_files(db_client: Annotated[DatabaseClient, Depends(DatabaseClient)]) -> list[FileMetadata]:
     raw_metadata = db_client.get_all_metadata()
+    logger.info("Retrieved %d records from database", len(raw_metadata))
     results = [FileMetadata.model_validate(metadata.model_dump()) for metadata in raw_metadata]
     return results
 
 
 @app.get("/files/search")
 async def search_files(db_client: Annotated[DatabaseClient, Depends(DatabaseClient)], tag: str) -> list[FileMetadata]:
     raw_metadata = db_client.select_metadata_by_tag(tag)
+    logger.info("Retrieved %d records from database with tag %s", len(raw_metadata), tag)
     results = [FileMetadata.model_validate(metadata.model_dump()) for metadata in raw_metadata]
     return results
 
@@ -118,12 +129,16 @@ async def update_file_tags(
 
 
 @app.delete("/file/{name}")
-async def delete_file(db_client: Annotated[DatabaseClient, Depends(DatabaseClient)], name: str) -> Response:
+async def delete_file(
+    db_client: Annotated[DatabaseClient, Depends(DatabaseClient)], name: str, background_tasks: BackgroundTasks
+) -> Response:
     record: FileMetadataRecord | None = db_client.get_metadata(name)
     if record is None:
         return Response(content=json.dumps({"error": "File not found"}), status_code=404, media_type="application/json")
     s3_client.delete(record.object_key)
     db_client.delete_metadata(record)
+    if record.local_path:
+        background_tasks.add_task(cache.delete_file, record.local_path)
     return Response(
         content=json.dumps({"message": "File deleted successfully", "record": record.model_dump()}),
         status_code=200,

src/smolvault/models.py

@@ -22,7 +22,7 @@ def file_sha256(self) -> str:
     @computed_field  # type: ignore
     @cached_property
     def link(self) -> str:
-        return f"http://pi.local:8000/file/{urllib.parse.quote_plus(self.name)}"
+        return f"http://pi.local:8000/file/original?filename={urllib.parse.quote_plus(self.name)}"
 
     @computed_field  # type: ignore
     @cached_property

tasks.py

@@ -0,0 +1,24 @@
+import sqlite3
+
+from invoke.context import Context
+from invoke.tasks import task
+from rich import print
+
+
+@task
+def lint(c: Context) -> None:
+    c.run("poetry run ruff check src/smolvault tests", echo=True, pty=True)
+
+
+@task
+def fmt(c: Context) -> None:
+    c.run("poetry run ruff format src/smolvault tests", echo=True, pty=True)
+
+
+@task
+def show_table(c: Context) -> None:
+    conn = sqlite3.connect("file_metadata.db")
+    cursor = conn.cursor()
+    cursor.execute("SELECT * FROM filemetadatarecord")
+    print(cursor.fetchall())
+    conn.close()

tests/conftest.py

@@ -1,12 +1,13 @@
 import os
+import pathlib
 from collections.abc import Generator
 from datetime import datetime
 from typing import Any
 from zoneinfo import ZoneInfo
 
 import boto3
 import pytest
-from httpx import AsyncClient
+from httpx import ASGITransport, AsyncClient
 from moto import mock_aws
 from mypy_boto3_s3 import S3Client
 from smolvault.clients.database import DatabaseClient, FileMetadataRecord, FileTag  # noqa: F401
@@ -17,14 +18,14 @@
 
 class TestDatabaseClient(DatabaseClient):
     def __init__(self) -> None:
-        self.engine = create_engine("sqlite:///test.db", echo=True, connect_args={"check_same_thread": False})
+        self.engine = create_engine("sqlite:///test.db", echo=False, connect_args={"check_same_thread": False})
         SQLModel.metadata.create_all(self.engine)
 
 
 @pytest.fixture(scope="module")
 def client() -> AsyncClient:
     app.dependency_overrides[DatabaseClient] = TestDatabaseClient
-    return AsyncClient(app=app, base_url="http://testserver", timeout=5.0)
+    return AsyncClient(transport=ASGITransport(app=app), base_url="http://testserver")  # type: ignore
 
 
 @pytest.fixture(scope="session")
@@ -55,13 +56,13 @@ def _test_bucket(aws: S3Client) -> None:
 @pytest.fixture()
 def _bucket_w_camera_img(_test_bucket: None) -> None:
     client = boto3.client("s3")
-    with open("tests/mock_data/camera.png", "rb") as f:
+    with pathlib.Path("tests/mock_data/camera.png").open("rb") as f:
         client.put_object(Bucket="test-bucket", Key="camera.png", Body=f.read())
 
 
 @pytest.fixture(scope="session")
 def camera_img() -> bytes:
-    with open("tests/mock_data/camera.png", "rb") as f:
+    with pathlib.Path("tests/mock_data/camera.png").open("rb") as f:
         return f.read()
 
 
@@ -72,7 +73,7 @@ def file_metadata_record() -> FileMetadataRecord:
         file_sha256="ddf2ef1fce9d6289051b8415d9b6ace81743288db15570179e409b3169055055",
         size=19467,
         object_key="camera.png",
-        link="http://pi.local:8000/file/camera.png",
+        link="http://pi.local:8000/file/original?filename=camera.png",
         upload_timestamp=datetime.now(ZoneInfo("UTC")).isoformat(),
         tags="camera,photo",
     )

tests/test_delete_file.py

@@ -0,0 +1,29 @@
+from typing import Any
+from uuid import uuid4
+
+import pytest
+from httpx import AsyncClient
+from smolvault.models import FileUploadDTO
+
+
+@pytest.mark.asyncio()
+@pytest.mark.usefixtures("_test_bucket")
+async def test_delete_file(client: AsyncClient, camera_img: bytes) -> None:
+    # first upload the file
+    filename = f"{uuid4().hex[:6]}-camera.png"
+    expected_obj = FileUploadDTO(name=filename, size=len(camera_img), content=camera_img, tags="camera,photo")
+    expected = expected_obj.model_dump(exclude={"content", "upload_timestamp", "tags"})
+    response = await client.post(
+        "/file/upload", files={"file": (filename, camera_img, "image/png")}, data={"tags": "camera,photo"}
+    )
+    actual: dict[str, Any] = response.json()
+    actual.pop("upload_timestamp")
+    assert response.status_code == 201
+    assert actual == expected
+
+    # now delete the file
+    response = await client.delete(f"/file/{filename}")
+    actual = response.json()
+    assert response.status_code == 200
+    assert actual["message"] == "File deleted successfully"
+    assert actual["record"]["file_name"] == filename