✨(api) add Pandas-based StatiqueImporter

We designed an efficient ORM-free Pandas-based Statique importer. It is
supposed to be 10x faster than using ORM-based routines as implemented
in /statique/bulk endpoint.

docker-compose.yml

@@ -40,6 +40,7 @@ services:
     restart: always
     volumes:
       - ./src/api:/app
+      - ./data:/data
     depends_on:
       - postgresql
 

src/api/CHANGELOG.md

@@ -8,6 +8,11 @@ and this project adheres to
 
 ## [Unreleased]
 
+### Added
+
+- Implement Pandas-based `StatiqueImporter`
+- CLI: add `import-statique` command
+
 ### Changed
 
 - Upgrade fastapi to `0.112.2`

src/api/Pipfile

@@ -32,6 +32,7 @@ black = "==24.8.0"
 csvkit = "==2.0.1"
 honcho = "==1.1.0"
 mypy = "==1.11.2"
+pandas-stubs = "==2.2.2.240807"
 polyfactory = "==2.16.2"
 pytest = "==8.3.2"
 pytest-cov = "==5.0.0"

src/api/qualicharge/cli.py

@@ -1,23 +1,34 @@
 """QualiCharge CLI."""
 
+import logging
+from pathlib import Path
 from typing import Optional, Sequence, cast
 
+import pandas as pd
 import questionary
 import typer
+from psycopg import Error as PGError
 from rich import print
 from rich.console import Console
+from rich.logging import RichHandler
 from rich.table import Table
 from sqlalchemy import Column as SAColumn
+from sqlalchemy.exc import IntegrityError, OperationalError, ProgrammingError
 from sqlmodel import Session as SMSession
 from sqlmodel import select
 
 from .auth.models import UserCreate
 from .auth.schemas import Group, ScopesEnum, User
 from .conf import settings
 from .db import get_session
+from .exceptions import IntegrityError as QCIntegrityError
 from .fixtures.operational_units import prefixes
 from .schemas.core import OperationalUnit
+from .schemas.sql import StatiqueImporter
 
+logging.basicConfig(
+    level=logging.INFO, format="%(message)s", datefmt="[%X]", handlers=[RichHandler()]
+)
 app = typer.Typer(name="qualicharge", no_args_is_help=True)
 console = Console()
 
@@ -416,6 +427,27 @@ def delete_user(ctx: typer.Context, username: str, force: bool = False):
     print(f"[bold yellow]User {username} deleted.[/bold yellow]")
 
 
+@app.command()
+def import_static(ctx: typer.Context, input_file: Path):
+    """Import Statique file (parquet format)."""
+    session: SMSession = ctx.obj
+
+    # Load dataset
+    console.log(f"Reading input file: {input_file}")
+    static = pd.read_parquet(input_file)
+    console.log(f"Read {len(static.index)} rows")
+    importer = StatiqueImporter(static, session.connection())
+
+    console.log("Save to configured database")
+    try:
+        importer.save()
+    except (ProgrammingError, IntegrityError, OperationalError, PGError) as err:
+        session.rollback()
+        raise QCIntegrityError("Input file importation failed. Rolling back.") from err
+    session.commit()
+    console.log("Saved (or updated) all entries successfully.")
+
+
 @app.callback()
 def main(ctx: typer.Context):
     """Attach database session to the context object."""

src/api/qualicharge/exceptions.py

@@ -39,3 +39,7 @@ class IntegrityError(QualiChargeExceptionMixin, Exception):
 
 class ObjectDoesNotExist(QualiChargeExceptionMixin, Exception):
     """Raised when queried object does not exist."""
+
+
+class ProgrammingError(QualiChargeExceptionMixin, Exception):
+    """Raised when QC object API is badly used."""