Merge pull request #73 from ricardogsilva/69-add-temporal-properties-…

…to-stations

Bootstrap observation variables and add temporal properties to observation stations

arpav_ppcv/database.py

@@ -166,11 +166,8 @@ def create_many_stations(
         geom = shapely.io.from_geojson(station_create.geom.model_dump_json())
         wkbelement = from_shape(geom)
         db_station = models.Station(
-            code=station_create.code,
+            **station_create.model_dump(exclude={"geom"}),
             geom=wkbelement,
-            altitude_m=station_create.altitude_m,
-            name=station_create.name,
-            type_=station_create.type_,
         )
         db_records.append(db_station)
         session.add(db_station)

arpav_ppcv/main.py

@@ -15,30 +15,35 @@
 import anyio
 import django
 import httpx
+import sqlmodel
 import typer
 import yaml
 from django.conf import settings as django_settings
 from django.core import management
 from rich import print
 from rich.padding import Padding
 from rich.panel import Panel
+from sqlalchemy.exc import IntegrityError
 
 from . import (
     config,
     database,
 )
 from .cliapp.app import app as cli_app
 from .observations_harvester.cliapp import app as observations_harvester_app
+from .schemas import models as observations_models
 from .thredds import crawler
 from .webapp.legacy.django_settings import get_custom_django_settings
 
 app = typer.Typer()
 db_app = typer.Typer()
 dev_app = typer.Typer()
+bootstrap_app = typer.Typer()
 app.add_typer(cli_app, name="app")
 app.add_typer(db_app, name="db")
 app.add_typer(dev_app, name="dev")
 app.add_typer(observations_harvester_app, name="observations-harvester")
+app.add_typer(bootstrap_app, name="bootstrap")
 
 
 @app.callback()
@@ -229,4 +234,60 @@ def import_thredds_datasets(
                 contents,
                 wildcard_filter,
                 force_download,
-            )
+            )
+
+
+@bootstrap_app.command("observation-variables")
+def bootstrap_observation_variables(
+        ctx: typer.Context,
+):
+    """Create initial observation variables."""
+    variables = [
+        observations_models.VariableCreate(
+            name="TDd",
+            description="Mean temperature",
+            unit="ºC"
+        ),
+        observations_models.VariableCreate(
+            name="TXd",
+            description="Max temperature",
+            unit="ºC"
+        ),
+        observations_models.VariableCreate(
+            name="TNd",
+            description="Min temperature",
+            unit="ºC"
+        ),
+        observations_models.VariableCreate(
+            name="PRCPTOT",
+            description="Total precipitation",
+            unit="mm"
+        ),
+        observations_models.VariableCreate(
+            name="TR",
+            description="Tropical nights",
+            unit="mm"
+        ),
+        observations_models.VariableCreate(
+            name="SU30",
+            description="Hot days",
+            unit="mm"
+        ),
+        observations_models.VariableCreate(
+            name="FD",
+            description="Cold days",
+            unit="mm"
+        ),
+    ]
+    with sqlmodel.Session(ctx.obj["engine"]) as session:
+        for var_create in variables:
+            try:
+                db_variable = database.create_variable(session, var_create)
+                print(f"Created observation variable {db_variable.name!r}")
+            except IntegrityError as err:
+                print(
+                    f"Could not create observation "
+                    f"variable {var_create.name!r}: {err}"
+                )
+                session.rollback()
+    print("Done!")

arpav_ppcv/migrations/versions/b02523194bd6_add_station_temporal_columns.py

@@ -0,0 +1,33 @@
+"""add station temporal columns
+
+Revision ID: b02523194bd6
+Revises: b9a2363d4257
+Create Date: 2024-05-07 15:06:56.860040
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'b02523194bd6'
+down_revision: Union[str, None] = 'b9a2363d4257'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('station', sa.Column('active_since', sa.Date(), nullable=True))
+    op.add_column('station', sa.Column('active_until', sa.Date(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('station', 'active_until')
+    op.drop_column('station', 'active_since')
+    # ### end Alembic commands ###

arpav_ppcv/observations_harvester/cliapp.py

@@ -10,17 +10,6 @@
 app = typer.Typer()
 
 
-@app.command()
-def refresh_variables(ctx: typer.Context) -> None:
-    client = httpx.Client()
-    with sqlmodel.Session(ctx.obj["engine"]) as session:
-        created, updated = operations.refresh_variables(client, session)
-        print(f"Created {len(created)} variables:")
-        print("\n".join(v.name for v in created))
-        print(f"Updated {len(updated)} variables:")
-        print("\n".join(v.name for v in updated))
-
-
 @app.command()
 def refresh_stations(ctx: typer.Context) -> None:
     client = httpx.Client()

arpav_ppcv/observations_harvester/operations.py

@@ -18,58 +18,6 @@
 logger = logging.getLogger(__name__)
 
 
-def harvest_variables(
-        client: httpx.Client,
-        db_session: sqlmodel.Session
-) -> tuple[
-    list[models.VariableCreate],
-    list[tuple[models.Variable, models.VariableUpdate]]
-]:
-    """
-    Queries remote API and returns a tuple with variables to create and update.
-    """
-    existing_variables = {v.name: v for v in database.collect_all_variables(db_session)}
-    response = client.get(
-        "https://api.arpa.veneto.it/REST/v1/clima_indicatori/tipo_indicatore",
-    )
-    response.raise_for_status()
-    to_create = []
-    to_update = []
-    for var_info in response.json().get("data", []):
-        variable_create = models.VariableCreate(
-            name=var_info["indicatore"],
-            description=var_info["descrizione"],
-            unit=var_info.get("unita", ""),
-        )
-        if variable_create.name not in existing_variables:
-            to_create.append(variable_create)
-        else:
-            existing_variable = existing_variables[variable_create.name]
-            if existing_variable.description != variable_create.description:
-                to_update.append(
-                    (
-                        existing_variable,
-                        models.VariableUpdate(description=variable_create.description)
-                    )
-                )
-    return to_create, to_update
-
-
-def refresh_variables(
-        client: httpx.Client,
-        db_session: sqlmodel.Session
-) -> tuple[list[models.Variable], list[models.Variable]]:
-    to_create, to_update = harvest_variables(client, db_session)
-    logger.info(f"About to create {len(to_create)} variables...")
-    created_variables = database.create_many_variables(db_session, to_create)
-    logger.info(f"About to update {len(to_update)} variables...")
-    updated_variables = []
-    for db_var, var_update in to_update:
-        updated = database.update_variable(db_session, db_var, var_update)
-        updated_variables.append(updated)
-    return created_variables, updated_variables
-
-
 def harvest_stations(
         client: httpx.Client, db_session: sqlmodel.Session
 ) -> list[models.StationCreate]:
@@ -93,6 +41,24 @@ def harvest_stations(
         response.raise_for_status()
         for raw_station in response.json().get("data", []):
             station_code = str(raw_station["statcd"])
+            if (raw_start := raw_station.get("iniziovalidita")):
+                try:
+                    active_since = dt.date(*(int(i) for i in raw_start.split("-")))
+                except TypeError:
+                    logger.warning(
+                        f"Could not extract a valid date from the input {raw_start!r}")
+                    active_since = None
+            else:
+                active_since = None
+            if (raw_end := raw_station.get("finevalidita")):
+                try:
+                    active_until = dt.date(*raw_end.split("-"))
+                except TypeError:
+                    logger.warning(
+                        f"Could not extract a valid date from the input {raw_end!r}")
+                    active_until = None
+            else:
+                active_until = None
             if (
                     station_code not in existing_stations and
                     station_code not in stations_create
@@ -109,6 +75,8 @@ def harvest_stations(
                     altitude_m=raw_station["altitude"],
                     name=raw_station["statnm"],
                     type_=raw_station["stattype"].lower().replace(" ", "_"),
+                    active_since=active_since,
+                    active_until=active_until,
                 )
                 stations_create[station_create.code] = station_create
     return list(stations_create.values())

arpav_ppcv/schemas/models.py

@@ -28,6 +28,8 @@ class StationBase(sqlmodel.SQLModel):
         )
     )
     code: str = sqlmodel.Field(unique=True)
+    active_since: Optional[dt.date] = None
+    active_until: Optional[dt.date] = None
 
 
 class Station(StationBase, table=True):
@@ -55,6 +57,8 @@ class StationCreate(sqlmodel.SQLModel):
     altitude_m: Optional[float] = None
     name: Optional[str] = ""
     type_: Optional[str] = ""
+    active_since: Optional[dt.date] = None
+    active_until: Optional[dt.date] = None
 
 
 class StationUpdate(sqlmodel.SQLModel):
@@ -63,6 +67,8 @@ class StationUpdate(sqlmodel.SQLModel):
     altitude_m: Optional[float] = None
     name: Optional[str] = None
     type_: Optional[str] = None
+    active_since: Optional[dt.date] = None
+    active_until: Optional[dt.date] = None
 
 
 class VariableBase(sqlmodel.SQLModel):