From fed1c65e0c7fbc718248db4abf892d8da7cc9207 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Tue, 7 May 2024 17:58:24 +0100 Subject: [PATCH] Added a CLI command to bootstrap observations variables and enhanced stations with temporal properties --- arpav_ppcv/database.py | 5 +- arpav_ppcv/main.py | 63 +++++++++++++++- ...2523194bd6_add_station_temporal_columns.py | 33 +++++++++ arpav_ppcv/observations_harvester/cliapp.py | 11 --- .../observations_harvester/operations.py | 72 ++++++------------- arpav_ppcv/schemas/models.py | 6 ++ 6 files changed, 122 insertions(+), 68 deletions(-) create mode 100644 arpav_ppcv/migrations/versions/b02523194bd6_add_station_temporal_columns.py diff --git a/arpav_ppcv/database.py b/arpav_ppcv/database.py index ec13e1e5..0dba088d 100644 --- a/arpav_ppcv/database.py +++ b/arpav_ppcv/database.py @@ -166,11 +166,8 @@ def create_many_stations( geom = shapely.io.from_geojson(station_create.geom.model_dump_json()) wkbelement = from_shape(geom) db_station = models.Station( - code=station_create.code, + **station_create.model_dump(exclude={"geom"}), geom=wkbelement, - altitude_m=station_create.altitude_m, - name=station_create.name, - type_=station_create.type_, ) db_records.append(db_station) session.add(db_station) diff --git a/arpav_ppcv/main.py b/arpav_ppcv/main.py index fa48a3d5..84c4821c 100644 --- a/arpav_ppcv/main.py +++ b/arpav_ppcv/main.py @@ -15,6 +15,7 @@ import anyio import django import httpx +import sqlmodel import typer import yaml from django.conf import settings as django_settings @@ -22,6 +23,7 @@ from rich import print from rich.padding import Padding from rich.panel import Panel +from sqlalchemy.exc import IntegrityError from . import ( config, @@ -29,16 +31,19 @@ ) from .cliapp.app import app as cli_app from .observations_harvester.cliapp import app as observations_harvester_app +from .schemas import models as observations_models from .thredds import crawler from .webapp.legacy.django_settings import get_custom_django_settings app = typer.Typer() db_app = typer.Typer() dev_app = typer.Typer() +bootstrap_app = typer.Typer() app.add_typer(cli_app, name="app") app.add_typer(db_app, name="db") app.add_typer(dev_app, name="dev") app.add_typer(observations_harvester_app, name="observations-harvester") +app.add_typer(bootstrap_app, name="bootstrap") @app.callback() @@ -229,4 +234,60 @@ def import_thredds_datasets( contents, wildcard_filter, force_download, - ) \ No newline at end of file + ) + + +@bootstrap_app.command("observation-variables") +def bootstrap_observation_variables( + ctx: typer.Context, +): + """Create initial observation variables.""" + variables = [ + observations_models.VariableCreate( + name="TDd", + description="Mean temperature", + unit="ºC" + ), + observations_models.VariableCreate( + name="TXd", + description="Max temperature", + unit="ºC" + ), + observations_models.VariableCreate( + name="TNd", + description="Min temperature", + unit="ºC" + ), + observations_models.VariableCreate( + name="PRCPTOT", + description="Total precipitation", + unit="mm" + ), + observations_models.VariableCreate( + name="TR", + description="Tropical nights", + unit="mm" + ), + observations_models.VariableCreate( + name="SU30", + description="Hot days", + unit="mm" + ), + observations_models.VariableCreate( + name="FD", + description="Cold days", + unit="mm" + ), + ] + with sqlmodel.Session(ctx.obj["engine"]) as session: + for var_create in variables: + try: + db_variable = database.create_variable(session, var_create) + print(f"Created observation variable {db_variable.name!r}") + except IntegrityError as err: + print( + f"Could not create observation " + f"variable {var_create.name!r}: {err}" + ) + session.rollback() + print("Done!") diff --git a/arpav_ppcv/migrations/versions/b02523194bd6_add_station_temporal_columns.py b/arpav_ppcv/migrations/versions/b02523194bd6_add_station_temporal_columns.py new file mode 100644 index 00000000..c7decc88 --- /dev/null +++ b/arpav_ppcv/migrations/versions/b02523194bd6_add_station_temporal_columns.py @@ -0,0 +1,33 @@ +"""add station temporal columns + +Revision ID: b02523194bd6 +Revises: b9a2363d4257 +Create Date: 2024-05-07 15:06:56.860040 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +# revision identifiers, used by Alembic. +revision: str = 'b02523194bd6' +down_revision: Union[str, None] = 'b9a2363d4257' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('station', sa.Column('active_since', sa.Date(), nullable=True)) + op.add_column('station', sa.Column('active_until', sa.Date(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('station', 'active_until') + op.drop_column('station', 'active_since') + # ### end Alembic commands ### diff --git a/arpav_ppcv/observations_harvester/cliapp.py b/arpav_ppcv/observations_harvester/cliapp.py index 33231260..22aacf45 100644 --- a/arpav_ppcv/observations_harvester/cliapp.py +++ b/arpav_ppcv/observations_harvester/cliapp.py @@ -10,17 +10,6 @@ app = typer.Typer() -@app.command() -def refresh_variables(ctx: typer.Context) -> None: - client = httpx.Client() - with sqlmodel.Session(ctx.obj["engine"]) as session: - created, updated = operations.refresh_variables(client, session) - print(f"Created {len(created)} variables:") - print("\n".join(v.name for v in created)) - print(f"Updated {len(updated)} variables:") - print("\n".join(v.name for v in updated)) - - @app.command() def refresh_stations(ctx: typer.Context) -> None: client = httpx.Client() diff --git a/arpav_ppcv/observations_harvester/operations.py b/arpav_ppcv/observations_harvester/operations.py index 224a5a50..7920449a 100644 --- a/arpav_ppcv/observations_harvester/operations.py +++ b/arpav_ppcv/observations_harvester/operations.py @@ -18,58 +18,6 @@ logger = logging.getLogger(__name__) -def harvest_variables( - client: httpx.Client, - db_session: sqlmodel.Session -) -> tuple[ - list[models.VariableCreate], - list[tuple[models.Variable, models.VariableUpdate]] -]: - """ - Queries remote API and returns a tuple with variables to create and update. - """ - existing_variables = {v.name: v for v in database.collect_all_variables(db_session)} - response = client.get( - "https://api.arpa.veneto.it/REST/v1/clima_indicatori/tipo_indicatore", - ) - response.raise_for_status() - to_create = [] - to_update = [] - for var_info in response.json().get("data", []): - variable_create = models.VariableCreate( - name=var_info["indicatore"], - description=var_info["descrizione"], - unit=var_info.get("unita", ""), - ) - if variable_create.name not in existing_variables: - to_create.append(variable_create) - else: - existing_variable = existing_variables[variable_create.name] - if existing_variable.description != variable_create.description: - to_update.append( - ( - existing_variable, - models.VariableUpdate(description=variable_create.description) - ) - ) - return to_create, to_update - - -def refresh_variables( - client: httpx.Client, - db_session: sqlmodel.Session -) -> tuple[list[models.Variable], list[models.Variable]]: - to_create, to_update = harvest_variables(client, db_session) - logger.info(f"About to create {len(to_create)} variables...") - created_variables = database.create_many_variables(db_session, to_create) - logger.info(f"About to update {len(to_update)} variables...") - updated_variables = [] - for db_var, var_update in to_update: - updated = database.update_variable(db_session, db_var, var_update) - updated_variables.append(updated) - return created_variables, updated_variables - - def harvest_stations( client: httpx.Client, db_session: sqlmodel.Session ) -> list[models.StationCreate]: @@ -93,6 +41,24 @@ def harvest_stations( response.raise_for_status() for raw_station in response.json().get("data", []): station_code = str(raw_station["statcd"]) + if (raw_start := raw_station.get("iniziovalidita")): + try: + active_since = dt.date(*(int(i) for i in raw_start.split("-"))) + except TypeError: + logger.warning( + f"Could not extract a valid date from the input {raw_start!r}") + active_since = None + else: + active_since = None + if (raw_end := raw_station.get("finevalidita")): + try: + active_until = dt.date(*raw_end.split("-")) + except TypeError: + logger.warning( + f"Could not extract a valid date from the input {raw_end!r}") + active_until = None + else: + active_until = None if ( station_code not in existing_stations and station_code not in stations_create @@ -109,6 +75,8 @@ def harvest_stations( altitude_m=raw_station["altitude"], name=raw_station["statnm"], type_=raw_station["stattype"].lower().replace(" ", "_"), + active_since=active_since, + active_until=active_until, ) stations_create[station_create.code] = station_create return list(stations_create.values()) diff --git a/arpav_ppcv/schemas/models.py b/arpav_ppcv/schemas/models.py index 8737ce39..4051bf99 100644 --- a/arpav_ppcv/schemas/models.py +++ b/arpav_ppcv/schemas/models.py @@ -28,6 +28,8 @@ class StationBase(sqlmodel.SQLModel): ) ) code: str = sqlmodel.Field(unique=True) + active_since: Optional[dt.date] = None + active_until: Optional[dt.date] = None class Station(StationBase, table=True): @@ -55,6 +57,8 @@ class StationCreate(sqlmodel.SQLModel): altitude_m: Optional[float] = None name: Optional[str] = "" type_: Optional[str] = "" + active_since: Optional[dt.date] = None + active_until: Optional[dt.date] = None class StationUpdate(sqlmodel.SQLModel): @@ -63,6 +67,8 @@ class StationUpdate(sqlmodel.SQLModel): altitude_m: Optional[float] = None name: Optional[str] = None type_: Optional[str] = None + active_since: Optional[dt.date] = None + active_until: Optional[dt.date] = None class VariableBase(sqlmodel.SQLModel):