From 467f3fd436072cc4fae075ba363aca3366e5265f Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Mon, 8 Jul 2024 11:46:17 +0100 Subject: [PATCH 1/6] Allow filtering configuration parameters by name --- arpav_ppcv/database.py | 49 +++++++++++++------ arpav_ppcv/webapp/api_v2/routers/coverages.py | 2 + 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/arpav_ppcv/database.py b/arpav_ppcv/database.py index 7cdbcfd7..421677ba 100644 --- a/arpav_ppcv/database.py +++ b/arpav_ppcv/database.py @@ -710,11 +710,16 @@ def list_configuration_parameters( limit: int = 20, offset: int = 0, include_total: bool = False, + name_filter: str | None = None, ) -> tuple[Sequence[coverages.ConfigurationParameter], Optional[int]]: """List existing configuration parameters.""" statement = sqlmodel.select(coverages.ConfigurationParameter).order_by( coverages.ConfigurationParameter.name ) + if name_filter is not None: + statement = _add_substring_filter( + statement, name_filter, coverages.ConfigurationParameter.name + ) items = session.exec(statement.offset(offset).limit(limit)).all() num_items = _get_total_num_records(session, statement) if include_total else None return items, num_items @@ -853,20 +858,20 @@ def list_coverage_configurations( coverages.CoverageConfiguration.name ) if name_filter is not None: - statement = statement.where( - coverages.CoverageConfiguration.name.ilike(name_filter) + statement = _add_substring_filter( + statement, name_filter, coverages.CoverageConfiguration.name ) if english_display_name_filter is not None: - statement = statement.where( - coverages.CoverageConfiguration.display_name_english.ilike( - english_display_name_filter - ) + statement = _add_substring_filter( + statement, + english_display_name_filter, + coverages.CoverageConfiguration.display_name_english, ) if italian_display_name_filter is not None: - statement = statement.where( - coverages.CoverageConfiguration.display_name_italian.ilike( - italian_display_name_filter - ) + statement = _add_substring_filter( + statement, + italian_display_name_filter, + coverages.CoverageConfiguration.display_name_italian, ) items = session.exec(statement.offset(offset).limit(limit)).all() num_items = _get_total_num_records(session, statement) if include_total else None @@ -1093,14 +1098,16 @@ def list_municipalities( municipalities.Municipality.name ) if name_filter is not None: - statement = statement.where(municipalities.Municipality.name.ilike(name_filter)) + statement = _add_substring_filter( + statement, name_filter, municipalities.Municipality.name + ) if province_name_filter is not None: - statement = statement.where( - municipalities.Municipality.province_name.ilike(province_name_filter) + statement = _add_substring_filter( + statement, province_name_filter, municipalities.Municipality.province_name ) if region_name_filter is not None: - statement = statement.where( - municipalities.Municipality.region_name.ilike(region_name_filter) + statement = _add_substring_filter( + statement, region_name_filter, municipalities.Municipality.region_name ) if polygon_intersection_filter is not None: statement = statement.where( @@ -1187,3 +1194,15 @@ def _get_total_num_records(session: sqlmodel.Session, statement): return session.exec( sqlmodel.select(sqlmodel.func.count()).select_from(statement) ).first() + + +def _add_substring_filter(statement, value: str, *columns): + filter_ = value.replace("%", "") + filter_ = f"%{filter_}%" + if len(columns) == 1: + result = statement.where(columns[0].ilike(filter_)) # type: ignore[attr-defined] + elif len(columns) > 1: + result = statement.where(sqlalchemy.or_(*[c.ilike(filter_) for c in columns])) + else: + raise RuntimeError("Invalid columns argument") + return result diff --git a/arpav_ppcv/webapp/api_v2/routers/coverages.py b/arpav_ppcv/webapp/api_v2/routers/coverages.py index 04b1ca3d..3761c28c 100644 --- a/arpav_ppcv/webapp/api_v2/routers/coverages.py +++ b/arpav_ppcv/webapp/api_v2/routers/coverages.py @@ -52,6 +52,7 @@ async def list_configuration_parameters( request: Request, db_session: Annotated[Session, Depends(dependencies.get_db_session)], list_params: Annotated[dependencies.CommonListFilterParameters, Depends()], + name_contains: str | None = None, ): """List configuration parameters.""" config_params, filtered_total = db.list_configuration_parameters( @@ -59,6 +60,7 @@ async def list_configuration_parameters( limit=list_params.limit, offset=list_params.offset, include_total=True, + name_filter=name_contains, ) _, unfiltered_total = db.list_configuration_parameters( db_session, limit=1, offset=0, include_total=True From c7a56a6afc2d27115c1fc7357ab83bbeaf4e10d6 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Mon, 15 Jul 2024 17:06:19 +0100 Subject: [PATCH 2/6] Allow filtering list items in the admin --- arpav_ppcv/database.py | 10 ++++++++++ arpav_ppcv/webapp/admin/views/coverages.py | 2 ++ arpav_ppcv/webapp/admin/views/observations.py | 4 +++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/arpav_ppcv/database.py b/arpav_ppcv/database.py index 421677ba..227e89d3 100644 --- a/arpav_ppcv/database.py +++ b/arpav_ppcv/database.py @@ -120,11 +120,16 @@ def list_variables( limit: int = 20, offset: int = 0, include_total: bool = False, + name_filter: Optional[str] = None, ) -> tuple[Sequence[observations.Variable], Optional[int]]: """List existing variables.""" statement = sqlmodel.select(observations.Variable).order_by( observations.Variable.name ) + if name_filter is not None: + statement = _add_substring_filter( + statement, name_filter, observations.Variable.name + ) items = session.exec(statement.offset(offset).limit(limit)).all() num_items = _get_total_num_records(session, statement) if include_total else None return items, num_items @@ -236,6 +241,7 @@ def list_stations( limit: int = 20, offset: int = 0, include_total: bool = False, + name_filter: Optional[str] = None, polygon_intersection_filter: shapely.Polygon = None, variable_id_filter: Optional[uuid.UUID] = None, variable_aggregation_type: Optional[ @@ -250,6 +256,10 @@ def list_stations( statement = sqlmodel.select(observations.Station).order_by( observations.Station.code ) + if name_filter is not None: + statement = _add_substring_filter( + statement, name_filter, observations.Station.name + ) if polygon_intersection_filter is not None: statement = statement.where( func.ST_Intersects( diff --git a/arpav_ppcv/webapp/admin/views/coverages.py b/arpav_ppcv/webapp/admin/views/coverages.py index 5127a37f..bd730e13 100644 --- a/arpav_ppcv/webapp/admin/views/coverages.py +++ b/arpav_ppcv/webapp/admin/views/coverages.py @@ -245,6 +245,7 @@ async def find_all( database.list_configuration_parameters, limit=limit, offset=skip, + name_filter=str(where) if where not in (None, "") else None, include_total=False, ) db_conf_params, _ = await anyio.to_thread.run_sync( @@ -439,6 +440,7 @@ async def find_all( database.list_coverage_configurations, limit=limit, offset=skip, + name_filter=str(where) if where not in (None, "") else None, include_total=False, ) db_cov_confs, _ = await anyio.to_thread.run_sync( diff --git a/arpav_ppcv/webapp/admin/views/observations.py b/arpav_ppcv/webapp/admin/views/observations.py index 9bf0c60a..323342eb 100644 --- a/arpav_ppcv/webapp/admin/views/observations.py +++ b/arpav_ppcv/webapp/admin/views/observations.py @@ -297,6 +297,7 @@ async def find_all( db.list_variables, limit=limit, offset=skip, + name_filter=str(where) if where not in (None, "") else None, include_total=False, ) db_vars, _ = await anyio.to_thread.run_sync( @@ -319,7 +320,7 @@ class StationView(ModelView): fields.UuidField("id"), starlette_admin.StringField("name", required=True), starlette_admin.StringField("code", required=True), - starlette_admin.StringField("type", required=True), + starlette_admin.StringField("type_", required=True), starlette_admin.FloatField("longitude", required=True), starlette_admin.FloatField("latitude", required=True), starlette_admin.DateField("active_since"), @@ -448,6 +449,7 @@ async def find_all( limit=limit, offset=skip, include_total=False, + name_filter=str(where) if where not in (None, "") else None, ) db_stations, _ = await anyio.to_thread.run_sync( list_stations, request.state.session From 18ea42f197f83d22f659825e6fcd4c3053f38dec Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 18 Jul 2024 14:56:25 +0100 Subject: [PATCH 3/6] wip - working on filtering coverage configurations from possible values --- arpav_ppcv/database.py | 83 +++- arpav_ppcv/schemas/coverages.py | 21 +- arpav_ppcv/webapp/api_v2/routers/coverages.py | 13 + tests/notebooks/generic.ipynb | 441 +++++++----------- 4 files changed, 265 insertions(+), 293 deletions(-) diff --git a/arpav_ppcv/database.py b/arpav_ppcv/database.py index 227e89d3..dac7f165 100644 --- a/arpav_ppcv/database.py +++ b/arpav_ppcv/database.py @@ -665,17 +665,42 @@ def get_configuration_parameter_value( ) +def get_configuration_parameter_value_by_names( + session: sqlmodel.Session, + parameter_name: str, + value_name: str, +) -> Optional[coverages.ConfigurationParameterValue]: + """Get configuration parameter value by name of parameter and name of value.""" + return session.exec( + sqlmodel.select(coverages.ConfigurationParameterValue) + .join(coverages.ConfigurationParameter) + .where( + coverages.ConfigurationParameter.name == parameter_name, + coverages.ConfigurationParameterValue.name == value_name, + ) + ).first() + + def list_configuration_parameter_values( session: sqlmodel.Session, *, limit: int = 20, offset: int = 0, include_total: bool = False, + used_by_coverage_configuration: coverages.CoverageConfiguration | None = None, ) -> tuple[Sequence[coverages.ConfigurationParameterValue], Optional[int]]: """List existing configuration parameters.""" statement = sqlmodel.select(coverages.ConfigurationParameterValue).order_by( coverages.ConfigurationParameterValue.name ) + if used_by_coverage_configuration is not None: + statement = ( + statement.join(coverages.ConfigurationParameterPossibleValue) + .join(coverages.CoverageConfiguration) + .where( + coverages.CoverageConfiguration.id == used_by_coverage_configuration.id + ) + ) items = session.exec(statement.offset(offset).limit(limit)).all() num_items = _get_total_num_records(session, statement) if include_total else None return items, num_items @@ -683,12 +708,19 @@ def list_configuration_parameter_values( def collect_all_configuration_parameter_values( session: sqlmodel.Session, + used_by_coverage_configuration: coverages.CoverageConfiguration | None = None, ) -> Sequence[coverages.ConfigurationParameterValue]: _, num_total = list_configuration_parameter_values( - session, limit=1, include_total=True + session, + limit=1, + include_total=True, + used_by_coverage_configuration=used_by_coverage_configuration, ) result, _ = list_configuration_parameter_values( - session, limit=num_total, include_total=False + session, + limit=num_total, + include_total=False, + used_by_coverage_configuration=used_by_coverage_configuration, ) return result @@ -862,6 +894,9 @@ def list_coverage_configurations( name_filter: Optional[str] = None, english_display_name_filter: Optional[str] = None, italian_display_name_filter: Optional[str] = None, + configuration_parameter_value_filter: Optional[ + list[coverages.ConfigurationParameterValue] + ] = None, ) -> tuple[Sequence[coverages.CoverageConfiguration], Optional[int]]: """List existing coverage configurations.""" statement = sqlmodel.select(coverages.CoverageConfiguration).order_by( @@ -893,6 +928,9 @@ def collect_all_coverage_configurations( name_filter: Optional[str] = None, english_display_name_filter: Optional[str] = None, italian_display_name_filter: Optional[str] = None, + configuration_parameter_value_filter: Optional[ + list[coverages.ConfigurationParameterValue] + ] = None, ) -> Sequence[coverages.CoverageConfiguration]: _, num_total = list_coverage_configurations( session, @@ -1055,9 +1093,11 @@ def list_allowed_coverage_identifiers( session: sqlmodel.Session, *, coverage_configuration_id: uuid.UUID, + matching_configuration_parameters: list[coverages.ConfigurationParameterValue] + | None = None, ) -> list[str]: """Build list of legal coverage identifiers.""" - result = [] + allowed_identifiers = [] db_cov_conf = get_coverage_configuration(session, coverage_configuration_id) if db_cov_conf is not None: pattern_parts = re.findall( @@ -1083,7 +1123,21 @@ def list_allowed_coverage_identifiers( values_to_combine[index] = [pattern_parts[index]] for combination in itertools.product(*values_to_combine): dataset_id = "-".join((db_cov_conf.name, *combination)) - result.append(dataset_id) + allowed_identifiers.append(dataset_id) + + result = [] + if len(matching_configuration_parameters or []) > 0: + for id_ in allowed_identifiers: + parsed_parts = id_.split("-")[1:] + for idx, param_name in enumerate(pattern_parts): + param_value = parsed_parts[idx] + pv = get_configuration_parameter_value_by_names( + session, param_name, param_value + ) + if pv in matching_configuration_parameters: + result.append(id_) + else: + result = allowed_identifiers return result @@ -1165,6 +1219,10 @@ def create_many_municipalities( return db_records +def get_relevant_coverage_configurations(): + ... + + def list_coverage_identifiers( session: sqlmodel.Session, *, @@ -1172,8 +1230,12 @@ def list_coverage_identifiers( offset: int = 0, include_total: bool = False, name_filter: list[str] | None = None, + matching_configuration_parameters: list[coverages.ConfigurationParameterValue] + | None = None, ) -> tuple[list[coverages.CoverageInternal], Optional[int]]: - all_cov_ids = collect_all_coverage_identifiers(session) + all_cov_ids = collect_all_coverage_identifiers( + session, matching_configuration_parameters + ) if name_filter is not None: for fragment in name_filter: all_cov_ids = [ @@ -1187,13 +1249,18 @@ def list_coverage_identifiers( def collect_all_coverage_identifiers( session: sqlmodel.Session, + matching_configuration_parameters: list[coverages.ConfigurationParameterValue] + | None = None, ) -> list[coverages.CoverageInternal]: cov_confs = collect_all_coverage_configurations(session) cov_ids = [] for cov_conf in cov_confs: - for cov_id in list_allowed_coverage_identifiers( - session, coverage_configuration_id=cov_conf.id - ): + allowed_identifiers = list_allowed_coverage_identifiers( + session, + coverage_configuration_id=cov_conf.id, + matching_configuration_parameters=matching_configuration_parameters, + ) + for cov_id in allowed_identifiers: cov_ids.append( coverages.CoverageInternal(configuration=cov_conf, identifier=cov_id) ) diff --git a/arpav_ppcv/schemas/coverages.py b/arpav_ppcv/schemas/coverages.py index 1167b6d5..3739365c 100644 --- a/arpav_ppcv/schemas/coverages.py +++ b/arpav_ppcv/schemas/coverages.py @@ -235,10 +235,13 @@ class CoverageConfiguration(sqlmodel.SQLModel, table=True): @pydantic.computed_field() @property def coverage_id_pattern(self) -> str: - id_parts = ["{name}"] - for match_obj in re.finditer(r"(\{\w+\})", self.thredds_url_pattern): - id_parts.append(match_obj.group(1)) - return "-".join(id_parts) + other_parts = set() + for pv in self.possible_values: + other_parts.add( + pv.configuration_parameter_value.configuration_parameter.name + ) + all_parts = ["name"] + sorted(list(other_parts)) + return "-".join(f"{{{part}}}" for part in all_parts) def get_thredds_url_fragment(self, coverage_identifier: str) -> str: try: @@ -269,7 +272,9 @@ def build_coverage_identifier( id_parts.append(conf_param_value.name) break else: - raise ValueError(f"Invalid param_name {param_name!r}") + raise ValueError( + f"Could not find suitable value for {param_name!r}" + ) else: continue return "-".join(id_parts) @@ -293,15 +298,17 @@ def retrieve_used_values( raise ValueError(f"Invalid parameter/value pair: {(param_name, value)}") return result - def retrieve_configuration_parameters(self, coverage_identifier) -> dict[str, str]: + def retrieve_configuration_parameters( + self, coverage_identifier: str + ) -> dict[str, str]: pattern_parts = re.finditer( r"\{(\w+)\}", self.coverage_id_pattern.partition("-")[-1] ) id_parts = coverage_identifier.split("-")[1:] result = {} for index, pattern_match_obj in enumerate(pattern_parts): - id_part = id_parts[index] configuration_parameter_name = pattern_match_obj.group(1) + id_part = id_parts[index] result[configuration_parameter_name] = id_part return result diff --git a/arpav_ppcv/webapp/api_v2/routers/coverages.py b/arpav_ppcv/webapp/api_v2/routers/coverages.py index 3761c28c..7eaf0cc4 100644 --- a/arpav_ppcv/webapp/api_v2/routers/coverages.py +++ b/arpav_ppcv/webapp/api_v2/routers/coverages.py @@ -158,6 +158,9 @@ def get_coverage_configuration( ) +# PossibleValue: pydantic.StringConstraints(pattern="^[\w-_]+:[\w-_]+$") + + @router.get( "/coverage-identifiers", response_model=coverage_schemas.CoverageIdentifierList, @@ -168,7 +171,17 @@ def list_coverage_identifiers( db_session: Annotated[Session, Depends(dependencies.get_db_session)], list_params: Annotated[dependencies.CommonListFilterParameters, Depends()], name_contains: Annotated[list[str], Query()] = None, + possible_value: Annotated[ + list[ + Annotated[ + str, + pydantic.StringConstraints(pattern=r"^[0-9a-zA-Z_]+:[0-9a-zA-Z_]+$"), + ] + ], + Query(), + ] = None, ): + logger.debug(f"{possible_value=}") cov_internals, filtered_total = db.list_coverage_identifiers( db_session, limit=list_params.limit, diff --git a/tests/notebooks/generic.ipynb b/tests/notebooks/generic.ipynb index 773f91c2..309e8d49 100644 --- a/tests/notebooks/generic.ipynb +++ b/tests/notebooks/generic.ipynb @@ -51,7 +51,7 @@ "metadata": {}, "outputs": [], "source": [ - "coverage_identifier = \"tas_seasonal_absolute_model_ensemble-rcp26-DJF\"\n", + "coverage_identifier = \"tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp26-DJF\"\n", "point_coords = \"POINT(11.5469 44.9524)\"\n", "date_range = \"../..\"\n", "\n", @@ -63,366 +63,251 @@ }, { "cell_type": "code", - "execution_count": 3, - "id": "4ac51747-ee8e-468a-a791-5f815b80286a", + "execution_count": 6, + "id": "6fa57746-a891-449a-b8c0-313de8470cd5", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "['tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp26-DJF',\n", + " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp26-MAM',\n", + " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp26-JJA',\n", + " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp26-SON']" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "settings.debug = False" + "db.list_allowed_coverage_identifiers(\n", + " session, \n", + " coverage_configuration_id=cov.configuration.id, \n", + " select_configuration_parameter_values=[\n", + " db.get_configuration_parameter_value_by_names(session, \"scenario\", \"rcp26\"),\n", + " db.get_configuration_parameter_value_by_names(session, \"time_period\", \"MAM\"),\n", + " ]\n", + ")" ] }, { "cell_type": "code", - "execution_count": 4, - "id": "93c28235-3793-4b3b-acd0-c00a56252cdf", - "metadata": { - "scrolled": true - }, + "execution_count": 7, + "id": "c90bcb97-aff2-454c-bc88-eaf30676cdd5", + "metadata": {}, "outputs": [ { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:asyncio:Using selector: EpollSelector\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "elapsed: 0.49170613288879395\n" - ] + "data": { + "text/plain": [ + "([CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='ensembletwbc/std/clipped/eca_cdd_an_avgagree_{time_window}_{scenario}_{year_period}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_agree_model_ensemble', wms_main_layer_name='consecutive_dry_days_index_per_time_period-uncertainty_group', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d50ef8e7-1a84-435e-84e6-294664b0964c'), unit='gg', display_name_english=None, palette='uncert-stippled/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_agree_model_ensemble-rcp26-tw1-DJF'),\n", + " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='ensembletwbc/std/clipped/eca_cdd_an_avgagree_{time_window}_{scenario}_{year_period}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_agree_model_ensemble', wms_main_layer_name='consecutive_dry_days_index_per_time_period-uncertainty_group', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d50ef8e7-1a84-435e-84e6-294664b0964c'), unit='gg', display_name_english=None, palette='uncert-stippled/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_agree_model_ensemble-rcp26-tw1-MAM'),\n", + " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='ensembletwbc/std/clipped/eca_cdd_an_avgagree_{time_window}_{scenario}_{year_period}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_agree_model_ensemble', wms_main_layer_name='consecutive_dry_days_index_per_time_period-uncertainty_group', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d50ef8e7-1a84-435e-84e6-294664b0964c'), unit='gg', display_name_english=None, palette='uncert-stippled/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_agree_model_ensemble-rcp26-tw1-JJA'),\n", + " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='ensembletwbc/std/clipped/eca_cdd_an_avgagree_{time_window}_{scenario}_{year_period}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_agree_model_ensemble', wms_main_layer_name='consecutive_dry_days_index_per_time_period-uncertainty_group', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d50ef8e7-1a84-435e-84e6-294664b0964c'), unit='gg', display_name_english=None, palette='uncert-stippled/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_agree_model_ensemble-rcp26-tw1-SON'),\n", + " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='ensembletwbc/std/clipped/eca_cdd_an_avgagree_{time_window}_{scenario}_{year_period}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_agree_model_ensemble', wms_main_layer_name='consecutive_dry_days_index_per_time_period-uncertainty_group', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d50ef8e7-1a84-435e-84e6-294664b0964c'), unit='gg', display_name_english=None, palette='uncert-stippled/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_agree_model_ensemble-rcp26-tw2-DJF'),\n", + " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='ensembletwbc/std/clipped/eca_cdd_an_avgagree_{time_window}_{scenario}_{year_period}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_agree_model_ensemble', wms_main_layer_name='consecutive_dry_days_index_per_time_period-uncertainty_group', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d50ef8e7-1a84-435e-84e6-294664b0964c'), unit='gg', display_name_english=None, palette='uncert-stippled/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_agree_model_ensemble-rcp26-tw2-MAM'),\n", + " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='ensembletwbc/std/clipped/eca_cdd_an_avgagree_{time_window}_{scenario}_{year_period}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_agree_model_ensemble', wms_main_layer_name='consecutive_dry_days_index_per_time_period-uncertainty_group', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d50ef8e7-1a84-435e-84e6-294664b0964c'), unit='gg', display_name_english=None, palette='uncert-stippled/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_agree_model_ensemble-rcp26-tw2-JJA'),\n", + " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='ensembletwbc/std/clipped/eca_cdd_an_avgagree_{time_window}_{scenario}_{year_period}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_agree_model_ensemble', wms_main_layer_name='consecutive_dry_days_index_per_time_period-uncertainty_group', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d50ef8e7-1a84-435e-84e6-294664b0964c'), unit='gg', display_name_english=None, palette='uncert-stippled/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_agree_model_ensemble-rcp26-tw2-SON'),\n", + " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='indici5rcm/clipped/eca_cdd_an_EC-EARTH_CCLM4-8-17_{scenario}_{year_period}_{time_window}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17', wms_main_layer_name='consecutive_dry_days_index_per_time_period', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d04c8d32-277b-4d50-ba68-02836523c3af'), unit='gg', display_name_english=None, palette='default/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17-rcp26-tw1-DJF'),\n", + " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='indici5rcm/clipped/eca_cdd_an_EC-EARTH_CCLM4-8-17_{scenario}_{year_period}_{time_window}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17', wms_main_layer_name='consecutive_dry_days_index_per_time_period', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d04c8d32-277b-4d50-ba68-02836523c3af'), unit='gg', display_name_english=None, palette='default/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17-rcp26-tw1-MAM'),\n", + " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='indici5rcm/clipped/eca_cdd_an_EC-EARTH_CCLM4-8-17_{scenario}_{year_period}_{time_window}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17', wms_main_layer_name='consecutive_dry_days_index_per_time_period', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d04c8d32-277b-4d50-ba68-02836523c3af'), unit='gg', display_name_english=None, palette='default/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17-rcp26-tw1-JJA'),\n", + " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='indici5rcm/clipped/eca_cdd_an_EC-EARTH_CCLM4-8-17_{scenario}_{year_period}_{time_window}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17', wms_main_layer_name='consecutive_dry_days_index_per_time_period', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d04c8d32-277b-4d50-ba68-02836523c3af'), unit='gg', display_name_english=None, palette='default/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17-rcp26-tw1-SON'),\n", + " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='indici5rcm/clipped/eca_cdd_an_EC-EARTH_CCLM4-8-17_{scenario}_{year_period}_{time_window}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17', wms_main_layer_name='consecutive_dry_days_index_per_time_period', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d04c8d32-277b-4d50-ba68-02836523c3af'), unit='gg', display_name_english=None, palette='default/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17-rcp26-tw2-DJF'),\n", + " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='indici5rcm/clipped/eca_cdd_an_EC-EARTH_CCLM4-8-17_{scenario}_{year_period}_{time_window}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17', wms_main_layer_name='consecutive_dry_days_index_per_time_period', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d04c8d32-277b-4d50-ba68-02836523c3af'), unit='gg', display_name_english=None, palette='default/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17-rcp26-tw2-MAM'),\n", + " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='indici5rcm/clipped/eca_cdd_an_EC-EARTH_CCLM4-8-17_{scenario}_{year_period}_{time_window}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17', wms_main_layer_name='consecutive_dry_days_index_per_time_period', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d04c8d32-277b-4d50-ba68-02836523c3af'), unit='gg', display_name_english=None, palette='default/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17-rcp26-tw2-JJA'),\n", + " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='indici5rcm/clipped/eca_cdd_an_EC-EARTH_CCLM4-8-17_{scenario}_{year_period}_{time_window}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17', wms_main_layer_name='consecutive_dry_days_index_per_time_period', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d04c8d32-277b-4d50-ba68-02836523c3af'), unit='gg', display_name_english=None, palette='default/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17-rcp26-tw2-SON'),\n", + " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='indici5rcm/clipped/eca_cdd_an_EC-EARTH_RACMO22E_{scenario}_{year_period}_{time_window}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_model_ec_earth_racmo22e', wms_main_layer_name='consecutive_dry_days_index_per_time_period', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('bb1697d1-d3bd-46aa-88be-f912a0c39ab9'), unit='gg', display_name_english=None, palette='default/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_model_ec_earth_racmo22e-rcp26-tw1-DJF'),\n", + " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='indici5rcm/clipped/eca_cdd_an_EC-EARTH_RACMO22E_{scenario}_{year_period}_{time_window}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_model_ec_earth_racmo22e', wms_main_layer_name='consecutive_dry_days_index_per_time_period', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('bb1697d1-d3bd-46aa-88be-f912a0c39ab9'), unit='gg', display_name_english=None, palette='default/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_model_ec_earth_racmo22e-rcp26-tw1-MAM'),\n", + " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='indici5rcm/clipped/eca_cdd_an_EC-EARTH_RACMO22E_{scenario}_{year_period}_{time_window}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_model_ec_earth_racmo22e', wms_main_layer_name='consecutive_dry_days_index_per_time_period', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('bb1697d1-d3bd-46aa-88be-f912a0c39ab9'), unit='gg', display_name_english=None, palette='default/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_model_ec_earth_racmo22e-rcp26-tw1-JJA'),\n", + " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='indici5rcm/clipped/eca_cdd_an_EC-EARTH_RACMO22E_{scenario}_{year_period}_{time_window}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_model_ec_earth_racmo22e', wms_main_layer_name='consecutive_dry_days_index_per_time_period', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('bb1697d1-d3bd-46aa-88be-f912a0c39ab9'), unit='gg', display_name_english=None, palette='default/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_model_ec_earth_racmo22e-rcp26-tw1-SON')],\n", + " None)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "new_time_start = time.time()\n", - "coverage_series, observation_series = operations.get_coverage_time_series(\n", - " settings=settings,\n", - " session=session,\n", - " http_client=httpx.AsyncClient(),\n", - " coverage=cov,\n", - " point_geom=shapely.io.from_wkt(point_coords),\n", - " temporal_range=date_range,\n", - " coverage_smoothing_strategies=[\n", - " CoverageDataSmoothingStrategy.NO_SMOOTHING,\n", - " CoverageDataSmoothingStrategy.MOVING_AVERAGE_11_YEARS,\n", - " CoverageDataSmoothingStrategy.LOESS_SMOOTHING\n", - " ],\n", - " observation_smoothing_strategies=[\n", - " ObservationDataSmoothingStrategy.NO_SMOOTHING,\n", - " ObservationDataSmoothingStrategy.MOVING_AVERAGE_5_YEARS\n", - " ],\n", - " include_coverage_data=True,\n", - " include_observation_data=False,\n", - " include_coverage_uncertainty=True,\n", - " include_coverage_related_data=True\n", - ")\n", - "new_time_end = time.time()\n", - "print(f\"elapsed: {new_time_end - new_time_start}\")" + "db.list_coverage_identifiers(\n", + " session,\n", + " matching_configuration_parameters=[\n", + " db.get_configuration_parameter_value_by_names(session, \"scenario\", \"rcp26\"),\n", + " db.get_configuration_parameter_value_by_names(session, \"climatological_model\", \"model_ensemble\"),\n", + " ]\n", + ")" ] }, { "cell_type": "code", - "execution_count": 5, - "id": "63fa30a2-13f9-4157-b3f7-dc131541d3b2", + "execution_count": 3, + "id": "4ac51747-ee8e-468a-a791-5f815b80286a", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[('tas_seasonal_absolute_model_ec_earth_cclm4_8_17-rcp26-DJF', 'NO_SMOOTHING'),\n", - " ('tas_seasonal_absolute_model_ec_earth_cclm4_8_17-rcp26-DJF',\n", - " 'MOVING_AVERAGE_11_YEARS'),\n", - " ('tas_seasonal_absolute_model_ec_earth_cclm4_8_17-rcp26-DJF',\n", - " 'LOESS_SMOOTHING'),\n", - " ('tas_seasonal_absolute_model_ensemble-rcp26-DJF', 'NO_SMOOTHING'),\n", - " ('tas_seasonal_absolute_model_ensemble-rcp26-DJF', 'MOVING_AVERAGE_11_YEARS'),\n", - " ('tas_seasonal_absolute_model_ensemble-rcp26-DJF', 'LOESS_SMOOTHING'),\n", - " ('tas_seasonal_absolute_model_ec_earth_racmo22e-rcp26-DJF', 'NO_SMOOTHING'),\n", - " ('tas_seasonal_absolute_model_ec_earth_racmo22e-rcp26-DJF',\n", - " 'MOVING_AVERAGE_11_YEARS'),\n", - " ('tas_seasonal_absolute_model_ec_earth_racmo22e-rcp26-DJF',\n", - " 'LOESS_SMOOTHING'),\n", - " ('tas_seasonal_absolute_model_ec_earth_rca4-rcp26-DJF', 'NO_SMOOTHING'),\n", - " ('tas_seasonal_absolute_model_ec_earth_rca4-rcp26-DJF',\n", - " 'MOVING_AVERAGE_11_YEARS'),\n", - " ('tas_seasonal_absolute_model_ec_earth_rca4-rcp26-DJF', 'LOESS_SMOOTHING'),\n", - " ('tas_seasonal_absolute_model_ensemble_upper_uncertainty-rcp26-DJF',\n", - " 'NO_SMOOTHING'),\n", - " ('tas_seasonal_absolute_model_ensemble_upper_uncertainty-rcp26-DJF',\n", - " 'MOVING_AVERAGE_11_YEARS'),\n", - " ('tas_seasonal_absolute_model_ensemble_upper_uncertainty-rcp26-DJF',\n", - " 'LOESS_SMOOTHING'),\n", - " ('tas_seasonal_absolute_model_mpi_esm_lr_remo2009-rcp26-DJF', 'NO_SMOOTHING'),\n", - " ('tas_seasonal_absolute_model_mpi_esm_lr_remo2009-rcp26-DJF',\n", - " 'MOVING_AVERAGE_11_YEARS'),\n", - " ('tas_seasonal_absolute_model_mpi_esm_lr_remo2009-rcp26-DJF',\n", - " 'LOESS_SMOOTHING'),\n", - " ('tas_seasonal_absolute_model_hadgem2_es_racmo22e-rcp26-DJF', 'NO_SMOOTHING'),\n", - " ('tas_seasonal_absolute_model_hadgem2_es_racmo22e-rcp26-DJF',\n", - " 'MOVING_AVERAGE_11_YEARS'),\n", - " ('tas_seasonal_absolute_model_hadgem2_es_racmo22e-rcp26-DJF',\n", - " 'LOESS_SMOOTHING'),\n", - " ('tas_seasonal_absolute_model_ensemble_lower_uncertainty-rcp26-DJF',\n", - " 'NO_SMOOTHING'),\n", - " ('tas_seasonal_absolute_model_ensemble_lower_uncertainty-rcp26-DJF',\n", - " 'MOVING_AVERAGE_11_YEARS'),\n", - " ('tas_seasonal_absolute_model_ensemble_lower_uncertainty-rcp26-DJF',\n", - " 'LOESS_SMOOTHING')]" + "'{name}-{climatological_model}-{measure}-{scenario}-{year_period}'" ] }, - "execution_count": 5, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "[(c.identifier, s.value) for c, s in coverage_series.keys()]" + "cov.configuration.coverage_id_pattern" ] }, { "cell_type": "code", - "execution_count": 5, - "id": "b3811df1-eb09-45eb-8bd6-e85286007a66", + "execution_count": 4, + "id": "705a7335-18df-474e-912a-8bd6dfca940d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'climatological_model': 'model_ensemble',\n", + " 'measure': 'absolute',\n", + " 'scenario': 'rcp26',\n", + " 'year_period': 'DJF'}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "raw_data = await operations.retrieve_multiple_ncss_datasets(\n", - " settings, \n", - " httpx.AsyncClient(), \n", - " [\n", - " CoverageInternal(\n", - " identifier=coverage_identifier, \n", - " configuration=db.get_coverage_configuration_by_coverage_identifier(\n", - " session, coverage_identifier)\n", - " ),\n", - " ],\n", - " shapely.io.from_wkt(point_coords),\n", - " (None, None)\n", - ")\n", - " " + "cov.configuration.retrieve_configuration_parameters(cov.identifier)" ] }, { "cell_type": "code", - "execution_count": 6, - "id": "d369153d-c155-4e27-ba7c-7228a56dd0f9", + "execution_count": 5, + "id": "db7e48fd-98fb-439e-8291-a9fd005aec21", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[ConfigurationParameterPossibleValue(configuration_parameter_value_id=UUID('c5381cd2-ca57-48a9-a096-71568e812366'), coverage_configuration_id=UUID('06c104d2-a502-400b-9728-b8e2d9ee65d9')),\n", + " ConfigurationParameterPossibleValue(configuration_parameter_value_id=UUID('88fa9565-8af6-4ab5-afeb-bfa00cd97955'), coverage_configuration_id=UUID('06c104d2-a502-400b-9728-b8e2d9ee65d9')),\n", + " ConfigurationParameterPossibleValue(configuration_parameter_value_id=UUID('70c82437-5e90-4181-96c7-262a097d18ba'), coverage_configuration_id=UUID('06c104d2-a502-400b-9728-b8e2d9ee65d9')),\n", + " ConfigurationParameterPossibleValue(configuration_parameter_value_id=UUID('4842fa35-b13a-446d-ad9f-43a45b82bd23'), coverage_configuration_id=UUID('06c104d2-a502-400b-9728-b8e2d9ee65d9'))]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "for cov, data_ in raw_data.items():\n", - " df = operations._parse_ncss_dataset(data_, cov.configuration.netcdf_main_dataset_name, None, None, cov.identifier)" + "cov.configuration.retrieve_used_values(cov.identifier)" ] }, { "cell_type": "code", "execution_count": 7, - "id": "ca25cce2-36fb-44fb-a60c-b3f8ede028b8", + "id": "5dd3f90c-fc59-4bec-a613-2297537f190f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'ensymbc/clipped/tas_avg_rcp26_DJF_ts19762100_ls_VFVG.nc'" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "import pyloess" + "cov.configuration.get_thredds_url_fragment(cov.identifier)" ] }, { "cell_type": "code", - "execution_count": 10, - "id": "16946c48-0c1e-46bf-9cd2-453ee8f5bc2b", + "execution_count": 6, + "id": "9f396350-e5ea-455c-a636-87fd763f3d89", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp26-DJF'" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "smoothed = pyloess.loess(df.index.year.astype(\"int\").values, df[coverage_identifier], span=0.75, degree=2)" + "cov.configuration.build_coverage_identifier(\n", + " parameters=[\n", + " db.get_configuration_parameter_value_by_names(session, \"climatological_model\", \"model_ensemble\"),\n", + " db.get_configuration_parameter_value_by_names(session, \"measure\", \"absolute\"),\n", + " db.get_configuration_parameter_value_by_names(session, \"scenario\", \"rcp26\"),\n", + " db.get_configuration_parameter_value_by_names(session, \"year_period\", \"DJF\"),\n", + " ]\n", + ")" ] }, { "cell_type": "code", - "execution_count": 11, - "id": "60dd97fa-6a8f-4ed8-b25b-d13a38d9d89b", - "metadata": { - "scrolled": true - }, + "execution_count": 7, + "id": "ec2c0ead-cd94-46d5-bacc-67ab35a0238f", + "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "array([[1976. , 3.39045092],\n", - " [1977. , 3.41520367],\n", - " [1978. , 3.43951448],\n", - " [1979. , 3.46338917],\n", - " [1980. , 3.48683499],\n", - " [1981. , 3.50985909],\n", - " [1982. , 3.53246727],\n", - " [1983. , 3.55466547],\n", - " [1984. , 3.57645893],\n", - " [1985. , 3.59785423],\n", - " [1986. , 3.6188608 ],\n", - " [1987. , 3.63948834],\n", - " [1988. , 3.65974729],\n", - " [1989. , 3.67964993],\n", - " [1990. , 3.69921081],\n", - " [1991. , 3.71844372],\n", - " [1992. , 3.73736291],\n", - " [1993. , 3.75598335],\n", - " [1994. , 3.77432163],\n", - " [1995. , 3.7923959 ],\n", - " [1996. , 3.81022421],\n", - " [1997. , 3.82782181],\n", - " [1998. , 3.8452036 ],\n", - " [1999. , 3.86238304],\n", - " [2000. , 3.87937367],\n", - " [2001. , 3.89618642],\n", - " [2002. , 3.91283006],\n", - " [2003. , 3.92931283],\n", - " [2004. , 3.94564078],\n", - " [2005. , 3.96181679],\n", - " [2006. , 3.97784126],\n", - " [2007. , 3.99370947],\n", - " [2008. , 4.00941002],\n", - " [2009. , 4.02492491],\n", - " [2010. , 4.04022963],\n", - " [2011. , 4.05528821],\n", - " [2012. , 4.07005002],\n", - " [2013. , 4.08445509],\n", - " [2014. , 4.09843551],\n", - " [2015. , 4.11190995],\n", - " [2016. , 4.12477485],\n", - " [2017. , 4.13690201],\n", - " [2018. , 4.14813488],\n", - " [2019. , 4.15831013],\n", - " [2020. , 4.16730638],\n", - " [2021. , 4.17513242],\n", - " [2022. , 4.18196441],\n", - " [2023. , 4.19625107],\n", - " [2024. , 4.21062102],\n", - " [2025. , 4.22493585],\n", - " [2026. , 4.23921175],\n", - " [2027. , 4.25354211],\n", - " [2028. , 4.26806956],\n", - " [2029. , 4.28300472],\n", - " [2030. , 4.29854456],\n", - " [2031. , 4.31481714],\n", - " [2032. , 4.33193141],\n", - " [2033. , 4.34987622],\n", - " [2034. , 4.36855939],\n", - " [2035. , 4.38791159],\n", - " [2036. , 4.40782196],\n", - " [2037. , 4.42815529],\n", - " [2038. , 4.44861772],\n", - " [2039. , 4.46891522],\n", - " [2040. , 4.48884042],\n", - " [2041. , 4.50831509],\n", - " [2042. , 4.52728011],\n", - " [2043. , 4.54563333],\n", - " [2044. , 4.56325833],\n", - " [2045. , 4.58006323],\n", - " [2046. , 4.59600293],\n", - " [2047. , 4.61099795],\n", - " [2048. , 4.6248397 ],\n", - " [2049. , 4.63730183],\n", - " [2050. , 4.64824497],\n", - " [2051. , 4.65766433],\n", - " [2052. , 4.66565783],\n", - " [2053. , 4.67241102],\n", - " [2054. , 4.67526526],\n", - " [2055. , 4.67849819],\n", - " [2056. , 4.68235967],\n", - " [2057. , 4.68683354],\n", - " [2058. , 4.69177381],\n", - " [2059. , 4.69701813],\n", - " [2060. , 4.70242627],\n", - " [2061. , 4.70788883],\n", - " [2062. , 4.71332622],\n", - " [2063. , 4.71868373],\n", - " [2064. , 4.72392387],\n", - " [2065. , 4.72901914],\n", - " [2066. , 4.73395416],\n", - " [2067. , 4.7387202 ],\n", - " [2068. , 4.74331141],\n", - " [2069. , 4.74772083],\n", - " [2070. , 4.75194145],\n", - " [2071. , 4.75596588],\n", - " [2072. , 4.75978888],\n", - " [2073. , 4.76340877],\n", - " [2074. , 4.76682503],\n", - " [2075. , 4.77004131],\n", - " [2076. , 4.77306356],\n", - " [2077. , 4.7759003 ],\n", - " [2078. , 4.77855902],\n", - " [2079. , 4.78104787],\n", - " [2080. , 4.78337515],\n", - " [2081. , 4.78554851],\n", - " [2082. , 4.7875757 ],\n", - " [2083. , 4.78946472],\n", - " [2084. , 4.791222 ],\n", - " [2085. , 4.79285211],\n", - " [2086. , 4.79436069],\n", - " [2087. , 4.7957522 ],\n", - " [2088. , 4.79703161],\n", - " [2089. , 4.79820364],\n", - " [2090. , 4.79927258],\n", - " [2091. , 4.8002431 ],\n", - " [2092. , 4.80111915],\n", - " [2093. , 4.80190698],\n", - " [2094. , 4.80260852],\n", - " [2095. , 4.80322256],\n", - " [2096. , 4.80374603],\n", - " [2097. , 4.80417534],\n", - " [2098. , 4.8045086 ],\n", - " [2099. , 4.80474447]])" + "" ] }, - "execution_count": 11, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "smoothed" + "cov.configuration.get_seasonal_aggregation_query_filter(coverage_identifier)" ] }, { "cell_type": "code", - "execution_count": 12, - "id": "cf2d3db8-41e6-47e1-b93e-efa30e882e0f", + "execution_count": 9, + "id": "02edb671-e3da-4834-b3b6-0284428c89d8", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "array([3.39045092, 3.41520367, 3.43951448, 3.46338917, 3.48683499,\n", - " 3.50985909, 3.53246727, 3.55466547, 3.57645893, 3.59785423,\n", - " 3.6188608 , 3.63948834, 3.65974729, 3.67964993, 3.69921081,\n", - " 3.71844372, 3.73736291, 3.75598335, 3.77432163, 3.7923959 ,\n", - " 3.81022421, 3.82782181, 3.8452036 , 3.86238304, 3.87937367,\n", - " 3.89618642, 3.91283006, 3.92931283, 3.94564078, 3.96181679,\n", - " 3.97784126, 3.99370947, 4.00941002, 4.02492491, 4.04022963,\n", - " 4.05528821, 4.07005002, 4.08445509, 4.09843551, 4.11190995,\n", - " 4.12477485, 4.13690201, 4.14813488, 4.15831013, 4.16730638,\n", - " 4.17513242, 4.18196441, 4.19625107, 4.21062102, 4.22493585,\n", - " 4.23921175, 4.25354211, 4.26806956, 4.28300472, 4.29854456,\n", - " 4.31481714, 4.33193141, 4.34987622, 4.36855939, 4.38791159,\n", - " 4.40782196, 4.42815529, 4.44861772, 4.46891522, 4.48884042,\n", - " 4.50831509, 4.52728011, 4.54563333, 4.56325833, 4.58006323,\n", - " 4.59600293, 4.61099795, 4.6248397 , 4.63730183, 4.64824497,\n", - " 4.65766433, 4.66565783, 4.67241102, 4.67526526, 4.67849819,\n", - " 4.68235967, 4.68683354, 4.69177381, 4.69701813, 4.70242627,\n", - " 4.70788883, 4.71332622, 4.71868373, 4.72392387, 4.72901914,\n", - " 4.73395416, 4.7387202 , 4.74331141, 4.74772083, 4.75194145,\n", - " 4.75596588, 4.75978888, 4.76340877, 4.76682503, 4.77004131,\n", - " 4.77306356, 4.7759003 , 4.77855902, 4.78104787, 4.78337515,\n", - " 4.78554851, 4.7875757 , 4.78946472, 4.791222 , 4.79285211,\n", - " 4.79436069, 4.7957522 , 4.79703161, 4.79820364, 4.79927258,\n", - " 4.8002431 , 4.80111915, 4.80190698, 4.80260852, 4.80322256,\n", - " 4.80374603, 4.80417534, 4.8045086 , 4.80474447])" + "['tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp26-DJF',\n", + " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp26-MAM',\n", + " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp26-JJA',\n", + " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp26-SON',\n", + " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp45-DJF',\n", + " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp45-MAM',\n", + " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp45-JJA',\n", + " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp45-SON',\n", + " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp85-DJF',\n", + " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp85-MAM',\n", + " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp85-JJA',\n", + " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp85-SON']" ] }, - "execution_count": 12, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "smoothed[:, 1]" + "db.list_allowed_coverage_identifiers(session, coverage_configuration_id=cov.configuration.id)" ] } ], From 385cbae8905b1f7e631d4b4af0ea430c94c8aa2b Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 18 Jul 2024 17:18:35 +0100 Subject: [PATCH 4/6] querying coverage configurations by their existing possible values --- arpav_ppcv/database.py | 50 ++++++ tests/notebooks/generic.ipynb | 277 ++++++++++++---------------------- 2 files changed, 148 insertions(+), 179 deletions(-) diff --git a/arpav_ppcv/database.py b/arpav_ppcv/database.py index dac7f165..e5e17b18 100644 --- a/arpav_ppcv/database.py +++ b/arpav_ppcv/database.py @@ -918,6 +918,56 @@ def list_coverage_configurations( italian_display_name_filter, coverages.CoverageConfiguration.display_name_italian, ) + if len(conf_params := configuration_parameter_value_filter or []) > 0: + possible_values_cte = ( + sqlmodel.select( + coverages.CoverageConfiguration.id, + func.jsonb_agg( + func.json_build_object( + coverages.ConfigurationParameter.name, + coverages.ConfigurationParameterValue.name, + ) + ).label("possible_values"), + ) + .join( + coverages.ConfigurationParameterPossibleValue, + coverages.CoverageConfiguration.id + == coverages.ConfigurationParameterPossibleValue.coverage_configuration_id, + ) + .join( + coverages.ConfigurationParameterValue, + coverages.ConfigurationParameterValue.id + == coverages.ConfigurationParameterPossibleValue.configuration_parameter_value_id, + ) + .join( + coverages.ConfigurationParameter, + coverages.ConfigurationParameter.id + == coverages.ConfigurationParameterValue.configuration_parameter_id, + ) + .group_by(coverages.CoverageConfiguration.id) + ).cte("cov_conf_possible_values") + statement = statement.join( + possible_values_cte, + possible_values_cte.c.id == coverages.CoverageConfiguration.id, + ) + for conf_param_value in conf_params: + param_name = conf_param_value.configuration_parameter.name + param_value = conf_param_value.name + statement = statement.where( + func.jsonb_path_exists( + possible_values_cte, + # the below expression is a a fragment of SQL/JSON Path + # language, which represents postgresql's way of filtering a + # JSON array. More detail at: + # + # https://www.postgresql.org/docs/current/functions-json.html#FUNCTIONS-SQLJSON-PATH + # + # in this case it reads like: + # return True if the current element of the array, which is an object, has a key named + # `param_name` with a corresponding value of `param_value` + f'$[*] ? (@.{param_name} == "{param_value}")', + ) + ) items = session.exec(statement.offset(offset).limit(limit)).all() num_items = _get_total_num_records(session, statement) if include_total else None return items, num_items diff --git a/tests/notebooks/generic.ipynb b/tests/notebooks/generic.ipynb index 309e8d49..ce57322c 100644 --- a/tests/notebooks/generic.ipynb +++ b/tests/notebooks/generic.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 34, "id": "dbcc00b1-2fc1-43ff-9549-ebca8cf03262", "metadata": {}, "outputs": [], @@ -19,7 +19,10 @@ "import pandas as pd\n", "import shapely.io\n", "import sqlmodel\n", - "from loess.loess_1d import loess_1d\n", + "from sqlalchemy import (\n", + " bindparam, \n", + " func\n", + ")\n", "\n", "from arpav_ppcv import (\n", " database as db,\n", @@ -32,6 +35,7 @@ " ObservationAggregationType,\n", " Season,\n", ")\n", + "from arpav_ppcv.schemas import coverages\n", "from arpav_ppcv.schemas.coverages import CoverageInternal\n", "\n", "logging.basicConfig(level=logging.DEBUG)\n", @@ -63,251 +67,166 @@ }, { "cell_type": "code", - "execution_count": 6, - "id": "6fa57746-a891-449a-b8c0-313de8470cd5", + "execution_count": 29, + "id": "d15772a0-a1c1-4258-9f2f-4402d074cbee", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp26-DJF',\n", - " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp26-MAM',\n", - " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp26-JJA',\n", - " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp26-SON']" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "db.list_allowed_coverage_identifiers(\n", - " session, \n", - " coverage_configuration_id=cov.configuration.id, \n", - " select_configuration_parameter_values=[\n", - " db.get_configuration_parameter_value_by_names(session, \"scenario\", \"rcp26\"),\n", - " db.get_configuration_parameter_value_by_names(session, \"time_period\", \"MAM\"),\n", - " ]\n", - ")" + "pv_cte = (\n", + " sqlmodel\n", + " .select(\n", + " coverages.CoverageConfiguration.id,\n", + " func.jsonb_agg(\n", + " func.json_build_object(\n", + " coverages.ConfigurationParameter.name,\n", + " coverages.ConfigurationParameterValue.name,\n", + " )\n", + " ).label(\"possible_values\")\n", + " )\n", + " .join(\n", + " coverages.ConfigurationParameterPossibleValue,\n", + " coverages.CoverageConfiguration.id ==\n", + " coverages.ConfigurationParameterPossibleValue.coverage_configuration_id\n", + " )\n", + " .join(\n", + " coverages.ConfigurationParameterValue,\n", + " coverages.ConfigurationParameterValue.id ==\n", + " coverages.ConfigurationParameterPossibleValue.configuration_parameter_value_id\n", + " )\n", + " .join(\n", + " coverages.ConfigurationParameter,\n", + " coverages.ConfigurationParameter.id ==\n", + " coverages.ConfigurationParameterValue.configuration_parameter_id\n", + " ).group_by(coverages.CoverageConfiguration.id)\n", + ").cte(\"cov_conf_possible_values\")" ] }, { "cell_type": "code", - "execution_count": 7, - "id": "c90bcb97-aff2-454c-bc88-eaf30676cdd5", + "execution_count": 30, + "id": "74f05366-c299-4964-8efe-de36cfe61de4", "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "([CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='ensembletwbc/std/clipped/eca_cdd_an_avgagree_{time_window}_{scenario}_{year_period}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_agree_model_ensemble', wms_main_layer_name='consecutive_dry_days_index_per_time_period-uncertainty_group', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d50ef8e7-1a84-435e-84e6-294664b0964c'), unit='gg', display_name_english=None, palette='uncert-stippled/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_agree_model_ensemble-rcp26-tw1-DJF'),\n", - " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='ensembletwbc/std/clipped/eca_cdd_an_avgagree_{time_window}_{scenario}_{year_period}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_agree_model_ensemble', wms_main_layer_name='consecutive_dry_days_index_per_time_period-uncertainty_group', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d50ef8e7-1a84-435e-84e6-294664b0964c'), unit='gg', display_name_english=None, palette='uncert-stippled/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_agree_model_ensemble-rcp26-tw1-MAM'),\n", - " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='ensembletwbc/std/clipped/eca_cdd_an_avgagree_{time_window}_{scenario}_{year_period}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_agree_model_ensemble', wms_main_layer_name='consecutive_dry_days_index_per_time_period-uncertainty_group', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d50ef8e7-1a84-435e-84e6-294664b0964c'), unit='gg', display_name_english=None, palette='uncert-stippled/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_agree_model_ensemble-rcp26-tw1-JJA'),\n", - " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='ensembletwbc/std/clipped/eca_cdd_an_avgagree_{time_window}_{scenario}_{year_period}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_agree_model_ensemble', wms_main_layer_name='consecutive_dry_days_index_per_time_period-uncertainty_group', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d50ef8e7-1a84-435e-84e6-294664b0964c'), unit='gg', display_name_english=None, palette='uncert-stippled/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_agree_model_ensemble-rcp26-tw1-SON'),\n", - " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='ensembletwbc/std/clipped/eca_cdd_an_avgagree_{time_window}_{scenario}_{year_period}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_agree_model_ensemble', wms_main_layer_name='consecutive_dry_days_index_per_time_period-uncertainty_group', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d50ef8e7-1a84-435e-84e6-294664b0964c'), unit='gg', display_name_english=None, palette='uncert-stippled/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_agree_model_ensemble-rcp26-tw2-DJF'),\n", - " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='ensembletwbc/std/clipped/eca_cdd_an_avgagree_{time_window}_{scenario}_{year_period}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_agree_model_ensemble', wms_main_layer_name='consecutive_dry_days_index_per_time_period-uncertainty_group', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d50ef8e7-1a84-435e-84e6-294664b0964c'), unit='gg', display_name_english=None, palette='uncert-stippled/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_agree_model_ensemble-rcp26-tw2-MAM'),\n", - " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='ensembletwbc/std/clipped/eca_cdd_an_avgagree_{time_window}_{scenario}_{year_period}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_agree_model_ensemble', wms_main_layer_name='consecutive_dry_days_index_per_time_period-uncertainty_group', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d50ef8e7-1a84-435e-84e6-294664b0964c'), unit='gg', display_name_english=None, palette='uncert-stippled/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_agree_model_ensemble-rcp26-tw2-JJA'),\n", - " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='ensembletwbc/std/clipped/eca_cdd_an_avgagree_{time_window}_{scenario}_{year_period}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_agree_model_ensemble', wms_main_layer_name='consecutive_dry_days_index_per_time_period-uncertainty_group', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d50ef8e7-1a84-435e-84e6-294664b0964c'), unit='gg', display_name_english=None, palette='uncert-stippled/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_agree_model_ensemble-rcp26-tw2-SON'),\n", - " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='indici5rcm/clipped/eca_cdd_an_EC-EARTH_CCLM4-8-17_{scenario}_{year_period}_{time_window}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17', wms_main_layer_name='consecutive_dry_days_index_per_time_period', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d04c8d32-277b-4d50-ba68-02836523c3af'), unit='gg', display_name_english=None, palette='default/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17-rcp26-tw1-DJF'),\n", - " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='indici5rcm/clipped/eca_cdd_an_EC-EARTH_CCLM4-8-17_{scenario}_{year_period}_{time_window}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17', wms_main_layer_name='consecutive_dry_days_index_per_time_period', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d04c8d32-277b-4d50-ba68-02836523c3af'), unit='gg', display_name_english=None, palette='default/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17-rcp26-tw1-MAM'),\n", - " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='indici5rcm/clipped/eca_cdd_an_EC-EARTH_CCLM4-8-17_{scenario}_{year_period}_{time_window}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17', wms_main_layer_name='consecutive_dry_days_index_per_time_period', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d04c8d32-277b-4d50-ba68-02836523c3af'), unit='gg', display_name_english=None, palette='default/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17-rcp26-tw1-JJA'),\n", - " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='indici5rcm/clipped/eca_cdd_an_EC-EARTH_CCLM4-8-17_{scenario}_{year_period}_{time_window}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17', wms_main_layer_name='consecutive_dry_days_index_per_time_period', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d04c8d32-277b-4d50-ba68-02836523c3af'), unit='gg', display_name_english=None, palette='default/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17-rcp26-tw1-SON'),\n", - " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='indici5rcm/clipped/eca_cdd_an_EC-EARTH_CCLM4-8-17_{scenario}_{year_period}_{time_window}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17', wms_main_layer_name='consecutive_dry_days_index_per_time_period', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d04c8d32-277b-4d50-ba68-02836523c3af'), unit='gg', display_name_english=None, palette='default/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17-rcp26-tw2-DJF'),\n", - " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='indici5rcm/clipped/eca_cdd_an_EC-EARTH_CCLM4-8-17_{scenario}_{year_period}_{time_window}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17', wms_main_layer_name='consecutive_dry_days_index_per_time_period', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d04c8d32-277b-4d50-ba68-02836523c3af'), unit='gg', display_name_english=None, palette='default/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17-rcp26-tw2-MAM'),\n", - " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='indici5rcm/clipped/eca_cdd_an_EC-EARTH_CCLM4-8-17_{scenario}_{year_period}_{time_window}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17', wms_main_layer_name='consecutive_dry_days_index_per_time_period', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d04c8d32-277b-4d50-ba68-02836523c3af'), unit='gg', display_name_english=None, palette='default/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17-rcp26-tw2-JJA'),\n", - " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='indici5rcm/clipped/eca_cdd_an_EC-EARTH_CCLM4-8-17_{scenario}_{year_period}_{time_window}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17', wms_main_layer_name='consecutive_dry_days_index_per_time_period', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('d04c8d32-277b-4d50-ba68-02836523c3af'), unit='gg', display_name_english=None, palette='default/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_model_ec_earth_cclm4_8_17-rcp26-tw2-SON'),\n", - " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='indici5rcm/clipped/eca_cdd_an_EC-EARTH_RACMO22E_{scenario}_{year_period}_{time_window}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_model_ec_earth_racmo22e', wms_main_layer_name='consecutive_dry_days_index_per_time_period', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('bb1697d1-d3bd-46aa-88be-f912a0c39ab9'), unit='gg', display_name_english=None, palette='default/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_model_ec_earth_racmo22e-rcp26-tw1-DJF'),\n", - " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='indici5rcm/clipped/eca_cdd_an_EC-EARTH_RACMO22E_{scenario}_{year_period}_{time_window}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_model_ec_earth_racmo22e', wms_main_layer_name='consecutive_dry_days_index_per_time_period', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('bb1697d1-d3bd-46aa-88be-f912a0c39ab9'), unit='gg', display_name_english=None, palette='default/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_model_ec_earth_racmo22e-rcp26-tw1-MAM'),\n", - " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='indici5rcm/clipped/eca_cdd_an_EC-EARTH_RACMO22E_{scenario}_{year_period}_{time_window}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_model_ec_earth_racmo22e', wms_main_layer_name='consecutive_dry_days_index_per_time_period', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('bb1697d1-d3bd-46aa-88be-f912a0c39ab9'), unit='gg', display_name_english=None, palette='default/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_model_ec_earth_racmo22e-rcp26-tw1-JJA'),\n", - " CoverageInternal(configuration=CoverageConfiguration(netcdf_main_dataset_name='cdd', observation_variable_aggregation_type=None, thredds_url_pattern='indici5rcm/clipped/eca_cdd_an_EC-EARTH_RACMO22E_{scenario}_{year_period}_{time_window}_ls_VFVGTAA.nc', uncertainty_lower_bounds_coverage_configuration_id=None, name='cdd_30yr_anomaly_annual_model_ec_earth_racmo22e', wms_main_layer_name='consecutive_dry_days_index_per_time_period', uncertainty_upper_bounds_coverage_configuration_id=None, id=UUID('bb1697d1-d3bd-46aa-88be-f912a0c39ab9'), unit='gg', display_name_english=None, palette='default/div-BrBG-inv', display_name_italian=None, color_scale_min=-40.0, description_english=None, color_scale_max=40.0, description_italian=None, observation_variable_id=None, coverage_id_pattern='{name}-{scenario}-{time_window}-{year_period}'), identifier='cdd_30yr_anomaly_annual_model_ec_earth_racmo22e-rcp26-tw1-SON')],\n", - " None)" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "SELECT coverageconfiguration.id, jsonb_agg(json_build_object(configurationparameter.name, configurationparametervalue.name)) AS possible_values \n", + "FROM coverageconfiguration JOIN configurationparameterpossiblevalue ON coverageconfiguration.id = configurationparameterpossiblevalue.coverage_configuration_id JOIN configurationparametervalue ON configurationparametervalue.id = configurationparameterpossiblevalue.configuration_parameter_value_id JOIN configurationparameter ON configurationparameter.id = configurationparametervalue.configuration_parameter_id GROUP BY coverageconfiguration.id\n" + ] } ], "source": [ - "db.list_coverage_identifiers(\n", - " session,\n", - " matching_configuration_parameters=[\n", - " db.get_configuration_parameter_value_by_names(session, \"scenario\", \"rcp26\"),\n", - " db.get_configuration_parameter_value_by_names(session, \"climatological_model\", \"model_ensemble\"),\n", - " ]\n", - ")" + "print(pv_cte)" ] }, { "cell_type": "code", - "execution_count": 3, - "id": "4ac51747-ee8e-468a-a791-5f815b80286a", + "execution_count": 39, + "id": "f7966b1d-9d38-4b05-a5ae-ab21a5c0f8b9", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'{name}-{climatological_model}-{measure}-{scenario}-{year_period}'" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "cov.configuration.coverage_id_pattern" + "pvs = {\n", + " \"climatological_model\": \"model_ensemble\",\n", + " \"scenario\": \"rcp26\"\n", + "}" ] }, { "cell_type": "code", - "execution_count": 4, - "id": "705a7335-18df-474e-912a-8bd6dfca940d", + "execution_count": 38, + "id": "8288a6f8-257d-49b6-a7a6-04b429b15615", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'climatological_model': 'model_ensemble',\n", - " 'measure': 'absolute',\n", - " 'scenario': 'rcp26',\n", - " 'year_period': 'DJF'}" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "cov.configuration.retrieve_configuration_parameters(cov.identifier)" + "statement = (\n", + " sqlmodel.select(coverages.CoverageConfiguration)\n", + " .join(pv_cte, pv_cte.c.id == coverages.CoverageConfiguration.id)\n", + " .where(func.jsonb_path_exists(pv_cte.c.possible_values, '$[*] ? (@.climatological_model == \"model_ensemble\")'))\n", + " .where(func.jsonb_path_exists(pv_cte.c.possible_values, '$[*] ? (@.scenario == \"rcp26\")'))\n", + ")" ] }, { "cell_type": "code", - "execution_count": 5, - "id": "db7e48fd-98fb-439e-8291-a9fd005aec21", + "execution_count": 40, + "id": "d1446147-b649-4fdf-b0e6-da0c0a23611e", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[ConfigurationParameterPossibleValue(configuration_parameter_value_id=UUID('c5381cd2-ca57-48a9-a096-71568e812366'), coverage_configuration_id=UUID('06c104d2-a502-400b-9728-b8e2d9ee65d9')),\n", - " ConfigurationParameterPossibleValue(configuration_parameter_value_id=UUID('88fa9565-8af6-4ab5-afeb-bfa00cd97955'), coverage_configuration_id=UUID('06c104d2-a502-400b-9728-b8e2d9ee65d9')),\n", - " ConfigurationParameterPossibleValue(configuration_parameter_value_id=UUID('70c82437-5e90-4181-96c7-262a097d18ba'), coverage_configuration_id=UUID('06c104d2-a502-400b-9728-b8e2d9ee65d9')),\n", - " ConfigurationParameterPossibleValue(configuration_parameter_value_id=UUID('4842fa35-b13a-446d-ad9f-43a45b82bd23'), coverage_configuration_id=UUID('06c104d2-a502-400b-9728-b8e2d9ee65d9'))]" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "cov.configuration.retrieve_used_values(cov.identifier)" + "statement = (\n", + " sqlmodel.select(coverages.CoverageConfiguration)\n", + " .join(pv_cte, pv_cte.c.id == coverages.CoverageConfiguration.id)\n", + ")" ] }, { "cell_type": "code", - "execution_count": 7, - "id": "5dd3f90c-fc59-4bec-a613-2297537f190f", + "execution_count": 41, + "id": "035e71d5-3729-4f3f-9c5b-b94e18afae82", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'ensymbc/clipped/tas_avg_rcp26_DJF_ts19762100_ls_VFVG.nc'" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "cov.configuration.get_thredds_url_fragment(cov.identifier)" + "for value_name, value in pvs.items():\n", + " statement = (\n", + " statement\n", + " .where(func.jsonb_path_exists(pv_cte.c.possible_values, f'$[*] ? (@.{value_name} == \"{value}\")'))\n", + " )" ] }, { "cell_type": "code", - "execution_count": 6, - "id": "9f396350-e5ea-455c-a636-87fd763f3d89", + "execution_count": 42, + "id": "72889317-a72d-4417-92d1-65694d953fd9", "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp26-DJF'" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "WITH cov_conf_possible_values AS \n", + "(SELECT coverageconfiguration.id AS id, jsonb_agg(json_build_object(configurationparameter.name, configurationparametervalue.name)) AS possible_values \n", + "FROM coverageconfiguration JOIN configurationparameterpossiblevalue ON coverageconfiguration.id = configurationparameterpossiblevalue.coverage_configuration_id JOIN configurationparametervalue ON configurationparametervalue.id = configurationparameterpossiblevalue.configuration_parameter_value_id JOIN configurationparameter ON configurationparameter.id = configurationparametervalue.configuration_parameter_id GROUP BY coverageconfiguration.id)\n", + " SELECT coverageconfiguration.id, coverageconfiguration.name, coverageconfiguration.display_name_english, coverageconfiguration.display_name_italian, coverageconfiguration.description_english, coverageconfiguration.description_italian, coverageconfiguration.netcdf_main_dataset_name, coverageconfiguration.thredds_url_pattern, coverageconfiguration.wms_main_layer_name, coverageconfiguration.unit, coverageconfiguration.palette, coverageconfiguration.color_scale_min, coverageconfiguration.color_scale_max, coverageconfiguration.observation_variable_id, coverageconfiguration.observation_variable_aggregation_type, coverageconfiguration.uncertainty_lower_bounds_coverage_configuration_id, coverageconfiguration.uncertainty_upper_bounds_coverage_configuration_id \n", + "FROM coverageconfiguration JOIN cov_conf_possible_values ON cov_conf_possible_values.id = coverageconfiguration.id \n", + "WHERE jsonb_path_exists(cov_conf_possible_values.possible_values, '$[*] ? (@.climatological_model == \"model_ensemble\")') AND jsonb_path_exists(cov_conf_possible_values.possible_values, '$[*] ? (@.scenario == \"rcp26\")')\n" + ] } ], "source": [ - "cov.configuration.build_coverage_identifier(\n", - " parameters=[\n", - " db.get_configuration_parameter_value_by_names(session, \"climatological_model\", \"model_ensemble\"),\n", - " db.get_configuration_parameter_value_by_names(session, \"measure\", \"absolute\"),\n", - " db.get_configuration_parameter_value_by_names(session, \"scenario\", \"rcp26\"),\n", - " db.get_configuration_parameter_value_by_names(session, \"year_period\", \"DJF\"),\n", - " ]\n", - ")" + "print(statement.compile(db.get_engine(settings), compile_kwargs={\"literal_binds\": True}))" ] }, { "cell_type": "code", - "execution_count": 7, - "id": "ec2c0ead-cd94-46d5-bacc-67ab35a0238f", + "execution_count": 43, + "id": "867b4a0f-232f-4c93-b7e6-ebccdb36af9d", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "cov.configuration.get_seasonal_aggregation_query_filter(coverage_identifier)" + "session.rollback()" ] }, { "cell_type": "code", - "execution_count": 9, - "id": "02edb671-e3da-4834-b3b6-0284428c89d8", + "execution_count": 44, + "id": "878c6ea7-8946-4779-925a-c89db88703f1", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "['tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp26-DJF',\n", - " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp26-MAM',\n", - " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp26-JJA',\n", - " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp26-SON',\n", - " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp45-DJF',\n", - " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp45-MAM',\n", - " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp45-JJA',\n", - " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp45-SON',\n", - " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp85-DJF',\n", - " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp85-MAM',\n", - " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp85-JJA',\n", - " 'tas_seasonal_absolute_model_ensemble-model_ensemble-absolute-rcp85-SON']" + "[CoverageConfiguration(description_english='TAS seasonal absolute model ensemble', color_scale_max=32.0, description_italian='TAS valore assoluto di stagione media ensemble', observation_variable_id=UUID('32706ce0-0134-4122-91a0-9d6f237350b3'), netcdf_main_dataset_name='tas', observation_variable_aggregation_type=, thredds_url_pattern='ensymbc/clipped/tas_avg_{scenario}_{year_period}_ts19762100_ls_VFVG.nc', uncertainty_lower_bounds_coverage_configuration_id=UUID('9c2bfa7a-d108-4d2d-8c99-f26ae9329e79'), name='tas_seasonal_absolute_model_ensemble', wms_main_layer_name='tas', uncertainty_upper_bounds_coverage_configuration_id=UUID('d33d6b43-0379-447a-ad9d-b2f36f7dce3e'), id=UUID('06c104d2-a502-400b-9728-b8e2d9ee65d9'), unit='ºC', display_name_english='TAS seasonal absolute model ensemble', palette='default/seq-YlOrRd', display_name_italian='TAS valore assoluto di stagione media ensemble', color_scale_min=-3.0, coverage_id_pattern='{name}-{climatological_model}-{measure}-{scenario}-{year_period}')]" ] }, - "execution_count": 9, + "execution_count": 44, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "db.list_allowed_coverage_identifiers(session, coverage_configuration_id=cov.configuration.id)" + "session.exec(statement).all()" ] } ], From e280436e8503b7e08237ee6be8573a154558e766 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Fri, 19 Jul 2024 17:00:36 +0100 Subject: [PATCH 5/6] Added possible values as query param in coverage-related list endpoints --- arpav_ppcv/database.py | 124 ++++++++---------- arpav_ppcv/webapp/api_v2/routers/coverages.py | 36 ++++- tests/conftest.py | 35 +++++ tests/notebooks/generic.ipynb | 97 +++++++++++++- tests/test_database.py | 82 ++++++++++++ 5 files changed, 305 insertions(+), 69 deletions(-) diff --git a/arpav_ppcv/database.py b/arpav_ppcv/database.py index e5e17b18..8db1d2f9 100644 --- a/arpav_ppcv/database.py +++ b/arpav_ppcv/database.py @@ -894,7 +894,7 @@ def list_coverage_configurations( name_filter: Optional[str] = None, english_display_name_filter: Optional[str] = None, italian_display_name_filter: Optional[str] = None, - configuration_parameter_value_filter: Optional[ + configuration_parameter_values_filter: Optional[ list[coverages.ConfigurationParameterValue] ] = None, ) -> tuple[Sequence[coverages.CoverageConfiguration], Optional[int]]: @@ -918,7 +918,7 @@ def list_coverage_configurations( italian_display_name_filter, coverages.CoverageConfiguration.display_name_italian, ) - if len(conf_params := configuration_parameter_value_filter or []) > 0: + if len(conf_params := configuration_parameter_values_filter or []) > 0: possible_values_cte = ( sqlmodel.select( coverages.CoverageConfiguration.id, @@ -955,8 +955,8 @@ def list_coverage_configurations( param_value = conf_param_value.name statement = statement.where( func.jsonb_path_exists( - possible_values_cte, - # the below expression is a a fragment of SQL/JSON Path + possible_values_cte.c.possible_values, + # the below expression is a fragment of SQL/JSON Path # language, which represents postgresql's way of filtering a # JSON array. More detail at: # @@ -978,7 +978,7 @@ def collect_all_coverage_configurations( name_filter: Optional[str] = None, english_display_name_filter: Optional[str] = None, italian_display_name_filter: Optional[str] = None, - configuration_parameter_value_filter: Optional[ + configuration_parameter_values_filter: Optional[ list[coverages.ConfigurationParameterValue] ] = None, ) -> Sequence[coverages.CoverageConfiguration]: @@ -989,6 +989,7 @@ def collect_all_coverage_configurations( name_filter=name_filter, english_display_name_filter=english_display_name_filter, italian_display_name_filter=italian_display_name_filter, + configuration_parameter_values_filter=configuration_parameter_values_filter, ) result, _ = list_coverage_configurations( session, @@ -997,6 +998,7 @@ def collect_all_coverage_configurations( name_filter=name_filter, english_display_name_filter=english_display_name_filter, italian_display_name_filter=italian_display_name_filter, + configuration_parameter_values_filter=configuration_parameter_values_filter, ) return result @@ -1139,56 +1141,45 @@ def update_coverage_configuration( return db_coverage_configuration -def list_allowed_coverage_identifiers( - session: sqlmodel.Session, - *, - coverage_configuration_id: uuid.UUID, - matching_configuration_parameters: list[coverages.ConfigurationParameterValue] - | None = None, +def generate_coverage_identifiers( + coverage_configuration: coverages.CoverageConfiguration, + configuration_parameter_values_filter: Optional[ + list[coverages.ConfigurationParameterValue] + ] = None, ) -> list[str]: - """Build list of legal coverage identifiers.""" + """Build list of legal coverage identifiers for a coverage configuration.""" + + params_to_filter = {} + for cpv in configuration_parameter_values_filter or []: + values = params_to_filter.setdefault(cpv.configuration_parameter.name, []) + values.append(cpv.name) + pattern_parts = re.findall( + r"\{(\w+)\}", coverage_configuration.coverage_id_pattern.partition("-")[-1] + ) + values_to_combine = [] + for part in pattern_parts: + part_values = [] + for possible_value in coverage_configuration.possible_values: + this_param_name = possible_value.configuration_parameter_value.configuration_parameter.name + if this_param_name == part: + # check if this param's value is to be filtered out or not + this_value = possible_value.configuration_parameter_value.name + if this_param_name in params_to_filter: + if this_value in params_to_filter.get(this_param_name, []): + part_values.append(this_value) + else: + part_values.append(this_value) + values_to_combine.append(part_values) + # account for the possibility that there is an error in the + # coverage_id_pattern, where some of the parts are not actually configured allowed_identifiers = [] - db_cov_conf = get_coverage_configuration(session, coverage_configuration_id) - if db_cov_conf is not None: - pattern_parts = re.findall( - r"\{(\w+)\}", db_cov_conf.coverage_id_pattern.partition("-")[-1] - ) - values_to_combine = [] - for part in pattern_parts: - part_values = [] - for possible_value in db_cov_conf.possible_values: - param_name_matches = ( - possible_value.configuration_parameter_value.configuration_parameter.name - == part - ) - if param_name_matches: - part_values.append( - possible_value.configuration_parameter_value.name - ) - values_to_combine.append(part_values) - # account for the possibility that there is an error in the - # coverage_id_pattern, where some of the parts are not actually configured - for index, container in enumerate(values_to_combine): - if len(container) == 0: - values_to_combine[index] = [pattern_parts[index]] - for combination in itertools.product(*values_to_combine): - dataset_id = "-".join((db_cov_conf.name, *combination)) - allowed_identifiers.append(dataset_id) - - result = [] - if len(matching_configuration_parameters or []) > 0: - for id_ in allowed_identifiers: - parsed_parts = id_.split("-")[1:] - for idx, param_name in enumerate(pattern_parts): - param_value = parsed_parts[idx] - pv = get_configuration_parameter_value_by_names( - session, param_name, param_value - ) - if pv in matching_configuration_parameters: - result.append(id_) - else: - result = allowed_identifiers - return result + for index, container in enumerate(values_to_combine): + if len(container) == 0: + values_to_combine[index] = [pattern_parts[index]] + for combination in itertools.product(*values_to_combine): + dataset_id = "-".join((coverage_configuration.name, *combination)) + allowed_identifiers.append(dataset_id) + return allowed_identifiers def list_municipalities( @@ -1269,10 +1260,6 @@ def create_many_municipalities( return db_records -def get_relevant_coverage_configurations(): - ... - - def list_coverage_identifiers( session: sqlmodel.Session, *, @@ -1280,11 +1267,12 @@ def list_coverage_identifiers( offset: int = 0, include_total: bool = False, name_filter: list[str] | None = None, - matching_configuration_parameters: list[coverages.ConfigurationParameterValue] - | None = None, + configuration_parameter_values_filter: Optional[ + list[coverages.ConfigurationParameterValue] + ] = None, ) -> tuple[list[coverages.CoverageInternal], Optional[int]]: all_cov_ids = collect_all_coverage_identifiers( - session, matching_configuration_parameters + session, configuration_parameter_values_filter ) if name_filter is not None: for fragment in name_filter: @@ -1299,16 +1287,18 @@ def list_coverage_identifiers( def collect_all_coverage_identifiers( session: sqlmodel.Session, - matching_configuration_parameters: list[coverages.ConfigurationParameterValue] - | None = None, + configuration_parameter_values_filter: Optional[ + list[coverages.ConfigurationParameterValue] + ] = None, ) -> list[coverages.CoverageInternal]: - cov_confs = collect_all_coverage_configurations(session) + cov_confs = collect_all_coverage_configurations( + session, + configuration_parameter_values_filter=configuration_parameter_values_filter, + ) cov_ids = [] for cov_conf in cov_confs: - allowed_identifiers = list_allowed_coverage_identifiers( - session, - coverage_configuration_id=cov_conf.id, - matching_configuration_parameters=matching_configuration_parameters, + allowed_identifiers = generate_coverage_identifiers( + cov_conf, configuration_parameter_values_filter ) for cov_id in allowed_identifiers: cov_ids.append( diff --git a/arpav_ppcv/webapp/api_v2/routers/coverages.py b/arpav_ppcv/webapp/api_v2/routers/coverages.py index 7eaf0cc4..21bbda5e 100644 --- a/arpav_ppcv/webapp/api_v2/routers/coverages.py +++ b/arpav_ppcv/webapp/api_v2/routers/coverages.py @@ -83,6 +83,15 @@ async def list_coverage_configurations( request: Request, db_session: Annotated[Session, Depends(dependencies.get_db_session)], list_params: Annotated[dependencies.CommonListFilterParameters, Depends()], + possible_value: Annotated[ + list[ + Annotated[ + str, + pydantic.StringConstraints(pattern=r"^[0-9a-zA-Z_]+:[0-9a-zA-Z_]+$"), + ] + ], + Query(), + ] = None, ): """### List coverage configurations. @@ -119,11 +128,24 @@ async def list_coverage_configurations( endpoint. """ + conf_param_values_filter = [] + for possible in possible_value or []: + param_name, param_value = possible.partition(":")[::2] + db_parameter_value = db.get_configuration_parameter_value_by_names( + db_session, param_name, param_value + ) + if db_parameter_value is not None: + conf_param_values_filter.append(db_parameter_value) + else: + logger.debug( + f"ignoring unknown parameter/value pair {param_name}:{param_value}" + ) coverage_configurations, filtered_total = db.list_coverage_configurations( db_session, limit=list_params.limit, offset=list_params.offset, include_total=True, + configuration_parameter_values_filter=conf_param_values_filter or None, ) _, unfiltered_total = db.list_coverage_configurations( db_session, limit=1, offset=0, include_total=True @@ -181,13 +203,25 @@ def list_coverage_identifiers( Query(), ] = None, ): - logger.debug(f"{possible_value=}") + conf_param_values_filter = [] + for possible in possible_value or []: + param_name, param_value = possible.partition(":")[::2] + db_parameter_value = db.get_configuration_parameter_value_by_names( + db_session, param_name, param_value + ) + if db_parameter_value is not None: + conf_param_values_filter.append(db_parameter_value) + else: + logger.debug( + f"ignoring unknown parameter/value pair {param_name}:{param_value}" + ) cov_internals, filtered_total = db.list_coverage_identifiers( db_session, limit=list_params.limit, offset=list_params.offset, include_total=True, name_filter=name_contains, + configuration_parameter_values_filter=conf_param_values_filter or None, ) _, unfiltered_total = db.list_coverage_identifiers( db_session, limit=1, offset=0, include_total=True diff --git a/tests/conftest.py b/tests/conftest.py index 145d0188..469465bc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -218,6 +218,41 @@ def sample_configuration_parameters(arpav_db_session): return db_conf_params +@pytest.fixture() +def sample_coverage_configurations( + arpav_db_session, sample_configuration_parameters +) -> list[coverages.CoverageConfiguration]: + db_cov_confs = [] + for i in range(10): + params_to_use = random.choices(sample_configuration_parameters, k=2) + param_values_to_use = [] + for param in params_to_use: + possible_value = coverages.ConfigurationParameterPossibleValue( + configuration_parameter_value=random.choice(param.allowed_values) + ) + param_values_to_use.append(possible_value) + db_cov_confs.append( + coverages.CoverageConfiguration( + name=f"coverage_configuration{i}", + netcdf_main_dataset_name="some-dataset-name", + thredds_url_pattern=( + f"the_thredds-param_" + f"{{" + f"{random.choice(param_values_to_use).configuration_parameter_value.configuration_parameter.name}" + f"}}" + ), + palette="fake", + possible_values=param_values_to_use, + ) + ) + for db_cov_conf in db_cov_confs: + arpav_db_session.add(db_cov_conf) + arpav_db_session.commit() + for db_cov_conf in db_cov_confs: + arpav_db_session.refresh(db_cov_conf) + return db_cov_confs + + def _override_get_settings(): standard_settings = config.get_settings() return standard_settings diff --git a/tests/notebooks/generic.ipynb b/tests/notebooks/generic.ipynb index ce57322c..11c99d45 100644 --- a/tests/notebooks/generic.ipynb +++ b/tests/notebooks/generic.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 34, + "execution_count": 1, "id": "dbcc00b1-2fc1-43ff-9549-ebca8cf03262", "metadata": {}, "outputs": [], @@ -65,6 +65,101 @@ ")" ] }, + { + "cell_type": "code", + "execution_count": 2, + "id": "d2de401b-4028-45d2-bf01-4c0cbb806d80", + "metadata": {}, + "outputs": [], + "source": [ + "param_values = [\n", + " db.get_configuration_parameter_value_by_names(session, \"climatological_model\", \"model_ensemble\"),\n", + " db.get_configuration_parameter_value_by_names(session, \"scenario\", \"rcp26\")\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "f379b854-b318-475f-95c7-82b0a2e48f6b", + "metadata": {}, + "outputs": [], + "source": [ + "cov_conf = db.get_coverage_configuration_by_name(session, \"tas_seasonal_absolute_model_ensemble\")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "46f23579-47c7-4123-a5a0-7368dcaee640", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'rcp26'" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cov_conf.possible_values[0].configuration_parameter_value.name" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "d505e1b4-14e9-459a-b942-2226c6bfc82e", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "DEBUG:arpav_ppcv.database:locals()={'session': , 'coverage_configuration': CoverageConfiguration(netcdf_main_dataset_name='tas', observation_variable_aggregation_type=, thredds_url_pattern='ensymbc/clipped/tas_avg_{scenario}_{year_period}_ts19762100_ls_VFVG.nc', uncertainty_lower_bounds_coverage_configuration_id=UUID('9c2bfa7a-d108-4d2d-8c99-f26ae9329e79'), name='tas_seasonal_absolute_model_ensemble', wms_main_layer_name='tas', uncertainty_upper_bounds_coverage_configuration_id=UUID('d33d6b43-0379-447a-ad9d-b2f36f7dce3e'), id=UUID('06c104d2-a502-400b-9728-b8e2d9ee65d9'), unit='ºC', display_name_english='TAS seasonal absolute model ensemble', palette='default/seq-YlOrRd', display_name_italian='TAS valore assoluto di stagione media ensemble', color_scale_min=-3.0, description_english='TAS seasonal absolute model ensemble', color_scale_max=32.0, description_italian='TAS valore assoluto di stagione media ensemble', observation_variable_id=UUID('32706ce0-0134-4122-91a0-9d6f237350b3'), coverage_id_pattern='{name}-{climatological_model}-{measure}-{scenario}-{year_period}'), 'configuration_parameter_values_filter': [ConfigurationParameterValue(id=UUID('c5381cd2-ca57-48a9-a096-71568e812366'), display_name_italian='Insieme di 5 modelli', description_italian='Insieme di cinque modelli climatologici: EC-EARTH CCLM4-8-17, EC-EARTH RACMO22E, EC-EARTH RCA4, HadGEM RACMO22E, MPI-ESM-LR-REMO2009', display_name_english='5 Model ensemble', name='model_ensemble', description_english='Ensemble of five climatological models: EC-EARTH CCLM4-8-17, EC-EARTH RACMO22E, EC-EARTH RCA4, HadGEM RACMO22E, MPI-ESM-LR-REMO2009', configuration_parameter_id=UUID('d6cfbf61-7f87-43a9-8e04-2c9d275eacc2')), ConfigurationParameterValue(id=UUID('70c82437-5e90-4181-96c7-262a097d18ba'), display_name_italian='RCP2.6', description_italian='Scenario Representation Concentration Pathway (RCP) che presuppone valori di forzante climatica di 2,6 W/m2', display_name_english='RCP2.6', name='rcp26', description_english='Representation Concentration Pathway (RCP) scenario that assumes climate forcing values of 2.6 W/m2', configuration_parameter_id=UUID('92bbb178-334f-4f30-bf2c-fb7cba7cd7e0'))]}\n", + "DEBUG:arpav_ppcv.database:pattern_parts=['climatological_model', 'measure', 'scenario', 'year_period']\n" + ] + }, + { + "data": { + "text/plain": [ + "['tas_seasonal_absolute_model_ensemble-model_ensemble-measure-rcp26-year_period']" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db.generate_coverage_identifiers(session, coverage_configuration=cov_conf, configuration_parameter_values_filter=param_values)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "978549f0-e1d6-415c-833b-3169b7448920", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "([CoverageConfiguration(netcdf_main_dataset_name='tas', observation_variable_aggregation_type=, thredds_url_pattern='ensymbc/clipped/tas_avg_{scenario}_{year_period}_ts19762100_ls_VFVG.nc', uncertainty_lower_bounds_coverage_configuration_id=UUID('9c2bfa7a-d108-4d2d-8c99-f26ae9329e79'), name='tas_seasonal_absolute_model_ensemble', wms_main_layer_name='tas', uncertainty_upper_bounds_coverage_configuration_id=UUID('d33d6b43-0379-447a-ad9d-b2f36f7dce3e'), id=UUID('06c104d2-a502-400b-9728-b8e2d9ee65d9'), unit='ºC', display_name_english='TAS seasonal absolute model ensemble', palette='default/seq-YlOrRd', display_name_italian='TAS valore assoluto di stagione media ensemble', color_scale_min=-3.0, description_english='TAS seasonal absolute model ensemble', color_scale_max=32.0, description_italian='TAS valore assoluto di stagione media ensemble', observation_variable_id=UUID('32706ce0-0134-4122-91a0-9d6f237350b3'), coverage_id_pattern='{name}-{climatological_model}-{measure}-{scenario}-{year_period}')],\n", + " None)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db.list_coverage_configurations(session, configuration_parameter_value_filter=param_values)" + ] + }, { "cell_type": "code", "execution_count": 29, diff --git a/tests/test_database.py b/tests/test_database.py index af182db9..461e7e08 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -30,6 +30,88 @@ def test_list_stations(arpav_db_session, sample_stations, limit, offset, include assert db_station.code == expected_codes[index] +@pytest.mark.parametrize( + "limit, offset, include_total", + [ + pytest.param(10, 0, False), + ], +) +def test_list_coverage_configurations( + arpav_db_session, sample_coverage_configurations, limit, offset, include_total +): + db_cov_confs, total = database.list_coverage_configurations( + arpav_db_session, limit=limit, offset=offset, include_total=include_total + ) + ordered_cov_confs = sorted( + sample_coverage_configurations, key=lambda cov_conf: cov_conf.name + ) + if include_total: + assert total == len(sample_coverage_configurations) + else: + assert total is None + for index, db_cov_conf in enumerate(db_cov_confs): + assert db_cov_conf.name == ordered_cov_confs[index].name + + +@pytest.mark.parametrize( + "params_to_generate, params_to_use, expected_identifiers", + [ + pytest.param( + {"vvv": ["v1", "v2", "v3"], "sss": ["s1", "s2", "s3"]}, + ["vvv-v1", "vvv-v2", "sss-s1"], + ["cc-s1-v1", "cc-s1-v2"], + ), + pytest.param( + {"vvv": ["v1", "v2", "v3"], "sss": ["s1", "s2", "s3"]}, + ["vvv-v1", "vvv-v2", "sss-s1", "sss-s2"], + ["cc-s1-v1", "cc-s1-v2", "cc-s2-v1", "cc-s2-v2"], + ), + ], +) +def test_generate_coverage_identifiers( + arpav_db_session, + params_to_generate: dict[str, list[str]], + params_to_use: list[str], + expected_identifiers: list[str], +): + params = {} + for p, values in params_to_generate.items(): + params[p] = coverages.ConfigurationParameter( + name=p, + allowed_values=[ + coverages.ConfigurationParameterValue(name=v) for v in values + ], + ) + for param_to_create in params.values(): + arpav_db_session.add(param_to_create) + arpav_db_session.commit() + for param_created in params.values(): + arpav_db_session.refresh(param_created) + + to_use = [] + for spec_to_use in params_to_use: + param_name, param_value = spec_to_use.partition("-")[::2] + param = params[param_name] + for allowed_value in param.allowed_values: + if allowed_value.name == param_value: + to_use.append( + coverages.ConfigurationParameterPossibleValue( + configuration_parameter_value=allowed_value + ) + ) + + cov_conf = coverages.CoverageConfiguration( + name="cc", + netcdf_main_dataset_name="some-dataset-name", + thredds_url_pattern="the_thredds-param_{vvv}", + palette="fake", + possible_values=to_use, + ) + arpav_db_session.add(cov_conf) + arpav_db_session.commit() + arpav_db_session.refresh(cov_conf) + + @pytest.mark.parametrize( "limit, offset, include_total", [ From 17e32a11d743bdd9df5ce8d29d331efd948353e4 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Fri, 19 Jul 2024 17:13:52 +0100 Subject: [PATCH 6/6] fix bug in generation of test samples --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 469465bc..f583093e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -224,7 +224,7 @@ def sample_coverage_configurations( ) -> list[coverages.CoverageConfiguration]: db_cov_confs = [] for i in range(10): - params_to_use = random.choices(sample_configuration_parameters, k=2) + params_to_use = random.sample(sample_configuration_parameters, k=2) param_values_to_use = [] for param in params_to_use: possible_value = coverages.ConfigurationParameterPossibleValue(