diff --git a/src/ert/dark_storage/common.py b/src/ert/dark_storage/common.py index cc6da63f5d5..3039e9659e1 100644 --- a/src/ert/dark_storage/common.py +++ b/src/ert/dark_storage/common.py @@ -1,35 +1,32 @@ -import logging from typing import Any, Dict, List, Optional, Sequence, Union +from uuid import UUID import pandas as pd from ert.config import EnkfObservationImplementationType from ert.libres_facade import LibresFacade -from ert.storage import EnsembleReader +from ert.storage import EnsembleReader, StorageReader -_logger = logging.getLogger(__name__) +def ensemble_parameter_names(storage: StorageReader, ensemble_id: UUID) -> List[str]: + return storage.get_ensemble(ensemble_id).get_gen_kw_keyset() -def ensemble_parameter_names(res: LibresFacade) -> List[str]: - return res.gen_kw_keys() - -def ensemble_parameters(res: LibresFacade) -> List[Dict[str, Any]]: +def ensemble_parameters( + storage: StorageReader, ensemble_id: UUID +) -> List[Dict[str, Any]]: return [ {"name": key, "userdata": {"data_origin": "GEN_KW"}, "labels": []} - for key in ensemble_parameter_names(res) + for key in ensemble_parameter_names(storage, ensemble_id) ] -def get_response_names(res: LibresFacade, ensemble: EnsembleReader) -> List[str]: +def get_response_names(ensemble: EnsembleReader) -> List[str]: result = ensemble.get_summary_keyset() - result.extend(res.get_gen_data_keys().copy()) + result.extend(ensemble.get_gen_data_keyset().copy()) return result -#### - - def data_for_key( ensemble: EnsembleReader, key: str, @@ -70,9 +67,6 @@ def data_for_key( return data -##################### - - def observations_for_obs_keys( res: LibresFacade, obs_keys: List[str] ) -> List[Dict[str, Any]]: diff --git a/src/ert/dark_storage/endpoints/ensembles.py b/src/ert/dark_storage/endpoints/ensembles.py index 8a05f0bdbce..8d0a53fc02c 100644 --- a/src/ert/dark_storage/endpoints/ensembles.py +++ b/src/ert/dark_storage/endpoints/ensembles.py @@ -5,11 +5,10 @@ from ert.dark_storage import json_schema as js from ert.dark_storage.common import ensemble_parameter_names, get_response_names -from ert.dark_storage.enkf import LibresFacade, get_res, get_storage +from ert.dark_storage.enkf import get_storage from ert.storage import StorageAccessor router = APIRouter(tags=["ensemble"]) -DEFAULT_LIBRESFACADE = Depends(get_res) DEFAULT_STORAGE = Depends(get_storage) DEFAULT_BODY = Body(...) @@ -17,7 +16,6 @@ @router.post("/experiments/{experiment_id}/ensembles", response_model=js.EnsembleOut) def post_ensemble( *, - res: LibresFacade = DEFAULT_LIBRESFACADE, ens_in: js.EnsembleIn, experiment_id: UUID, ) -> js.EnsembleOut: @@ -27,20 +25,19 @@ def post_ensemble( @router.get("/ensembles/{ensemble_id}", response_model=js.EnsembleOut) def get_ensemble( *, - res: LibresFacade = DEFAULT_LIBRESFACADE, - db: StorageAccessor = DEFAULT_STORAGE, + storage: StorageAccessor = DEFAULT_STORAGE, ensemble_id: UUID, ) -> js.EnsembleOut: - ens = db.get_ensemble(ensemble_id) + ensemble = storage.get_ensemble(ensemble_id) return js.EnsembleOut( id=ensemble_id, children=[], parent=None, - experiment_id=ens.experiment_id, - userdata={"name": ens.name}, - size=ens.ensemble_size, - parameter_names=ensemble_parameter_names(res), - response_names=get_response_names(res, ens), + experiment_id=ensemble.experiment_id, + userdata={"name": ensemble.name}, + size=ensemble.ensemble_size, + parameter_names=ensemble_parameter_names(storage, ensemble_id), + response_names=get_response_names(ensemble), child_ensemble_ids=[], ) @@ -48,7 +45,6 @@ def get_ensemble( @router.put("/ensembles/{ensemble_id}/userdata") async def replace_ensemble_userdata( *, - res: LibresFacade = DEFAULT_LIBRESFACADE, ensemble_id: UUID, body: Any = DEFAULT_BODY, ) -> None: @@ -58,7 +54,6 @@ async def replace_ensemble_userdata( @router.patch("/ensembles/{ensemble_id}/userdata") async def patch_ensemble_userdata( *, - res: LibresFacade = DEFAULT_LIBRESFACADE, ensemble_id: UUID, body: Any = DEFAULT_BODY, ) -> None: @@ -68,7 +63,6 @@ async def patch_ensemble_userdata( @router.get("/ensembles/{ensemble_id}/userdata", response_model=Mapping[str, Any]) async def get_ensemble_userdata( *, - res: LibresFacade = DEFAULT_LIBRESFACADE, ensemble_id: UUID, ) -> Mapping[str, Any]: raise NotImplementedError diff --git a/src/ert/dark_storage/endpoints/observations.py b/src/ert/dark_storage/endpoints/observations.py index 830049fe5f0..3dabf2e901b 100644 --- a/src/ert/dark_storage/endpoints/observations.py +++ b/src/ert/dark_storage/endpoints/observations.py @@ -18,7 +18,6 @@ ) def post_observation( *, - res: LibresFacade = DEFAULT_LIBRESFACADE, obs_in: js.ObservationIn, experiment_id: UUID, ) -> js.ObservationOut: @@ -48,7 +47,7 @@ def get_observations( "/ensembles/{ensemble_id}/observations", response_model=List[js.ObservationOut] ) def get_observations_with_transformation( - *, res: LibresFacade = DEFAULT_LIBRESFACADE, ensemble_id: UUID + *, ensemble_id: UUID ) -> List[js.ObservationOut]: raise NotImplementedError @@ -56,7 +55,6 @@ def get_observations_with_transformation( @router.put("/observations/{obs_id}/userdata") async def replace_observation_userdata( *, - res: LibresFacade = DEFAULT_LIBRESFACADE, obs_id: UUID, body: Any = DEFAULT_BODY, ) -> None: @@ -66,7 +64,6 @@ async def replace_observation_userdata( @router.patch("/observations/{obs_id}/userdata") async def patch_observation_userdata( *, - res: LibresFacade = DEFAULT_LIBRESFACADE, obs_id: UUID, body: Any = DEFAULT_BODY, ) -> None: @@ -76,7 +73,6 @@ async def patch_observation_userdata( @router.get("/observations/{obs_id}/userdata", response_model=Mapping[str, Any]) async def get_observation_userdata( *, - res: LibresFacade = DEFAULT_LIBRESFACADE, obs_id: UUID, ) -> Mapping[str, Any]: raise NotImplementedError diff --git a/src/ert/dark_storage/endpoints/records.py b/src/ert/dark_storage/endpoints/records.py index 1e6253f8eef..5de1cf58247 100644 --- a/src/ert/dark_storage/endpoints/records.py +++ b/src/ert/dark_storage/endpoints/records.py @@ -30,7 +30,6 @@ @router.post("/ensembles/{ensemble_id}/records/{name}/file") async def post_ensemble_record_file( *, - res: LibresFacade = DEFAULT_LIBRESFACADE, name: str, ensemble_id: UUID, realization_index: Optional[int] = None, @@ -42,7 +41,6 @@ async def post_ensemble_record_file( @router.put("/ensembles/{ensemble_id}/records/{name}/blob") async def add_block( *, - res: LibresFacade = DEFAULT_LIBRESFACADE, name: str, ensemble_id: UUID, block_index: int, @@ -55,7 +53,6 @@ async def add_block( @router.post("/ensembles/{ensemble_id}/records/{name}/blob") async def create_blob( *, - res: LibresFacade = DEFAULT_LIBRESFACADE, name: str, ensemble_id: UUID, realization_index: Optional[int] = None, @@ -66,7 +63,6 @@ async def create_blob( @router.patch("/ensembles/{ensemble_id}/records/{name}/blob") async def finalize_blob( *, - res: LibresFacade = DEFAULT_LIBRESFACADE, name: str, ensemble_id: UUID, realization_index: Optional[int] = None, @@ -79,7 +75,6 @@ async def finalize_blob( ) async def post_ensemble_record_matrix( *, - res: LibresFacade = DEFAULT_LIBRESFACADE, ensemble_id: UUID, name: str, prior: Optional[str] = None, @@ -93,7 +88,6 @@ async def post_ensemble_record_matrix( @router.put("/ensembles/{ensemble_id}/records/{name}/userdata") async def replace_record_userdata( *, - res: LibresFacade = DEFAULT_LIBRESFACADE, ensemble_id: UUID, name: str, realization_index: Optional[int] = None, @@ -105,7 +99,6 @@ async def replace_record_userdata( @router.patch("/ensembles/{ensemble_id}/records/{name}/userdata") async def patch_record_userdata( *, - res: LibresFacade = DEFAULT_LIBRESFACADE, ensemble_id: UUID, name: str, realization_index: Optional[int] = None, @@ -119,7 +112,6 @@ async def patch_record_userdata( ) async def get_record_userdata( *, - res: LibresFacade = DEFAULT_LIBRESFACADE, ensemble_id: UUID, name: str, realization_index: Optional[int] = None, @@ -130,7 +122,6 @@ async def get_record_userdata( @router.post("/ensembles/{ensemble_id}/records/{name}/observations") async def post_record_observations( *, - res: LibresFacade = DEFAULT_LIBRESFACADE, ensemble_id: UUID, name: str, realization_index: Optional[int] = None, @@ -214,7 +205,6 @@ async def get_ensemble_record( @router.get("/ensembles/{ensemble_id}/records/{name}/labels", response_model=List[str]) async def get_record_labels( *, - res: LibresFacade = DEFAULT_LIBRESFACADE, ensemble_id: UUID, name: str, ) -> List[str]: @@ -223,31 +213,26 @@ async def get_record_labels( @router.get("/ensembles/{ensemble_id}/parameters", response_model=List[Dict[str, Any]]) async def get_ensemble_parameters( - *, res: LibresFacade = DEFAULT_LIBRESFACADE, ensemble_id: UUID + *, storage: StorageReader = DEFAULT_STORAGE, ensemble_id: UUID ) -> List[Dict[str, Any]]: - return ensemble_parameters(res) + return ensemble_parameters(storage, ensemble_id) @router.get( "/ensembles/{ensemble_id}/records", response_model=Mapping[str, js.RecordOut] ) -async def get_ensemble_records( - *, res: LibresFacade = DEFAULT_LIBRESFACADE, ensemble_id: UUID -) -> Mapping[str, js.RecordOut]: +async def get_ensemble_records(*, ensemble_id: UUID) -> Mapping[str, js.RecordOut]: raise NotImplementedError @router.get("/records/{record_id}", response_model=js.RecordOut) -async def get_record( - *, res: LibresFacade = DEFAULT_LIBRESFACADE, record_id: UUID -) -> js.RecordOut: +async def get_record(*, record_id: UUID) -> js.RecordOut: raise NotImplementedError @router.get("/records/{record_id}/data") async def get_record_data( *, - res: LibresFacade = DEFAULT_LIBRESFACADE, record_id: UUID, accept: Optional[str] = DEFAULT_HEADER, ) -> Any: diff --git a/src/ert/libres_facade.py b/src/ert/libres_facade.py index 5af8bba4915..ac142835005 100644 --- a/src/ert/libres_facade.py +++ b/src/ert/libres_facade.py @@ -199,7 +199,6 @@ def get_data_key_for_obs_key(self, observation_key: str) -> str: else: return obs.data_key - # duplicate in local_ensemble def get_gen_data_keys(self) -> List[str]: ensemble_config = self.config.ensemble_config gen_data_keys = ensemble_config.get_keylist_gen_data() diff --git a/src/ert/shared/storage/extraction.py b/src/ert/shared/storage/extraction.py index 880d3bfacdd..242eadde041 100644 --- a/src/ert/shared/storage/extraction.py +++ b/src/ert/shared/storage/extraction.py @@ -10,12 +10,6 @@ logger = logging.getLogger() - -def create_observations(ert: LibresFacade) -> List[Dict[str, Dict[str, Any]]]: - keys = [i.observation_key for i in ert.get_observations()] - return observations_for_obs_keys(ert, keys) - - _PRIOR_NAME_MAP = { "NORMAL": "normal", "LOGNORMAL": "lognormal", @@ -31,6 +25,11 @@ def create_observations(ert: LibresFacade) -> List[Dict[str, Dict[str, Any]]]: } +def create_observations(ert: LibresFacade) -> List[Dict[str, Dict[str, Any]]]: + keys = [i.observation_key for i in ert.get_observations()] + return observations_for_obs_keys(ert, keys) + + def create_priors(ert: LibresFacade) -> Mapping[str, Dict[str, Union[str, float]]]: priors = {} for group, gen_kw_priors in ert.gen_kw_priors().items(): diff --git a/src/ert/storage/local_ensemble.py b/src/ert/storage/local_ensemble.py index 5d3c14f24fc..822bdb18c39 100644 --- a/src/ert/storage/local_ensemble.py +++ b/src/ert/storage/local_ensemble.py @@ -230,8 +230,6 @@ def load_responses( def get_active_realizations(self) -> List[int]: return self.realization_list(RealizationStorageState.HAS_DATA) - ### summary data - def load_all_summary_data( self, keys: Optional[List[str]] = None, @@ -279,7 +277,6 @@ def gather_summary_data( ) return data.unstack(level="Realization") - #### gen data def _get_gen_data_config(self, key: str) -> GenDataConfig: config = self.experiment.response_configuration[key] assert isinstance(config, GenDataConfig) @@ -326,8 +323,6 @@ def load_gen_data( columns=realizations, ) - ###### gen_kw - def get_gen_kw_keyset(self) -> List[str]: gen_kw_keys = [ k diff --git a/tests/performance_tests/test_dark_storage_performance.py b/tests/performance_tests/test_dark_storage_performance.py index 9d311064c8d..e8c0d72e580 100644 --- a/tests/performance_tests/test_dark_storage_performance.py +++ b/tests/performance_tests/test_dark_storage_performance.py @@ -18,10 +18,9 @@ def run_in_loop(coro: Awaitable[T]) -> T: return get_event_loop().run_until_complete(coro) -def get_single_record_csv(ert, storage, ensemble_id1, keyword, poly_ran): +def get_single_record_csv(storage, ensemble_id1, keyword, poly_ran): csv = run_in_loop( records.get_ensemble_record( - res=ert, db=storage, name=keyword, ensemble_id=ensemble_id1, @@ -70,10 +69,9 @@ def get_observations(ert, storage, ensemble_id1, keyword: str, poly_ran): raise AssertionError(f"should never get here, keyword is {keyword}") -def get_single_record_parquet(ert, storage, ensemble_id1, keyword, poly_ran): +def get_single_record_parquet(storage, ensemble_id1, keyword, poly_ran): parquet = run_in_loop( records.get_ensemble_record( - res=ert, db=storage, name=keyword, ensemble_id=ensemble_id1, @@ -86,10 +84,9 @@ def get_single_record_parquet(ert, storage, ensemble_id1, keyword, poly_ran): assert len(record_df1_indexed.index) == 1 -def get_record_parquet(ert, storage, ensemble_id1, keyword, poly_ran): +def get_record_parquet(storage, ensemble_id1, keyword, poly_ran): parquet = run_in_loop( records.get_ensemble_record( - res=ert, db=storage, name=keyword, ensemble_id=ensemble_id1, @@ -101,21 +98,19 @@ def get_record_parquet(ert, storage, ensemble_id1, keyword, poly_ran): assert len(record_df1.index) == poly_ran["reals"] -def get_record_csv(ert, storage, ensemble_id1, keyword, poly_ran): +def get_record_csv(storage, ensemble_id1, keyword, poly_ran): csv = run_in_loop( - records.get_ensemble_record( - res=ert, db=storage, name=keyword, ensemble_id=ensemble_id1 - ) + records.get_ensemble_record(db=storage, name=keyword, ensemble_id=ensemble_id1) ).body record_df1 = pd.read_csv(io.BytesIO(csv), index_col=0, float_precision="round_trip") assert len(record_df1.columns) == poly_ran["gen_data_entries"] assert len(record_df1.index) == poly_ran["reals"] -def get_result(ert, storage, ensemble_id1, keyword, poly_ran): +def get_result(storage, ensemble_id1, keyword, poly_ran): csv = run_in_loop( responses.get_ensemble_response_dataframe( - res=ert, db=storage, ensemble_id=ensemble_id1, response_name=keyword + db=storage, ensemble_id=ensemble_id1, response_name=keyword ) ).body response_df1 = pd.read_csv( @@ -125,9 +120,9 @@ def get_result(ert, storage, ensemble_id1, keyword, poly_ran): assert len(response_df1.index) == poly_ran["reals"] -def get_parameters(ert, storage, ensemble_id1, keyword, poly_ran): +def get_parameters(storage, ensemble_id1, keyword, poly_ran): parameters_json = run_in_loop( - records.get_ensemble_parameters(res=ert, ensemble_id=ensemble_id1) + records.get_ensemble_parameters(storage=storage, ensemble_id=ensemble_id1) ) assert ( len(parameters_json) @@ -142,7 +137,6 @@ def get_parameters(ert, storage, ensemble_id1, keyword, poly_ran): get_record_parquet, get_record_csv, get_single_record_parquet, - get_observations, get_parameters, ], ) @@ -170,7 +164,46 @@ def test_direct_dark_performance( ensemble_id_default = None for ensemble_id in experiment_json[0].ensemble_ids: ensemble_json = ensembles.get_ensemble( - res=enkf_facade, db=storage, ensemble_id=ensemble_id + storage=storage, ensemble_id=ensemble_id + ) + if ensemble_json.userdata["name"] == "default": + ensemble_json_default = ensemble_json + ensemble_id_default = ensemble_id + assert key in ensemble_json_default.response_names + benchmark(function, storage, ensemble_id_default, key, template_config) + + +@pytest.mark.parametrize( + "function", + [ + get_observations, + ], +) +@pytest.mark.parametrize( + "keyword", ["summary", "gen_data", "summary_with_obs", "gen_data_with_obs"] +) +@pytest.mark.integration_test +def test_direct_dark_performance_with_libres_facade( + benchmark, template_config, monkeypatch, function, keyword +): + key = { + "summary": "PSUM1", + "gen_data": "POLY_RES_1@0", + "summary_with_obs": "PSUM0", + "gen_data_with_obs": "POLY_RES_0@0", + }[keyword] + + with template_config["folder"].as_cwd(): + config = ErtConfig.from_file("poly.ert") + ert = EnKFMain(config) + enkf_facade = LibresFacade(ert) + storage = open_storage(enkf_facade.enspath) + experiment_json = experiments.get_experiments(res=enkf_facade, db=storage) + ensemble_json_default = None + ensemble_id_default = None + for ensemble_id in experiment_json[0].ensemble_ids: + ensemble_json = ensembles.get_ensemble( + storage=storage, ensemble_id=ensemble_id ) if ensemble_json.userdata["name"] == "default": ensemble_json_default = ensemble_json