From 202330777be2b3d0cecb8fc1891e68af86a2856f Mon Sep 17 00:00:00 2001 From: Zohar Malamant Date: Fri, 23 Apr 2021 10:15:49 +0200 Subject: [PATCH] Webviz-ert compatible with newnew storage co-author: Zohar Malamant --- tests/conftest.py | 90 ++- tests/data/snake_oil_data.py | 563 +++++++----------- tests/models/test_ensemble_model.py | 1 - tests/models/test_response_model.py | 7 +- tests/plots/test_controller.py | 12 +- .../ensemble_selector_controller.py | 6 +- .../controllers/multi_parameter_controller.py | 4 +- .../controllers/multi_response_controller.py | 4 +- .../observation_response_controller.py | 15 +- .../parameter_comparison_controller.py | 2 +- .../parameter_selector_controller.py | 2 +- .../response_correlation_controller.py | 14 +- webviz_ert/data_loader/__init__.py | 304 +++++++--- webviz_ert/models/__init__.py | 6 +- webviz_ert/models/ensemble_model.py | 71 ++- webviz_ert/models/observation.py | 15 +- webviz_ert/models/parameter_model.py | 18 +- webviz_ert/models/plot_model.py | 20 +- webviz_ert/models/response.py | 155 ++--- webviz_ert/plugins/_webviz_ert.py | 2 +- 20 files changed, 621 insertions(+), 690 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 7ec282f5..67c3eb99 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,69 +21,59 @@ def mock_data(mocker): "webviz_ert.data_loader.get_info", side_effect=lambda _: {"baseurl": "http://127.0.0.1:5000", "auth": ""}, ) - mocker.patch( - "webviz_ert.data_loader.get_url", side_effect=lambda _: "http://127.0.0.1:5000" - ) - mocker.patch("webviz_ert.data_loader.get_auth", side_effect=lambda _: "") - mocker.patch("webviz_ert.data_loader.pandas.read_csv", side_effect=_pandas_read_csv) + mocker.patch("webviz_ert.data_loader._requests_get", side_effect=_requests_get) - mocker.patch( - "webviz_ert.models.ensemble_model.get_ensemble_url", side_effect=_ensemble_url - ) - mocker.patch( - "webviz_ert.models.response.get_response_url", side_effect=_response_url - ) - mocker.patch( - "webviz_ert.models.parameter_model.get_parameter_data_url", - side_effect=_parameter_data_url, - ) + mocker.patch("webviz_ert.data_loader._requests_post", side_effect=_requests_post) -def _pandas_read_csv(*args, **kwargs): - data = args[0] - if "header" in kwargs: - return pd.DataFrame(data=list(data), columns=kwargs["header"]) - return pd.DataFrame(data=list(data), columns=["value"]) +class _MockResponse: + def __init__(self, url, data, status_code): + self.data = data + self.status_code = status_code + def json(self): + return self.data -def _requests_get(url, **kwargs): - class MockResponse: - def __init__(self, data, status_code): - self.data = data - self.status_code = status_code + @property + def text(self): + return self.data - def json(self): - return self.data + @property + def raw(self): + return self.data - @property - def text(self): - return self.data + @property + def content(self): + return self.data - @property - def raw(self): - return self.data + def raise_for_status(self): + if self.status_code == 400: + raise HTTPError( + "Mocked requests raised HTTPError 400 due to missing data in test-data set!\n" + f"{url}" + ) - def raise_for_status(self): - if self.status_code == 400: - raise HTTPError( - "Mocked requests raised HTTPError 400 due to missing data in test-data set!\n" - f"{args[0]}" - ) +def _requests_get(url, **kwargs): + if kwargs.get("params") is not None: + url += "?" + for param, value in kwargs["params"].items(): + url += f"{param}={value}" if url in ensembles_response: - return MockResponse(ensembles_response[url], 200) - return MockResponse({}, 400) + return _MockResponse(url, ensembles_response[url], 200) + return _MockResponse(url, {}, 400) -def _ensemble_url(ensemble_id, project_id=None): - return f"http://127.0.0.1:5000/ensembles/{ensemble_id}" +from webviz_ert.data_loader import GET_ENSEMBLE -def _response_url(ensemble_id, response_id, project_id=None): - return f"http://127.0.0.1:5000/ensembles/{ensemble_id}/responses/{response_id}" +def _requests_post(url, **kwargs): + payload = kwargs["json"] + if payload["query"] == GET_ENSEMBLE: + variables = payload["variables"] + ensemble_id = variables["id"] + url = f"http://127.0.0.1:5000/ensembles/{ensemble_id}" - -def _parameter_data_url(ensemble_id, parameter_id, project_id=None): - return ( - f"http://127.0.0.1:5000/ensembles/{ensemble_id}/parameters/{parameter_id}/data" - ) + if url in ensembles_response: + return _MockResponse(url, ensembles_response[url], 200) + return _MockResponse(url, {}, 400) diff --git a/tests/data/snake_oil_data.py b/tests/data/snake_oil_data.py index e7a98d1a..fd307d9d 100644 --- a/tests/data/snake_oil_data.py +++ b/tests/data/snake_oil_data.py @@ -1,3 +1,6 @@ +import pandas as pd + + class DataContent: def __init__(self, content): self._content = content.encode() @@ -8,379 +11,223 @@ def content(self): ensembles_response = { - "http://127.0.0.1:5000/ensembles": { - "ensembles": [ - { - "children": [ - { - "name": "default_smoother_update", - "id": 2, - } - ], - "name": "default", - "parent": {}, - "id": 1, - "time_created": "2020-04-29T09:36:26", - }, - { - "children": [], - "name": "default_smoother_update", - "parent": { - "name": "default", - "id": 1, - }, - "id": 2, - "time_created": "2020-04-29T09:43:25", - }, - ] - }, - "http://127.0.0.1:5000/ensembles/1": { - "children": [ - { - "name": "default_smoother_update", - "id": 2, - } - ], - "name": "default", - "parameters": [ - { - "group": "SNAKE_OIL_PARAM", - "key": "BPR_138_PERSISTENCE", - "prior": { - "function": "UNIFORM", - "parameter_names": ["MIN", "MAX"], - "parameter_values": [0.2, 0.7], - }, - "id": 1, - }, - { - "group": "SNAKE_OIL_PARAM", - "key": "OP1_DIVERGENCE_SCALE", - "prior": { - "function": "UNIFORM", - "parameter_names": ["MIN", "MAX"], - "parameter_values": [0.2, 0.7], - }, - "id": 2, - }, - ], - "parent": {}, - "realizations": [ - {"name": 0, "id": 0}, - {"name": 1, "id": 1}, - {"name": 2, "id": 2}, - ], - "id": 1, - "responses": [ - { - "name": "SNAKE_OIL_GPR_DIFF", - "id": "SNAKE_OIL_GPR_DIFF", - }, - ], - "time_created": "2020-04-29T09:36:26", + "http://127.0.0.1:5000/gql": { + "data": { + "experiments": [ + { + "name": "exp1", + "ensembles": [ + { + "children": [ + 2, + ], + "name": "default", + "parent": None, + "id": 1, + "time_created": "2020-04-29T09:36:26", + }, + { + "children": [], + "name": "default_smoother_update", + "parent": { + "name": "default", + "id": 1, + }, + "id": 2, + "time_created": "2020-04-29T09:43:25", + }, + ], + } + ] + } }, - "http://127.0.0.1:5000/ensembles/2": { - "children": [], - "name": "default_smoother_update", - "parameters": [ - { - "group": "SNAKE_OIL_PARAM", - "key": "BPR_138_PERSISTENCE", - "prior": { - "function": "UNIFORM", - "parameter_names": ["MIN", "MAX"], - "parameter_values": [0.2, 0.7], - }, - "id": 1, - }, - { - "group": "SNAKE_OIL_PARAM", - "key": "OP1_DIVERGENCE_SCALE", - "prior": { - "function": "UNIFORM", - "parameter_names": ["MIN", "MAX"], - "parameter_values": [0.2, 0.7], - }, - "id": 2, - }, - ], - "parent": { - "name": "default", - "id": 1, + "http://127.0.0.1:5000/experiments/exp1_id/priors": { + "BPR_138_PERSISTENCE": { + "function": "UNIFORM", + "parameter_names": ["MIN", "MAX"], + "parameter_values": [0.2, 0.7], }, - "realizations": [ - {"name": 0, "id": 0}, - {"name": 1, "id": 1}, - {"name": 2, "id": 2}, - ], - "id": 2, - "responses": [ - { - "name": "SNAKE_OIL_GPR_DIFF", - "id": "SNAKE_OIL_GPR_DIFF", - }, - ], - "time_created": "2020-04-29T10:36:26", - }, - "http://127.0.0.1:5000/ensembles/3": { - "children": [], - "name": "default3", - "parameters": [], - "parent": {}, - "id": 3, - "realizations": [], - "responses": [ - { - "name": "SNAKE_OIL_GPR_DIFF", - "id": "SNAKE_OIL_GPR_DIFF", - }, - { - "name": "FOPR", - "id": "FOPR", - }, - { - "name": "WOPR:OP1", - "id": "WOPR:OP1", - }, - ], - "time_created": "2020-04-29T10:57:47", - }, - "http://127.0.0.1:5000/ensembles/4": { - "children": [], - "name": "default4", - "parameters": [], - "parent": {}, - "id": 4, - "realizations": [], - "responses": [ - { - "name": "SNAKE_OIL_GPR_DIFF", - "id": "SNAKE_OIL_GPR_DIFF", - }, - { - "name": "FOPR", - "id": "FOPR", - }, - ], - "time_created": "2020-04-29T10:59:37", - }, - "http://127.0.0.1:5000/ensembles/1/parameters/1": { - "alldata_url": "http://127.0.0.1:5000/ensembles/1/parameters/1/data", - "group": "SNAKE_OIL_PARAM", - "key": "BPR_138_PERSISTENCE", - "parameter_realizations": [ - { - "data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], - "name": 0, - "realization": {"id": 0}, - }, - { - "data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], - "name": 1, - "realization": {"id": 1}, - }, - { - "data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], - "name": 2, - "realization": {"id": 2}, - }, - ], - "prior": { + "OP1_DIVERGENCE_SCALE": { "function": "UNIFORM", "parameter_names": ["MIN", "MAX"], "parameter_values": [0.2, 0.7], }, - "id": 1, - }, - "http://127.0.0.1:5000/ensembles/1/realizations/0": { - "name": 0, - "parameters": [ - { - "data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], - "name": "BPR_138_PERSISTENCE", - }, - { - "data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], - "name": "BPR_555_PERSISTENCE", - }, - { - "data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], - "name": "OP1_DIVERGENCE_SCALE", - }, - ], - "responses": [ - { - "data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], - "name": "SNAKE_OIL_GPR_DIFF", - }, - { - "data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], - "name": "SNAKE_OIL_OPR_DIFF", - }, - { - "data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], - "name": "SNAKE_OIL_WPR_DIFF", - }, - ], }, - "http://127.0.0.1:5000/ensembles/1/responses/SNAKE_OIL_GPR_DIFF": { - "alldata_url": "http://127.0.0.1:5000/ensembles/1/responses/SNAKE_OIL_GPR_DIFF/data", - "axis": {"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}, - "ensemble_id": "1", - "name": "SNAKE_OIL_GPR_DIFF", - "realizations": [ - { - "data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], - "name": 0, - "id": "0", - "summarized_misfits": {}, - "univariate_misfits": {}, + "http://127.0.0.1:5000/ensembles/1": { + "data": { + "ensemble": { + "children": [{"ensembleResult": {"id": 2}}], + "experiment": {"id": "exp1_id"}, + "parent": None, + "id": 1, + "timeCreated": "2020-04-29T09:36:26", + "size": 1, + "Metadata": '{"name": "default"}', } - ], + } }, - "http://127.0.0.1:5000/ensembles/1/responses/FOPR": { - "alldata_url": "http://127.0.0.1:5000/ensembles/1/responses/FOPR/data", - "axis": {"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}, - "ensemble_id": "1", - "name": "FOPR", - "observations": [ - { - "data": { - "x_axis": {"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}, - "std": {"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}, - "values": {"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}, - "active": { - "data": [ - True, - True, - False, - False, - True, - True, - True, - True, - True, - False, - ] - }, - }, - "name": "FOPR", - } - ], - "realizations": [ - { - "data": "http://127.0.0.1:5000/data/726", - "name": 0, - "id": 0, - "summarized_misfits": {"FOPR": 946.263115564503}, - "univariate_misfits": { - "FOPR": [ - { - "obs_index": 0, - "sign": True, - "value": 1.3776484533744848, - "obs_location": 0, - }, - { - "obs_index": 1, - "sign": True, - "value": 1.384794184010784, - "obs_location": 1, - }, - { - "obs_index": 2, - "sign": True, - "value": 1.3966335691013885, - "obs_location": 2, - }, - ] - }, - } - ], + "http://127.0.0.1:5000/ensembles/1/parameters": [ + "BPR_138_PERSISTENCE", + "OP1_DIVERGENCE_SCALE", + ], + "http://127.0.0.1:5000/ensembles/1/responses": { + "SNAKE_OIL_GPR_DIFF": { + "name": "SNAKE_OIL_GPR_DIFF", + "id": "SNAKE_OIL_GPR_DIFF", + }, }, - "http://127.0.0.1:5000/ensembles/3/responses/SNAKE_OIL_GPR_DIFF": { - "alldata_url": "http://127.0.0.1:5000/ensembles/3/responses/SNAKE_OIL_GPR_DIFF/data", - "axis": {"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}, - "ensemble_id": "1", - "name": "SNAKE_OIL_GPR_DIFF", - "realizations": [], + "http://127.0.0.1:5000/ensembles/2": { + "data": { + "ensemble": { + "experiment": {"id": "exp1_id"}, + "children": [], + "parent": {"ensembleReference": {"id": 1}}, + "id": 2, + "timeCreated": "2020-04-29T10:36:26", + "size": 1, + "Metadata": '{"name": "default_smoother_update"}', + } + } }, - "http://127.0.0.1:5000/ensembles/3/responses/FOPR": { - "alldata_url": "http://127.0.0.1:5000/ensembles/3/responses/FOPR/data", - "axis": {"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}, - "ensemble_id": "3", - "name": "FOPR", - "realizations": [], + "http://127.0.0.1:5000/ensembles/2/responses": { + "SNAKE_OIL_GPR_DIFF": { + "name": "SNAKE_OIL_GPR_DIFF", + "id": "SNAKE_OIL_GPR_DIFF", + }, }, - "http://127.0.0.1:5000/ensembles/3/responses/WOPR:OP1": { - "alldata_url": "http://127.0.0.1:5000/ensembles/3/responses/WOPR:OP1/data", - "axis": {"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}, - "ensemble_id": "3", - "name": "WOPR:OP1", - "realizations": [], - "observations": [ - { - "data": { - "x_axis": {"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}, - "std": {"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}, - "values": {"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}, - "active": { - "data": [ - True, - True, - False, - False, - True, - True, - True, - True, - True, - False, - ] - }, - }, - "name": "WOPR:OP1", + "http://127.0.0.1:5000/ensembles/3": { + "data": { + "ensemble": { + "experiment": {"id": "exp1_id"}, + "children": [], + "name": "default3", + "parent": None, + "id": 3, + "timeCreated": "2020-04-29T10:57:47", + "size": 1, + "Metadata": '{"name": "default3"}', } - ], + } }, - "http://127.0.0.1:5000/ensembles/4/responses/SNAKE_OIL_GPR_DIFF": { - "alldata_url": "http://127.0.0.1:5000/ensembles/4/responses/SNAKE_OIL_GPR_DIFF/data", - "axis": {"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}, - "ensemble_id": "1", - "name": "SNAKE_OIL_GPR_DIFF", - "realizations": [], + "http://127.0.0.1:5000/ensembles/3/responses": { + "SNAKE_OIL_GPR_DIFF": { + "name": "SNAKE_OIL_GPR_DIFF", + "id": "SNAKE_OIL_GPR_DIFF", + }, + "FOPR": { + "name": "FOPR", + "id": "FOPR", + }, + "WOPR:OP1": { + "name": "WOPR:OP1", + "id": "WOPR:OP1", + }, }, - "http://127.0.0.1:5000/ensembles/4/responses/FOPR": { - "alldata_url": "http://127.0.0.1:5000/ensembles/4/responses/FOPR/data", - "axis": {"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}, - "ensemble_id": "4", - "name": "FOPR", - "realizations": [], - "observations": [ - { - "data": { - "x_axis": {"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}, - "std": {"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}, - "values": {"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}, - "active": { - "data": [ - True, - True, - False, - False, - True, - True, - True, - True, - True, - False, - ] - }, - }, - "name": "FOPR", + "http://127.0.0.1:5000/ensembles/4": { + "data": { + "ensemble": { + "experiment": {"id": "exp1_id"}, + "children": [], + "parent": None, + "id": 4, + "timeCreated": "2020-04-29T10:59:37", + "size": 1, + "Metadata": '{"name": "default4"}', } - ], + } + }, + "http://127.0.0.1:5000/ensembles/4/responses": { + "SNAKE_OIL_GPR_DIFF": { + "name": "SNAKE_OIL_GPR_DIFF", + "id": "SNAKE_OIL_GPR_DIFF", + }, + "FOPR": { + "name": "FOPR", + "id": "FOPR", + }, }, - "http://127.0.0.1:5000/ensembles/1/parameters/1/data": [0, 0.1], - "http://127.0.0.1:5000/ensembles/1/parameters/2/data": [0, 0.1], + "http://127.0.0.1:5000/ensembles/1/records/BPR_138_PERSISTENCE": pd.DataFrame( + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + columns=[0], + index=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + ) + .transpose() + .to_csv() + .encode(), + "http://127.0.0.1:5000/ensembles/1/records/OP1_DIVERGENCE_SCALE": pd.DataFrame( + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + columns=[0], + index=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + ) + .transpose() + .to_csv() + .encode(), + "http://127.0.0.1:5000/ensembles/1/records/SNAKE_OIL_GPR_DIFF?realization_index=0": pd.DataFrame( + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + columns=[0], + index=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + ) + .transpose() + .to_csv() + .encode(), + "http://127.0.0.1:5000/ensembles/1/records/SNAKE_OIL_GPR_DIFF/observations?realization_index=0": [], + "http://127.0.0.1:5000/ensembles/3/records/SNAKE_OIL_GPR_DIFF?realization_index=0": pd.DataFrame( + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + columns=[0], + index=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + ) + .transpose() + .to_csv() + .encode(), + "http://127.0.0.1:5000/ensembles/3/records/SNAKE_OIL_GPR_DIFF/observations?realization_index=0": [], + "http://127.0.0.1:5000/ensembles/3/responses/FOPR?realization_index=0": pd.DataFrame( + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + columns=[0], + index=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + ) + .transpose() + .to_csv() + .encode(), + "http://127.0.0.1:5000/ensembles/3/records/FOPR/observations?realization_index=0": [], + "http://127.0.0.1:5000/ensembles/3/records/WOPR:OP1?realization_index=0": pd.DataFrame( + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + columns=[0], + index=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + ) + .transpose() + .to_csv() + .encode(), + "http://127.0.0.1:5000/ensembles/3/records/WOPR:OP1/observations?realization_index=0": [ + { + "x_axis": {"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}, + "errors": {"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}, + "values": {"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}, + "name": "WOPR:OP1", + } + ], + "http://127.0.0.1:5000/ensembles/4/records/SNAKE_OIL_GPR_DIFF?realization_index=0": pd.DataFrame( + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + columns=[0], + index=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + ) + .transpose() + .to_csv() + .encode(), + "http://127.0.0.1:5000/ensembles/4/records/SNAKE_OIL_GPR_DIFF/observations?realization_index=0": [], + "http://127.0.0.1:5000/ensembles/4/responses/FOPR?realization_index=0": pd.DataFrame( + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + columns=[0], + index=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + ) + .transpose() + .to_csv() + .encode(), + "http://127.0.0.1:5000/ensembles/4/records/FOPR/observations?realization_index=0": [ + { + "x_axis": {"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}, + "errors": {"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}, + "values": {"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}, + "name": "FOPR", + } + ], } diff --git a/tests/models/test_ensemble_model.py b/tests/models/test_ensemble_model.py index 577bd1ec..803a3d4e 100644 --- a/tests/models/test_ensemble_model.py +++ b/tests/models/test_ensemble_model.py @@ -1,5 +1,4 @@ from webviz_ert.models import EnsembleModel -from webviz_ert.data_loader import get_url def test_ensemble_model(mock_data): diff --git a/tests/models/test_response_model.py b/tests/models/test_response_model.py index 734b2aff..b470034e 100644 --- a/tests/models/test_response_model.py +++ b/tests/models/test_response_model.py @@ -1,12 +1,13 @@ -from webviz_ert.data_loader import get_ensemble_url from webviz_ert.models import Response -def test_ensemble_model(mock_data): +def test_response_model(mock_data): resp_model = Response( "SNAKE_OIL_GPR_DIFF", ensemble_id=1, response_id="SNAKE_OIL_GPR_DIFF", + project_id="", + ensemble_size=1, ) assert resp_model.name == "SNAKE_OIL_GPR_DIFF" - assert len(resp_model.realizations) == 1 + assert len(resp_model.data.columns) == 1 diff --git a/tests/plots/test_controller.py b/tests/plots/test_controller.py index f0345d49..ed4ce069 100644 --- a/tests/plots/test_controller.py +++ b/tests/plots/test_controller.py @@ -141,7 +141,7 @@ def test_multi_histogram_plot_representation(): name_dict[key] = f"{ensemble_name}" colors_dict[key] = color priors = { - (0, "default"): (PriorModel("UNIFORM", ["STD", "MEAN"], [0, 1]), colors[0]) + (0, "default"): (PriorModel("uniform", ["std", "mean"], [0, 1]), colors[0]) } plot = MultiHistogramPlotModel( @@ -202,16 +202,12 @@ def test_parallel_coordinates_representation(): def test_univariate_misfits_boxplot_representation(): data = np.random.rand(200).reshape(-1, 20) missfits_df = pd.DataFrame(data=data, index=range(10), columns=range(20)) - missfits_df["x_axis"] = np.arange(10) - plots = _get_univariate_misfits_boxplots( - missfits_df.copy(), assets.ERTSTYLE["ensemble-selector"]["color_wheel"][0] - ) - - assert len(plots) == 10 + plots = _get_univariate_misfits_boxplots(missfits_df.copy(), color="rgb(255,0,0)") + assert len(plots) == 20 for id_plot, plot in enumerate(plots): np.testing.assert_equal(0.3, plot.repr.jitter) np.testing.assert_equal("all", plot.repr.boxpoints) - x_pos = int(missfits_df["x_axis"][id_plot]) + x_pos = missfits_df.columns[id_plot] name = f"{x_pos}" assert name == plot.repr.name diff --git a/webviz_ert/controllers/ensemble_selector_controller.py b/webviz_ert/controllers/ensemble_selector_controller.py index 29494092..274ef035 100644 --- a/webviz_ert/controllers/ensemble_selector_controller.py +++ b/webviz_ert/controllers/ensemble_selector_controller.py @@ -1,4 +1,4 @@ -from typing import List, Dict, Mapping, Union, Optional, TYPE_CHECKING +from typing import List, Dict, Mapping, Union, Optional, TYPE_CHECKING, MutableMapping from webviz_ert.plugins._webviz_ert import WebvizErtPluginABC import dash from dash.dependencies import Input, Output, State @@ -11,7 +11,7 @@ from webviz_ert.models import EnsembleModel -def _construct_graph(ensembles: Mapping[int, "EnsembleModel"]) -> List[Dict]: +def _construct_graph(ensembles: MutableMapping[str, "EnsembleModel"]) -> List[Dict]: queue = [ ensemble @@ -62,7 +62,7 @@ def ensemble_selector_controller(parent: WebvizErtPluginABC, app: dash.Dash) -> ], ) def update_ensemble_selector_graph( - selected_ensembles: Optional[Mapping[int, Dict]], + selected_ensembles: Optional[Mapping[str, Dict]], _: int, elements: Optional[List[Dict]], ) -> List[Dict]: diff --git a/webviz_ert/controllers/multi_parameter_controller.py b/webviz_ert/controllers/multi_parameter_controller.py index f042eed0..84f5bd35 100644 --- a/webviz_ert/controllers/multi_parameter_controller.py +++ b/webviz_ert/controllers/multi_parameter_controller.py @@ -54,7 +54,7 @@ def update_histogram( _: Any, __: Any, legend: List[str], - selected_ensembles: Optional[Mapping[int, Dict]], + selected_ensembles: Optional[Mapping[str, Dict]], parameter: str, bin_count: int, ) -> Tuple[go.Figure, int]: @@ -101,7 +101,7 @@ def update_histogram( def set_parameter_from_btn( parameter: str, plotting_options: List[Mapping[str, str]], - selected_ensembles: Optional[Mapping[int, Dict]], + selected_ensembles: Optional[Mapping[str, Dict]], ) -> List[Mapping[str, str]]: if not selected_ensembles: raise PreventUpdate diff --git a/webviz_ert/controllers/multi_response_controller.py b/webviz_ert/controllers/multi_response_controller.py index 4314cc63..04b35022 100644 --- a/webviz_ert/controllers/multi_response_controller.py +++ b/webviz_ert/controllers/multi_response_controller.py @@ -150,13 +150,13 @@ def multi_response_controller(parent: WebvizErtPluginABC, app: dash.Dash) -> Non def update_graph( plot_type: str, _: Any, - selected_ensembles: Optional[Mapping[int, Dict]], + selected_ensembles: Optional[Mapping[str, Dict]], response: Optional[str], ) -> go.Figure: if response in [None, ""] or not selected_ensembles: raise PreventUpdate - def _generate_plot(ensemble_id: int, color: str) -> Optional[ResponsePlotModel]: + def _generate_plot(ensemble_id: str, color: str) -> Optional[ResponsePlotModel]: ensemble = load_ensemble(parent, ensemble_id) if response not in ensemble.responses: return None diff --git a/webviz_ert/controllers/observation_response_controller.py b/webviz_ert/controllers/observation_response_controller.py index 9249388c..45fcb9bf 100644 --- a/webviz_ert/controllers/observation_response_controller.py +++ b/webviz_ert/controllers/observation_response_controller.py @@ -21,13 +21,12 @@ def _get_univariate_misfits_boxplots( ) -> List[BoxPlotModel]: if misfits_df is None: return [] - - x_axis = misfits_df.pop("x_axis") + x_axis = misfits_df.columns misfits_data = list() - for misfits in misfits_df.T: + for misfits in misfits_df: plot = BoxPlotModel( - y_axis=misfits_df.T[misfits].values, - name=f"{x_axis.loc[misfits]}", + y_axis=misfits_df[misfits].values, + name=f"{misfits}", color=color, ) misfits_data.append(plot) @@ -59,7 +58,7 @@ def observation_response_controller(parent: WebvizErtPluginABC, app: dash.Dash) [Input(parent.uuid("ensemble-selection-store"), "data")], ) def set_response_options( - selected_ensembles: Optional[Mapping[int, Dict]] + selected_ensembles: Optional[Mapping[str, Dict]] ) -> List[Dict]: if not selected_ensembles: raise PreventUpdate @@ -100,7 +99,7 @@ def update_graph( response: Optional[str], yaxis_type: List[str], misfits_type: str, - selected_ensembles: Optional[Mapping[int, Dict]], + selected_ensembles: Optional[Mapping[str, Dict]], ) -> go.Figure: if not response or response == "" or not selected_ensembles: raise PreventUpdate @@ -126,7 +125,7 @@ def update_graph( ) return plot.repr - def _generate_plot(ensemble_id: int, color: str) -> ResponsePlotModel: + def _generate_plot(ensemble_id: str, color: str) -> ResponsePlotModel: ensemble = load_ensemble(parent, ensemble_id) plot = _create_misfits_plot(ensemble.responses[response], [], color) return plot diff --git a/webviz_ert/controllers/parameter_comparison_controller.py b/webviz_ert/controllers/parameter_comparison_controller.py index 1664d736..d57d45ce 100644 --- a/webviz_ert/controllers/parameter_comparison_controller.py +++ b/webviz_ert/controllers/parameter_comparison_controller.py @@ -35,7 +35,7 @@ def update_parallel_coor( _: Any, __: Any, selected_parameters: Optional[List[str]], - selected_ensembles: Optional[Mapping[int, Dict]], + selected_ensembles: Optional[Mapping[str, Dict]], ) -> go.Figure: if not selected_ensembles or not selected_parameters: raise PreventUpdate diff --git a/webviz_ert/controllers/parameter_selector_controller.py b/webviz_ert/controllers/parameter_selector_controller.py index 0fe64fd1..d46fbb73 100644 --- a/webviz_ert/controllers/parameter_selector_controller.py +++ b/webviz_ert/controllers/parameter_selector_controller.py @@ -51,7 +51,7 @@ def parameter_selector_controller( [State(parameter_type_store_id, "data")], ) def update_parameters_options( - selected_ensembles: Optional[Mapping[int, Dict]], + selected_ensembles: Optional[Mapping[str, Dict]], filter_search: str, selected: Optional[List[str]], *args: List[str], diff --git a/webviz_ert/controllers/response_correlation_controller.py b/webviz_ert/controllers/response_correlation_controller.py index b7589a52..c7566138 100644 --- a/webviz_ert/controllers/response_correlation_controller.py +++ b/webviz_ert/controllers/response_correlation_controller.py @@ -58,7 +58,7 @@ def update_correlation_plot( correlation_metric: str, parameters: List[str], responses: List[str], - ensembles: Optional[Mapping[int, Dict]], + ensembles: Optional[Mapping[str, Dict]], ) -> Optional[Tuple[go.Figure, go.Figure]]: if not ( ensembles @@ -153,7 +153,7 @@ def update_response_overview_plot( ___: Any, ____: Any, responses: Optional[List[str]], - ensembles: Optional[Mapping[int, Dict]], + ensembles: Optional[Mapping[str, Dict]], corr_xindex: Dict, corr_param_resp: Dict, ) -> Optional[go.Figure]: @@ -167,7 +167,7 @@ def update_response_overview_plot( ensemble = load_ensemble(parent, ensemble_id) response = ensemble.responses[selected_response] x_axis = response.axis - if x_axis: + if x_axis is not None: if str(x_axis[0]).isnumeric(): style = deepcopy(assets.ERTSTYLE["response-plot"]["response-index"]) else: @@ -200,7 +200,7 @@ def update_response_overview_plot( fig.update_layout(_layout) x_index = corr_xindex.get(selected_response, 0) - if x_axis: + if x_axis is not None: fig.add_shape( type="line", x0=x_axis[x_index], @@ -243,7 +243,7 @@ def update_response_parameter_scatterplot( corr_param_resp: Dict, parameters: List[str], responses: List[str], - ensembles: Optional[Mapping[int, Dict]], + ensembles: Optional[Mapping[str, Dict]], ) -> Optional[Tuple[go.Figure, Component]]: if not ( @@ -269,7 +269,7 @@ def update_response_parameter_scatterplot( y_data = ensemble.parameters[selected_parameter].data_df() response = ensemble.responses[selected_response] - if response.axis: + if response.axis is not None: x_index = corr_xindex.get(selected_response, 0) x_data = response.data_df().iloc[x_index] style = deepcopy(assets.ERTSTYLE["response-plot"]["response-index"]) @@ -327,7 +327,7 @@ def update_response_parameter_scatterplot( final_text = [] for response_name in responses: x_axis = ensemble.responses[response_name].axis - if x_axis: + if x_axis is not None: x_value = x_axis[corr_xindex.get(response_name, 0)] if response_name == selected_response: res_text = f"**{response_name} @ {x_value}**, " diff --git a/webviz_ert/data_loader/__init__.py b/webviz_ert/data_loader/__init__.py index 3f7d33bb..78d72463 100644 --- a/webviz_ert/data_loader/__init__.py +++ b/webviz_ert/data_loader/__init__.py @@ -1,12 +1,14 @@ -from typing import Mapping, Any, Union, Mapping, Optional, List import os +import json +from typing import Any, Union, Mapping, Optional, List, MutableMapping, Tuple +from collections import defaultdict +from pprint import pformat import requests import logging -import pandas +import pandas as pd +import io - -def _requests_get(*args: Union[str, bytes], **kwargs: Any) -> requests.models.Response: - return requests.get(*args, **kwargs) +logger = logging.getLogger() def get_info(project_id: str = None) -> Mapping[str, str]: @@ -15,74 +17,228 @@ def get_info(project_id: str = None) -> Mapping[str, str]: return get_info(project_id) -os.environ["NO_PROXY"] = "localhost,127.0.0.1" -data_cache = {} - - -def get_url(project_id: str = None) -> str: - return get_info(project_id)["baseurl"] - - -def get_auth(project_id: str = None) -> str: - return get_info(project_id)["auth"] - - -def get_csv_data(data_url: str, project_id: str = None) -> Optional[pandas.DataFrame]: - response = _requests_get(data_url, auth=get_auth(project_id), stream=True) - response.raise_for_status() - return pandas.read_csv(response.raw, names=["value"]) - - -def get_ensembles(project_id: str = None) -> List[Mapping[str, Any]]: - server_url = get_url(project_id) - data_cache["ensembles"] = get_schema(f"{server_url}/ensembles", project_id)[ - "ensembles" - ] - return data_cache["ensembles"] - - -def get_ensemble_url(ensemble_id: int, project_id: Optional[str] = None) -> str: - from ert_shared.storage.paths import ensemble - - server_url = get_url(project_id) - ensemble_url = ensemble(ensemble_id) - return f"{server_url}{ensemble_url}" - - -def get_response_url( - ensemble_id: int, response_id: int, project_id: Optional[str] = None -) -> str: - from ert_shared.storage.paths import response - - server_url = get_url(project_id) - response_url = response(ensemble_id, response_id) - return f"{server_url}{response_url}" - - -def get_parameter_url( - ensemble_id: int, parameter_id: int, project_id: Optional[str] = None -) -> str: - from ert_shared.storage.paths import parameter - - server_url = get_url(project_id) - parameter_url = parameter(ensemble_id, parameter_id) - return f"{server_url}{parameter_url}" - - -def get_parameter_data_url( - ensemble_id: int, parameter_id: int, project_id: Optional[str] = None -) -> str: - from ert_shared.storage.paths import parameter_data - - server_url = get_url(project_id) - parameter_url = parameter_data(ensemble_id, parameter_id) - return f"{server_url}{parameter_url}" - +# these are needed to mock for testing +def _requests_get(*args: Union[str, bytes], **kwargs: Any) -> requests.models.Response: + return requests.get(*args, **kwargs) -def get_schema(api_url: str, project_id: Optional[str] = None) -> Mapping[str, Any]: - logging.info(f"Getting json from {api_url}...") - http_response = _requests_get(api_url, auth=get_auth(project_id)) - http_response.raise_for_status() - logging.info(" done!") - return http_response.json() +def _requests_post(*args: Union[str, bytes], **kwargs: Any) -> requests.models.Response: + return requests.post(*args, **kwargs) + + +GET_REALIZATION = """\ +query($ensembleId: ID!) { + ensemble(id: $ensembleId) { + name + responses { + name + data_uri + } + parameters { + name + data_uri + } + } +} +""" + +GET_ALL_ENSEMBLES = """\ +query { + experiments { + name + ensembles { + id + timeCreated + parentEnsemble { + id + } + childEnsembles { + id + } + } + } +} +""" + +GET_ENSEMBLE = """\ +query ($id: ID!) { + ensemble(id: $id) { + id + size + timeCreated + children { + ensembleResult{ + id + } + } + Metadata + parent { + ensembleReference{ + id + } + } + experiment { + id + } + } +} +""" + + +data_cache: dict = {} +ServerIdentifier = Tuple[str, Optional[str]] # (baseurl, optional token) + + +class DataLoaderException(Exception): + pass + + +class DataLoader: + _instances: MutableMapping[ServerIdentifier, "DataLoader"] = {} + + baseurl: str + token: str + _graphql_cache: MutableMapping[str, MutableMapping[dict, Any]] + + def __new__(cls, baseurl: str, token: Optional[str] = None) -> "DataLoader": + if (baseurl, token) in cls._instances: + return cls._instances[(baseurl, token)] + + loader = super().__new__(cls) + loader.baseurl = baseurl + loader.token = token + loader._graphql_cache = defaultdict(dict) + cls._instances[(baseurl, token)] = loader + return loader + + def _query(self, query: str, **kwargs: Any) -> dict: + """ + Cachable GraphQL helper + """ + # query_cache = self._graphql_cache[query].get(kwargs) + # if query_cache is not None: + # return query_cache + resp = _requests_post( + f"{self.baseurl}/gql", + json={ + "query": query, + "variables": kwargs, + }, + ) + try: + doc = resp.json() + except json.JSONDecodeError: + doc = resp.content + if resp.status_code != 200 or isinstance(doc, bytes): + raise RuntimeError( + f"ERT Storage query returned with '{resp.status_code}':\n{pformat(doc)}" + ) + + # self._graphql_cache[query][kwargs] = doc + # print(f"--- Query ---\n{query}\n--- Response ---\n{pformat(doc)}\n---------\n") + return doc["data"] + + def _get( + self, url: str, headers: dict = None, params: dict = None + ) -> requests.Response: + resp = _requests_get(f"{self.baseurl}/{url}", headers=headers, params=params) + if resp.status_code != 200: + raise DataLoaderException( + f"""Error fetching data from {self.baseurl}/{url} + The request return with status code: {resp.status_code} + {str(resp.content)} + """ + ) + return resp + + def get_all_ensembles(self) -> list: + experiments = self._query(GET_ALL_ENSEMBLES)["experiments"] + return [ + {"name": exp["name"], **ens} + for exp in experiments + for ens in exp["ensembles"] + ] + + def get_ensemble(self, ensemble_id: str) -> dict: + return self._query(GET_ENSEMBLE, id=ensemble_id)["ensemble"] + + def get_ensemble_responses(self, ensemble_id: str) -> dict: + return self._get(url=f"ensembles/{ensemble_id}/responses").json() + + def get_ensemble_metadata(self, ensemble_id: str) -> dict: + return self._get(url=f"ensembles/{ensemble_id}/metadata").json() + + def get_ensemble_parameters(self, ensemble_id: str) -> list: + return self._get(url=f"ensembles/{ensemble_id}/parameters").json() + + def get_experiment_priors(self, experiment_id: str) -> dict: + return self._get(url=f"experiments/{experiment_id}/priors").json() + + def get_ensemble_parameter_data( + self, ensemble_id: str, parameter_name: str + ) -> pd.DataFrame: + resp = self._get( + url=f"ensembles/{ensemble_id}/records/{parameter_name}", + headers={"accept": "application/x-dataframe"}, + ) + stream = io.BytesIO(resp.content) + df = pd.read_csv(stream, index_col=0, float_precision="round_trip") + return df + + def get_ensemble_record_data( + self, ensemble_id: str, record_name: str, ensemble_size: int + ) -> pd.DataFrame: + dfs = [] + for rel_idx in range(ensemble_size): + try: + resp = self._get( + url=f"ensembles/{ensemble_id}/records/{record_name}", + headers={"accept": "application/x-dataframe"}, + params={"realization_index": rel_idx}, + ) + stream = io.BytesIO(resp.content) + df = pd.read_csv( + stream, index_col=0, float_precision="round_trip" + ).transpose() + df.columns = [rel_idx] + dfs.append(df) + + except DataLoaderException as e: + logger.error(e) + + if dfs == []: + raise DataLoaderException(f"No data found for {record_name}") + + return pd.concat(dfs, axis=1) + + def get_ensemble_record_observations( + self, ensemble_id: str, record_name: str + ) -> List[dict]: + return self._get( + url=f"ensembles/{ensemble_id}/records/{record_name}/observations", + # Hard coded to zero, as all realizations are connected to the same observations + params={"realization_index": 0}, + ).json() + + def compute_misfit( + self, ensemble_id: str, response_name: str, summary: bool + ) -> pd.DataFrame: + resp = self._get( + "compute/misfits", + params={ + "ensemble_id": ensemble_id, + "response_name": response_name, + "summary_misfits": summary, + }, + ) + stream = io.BytesIO(resp.content) + df = pd.read_csv(stream, index_col=0, float_precision="round_trip") + return df + + +def get_data_loader(project_id: Optional[str] = None) -> DataLoader: + return DataLoader(*(get_info(project_id).values())) + + +def get_ensembles(project_id: Optional[str] = None) -> list: + return get_data_loader(project_id).get_all_ensembles() diff --git a/webviz_ert/models/__init__.py b/webviz_ert/models/__init__.py index 595c7e58..d05e6a62 100644 --- a/webviz_ert/models/__init__.py +++ b/webviz_ert/models/__init__.py @@ -7,7 +7,7 @@ def indexes_to_axis( indexes: Optional[List[Union[int, str, datetime.datetime]]] ) -> Optional[List[Union[int, str, datetime.datetime]]]: try: - if indexes and type(indexes[0]) is str: + if indexes and type(indexes[0]) is str and not str(indexes[0]).isnumeric(): return list(map(lambda dt: dateutil.parser.isoparse(str(dt)), indexes)) return indexes except ValueError as e: @@ -20,12 +20,12 @@ def indexes_to_axis( def load_ensemble( - parent_page: "WebvizErtPluginABC", ensemble_id: int + parent_page: "WebvizErtPluginABC", ensemble_id: str ) -> "EnsembleModel": ensemble = parent_page.ensembles.get( ensemble_id, EnsembleModel( - ensemble_id=int(ensemble_id), project_id=parent_page.project_identifier + ensemble_id=ensemble_id, project_id=parent_page.project_identifier ), ) parent_page.ensembles[ensemble_id] = ensemble diff --git a/webviz_ert/models/ensemble_model.py b/webviz_ert/models/ensemble_model.py index ccfac6b1..8fd939f5 100644 --- a/webviz_ert/models/ensemble_model.py +++ b/webviz_ert/models/ensemble_model.py @@ -1,29 +1,33 @@ +import json import pandas as pd from typing import Mapping, List, Dict, Union, Any, Optional -from webviz_ert.data_loader import get_ensemble_url, get_schema +from webviz_ert.data_loader import ( + get_data_loader, +) from webviz_ert.models import Response, PriorModel, ParametersModel -def get_parameter_models( - parameters_schema: Dict, ensemble_id: int, project_id: str +def _create_parameter_models( + parameters_names: list, priors: dict, ensemble_id: str, project_id: str ) -> Optional[Mapping[str, ParametersModel]]: parameters = {} - for param in parameters_schema: - group = param["group"] - key = param["key"] + for param in parameters_names: + key = param + prior_schema = priors.get(key, None) prior = None - if param["prior"]: + if prior_schema: prior = PriorModel( - param["prior"]["function"], - param["prior"]["parameter_names"], - param["prior"]["parameter_values"], + prior_schema["function"], + [x[0] for x in prior_schema.items() if isinstance(x[1], (float, int))], + [x[1] for x in prior_schema.items() if isinstance(x[1], (float, int))], ) + parameters[key] = ParametersModel( - group=group, + group="", # TODO? key=key, prior=prior, - param_id=param["id"], + param_id="", # TODO? project_id=project_id, ensemble_id=ensemble_id, ) @@ -31,21 +35,29 @@ def get_parameter_models( class EnsembleModel: - def __init__(self, ensemble_id: int, project_id: str): - self._schema = get_schema(get_ensemble_url(ensemble_id)) + def __init__(self, ensemble_id: str, project_id: str): + self._data_loader = get_data_loader(project_id) + self._schema = self._data_loader.get_ensemble(ensemble_id) + self._experiment_id = self._schema["experiment"]["id"] self._project_id = project_id - self._name = self._schema["name"] + self._metadata = json.loads(self._schema["Metadata"]) + self._name = self._metadata["name"] self._id = ensemble_id self._children = self._schema["children"] self._parent = self._schema["parent"] - self._time_created = self._schema["time_created"] + self._size = self._schema["size"] + self._time_created = self._schema["timeCreated"] self.responses = { - resp_schema["name"]: Response( - name=resp_schema["name"], + resp_name: Response( + name=resp_name, response_id=resp_schema["id"], ensemble_id=ensemble_id, + project_id=project_id, + ensemble_size=self._size, ) - for resp_schema in self._schema["responses"] + for resp_name, resp_schema in self._data_loader.get_ensemble_responses( + ensemble_id + ).items() } self._parameters: Optional[Mapping[str, ParametersModel]] = None self._cached_children: Optional[List["EnsembleModel"]] = None @@ -55,7 +67,10 @@ def __init__(self, ensemble_id: int, project_id: str): def children(self) -> Optional[List["EnsembleModel"]]: if not self._cached_children: self._cached_children = [ - EnsembleModel(ensemble_id=child["id"], project_id=self._project_id) + EnsembleModel( + ensemble_id=child["ensembleResult"]["id"], + project_id=self._project_id, + ) for child in self._children ] return self._cached_children @@ -66,7 +81,8 @@ def parent(self) -> Optional["EnsembleModel"]: return None if not self._cached_parent: self._cached_parent = EnsembleModel( - ensemble_id=self._parent["id"], project_id=self._project_id + ensemble_id=self._parent["ensembleReference"]["id"], + project_id=self._project_id, ) return self._cached_parent @@ -75,8 +91,15 @@ def parameters( self, ) -> Optional[Mapping[str, ParametersModel]]: if not self._parameters: - self._parameters = get_parameter_models( - self._schema["parameters"], + parameter_names = self._data_loader.get_ensemble_parameters(self._id) + parameter_priors = ( + self._data_loader.get_experiment_priors(self._experiment_id) + if not self._parent + else {} + ) + self._parameters = _create_parameter_models( + parameter_names, + parameter_priors, ensemble_id=self._id, project_id=self._project_id, ) @@ -92,7 +115,7 @@ def parameters_df(self, parameter_list: Optional[List[str]] = None) -> pd.DataFr return pd.DataFrame(data=data) @property - def id(self) -> int: + def id(self) -> str: return self._id def __str__(self) -> str: diff --git a/webviz_ert/models/observation.py b/webviz_ert/models/observation.py index cc0bb296..1c247851 100644 --- a/webviz_ert/models/observation.py +++ b/webviz_ert/models/observation.py @@ -6,18 +6,11 @@ class Observation: def __init__(self, observation_schema: Dict): self.name = str(observation_schema["name"]) - self._x_axis = [] - self._std = [] - self._values = [] + self._x_axis = observation_schema["x_axis"] + self._std = observation_schema["errors"] + self._values = observation_schema["values"] self._attributes = "" - self._active = [] - - if "data" in observation_schema: - data = observation_schema["data"] - self._x_axis = data["x_axis"]["data"] - self._std = data["std"]["data"] - self._values = data["values"]["data"] - self._active = data["active"]["data"] + self._active = [True for _ in self._x_axis] if "attributes" in observation_schema: for k, v in observation_schema["attributes"].items(): diff --git a/webviz_ert/models/parameter_model.py b/webviz_ert/models/parameter_model.py index 6a0a8bd1..a7d7c8b1 100644 --- a/webviz_ert/models/parameter_model.py +++ b/webviz_ert/models/parameter_model.py @@ -1,9 +1,6 @@ import pandas as pd -from typing import List, Any, Optional -from webviz_ert.data_loader import ( - get_csv_data, - get_parameter_data_url, -) +from typing import List, Any, Optional, Union +from webviz_ert.data_loader import get_data_loader class PriorModel: @@ -11,9 +8,8 @@ def __init__( self, function: str, function_parameter_names: List[str], - function_parameter_values: List[str], + function_parameter_values: List[Union[float, int]], ): - self.function = function self.function_parameter_names = function_parameter_names self.function_parameter_values = function_parameter_values @@ -29,14 +25,12 @@ def __init__(self, **kwargs: Any): self._ensemble_id = kwargs["ensemble_id"] self._realizations = kwargs.get("realizations") self._data_df = pd.DataFrame() + self._data_loader = get_data_loader(self._project_id) def data_df(self) -> pd.DataFrame: if self._data_df.empty: - _data_df = get_csv_data( - get_parameter_data_url( - ensemble_id=self._ensemble_id, parameter_id=self._id - ), - project_id=self._project_id, + _data_df = self._data_loader.get_ensemble_parameter_data( + ensemble_id=self._ensemble_id, parameter_name=self.key ) if _data_df is not None: _data_df = _data_df.transpose() diff --git a/webviz_ert/models/plot_model.py b/webviz_ert/models/plot_model.py index debb4a2c..eb6727ec 100644 --- a/webviz_ert/models/plot_model.py +++ b/webviz_ert/models/plot_model.py @@ -75,17 +75,17 @@ def _DERRF(*args: Any) -> List: PRIOR_FUNCTIONS = { - "NORMAL": norm.pdf, - "LOGNORMAL": lognorm.pdf, - "TRUNCATED_NORMAL": _TRUNC_NORMAL, - "UNIFORM": _UNIFORM, + "normal": norm.pdf, + "lognormal": lognorm.pdf, + "ert_truncnormal": _TRUNC_NORMAL, + "uniform": _UNIFORM, "DUNIF": _DUNIFORM, - "LOGUNIF": loguniform.pdf, - "TRIANGULAR": _TRIANGULAR, - "CONST": _CONST, - "RAW": _RAW, - "ERRF": _ERRF, - "DERRF": _DERRF, + "loguniform": loguniform.pdf, + "trig": _TRIANGULAR, + "const": _CONST, + "stdnormal": _RAW, + "ert_ert": _ERRF, + "ert_derf": _DERRF, } diff --git a/webviz_ert/models/response.py b/webviz_ert/models/response.py index 41fdb4c9..3f9fd6f7 100644 --- a/webviz_ert/models/response.py +++ b/webviz_ert/models/response.py @@ -1,147 +1,80 @@ from typing import List, Mapping, Optional, Any, Union, Dict import datetime import pandas as pd -from webviz_ert.data_loader import get_schema, get_response_url +from webviz_ert.data_loader import get_data_loader, DataLoader from webviz_ert.models import Realization, Observation, indexes_to_axis class Response: - def __init__(self, name: str, response_id: int, ensemble_id: int): - self._schema: Optional[Mapping[str, Any]] = None # get_schema(api_url=ref_url) - self._id = response_id - self._ensemble_id = ensemble_id - self.name = name - self._axis: Optional[List[Union[int, str, datetime.datetime]]] = None - self._data = None - self._realizations: Optional[List[Realization]] = None + def __init__( + self, + name: str, + response_id: str, + ensemble_id: str, + project_id: str, + ensemble_size: int, + ): + self._data_loader: DataLoader = get_data_loader(project_id) + self._document: Optional[Mapping[str, Any]] = None + self._id: str = response_id + self._ensemble_id: str = ensemble_id + self.name: str = name + self._data: Optional[pd.DataFrame] = None self._observations: Optional[List[Observation]] = None - - def _update_schema(self) -> None: - if not self._schema: - self._schema = get_schema( - api_url=get_response_url( - ensemble_id=self._ensemble_id, response_id=self._id - ) - ) + self._univariate_misfits_df: Optional[pd.DataFrame] = None + self._summary_misfits_df: Optional[pd.DataFrame] = None + self._ensemble_size: int = ensemble_size @property - def ensemble_id(self) -> int: + def ensemble_id(self) -> str: return self._ensemble_id @property def axis(self) -> Optional[List[Union[int, str, datetime.datetime]]]: - if self._axis is None: - self._update_schema() - if self._schema: - indexes = self._schema["axis"]["data"] - self._axis = indexes_to_axis(indexes) - return self._axis + return self.data.index @property def data(self) -> pd.DataFrame: - self._update_schema() - if self._schema: - self._data_url = self._schema["alldata_url"] - if self._data is None and self._realizations is not None: - self._data = pd.read_csv(self._data_url, header=None).T - if self._data: - self._data.columns = [ - realization.name for realization in self._realizations - ] + if self._data is None: + self._data = self._data_loader.get_ensemble_record_data( + self._ensemble_id, self.name, self._ensemble_size + ) return self._data def univariate_misfits_df( self, selection: Optional[List[int]] = None ) -> pd.DataFrame: - if not self.realizations: - return None + if self._univariate_misfits_df is None: + self._univariate_misfits_df = self._data_loader.compute_misfit( + self._ensemble_id, self.name, summary=False + ) if selection: - data = { - realization.name: realization.univariate_misfits_df["value_sign"] - for realization in self.realizations - if realization.name in selection - and realization.univariate_misfits_df is not None - } - else: - data = { - realization.name: realization.univariate_misfits_df["value_sign"] - for realization in self.realizations - if realization.univariate_misfits_df is not None - } - if data: - misfits_df = pd.DataFrame(data=data) - misfits_df["x_axis"] = self.realizations[0].univariate_misfits_df[ - "obs_location" - ] - if misfits_df["x_axis"].dtype == "object": - misfits_df["x_axis"] = pd.to_datetime( - misfits_df["x_axis"], infer_datetime_format=True - ) - misfits_df.index.name = self.name - return misfits_df - return None + return self._univariate_misfits_df.iloc[selection, :] + return self._univariate_misfits_df def summary_misfits_df(self, selection: Optional[List[int]] = None) -> pd.DataFrame: - if not self.realizations: - return None + if self._summary_misfits_df is None: + self._summary_misfits_df = self._data_loader.compute_misfit( + self._ensemble_id, self.name, summary=True + ) if selection: - data = { - realization.name: [realization.summarized_misfits_value] - for realization in self.realizations - if realization.name in selection - and bool(realization.summarized_misfits_value) - } - else: - data = { - realization.name: [realization.summarized_misfits_value] - for realization in self.realizations - if bool(realization.summarized_misfits_value) - } - if bool(data): - misfits_df = pd.DataFrame(data=data) - misfits_df.index.name = self.name - return misfits_df.astype("float64") - return None + self._summary_misfits_df.iloc[selection, :] + return self._summary_misfits_df def data_df(self, selection: Optional[List[int]] = None) -> pd.DataFrame: - if self.realizations: - if selection: - data = { - realization.name: realization.data - for realization in self.realizations - if realization.name in selection - } - else: - data = { - realization.name: realization.data - for realization in self.realizations - } - return pd.DataFrame(data=data).astype("float64") - - @property - def realizations(self) -> Optional[List[Realization]]: - self._update_schema() - if self._schema: - if "realizations" in self._schema: - self._realizations_schema = self._schema["realizations"] - - if self._realizations is None: - self._realizations = [ - Realization(realization_schema=realization_schema) - for realization_schema in self._realizations_schema - ] - return self._realizations + if selection: + self.data.iloc[selection, :] + return self.data @property def observations(self) -> Optional[List[Observation]]: - self._update_schema() - if not self._schema or "observations" not in self._schema: - return [] - _observations_schema = self._schema["observations"] - if self._observations is None and _observations_schema is not None: + if self._observations is None: + _observations_schemas = self._data_loader.get_ensemble_record_observations( + self._ensemble_id, self.name + ) self._observations = [] - for observation_schema in _observations_schema: + for observation_schema in _observations_schemas: self._observations.append( Observation(observation_schema=observation_schema) ) diff --git a/webviz_ert/plugins/_webviz_ert.py b/webviz_ert/plugins/_webviz_ert.py index f9dea581..2afbb8dc 100644 --- a/webviz_ert/plugins/_webviz_ert.py +++ b/webviz_ert/plugins/_webviz_ert.py @@ -10,5 +10,5 @@ class WebvizErtPluginABC(WebvizPluginABC): def __init__(self, app: dash.Dash, project_identifier: str): super().__init__() self.project_identifier = project_identifier - self.ensembles: MutableMapping[int, "EnsembleModel"] = {} + self.ensembles: MutableMapping[str, "EnsembleModel"] = {} self.parameter_models: MutableMapping[str, "ParametersModel"] = {}