From 45f846fcd872b7d7458f94dd627cea0503ee330b Mon Sep 17 00:00:00 2001 From: Hans Kallekleiv <16436291+HansKallekleiv@users.noreply.github.com> Date: Thu, 14 Sep 2023 12:25:15 +0200 Subject: [PATCH 1/8] set up test folders --- backend/tests/unit/__init__.py | 0 backend/tests/unit/services/__init__.py | 0 backend/tests/unit/services/sumo_access/__init__.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 backend/tests/unit/__init__.py create mode 100644 backend/tests/unit/services/__init__.py create mode 100644 backend/tests/unit/services/sumo_access/__init__.py diff --git a/backend/tests/unit/__init__.py b/backend/tests/unit/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/tests/unit/services/__init__.py b/backend/tests/unit/services/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/tests/unit/services/sumo_access/__init__.py b/backend/tests/unit/services/sumo_access/__init__.py new file mode 100644 index 000000000..e69de29bb From 142a53f47f837e40677700d0f2c36958e763ae77 Mon Sep 17 00:00:00 2001 From: Hans Kallekleiv <16436291+HansKallekleiv@users.noreply.github.com> Date: Thu, 14 Sep 2023 20:51:39 +0200 Subject: [PATCH 2/8] set up some simple tests --- .../services/sumo_access/parameter_access.py | 86 +------------ backend/src/services/utils/parameter_utils.py | 92 ++++++++++++++ .../tests/integration/services/conftest.py | 18 +++ .../sumo_access/test_parameter_access.py | 19 +++ .../{sumo_access => utils}/__init__.py | 0 .../services/utils/test_parameter_utils.py | 120 ++++++++++++++++++ 6 files changed, 250 insertions(+), 85 deletions(-) create mode 100644 backend/src/services/utils/parameter_utils.py create mode 100644 backend/tests/integration/services/conftest.py create mode 100644 backend/tests/integration/services/sumo_access/test_parameter_access.py rename backend/tests/unit/services/{sumo_access => utils}/__init__.py (100%) create mode 100644 backend/tests/unit/services/utils/test_parameter_utils.py diff --git a/backend/src/services/sumo_access/parameter_access.py b/backend/src/services/sumo_access/parameter_access.py index 058c40d1c..b48adc63f 100644 --- a/backend/src/services/sumo_access/parameter_access.py +++ b/backend/src/services/sumo_access/parameter_access.py @@ -1,8 +1,6 @@ import logging from typing import List, Optional -import numpy as np -import pandas as pd from fmu.sumo.explorer.explorer import CaseCollection, SumoClient from .queries.parameters import get_parameters_for_iteration, SumoEnsembleParameter @@ -10,10 +8,8 @@ from .parameter_types import ( EnsembleParameter, EnsembleParameters, - EnsembleSensitivity, - EnsembleSensitivityCase, - SensitivityType, ) +from ..utils.parameter_utils import create_ensemble_sensitivities, create_ensemble_parameter LOGGER = logging.getLogger(__name__) @@ -47,83 +43,3 @@ def get_parameter(self, parameter_name: str) -> EnsembleParameter: """Retrieve a single parameter for an ensemble""" parameters = self.get_parameters_and_sensitivities() return next(parameter for parameter in parameters.parameters if parameter.name == parameter_name) - - -def create_ensemble_parameter( - sumo_parameter: SumoEnsembleParameter, -) -> EnsembleParameter: - """Create an EnsembleParameter from a Sumo parameter object""" - - return EnsembleParameter( - name=sumo_parameter.name, - is_logarithmic=sumo_parameter.name.startswith("LOG10_"), - is_numerical=is_array_numeric(np.array(sumo_parameter.values)), - is_constant=all(value == sumo_parameter.values[0] for value in sumo_parameter.values), - group_name=sumo_parameter.groupname, - descriptive_name=f"{sumo_parameter.name} (log)" - if sumo_parameter.name.startswith("LOG10_") - else sumo_parameter.name, - values=sumo_parameter.values, - realizations=sumo_parameter.realizations, - ) - - -def create_ensemble_sensitivities( - sumo_ensemble_parameters: List[SumoEnsembleParameter], -) -> Optional[List[EnsembleSensitivity]]: - """Extract sensitivities from a list of SumoEnsembleParameter objects""" - sensitivities = [] - - sens_name_parameter = next( - (parameter for parameter in sumo_ensemble_parameters if parameter.name == "SENSNAME"), - None, - ) - sens_case_parameter = next( - (parameter for parameter in sumo_ensemble_parameters if parameter.name == "SENSCASE"), - None, - ) - if sens_case_parameter is None or sens_name_parameter is None: - return None - df = pd.DataFrame( - { - "name": sens_name_parameter.values, - "case": sens_case_parameter.values, - "REAL": sens_case_parameter.realizations, - } - ) - for name, group in df.groupby("name"): - sensitivities.append( - EnsembleSensitivity( - name=name, - type=find_sensitivity_type(list(group["case"].unique())), - cases=create_ensemble_sensitivity_cases(group), - ) - ) - return sensitivities if sensitivities else None - - -def find_sensitivity_type(sens_case_names: List[str]) -> SensitivityType: - """Find the sensitivity type based on the sensitivity case names""" - if len(sens_case_names) == 1 and sens_case_names[0] == "p10_p90": - return SensitivityType.MONTECARLO - return SensitivityType.SCENARIO - - -def create_ensemble_sensitivity_cases( - df: pd.DataFrame, -) -> List[EnsembleSensitivityCase]: - """Create a list of EnsembleSensitivityCase objects from a dataframe""" - cases = [] - for case_name, case_df in df.groupby("case"): - cases.append( - EnsembleSensitivityCase( - name=case_name, - realizations=case_df["REAL"].unique().tolist(), - ) - ) - return cases - - -def is_array_numeric(array: np.ndarray) -> bool: - """Check if an array is numeric""" - return np.issubdtype(array.dtype, np.number) diff --git a/backend/src/services/utils/parameter_utils.py b/backend/src/services/utils/parameter_utils.py new file mode 100644 index 000000000..29879ff27 --- /dev/null +++ b/backend/src/services/utils/parameter_utils.py @@ -0,0 +1,92 @@ +from typing import List, Optional + +import numpy as np +import pandas as pd + +from ..sumo_access.queries.parameters import SumoEnsembleParameter +from ..sumo_access.parameter_types import ( + EnsembleParameter, + EnsembleSensitivity, + EnsembleSensitivityCase, + SensitivityType, +) + + +def is_array_numeric(array: np.ndarray) -> bool: + """Check if an array is numeric""" + return np.issubdtype(array.dtype, np.number) + + +def create_ensemble_parameter( + sumo_parameter: SumoEnsembleParameter, +) -> EnsembleParameter: + """Create an EnsembleParameter from a Sumo parameter object""" + + return EnsembleParameter( + name=sumo_parameter.name, + is_logarithmic=sumo_parameter.name.startswith("LOG10_"), + is_numerical=is_array_numeric(np.array(sumo_parameter.values)), + is_constant=all(value == sumo_parameter.values[0] for value in sumo_parameter.values), + group_name=sumo_parameter.groupname, + descriptive_name=f"{sumo_parameter.name} (log)" + if sumo_parameter.name.startswith("LOG10_") + else sumo_parameter.name, + values=sumo_parameter.values, + realizations=sumo_parameter.realizations, + ) + + +def create_ensemble_sensitivities( + sumo_ensemble_parameters: List[SumoEnsembleParameter], +) -> Optional[List[EnsembleSensitivity]]: + """Extract sensitivities from a list of SumoEnsembleParameter objects""" + sensitivities = [] + + sens_name_parameter = next( + (parameter for parameter in sumo_ensemble_parameters if parameter.name == "SENSNAME"), + None, + ) + sens_case_parameter = next( + (parameter for parameter in sumo_ensemble_parameters if parameter.name == "SENSCASE"), + None, + ) + if sens_case_parameter is None or sens_name_parameter is None: + return None + df = pd.DataFrame( + { + "name": sens_name_parameter.values, + "case": sens_case_parameter.values, + "REAL": sens_case_parameter.realizations, + } + ) + for name, group in df.groupby("name"): + sensitivities.append( + EnsembleSensitivity( + name=name, + type=find_sensitivity_type(list(group["case"].unique())), + cases=create_ensemble_sensitivity_cases(group), + ) + ) + return sensitivities if sensitivities else None + + +def find_sensitivity_type(sens_case_names: List[str]) -> SensitivityType: + """Find the sensitivity type based on the sensitivity case names""" + if len(sens_case_names) == 1 and sens_case_names[0] == "p10_p90": + return SensitivityType.MONTECARLO + return SensitivityType.SCENARIO + + +def create_ensemble_sensitivity_cases( + df: pd.DataFrame, +) -> List[EnsembleSensitivityCase]: + """Create a list of EnsembleSensitivityCase objects from a dataframe""" + cases = [] + for case_name, case_df in df.groupby("case"): + cases.append( + EnsembleSensitivityCase( + name=case_name, + realizations=case_df["REAL"].unique().tolist(), + ) + ) + return cases diff --git a/backend/tests/integration/services/conftest.py b/backend/tests/integration/services/conftest.py new file mode 100644 index 000000000..f29f4599c --- /dev/null +++ b/backend/tests/integration/services/conftest.py @@ -0,0 +1,18 @@ +import os + +import pytest +from sumo.wrapper import SumoClient + + +@pytest.fixture(name="sumo_token") +def fixture_sumo_token() -> str: + token = os.getenv("SUMO_TOKEN") + if token is None: + client = SumoClient(env="dev") + token = client.authenticate() + return token + + +@pytest.fixture(name="sumo_case_uuid") +def fixture_sumo_case_uuid() -> str: + return "10f41041-2c17-4374-a735-bb0de62e29dc" diff --git a/backend/tests/integration/services/sumo_access/test_parameter_access.py b/backend/tests/integration/services/sumo_access/test_parameter_access.py new file mode 100644 index 000000000..03cc587ea --- /dev/null +++ b/backend/tests/integration/services/sumo_access/test_parameter_access.py @@ -0,0 +1,19 @@ +import pytest + +import services.sumo_access.parameter_access as parameter_access + + +@pytest.fixture(name="parameter_access_instance") +def get_parameter_access_instance(sumo_token, sumo_case_uuid) -> str: + return parameter_access.ParameterAccess(sumo_token, sumo_case_uuid, "iter-0") + + +def test_init(parameter_access_instance, sumo_case_uuid): + assert len(parameter_access_instance.case_collection) == 1 + assert parameter_access_instance.case_collection[0].uuid == sumo_case_uuid + + +def test_get_parameters_and_sensitivities(parameter_access_instance): + parameters_and_sensitivities = parameter_access_instance.get_parameters_and_sensitivities() + assert len(parameters_and_sensitivities.parameters) == 113 + assert len(parameters_and_sensitivities.sensitivities) == 1 diff --git a/backend/tests/unit/services/sumo_access/__init__.py b/backend/tests/unit/services/utils/__init__.py similarity index 100% rename from backend/tests/unit/services/sumo_access/__init__.py rename to backend/tests/unit/services/utils/__init__.py diff --git a/backend/tests/unit/services/utils/test_parameter_utils.py b/backend/tests/unit/services/utils/test_parameter_utils.py new file mode 100644 index 000000000..88c1007b0 --- /dev/null +++ b/backend/tests/unit/services/utils/test_parameter_utils.py @@ -0,0 +1,120 @@ +import pytest + +import pandas as pd +from services.utils.parameter_utils import ( + create_ensemble_parameter, + create_ensemble_sensitivities, + find_sensitivity_type, + create_ensemble_sensitivity_cases, + EnsembleParameter, + EnsembleSensitivity, + EnsembleSensitivityCase, + SensitivityType, +) + + +from services.sumo_access.queries.parameters import SumoEnsembleParameter + + +@pytest.mark.parametrize( + "sumo_param_input, expected_output", + [ + ( + SumoEnsembleParameter( + name="LOG10_PARAM", groupname="group1", values=[1.0, 1.0, 1.0], realizations=[0, 1, 2] + ), + EnsembleParameter( + name="LOG10_PARAM", + is_logarithmic=True, + is_numerical=True, + is_constant=True, + group_name="group1", + descriptive_name="LOG10_PARAM (log)", + values=[1.0, 1.0, 1.0], + realizations=[0, 1, 2], + ), + ), + ( + SumoEnsembleParameter(name="PARAM", values=[1.0, 2.0, 3.0], realizations=[0, 1, 2]), + EnsembleParameter( + name="PARAM", + is_logarithmic=False, + is_numerical=True, + is_constant=False, + group_name=None, + descriptive_name="PARAM", + values=[1.0, 2.0, 3.0], + realizations=[0, 1, 2], + ), + ), + ], +) +def test_create_ensemble_parameter(sumo_param_input, expected_output): + ensemble_param = create_ensemble_parameter(sumo_param_input) + assert ensemble_param == expected_output + + +@pytest.mark.parametrize( + "sumo_param_input, expected_output", + [ + ( + [ + SumoEnsembleParameter( + name="SENSNAME", groupname=None, values=["SENS_A", "SENS_B", "SENS_B"], realizations=[0, 1, 2] + ), + SumoEnsembleParameter( + name="SENSCASE", groupname=None, values=["p10_p90", "low", "high"], realizations=[0, 1, 2] + ), + ], + [ + EnsembleSensitivity( + name="SENS_A", + type=SensitivityType.MONTECARLO, + cases=[EnsembleSensitivityCase(name="p10_p90", realizations=[0])], + ), + EnsembleSensitivity( + name="SENS_B", + type=SensitivityType.SCENARIO, + cases=[ + EnsembleSensitivityCase(name="high", realizations=[2]), + EnsembleSensitivityCase(name="low", realizations=[1]), + ], + ), + ], + ), + ], +) +def test_create_ensemble_sensitivities(sumo_param_input, expected_output): + sensitivities = create_ensemble_sensitivities(sumo_param_input) + + assert sensitivities == expected_output + + +@pytest.mark.parametrize( + "sens_case_names, expected_output", + [ + (["p10_p90"], SensitivityType.MONTECARLO), + (["case1", "case2"], SensitivityType.SCENARIO), + ], +) +def test_find_sensitivity_type(sens_case_names, expected_output): + assert find_sensitivity_type(sens_case_names) == expected_output + + +@pytest.mark.parametrize( + "sens_case_input, expected_output", + [ + ( + {"case": ["case1", "case1", "case2", "case2"], "REAL": [0, 1, 2, 3]}, + [ + EnsembleSensitivityCase(name="case1", realizations=[0, 1]), + EnsembleSensitivityCase(name="case2", realizations=[2, 3]), + ], + ), + ], +) +def test_create_ensemble_sensitivity_cases(sens_case_input, expected_output): + df = pd.DataFrame(sens_case_input) + cases = create_ensemble_sensitivity_cases(df) + + assert cases == expected_output From 0ed89b7f24b88ee13e92eb6579bd335e9a8722b2 Mon Sep 17 00:00:00 2001 From: Hans Kallekleiv <16436291+HansKallekleiv@users.noreply.github.com> Date: Fri, 15 Sep 2023 13:43:17 +0200 Subject: [PATCH 3/8] lint --- backend/src/services/sumo_access/parameter_access.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/services/sumo_access/parameter_access.py b/backend/src/services/sumo_access/parameter_access.py index b48adc63f..4ffa6ce43 100644 --- a/backend/src/services/sumo_access/parameter_access.py +++ b/backend/src/services/sumo_access/parameter_access.py @@ -1,5 +1,5 @@ import logging -from typing import List, Optional +from typing import List from fmu.sumo.explorer.explorer import CaseCollection, SumoClient From 9949fddacdab938d6040f3259902c3aa07e449be Mon Sep 17 00:00:00 2001 From: Hans Kallekleiv <16436291+HansKallekleiv@users.noreply.github.com> Date: Fri, 15 Sep 2023 14:47:46 +0200 Subject: [PATCH 4/8] mypy --- .../services/sumo_access/test_parameter_access.py | 10 +++++----- .../unit/services/utils/test_parameter_utils.py | 13 +++++++++---- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/backend/tests/integration/services/sumo_access/test_parameter_access.py b/backend/tests/integration/services/sumo_access/test_parameter_access.py index 03cc587ea..49802e0e2 100644 --- a/backend/tests/integration/services/sumo_access/test_parameter_access.py +++ b/backend/tests/integration/services/sumo_access/test_parameter_access.py @@ -1,19 +1,19 @@ import pytest -import services.sumo_access.parameter_access as parameter_access +from services.sumo_access.parameter_access import ParameterAccess @pytest.fixture(name="parameter_access_instance") -def get_parameter_access_instance(sumo_token, sumo_case_uuid) -> str: - return parameter_access.ParameterAccess(sumo_token, sumo_case_uuid, "iter-0") +def get_parameter_access_instance(sumo_token: str, sumo_case_uuid: str) -> ParameterAccess: + return ParameterAccess(sumo_token, sumo_case_uuid, "iter-0") -def test_init(parameter_access_instance, sumo_case_uuid): +def test_init(parameter_access_instance: ParameterAccess, sumo_case_uuid: str) -> None: assert len(parameter_access_instance.case_collection) == 1 assert parameter_access_instance.case_collection[0].uuid == sumo_case_uuid -def test_get_parameters_and_sensitivities(parameter_access_instance): +def test_get_parameters_and_sensitivities(parameter_access_instance: ParameterAccess) -> None: parameters_and_sensitivities = parameter_access_instance.get_parameters_and_sensitivities() assert len(parameters_and_sensitivities.parameters) == 113 assert len(parameters_and_sensitivities.sensitivities) == 1 diff --git a/backend/tests/unit/services/utils/test_parameter_utils.py b/backend/tests/unit/services/utils/test_parameter_utils.py index 88c1007b0..23c15d15c 100644 --- a/backend/tests/unit/services/utils/test_parameter_utils.py +++ b/backend/tests/unit/services/utils/test_parameter_utils.py @@ -1,3 +1,4 @@ +from typing import List, Dict, Union import pytest import pandas as pd @@ -49,7 +50,7 @@ ), ], ) -def test_create_ensemble_parameter(sumo_param_input, expected_output): +def test_create_ensemble_parameter(sumo_param_input: SumoEnsembleParameter, expected_output: EnsembleParameter) -> None: ensemble_param = create_ensemble_parameter(sumo_param_input) assert ensemble_param == expected_output @@ -84,7 +85,9 @@ def test_create_ensemble_parameter(sumo_param_input, expected_output): ), ], ) -def test_create_ensemble_sensitivities(sumo_param_input, expected_output): +def test_create_ensemble_sensitivities( + sumo_param_input: SumoEnsembleParameter, expected_output: EnsembleParameter +) -> None: sensitivities = create_ensemble_sensitivities(sumo_param_input) assert sensitivities == expected_output @@ -97,7 +100,7 @@ def test_create_ensemble_sensitivities(sumo_param_input, expected_output): (["case1", "case2"], SensitivityType.SCENARIO), ], ) -def test_find_sensitivity_type(sens_case_names, expected_output): +def test_find_sensitivity_type(sens_case_names: List[str], expected_output: SensitivityType) -> None: assert find_sensitivity_type(sens_case_names) == expected_output @@ -113,7 +116,9 @@ def test_find_sensitivity_type(sens_case_names, expected_output): ), ], ) -def test_create_ensemble_sensitivity_cases(sens_case_input, expected_output): +def test_create_ensemble_sensitivity_cases( + sens_case_input: Dict[str, list[Union[str, float]]], expected_output: List[EnsembleSensitivityCase] +) -> None: df = pd.DataFrame(sens_case_input) cases = create_ensemble_sensitivity_cases(df) From a25b484801ba1ef69544854e319a1a8af1eb3cef Mon Sep 17 00:00:00 2001 From: Hans Kallekleiv <16436291+HansKallekleiv@users.noreply.github.com> Date: Fri, 15 Sep 2023 14:51:00 +0200 Subject: [PATCH 5/8] Add unit tests to github action --- .github/workflows/webviz.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/webviz.yml b/.github/workflows/webviz.yml index 6192c2853..728685514 100644 --- a/.github/workflows/webviz.yml +++ b/.github/workflows/webviz.yml @@ -93,10 +93,10 @@ jobs: bandit --recursive src/ mypy src/ tests/ - # - name: 🤖 Run tests - # working-directory: ./backend - # run: | - # pytest ./tests + - name: 🤖 Run tests + working-directory: ./backend + run: | + pytest ./tests/unit build_docker_images: runs-on: ubuntu-latest From 2903a22db0c31b41ea043ce8ac208f0ed87dd91a Mon Sep 17 00:00:00 2001 From: Hans Kallekleiv <16436291+HansKallekleiv@users.noreply.github.com> Date: Mon, 18 Sep 2023 11:22:11 +0200 Subject: [PATCH 6/8] move helpers to access layher --- .../services/sumo_access/parameter_access.py | 94 ++++++++++++++++++- .../sumo_access/queries/parameters.py | 2 +- .../test_parameter_access_helpers.py} | 5 +- backend/tests/unit/services/utils/__init__.py | 0 4 files changed, 94 insertions(+), 7 deletions(-) rename backend/tests/unit/services/{utils/test_parameter_utils.py => sumo_access/test_parameter_access_helpers.py} (97%) delete mode 100644 backend/tests/unit/services/utils/__init__.py diff --git a/backend/src/services/sumo_access/parameter_access.py b/backend/src/services/sumo_access/parameter_access.py index 4ffa6ce43..b1338b0ef 100644 --- a/backend/src/services/sumo_access/parameter_access.py +++ b/backend/src/services/sumo_access/parameter_access.py @@ -1,15 +1,21 @@ import logging -from typing import List +from typing import List, Optional +import numpy as np +import pandas as pd from fmu.sumo.explorer.explorer import CaseCollection, SumoClient -from .queries.parameters import get_parameters_for_iteration, SumoEnsembleParameter -from ._helpers import create_sumo_client_instance +from .queries.parameters import SumoEnsembleParameter, get_parameters_for_iteration from .parameter_types import ( EnsembleParameter, EnsembleParameters, + EnsembleSensitivity, + EnsembleSensitivityCase, + SensitivityType, ) -from ..utils.parameter_utils import create_ensemble_sensitivities, create_ensemble_parameter + + +from ._helpers import create_sumo_client_instance LOGGER = logging.getLogger(__name__) @@ -43,3 +49,83 @@ def get_parameter(self, parameter_name: str) -> EnsembleParameter: """Retrieve a single parameter for an ensemble""" parameters = self.get_parameters_and_sensitivities() return next(parameter for parameter in parameters.parameters if parameter.name == parameter_name) + + +def is_array_numeric(array: np.ndarray) -> bool: + """Check if an array is numeric""" + return np.issubdtype(array.dtype, np.number) + + +def create_ensemble_parameter( + sumo_parameter: SumoEnsembleParameter, +) -> EnsembleParameter: + """Create an EnsembleParameter from a Sumo parameter object""" + + return EnsembleParameter( + name=sumo_parameter.name, + is_logarithmic=sumo_parameter.name.startswith("LOG10_"), + is_numerical=is_array_numeric(np.array(sumo_parameter.values)), + is_constant=all(value == sumo_parameter.values[0] for value in sumo_parameter.values), + group_name=sumo_parameter.groupname, + descriptive_name=f"{sumo_parameter.name} (log)" + if sumo_parameter.name.startswith("LOG10_") + else sumo_parameter.name, + values=sumo_parameter.values, + realizations=sumo_parameter.realizations, + ) + + +def create_ensemble_sensitivities( + sumo_ensemble_parameters: List[SumoEnsembleParameter], +) -> Optional[List[EnsembleSensitivity]]: + """Extract sensitivities from a list of SumoEnsembleParameter objects""" + sensitivities = [] + + sens_name_parameter = next( + (parameter for parameter in sumo_ensemble_parameters if parameter.name == "SENSNAME"), + None, + ) + sens_case_parameter = next( + (parameter for parameter in sumo_ensemble_parameters if parameter.name == "SENSCASE"), + None, + ) + if sens_case_parameter is None or sens_name_parameter is None: + return None + df = pd.DataFrame( + { + "name": sens_name_parameter.values, + "case": sens_case_parameter.values, + "REAL": sens_case_parameter.realizations, + } + ) + for name, group in df.groupby("name"): + sensitivities.append( + EnsembleSensitivity( + name=name, + type=find_sensitivity_type(list(group["case"].unique())), + cases=create_ensemble_sensitivity_cases(group), + ) + ) + return sensitivities if sensitivities else None + + +def find_sensitivity_type(sens_case_names: List[str]) -> SensitivityType: + """Find the sensitivity type based on the sensitivity case names""" + if len(sens_case_names) == 1 and sens_case_names[0] == "p10_p90": + return SensitivityType.MONTECARLO + return SensitivityType.SCENARIO + + +def create_ensemble_sensitivity_cases( + df: pd.DataFrame, +) -> List[EnsembleSensitivityCase]: + """Create a list of EnsembleSensitivityCase objects from a dataframe""" + cases = [] + for case_name, case_df in df.groupby("case"): + cases.append( + EnsembleSensitivityCase( + name=case_name, + realizations=case_df["REAL"].unique().tolist(), + ) + ) + return cases diff --git a/backend/src/services/sumo_access/queries/parameters.py b/backend/src/services/sumo_access/queries/parameters.py index 37805a608..f1b8e94bf 100644 --- a/backend/src/services/sumo_access/queries/parameters.py +++ b/backend/src/services/sumo_access/queries/parameters.py @@ -5,7 +5,7 @@ class SumoEnsembleParameter(BaseModel): name: str - groupname: Optional[str] + groupname: Optional[str] = None values: Union[List[float], List[int], List[str]] realizations: List[int] diff --git a/backend/tests/unit/services/utils/test_parameter_utils.py b/backend/tests/unit/services/sumo_access/test_parameter_access_helpers.py similarity index 97% rename from backend/tests/unit/services/utils/test_parameter_utils.py rename to backend/tests/unit/services/sumo_access/test_parameter_access_helpers.py index 23c15d15c..f13f834f7 100644 --- a/backend/tests/unit/services/utils/test_parameter_utils.py +++ b/backend/tests/unit/services/sumo_access/test_parameter_access_helpers.py @@ -2,18 +2,19 @@ import pytest import pandas as pd -from services.utils.parameter_utils import ( +from services.sumo_access.parameter_access import ( create_ensemble_parameter, create_ensemble_sensitivities, find_sensitivity_type, create_ensemble_sensitivity_cases, +) +from services.sumo_access.parameter_types import ( EnsembleParameter, EnsembleSensitivity, EnsembleSensitivityCase, SensitivityType, ) - from services.sumo_access.queries.parameters import SumoEnsembleParameter diff --git a/backend/tests/unit/services/utils/__init__.py b/backend/tests/unit/services/utils/__init__.py deleted file mode 100644 index e69de29bb..000000000 From 293e18eb09c255c005122135541e1ef1b23a2a6a Mon Sep 17 00:00:00 2001 From: Hans Kallekleiv <16436291+HansKallekleiv@users.noreply.github.com> Date: Mon, 18 Sep 2023 11:27:29 +0200 Subject: [PATCH 7/8] revert access to orig --- .../src/services/sumo_access/parameter_access.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/backend/src/services/sumo_access/parameter_access.py b/backend/src/services/sumo_access/parameter_access.py index b1338b0ef..058c40d1c 100644 --- a/backend/src/services/sumo_access/parameter_access.py +++ b/backend/src/services/sumo_access/parameter_access.py @@ -5,7 +5,8 @@ import pandas as pd from fmu.sumo.explorer.explorer import CaseCollection, SumoClient -from .queries.parameters import SumoEnsembleParameter, get_parameters_for_iteration +from .queries.parameters import get_parameters_for_iteration, SumoEnsembleParameter +from ._helpers import create_sumo_client_instance from .parameter_types import ( EnsembleParameter, EnsembleParameters, @@ -14,9 +15,6 @@ SensitivityType, ) - -from ._helpers import create_sumo_client_instance - LOGGER = logging.getLogger(__name__) @@ -51,11 +49,6 @@ def get_parameter(self, parameter_name: str) -> EnsembleParameter: return next(parameter for parameter in parameters.parameters if parameter.name == parameter_name) -def is_array_numeric(array: np.ndarray) -> bool: - """Check if an array is numeric""" - return np.issubdtype(array.dtype, np.number) - - def create_ensemble_parameter( sumo_parameter: SumoEnsembleParameter, ) -> EnsembleParameter: @@ -129,3 +122,8 @@ def create_ensemble_sensitivity_cases( ) ) return cases + + +def is_array_numeric(array: np.ndarray) -> bool: + """Check if an array is numeric""" + return np.issubdtype(array.dtype, np.number) From 723ada72173cd9a51b9226db60481f2fb31dcd21 Mon Sep 17 00:00:00 2001 From: Hans Kallekleiv <16436291+HansKallekleiv@users.noreply.github.com> Date: Mon, 18 Sep 2023 11:28:41 +0200 Subject: [PATCH 8/8] revert --- backend/src/services/utils/parameter_utils.py | 92 ------------------- 1 file changed, 92 deletions(-) delete mode 100644 backend/src/services/utils/parameter_utils.py diff --git a/backend/src/services/utils/parameter_utils.py b/backend/src/services/utils/parameter_utils.py deleted file mode 100644 index 29879ff27..000000000 --- a/backend/src/services/utils/parameter_utils.py +++ /dev/null @@ -1,92 +0,0 @@ -from typing import List, Optional - -import numpy as np -import pandas as pd - -from ..sumo_access.queries.parameters import SumoEnsembleParameter -from ..sumo_access.parameter_types import ( - EnsembleParameter, - EnsembleSensitivity, - EnsembleSensitivityCase, - SensitivityType, -) - - -def is_array_numeric(array: np.ndarray) -> bool: - """Check if an array is numeric""" - return np.issubdtype(array.dtype, np.number) - - -def create_ensemble_parameter( - sumo_parameter: SumoEnsembleParameter, -) -> EnsembleParameter: - """Create an EnsembleParameter from a Sumo parameter object""" - - return EnsembleParameter( - name=sumo_parameter.name, - is_logarithmic=sumo_parameter.name.startswith("LOG10_"), - is_numerical=is_array_numeric(np.array(sumo_parameter.values)), - is_constant=all(value == sumo_parameter.values[0] for value in sumo_parameter.values), - group_name=sumo_parameter.groupname, - descriptive_name=f"{sumo_parameter.name} (log)" - if sumo_parameter.name.startswith("LOG10_") - else sumo_parameter.name, - values=sumo_parameter.values, - realizations=sumo_parameter.realizations, - ) - - -def create_ensemble_sensitivities( - sumo_ensemble_parameters: List[SumoEnsembleParameter], -) -> Optional[List[EnsembleSensitivity]]: - """Extract sensitivities from a list of SumoEnsembleParameter objects""" - sensitivities = [] - - sens_name_parameter = next( - (parameter for parameter in sumo_ensemble_parameters if parameter.name == "SENSNAME"), - None, - ) - sens_case_parameter = next( - (parameter for parameter in sumo_ensemble_parameters if parameter.name == "SENSCASE"), - None, - ) - if sens_case_parameter is None or sens_name_parameter is None: - return None - df = pd.DataFrame( - { - "name": sens_name_parameter.values, - "case": sens_case_parameter.values, - "REAL": sens_case_parameter.realizations, - } - ) - for name, group in df.groupby("name"): - sensitivities.append( - EnsembleSensitivity( - name=name, - type=find_sensitivity_type(list(group["case"].unique())), - cases=create_ensemble_sensitivity_cases(group), - ) - ) - return sensitivities if sensitivities else None - - -def find_sensitivity_type(sens_case_names: List[str]) -> SensitivityType: - """Find the sensitivity type based on the sensitivity case names""" - if len(sens_case_names) == 1 and sens_case_names[0] == "p10_p90": - return SensitivityType.MONTECARLO - return SensitivityType.SCENARIO - - -def create_ensemble_sensitivity_cases( - df: pd.DataFrame, -) -> List[EnsembleSensitivityCase]: - """Create a list of EnsembleSensitivityCase objects from a dataframe""" - cases = [] - for case_name, case_df in df.groupby("case"): - cases.append( - EnsembleSensitivityCase( - name=case_name, - realizations=case_df["REAL"].unique().tolist(), - ) - ) - return cases