Skip to content

Commit

Permalink
CLN: Simplify InitializeCase
Browse files Browse the repository at this point in the history
This class had functionality built into it that did not end up being
used or may have been undesirable to allow users to use (hence was not
used). This functionality was cleaned up, the tests adjusted, and some
other tweaks and adjustments.

The items removed that were marked deprecated should be OK to delete as
they were relevant only to the create_case_data.py workflow job and not
used or imported by users. The workflow job did not use these items.
  • Loading branch information
mferrera committed Feb 28, 2024
1 parent a24774b commit 50cba44
Show file tree
Hide file tree
Showing 5 changed files with 188 additions and 297 deletions.
154 changes: 29 additions & 125 deletions src/fmu/dataio/dataio.py
Original file line number Diff line number Diff line change
Expand Up @@ -796,13 +796,13 @@ def export(
#
# The InitializeCase is used for making the case matadata prior to any other actions,
# e.g. forward jobs. However, case metadata file may already exist, and in that case
# this class should only emit a message or warning
# this class should only emit a message or warning.
# ######################################################################################


@dataclass
class InitializeCase: # pylint: disable=too-few-public-methods
"""Instantate InitializeCase object.
"""Initialize metadata for an FMU Case.
In ERT this is typically ran as an hook workflow in advance.
Expand All @@ -812,47 +812,31 @@ class InitializeCase: # pylint: disable=too-few-public-methods
some predefined main level keys. If config is None or the env variable
FMU_GLOBAL_CONFIG pointing to a file is provided, then it will attempt to
parse that file instead.
rootfolder: To override the automatic and actual ``rootpath``. Absolute path to
the case root, including case name. If not provided (which is not
recommended), the rootpath will be attempted parsed from the file structure
or by other means.
rootfolder: Absolute path to the case root, including case name.
casename: Name of case (experiment)
caseuser: Username provided
restart_from: ID of eventual restart (deprecated)
description: Description text as string or list of strings.
description (Optional): Description text as string or list of strings.
"""

# class variables
meta_format: ClassVar[Literal["yaml", "json"]] = "yaml"

# instance
config: dict
rootfolder: Optional[Union[str, Path]] = None
casename: Optional[str] = None
caseuser: Optional[str] = None
restart_from: Optional[str] = None
rootfolder: str | Path
casename: str
caseuser: str

description: Optional[Union[str, list]] = None
verbosity: str = "DEPRECATED"

_metadata: dict = field(default_factory=dict, init=False)
_metafile: Path = field(default_factory=Path, init=False)
_pwd: Path = field(default_factory=Path, init=False)
_casepath: Path = field(default_factory=Path, init=False)

def __post_init__(self) -> None:
if self.verbosity != "DEPRECATED":
warn(
"Using the 'verbosity' key is now deprecated and will have no "
"effect and will be removed in near future. Please remove it from the "
"argument list. Set logging level from client script in the standard "
"manner instead.",
UserWarning,
)

if not self.config or GLOBAL_ENVNAME in os.environ:
cnf = some_config_from_env(GLOBAL_ENVNAME)
assert cnf is not None
self.config = cnf
self._pwd = Path().absolute()
self._casepath = Path(self.rootfolder)
self._metafile = self._casepath / "share/metadata/fmu_case.yml"

# For this class, the global config must be valid; hence error if not
try:
Expand All @@ -862,101 +846,39 @@ def __post_init__(self) -> None:
raise
logger.info("Ran __post_init__ for InitializeCase")

def _update_settings(self, newsettings: dict) -> None:
"""Update instance settings (properties) from other routines."""
logger.info("Try new settings %s", newsettings)
def _establish_metadata_files(self) -> bool:
"""Checks if the metadata files and directories are established and creates
relevant directories and files if not.
# derive legal input from dataclass signature
annots = getattr(self, "__annotations__", {})
legals = {key: val for key, val in annots.items() if not key.startswith("_")}

for setting, value in newsettings.items():
if setting == "restart_from":
warn(
"The 'restart_from' argument is deprecated and will be removed in "
"a future version. Please refer to the fmu-dataio documentation "
"for information on how to record information about restart "
"source.",
DeprecationWarning,
)
if _validate_variable(setting, value, legals):
setattr(self, setting, value)
logger.info("New setting OK for %s", setting)

def _establish_pwd_casepath(self) -> None:
"""Establish state variables pwd and casepath.
See ExportData's method but this is much simpler (e.g. no RMS context)
Returns:
False if fmu_case.yml exists (not established), True if it doesn't.
"""
self._pwd = Path().absolute()

if self.rootfolder:
self._casepath = Path(self.rootfolder)
else:
logger.info("Emit UserWarning")
warn(
"The rootfolder is defaulted, but it is strongly recommended to give "
"an explicit rootfolder",
UserWarning,
)
self._casepath = self._pwd.parent.parent

logger.info("Set PWD (case): %s", str(self._pwd))
logger.info("Set rootpath (case): %s", str(self._casepath))

def _establish_case_metadata(self, force: bool = False) -> bool:
if not self._casepath.exists():
self._casepath.mkdir(parents=True, exist_ok=True)
if not self._metafile.parent.exists():
self._metafile.parent.mkdir(parents=True, exist_ok=True)
logger.info("Created rootpath (case) %s", self._casepath)

metadata_path = self._casepath / "share/metadata"
self._metafile = metadata_path / "fmu_case.yml"
logger.info("The requested metafile is %s", self._metafile)

if force:
logger.info("Forcing a new metafile")

if not self._metafile.is_file() or force:
metadata_path.mkdir(parents=True, exist_ok=True)
return True

return False
return not self._metafile.exists()

# ==================================================================================
# Public methods:
# ==================================================================================

def generate_metadata(
self,
force: bool = False,
skip_null: bool = True,
**kwargs: object,
) -> dict | None:
def generate_metadata(self) -> dict:
"""Generate case metadata.
Args:
force: Overwrite existing case metadata if True. Default is False. If force
is False and case metadata already exists, a warning will issued and
None will be returned.
skip_null: Fields with None/missing values will be skipped if True (default)
**kwargs: See InitializeCase() arguments; initial will be overrided by
settings here.
Returns:
A dictionary with case metadata or None
A dictionary with case metadata or an empty dictionary if the metadata
already exists.
"""
self._update_settings(kwargs)
self._establish_pwd_casepath()

if not self._establish_case_metadata(force=force):
if not self._establish_metadata_files():
exists_warning = (
"The case metadata file already exists and will not be overwritten. "
"To make new case metadata delete the old case or run on a different "
"runpath."
)
logger.warning(exists_warning)
warn(exists_warning, UserWarning)
return None
return {}

meta = _metadata.default_meta_dollars()
meta["class"] = "case"
Expand All @@ -979,40 +901,22 @@ def generate_metadata(
mcase["user"] = {"id": self.caseuser}

mcase["description"] = generate_description(self.description)
mcase["restart_from"] = self.restart_from

meta["tracklog"] = _metadata.generate_meta_tracklog()

if skip_null:
meta = drop_nones(meta)
meta = drop_nones(meta)

self._metadata = meta

logger.info("The case metadata are now ready!")
return deepcopy(self._metadata)

# alias
generate_case_metadata = generate_metadata

def export(
self,
force: bool = False,
skip_null: bool = True,
**kwargs: dict[str, Any],
) -> str | None:
def export(self) -> str:
"""Export case metadata to file.
Args:
force: Overwrite existing case metadata if True. Default is False. If force
is False and case metadata already exists, a warning will issued and
None will be returned.
skip_null: Fields with None/missing values will be skipped if True (default)
**kwargs: See InitializeCase() arguments; initial will be overrided by
settings here.
Returns:
Full path of metadata file or None
Full path of metadata file.
"""
if self.generate_case_metadata(force=force, skip_null=skip_null, **kwargs):
if self.generate_metadata():
export_metadata_file(
self._metafile, self._metadata, savefmt=self.meta_format
)
Expand Down
13 changes: 3 additions & 10 deletions src/fmu/dataio/scripts/create_case_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,19 +115,12 @@ def create_metadata(args: argparse.Namespace) -> str:
) as f:
global_variables = yaml.safe_load(f)

# fmu.dataio.InitializeCase class is scheduled to be renamed.
case = InitializeCase(config=global_variables)
case_metadata_path = case.export(
return InitializeCase(
config=global_variables,
rootfolder=args.ert_caseroot,
casename=args.ert_casename,
caseuser=args.ert_username,
description=None, # type: ignore
# BUG(JB): description must be str accoring to dataclass
)

logger.info("Case metadata has been made: %s", case_metadata_path)
assert case_metadata_path is not None
return case_metadata_path
).export()


def register_on_sumo(
Expand Down
42 changes: 21 additions & 21 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,21 +20,21 @@

logger = logging.getLogger(__name__)

RUN1 = "tests/data/drogon/ertrun1/realization-0/iter-0"
RUN2 = "tests/data/drogon/ertrun1"
RUN_PRED = "tests/data/drogon/ertrun1/realization-0/pred"
ERTRUN = "tests/data/drogon/ertrun1"
ERTRUN_REAL0_ITER0 = f"{ERTRUN}/realization-0/iter-0"
ERTRUN_PRED = f"{ERTRUN}/realization-0/pred"

RUN1_ENV_PREHOOK = {
ERTRUN_ENV_PREHOOK = {
f"_ERT_{FmuEnv.EXPERIMENT_ID.name}": "6a8e1e0f-9315-46bb-9648-8de87151f4c7",
f"_ERT_{FmuEnv.ENSEMBLE_ID.name}": "b027f225-c45d-477d-8f33-73695217ba14",
f"_ERT_{FmuEnv.SIMULATION_MODE.name}": "test_run",
}
RUN1_ENV_FORWARD = {
ERTRUN_ENV_FORWARD = {
f"_ERT_{FmuEnv.ITERATION_NUMBER.name}": "0",
f"_ERT_{FmuEnv.REALIZATION_NUMBER.name}": "0",
f"_ERT_{FmuEnv.RUNPATH.name}": "---", # set dynamically due to pytest tmp rotation
}
RUN1_ENV_FULLRUN = {**RUN1_ENV_PREHOOK, **RUN1_ENV_FORWARD}
ERTRUN_ENV_FULLRUN = {**ERTRUN_ENV_PREHOOK, **ERTRUN_ENV_FORWARD}

ERT_RUNPATH = f"_ERT_{FmuEnv.RUNPATH.name}"

Expand Down Expand Up @@ -65,19 +65,19 @@ def _fmu_run1_env_variables(monkeypatch, usepath="", case_only=False):
Will here monkeypatch the ENV variables, with a particular setting for RUNPATH
(trough `usepath`) which may vary dynamically due to pytest tmp area rotation.
"""
env = RUN1_ENV_FULLRUN if not case_only else RUN1_ENV_PREHOOK
env = ERTRUN_ENV_FULLRUN if not case_only else ERTRUN_ENV_PREHOOK
for key, value in env.items():
env_value = str(usepath) if "RUNPATH" in key else value
monkeypatch.setenv(key, env_value)
logger.debug("Setting env %s as %s", key, env_value)


@pytest.fixture(name="fmurun", scope="function")
def fixture_fmurun(tmp_path_factory, monkeypatch, rootpath):
@pytest.fixture(scope="function")
def fmurun(tmp_path_factory, monkeypatch, rootpath):
"""A tmp folder structure for testing; here a new fmurun without case metadata."""
tmppath = tmp_path_factory.mktemp("data")
newpath = tmppath / RUN1
shutil.copytree(rootpath / RUN1, newpath)
newpath = tmppath / ERTRUN_REAL0_ITER0
shutil.copytree(rootpath / ERTRUN_REAL0_ITER0, newpath)

_fmu_run1_env_variables(monkeypatch, usepath=newpath, case_only=False)

Expand All @@ -89,8 +89,8 @@ def fixture_fmurun(tmp_path_factory, monkeypatch, rootpath):
def fixture_fmurun_prehook(tmp_path_factory, monkeypatch, rootpath):
"""A tmp folder structure for testing; here a new fmurun without case metadata."""
tmppath = tmp_path_factory.mktemp("data")
newpath = tmppath / RUN2
shutil.copytree(rootpath / RUN2, newpath)
newpath = tmppath / ERTRUN
shutil.copytree(rootpath / ERTRUN, newpath)

_fmu_run1_env_variables(monkeypatch, usepath=newpath, case_only=True)

Expand All @@ -102,8 +102,8 @@ def fixture_fmurun_prehook(tmp_path_factory, monkeypatch, rootpath):
def fixture_fmurun_w_casemetadata(tmp_path_factory, monkeypatch, rootpath):
"""Create a tmp folder structure for testing; here existing fmurun w/ case meta!"""
tmppath = tmp_path_factory.mktemp("data3")
newpath = tmppath / RUN2
shutil.copytree(rootpath / RUN2, newpath)
newpath = tmppath / ERTRUN
shutil.copytree(rootpath / ERTRUN, newpath)
rootpath = newpath / "realization-0/iter-0"

_fmu_run1_env_variables(monkeypatch, usepath=rootpath, case_only=False)
Expand All @@ -116,8 +116,8 @@ def fixture_fmurun_w_casemetadata(tmp_path_factory, monkeypatch, rootpath):
def fixture_fmurun_w_casemetadata_pred(tmp_path_factory, monkeypatch, rootpath):
"""Create a tmp folder structure for testing; here existing fmurun w/ case meta!"""
tmppath = tmp_path_factory.mktemp("data3")
newpath = tmppath / RUN2
shutil.copytree(rootpath / RUN2, newpath)
newpath = tmppath / ERTRUN
shutil.copytree(rootpath / ERTRUN, newpath)
rootpath = newpath / "realization-0/pred"

_fmu_run1_env_variables(monkeypatch, usepath=rootpath, case_only=False)
Expand All @@ -130,8 +130,8 @@ def fixture_fmurun_w_casemetadata_pred(tmp_path_factory, monkeypatch, rootpath):
def fixture_fmurun_pred(tmp_path_factory, rootpath):
"""Create a tmp folder structure for testing; here a new fmurun for prediction."""
tmppath = tmp_path_factory.mktemp("data_pred")
newpath = tmppath / RUN_PRED
shutil.copytree(rootpath / RUN_PRED, newpath)
newpath = tmppath / ERTRUN_PRED
shutil.copytree(rootpath / ERTRUN_PRED, newpath)
logger.debug("Ran %s", _current_function_name())
return newpath

Expand All @@ -144,8 +144,8 @@ def fixture_rmsrun_fmu_w_casemetadata(tmp_path_factory, rootpath):
in a FMU setup where case metadata are present
"""
tmppath = tmp_path_factory.mktemp("data3")
newpath = tmppath / RUN2
shutil.copytree(rootpath / RUN2, newpath)
newpath = tmppath / ERTRUN
shutil.copytree(rootpath / ERTRUN, newpath)
rmspath = newpath / "realization-0/iter-0/rms/model"
rmspath.mkdir(parents=True, exist_ok=True)
logger.debug("Active folder is %s", rmspath)
Expand Down
12 changes: 6 additions & 6 deletions tests/test_units/test_fmuprovider_class.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,25 +123,25 @@ def test_fmuprovider_prehook_case(tmp_path, globalconfig2, fmurun_prehook):
I *may* be that this behaviour can be removed in near future, since the ERT env
variables now will tell us that this is an active ERT run.
"""

icase = dataio.InitializeCase(config=globalconfig2)

caseroot = tmp_path / "prehook"
caseroot.mkdir(parents=True)

os.chdir(caseroot)

exp = icase.export(
icase = dataio.InitializeCase(
config=globalconfig2,
rootfolder=caseroot,
casename="MyCaseName",
caseuser="MyUser",
description="Some description",
)
exp = icase.export()

casemetafile = caseroot / "share/metadata/fmu_case.yml"

assert casemetafile.is_file()
assert exp == str(casemetafile.resolve())
os.chdir(fmurun_prehook)

os.chdir(fmurun_prehook)
logger.debug("Case root proposed is: %s", caseroot)
myfmu = FmuProvider(
model="Model567",
Expand Down
Loading

0 comments on commit 50cba44

Please sign in to comment.