diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4da18a66..e4a63b0f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -103,6 +103,10 @@ Native multi agents support: when creating the data. - [BREAKING] deprecation of `backend.check_kirchoff` in favor of `backend.check_kirchhoff` (fix the typo in the name) +- [BREAKING] change the name of the generated classes: now by default the backend class + name is added. This behaviour can be turned off by passing `_add_cls_nm_bk=False` + when calling `grid2op.make(...)`. If you develop a new Backend, you can also + customize the added name by overloading the `get_class_added_name` class method. - [FIXED] issue https://github.com/Grid2op/grid2op/issues/657 - [FIXED] missing an import on the `MaskedEnvironment` class - [FIXED] a bug when trying to set the load_p, load_q, gen_p, gen_v by names. @@ -114,7 +118,7 @@ Native multi agents support: with `gridobj.get_line_info(...)`, `gridobj.get_load_info(...)`, `gridobj.get_gen_info(...)` or , `gridobj.get_storage_info(...)` - [ADDED] codacy badge on the readme -- [ADDED] a method to check the KCL (`obs.check_kirchoff`) directly from the observation +- [ADDED] a method to check the KCL (`obs.check_kirchhoff`) directly from the observation (previously it was only possible to do it from the backend). This should be used for testing purpose only - [IMPROVED] possibility to set the injections values with names @@ -132,6 +136,8 @@ Native multi agents support: - [IMPROVED] some type hints for some agent class - [IMPROVED] the `backend.update_from_obs` function to work even when observation does not have shunt information but there are not shunts on the grid. +- [IMPROVED] consistency of `MultiMixEnv` in case of automatic_classes (only one + class is generated for all mixes) [1.10.4] - 2024-10-15 ------------------------- diff --git a/docs/conf.py b/docs/conf.py index d25f97a1..fc753b64 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -22,7 +22,7 @@ author = 'Benjamin Donnot' # The full version, including alpha/beta/rc tags -release = '1.11.0.dev0' +release = '1.11.0.dev1' version = '1.11' diff --git a/grid2op/Backend/backend.py b/grid2op/Backend/backend.py index 7c31344c..15539fed 100644 --- a/grid2op/Backend/backend.py +++ b/grid2op/Backend/backend.py @@ -2245,3 +2245,22 @@ def assert_grid_correct_after_powerflow(self) -> None: raise EnvError( 'Some components of "backend.get_topo_vect()" are not finite. This should be integer.' ) + + def get_class_added_name(self) -> str: + """ + .. versionadded: 1.11.0 + + This function allows to customize the name added in the generated classes + by default. + + It can be usefull for example if multiple instance of your backend can have different + ordering even if they are loaded with the same backend class. + + This should not be modified except if you code a specific backend class. + + Returns + ------- + ``str``: + The added name added to the class + """ + return type(self).__name__ diff --git a/grid2op/Backend/pandaPowerBackend.py b/grid2op/Backend/pandaPowerBackend.py index 98711ce4..61062057 100644 --- a/grid2op/Backend/pandaPowerBackend.py +++ b/grid2op/Backend/pandaPowerBackend.py @@ -575,7 +575,47 @@ def _aux_run_pf_init(self): raise pp.powerflow.LoadflowNotConverged except pp.powerflow.LoadflowNotConverged: self._aux_runpf_pp(True) - + + def _init_big_topo_to_bk(self): + self._big_topo_to_backend = [(None, None, None) for _ in range(self.dim_topo)] + for load_id, pos_big_topo in enumerate(self.load_pos_topo_vect): + self._big_topo_to_backend[pos_big_topo] = (load_id, load_id, 0) + for gen_id, pos_big_topo in enumerate(self.gen_pos_topo_vect): + self._big_topo_to_backend[pos_big_topo] = (gen_id, gen_id, 1) + for l_id, pos_big_topo in enumerate(self.line_or_pos_topo_vect): + if l_id < self.__nb_powerline: + self._big_topo_to_backend[pos_big_topo] = (l_id, l_id, 2) + else: + self._big_topo_to_backend[pos_big_topo] = ( + l_id, + l_id - self.__nb_powerline, + 3, + ) + for l_id, pos_big_topo in enumerate(self.line_ex_pos_topo_vect): + if l_id < self.__nb_powerline: + self._big_topo_to_backend[pos_big_topo] = (l_id, l_id, 4) + else: + self._big_topo_to_backend[pos_big_topo] = ( + l_id, + l_id - self.__nb_powerline, + 5, + ) + + def _init_topoid_objid(self): + self._big_topo_to_obj = [(None, None) for _ in range(self.dim_topo)] + nm_ = "load" + for load_id, pos_big_topo in enumerate(self.load_pos_topo_vect): + self._big_topo_to_obj[pos_big_topo] = (load_id, nm_) + nm_ = "gen" + for gen_id, pos_big_topo in enumerate(self.gen_pos_topo_vect): + self._big_topo_to_obj[pos_big_topo] = (gen_id, nm_) + nm_ = "lineor" + for l_id, pos_big_topo in enumerate(self.line_or_pos_topo_vect): + self._big_topo_to_obj[pos_big_topo] = (l_id, nm_) + nm_ = "lineex" + for l_id, pos_big_topo in enumerate(self.line_ex_pos_topo_vect): + self._big_topo_to_obj[pos_big_topo] = (l_id, nm_) + def _init_private_attrs(self) -> None: # number of elements per substation self.sub_info = np.zeros(self.n_sub, dtype=dt_int) @@ -740,6 +780,7 @@ def _init_private_attrs(self) -> None: self._nb_bus_before = None # store the topoid -> objid + self._init_topoid_objid() self._big_topo_to_obj = [(None, None) for _ in range(self.dim_topo)] nm_ = "load" for load_id, pos_big_topo in enumerate(self.load_pos_topo_vect): @@ -755,29 +796,7 @@ def _init_private_attrs(self) -> None: self._big_topo_to_obj[pos_big_topo] = (l_id, nm_) # store the topoid -> objid - self._big_topo_to_backend = [(None, None, None) for _ in range(self.dim_topo)] - for load_id, pos_big_topo in enumerate(self.load_pos_topo_vect): - self._big_topo_to_backend[pos_big_topo] = (load_id, load_id, 0) - for gen_id, pos_big_topo in enumerate(self.gen_pos_topo_vect): - self._big_topo_to_backend[pos_big_topo] = (gen_id, gen_id, 1) - for l_id, pos_big_topo in enumerate(self.line_or_pos_topo_vect): - if l_id < self.__nb_powerline: - self._big_topo_to_backend[pos_big_topo] = (l_id, l_id, 2) - else: - self._big_topo_to_backend[pos_big_topo] = ( - l_id, - l_id - self.__nb_powerline, - 3, - ) - for l_id, pos_big_topo in enumerate(self.line_ex_pos_topo_vect): - if l_id < self.__nb_powerline: - self._big_topo_to_backend[pos_big_topo] = (l_id, l_id, 4) - else: - self._big_topo_to_backend[pos_big_topo] = ( - l_id, - l_id - self.__nb_powerline, - 5, - ) + self._init_big_topo_to_bk() self.theta_or = np.full(self.n_line, fill_value=np.NaN, dtype=dt_float) self.theta_ex = np.full(self.n_line, fill_value=np.NaN, dtype=dt_float) diff --git a/grid2op/Environment/baseEnv.py b/grid2op/Environment/baseEnv.py index 8dd40cd3..042e7352 100644 --- a/grid2op/Environment/baseEnv.py +++ b/grid2op/Environment/baseEnv.py @@ -28,7 +28,7 @@ HighResSimCounter) from grid2op.Backend import Backend from grid2op.dtypes import dt_int, dt_float, dt_bool -from grid2op.Space import GridObjects, RandomObject +from grid2op.Space import GridObjects, RandomObject, GRID2OP_CLASSES_ENV_FOLDER from grid2op.Exceptions import (Grid2OpException, EnvError, InvalidRedispatching, @@ -353,7 +353,7 @@ def __init__( self._local_dir_cls = _local_dir_cls # suppose it's the second path to the environment, so the classes are already in the files self._read_from_local_dir = _read_from_local_dir if self._read_from_local_dir is not None: - if os.path.split(self._read_from_local_dir)[1] == "_grid2op_classes": + if os.path.split(self._read_from_local_dir)[1] == GRID2OP_CLASSES_ENV_FOLDER: # legacy behaviour (using experimental_read_from_local_dir kwargs in env.make) self._do_not_erase_local_dir_cls = True else: @@ -4081,7 +4081,7 @@ def _aux_gen_classes(cls_other, sys_path, _add_class_output=False): sys.path.append(sub_repo) sub_repo_mod = None - if tmp_nm == "_grid2op_classes": + if tmp_nm == GRID2OP_CLASSES_ENV_FOLDER: # legacy "experimental_read_from_local_dir" # issue was the module "_grid2op_classes" had the same name # regardless of the environment, so grid2op was "confused" @@ -4118,7 +4118,7 @@ def _aux_gen_classes(cls_other, sys_path, _add_class_output=False): cls_res = getattr(module, cls_other.__name__) return str_import, cls_res - def generate_classes(self, *, local_dir_id=None, _guard=None, _is_base_env__=True, sys_path=None): + def generate_classes(self, *, local_dir_id=None, _guard=None, sys_path=None, _is_base_env__=True): """ Use with care, but can be incredibly useful ! @@ -4203,9 +4203,9 @@ def generate_classes(self, *, local_dir_id=None, _guard=None, _is_base_env__=Tru "(eg no the top level env) if I don't know the path of " "the top level environment.") if local_dir_id is not None: - sys_path = os.path.join(self.get_path_env(), "_grid2op_classes", local_dir_id) + sys_path = os.path.join(self.get_path_env(), GRID2OP_CLASSES_ENV_FOLDER, local_dir_id) else: - sys_path = os.path.join(self.get_path_env(), "_grid2op_classes") + sys_path = os.path.join(self.get_path_env(), GRID2OP_CLASSES_ENV_FOLDER) if _is_base_env__: if os.path.exists(sys_path): diff --git a/grid2op/Environment/environment.py b/grid2op/Environment/environment.py index cdcb373b..5468db5e 100644 --- a/grid2op/Environment/environment.py +++ b/grid2op/Environment/environment.py @@ -174,8 +174,8 @@ def __init__( # this means that the "make" call is issued from the # creation of a MultiMix. # So I use the base name instead. - self.name = "".join(_overload_name_multimix[2:]) - self.multimix_mix_name = name + self.name = _overload_name_multimix.name_env + _overload_name_multimix.add_to_name + self.multimix_mix_name = None # set in creation of the MultiMixEnv instead self._overload_name_multimix = _overload_name_multimix else: self.name = name diff --git a/grid2op/Environment/multiMixEnv.py b/grid2op/Environment/multiMixEnv.py index be250847..af140350 100644 --- a/grid2op/Environment/multiMixEnv.py +++ b/grid2op/Environment/multiMixEnv.py @@ -10,17 +10,65 @@ import warnings import numpy as np import copy -from typing import Any, Dict, Tuple, Union, List, Literal +import re +from typing import Any, Dict, Tuple, Union, List, Literal, Optional from grid2op.dtypes import dt_int, dt_float -from grid2op.Space import GridObjects, RandomObject, DEFAULT_N_BUSBAR_PER_SUB +from grid2op.Space import GridObjects, RandomObject, DEFAULT_N_BUSBAR_PER_SUB, GRID2OP_CLASSES_ENV_FOLDER from grid2op.Exceptions import EnvError, Grid2OpException +from grid2op.Backend import Backend from grid2op.Observation import BaseObservation from grid2op.MakeEnv.PathUtils import USE_CLASS_IN_FILE from grid2op.Environment.baseEnv import BaseEnv from grid2op.typing_variables import STEP_INFO_TYPING, RESET_OPTIONS_TYPING +class _OverloadNameMultiMixInfo: + VALUE_ERROR_GETITEM : str = "You can only access member with integer and not with {}" + + def __init__(self, + path_cls=None, + path_env=None, + name_env=None, + add_to_name="", + mix_id=0, + ): + self.path_cls = path_cls + self.path_env = path_env + self.name_env = name_env + self.add_to_name = add_to_name + self.local_dir_tmpfolder = None + self.mix_id = mix_id + + def __getitem__(self, arg): + cls = type(self) + try: + arg_ = int(arg) + except ValueError as exc_: + raise ValueError(cls.VALUE_ERROR_GETITEM.format(type(arg))) from exc_ + + if arg_ != arg: + raise ValueError(cls.VALUE_ERROR_GETITEM.format(type(arg))) + + if arg_ < 0: + # for stuff like "overload[-1]" + arg_ += 6 + + if arg_ == 0: + return self.path_cls + if arg_ == 1: + return self.path_env + if arg_ == 2: + return self.name_env + if arg_ == 3: + return self.add_to_name + if arg_ == 4: + return self.local_dir_tmpfolder + if arg_ == 5: + return self.mix_id + raise IndexError("_OverloadNameMultiMixInfo can only be used with index being 0, 1, 2, 3, 4 or 5") + + class MultiMixEnvironment(GridObjects, RandomObject): """ This class represent a single powergrid configuration, @@ -165,6 +213,7 @@ def __init__( logger=None, experimental_read_from_local_dir=None, n_busbar=DEFAULT_N_BUSBAR_PER_SUB, + _add_cls_nm_bk=True, _add_to_name="", # internal, for test only, do not use ! _compat_glop_version=None, # internal, for test only, do not use ! _test=False, @@ -174,74 +223,87 @@ def __init__( RandomObject.__init__(self) self.current_env = None self.env_index = None - self.mix_envs = [] + self.mix_envs = {} self._env_dir = os.path.abspath(envs_dir) self.__closed = False self._do_not_erase_local_dir_cls = False - self._local_dir_cls = None + self._local_dir_cls = None if not os.path.exists(envs_dir): raise EnvError(f"There is nothing at {envs_dir}") # Special case handling for backend # TODO: with backend.copy() instead ! backendClass = None backend_kwargs = {} + self._ptr_backend_obj_first_env : Optional[Backend]= None + _added_bk_name = "" + if "backend" in kwargs: backendClass = type(kwargs["backend"]) if hasattr(kwargs["backend"], "_my_kwargs"): # was introduced in grid2op 1.7.1 backend_kwargs = kwargs["backend"]._my_kwargs + _added_bk_name = kwargs["backend"].get_class_added_name() + self._ptr_backend_obj_first_env = kwargs["backend"] del kwargs["backend"] - - li_mix_nms = [mix_name for mix_name in sorted(os.listdir(envs_dir)) if os.path.isdir(os.path.join(envs_dir, mix_name))] + + li_mix_nms = [mix_name for mix_name in sorted(os.listdir(envs_dir)) + if (mix_name != GRID2OP_CLASSES_ENV_FOLDER and + mix_name != "__pycache__" and + os.path.isdir(os.path.join(envs_dir, mix_name)) + )] if not li_mix_nms: raise EnvError("We did not find any mix in this multi-mix environment.") # Make sure GridObject class attributes are set from first env # Should be fine since the grid is the same for all envs - multi_env_name = (None, envs_dir, os.path.basename(os.path.abspath(envs_dir)), _add_to_name) + self.multi_env_name = _OverloadNameMultiMixInfo(None, envs_dir, os.path.basename(os.path.abspath(envs_dir)), _add_to_name) + env_for_init = self._aux_create_a_mix(envs_dir, li_mix_nms[0], + True, # first mix logger, backendClass, backend_kwargs, + _add_cls_nm_bk, _add_to_name, _compat_glop_version, n_busbar, _test, experimental_read_from_local_dir, - multi_env_name, - kwargs) - - cls_res_me = self._aux_add_class_file(env_for_init) + self.multi_env_name, + kwargs) + cls_res_me = self._aux_add_class_file(env_for_init) if cls_res_me is not None: self.__class__ = cls_res_me else: self.__class__ = type(self).init_grid(type(env_for_init.backend), _local_dir_cls=env_for_init._local_dir_cls) - self.mix_envs.append(env_for_init) - self._local_dir_cls = env_for_init._local_dir_cls - + self.mix_envs[li_mix_nms[0]] = env_for_init # TODO reuse same observation_space and action_space in all the envs maybe ? - multi_env_name = (type(env_for_init)._PATH_GRID_CLASSES, *multi_env_name[1:]) + self.multi_env_name.path_cls = type(env_for_init)._PATH_GRID_CLASSES + self.multi_env_name.name_env = env_for_init.env_name + i = -1 try: - for mix_name in li_mix_nms[1:]: + for i, mix_name in enumerate(li_mix_nms[1:]): mix_path = os.path.join(envs_dir, mix_name) if not os.path.isdir(mix_path): continue mix = self._aux_create_a_mix(envs_dir, mix_name, + False, # first mix logger, backendClass, backend_kwargs, + _add_cls_nm_bk, # _add_cls_nm_bk already added in _add_to_name ? _add_to_name, _compat_glop_version, n_busbar, _test, experimental_read_from_local_dir, - multi_env_name, + self.multi_env_name, kwargs) - self.mix_envs.append(mix) + self.mix_envs[mix_name] = mix except Exception as exc_: - err_msg = "MultiMix environment creation failed at the creation of the first mix. Error: {}".format(exc_) + err_msg = f"MultiMix environment creation failed at the creation of mix {mix_name} (mix {i+1+1} / {len(li_mix_nms)})" raise EnvError(err_msg) from exc_ if len(self.mix_envs) == 0: @@ -250,17 +312,20 @@ def __init__( # tell every mix the "MultiMix" is responsible for deleting the # folder that stores the classes definition - for el in self.mix_envs: + for el in self.mix_envs.values(): el._do_not_erase_local_dir_cls = True self.env_index = 0 - self.current_env = self.mix_envs[self.env_index] - + self.all_names = li_mix_nms + self.current_env = self.mix_envs[self.all_names[self.env_index]] # legacy behaviour (using experimental_read_from_local_dir kwargs in env.make) if self._read_from_local_dir is not None: - if os.path.split(self._read_from_local_dir)[1] == "_grid2op_classes": + if os.path.split(self._read_from_local_dir)[1] == GRID2OP_CLASSES_ENV_FOLDER: self._do_not_erase_local_dir_cls = True else: self._do_not_erase_local_dir_cls = True + + # to prevent the cleaning of this tmp folder + self.multi_env_name.local_dir_tmpfolder = None def _aux_aux_add_class_file(self, sys_path, env_for_init): # used for the old behaviour (setting experimental_read_from_local_dir=True in make) @@ -286,24 +351,43 @@ def _aux_add_class_file(self, env_for_init): if env_for_init.classes_are_in_files() and env_for_init._local_dir_cls is not None: sys_path = os.path.abspath(env_for_init._local_dir_cls.name) self._local_dir_cls = env_for_init._local_dir_cls + self.multi_env_name.local_dir_tmpfolder = self._local_dir_cls env_for_init._local_dir_cls = None # then generate the proper classes cls_res_me = self._aux_aux_add_class_file(sys_path, env_for_init) return cls_res_me return None - + + def _aux_make_backend_from_cls(self, backendClass, backend_kwargs): + # Special case for backend + try: + # should pass with grid2op >= 1.7.1 + bk = backendClass(**backend_kwargs) + except TypeError as exc_: + # with grid2Op version prior to 1.7.1 + # you might have trouble with + # "TypeError: __init__() got an unexpected keyword argument 'can_be_copied'" + msg_ = ("Impossible to create a backend for each mix using the " + "backend key-word arguments. Falling back to creating " + "with no argument at all (default behaviour with grid2op <= 1.7.0).") + warnings.warn(msg_) + bk = backendClass() + return bk + def _aux_create_a_mix(self, envs_dir, mix_name, + is_first_mix, logger, backendClass, backend_kwargs, + _add_cls_nm_bk, _add_to_name, _compat_glop_version, n_busbar, _test, experimental_read_from_local_dir, - multi_env_name, + multi_env_name : _OverloadNameMultiMixInfo, kwargs ): # Inline import to prevent cyclical import @@ -315,44 +399,46 @@ def _aux_create_a_mix(self, else None ) mix_path = os.path.join(envs_dir, mix_name) - # Special case for backend - if backendClass is not None: - try: - # should pass with grid2op >= 1.7.1 - bk = backendClass(**backend_kwargs) - except TypeError as exc_: - # with grid2Op version prior to 1.7.1 - # you might have trouble with - # "TypeError: __init__() got an unexpected keyword argument 'can_be_copied'" - msg_ = ("Impossible to create a backend for each mix using the " - "backend key-word arguments. Falling back to creating " - "with no argument at all (default behaviour with grid2op <= 1.7.0).") - warnings.warn(msg_) - bk = backendClass() - mix = make( - mix_path, - backend=bk, - _add_to_name=_add_to_name, - _compat_glop_version=_compat_glop_version, - n_busbar=n_busbar, - test=_test, - logger=this_logger, - experimental_read_from_local_dir=experimental_read_from_local_dir, - _overload_name_multimix=multi_env_name, - **kwargs, - ) + kwargs_make = dict( + _add_cls_nm_bk=_add_cls_nm_bk, + _add_to_name=_add_to_name, + _compat_glop_version=_compat_glop_version, + n_busbar=n_busbar, + test=_test, + logger=this_logger, + experimental_read_from_local_dir=experimental_read_from_local_dir, + _overload_name_multimix=multi_env_name, + **kwargs) + if is_first_mix: + # in the first mix either I need to create the backend, or + # pass the backend given in argument + if self._ptr_backend_obj_first_env is not None: + # I reuse the backend passed as object on the first mix + bk = self._ptr_backend_obj_first_env + kwargs_make["backend"] = bk + elif backendClass is not None: + # Special case for backend + bk = self._aux_make_backend_from_cls(backendClass, backend_kwargs) + kwargs_make["backend"] = bk else: - mix = make( - mix_path, - n_busbar=n_busbar, - _add_to_name=_add_to_name, - _compat_glop_version=_compat_glop_version, - test=_test, - logger=this_logger, - experimental_read_from_local_dir=experimental_read_from_local_dir, - _overload_name_multimix=multi_env_name, - **kwargs, - ) + # in the other mixes, things are created with either a copy of the backend + # or a new backend from the kwargs + if self._ptr_backend_obj_first_env._can_be_copied: + bk = self._ptr_backend_obj_first_env.copy() + bk._is_loaded = False + elif backendClass is not None: + # Special case for backend + bk = self._aux_make_backend_from_cls(self.mix_envs[self.all_names[0]]._raw_backend_class, + self._ptr_backend_obj_first_env._my_kwargs) + kwargs_make["backend"] = bk + + mix = make(mix_path, **kwargs_make) + mix.multimix_mix_name = mix_name + multi_env_name.mix_id += 1 + if is_first_mix and self._ptr_backend_obj_first_env is None: + # if the "backend" kwargs has not been provided in the user call to "make" + # then I save a "pointer" to the backend of the first mix + self._ptr_backend_obj_first_env = mix.backend return mix def get_path_env(self): @@ -396,7 +482,7 @@ def __iter__(self): def __next__(self): if self.env_index < len(self.mix_envs): - r = self.mix_envs[self.env_index] + r = self.mix_envs[self.all_names[self.env_index]] self.env_index = self.env_index + 1 return r else: @@ -410,16 +496,16 @@ def __getattr__(self, name): return getattr(self.current_env, name) def keys(self): - for mix in self.mix_envs: - yield mix.multimix_mix_name + for mix in self.mix_envs.keys(): + yield mix def values(self): - for mix in self.mix_envs: + for mix in self.mix_envs.values(): yield mix def items(self): - for mix in self.mix_envs: - yield mix.multimix_mix_name, mix + for mix in self.mix_envs.items(): + yield mix def copy(self): if self.__closed: @@ -442,8 +528,8 @@ def copy(self): continue setattr(res, k, copy.deepcopy(getattr(self, k))) # now deal with the mixes - res.mix_envs = [mix.copy() for mix in mix_envs] - res.current_env = res.mix_envs[res.env_index] + res.mix_envs = {el: mix.copy() for el, mix in mix_envs.items()} + res.current_env = res.mix_envs[res.all_names[res.env_index]] # finally deal with the ownership of the class folder res._local_dir_cls = _local_dir_cls res._do_not_erase_local_dir_cls = True @@ -473,12 +559,7 @@ def __getitem__(self, key): if self.__closed: raise EnvError("This environment is closed, you cannot use it.") # Search for key - for mix in self.mix_envs: - if mix.multimix_mix_name == key: - return mix - - # Not found by name - raise KeyError + return self.mix_envs[key] def reset(self, *, @@ -502,7 +583,7 @@ def reset(self, else: self.env_index = (self.env_index + 1) % len(self.mix_envs) - self.current_env = self.mix_envs[self.env_index] + self.current_env = self.mix_envs[self.all_names[self.env_index]] return self.current_env.reset(seed=seed, options=options) def seed(self, seed=None): @@ -536,7 +617,7 @@ def seed(self, seed=None): s = super().seed(seed) seeds = [s] max_dt_int = np.iinfo(dt_int).max - for env in self.mix_envs: + for env in self.mix_envs.values(): env_seed = self.space_prng.randint(max_dt_int) env_seeds = env.seed(env_seed) seeds.append(env_seeds) @@ -545,25 +626,25 @@ def seed(self, seed=None): def set_chunk_size(self, new_chunk_size): if self.__closed: raise EnvError("This environment is closed, you cannot use it.") - for mix in self.mix_envs: + for mix in self.mix_envs.values(): mix.set_chunk_size(new_chunk_size) def set_id(self, id_): if self.__closed: raise EnvError("This environment is closed, you cannot use it.") - for mix in self.mix_envs: + for mix in self.mix_envs.values(): mix.set_id(id_) def deactivate_forecast(self): if self.__closed: raise EnvError("This environment is closed, you cannot use it.") - for mix in self.mix_envs: + for mix in self.mix_envs.values(): mix.deactivate_forecast() def reactivate_forecast(self): if self.__closed: raise EnvError("This environment is closed, you cannot use it.") - for mix in self.mix_envs: + for mix in self.mix_envs.values(): mix.reactivate_forecast() def set_thermal_limit(self, thermal_limit): @@ -573,7 +654,7 @@ def set_thermal_limit(self, thermal_limit): """ if self.__closed: raise EnvError("This environment is closed, you cannot use it.") - for mix in self.mix_envs: + for mix in self.mix_envs.values(): mix.set_thermal_limit(thermal_limit) def __enter__(self): @@ -596,7 +677,7 @@ def close(self): if self.__closed: return - for mix in self.mix_envs: + for mix in self.mix_envs.values(): mix.close() self.__closed = True @@ -613,7 +694,7 @@ def close(self): def attach_layout(self, grid_layout): if self.__closed: raise EnvError("This environment is closed, you cannot use it.") - for mix in self.mix_envs: + for mix in self.mix_envs.values(): mix.attach_layout(grid_layout) def __del__(self): @@ -622,12 +703,12 @@ def __del__(self): self.close() def generate_classes(self): - mix_for_classes = self.mix_envs[0] - path_cls = os.path.join(mix_for_classes.get_path_env(), "_grid2op_classes") + mix_for_classes = self.mix_envs[self.all_names[0]] + path_cls = os.path.join(self.multi_env_name.path_env, GRID2OP_CLASSES_ENV_FOLDER) if not os.path.exists(path_cls): try: os.mkdir(path_cls) except FileExistsError: pass - mix_for_classes.generate_classes() + mix_for_classes.generate_classes(sys_path=path_cls) self._aux_aux_add_class_file(path_cls, mix_for_classes) diff --git a/grid2op/MakeEnv/Make.py b/grid2op/MakeEnv/Make.py index 11a202e5..89154b38 100644 --- a/grid2op/MakeEnv/Make.py +++ b/grid2op/MakeEnv/Make.py @@ -248,6 +248,7 @@ def _aux_make_multimix( test=False, experimental_read_from_local_dir=False, n_busbar=2, + _add_cls_nm_bk=True, _add_to_name="", _compat_glop_version=None, _overload_name_multimix=None, @@ -263,6 +264,7 @@ def _aux_make_multimix( experimental_read_from_local_dir=experimental_read_from_local_dir, n_busbar=n_busbar, _test=test, + _add_cls_nm_bk=_add_cls_nm_bk, _add_to_name=_add_to_name, _compat_glop_version=_compat_glop_version, logger=logger, @@ -286,6 +288,7 @@ def make( logger: Optional[logging.Logger]=None, experimental_read_from_local_dir : bool=False, n_busbar=2, + _add_cls_nm_bk=True, _add_to_name : str="", _compat_glop_version : Optional[str]=None, _overload_name_multimix : Optional[str]=None, # do not use ! @@ -333,6 +336,14 @@ def make( Other keyword argument to give more control on the environment you are creating. See the Parameters information of the :func:`make_from_dataset_path`. + _add_cls_nm_bk: ``bool`` + Internal (and new in version 1.11.0). This flag (True by default, which is a breaking + change from 1.11.0 compared to previous versions) will add the backend + name in the generated class name. + + It is deactivated if classes are automatically generated by default `use_class_in_files` + is ``True`` + _add_to_name: Internal, do not use (and can only be used when setting "test=True"). If `experimental_read_from_local_dir` is set to True, this has no effect. @@ -432,6 +443,7 @@ def make_from_path_fn_(*args, **kwargs): return make_from_path_fn( dataset_path=dataset, + _add_cls_nm_bk=_add_cls_nm_bk, _add_to_name=_add_to_name_tmp, _compat_glop_version=_compat_glop_version_tmp, _overload_name_multimix=_overload_name_multimix, @@ -482,6 +494,7 @@ def make_from_path_fn_(*args, **kwargs): dataset_path=ds_path, logger=logger, n_busbar=n_busbar, + _add_cls_nm_bk=_add_cls_nm_bk, _add_to_name=_add_to_name, _compat_glop_version=_compat_glop_version, experimental_read_from_local_dir=experimental_read_from_local_dir, @@ -497,6 +510,7 @@ def make_from_path_fn_(*args, **kwargs): real_ds_path, logger=logger, n_busbar=n_busbar, + _add_cls_nm_bk=_add_cls_nm_bk, experimental_read_from_local_dir=experimental_read_from_local_dir, _overload_name_multimix=_overload_name_multimix, **kwargs @@ -519,5 +533,6 @@ def make_from_path_fn_(*args, **kwargs): n_busbar=n_busbar, experimental_read_from_local_dir=experimental_read_from_local_dir, _overload_name_multimix=_overload_name_multimix, + _add_cls_nm_bk=_add_cls_nm_bk, **kwargs ) diff --git a/grid2op/MakeEnv/MakeFromPath.py b/grid2op/MakeEnv/MakeFromPath.py index c051bf67..640c93be 100644 --- a/grid2op/MakeEnv/MakeFromPath.py +++ b/grid2op/MakeEnv/MakeFromPath.py @@ -10,6 +10,7 @@ import time import copy import importlib.util +from typing import Dict, Tuple, Type, Union, Optional import numpy as np import json import warnings @@ -25,6 +26,7 @@ FromChronix2grid, GridStateFromFile, GridValue) +from grid2op.Space import GRID2OP_CLASSES_ENV_FOLDER from grid2op.Action import BaseAction, DontAct from grid2op.Exceptions import EnvError from grid2op.Observation import CompleteObservation, BaseObservation @@ -33,6 +35,7 @@ from grid2op.VoltageControler import ControlVoltageFromFile from grid2op.Opponent import BaseOpponent, BaseActionBudget, NeverAttackBudget from grid2op.operator_attention import LinearAttentionBudget +from grid2op.typing_variables import DICT_CONFIG_TYPING from grid2op.MakeEnv.get_default_aux import _get_default_aux from grid2op.MakeEnv.PathUtils import _aux_fix_backend_internal_classes @@ -127,6 +130,7 @@ def make_from_dataset_path( logger=None, experimental_read_from_local_dir=False, n_busbar=2, + _add_cls_nm_bk=True, _add_to_name="", _compat_glop_version=None, _overload_name_multimix=None, @@ -282,13 +286,13 @@ def make_from_dataset_path( """ # Compute and find root folder _check_path(dataset_path, "Dataset root directory") - dataset_path_abs = os.path.abspath(dataset_path) + dataset_path_abs : str = os.path.abspath(dataset_path) # Compute env name from directory name - name_env = os.path.split(dataset_path_abs)[1] + name_env : str = os.path.split(dataset_path_abs)[1] # Compute and find chronics folder - chronics_path = _get_default_aux( + chronics_path : str = _get_default_aux( "chronics_path", kwargs, defaultClassApp=str, @@ -310,7 +314,7 @@ def make_from_dataset_path( exc_chronics = exc_ # Compute and find grid layout file - grid_layout_path_abs = os.path.abspath( + grid_layout_path_abs : str = os.path.abspath( os.path.join(dataset_path_abs, NAME_GRID_LAYOUT_FILE) ) try: @@ -333,7 +337,7 @@ def make_from_dataset_path( spec = importlib.util.spec_from_file_location("config.config", config_path_abs) config_module = importlib.util.module_from_spec(spec) spec.loader.exec_module(config_module) - config_data = config_module.config + config_data : DICT_CONFIG_TYPING = config_module.config except Exception as exc_: print(exc_) raise EnvError( @@ -344,7 +348,7 @@ def make_from_dataset_path( graph_layout = None try: with open(grid_layout_path_abs) as layout_fp: - graph_layout = json.load(layout_fp) + graph_layout : Dict[str, Tuple[float, float]]= json.load(layout_fp) except Exception as exc_: warnings.warn( "Dataset {} doesn't have a valid graph layout. Expect some failures when attempting " @@ -354,7 +358,7 @@ def make_from_dataset_path( # Get thermal limits thermal_limits = None if "thermal_limits" in config_data: - thermal_limits = config_data["thermal_limits"] + thermal_limits : Union[np.ndarray, Dict[str, float]]= config_data["thermal_limits"] # Get chronics_to_backend name_converter = None @@ -378,9 +382,9 @@ def make_from_dataset_path( # Get default backend class backend_class_cfg = PandaPowerBackend if "backend_class" in config_data and config_data["backend_class"] is not None: - backend_class_cfg = config_data["backend_class"] + backend_class_cfg : Type[Backend] = config_data["backend_class"] ## Create the backend, to compute the powerflow - backend = _get_default_aux( + backend : Backend = _get_default_aux( "backend", kwargs, defaultClass=backend_class_cfg, @@ -389,7 +393,7 @@ def make_from_dataset_path( ) # Compute and find backend/grid file - grid_path = _get_default_aux( + grid_path : str = _get_default_aux( "grid_path", kwargs, defaultClassApp=str, @@ -419,9 +423,9 @@ def make_from_dataset_path( "observation_class" in config_data and config_data["observation_class"] is not None ): - observation_class_cfg = config_data["observation_class"] + observation_class_cfg : Type[BaseObservation] = config_data["observation_class"] ## Setup the type of observation the agent will receive - observation_class = _get_default_aux( + observation_class : Type[BaseObservation] = _get_default_aux( "observation_class", kwargs, defaultClass=observation_class_cfg, @@ -433,7 +437,7 @@ def make_from_dataset_path( ## Create the parameters of the game, thermal limits threshold, # simulate cascading failure, powerflow mode etc. (the gamification of the game) if "param" in kwargs: - param = _get_default_aux( + param : Parameters = _get_default_aux( "param", kwargs, defaultClass=Parameters, @@ -493,12 +497,12 @@ def make_from_dataset_path( if "rules_class" in config_data and config_data["rules_class"] is not None: warnings.warn("You used the deprecated rules_class in your config. Please change its " "name to 'gamerules_class' to mimic the grid2op.make kwargs.") - rules_class_cfg = config_data["rules_class"] + rules_class_cfg : Type[BaseRules] = config_data["rules_class"] if "gamerules_class" in config_data and config_data["gamerules_class"] is not None: - rules_class_cfg = config_data["gamerules_class"] + rules_class_cfg : Type[BaseRules] = config_data["gamerules_class"] ## Create the rules of the game (mimic the operationnal constraints) - gamerules_class = _get_default_aux( + gamerules_class : Type[BaseRules] = _get_default_aux( "gamerules_class", kwargs, defaultClass=rules_class_cfg, @@ -510,10 +514,10 @@ def make_from_dataset_path( # Get default reward class reward_class_cfg = L2RPNReward if "reward_class" in config_data and config_data["reward_class"] is not None: - reward_class_cfg = config_data["reward_class"] + reward_class_cfg : Type[BaseReward] = config_data["reward_class"] ## Setup the reward the agent will receive - reward_class = _get_default_aux( + reward_class : Type[BaseReward] = _get_default_aux( "reward_class", kwargs, defaultClass=reward_class_cfg, @@ -886,16 +890,41 @@ def make_from_dataset_path( classes_in_file_kwargs = bool(kwargs["class_in_file"]) use_class_in_files = classes_in_file_kwargs + # new in 1.11.0: + if _add_cls_nm_bk: + _add_to_name = backend.get_class_added_name() + _add_to_name + do_not_erase_cls : Optional[bool] = None + + # new in 1.11.0 + if _overload_name_multimix is not None: + # this is a multimix + # AND this is the first mix of a multi mix + # I change the env name to add the "add_to_name" + + if _overload_name_multimix.mix_id == 0: + # this is the first mix I need to assign proper names + _overload_name_multimix.name_env = _overload_name_multimix.name_env + _add_to_name + _overload_name_multimix.add_to_name = "" + else: + # this is not the first mix + # for the other mix I need to read the data from files and NOT + # create the classes + use_class_in_files = False + if use_class_in_files: # new behaviour - sys_path = os.path.join(os.path.split(grid_path_abs)[0], "_grid2op_classes") - if not os.path.exists(sys_path): + if _overload_name_multimix is None: + sys_path_cls = os.path.join(os.path.split(grid_path_abs)[0], GRID2OP_CLASSES_ENV_FOLDER) + else: + sys_path_cls = os.path.join(_overload_name_multimix[1], GRID2OP_CLASSES_ENV_FOLDER) + if not os.path.exists(sys_path_cls): try: - os.mkdir(sys_path) + os.mkdir(sys_path_cls) except FileExistsError: # if another process created it, no problem pass - init_nm = os.path.join(sys_path, "__init__.py") + + init_nm = os.path.join(sys_path_cls, "__init__.py") if not os.path.exists(init_nm): try: with open(init_nm, "w", encoding="utf-8") as f: @@ -904,8 +933,14 @@ def make_from_dataset_path( pass import tempfile - this_local_dir = tempfile.TemporaryDirectory(dir=sys_path) - + if _overload_name_multimix is None or _overload_name_multimix[0] is None: + this_local_dir = tempfile.TemporaryDirectory(dir=sys_path_cls) + this_local_dir_name = this_local_dir.name + else: + this_local_dir_name = _overload_name_multimix[0] + this_local_dir = None + do_not_erase_cls = True + if experimental_read_from_local_dir: warnings.warn("With the automatic class generation, we removed the possibility to " "set `experimental_read_from_local_dir` to True.") @@ -922,51 +957,51 @@ def make_from_dataset_path( if graph_layout is not None and graph_layout: type(backend).attach_layout(graph_layout) - if not os.path.exists(this_local_dir.name): - raise EnvError(f"Path {this_local_dir.name} has not been created by the tempfile package") + if not os.path.exists(this_local_dir_name): + raise EnvError(f"Path {this_local_dir_name} has not been created by the tempfile package") init_env = Environment(init_env_path=os.path.abspath(dataset_path), - init_grid_path=grid_path_abs, - chronics_handler=data_feeding_fake, - backend=backend, - parameters=param, - name=name_env + _add_to_name, - names_chronics_to_backend=names_chronics_to_backend, - actionClass=action_class, - observationClass=observation_class, - rewardClass=reward_class, - legalActClass=gamerules_class, - voltagecontrolerClass=volagecontroler_class, - other_rewards=other_rewards, - opponent_space_type=opponent_space_type, - opponent_action_class=opponent_action_class, - opponent_class=opponent_class, - opponent_init_budget=opponent_init_budget, - opponent_attack_duration=opponent_attack_duration, - opponent_attack_cooldown=opponent_attack_cooldown, - opponent_budget_per_ts=opponent_budget_per_ts, - opponent_budget_class=opponent_budget_class, - kwargs_opponent=kwargs_opponent, - has_attention_budget=has_attention_budget, - attention_budget_cls=attention_budget_class, - kwargs_attention_budget=kwargs_attention_budget, - logger=logger, - n_busbar=n_busbar, # TODO n_busbar_per_sub different num per substations: read from a config file maybe (if not provided by the user) - _compat_glop_version=_compat_glop_version, - _read_from_local_dir=None, # first environment to generate the classes and save them - _local_dir_cls=None, - _overload_name_multimix=_overload_name_multimix, - kwargs_observation=kwargs_observation, - observation_bk_class=observation_backend_class, - observation_bk_kwargs=observation_backend_kwargs - ) - if not os.path.exists(this_local_dir.name): - raise EnvError(f"Path {this_local_dir.name} has not been created by the tempfile package") - init_env.generate_classes(local_dir_id=this_local_dir.name) + init_grid_path=grid_path_abs, + chronics_handler=data_feeding_fake, + backend=backend, + parameters=param, + name=name_env + _add_to_name, + names_chronics_to_backend=names_chronics_to_backend, + actionClass=action_class, + observationClass=observation_class, + rewardClass=reward_class, + legalActClass=gamerules_class, + voltagecontrolerClass=volagecontroler_class, + other_rewards=other_rewards, + opponent_space_type=opponent_space_type, + opponent_action_class=opponent_action_class, + opponent_class=opponent_class, + opponent_init_budget=opponent_init_budget, + opponent_attack_duration=opponent_attack_duration, + opponent_attack_cooldown=opponent_attack_cooldown, + opponent_budget_per_ts=opponent_budget_per_ts, + opponent_budget_class=opponent_budget_class, + kwargs_opponent=kwargs_opponent, + has_attention_budget=has_attention_budget, + attention_budget_cls=attention_budget_class, + kwargs_attention_budget=kwargs_attention_budget, + logger=logger, + n_busbar=n_busbar, # TODO n_busbar_per_sub different num per substations: read from a config file maybe (if not provided by the user) + _compat_glop_version=_compat_glop_version, + _read_from_local_dir=None, # first environment to generate the classes and save them + _local_dir_cls=None, + _overload_name_multimix=_overload_name_multimix, + kwargs_observation=kwargs_observation, + observation_bk_class=observation_backend_class, + observation_bk_kwargs=observation_backend_kwargs + ) + if not os.path.exists(this_local_dir_name): + raise EnvError(f"Path {this_local_dir_name} has not been created by the tempfile package") + init_env.generate_classes(local_dir_id=this_local_dir_name) # fix `my_bk_act_class` and `_complete_action_class` _aux_fix_backend_internal_classes(type(backend), this_local_dir) init_env.backend = None # to avoid to close the backend when init_env is deleted init_env._local_dir_cls = None - classes_path = this_local_dir.name + classes_path = this_local_dir_name allow_loaded_backend = True else: # legacy behaviour (<= 1.10.1 behaviour) @@ -974,15 +1009,10 @@ def make_from_dataset_path( if experimental_read_from_local_dir: if _overload_name_multimix is not None: # I am in a multimix - if _overload_name_multimix[0] is None: - # first mix: path is correct - sys_path = os.path.join(os.path.split(grid_path_abs)[0], "_grid2op_classes") - else: - # other mixes I need to retrieve the properties of the first mix - sys_path = _overload_name_multimix[0] + sys_path = os.path.join(_overload_name_multimix.path_env, GRID2OP_CLASSES_ENV_FOLDER) else: # I am not in a multimix - sys_path = os.path.join(os.path.split(grid_path_abs)[0], "_grid2op_classes") + sys_path = os.path.join(os.path.split(grid_path_abs)[0], GRID2OP_CLASSES_ENV_FOLDER) if not os.path.exists(sys_path): raise RuntimeError( "Attempting to load the grid classes from the env path. Yet the directory " @@ -1001,8 +1031,21 @@ def make_from_dataset_path( import sys sys.path.append(os.path.split(os.path.abspath(sys_path))[0]) classes_path = sys_path + + # new in 1.11.0 + if _overload_name_multimix is not None: + # case of multimix + _add_to_name = '' # already defined in the first mix + name_env = _overload_name_multimix.name_env + if _overload_name_multimix.mix_id >= 1 and _overload_name_multimix.local_dir_tmpfolder is not None: + # this is not the first mix + # for the other mix I need to read the data from files and NOT + # create the classes + this_local_dir = _overload_name_multimix.local_dir_tmpfolder + classes_path = this_local_dir.name + # Finally instantiate env from config & overrides - # including (if activated the new grid2op behaviour) + # including (if activated the new grid2op behaviour) env = Environment( init_env_path=os.path.abspath(dataset_path), init_grid_path=grid_path_abs, @@ -1040,6 +1083,8 @@ def make_from_dataset_path( observation_bk_class=observation_backend_class, observation_bk_kwargs=observation_backend_kwargs ) + if do_not_erase_cls is not None: + env._do_not_erase_local_dir_cls = do_not_erase_cls # Update the thermal limit if any if thermal_limits is not None: env.set_thermal_limit(thermal_limits) diff --git a/grid2op/Space/GridObjects.py b/grid2op/Space/GridObjects.py index f6c84dd4..c69f1291 100644 --- a/grid2op/Space/GridObjects.py +++ b/grid2op/Space/GridObjects.py @@ -33,7 +33,7 @@ # TODO tests of these methods and this class in general DEFAULT_N_BUSBAR_PER_SUB = 2 - +GRID2OP_CLASSES_ENV_FOLDER = "_grid2op_classes" class GridObjects: """ @@ -2885,7 +2885,7 @@ def _aux_init_grid_from_cls(cls, gridobj, name_res): # NB: these imports needs to be consistent with what is done in # base_env.generate_classes() super_module_nm, module_nm = os.path.split(gridobj._PATH_GRID_CLASSES) - if module_nm == "_grid2op_classes": + if module_nm == GRID2OP_CLASSES_ENV_FOLDER: # legacy "experimental_read_from_local_dir" # issue was the module "_grid2op_classes" had the same name # regardless of the environment, so grid2op was "confused" @@ -4494,11 +4494,11 @@ def _build_cls_from_import(name_cls, path_env): return None if not os.path.isdir(path_env): return None - if not os.path.exists(os.path.join(path_env, "_grid2op_classes")): + if not os.path.exists(os.path.join(path_env, GRID2OP_CLASSES_ENV_FOLDER)): return None sys.path.append(path_env) try: - module = importlib.import_module("_grid2op_classes") + module = importlib.import_module(GRID2OP_CLASSES_ENV_FOLDER) if hasattr(module, name_cls): my_class = getattr(module, name_cls) except (ModuleNotFoundError, ImportError) as exc_: diff --git a/grid2op/Space/__init__.py b/grid2op/Space/__init__.py index 69387627..8a71e1dd 100644 --- a/grid2op/Space/__init__.py +++ b/grid2op/Space/__init__.py @@ -1,5 +1,9 @@ -__all__ = ["RandomObject", "SerializableSpace", "GridObjects", "DEFAULT_N_BUSBAR_PER_SUB"] +__all__ = ["RandomObject", + "SerializableSpace", + "GridObjects", + "DEFAULT_N_BUSBAR_PER_SUB", + "GRID2OP_CLASSES_ENV_FOLDER"] from grid2op.Space.RandomObject import RandomObject from grid2op.Space.SerializableSpace import SerializableSpace -from grid2op.Space.GridObjects import GridObjects, DEFAULT_N_BUSBAR_PER_SUB +from grid2op.Space.GridObjects import GridObjects, DEFAULT_N_BUSBAR_PER_SUB, GRID2OP_CLASSES_ENV_FOLDER diff --git a/grid2op/__init__.py b/grid2op/__init__.py index 14ab5755..35522b93 100644 --- a/grid2op/__init__.py +++ b/grid2op/__init__.py @@ -11,7 +11,7 @@ Grid2Op """ -__version__ = '1.11.0.dev0' +__version__ = '1.11.0.dev1' __all__ = [ "Action", diff --git a/grid2op/tests/automatic_classes.py b/grid2op/tests/automatic_classes.py index f68c6f51..c50b91c5 100644 --- a/grid2op/tests/automatic_classes.py +++ b/grid2op/tests/automatic_classes.py @@ -95,6 +95,10 @@ class AutoClassInFileTester(unittest.TestCase): def get_env_name(self): return "l2rpn_case14_sandbox" + def get_env_name_cls(self): + # from grid2op 1.11.0 the backend name is in the class nameby default + return f"{self.get_env_name()}PandaPowerBackend" + def setUp(self) -> None: self.max_iter = 10 return super().setUp() @@ -131,7 +135,7 @@ def test_all_classes_from_file(self, name_observation_cls=None, name_action_cls=None): if classes_name is None: - classes_name = self.get_env_name() + classes_name = self.get_env_name_cls() if name_observation_cls is None: name_observation_cls = self._aux_get_obs_cls().format(classes_name) if name_action_cls is None: @@ -139,6 +143,7 @@ def test_all_classes_from_file(self, name_action_cls = name_action_cls.format(classes_name) env = self._aux_make_env(env) + names_cls = [f"ActionSpace_{classes_name}", f"_BackendAction_{classes_name}", f"CompleteAction_{classes_name}", @@ -163,7 +168,6 @@ def test_all_classes_from_file(self, "_actionClass", None, # VoltageOnlyAction not in env ] - # NB: these imports needs to be consistent with what is done in # base_env.generate_classes() and gridobj.init_grid(...) supermodule_nm, module_nm = os.path.split(env._read_from_local_dir) @@ -366,8 +370,8 @@ def test_all_classes_from_file_runner_1ep(self, env: Optional[Environment]=None) env = self._aux_make_env(env) this_agent = _ThisAgentTest(env.action_space, env._read_from_local_dir, - self._aux_get_obs_cls().format(self.get_env_name()), - self._aux_get_act_cls().format(self.get_env_name()), + self._aux_get_obs_cls().format(self.get_env_name_cls()), + self._aux_get_act_cls().format(self.get_env_name_cls()), ) runner = Runner(**env.get_params_for_runner(), agentClass=None, @@ -385,8 +389,8 @@ def test_all_classes_from_file_runner_2ep_seq(self, env: Optional[Environment]=N env = self._aux_make_env(env) this_agent = _ThisAgentTest(env.action_space, env._read_from_local_dir, - self._aux_get_obs_cls().format(self.get_env_name()), - self._aux_get_act_cls().format(self.get_env_name()), + self._aux_get_obs_cls().format(self.get_env_name_cls()), + self._aux_get_act_cls().format(self.get_env_name_cls()), ) runner = Runner(**env.get_params_for_runner(), agentClass=None, @@ -408,8 +412,8 @@ def test_all_classes_from_file_runner_2ep_par_fork(self, env: Optional[Environme env = self._aux_make_env(env) this_agent = _ThisAgentTest(env.action_space, env._read_from_local_dir, - self._aux_get_obs_cls().format(self.get_env_name()), - self._aux_get_act_cls().format(self.get_env_name()), + self._aux_get_obs_cls().format(self.get_env_name_cls()), + self._aux_get_act_cls().format(self.get_env_name_cls()), ) ctx = mp.get_context('fork') runner = Runner(**env.get_params_for_runner(), @@ -432,8 +436,8 @@ def test_all_classes_from_file_runner_2ep_par_spawn(self, env: Optional[Environm env = self._aux_make_env(env) this_agent = _ThisAgentTest(env.action_space, env._read_from_local_dir, - self._aux_get_obs_cls().format(self.get_env_name()), - self._aux_get_act_cls().format(self.get_env_name()), + self._aux_get_obs_cls().format(self.get_env_name_cls()), + self._aux_get_act_cls().format(self.get_env_name_cls()), ) ctx = mp.get_context('spawn') runner = Runner(**env.get_params_for_runner(), @@ -636,20 +640,20 @@ def test_all_classes_from_file(self, env = self._aux_make_env(env) try: super().test_all_classes_from_file(env, - classes_name=classes_name, - name_complete_obs_cls=name_complete_obs_cls, - name_observation_cls=name_observation_cls, - name_action_cls=name_action_cls - ) + classes_name=classes_name, + name_complete_obs_cls=name_complete_obs_cls, + name_observation_cls=name_observation_cls, + name_action_cls=name_action_cls + ) if isinstance(env, MultiMixEnvironment): # test each mix of a multi mix for mix in env: super().test_all_classes_from_file(mix, - classes_name=classes_name, - name_complete_obs_cls=name_complete_obs_cls, - name_observation_cls=name_observation_cls, - name_action_cls=name_action_cls - ) + classes_name=classes_name, + name_complete_obs_cls=name_complete_obs_cls, + name_observation_cls=name_observation_cls, + name_action_cls=name_action_cls + ) finally: if env_orig is None: # need to clean the env I created diff --git a/grid2op/tests/test_MultiMix.py b/grid2op/tests/test_MultiMix.py index 0f66ed0b..1024f758 100644 --- a/grid2op/tests/test_MultiMix.py +++ b/grid2op/tests/test_MultiMix.py @@ -82,7 +82,11 @@ def dummy(self): assert mme.current_obs is not None assert mme.current_env is not None for env in mme: - assert env.backend.dummy() == True + assert env.backend.dummy() == True, f"error for mix {env.multimix_mix_name}" + # the test below test that the backend is not initialized twice, + # if it was the case the name would be something like + # DummyBackend1_multimixDummyBackend1DummyBackend1 + assert type(env.backend).__name__ == "DummyBackend1_multimixDummyBackend1", f"{ type(env.backend).__name__} for mix {env.multimix_mix_name}" def test_creation_with_backend_are_not_shared(self): class DummyBackend2(PandaPowerBackend): @@ -298,9 +302,9 @@ def test_forecast_toggle(self): def test_bracket_access_by_name(self): mme = MultiMixEnvironment(PATH_DATA_MULTIMIX, _test=True) mix1_env = mme["case14_001"] - assert mix1_env.multimix_mix_name == "case14_001" + assert mix1_env.multimix_mix_name == "case14_001", f"{mix1_env.multimix_mix_name}" mix2_env = mme["case14_002"] - assert mix2_env.multimix_mix_name == "case14_002" + assert mix2_env.multimix_mix_name == "case14_002", f"{mix1_env.multimix_mix_name}" with self.assertRaises(KeyError): unknown_env = mme["unknown_raise"] diff --git a/grid2op/tests/test_Observation.py b/grid2op/tests/test_Observation.py index b4464e9b..0e85a27b 100644 --- a/grid2op/tests/test_Observation.py +++ b/grid2op/tests/test_Observation.py @@ -107,7 +107,7 @@ def setUp(self): "name_storage": [], "glop_version": grid2op.__version__, # "env_name": "rte_case14_test", - "env_name": "rte_case14_testTestBasisObsBehaviour", + "env_name": "rte_case14_testPandaPowerBackendTestBasisObsBehaviour", "sub_info": [3, 6, 4, 6, 5, 6, 3, 2, 5, 3, 3, 3, 4, 3], "load_to_subid": [1, 2, 13, 3, 4, 5, 8, 9, 10, 11, 12], "gen_to_subid": [1, 2, 5, 7, 0], diff --git a/grid2op/tests/test_add_class_name_backend.py b/grid2op/tests/test_add_class_name_backend.py new file mode 100644 index 00000000..35445223 --- /dev/null +++ b/grid2op/tests/test_add_class_name_backend.py @@ -0,0 +1,237 @@ +# Copyright (c) 2024, RTE (https://www.rte-france.com) +# See AUTHORS.txt +# This Source Code Form is subject to the terms of the Mozilla Public License, version 2.0. +# If a copy of the Mozilla Public License, version 2.0 was not distributed with this file, +# you can obtain one at http://mozilla.org/MPL/2.0/. +# SPDX-License-Identifier: MPL-2.0 +# This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. + +import copy +import numpy as np +from os import PathLike +from typing import Union +import grid2op +from grid2op.Backend import PandaPowerBackend +import unittest +import warnings + +import pdb + +from grid2op.Backend.pandaPowerBackend import NUMBA_ +from grid2op.Action._backendAction import _BackendAction + + +class _Aux_Test_PPDiffOrder(PandaPowerBackend): + def __init__(self, + detailed_infos_for_cascading_failures: bool = False, + lightsim2grid: bool = False, + dist_slack: bool = False, + max_iter: int = 10, + can_be_copied: bool = True, + with_numba: bool = NUMBA_, + seed=0): + super().__init__(detailed_infos_for_cascading_failures, + lightsim2grid, + dist_slack, + max_iter, + can_be_copied, + with_numba) + self._order_line = None + self._order_load = None + self._inv_order_line = None + self._inv_order_load = None + self.seed = seed + self._prng = np.random.default_rng(seed) + self.li_attr_del = ["gen_to_sub_pos", + "load_to_sub_pos", + "line_or_to_sub_pos", + "line_ex_to_sub_pos" + ] + self.li_pos_topo_vect = ["line_or_pos_topo_vect", + "line_ex_pos_topo_vect", + "load_pos_topo_vect", + "gen_pos_topo_vect", + ] + self._orig_topo_vect = None + self._new_topo_vect = None + + self._my_kwargs["seed"] = int(self.seed) + + def load_grid(self, path: Union[PathLike, str], filename: Union[PathLike, str, None] = None) -> None: + super().load_grid(path, filename) + if self.n_storage > 0: + self.li_attr_del.append("storage_to_sub_pos") + self.li_pos_topo_vect.append("storage_pos_topo_vect") + + self._orig_topo_vect = {el: getattr(type(self), el) for el in self.li_pos_topo_vect} + + # generate a different order + self._order_line = np.arange(self.n_line) + self._prng.shuffle(self._order_line) + self._order_load = np.arange(self.n_load) + self._prng.shuffle(self._order_load) + self._inv_order_load = np.argsort(self._order_load) + self._inv_order_line = np.argsort(self._order_line) + + # load the grid + self.load_to_subid = self.load_to_subid[self._order_load] + self.line_or_to_subid = self.line_or_to_subid[self._order_line] + self.line_ex_to_subid = self.line_ex_to_subid[self._order_line] + + # delete all the set attribute by the PandaPowerBackend class + for attr_nm in self.li_attr_del: + delattr(self, attr_nm) + setattr(self, attr_nm, None) + + # compute the "big topo" position + self._compute_pos_big_topo() + self.thermal_limit_a = self.thermal_limit_a[self._order_line] + self._new_topo_vect = {el: getattr(type(self), el) for el in self.li_pos_topo_vect} + self.name_load = self.name_load[self._order_load] + self.name_line = self.name_line[self._order_line] + + self._init_bus_load = self._init_bus_load[self._order_load] + self._init_bus_lor = self._init_bus_lor[self._order_line] + self._init_bus_lex = self._init_bus_lex[self._order_line] + self._init_big_topo_to_bk() + self._init_topoid_objid() + + def apply_action(self, backendAction: _BackendAction) -> None: + if backendAction is None: + return + reordered = copy.deepcopy(backendAction) + reordered.load_p.reorder(self._inv_order_load) + reordered.load_q.reorder(self._inv_order_load) + # probably won't work if topo is changed... + return super().apply_action(reordered) + + def _loads_info(self): + tmp = super()._loads_info() + res = [el[self._order_load] for el in tmp] + return res + + def _aux_get_line_info(self, colname1, colname2): + vect = super()._aux_get_line_info(colname1, colname2) + return vect[self._order_line] + + def get_class_added_name(self) -> str: + return type(self).__name__ + f"_{self.seed}" + +class TestAddClassNameBackend(unittest.TestCase): + def setUp(self) -> None: + self.tgt_load_p = np.array( [22. , 87. , 45.79999924, 7. , 12. , + 28.20000076, 8.69999981, 3.5 , 5.5 , 12.69999981, + 14.80000019]) + self.load_pos_topo_vect_diff_order = np.array([13, 44, 19, 41, 54, 36, 24, 9, 3, 47, 50]) + self.line_or_pos_topo_vect_diff_order = np.array([ 5, 37, 14, 6, 48, 15, 7, 38, 39, 27, + 1, 42, 28, 11, 31, 20, 51, 29, 2, 16]) + self.load_pos_topo_vect_corr_order = np.array([ 8, 12, 18, 23, 30, 40, 43, 46, 49, 53, 56]) + self.line_or_pos_topo_vect_corr_order = np.array([ 0, 1, 4, 5, 6, 10, 15, 24, 25, 26, + 36, 37, 42, 48, 52, 16, 17, 22, 32, 39]) + + self.load_pos_topo_vect_multi_do = np.array([ 23, 118, 165, 200, 364, 512, 76, 495, 429, 121, 35, 522, 174, + 203, 281, 389, 271, 377, 95, 89, 181, 447, 100, 298, 187, 432, + 450, 530, 484, 411, 184, 502, 246, 92, 241, 259, 230, 361, 220, + 491, 0, 453, 474, 141, 344, 330, 42, 456, 519, 54, 420, 386, + 471, 338, 256, 335, 132, 401, 86, 3, 66, 223, 150, 196, 227, + 80, 26, 305, 468, 138, 348, 515, 262, 319, 505, 57, 381, 69, + 333, 525, 479, 20, 162, 233, 128, 396, 6, 499, 417, 358, 171, + 438, 10, 191, 147, 528, 111, 441, 51]) + self.load_pos_topo_vect_multi_pp = np.array([ 2, 5, 9, 14, 22, 25, 30, 41, 50, 53, 56, 65, 68, + 75, 79, 85, 88, 91, 94, 99, 103, 117, 120, 123, 131, 137, + 140, 146, 149, 152, 164, 170, 173, 180, 183, 186, 190, 195, 199, + 202, 219, 222, 226, 229, 232, 240, 245, 255, 258, 261, 270, 275, + 287, 304, 307, 326, 332, 334, 337, 343, 347, 357, 360, 363, 374, + 380, 385, 388, 395, 398, 403, 416, 419, 428, 431, 437, 440, 446, + 449, 452, 455, 467, 470, 473, 478, 483, 490, 494, 498, 501, 504, + 509, 514, 518, 521, 524, 527, 529, 532]) + return super().setUp() + + def get_env_name(self): + return "l2rpn_case14_sandbox" + + def get_env_name_multi(self): + return "l2rpn_neurips_2020_track2" + + def debug_fake_backend(self): + tgt_load_bus = np.array([ 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13]) + env1 = grid2op.make(self.get_env_name(), test=True, backend=_Aux_Test_PPDiffOrder(seed=0), _add_cls_nm_bk=False, _add_to_name=type(self).__name__) + assert (env1.load_pos_topo_vect == self.load_pos_topo_vect_diff_order ).all() + assert (env1.line_or_pos_topo_vect == self.line_or_pos_topo_vect_diff_order).all() + env1.reset(seed=0, options={"time serie id": 0}) + assert np.abs(env1.backend._grid.load["p_mw"] - self.tgt_load_p).max() <= 1e-5 + assert np.all(env1.backend._grid.load["bus"] == tgt_load_bus) + + def test_legacy_behaviour_fails(self): + test_id = "0" + with warnings.catch_warnings(): + warnings.filterwarnings("ignore") + env0_0 = grid2op.make(self.get_env_name(), test=True, _add_cls_nm_bk=False, _add_to_name=type(self).__name__+test_id) + env0_1 = grid2op.make(self.get_env_name(), test=True, backend=_Aux_Test_PPDiffOrder(seed=0), _add_cls_nm_bk=False, _add_to_name=type(self).__name__+test_id) + assert type(env0_0).__name__ == type(env0_1).__name__ + assert (env0_0.load_pos_topo_vect == self.load_pos_topo_vect_corr_order ).all() + assert (env0_0.line_or_pos_topo_vect == self.line_or_pos_topo_vect_corr_order).all() + assert (env0_1.load_pos_topo_vect != self.load_pos_topo_vect_diff_order ).any() + assert (env0_1.line_or_pos_topo_vect != self.line_or_pos_topo_vect_diff_order).any() + + test_id = "1" + with warnings.catch_warnings(): + warnings.filterwarnings("ignore") + env1_0 = grid2op.make(self.get_env_name(), test=True, backend=_Aux_Test_PPDiffOrder(seed=0), _add_cls_nm_bk=False, _add_to_name=type(self).__name__+test_id) + env1_1 = grid2op.make(self.get_env_name(), test=True, _add_cls_nm_bk=False, _add_to_name=type(self).__name__+test_id) + assert type(env1_0).__name__ == type(env1_1).__name__ + assert (env1_0.load_pos_topo_vect == self.load_pos_topo_vect_diff_order ).all() + assert (env1_0.line_or_pos_topo_vect == self.line_or_pos_topo_vect_diff_order).all() + assert (env1_1.load_pos_topo_vect != self.load_pos_topo_vect_corr_order ).any() + assert (env1_1.line_or_pos_topo_vect != self.line_or_pos_topo_vect_corr_order).any() + + def test_basic_env(self): + test_id = "3" + with warnings.catch_warnings(): + warnings.filterwarnings("ignore") + env0 = grid2op.make(self.get_env_name(), test=True, backend=_Aux_Test_PPDiffOrder(seed=0), _add_to_name=type(self).__name__+test_id) + env1 = grid2op.make(self.get_env_name(), test=True, _add_to_name=type(self).__name__+test_id) + assert type(env0).__name__ != type(env1).__name__ + assert (env0.load_pos_topo_vect == self.load_pos_topo_vect_diff_order ).all() + assert (env0.line_or_pos_topo_vect == self.line_or_pos_topo_vect_diff_order).all() + assert (env1.load_pos_topo_vect == self.load_pos_topo_vect_corr_order ).all() + assert (env1.line_or_pos_topo_vect == self.line_or_pos_topo_vect_corr_order).all() + + test_id = "4" + with warnings.catch_warnings(): + warnings.filterwarnings("ignore") + env0 = grid2op.make(self.get_env_name(), test=True, _add_to_name=type(self).__name__+test_id) + env1 = grid2op.make(self.get_env_name(), test=True, backend=_Aux_Test_PPDiffOrder(seed=0), _add_to_name=type(self).__name__+test_id) + assert type(env0).__name__ != type(env1).__name__ + assert (env1.load_pos_topo_vect == self.load_pos_topo_vect_diff_order ).all() + assert (env1.line_or_pos_topo_vect == self.line_or_pos_topo_vect_diff_order).all() + assert (env0.load_pos_topo_vect == self.load_pos_topo_vect_corr_order ).all() + assert (env0.line_or_pos_topo_vect == self.line_or_pos_topo_vect_corr_order).all() + + def test_multi_env(self): + test_id = "5" + with warnings.catch_warnings(): + warnings.filterwarnings("ignore") + env0 = grid2op.make(self.get_env_name_multi(), test=True, backend=_Aux_Test_PPDiffOrder(seed=0), _add_to_name=type(self).__name__+test_id) + env1 = grid2op.make(self.get_env_name_multi(), test=True, _add_to_name=type(self).__name__+test_id) + assert (type(env0).load_pos_topo_vect == self.load_pos_topo_vect_multi_do).all() + for el in env0: + assert (type(el).load_pos_topo_vect == self.load_pos_topo_vect_multi_do).all() + assert (type(env1).load_pos_topo_vect == self.load_pos_topo_vect_multi_pp).all() + for el in env1: + assert (type(el).load_pos_topo_vect == self.load_pos_topo_vect_multi_pp).all() + + test_id = "6" + with warnings.catch_warnings(): + warnings.filterwarnings("ignore") + env0 = grid2op.make(self.get_env_name_multi(), test=True, _add_to_name=type(self).__name__+test_id) + env1 = grid2op.make(self.get_env_name_multi(), test=True, backend=_Aux_Test_PPDiffOrder(seed=0), _add_to_name=type(self).__name__+test_id) + assert (type(env1).load_pos_topo_vect == self.load_pos_topo_vect_multi_do).all() + for el in env1: + assert (type(el).load_pos_topo_vect == self.load_pos_topo_vect_multi_do).all() + assert (type(env0).load_pos_topo_vect == self.load_pos_topo_vect_multi_pp).all() + for el in env0: + assert (type(el).load_pos_topo_vect == self.load_pos_topo_vect_multi_pp).all() + +# TODO and as always, add Runner, MaskedEnv and TimedOutEnv +# TODO check with "automatic class generation" \ No newline at end of file diff --git a/grid2op/tests/test_generate_classes.py b/grid2op/tests/test_generate_classes.py index 98159248..f991fe7a 100644 --- a/grid2op/tests/test_generate_classes.py +++ b/grid2op/tests/test_generate_classes.py @@ -6,12 +6,14 @@ # SPDX-License-Identifier: MPL-2.0 # This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems. +import re import unittest import warnings from pathlib import Path from grid2op.Environment import Environment, MultiMixEnvironment from grid2op.tests.helper_path_test import * import grid2op +from grid2op.Space import GRID2OP_CLASSES_ENV_FOLDER import shutil import pdb @@ -19,16 +21,9 @@ class TestGenerateFile(unittest.TestCase): def _aux_assert_exists_then_delete(self, env): - if isinstance(env, MultiMixEnvironment): - # for mix in env: - # self._aux_assert_exists_then_delete(mix) - self._aux_assert_exists_then_delete(env.mix_envs[0]) - elif isinstance(env, Environment): - path = Path(env.get_path_env()) / "_grid2op_classes" - assert path.exists(), f"path {path} does not exists" - shutil.rmtree(path, ignore_errors=True) - else: - raise RuntimeError("Unknown env type") + path = Path(env.get_path_env()) / GRID2OP_CLASSES_ENV_FOLDER + assert path.exists(), f"path {path} does not exists" + shutil.rmtree(path, ignore_errors=True) def list_env(self): env_with_alert = os.path.join( @@ -56,6 +51,11 @@ def test_can_load(self): test=True, _add_to_name=_add_to_name) env.generate_classes() + cls_nm_tmp = f"PandaPowerBackend{_add_to_name}" + cls_nm_end = f"{cls_nm_tmp}$" + cls_nm_twice = f"{cls_nm_tmp}.+{cls_nm_end}" + assert re.search(cls_nm_end, type(env).__name__) is not None # name of the backend and "add_to_name" should appear once + assert re.search(cls_nm_twice, type(env).__name__) is None # they should not appear twice ! with warnings.catch_warnings(): warnings.filterwarnings("ignore") try: diff --git a/grid2op/tests/test_noisy_obs.py b/grid2op/tests/test_noisy_obs.py index e51a5ba3..a7780a99 100644 --- a/grid2op/tests/test_noisy_obs.py +++ b/grid2op/tests/test_noisy_obs.py @@ -113,7 +113,7 @@ def test_with_copy(self): def test_simulate(self): sim_o, *_ = self.obs.simulate(self.env.action_space()) - assert type(sim_o).env_name == "educ_case14_storage"+type(self).__name__ + assert type(sim_o).env_name == "educ_case14_storagePandaPowerBackend"+type(self).__name__ assert isinstance(sim_o, CompleteObservation) # test that it is reproducible diff --git a/grid2op/typing_variables.py b/grid2op/typing_variables.py index 856e7a76..ea19ec21 100644 --- a/grid2op/typing_variables.py +++ b/grid2op/typing_variables.py @@ -63,3 +63,7 @@ List[int], # give info for all substations Dict[str, int] # give information for some substation ] + +#: possible config key / values in the config.py file +# TODO improve that +DICT_CONFIG_TYPING = Dict[str, Any]