Merge pull request #540 from BDonnot/bd_dev

Bd dev

.circleci/config.yml

@@ -21,6 +21,9 @@ executors:
     python311:
         docker:
             - image: python:3.11-buster
+    python312:
+        docker:
+            - image: cimg/python:3.12.0
 
 jobs:
     test:
@@ -231,6 +234,14 @@ jobs:
                    python -m pip install -U .[test]
                    export _GRID2OP_FORCE_TEST=1
                    grid2op.testinstall
+            - run:
+               command: |
+                   source venv_test/bin/activate
+                   python -m pip install -U "numpy>=1.26,<1.27"
+                   python -m pip install -U .[test]
+                   export _GRID2OP_FORCE_TEST=1
+                   grid2op.testinstall
+
     install310:
         executor: python310
         resource_class: small
@@ -282,6 +293,14 @@ jobs:
                    python -m pip install -U .[test]
                    export _GRID2OP_FORCE_TEST=1
                    grid2op.testinstall
+            - run:
+               command: |
+                   source venv_test/bin/activate
+                   python -m pip install -U "numpy>=1.26,<1.27"
+                   python -m pip install -U .[test]
+                   export _GRID2OP_FORCE_TEST=1
+                   grid2op.testinstall
+
     install311:
         executor: python311
         resource_class: small
@@ -319,6 +338,36 @@ jobs:
                    python -m pip install -U .[test]
                    export _GRID2OP_FORCE_TEST=1
                    grid2op.testinstall
+            - run:
+               command: |
+                   source venv_test/bin/activate
+                   python -m pip install -U "numpy>=1.26,<1.27"
+                   python -m pip install -U .[test]
+                   export _GRID2OP_FORCE_TEST=1
+                   grid2op.testinstall
+    install312:
+        executor: python312
+        resource_class: small
+        steps:
+            - checkout
+            - run: 
+                command: |
+                    sudo  apt-get update
+                    sudo  apt-get install -y coinor-cbc
+            - run: python -m pip install virtualenv
+            - run: python -m virtualenv venv_test
+            - run:
+               command: |
+                   source venv_test/bin/activate
+                   python -m pip install -U pip setuptools wheel
+                #    python -m pip install -U numba  # not on python 3.12 at the moment
+            - run:
+               command: |
+                   source venv_test/bin/activate
+                   python -m pip install -U "numpy>=1.26,<1.27"
+                   python -m pip install -U .[test]
+                   export _GRID2OP_FORCE_TEST=1
+                   grid2op.testinstall
 
 workflows:
     version: 2.1
@@ -331,3 +380,4 @@ workflows:
           - install39
           - install310
           - install311
+        #   - install312  # failing because of dependencies of numba, torch etc. Tired of it so ignoring it !

.github/workflows/main.yml

@@ -35,6 +35,11 @@ jobs:
             abi: cp311,
             version: '3.11',
           }
+          - {
+            name: cp312,
+            abi: cp312,
+            version: '3.12',
+          }
 
     steps:
 

.gitignore

@@ -392,6 +392,11 @@ actspace_converter.py
 grid2op/data_test/input_data_local/
 test_sim2real_battery.py
 grid2op/tests/list_test_debug
+aux_test_make_backend_test_suite.py
+grid2op/tests/test_failing_simulator.txt
+old_pyproject.toml
+pp_bug_gen_alone.py
+test_dunder.py
 
 # profiling files
 **.prof

CHANGELOG.rst

@@ -42,8 +42,19 @@ Change Log
   exception (they should lead to a divergence)
 - [FIXED] some wrong behaviour in the `remove_line_status_from_topo` when no observation where provided
   and `check_cooldown` is `False`
+- [FIXED] a bug in PandaPowerBackend in AC powerflow: disconnected storage unit had no 0. as voltage
+- [FIXED] a bug in PandaPowerBackend in AC powerflow when a generator was alone a bus it made the powerflow
+  crash on some cases (*eg* without lightsim2grid, without numba)
+- [FIXED] a bug in PandaPowerBackend in DC (in some cases non connected grid were not spotted)
+- [FIXED] now the observations once reloaded have the correct `_is_done` flag (`obs._is_done = False`)
+  which allows to use the `obs.get_energy_graph()` for example. This fixes https://github.com/rte-france/Grid2Op/issues/538
 - [ADDED] now depends on the `typing_extensions` package
+- [ADDED] a complete test suite to help people develop new backend using "Test Driven Programming" 
+  techniques
+- [IMPROVED] now easier than ever to run the grid2op test suite with a new backend (for relevant tests)
 - [IMPROVED] type hints for `Backend` and `PandapowerBackend`
+- [IMPROVED] distribute python 3.12 wheel
+- [IMPROVED] test for python 3.12 and numpy 1.26 when appropriate (*eg* when numpy version is released)
 
 [1.9.5] - 2023-09-18
 ---------------------

README.md

@@ -296,11 +296,12 @@ The unit tests includes testing, on linux machines the correct integration of gr
 * python 3.9
 * python 3.10
 * python 3.11
+* python 3.12
 
-On all of these cases, we tested grid2op on all available numpy version >= 1.20 (**nb** available numpy versions depend
+On all of these cases, we tested grid2op on all available numpy versions >= 1.20 (**nb** available numpy versions depend
 on python version).
 
-The complete test suit is run on linux with the latest numpy version on python 3.8.
+The complete test suit is run on linux with the latest numpy version on python 3.10.
 
 ### Known issues
 

grid2op/Action/baseAction.py

@@ -1879,7 +1879,7 @@ def _digest_setbus(self, dict_):
                     msg += (
                         ' at least one of "loads_id", "generators_id", "lines_or_id", '
                     )
-                    msg += '"lines_ex_id" or "substations_id"'
+                    msg += '"lines_ex_id" or "substations_id" or "storages_id"'
                     msg += " as keys. None where found. Current used keys are: "
                     msg += "{}".format(sorted(ddict_.keys()))
                     raise AmbiguousAction(msg)
@@ -1922,7 +1922,7 @@ def _digest_change_bus(self, dict_):
                     msg += (
                         ' at least one of "loads_id", "generators_id", "lines_or_id", '
                     )
-                    msg += '"lines_ex_id" or "substations_id"'
+                    msg += '"lines_ex_id" or "substations_id" or "storages_id"'
                     msg += " as keys. None where found. Current used keys are: "
                     msg += "{}".format(sorted(ddict_.keys()))
                     raise AmbiguousAction(msg)

grid2op/Backend/backend.py

@@ -172,6 +172,39 @@ def __init__(self,
         for k, v in kwargs.items():
             self._my_kwargs[k] = v
 
+    def make_complete_path(self,
+                           path : Union[os.PathLike, str],
+                           filename : Optional[Union[os.PathLike, str]]=None) -> str:
+        """Auxiliary function to retrieve the full path of the grid.
+        
+        It is best used at the beginning of the `load_grid` function of a backend.
+
+        Returns
+        -------
+        _type_
+            _description_
+
+        Raises
+        ------
+        Grid2OpException
+            _description_
+        Grid2OpException
+            _description_
+        """
+        if path is None and filename is None:
+            raise Grid2OpException(
+                "You must provide at least one of path or file to load a powergrid."
+            )
+        if path is None:
+            full_path = filename
+        elif filename is None:
+            full_path = path
+        else:
+            full_path = os.path.join(path, filename)
+        if not os.path.exists(full_path):
+            raise Grid2OpException('There is no powergrid at "{}"'.format(full_path))
+        return full_path
+
     @property
     def is_loaded(self) -> bool:
         """Return whether or not this backend has been loaded, that is if `load_grid` has been called or not with this instance."""

grid2op/Backend/educPandaPowerBackend.py

@@ -133,18 +133,7 @@ def load_grid(self,
         """
 
         # first, handles different kind of path:
-        if path is None and filename is None:
-            raise RuntimeError(
-                "You must provide at least one of path or file to load a powergrid."
-            )
-        if path is None:
-            full_path = filename
-        elif filename is None:
-            full_path = path
-        else:
-            full_path = os.path.join(path, filename)
-        if not os.path.exists(full_path):
-            raise RuntimeError('There is no powergrid at "{}"'.format(full_path))
+        full_path = self.make_complete_path(path, filename)
 
         # then load the grid located at the full path and store it in `self._grid`
         # raise an exception if it can't be loaded

grid2op/Backend/pandaPowerBackend.py

@@ -99,12 +99,12 @@ class PandaPowerBackend(Backend):
 
     .. code-block:: python
 
-            import grid2op
-            from grid2op.Backend import PandaPowerBackend
-            backend = PandaPowerBackend()
+        import grid2op
+        from grid2op.Backend import PandaPowerBackend
+        backend = PandaPowerBackend()
 
-            env = grid2op.make(backend=backend)
-            # and use "env" as any open ai gym environment.
+        env = grid2op.make(backend=backend)
+        # and use "env" as "any open ai gym" environment.
 
     """
 
@@ -328,26 +328,13 @@ def load_grid(self,
         are set as "out of service" unless a topological action acts on these specific substations.
 
         """
-
-        if path is None and filename is None:
-            raise RuntimeError(
-                "You must provide at least one of path or file to load a powergrid."
-            )
-        if path is None:
-            full_path = filename
-        elif filename is None:
-            full_path = path
-        else:
-            full_path = os.path.join(path, filename)
-        if not os.path.exists(full_path):
-            raise RuntimeError('There is no powergrid at "{}"'.format(full_path))
+        full_path = self.make_complete_path(path, filename)
 
         with warnings.catch_warnings():
             # remove deprecationg warnings for old version of pandapower
             warnings.filterwarnings("ignore", category=DeprecationWarning)
             warnings.filterwarnings("ignore", category=FutureWarning)
             self._grid = pp.from_json(full_path)
-
         self._check_for_non_modeled_elements()
 
         # add the slack bus that is often not modeled as a generator, but i need it for this backend to work
@@ -559,7 +546,9 @@ def load_grid(self,
         add_topo = copy.deepcopy(self._grid.bus)
         add_topo.index += add_topo.shape[0]
         add_topo["in_service"] = False
-        self._grid.bus = pd.concat((self._grid.bus, add_topo))
+        # self._grid.bus = pd.concat((self._grid.bus, add_topo))
+        for ind, el in add_topo.iterrows():
+            pp.create_bus(self._grid, index=ind, **el)
 
         self._init_private_attrs()
 
@@ -1030,30 +1019,39 @@ def runpf(self, is_dc : bool=False) -> Tuple[bool, Union[Exception, None]]:
                                                             " produces 0. instead. Please check generators: "
                                                             f"{np.where(~self._grid.gen['in_service'])[0]}"
                                                             )
-
-                if is_dc:
-                    pp.rundcpp(self._grid, check_connectivity=False, init="flat")
-                    # if dc i start normally next time i call an ac powerflow
-                    self._nb_bus_before = None
-                else:
-                    pp.runpp(
-                        self._grid,
-                        check_connectivity=False,
-                        init=self._pf_init,
-                        numba=self.with_numba,
-                        lightsim2grid=self._lightsim2grid,
-                        max_iteration=self._max_iter,
-                        distributed_slack=self._dist_slack,
-                    )
-
+                try:
+                    if is_dc:
+                        pp.rundcpp(self._grid, check_connectivity=True, init="flat")
+                        # if I put check_connectivity=False then the test AAATestBackendAPI.test_22_islanded_grid_make_divergence
+                        # does not pass
+
+                        # if dc i start normally next time i call an ac powerflow
+                        self._nb_bus_before = None
+                    else:
+                        pp.runpp(
+                            self._grid,
+                            check_connectivity=False,
+                            init=self._pf_init,
+                            numba=self.with_numba,
+                            lightsim2grid=self._lightsim2grid,
+                            max_iteration=self._max_iter,
+                            distributed_slack=self._dist_slack,
+                        )
+                except IndexError as exc_:
+                    raise pp.powerflow.LoadflowNotConverged(f"Surprising behaviour of pandapower when a bus is not connected to "
+                                                            f"anything but present on the bus (with check_connectivity=False). "
+                                                            f"Error was {exc_}"
+                                                            )
+
                 # stores the computation time
                 if "_ppc" in self._grid:
                     if "et" in self._grid["_ppc"]:
                         self.comp_time += self._grid["_ppc"]["et"]
                 if self._grid.res_gen.isnull().values.any():
                     # TODO see if there is a better way here -> do not handle this here, but rather in Backend._next_grid_state
                     # sometimes pandapower does not detect divergence and put Nan.
-                    raise pp.powerflow.LoadflowNotConverged("Divergence due to Nan values in res_gen table.")
+                    raise pp.powerflow.LoadflowNotConverged("Divergence due to Nan values in res_gen table (most likely due to "
+                                                            "a non connected grid).")
 
             # if a connected bus has a no voltage, it's a divergence (grid was not connected)
             if self._grid.res_bus.loc[self._grid.bus["in_service"]]["va_degree"].isnull().any():
@@ -1514,7 +1512,7 @@ def shunt_info(self) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
             .values.astype(dt_float)
         )
         shunt_bus = type(self).global_bus_to_local(self._grid.shunt["bus"].values, self.shunt_to_subid)
-        shunt_v[~self._grid.shunt["in_service"].values] = 0
+        shunt_v[~self._grid.shunt["in_service"].values] = 0.
         shunt_bus[~self._grid.shunt["in_service"].values] = -1
         return shunt_p, shunt_q, shunt_v, shunt_bus
 
@@ -1543,6 +1541,7 @@ def _storages_info(self):
                 ].values.astype(dt_float)
                 * self.storage_pu_to_kv
             )
+            v_storage[~self._grid.storage["in_service"].values] = 0.
         else:
             p_storage = np.zeros(shape=0, dtype=dt_float)
             q_storage = np.zeros(shape=0, dtype=dt_float)

grid2op/Chronics/gridStateFromFile.py

@@ -234,6 +234,8 @@ def _assert_correct(self, dict_convert, order_backend):
         lend_dict_values = len(vals)
 
         if len_dict_keys != len_backend:
+            import pdb
+            pdb.set_trace()
             err_msg = "Conversion mismatch between backend data {} elements and converter data {} (keys)"
             raise IncorrectNumberOfElements(err_msg.format(len_backend, len_dict_keys))
         if lend_dict_values != len_backend:

grid2op/Episode/EpisodeData.py

@@ -908,7 +908,8 @@ def __init__(
             except IncorrectNumberOfElements as exc_:
                 # grid2op does not allow to load the object: there is a mismatch between what has been stored
                 # and what is currently used.
-                raise
+                raise Grid2OpException("grid2op does not allow to load the object: there is a mismatch "
+                                       "between what has been stored and what is currently used.") from exc_
             except NonFiniteElement:
                 self._game_over = i
                 break

grid2op/MakeEnv/get_default_aux.py

@@ -7,6 +7,8 @@
 # This file is part of Grid2Op, Grid2Op a testbed platform to model sequential decision making in power systems.
 
 import numbers
+import copy
+import warnings
 from tarfile import ENCODING
 
 from grid2op.Exceptions import *
@@ -86,6 +88,13 @@ def _get_default_aux(
     # first seek for the parameter in the kwargs, and check it's valid
     if name in kwargs:
         res = kwargs[name]
+        if defaultClassApp in (dict, list, set):# see https://github.com/rte-france/Grid2Op/issues/536
+            try:
+                res = copy.deepcopy(res)
+            except copy.Error:
+                warnings.warn(f"Impossible to copy mutable value for kwargs {name}. Make sure not to reuse "
+                              f"the same kwargs for creating two environments."
+                              "(more info on https://github.com/rte-france/Grid2Op/issues/536)")
         if isclass is None:
             # I don't know whether it's an object or a class
             error_msg_here = None

grid2op/Observation/baseObservation.py

@@ -3390,6 +3390,7 @@ def from_vect(self, vect, check_legit=True):
         self._reset_matrices()
         # and ensure everything is reloaded properly
         super().from_vect(vect, check_legit=check_legit)
+        self._is_done = False
 
     def to_dict(self):
         """