From acc3899eb30881dbce3e9421820b4193233ba8eb Mon Sep 17 00:00:00 2001 From: Torben Schiz <49746900+tjwsch@users.noreply.github.com> Date: Sat, 30 Mar 2024 21:11:22 +0100 Subject: [PATCH 01/41] Fix typo, remove redundent code and replace __import__ (#90) --- micro_manager/micro_manager.py | 7 +++---- tests/unit/test_micro_manager.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/micro_manager/micro_manager.py b/micro_manager/micro_manager.py index 28ca71a8..704e8196 100644 --- a/micro_manager/micro_manager.py +++ b/micro_manager/micro_manager.py @@ -19,6 +19,7 @@ import numpy as np import logging import time +import importlib from copy import deepcopy from typing import Dict from warnings import warn @@ -71,8 +72,6 @@ def __init__(self, config_file: str) -> None: self._rank, self._size) - micro_file_name = self._config.get_micro_file_name() - self._macro_mesh_name = self._config.get_macro_mesh_name() # Data names of data written to preCICE @@ -309,9 +308,9 @@ def _initialize(self) -> None: self._micro_sims = [None] * self._local_number_of_sims # DECLARATION micro_problem = getattr( - __import__( + importlib.import_module( self._config.get_micro_file_name(), - fromlist=["MicroSimulation"]), + "MicroSimulation"), "MicroSimulation") # Create micro simulation objects diff --git a/tests/unit/test_micro_manager.py b/tests/unit/test_micro_manager.py index 55548cc7..b6a43e1f 100644 --- a/tests/unit/test_micro_manager.py +++ b/tests/unit/test_micro_manager.py @@ -75,7 +75,7 @@ def test_read_write_data_from_precice(self): self.assertListEqual(data["macro-vector-data"].tolist(), fake_data["macro-vector-data"].tolist()) - def test_solve_mico_sims(self): + def test_solve_micro_sims(self): """ Test if the internal function _solve_micro_simulations works as expected. """ From 824028b01dcb1b0f74ae33ee9fdde120f7635ad9 Mon Sep 17 00:00:00 2001 From: Ishaan Desai Date: Mon, 1 Apr 2024 19:00:30 +0200 Subject: [PATCH 02/41] Fixes by pre-commit --- .pre-commit-config.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..3a121152 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,20 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-xml + - id: check-merge-conflict + - id: mixed-line-ending + - id: end-of-file-fixer + - id: trailing-whitespace +# autopep8 for Python formatting +- repo: https://github.com/hhatto/autopep8 + rev: v2.1.0 + hooks: + - id: autopep8 + args: [--recursive --diff --aggressive --aggressive --exit-code --ignore E402 --max-line-length 120 .] +# isort for python imports +- repo: https://github.com/pycqa/isort + rev: 5.12.0 + hooks: + - id: isort From 7ce6b34b478fbd4f834f442d4e82a258a95f54cd Mon Sep 17 00:00:00 2001 From: Ishaan Desai Date: Tue, 2 Apr 2024 15:24:59 +0200 Subject: [PATCH 03/41] Fix initialize() call of MicroSimulation class when adaptivity is off. Also fix formatter --- .github/workflows/check-pep8.yml | 22 --- .pre-commit-config.yaml | 9 +- micro_manager/micro_manager.py | 300 ++++++++++++++++++++++--------- 3 files changed, 220 insertions(+), 111 deletions(-) delete mode 100644 .github/workflows/check-pep8.yml diff --git a/.github/workflows/check-pep8.yml b/.github/workflows/check-pep8.yml deleted file mode 100644 index 46ef5313..00000000 --- a/.github/workflows/check-pep8.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: autopep8 -on: - push: - branches: - - main - - develop - pull_request: - branches: - - "*" -jobs: - autopep8: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: autopep8 - id: autopep8 - uses: peter-evans/autopep8@v1 - with: - args: --recursive --diff --aggressive --aggressive --exit-code --ignore E402 --max-line-length 120 . - - name: Fail if autopep8 made changes - if: ${{ steps.autopep8.outputs.exit-code == 2 }} - run: exit 1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3a121152..fdc41dbd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,12 +7,11 @@ repos: - id: mixed-line-ending - id: end-of-file-fixer - id: trailing-whitespace -# autopep8 for Python formatting -- repo: https://github.com/hhatto/autopep8 - rev: v2.1.0 +# black repo for python formatting +- repo: https://github.com/ambv/black + rev: 22.12.0 hooks: - - id: autopep8 - args: [--recursive --diff --aggressive --aggressive --exit-code --ignore E402 --max-line-length 120 .] + - id: black # isort for python imports - repo: https://github.com/pycqa/isort rev: 5.12.0 diff --git a/micro_manager/micro_manager.py b/micro_manager/micro_manager.py index 704e8196..5d7089df 100644 --- a/micro_manager/micro_manager.py +++ b/micro_manager/micro_manager.py @@ -12,23 +12,24 @@ """ import argparse +import importlib +import logging import os import sys -import precice -from mpi4py import MPI -import numpy as np -import logging import time -import importlib from copy import deepcopy from typing import Dict from warnings import warn -from .config import Config -from .micro_simulation import create_simulation_class -from .adaptivity.local_adaptivity import LocalAdaptivityCalculator +import numpy as np +import precice +from mpi4py import MPI + from .adaptivity.global_adaptivity import GlobalAdaptivityCalculator +from .adaptivity.local_adaptivity import LocalAdaptivityCalculator +from .config import Config from .domain_decomposition import DomainDecomposer +from .micro_simulation import create_simulation_class sys.path.append(os.getcwd()) @@ -51,11 +52,13 @@ def __init__(self, config_file: str) -> None: self._logger.setLevel(level=logging.INFO) # Create file handler which logs messages - fh = logging.FileHandler('micro-manager.log') + fh = logging.FileHandler("micro-manager.log") fh.setLevel(logging.INFO) # Create formatter and add it to handlers - formatter = logging.Formatter('[' + str(self._rank) + '] %(name)s - %(levelname)s - %(message)s') + formatter = logging.Formatter( + "[" + str(self._rank) + "] %(name)s - %(levelname)s - %(message)s" + ) fh.setFormatter(formatter) self._logger.addHandler(fh) # add the handlers to the logger @@ -67,10 +70,8 @@ def __init__(self, config_file: str) -> None: # Define the preCICE Participant self._participant = precice.Participant( - "Micro-Manager", - self._config.get_config_file_name(), - self._rank, - self._size) + "Micro-Manager", self._config.get_config_file_name(), self._rank, self._size + ) self._macro_mesh_name = self._config.get_macro_mesh_name() @@ -114,7 +115,9 @@ def __init__(self, config_file: str) -> None: if name in self._write_data_names: self._adaptivity_micro_data_names[name] = is_data_vector - self._adaptivity_in_every_implicit_step = self._config.is_adaptivity_required_in_every_implicit_iteration() + self._adaptivity_in_every_implicit_step = ( + self._config.is_adaptivity_required_in_every_implicit_iteration() + ) self._micro_sims_active_steps = None self._initialize() @@ -139,20 +142,35 @@ def solve(self) -> None: if self._is_adaptivity_on: similarity_dists = np.zeros( - (self._number_of_sims_for_adaptivity, - self._number_of_sims_for_adaptivity)) + ( + self._number_of_sims_for_adaptivity, + self._number_of_sims_for_adaptivity, + ) + ) # Start adaptivity calculation with all sims active is_sim_active = np.array([True] * self._number_of_sims_for_adaptivity) # Active sims do not have an associated sim - sim_is_associated_to = np.full((self._number_of_sims_for_adaptivity), -2, dtype=np.intc) + sim_is_associated_to = np.full( + (self._number_of_sims_for_adaptivity), -2, dtype=np.intc + ) # If micro simulations have been initialized, compute adaptivity based on initial data if self._micro_sims_init: # Compute adaptivity based on initial data of micro sims - similarity_dists, is_sim_active, sim_is_associated_to = self._adaptivity_controller.compute_adaptivity( - self._dt, self._micro_sims, similarity_dists, is_sim_active, sim_is_associated_to, self._data_for_adaptivity) + ( + similarity_dists, + is_sim_active, + sim_is_associated_to, + ) = self._adaptivity_controller.compute_adaptivity( + self._dt, + self._micro_sims, + similarity_dists, + is_sim_active, + sim_is_associated_to, + self._data_for_adaptivity, + ) while self._participant.is_coupling_ongoing(): # Write a checkpoint @@ -164,8 +182,18 @@ def solve(self) -> None: if self._is_adaptivity_on: if not self._adaptivity_in_every_implicit_step: - similarity_dists, is_sim_active, sim_is_associated_to = self._adaptivity_controller.compute_adaptivity( - self._dt, self._micro_sims, similarity_dists, is_sim_active, sim_is_associated_to, self._data_for_adaptivity) + ( + similarity_dists, + is_sim_active, + sim_is_associated_to, + ) = self._adaptivity_controller.compute_adaptivity( + self._dt, + self._micro_sims, + similarity_dists, + is_sim_active, + sim_is_associated_to, + self._data_for_adaptivity, + ) # Only checkpoint the adaptivity configuration if adaptivity is computed # once in every time window @@ -177,7 +205,13 @@ def solve(self) -> None: active_sim_ids = np.where(is_sim_active)[0] elif self._adaptivity_type == "global": active_sim_ids = np.where( - is_sim_active[self._global_ids_of_local_sims[0]:self._global_ids_of_local_sims[-1] + 1])[0] + is_sim_active[ + self._global_ids_of_local_sims[ + 0 + ] : self._global_ids_of_local_sims[-1] + + 1 + ] + )[0] for active_id in active_sim_ids: self._micro_sims_active_steps[active_id] += 1 @@ -186,20 +220,37 @@ def solve(self) -> None: if self._is_adaptivity_on: if self._adaptivity_in_every_implicit_step: - similarity_dists, is_sim_active, sim_is_associated_to = self._adaptivity_controller.compute_adaptivity( - self._dt, self._micro_sims, similarity_dists, is_sim_active, sim_is_associated_to, self._data_for_adaptivity) + ( + similarity_dists, + is_sim_active, + sim_is_associated_to, + ) = self._adaptivity_controller.compute_adaptivity( + self._dt, + self._micro_sims, + similarity_dists, + is_sim_active, + sim_is_associated_to, + self._data_for_adaptivity, + ) if self._adaptivity_type == "local": active_sim_ids = np.where(is_sim_active)[0] elif self._adaptivity_type == "global": active_sim_ids = np.where( - is_sim_active[self._global_ids_of_local_sims[0]:self._global_ids_of_local_sims[-1] + 1])[0] + is_sim_active[ + self._global_ids_of_local_sims[ + 0 + ] : self._global_ids_of_local_sims[-1] + + 1 + ] + )[0] for active_id in active_sim_ids: self._micro_sims_active_steps[active_id] += 1 micro_sims_output = self._solve_micro_simulations_with_adaptivity( - micro_sims_input, is_sim_active, sim_is_associated_to) + micro_sims_input, is_sim_active, sim_is_associated_to + ) else: micro_sims_output = self._solve_micro_simulations(micro_sims_input) @@ -226,8 +277,13 @@ def solve(self) -> None: sim_is_associated_to = np.copy(sim_is_associated_to_cp) else: # Time window has converged, now micro output can be generated - self._logger.info("Micro simulations {} - {} have converged at t = {}".format( - self._micro_sims[0].get_global_id(), self._micro_sims[-1].get_global_id(), t)) + self._logger.info( + "Micro simulations {} - {} have converged at t = {}".format( + self._micro_sims[0].get_global_id(), + self._micro_sims[-1].get_global_id(), + t, + ) + ) if self._micro_sims_have_output: if n % self._micro_n_out == 0: @@ -251,31 +307,46 @@ def _initialize(self) -> None: """ # Decompose the macro-domain and set the mesh access region for each partition in preCICE assert len(self._macro_bounds) / 2 == self._participant.get_mesh_dimensions( - self._macro_mesh_name), "Provided macro mesh bounds are of incorrect dimension" + self._macro_mesh_name + ), "Provided macro mesh bounds are of incorrect dimension" if self._is_parallel: domain_decomposer = DomainDecomposer( - self._logger, self._participant.get_mesh_dimensions(self._macro_mesh_name), self._rank, self._size) - coupling_mesh_bounds = domain_decomposer.decompose_macro_domain(self._macro_bounds, self._ranks_per_axis) + self._logger, + self._participant.get_mesh_dimensions(self._macro_mesh_name), + self._rank, + self._size, + ) + coupling_mesh_bounds = domain_decomposer.decompose_macro_domain( + self._macro_bounds, self._ranks_per_axis + ) else: coupling_mesh_bounds = self._macro_bounds - self._participant.set_mesh_access_region(self._macro_mesh_name, coupling_mesh_bounds) + self._participant.set_mesh_access_region( + self._macro_mesh_name, coupling_mesh_bounds + ) # initialize preCICE self._participant.initialize() - self._mesh_vertex_ids, mesh_vertex_coords = self._participant.get_mesh_vertex_ids_and_coordinates( - self._macro_mesh_name) - assert (mesh_vertex_coords.size != 0), "Macro mesh has no vertices." + ( + self._mesh_vertex_ids, + mesh_vertex_coords, + ) = self._participant.get_mesh_vertex_ids_and_coordinates(self._macro_mesh_name) + assert mesh_vertex_coords.size != 0, "Macro mesh has no vertices." self._local_number_of_sims, _ = mesh_vertex_coords.shape - self._logger.info("Number of local micro simulations = {}".format(self._local_number_of_sims)) + self._logger.info( + "Number of local micro simulations = {}".format(self._local_number_of_sims) + ) if self._local_number_of_sims == 0: if self._is_parallel: self._logger.info( "Rank {} has no micro simulations and hence will not do any computation.".format( - self._rank)) + self._rank + ) + ) self._is_rank_empty = True else: raise Exception("Micro Manager has no micro simulations.") @@ -293,13 +364,20 @@ def _initialize(self) -> None: for name, is_data_vector in self._adaptivity_data_names.items(): if is_data_vector: self._data_for_adaptivity[name] = np.zeros( - (self._local_number_of_sims, self._participant.get_data_dimensions( - self._macro_mesh_name, name))) + ( + self._local_number_of_sims, + self._participant.get_data_dimensions( + self._macro_mesh_name, name + ), + ) + ) else: - self._data_for_adaptivity[name] = np.zeros((self._local_number_of_sims)) + self._data_for_adaptivity[name] = np.zeros( + (self._local_number_of_sims) + ) # Create lists of local and global IDs - sim_id = np.sum(nms_all_ranks[:self._rank]) + sim_id = np.sum(nms_all_ranks[: self._rank]) self._global_ids_of_local_sims = [] # DECLARATION for i in range(self._local_number_of_sims): self._global_ids_of_local_sims.append(sim_id) @@ -309,22 +387,28 @@ def _initialize(self) -> None: micro_problem = getattr( importlib.import_module( - self._config.get_micro_file_name(), - "MicroSimulation"), - "MicroSimulation") + self._config.get_micro_file_name(), "MicroSimulation" + ), + "MicroSimulation", + ) # Create micro simulation objects for i in range(self._local_number_of_sims): - self._micro_sims[i] = create_simulation_class( - micro_problem)(self._global_ids_of_local_sims[i]) + self._micro_sims[i] = create_simulation_class(micro_problem)( + self._global_ids_of_local_sims[i] + ) - self._logger.info("Micro simulations with global IDs {} - {} created.".format( - self._global_ids_of_local_sims[0], self._global_ids_of_local_sims[-1])) + self._logger.info( + "Micro simulations with global IDs {} - {} created.".format( + self._global_ids_of_local_sims[0], self._global_ids_of_local_sims[-1] + ) + ) if self._is_adaptivity_on: if self._adaptivity_type == "local": self._adaptivity_controller = LocalAdaptivityCalculator( - self._config, self._logger) + self._config, self._logger + ) self._number_of_sims_for_adaptivity = self._local_number_of_sims elif self._adaptivity_type == "global": self._adaptivity_controller = GlobalAdaptivityCalculator( @@ -333,7 +417,8 @@ def _initialize(self) -> None: self._global_number_of_sims, self._global_ids_of_local_sims, self._rank, - self._comm) + self._comm, + ) self._number_of_sims_for_adaptivity = self._global_number_of_sims self._micro_sims_active_steps = np.zeros(self._local_number_of_sims) @@ -341,15 +426,25 @@ def _initialize(self) -> None: self._micro_sims_init = False # DECLARATION # Get initial data from micro simulations if initialize() method exists - if hasattr(micro_problem, 'initialize') and callable(getattr(micro_problem, 'initialize')): - if self._is_adaptivity_on: - self._micro_sims_init = True - initial_micro_output = self._micro_sims[0].initialize() # Call initialize() of the first simulation - if initial_micro_output is None: # Check if the detected initialize() method returns any data - warn("The initialize() call of the Micro simulation has not returned any initial data." - " The initialize call is stopped.") - self._micro_sims_init = False - else: + if hasattr(micro_problem, "initialize") and callable( + getattr(micro_problem, "initialize") + ): + self._micro_sims_init = True + initial_micro_output = self._micro_sims[ + 0 + ].initialize() # Call initialize() of the first simulation + if ( + initial_micro_output is None + ): # Check if the detected initialize() method returns any data + warn( + "The initialize() call of the Micro simulation has not returned any initial data." + " This means that the initialize() call has no effect on the adaptivity." + ) + self._micro_sims_init = False + for i in range(1, self._local_number_of_sims): + self._micro_sims[i].initialize() + else: + if self._is_adaptivity_on: # Save initial data from first micro simulation as we anyway have it for name in initial_micro_output.keys(): self._data_for_adaptivity[name][0] = initial_micro_output[name] @@ -358,13 +453,17 @@ def _initialize(self) -> None: for i in range(1, self._local_number_of_sims): initial_micro_output = self._micro_sims[i].initialize() for name in self._adaptivity_micro_data_names: - self._data_for_adaptivity[name][i] = initial_micro_output[name] - else: - self._logger.info( - "Micro simulation has the method initialize(), but it is not called, because adaptivity is off.") + self._data_for_adaptivity[name][i] = initial_micro_output[ + name + ] + else: + for i in range(1, self._local_number_of_sims): + self._micro_sims[i].initialize() self._micro_sims_have_output = False - if hasattr(micro_problem, 'output') and callable(getattr(micro_problem, 'output')): + if hasattr(micro_problem, "output") and callable( + getattr(micro_problem, "output") + ): self._micro_sims_have_output = True self._dt = self._participant.get_max_time_step_size() @@ -383,8 +482,13 @@ def _read_data_from_precice(self) -> list: read_data[name] = [] for name in self._read_data_names.keys(): - read_data.update({name: self._participant.read_data( - self._macro_mesh_name, name, self._mesh_vertex_ids, self._dt)}) + read_data.update( + { + name: self._participant.read_data( + self._macro_mesh_name, name, self._mesh_vertex_ids, self._dt + ) + } + ) if self._is_adaptivity_on: if name in self._adaptivity_macro_data_names: @@ -412,10 +516,16 @@ def _write_data_to_precice(self, data: list) -> None: for dname in self._write_data_names.keys(): self._participant.write_data( - self._macro_mesh_name, dname, self._mesh_vertex_ids, data_dict[dname]) + self._macro_mesh_name, + dname, + self._mesh_vertex_ids, + data_dict[dname], + ) else: for dname in self._write_data_names.keys(): - self._participant.write_data(self._macro_mesh_name, dname, [], np.array([])) + self._participant.write_data( + self._macro_mesh_name, dname, [], np.array([]) + ) def _solve_micro_simulations(self, micro_sims_input: list) -> list: """ @@ -446,10 +556,11 @@ def _solve_micro_simulations(self, micro_sims_input: list) -> list: return micro_sims_output def _solve_micro_simulations_with_adaptivity( - self, - micro_sims_input: list, - is_sim_active: np.ndarray, - sim_is_associated_to: np.ndarray) -> list: + self, + micro_sims_input: list, + is_sim_active: np.ndarray, + sim_is_associated_to: np.ndarray, + ) -> list: """ Solve all micro simulations and assemble the micro simulations outputs in a list of dicts format. @@ -471,9 +582,22 @@ def _solve_micro_simulations_with_adaptivity( """ if self._adaptivity_type == "global": active_sim_ids = np.where( - is_sim_active[self._global_ids_of_local_sims[0]:self._global_ids_of_local_sims[-1] + 1])[0] + is_sim_active[ + self._global_ids_of_local_sims[0] : self._global_ids_of_local_sims[ + -1 + ] + + 1 + ] + )[0] inactive_sim_ids = np.where( - is_sim_active[self._global_ids_of_local_sims[0]:self._global_ids_of_local_sims[-1] + 1] == False)[0] + is_sim_active[ + self._global_ids_of_local_sims[0] : self._global_ids_of_local_sims[ + -1 + ] + + 1 + ] + == False + )[0] elif self._adaptivity_type == "local": active_sim_ids = np.where(is_sim_active)[0] inactive_sim_ids = np.where(is_sim_active == False)[0] @@ -483,28 +607,37 @@ def _solve_micro_simulations_with_adaptivity( # Solve all active micro simulations for active_id in active_sim_ids: start_time = time.time() - micro_sims_output[active_id] = self._micro_sims[active_id].solve(micro_sims_input[active_id], self._dt) + micro_sims_output[active_id] = self._micro_sims[active_id].solve( + micro_sims_input[active_id], self._dt + ) end_time = time.time() # Mark the micro sim as active for export micro_sims_output[active_id]["active_state"] = 1 - micro_sims_output[active_id]["active_steps"] = self._micro_sims_active_steps[active_id] + micro_sims_output[active_id][ + "active_steps" + ] = self._micro_sims_active_steps[active_id] if self._is_micro_solve_time_required: micro_sims_output[active_id]["micro_sim_time"] = end_time - start_time # For each inactive simulation, copy data from most similar active simulation if self._adaptivity_type == "global": - self._adaptivity_controller.communicate_micro_output(is_sim_active, sim_is_associated_to, micro_sims_output) + self._adaptivity_controller.communicate_micro_output( + is_sim_active, sim_is_associated_to, micro_sims_output + ) elif self._adaptivity_type == "local": for inactive_id in inactive_sim_ids: micro_sims_output[inactive_id] = deepcopy( - micro_sims_output[sim_is_associated_to[inactive_id]]) + micro_sims_output[sim_is_associated_to[inactive_id]] + ) # Resolve micro sim output data for inactive simulations for inactive_id in inactive_sim_ids: micro_sims_output[inactive_id]["active_state"] = 0 - micro_sims_output[inactive_id]["active_steps"] = self._micro_sims_active_steps[inactive_id] + micro_sims_output[inactive_id][ + "active_steps" + ] = self._micro_sims_active_steps[inactive_id] if self._is_micro_solve_time_required: micro_sims_output[inactive_id]["micro_sim_time"] = 0 @@ -518,11 +651,10 @@ def _solve_micro_simulations_with_adaptivity( def main(): - parser = argparse.ArgumentParser(description='.') + parser = argparse.ArgumentParser(description=".") parser.add_argument( - 'config_file', - type=str, - help='Path to the JSON config file of the manager.') + "config_file", type=str, help="Path to the JSON config file of the manager." + ) args = parser.parse_args() config_file_path = args.config_file From 2ea70e3cadff697018552c5f7c7833c1b7a4c5d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Simonis?= Date: Wed, 10 Apr 2024 11:00:05 +0200 Subject: [PATCH 04/41] Fix doc image prefix (#94) * Fix image prefix * Add preCICE pre-commits --- .pre-commit-config.yaml | 7 +++++++ docs/README.md | 2 +- ...docs-tooling-micro-manager-manager-solution.png} | Bin 3 files changed, 8 insertions(+), 1 deletion(-) rename docs/images/{tooling-micro-manager-manager-solution.png => docs-tooling-micro-manager-manager-solution.png} (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fdc41dbd..8b92068d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,3 +17,10 @@ repos: rev: 5.12.0 hooks: - id: isort +- repo: https://github.com/precice/precice-pre-commit-hooks + rev: 'v3.3' + hooks: + - id: format-precice-config + files: "^.*/precice-config.xml" + - id: check-image-prefix + args: [ --prefix=docs-tooling-micro-manager- ] diff --git a/docs/README.md b/docs/README.md index 6fee76fc..02837369 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,7 +9,7 @@ summary: A tool to manage many micro simulations and couple them to a macro simu The Micro Manager manages many simulations on a micro scale and couples them to one simulation on a macro scale. For the coupling itself, it heavily relies on the coupling library [preCICE](https://precice.org/index.html). -![Micro Manager strategy schematic](images/tooling-micro-manager-manager-solution.png) +![Micro Manager strategy schematic](images/docs-tooling-micro-manager-manager-solution.png) ## What can it do? diff --git a/docs/images/tooling-micro-manager-manager-solution.png b/docs/images/docs-tooling-micro-manager-manager-solution.png similarity index 100% rename from docs/images/tooling-micro-manager-manager-solution.png rename to docs/images/docs-tooling-micro-manager-manager-solution.png From fa0e9ce90d8e9a7409dbe9514a99b1d2a8a557eb Mon Sep 17 00:00:00 2001 From: Ishaan Desai Date: Wed, 10 Apr 2024 13:17:56 +0200 Subject: [PATCH 05/41] Update GitHub Actions workflow files --- .github/workflows/check-links.yml | 14 ----- .github/workflows/check-markdown.yml | 13 ----- .github/workflows/pythonpublish.yml | 4 +- .github/workflows/run-adaptivity-test.yml | 8 +-- .github/workflows/run-checks.yml | 55 +++++++++++++++++++ .../run-domain-decomposition-tests.yml | 8 +-- .github/workflows/run-macro-micro-dummy.yml | 4 +- .github/workflows/run-unit-tests.yml | 2 +- 8 files changed, 68 insertions(+), 40 deletions(-) delete mode 100644 .github/workflows/check-links.yml delete mode 100644 .github/workflows/check-markdown.yml create mode 100644 .github/workflows/run-checks.yml diff --git a/.github/workflows/check-links.yml b/.github/workflows/check-links.yml deleted file mode 100644 index f7e0ce61..00000000 --- a/.github/workflows/check-links.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: Check links (manual) -on: workflow_dispatch -jobs: - check_links: - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@v2 - - name: Check links in markdown files (markdown-link-check) - uses: gaurav-nelson/github-action-markdown-link-check@v1 - with: - use-quiet-mode: 'yes' - use-verbose-mode: 'no' - config-file: '.markdown-link-check-config.json' diff --git a/.github/workflows/check-markdown.yml b/.github/workflows/check-markdown.yml deleted file mode 100644 index 39819fb4..00000000 --- a/.github/workflows/check-markdown.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Lint docs -on: [push, pull_request] -jobs: - check_md: - runs-on: ubuntu-latest - steps: - - name: Check out repository - uses: actions/checkout@v2 - - name: Lint markdown files (markdownlint) - uses: articulate/actions-markdownlint@v1 - with: - config: .markdownlint.json - files: '.' diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index 275817cd..6a11cc56 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -9,9 +9,9 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: python-version: '3.x' - name: Install dependencies diff --git a/.github/workflows/run-adaptivity-test.yml b/.github/workflows/run-adaptivity-test.yml index f708a48e..f99e518b 100644 --- a/.github/workflows/run-adaptivity-test.yml +++ b/.github/workflows/run-adaptivity-test.yml @@ -7,14 +7,14 @@ on: pull_request: branches: - "*" -jobs: +jobs: adaptivity_integration_tests: name: Run adaptivity integration tests runs-on: ubuntu-latest container: precice/precice:nightly steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: micro-manager @@ -50,7 +50,7 @@ jobs: container: precice/precice:nightly steps: - name: Checkout Repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: micro-manager @@ -76,7 +76,7 @@ jobs: container: precice/precice:nightly steps: - name: Checkout Repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: micro-manager diff --git a/.github/workflows/run-checks.yml b/.github/workflows/run-checks.yml new file mode 100644 index 00000000..7bc52561 --- /dev/null +++ b/.github/workflows/run-checks.yml @@ -0,0 +1,55 @@ +name: Run checks for markdown, links, and pre-commit +on: + push: + branches: + - main + - develop + pull_request: + branches: + - "*" +jobs: + check_md: + name: Lint markdown files + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + - name: Lint markdown files (markdownlint) + uses: articulate/actions-markdownlint@v1 + with: + config: .markdownlint.json + files: '.' + + check_links: + name: Check links in markdown files + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + - name: Check links in markdown files (markdown-link-check) + uses: gaurav-nelson/github-action-markdown-link-check@v1 + with: + use-quiet-mode: 'yes' + use-verbose-mode: 'no' + config-file: '.markdown-link-check-config.json' + + precommit: + name: pre-commit checks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + check-latest: true + - name: Install pre-commit + run: pip install pre-commit + - name: Run checks + run: pre-commit run -a -v + - name: Git status + if: always() + run: git status + - name: Full diff + if: always() + run: git diff diff --git a/.github/workflows/run-domain-decomposition-tests.yml b/.github/workflows/run-domain-decomposition-tests.yml index b44745ac..807a9661 100644 --- a/.github/workflows/run-domain-decomposition-tests.yml +++ b/.github/workflows/run-domain-decomposition-tests.yml @@ -7,14 +7,14 @@ on: pull_request: branches: - "*" -jobs: +jobs: domain_decomposition_integration_tests: name: Run domain decomposition integration tests runs-on: ubuntu-latest container: precice/precice:nightly steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: micro-manager @@ -42,7 +42,7 @@ jobs: timeout-minutes: 3 working-directory: micro-manager/tests/integration/test_unit_cube run: mpiexec -n 2 --allow-run-as-root python3 run_micro_manager.py --config micro-manager-config-parallel-1.json & python3 unit_cube.py - + - name: Run integration test (variant 2) timeout-minutes: 3 working-directory: micro-manager/tests/integration/test_unit_cube @@ -54,7 +54,7 @@ jobs: container: precice/precice:nightly steps: - name: Checkout Repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: micro-manager diff --git a/.github/workflows/run-macro-micro-dummy.yml b/.github/workflows/run-macro-micro-dummy.yml index c615fd75..9ac65245 100644 --- a/.github/workflows/run-macro-micro-dummy.yml +++ b/.github/workflows/run-macro-micro-dummy.yml @@ -7,7 +7,7 @@ on: pull_request: branches: - "*" -jobs: +jobs: run_dummy: name: Run dummy runs-on: ubuntu-latest @@ -15,7 +15,7 @@ jobs: steps: - name: Checkout Repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: micro-manager diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 420f3fd5..72b04c72 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest container: precice/precice:nightly steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: path: micro-manager From cad224bb07b4e2d72d0779923b15c3571efc9e9a Mon Sep 17 00:00:00 2001 From: Ishaan Desai Date: Wed, 10 Apr 2024 13:30:40 +0200 Subject: [PATCH 06/41] Add .html pattern to ignore in markdown link check --- .markdown-link-check-config.json | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.markdown-link-check-config.json b/.markdown-link-check-config.json index 3fff32c2..b539fcdd 100644 --- a/.markdown-link-check-config.json +++ b/.markdown-link-check-config.json @@ -1,3 +1,8 @@ { - "aliveStatusCodes": [429, 200] -} \ No newline at end of file + "aliveStatusCodes": [429, 200], + "ignorePatterns": [ + { + "pattern": "*.html" + } + ] +} From a564c36fc93fb868d61500db6fa6cf8923bfab66 Mon Sep 17 00:00:00 2001 From: Ishaan Desai Date: Wed, 10 Apr 2024 13:53:29 +0200 Subject: [PATCH 07/41] Files changes due to pre-commit run --- examples/cpp-dummy/.gitignore | 2 +- examples/cpp-dummy/run_micro_manager.py | 7 +- examples/macro_dummy.py | 38 +++- examples/precice-config-adaptivity.xml | 2 +- examples/precice-config.xml | 103 +++++------ examples/python-dummy/micro_dummy.py | 7 +- examples/python-dummy/run_micro_manager.py | 3 +- micro_manager/__init__.py | 2 +- micro_manager/adaptivity/adaptivity.py | 72 +++++--- micro_manager/adaptivity/global_adaptivity.py | 149 ++++++++++----- micro_manager/adaptivity/local_adaptivity.py | 58 ++++-- micro_manager/config.py | 94 +++++++--- micro_manager/domain_decomposition.py | 20 +- micro_manager/micro_simulation.py | 1 + setup.py | 45 ++--- .../integration/test_unit_cube/micro_dummy.py | 7 +- .../test_unit_cube/precice-config.xml | 111 ++++++----- .../test_unit_cube/run_micro_manager.py | 3 +- tests/integration/test_unit_cube/unit_cube.py | 8 +- tests/unit/.gitignore | 2 +- tests/unit/precice.py | 5 +- tests/unit/test_adaptivity_parallel.py | 80 ++++---- tests/unit/test_adaptivity_serial.py | 172 ++++++++++++++---- tests/unit/test_domain_decomposition.py | 25 ++- tests/unit/test_micro_manager.py | 69 ++++--- 25 files changed, 707 insertions(+), 378 deletions(-) diff --git a/examples/cpp-dummy/.gitignore b/examples/cpp-dummy/.gitignore index 9604e78e..e181e156 100644 --- a/examples/cpp-dummy/.gitignore +++ b/examples/cpp-dummy/.gitignore @@ -8,4 +8,4 @@ precice-run/ __pycache__ # Compiled files -*.so \ No newline at end of file +*.so diff --git a/examples/cpp-dummy/run_micro_manager.py b/examples/cpp-dummy/run_micro_manager.py index 6c7336bd..aad6f031 100644 --- a/examples/cpp-dummy/run_micro_manager.py +++ b/examples/cpp-dummy/run_micro_manager.py @@ -2,11 +2,14 @@ Script to run the Micro Manager """ -from micro_manager import MicroManager from argparse import ArgumentParser +from micro_manager import MicroManager + parser = ArgumentParser() -parser.add_argument("--config", required=True, help="Path to the micro manager configuration file") +parser.add_argument( + "--config", required=True, help="Path to the micro manager configuration file" +) args = parser.parse_args() manager = MicroManager(args.config) diff --git a/examples/macro_dummy.py b/examples/macro_dummy.py index acb3e56d..8cf5319c 100644 --- a/examples/macro_dummy.py +++ b/examples/macro_dummy.py @@ -33,19 +33,27 @@ def main(): vertex_ids = interface.set_mesh_vertices(read_mesh_name, coords) write_scalar_data = np.zeros(nv) - write_vector_data = np.zeros((nv, interface.get_data_dimensions(write_mesh_name, "macro-vector-data"))) + write_vector_data = np.zeros( + (nv, interface.get_data_dimensions(write_mesh_name, "macro-vector-data")) + ) for i in range(nv): write_scalar_data[i] = i - for d in range(interface.get_data_dimensions(write_mesh_name, "macro-vector-data")): + for d in range( + interface.get_data_dimensions(write_mesh_name, "macro-vector-data") + ): write_vector_data[i, d] = i if interface.requires_initial_data(): for name, dim in write_data_names.items(): if dim == 0: - interface.write_data(write_mesh_name, name, vertex_ids, write_scalar_data) + interface.write_data( + write_mesh_name, name, vertex_ids, write_scalar_data + ) elif dim == 1: - interface.write_data(write_mesh_name, name, vertex_ids, write_vector_data) + interface.write_data( + write_mesh_name, name, vertex_ids, write_vector_data + ) # initialize preCICE interface.initialize() @@ -61,13 +69,19 @@ def main(): for name, dim in read_data_names.items(): if dim == 0: - read_scalar_data = interface.read_data(read_mesh_name, name, vertex_ids, 1) + read_scalar_data = interface.read_data( + read_mesh_name, name, vertex_ids, 1 + ) elif dim == 1: - read_vector_data = interface.read_data(read_mesh_name, name, vertex_ids, 1) + read_vector_data = interface.read_data( + read_mesh_name, name, vertex_ids, 1 + ) write_scalar_data[:] = read_scalar_data[:] for i in range(nv): - for d in range(interface.get_data_dimensions(read_mesh_name, "micro-vector-data")): + for d in range( + interface.get_data_dimensions(read_mesh_name, "micro-vector-data") + ): write_vector_data[i, d] = read_vector_data[i, d] if t > 1: # to trigger adaptivity after some time # ensure that the data is different from the previous time step @@ -76,9 +90,13 @@ def main(): for name, dim in write_data_names.items(): if dim == 0: - interface.write_data(write_mesh_name, name, vertex_ids, write_scalar_data) + interface.write_data( + write_mesh_name, name, vertex_ids, write_scalar_data + ) elif dim == 1: - interface.write_data(write_mesh_name, name, vertex_ids, write_vector_data) + interface.write_data( + write_mesh_name, name, vertex_ids, write_vector_data + ) # do the coupling interface.advance(dt) @@ -96,5 +114,5 @@ def main(): interface.finalize() -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/examples/precice-config-adaptivity.xml b/examples/precice-config-adaptivity.xml index 80f35c3f..1e4180b9 100644 --- a/examples/precice-config-adaptivity.xml +++ b/examples/precice-config-adaptivity.xml @@ -5,7 +5,7 @@ - + diff --git a/examples/precice-config.xml b/examples/precice-config.xml index 9bebd403..d89f648f 100644 --- a/examples/precice-config.xml +++ b/examples/precice-config.xml @@ -1,55 +1,52 @@ - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/python-dummy/micro_dummy.py b/examples/python-dummy/micro_dummy.py index eeed2984..638e2051 100644 --- a/examples/python-dummy/micro_dummy.py +++ b/examples/python-dummy/micro_dummy.py @@ -5,7 +5,6 @@ class MicroSimulation: - def __init__(self, sim_id): """ Constructor of MicroSimulation class. @@ -23,8 +22,10 @@ def solve(self, macro_data, dt): for d in range(self._dims): self._micro_vector_data.append(macro_data["macro-vector-data"][d] + 1) - return {"micro-scalar-data": self._micro_scalar_data.copy(), - "micro-vector-data": self._micro_vector_data.copy()} + return { + "micro-scalar-data": self._micro_scalar_data.copy(), + "micro-vector-data": self._micro_vector_data.copy(), + } def set_state(self, state): self._state = state diff --git a/examples/python-dummy/run_micro_manager.py b/examples/python-dummy/run_micro_manager.py index 3cffab3a..d6309a1d 100644 --- a/examples/python-dummy/run_micro_manager.py +++ b/examples/python-dummy/run_micro_manager.py @@ -2,9 +2,10 @@ Script to run the Micro Manager """ -from micro_manager import MicroManager from argparse import ArgumentParser +from micro_manager import MicroManager + parser = ArgumentParser() parser.add_argument("--config", help="Path to the micro manager configuration file") args = parser.parse_args() diff --git a/micro_manager/__init__.py b/micro_manager/__init__.py index a4d9bf6c..0f581ef4 100644 --- a/micro_manager/__init__.py +++ b/micro_manager/__init__.py @@ -1,2 +1,2 @@ -from .micro_manager import MicroManager from .config import Config +from .micro_manager import MicroManager diff --git a/micro_manager/adaptivity/adaptivity.py b/micro_manager/adaptivity/adaptivity.py index ae2b9411..a977cabd 100644 --- a/micro_manager/adaptivity/adaptivity.py +++ b/micro_manager/adaptivity/adaptivity.py @@ -2,11 +2,12 @@ Functionality for adaptive initialization and control of micro simulations """ import sys -import numpy as np from math import exp from typing import Callable from warnings import warn +import numpy as np + class AdaptivityCalculator: def __init__(self, configurator, logger) -> None: @@ -30,9 +31,13 @@ def __init__(self, configurator, logger) -> None: self._coarse_tol = 0.0 self._ref_tol = 0.0 - self._similarity_measure = self._get_similarity_measure(configurator.get_adaptivity_similarity_measure()) + self._similarity_measure = self._get_similarity_measure( + configurator.get_adaptivity_similarity_measure() + ) - def _get_similarity_dists(self, dt: float, similarity_dists: np.ndarray, data: dict) -> np.ndarray: + def _get_similarity_dists( + self, dt: float, similarity_dists: np.ndarray, data: dict + ) -> np.ndarray: """ Calculate metric which determines if two micro simulations are similar enough to have one of them deactivated. @@ -66,9 +71,8 @@ def _get_similarity_dists(self, dt: float, similarity_dists: np.ndarray, data: d return exp(-self._hist_param * dt) * _similarity_dists + dt * data_diff def _update_active_sims( - self, - similarity_dists: np.ndarray, - is_sim_active: np.ndarray) -> np.ndarray: + self, similarity_dists: np.ndarray, is_sim_active: np.ndarray + ) -> np.ndarray: """ Update set of active micro simulations. Active micro simulations are compared to each other and if found similar, one of them is deactivated. @@ -88,12 +92,18 @@ def _update_active_sims( max_similarity_dist = np.amax(similarity_dists) if max_similarity_dist == 0.0: - warn("All similarity distances are zero, probably because all the data for adaptivity is the same. Coarsening tolerance will be manually set to minimum float number.") + warn( + "All similarity distances are zero, probably because all the data for adaptivity is the same. Coarsening tolerance will be manually set to minimum float number." + ) self._coarse_tol = sys.float_info.min else: - self._coarse_tol = self._coarse_const * self._refine_const * max_similarity_dist + self._coarse_tol = ( + self._coarse_const * self._refine_const * max_similarity_dist + ) - _is_sim_active = np.copy(is_sim_active) # Input is_sim_active is not longer used after this point + _is_sim_active = np.copy( + is_sim_active + ) # Input is_sim_active is not longer used after this point # Update the set of active micro sims for i in range(_is_sim_active.size): @@ -104,10 +114,11 @@ def _update_active_sims( return _is_sim_active def _associate_inactive_to_active( - self, - similarity_dists: np.ndarray, - is_sim_active: np.ndarray, - sim_is_associated_to: np.ndarray) -> np.ndarray: + self, + similarity_dists: np.ndarray, + is_sim_active: np.ndarray, + sim_is_associated_to: np.ndarray, + ) -> np.ndarray: """ Associate inactive micro simulations to most similar active micro simulation. @@ -144,10 +155,8 @@ def _associate_inactive_to_active( return _sim_is_associated_to def _check_for_activation( - self, - inactive_id: int, - similarity_dists: np.ndarray, - is_sim_active: np.ndarray) -> bool: + self, inactive_id: int, similarity_dists: np.ndarray, is_sim_active: np.ndarray + ) -> bool: """ Check if an inactive simulation needs to be activated. @@ -173,10 +182,8 @@ def _check_for_activation( return min(dists) > self._ref_tol def _check_for_deactivation( - self, - active_id: int, - similarity_dists: np.ndarray, - is_sim_active: np.ndarray) -> bool: + self, active_id: int, similarity_dists: np.ndarray, is_sim_active: np.ndarray + ) -> bool: """ Check if an active simulation needs to be deactivated. @@ -203,7 +210,9 @@ def _check_for_deactivation( return True return False - def _get_similarity_measure(self, similarity_measure: str) -> Callable[[np.ndarray], np.ndarray]: + def _get_similarity_measure( + self, similarity_measure: str + ) -> Callable[[np.ndarray], np.ndarray]: """ Get similarity measure to be used for similarity calculation @@ -217,17 +226,18 @@ def _get_similarity_measure(self, similarity_measure: str) -> Callable[[np.ndarr similarity_measure : function Function to be used for similarity calculation. Takes data as input and returns similarity measure """ - if similarity_measure == 'L1': + if similarity_measure == "L1": return self._l1 - elif similarity_measure == 'L2': + elif similarity_measure == "L2": return self._l2 - elif similarity_measure == 'L1rel': + elif similarity_measure == "L1rel": return self._l1rel - elif similarity_measure == 'L2rel': + elif similarity_measure == "L2rel": return self._l2rel else: raise ValueError( - 'Similarity measure not supported. Currently supported similarity measures are "L1", "L2", "L1rel", "L2rel".') + 'Similarity measure not supported. Currently supported similarity measures are "L1", "L2", "L1rel", "L2rel".' + ) def _l1(self, data: np.ndarray) -> np.ndarray: """ @@ -279,7 +289,9 @@ def _l1rel(self, data: np.ndarray) -> np.ndarray: pointwise_diff = data[np.newaxis, :] - data[:, np.newaxis] # divide by data to get relative difference # divide i,j by max(data[i],data[j]) to get relative difference - relative = np.nan_to_num((pointwise_diff / np.maximum(data[np.newaxis, :], data[:, np.newaxis]))) + relative = np.nan_to_num( + (pointwise_diff / np.maximum(data[np.newaxis, :], data[:, np.newaxis])) + ) return np.linalg.norm(relative, ord=1, axis=-1) def _l2rel(self, data: np.ndarray) -> np.ndarray: @@ -300,5 +312,7 @@ def _l2rel(self, data: np.ndarray) -> np.ndarray: pointwise_diff = data[np.newaxis, :] - data[:, np.newaxis] # divide by data to get relative difference # divide i,j by max(data[i],data[j]) to get relative difference - relative = np.nan_to_num((pointwise_diff / np.maximum(data[np.newaxis, :], data[:, np.newaxis]))) + relative = np.nan_to_num( + (pointwise_diff / np.maximum(data[np.newaxis, :], data[:, np.newaxis])) + ) return np.linalg.norm(relative, ord=2, axis=-1) diff --git a/micro_manager/adaptivity/global_adaptivity.py b/micro_manager/adaptivity/global_adaptivity.py index 155853a7..90690ec7 100644 --- a/micro_manager/adaptivity/global_adaptivity.py +++ b/micro_manager/adaptivity/global_adaptivity.py @@ -5,23 +5,26 @@ Note: All ID variables used in the methods of this class are global IDs, unless they have *local* in their name. """ -import numpy as np import hashlib from copy import deepcopy -from mpi4py import MPI from typing import Dict + +import numpy as np +from mpi4py import MPI + from .adaptivity import AdaptivityCalculator class GlobalAdaptivityCalculator(AdaptivityCalculator): def __init__( - self, - configurator, - logger, - global_number_of_sims: float, - global_ids: list, - rank: int, - comm) -> None: + self, + configurator, + logger, + global_number_of_sims: float, + global_ids: list, + rank: int, + comm, + ) -> None: """ Class constructor. @@ -52,7 +55,9 @@ def __init__( for i in range(local_number_of_sims): micro_sims_on_this_rank[i] = self._rank - self._rank_of_sim = np.zeros(global_number_of_sims, dtype=np.intc) # DECLARATION + self._rank_of_sim = np.zeros( + global_number_of_sims, dtype=np.intc + ) # DECLARATION self._comm.Allgatherv(micro_sims_on_this_rank, self._rank_of_sim) @@ -62,13 +67,14 @@ def __init__( self._is_sim_on_this_rank[i] = True def compute_adaptivity( - self, - dt: float, - micro_sims: list, - similarity_dists_nm1: np.ndarray, - is_sim_active_nm1: np.ndarray, - sim_is_associated_to_nm1: np.ndarray, - data_for_adaptivity: dict) -> tuple: + self, + dt: float, + micro_sims: list, + similarity_dists_nm1: np.ndarray, + is_sim_active_nm1: np.ndarray, + sim_is_associated_to_nm1: np.ndarray, + data_for_adaptivity: dict, + ) -> tuple: """ Compute adaptivity globally based on similarity distances and micro simulation states @@ -98,7 +104,9 @@ def compute_adaptivity( if name not in self._adaptivity_data_names: raise ValueError( "Data for adaptivity must be one of the following: {}".format( - self._adaptivity_data_names.keys())) + self._adaptivity_data_names.keys() + ) + ) # Gather adaptivity data from all ranks global_data_for_adaptivity = dict() @@ -106,29 +114,39 @@ def compute_adaptivity( data_as_list = self._comm.allgather(data_for_adaptivity[name]) global_data_for_adaptivity[name] = np.concatenate((data_as_list[:]), axis=0) - similarity_dists = self._get_similarity_dists(dt, similarity_dists_nm1, global_data_for_adaptivity) + similarity_dists = self._get_similarity_dists( + dt, similarity_dists_nm1, global_data_for_adaptivity + ) is_sim_active = self._update_active_sims(similarity_dists, is_sim_active_nm1) is_sim_active, sim_is_associated_to = self._update_inactive_sims( - similarity_dists, is_sim_active, sim_is_associated_to_nm1, micro_sims) + similarity_dists, is_sim_active, sim_is_associated_to_nm1, micro_sims + ) sim_is_associated_to = self._associate_inactive_to_active( - similarity_dists, is_sim_active, sim_is_associated_to) + similarity_dists, is_sim_active, sim_is_associated_to + ) self._logger.info( "{} active simulations, {} inactive simulations".format( np.count_nonzero( - is_sim_active[self._global_ids[0]:self._global_ids[-1] + 1]), + is_sim_active[self._global_ids[0] : self._global_ids[-1] + 1] + ), np.count_nonzero( - is_sim_active[self._global_ids[0]:self._global_ids[-1] + 1] == False))) + is_sim_active[self._global_ids[0] : self._global_ids[-1] + 1] + == False + ), + ) + ) return similarity_dists, is_sim_active, sim_is_associated_to def communicate_micro_output( - self, - is_sim_active: np.ndarray, - sim_is_associated_to: np.ndarray, - micro_output: list) -> None: + self, + is_sim_active: np.ndarray, + sim_is_associated_to: np.ndarray, + micro_output: list, + ) -> None: """ Communicate micro output from active simulation to their associated inactive simulations. Process to process (p2p) communication is done. @@ -144,9 +162,13 @@ def communicate_micro_output( micro_output : list List of dicts having individual output of each simulation. Only the active simulation outputs are entered. """ - inactive_local_ids = np.where(is_sim_active[self._global_ids[0]:self._global_ids[-1] + 1] == False)[0] + inactive_local_ids = np.where( + is_sim_active[self._global_ids[0] : self._global_ids[-1] + 1] == False + )[0] - local_sim_is_associated_to = sim_is_associated_to[self._global_ids[0]:self._global_ids[-1] + 1] + local_sim_is_associated_to = sim_is_associated_to[ + self._global_ids[0] : self._global_ids[-1] + 1 + ] # Keys are global IDs of active simulations associated to inactive # simulations on this rank. Values are global IDs of the inactive @@ -162,7 +184,9 @@ def communicate_micro_output( else: active_to_inactive_map[assoc_active_id] = [i] else: # If associated active simulation is on this rank, copy the output directly - micro_output[i] = deepcopy(micro_output[self._global_ids.index(assoc_active_id)]) + micro_output[i] = deepcopy( + micro_output[self._global_ids.index(assoc_active_id)] + ) assoc_active_ids = list(active_to_inactive_map.keys()) @@ -175,11 +199,12 @@ def communicate_micro_output( micro_output[local_id] = deepcopy(output) def _update_inactive_sims( - self, - similarity_dists: np.ndarray, - is_sim_active: np.ndarray, - sim_is_associated_to: np.ndarray, - micro_sims: list) -> tuple: + self, + similarity_dists: np.ndarray, + is_sim_active: np.ndarray, + sim_is_associated_to: np.ndarray, + micro_sims: list, + ) -> tuple: """ Update set of inactive micro simulations. Each inactive micro simulation is compared to all active ones and if it is not similar to any of them, it is activated. @@ -204,7 +229,9 @@ def _update_inactive_sims( """ self._ref_tol = self._refine_const * np.amax(similarity_dists) - _is_sim_active = np.copy(is_sim_active) # Input is_sim_active is not longer used after this point + _is_sim_active = np.copy( + is_sim_active + ) # Input is_sim_active is not longer used after this point _sim_is_associated_to = np.copy(sim_is_associated_to) _sim_is_associated_to_updated = np.copy(sim_is_associated_to) @@ -214,11 +241,15 @@ def _update_inactive_sims( if not _is_sim_active[i]: # if id is inactive if self._check_for_activation(i, similarity_dists, _is_sim_active): _is_sim_active[i] = True - _sim_is_associated_to_updated[i] = -2 # Active sim cannot have an associated sim + _sim_is_associated_to_updated[ + i + ] = -2 # Active sim cannot have an associated sim if self._is_sim_on_this_rank[i]: to_be_activated_ids.append(i) - local_sim_is_associated_to = _sim_is_associated_to[self._global_ids[0]:self._global_ids[-1] + 1] + local_sim_is_associated_to = _sim_is_associated_to[ + self._global_ids[0] : self._global_ids[-1] + 1 + ] # Keys are global IDs of active sims not on this rank, values are lists of local and # global IDs of inactive sims associated to the active sims which are on this rank @@ -230,20 +261,30 @@ def _update_inactive_sims( to_be_activated_local_id = self._global_ids.index(i) assoc_active_id = local_sim_is_associated_to[to_be_activated_local_id] - if self._is_sim_on_this_rank[assoc_active_id]: # Associated active simulation is on the same rank + if self._is_sim_on_this_rank[ + assoc_active_id + ]: # Associated active simulation is on the same rank assoc_active_local_id = self._global_ids.index(assoc_active_id) - micro_sims[to_be_activated_local_id].set_state(micro_sims[assoc_active_local_id].get_state()) + micro_sims[to_be_activated_local_id].set_state( + micro_sims[assoc_active_local_id].get_state() + ) else: # Associated active simulation is not on this rank if assoc_active_id in to_be_activated_map: - to_be_activated_map[assoc_active_id].append(to_be_activated_local_id) + to_be_activated_map[assoc_active_id].append( + to_be_activated_local_id + ) else: - to_be_activated_map[assoc_active_id] = [to_be_activated_local_id] + to_be_activated_map[assoc_active_id] = [ + to_be_activated_local_id + ] sim_states_and_global_ids = [] for sim in micro_sims: sim_states_and_global_ids.append((sim.get_state(), sim.get_global_id())) - recv_reqs = self._p2p_comm(list(to_be_activated_map.keys()), sim_states_and_global_ids) + recv_reqs = self._p2p_comm( + list(to_be_activated_map.keys()), sim_states_and_global_ids + ) # Use received micro sims to activate the required simulations for req in recv_reqs: @@ -273,7 +314,9 @@ def _create_tag(self, sim_id: int, src_rank: int, dest_rank: int) -> int: Unique tag. """ send_hashtag = hashlib.sha256() - send_hashtag.update((str(src_rank) + str(sim_id) + str(dest_rank)).encode('utf-8')) + send_hashtag.update( + (str(src_rank) + str(sim_id) + str(dest_rank)).encode("utf-8") + ) tag = int(send_hashtag.hexdigest()[:6], base=16) return tag @@ -294,9 +337,17 @@ def _p2p_comm(self, assoc_active_ids: list, data: list) -> list: recv_reqs : list List of MPI requests of receive operations. """ - send_map_local: Dict[int, int] = dict() # keys are global IDs, values are rank to send to - send_map: Dict[int, list] = dict() # keys are global IDs of sims to send, values are ranks to send the sims to - recv_map: Dict[int, int] = dict() # keys are global IDs to receive, values are ranks to receive from + send_map_local: Dict[ + int, int + ] = dict() # keys are global IDs, values are rank to send to + send_map: Dict[ + int, list + ] = ( + dict() + ) # keys are global IDs of sims to send, values are ranks to send the sims to + recv_map: Dict[ + int, int + ] = dict() # keys are global IDs to receive, values are ranks to receive from for i in assoc_active_ids: # Add simulation and its rank to receive map @@ -328,7 +379,9 @@ def _p2p_comm(self, assoc_active_ids: list, data: list) -> list: recv_reqs = [] for global_id, recv_rank in recv_map.items(): tag = self._create_tag(global_id, recv_rank, self._rank) - bufsize = 1 << 30 # allocate and use a temporary 1 MiB buffer size https://github.com/mpi4py/mpi4py/issues/389 + bufsize = ( + 1 << 30 + ) # allocate and use a temporary 1 MiB buffer size https://github.com/mpi4py/mpi4py/issues/389 req = self._comm.irecv(bufsize, source=recv_rank, tag=tag) recv_reqs.append(req) diff --git a/micro_manager/adaptivity/local_adaptivity.py b/micro_manager/adaptivity/local_adaptivity.py index 3fc45c2a..a2defb8c 100644 --- a/micro_manager/adaptivity/local_adaptivity.py +++ b/micro_manager/adaptivity/local_adaptivity.py @@ -4,6 +4,7 @@ each other. A global comparison is not done. """ import numpy as np + from .adaptivity import AdaptivityCalculator @@ -22,13 +23,14 @@ def __init__(self, configurator, logger) -> None: super().__init__(configurator, logger) def compute_adaptivity( - self, - dt, - micro_sims, - similarity_dists_nm1: np.ndarray, - is_sim_active_nm1: np.ndarray, - sim_is_associated_to_nm1: np.ndarray, - data_for_adaptivity: dict) -> tuple: + self, + dt, + micro_sims, + similarity_dists_nm1: np.ndarray, + is_sim_active_nm1: np.ndarray, + sim_is_associated_to_nm1: np.ndarray, + data_for_adaptivity: dict, + ) -> tuple: """ Compute adaptivity locally (within a rank). @@ -59,31 +61,41 @@ def compute_adaptivity( if name not in self._adaptivity_data_names: raise ValueError( "Data for adaptivity must be one of the following: {}".format( - self._adaptivity_data_names.keys())) + self._adaptivity_data_names.keys() + ) + ) - similarity_dists = self._get_similarity_dists(dt, similarity_dists_nm1, data_for_adaptivity) + similarity_dists = self._get_similarity_dists( + dt, similarity_dists_nm1, data_for_adaptivity + ) # Operation done globally if global adaptivity is chosen is_sim_active = self._update_active_sims(similarity_dists, is_sim_active_nm1) is_sim_active, sim_is_associated_to = self._update_inactive_sims( - similarity_dists, is_sim_active, sim_is_associated_to_nm1, micro_sims) + similarity_dists, is_sim_active, sim_is_associated_to_nm1, micro_sims + ) sim_is_associated_to = self._associate_inactive_to_active( - similarity_dists, is_sim_active, sim_is_associated_to) + similarity_dists, is_sim_active, sim_is_associated_to + ) self._logger.info( "{} active simulations, {} inactive simulations".format( - np.count_nonzero(is_sim_active), np.count_nonzero(is_sim_active == False))) + np.count_nonzero(is_sim_active), + np.count_nonzero(is_sim_active == False), + ) + ) return similarity_dists, is_sim_active, sim_is_associated_to def _update_inactive_sims( - self, - similarity_dists: np.ndarray, - is_sim_active: np.ndarray, - sim_is_associated_to: np.ndarray, - micro_sims: list) -> tuple: + self, + similarity_dists: np.ndarray, + is_sim_active: np.ndarray, + sim_is_associated_to: np.ndarray, + micro_sims: list, + ) -> tuple: """ Update set of inactive micro simulations. Each inactive micro simulation is compared to all active ones and if it is not similar to any of them, it is activated. @@ -108,7 +120,9 @@ def _update_inactive_sims( """ self._ref_tol = self._refine_const * np.amax(similarity_dists) - _is_sim_active = np.copy(is_sim_active) # Input is_sim_active is not longer used after this point + _is_sim_active = np.copy( + is_sim_active + ) # Input is_sim_active is not longer used after this point _sim_is_associated_to = np.copy(sim_is_associated_to) # Update the set of inactive micro sims @@ -116,8 +130,12 @@ def _update_inactive_sims( if not _is_sim_active[i]: # if id is inactive if self._check_for_activation(i, similarity_dists, _is_sim_active): associated_active_local_id = _sim_is_associated_to[i] - micro_sims[i].set_state(micro_sims[associated_active_local_id].get_state()) + micro_sims[i].set_state( + micro_sims[associated_active_local_id].get_state() + ) _is_sim_active[i] = True - _sim_is_associated_to[i] = -2 # Active sim cannot have an associated sim + _sim_is_associated_to[ + i + ] = -2 # Active sim cannot have an associated sim return _is_sim_active, _sim_is_associated_to diff --git a/micro_manager/config.py b/micro_manager/config.py index 897d95f0..700f70ac 100644 --- a/micro_manager/config.py +++ b/micro_manager/config.py @@ -64,36 +64,55 @@ def read_json(self, config_filename): data = json.load(read_file) # convert paths to python-importable paths - self._micro_file_name = data["micro_file_name"].replace("/", ".").replace("\\", ".").replace(".py", "") - - self._config_file_name = os.path.join(folder, data["coupling_params"]["config_file_name"]) + self._micro_file_name = ( + data["micro_file_name"] + .replace("/", ".") + .replace("\\", ".") + .replace(".py", "") + ) + + self._config_file_name = os.path.join( + folder, data["coupling_params"]["config_file_name"] + ) self._macro_mesh_name = data["coupling_params"]["macro_mesh_name"] try: self._write_data_names = data["coupling_params"]["write_data_names"] - assert isinstance(self._write_data_names, dict), "Write data entry is not a dictionary" + assert isinstance( + self._write_data_names, dict + ), "Write data entry is not a dictionary" for key, value in self._write_data_names.items(): if value == "scalar": self._write_data_names[key] = False elif value == "vector": self._write_data_names[key] = True else: - raise Exception("Write data dictionary as a value other than 'scalar' or 'vector'") + raise Exception( + "Write data dictionary as a value other than 'scalar' or 'vector'" + ) except BaseException: - self._logger.info("No write data names provided. Micro manager will only read data from preCICE.") + self._logger.info( + "No write data names provided. Micro manager will only read data from preCICE." + ) try: self._read_data_names = data["coupling_params"]["read_data_names"] - assert isinstance(self._read_data_names, dict), "Read data entry is not a dictionary" + assert isinstance( + self._read_data_names, dict + ), "Read data entry is not a dictionary" for key, value in self._read_data_names.items(): if value == "scalar": self._read_data_names[key] = False elif value == "vector": self._read_data_names[key] = True else: - raise Exception("Read data dictionary as a value other than 'scalar' or 'vector'") + raise Exception( + "Read data dictionary as a value other than 'scalar' or 'vector'" + ) except BaseException: - self._logger.info("No read data names provided. Micro manager will only write data to preCICE.") + self._logger.info( + "No read data names provided. Micro manager will only write data to preCICE." + ) self._macro_domain_bounds = data["simulation_params"]["macro_domain_bounds"] @@ -101,7 +120,8 @@ def read_json(self, config_filename): self._ranks_per_axis = data["simulation_params"]["decomposition"] except BaseException: self._logger.info( - "Domain decomposition is not specified, so the Micro Manager will expect to be run in serial.") + "Domain decomposition is not specified, so the Micro Manager will expect to be run in serial." + ) try: if data["simulation_params"]["adaptivity"]: @@ -110,7 +130,8 @@ def read_json(self, config_filename): self._adaptivity = False except BaseException: self._logger.info( - "Micro Manager will not adaptively run micro simulations, but instead will run all micro simulations in all time steps.") + "Micro Manager will not adaptively run micro simulations, but instead will run all micro simulations in all time steps." + ) if self._adaptivity: if data["simulation_params"]["adaptivity"]["type"] == "local": @@ -128,19 +149,32 @@ def read_json(self, config_filename): warn( "Only micro simulation data is used for similarity computation in adaptivity. This would lead to the" " same set of active and inactive simulations for the entire simulation time. If this is not intended," - " please include macro simulation data as well.") - - self._adaptivity_history_param = data["simulation_params"]["adaptivity"]["history_param"] - self._adaptivity_coarsening_constant = data["simulation_params"]["adaptivity"]["coarsening_constant"] - self._adaptivity_refining_constant = data["simulation_params"]["adaptivity"]["refining_constant"] + " please include macro simulation data as well." + ) + + self._adaptivity_history_param = data["simulation_params"]["adaptivity"][ + "history_param" + ] + self._adaptivity_coarsening_constant = data["simulation_params"][ + "adaptivity" + ]["coarsening_constant"] + self._adaptivity_refining_constant = data["simulation_params"][ + "adaptivity" + ]["refining_constant"] if "similarity_measure" in data["simulation_params"]["adaptivity"]: - self._adaptivity_similarity_measure = data["simulation_params"]["adaptivity"]["similarity_measure"] + self._adaptivity_similarity_measure = data["simulation_params"][ + "adaptivity" + ]["similarity_measure"] else: - self._logger.info("No similarity measure provided, using L1 norm as default") + self._logger.info( + "No similarity measure provided, using L1 norm as default" + ) self._adaptivity_similarity_measure = "L1" - adaptivity_every_implicit_iteration = data["simulation_params"]["adaptivity"]["every_implicit_iteration"] + adaptivity_every_implicit_iteration = data["simulation_params"][ + "adaptivity" + ]["every_implicit_iteration"] if adaptivity_every_implicit_iteration == "True": self._adaptivity_every_implicit_iteration = True @@ -148,30 +182,39 @@ def read_json(self, config_filename): self._adaptivity_every_implicit_iteration = False if not self._adaptivity_every_implicit_iteration: - self._logger.info("Micro Manager will compute adaptivity once at the start of every time window") + self._logger.info( + "Micro Manager will compute adaptivity once at the start of every time window" + ) self._write_data_names["active_state"] = False self._write_data_names["active_steps"] = False try: diagnostics_data_names = data["diagnostics"]["data_from_micro_sims"] - assert isinstance(diagnostics_data_names, dict), "Diagnostics data is not a dictionary" + assert isinstance( + diagnostics_data_names, dict + ), "Diagnostics data is not a dictionary" for key, value in diagnostics_data_names.items(): if value == "scalar": self._write_data_names[key] = False elif value == "vector": self._write_data_names[key] = True else: - raise Exception("Diagnostics data dictionary as a value other than 'scalar' or 'vector'") + raise Exception( + "Diagnostics data dictionary as a value other than 'scalar' or 'vector'" + ) except BaseException: - self._logger.info("No diagnostics data is defined. Micro Manager will not output any diagnostics data.") + self._logger.info( + "No diagnostics data is defined. Micro Manager will not output any diagnostics data." + ) try: self._micro_output_n = data["diagnostics"]["micro_output_n"] except BaseException: self._logger.info( "Output interval of micro simulations not specified, if output is available then it will be called " - "in every time window.") + "in every time window." + ) try: if data["diagnostics"]["output_micro_sim_solve_time"]: @@ -179,7 +222,8 @@ def read_json(self, config_filename): self._write_data_names["micro_sim_time"] = False except BaseException: self._logger.info( - "Micro manager will not output time required to solve each micro simulation in each time step.") + "Micro manager will not output time required to solve each micro simulation in each time step." + ) def get_config_file_name(self): """ diff --git a/micro_manager/domain_decomposition.py b/micro_manager/domain_decomposition.py index 91fd6bf8..cfc7cbe7 100644 --- a/micro_manager/domain_decomposition.py +++ b/micro_manager/domain_decomposition.py @@ -47,12 +47,15 @@ def decompose_macro_domain(self, macro_bounds: list, ranks_per_axis: list) -> li List containing the upper and lower bounds of the domain pertaining to this rank. Format is same as input parameter macro_bounds. """ - assert np.prod( - ranks_per_axis) == self._size, "Total number of processors provided in the Micro Manager configuration and in the MPI execution command do not match." + assert ( + np.prod(ranks_per_axis) == self._size + ), "Total number of processors provided in the Micro Manager configuration and in the MPI execution command do not match." dx = [] for d in range(self._dims): - dx.append(abs(macro_bounds[d * 2 + 1] - macro_bounds[d * 2]) / ranks_per_axis[d]) + dx.append( + abs(macro_bounds[d * 2 + 1] - macro_bounds[d * 2]) / ranks_per_axis[d] + ) rank_in_axis: list[int] = [0] * self._dims if ranks_per_axis[0] == 1: # if serial in x axis @@ -69,13 +72,18 @@ def decompose_macro_domain(self, macro_bounds: list, ranks_per_axis: list) -> li if ranks_per_axis[2] == 1: # if serial in z axis rank_in_axis[2] = 0 else: - rank_in_axis[2] = int(self._rank / (ranks_per_axis[0] * ranks_per_axis[1])) # z axis + rank_in_axis[2] = int( + self._rank / (ranks_per_axis[0] * ranks_per_axis[1]) + ) # z axis if ranks_per_axis[1] == 1: # if serial in y axis rank_in_axis[1] = 0 else: - rank_in_axis[1] = (self._rank - ranks_per_axis[0] * ranks_per_axis[1] - * rank_in_axis[2]) % ranks_per_axis[2] # y axis + rank_in_axis[1] = ( + self._rank - ranks_per_axis[0] * ranks_per_axis[1] * rank_in_axis[2] + ) % ranks_per_axis[ + 2 + ] # y axis mesh_bounds = [] for d in range(self._dims): diff --git a/micro_manager/micro_simulation.py b/micro_manager/micro_simulation.py index 902eaba1..dad1c872 100644 --- a/micro_manager/micro_simulation.py +++ b/micro_manager/micro_simulation.py @@ -19,6 +19,7 @@ def create_simulation_class(micro_simulation_class): Simulation : class Definition of class Simulation defined in this function. """ + class Simulation(micro_simulation_class): def __init__(self, global_id): micro_simulation_class.__init__(self, global_id) diff --git a/setup.py b/setup.py index 9d2f7032..1e19c417 100644 --- a/setup.py +++ b/setup.py @@ -1,33 +1,34 @@ import os -from setuptools import setup, find_packages - # from https://stackoverflow.com/a/9079062 import sys + +from setuptools import find_packages, setup + if sys.version_info[0] < 3: - raise Exception("micromanager only supports Python3. Did you run $python setup.py