Enable start at any report step

Enable master and slaves to start at any report step. In the previous
commits, only first report step was supported.

CMakeLists_files.cmake

@@ -109,8 +109,10 @@ list (APPEND MAIN_SOURCE_FILES
   opm/simulators/flow/partitionCells.cpp
   opm/simulators/flow/RSTConv.cpp
   opm/simulators/flow/RegionPhasePVAverage.cpp
+  opm/simulators/flow/ReservoirCoupling.cpp
   opm/simulators/flow/ReservoirCouplingMaster.cpp
   opm/simulators/flow/ReservoirCouplingSlave.cpp
+  opm/simulators/flow/ReservoirCouplingSpawnSlaves.cpp
   opm/simulators/flow/SimulatorReportBanners.cpp
   opm/simulators/flow/SimulatorSerializer.cpp
   opm/simulators/flow/SolutionContainers.cpp
@@ -841,6 +843,9 @@ list (APPEND PUBLIC_HEADER_FILES
   opm/simulators/flow/priVarsPacking.hpp
   opm/simulators/flow/RSTConv.hpp
   opm/simulators/flow/RegionPhasePVAverage.hpp
+  opm/simulators/flow/ReservoirCouplingMaster.hpp
+  opm/simulators/flow/ReservoirCouplingSlave.hpp
+  opm/simulators/flow/ReservoirCouplingSpawnSlaves.hpp
   opm/simulators/flow/SimulatorFullyImplicitBlackoil.hpp
   opm/simulators/flow/SimulatorReportBanners.hpp
   opm/simulators/flow/SimulatorSerializer.hpp

opm/simulators/flow/ReservoirCoupling.cpp

@@ -0,0 +1,78 @@
+/*
+  Copyright 2024 Equinor AS
+
+  This file is part of the Open Porous Media project (OPM).
+
+  OPM is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  OPM is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with OPM.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <config.h>
+
+#include <opm/simulators/flow/ReservoirCoupling.hpp>
+#include <opm/simulators/utils/ParallelCommunication.hpp>
+#include <opm/common/OpmLog/OpmLog.hpp>
+
+#include <fmt/format.h>
+
+namespace Opm {
+namespace ReservoirCoupling {
+
+void custom_error_handler_(MPI_Comm* comm, int* err, const std::string &msg)
+{
+    // It can be useful to have a custom error handler for debugging purposes.
+    // If not, MPI will use the default error handler which aborts the program, and
+    // it can be difficult to determine exactly where the error occurred. With a custom
+    // error handler you can at least set a breakpoint here to get a backtrace.
+    int rank;
+    MPI_Comm_rank(*comm, &rank);
+    char err_string[MPI_MAX_ERROR_STRING];
+    int len;
+    MPI_Error_string(*err, err_string, &len);
+    const std::string error_msg = fmt::format(
+        "Reservoir coupling MPI error handler {} rank {}: {}", msg, rank, err_string);
+    // NOTE: The output to terminal vie stderr or stdout has been redirected to files for
+    //   the slaves, see Main.cpp. So the following output will not be visible in the terminal
+    //   if we are called from a slave process.
+    // std::cerr << error_msg << std::endl;
+    OpmLog::error(error_msg);  // Output to log file, also shows the message on stdout for master.
+    MPI_Abort(*comm, *err);
+}
+
+void custom_error_handler_slave_(MPI_Comm* comm, int* err, ...)
+{
+    custom_error_handler_(comm, err, "slave");
+}
+
+void custom_error_handler_master_(MPI_Comm* comm, int* err, ...)
+{
+    custom_error_handler_(comm, err, "master");
+}
+
+void setErrhandler(MPI_Comm comm, bool is_master)
+{
+    MPI_Errhandler errhandler;
+    // NOTE: Lambdas with captures cannot be used as C function pointers, also
+    //   converting lambdas that use ellipsis "..." as last argument to a C function pointer
+    //   is not currently possible, so we need to use static functions instead.
+    if (is_master) {
+        MPI_Comm_create_errhandler(custom_error_handler_master_, &errhandler);
+    }
+    else {
+        MPI_Comm_create_errhandler(custom_error_handler_slave_, &errhandler);
+    }
+    MPI_Comm_set_errhandler(comm, errhandler);
+}
+
+} // namespace ReservoirCoupling
+} // namespace Opm

opm/simulators/flow/ReservoirCoupling.hpp

@@ -29,9 +29,12 @@ namespace ReservoirCoupling {
 
 enum class MessageTag : int {
     SlaveSimulationStartDate,
+    SlaveActivationDate,
     SlaveProcessTermination,
     SlaveNextReportDate,
     SlaveNextTimeStep,
+    MasterGroupNames,
+    MasterGroupNamesSize,
 };
 
 // Custom deleter for MPI_Comm
@@ -194,6 +197,10 @@ class TimePoint {
     }
 };
 
+// Helper functions
+void custom_error_handler_(MPI_Comm* comm, int* err, const std::string &msg);
+void setErrhandler(MPI_Comm comm, bool is_master);
+
 } // namespace ReservoirCoupling
 } // namespace Opm