Add SHUTDOWN_WAIT and SHUTDOWN profiling events and record them

multiscale · Sep 30, 2023 · b68dd26 · b68dd26
1 parent 1bb85f3
commit b68dd26
Show file tree

Hide file tree

Showing 8 changed files with 29 additions and 7 deletions.
diff --git a/libmuscle/cpp/src/libmuscle/communicator.cpp b/libmuscle/cpp/src/libmuscle/communicator.cpp
@@ -315,13 +315,13 @@ void Communicator::shutdown() {
         client.second->close();
 
     ProfileEvent wait_event(ProfileEventType::disconnect_wait, ProfileTimestamp());
-
     post_office_.wait_for_receivers();
-
     profiler_.record_event(std::move(wait_event));
 
+    ProfileEvent shutdown_event(ProfileEventType::shutdown, ProfileTimestamp());
     for (auto & server : servers_)
         server->close();
+    profiler_.record_event(std::move(shutdown_event));
 }
 
 Communicator::PortMessageCounts Communicator::get_message_counts() {

diff --git a/libmuscle/cpp/src/libmuscle/instance.cpp b/libmuscle/cpp/src/libmuscle/instance.cpp
@@ -849,11 +849,16 @@ bool Instance::Impl::have_f_init_connections_() {
  * @return true iff no ClosePort messages were received.
  */
 bool Instance::Impl::pre_receive_() {
+    ProfileEvent sw_event(ProfileEventType::shutdown_wait, ProfileTimestamp());
+
     bool all_ports_open = receive_settings_();
     pre_receive_f_init_();
     for (auto const & ref_msg : f_init_cache_)
         if (is_close_port(ref_msg.second.data()))
                 all_ports_open = false;
+
+    if (!all_ports_open)
+        profiler_->record_event(std::move(sw_event));
     return all_ports_open;
 }
 

diff --git a/libmuscle/cpp/src/libmuscle/profiling.hpp b/libmuscle/cpp/src/libmuscle/profiling.hpp
@@ -25,7 +25,9 @@ enum class ProfileEventType {
     receive_wait = 5,
     receive_transfer = 6,
     receive_decode = 7,
+    shutdown_wait = 9,
     disconnect_wait = 8,
+    shutdown = 10,
     deregister = 1
 };
 

diff --git a/libmuscle/python/libmuscle/communicator.py b/libmuscle/python/libmuscle/communicator.py
@@ -414,13 +414,13 @@ def shutdown(self) -> None:
             client.close()
 
         wait_event = ProfileEvent(ProfileEventType.DISCONNECT_WAIT, ProfileTimestamp())
-
         self._post_office.wait_for_receivers()
-
         self._profiler.record_event(wait_event)
 
+        shutdown_event = ProfileEvent(ProfileEventType.SHUTDOWN, ProfileTimestamp())
         for server in self._servers:
             server.close()
+        self._profiler.record_event(shutdown_event)
 
     def restore_message_counts(self, port_message_counts: Dict[str, List[int]]
                                ) -> None:

diff --git a/libmuscle/python/libmuscle/instance.py b/libmuscle/python/libmuscle/instance.py
@@ -1005,11 +1005,17 @@ def _pre_receive(self) -> bool:
         Returns:
             True iff no ClosePort messages were received.
         """
+        sw_event = ProfileEvent(ProfileEventType.SHUTDOWN_WAIT, ProfileTimestamp())
+
         all_ports_open = self.__receive_settings()
         self.__pre_receive_f_init()
         for message in self._f_init_cache.values():
             if isinstance(message.data, ClosePort):
                 all_ports_open = False
+
+        if not all_ports_open:
+            self._profiler.record_event(sw_event)
+
         return all_ports_open
 
     def __receive_settings(self) -> bool:

diff --git a/libmuscle/python/libmuscle/manager/profile_database.py b/libmuscle/python/libmuscle/manager/profile_database.py
@@ -103,7 +103,7 @@ def instance_stats(
         cur.execute(
                 "SELECT instance, start_time"
                 " FROM all_events"
-                " WHERE type = 'DISCONNECT_WAIT'")
+                " WHERE type = 'SHUTDOWN_WAIT'")
         stop_run = dict(cur.fetchall())
 
         if not stop_run:

diff --git a/libmuscle/python/libmuscle/profiling.py b/libmuscle/python/libmuscle/profiling.py
@@ -14,7 +14,9 @@ class ProfileEventType(Enum):
     RECEIVE_WAIT = 5
     RECEIVE_TRANSFER = 6
     RECEIVE_DECODE = 7
+    SHUTDOWN_WAIT = 9
     DISCONNECT_WAIT = 8
+    SHUTDOWN = 10
     DEREGISTER = 1
 
 

diff --git a/muscle3/profiling.py b/muscle3/profiling.py
@@ -97,14 +97,16 @@ def plot_resources(performance_file: Path) -> None:
 
 
 _EVENT_TYPES = (
-        'REGISTER', 'CONNECT', 'DISCONNECT_WAIT', 'DEREGISTER',
-        'SEND', 'RECEIVE_WAIT', 'RECEIVE_TRANSFER', 'RECEIVE_DECODE')
+        'REGISTER', 'CONNECT', 'SHUTDOWN_WAIT', 'DISCONNECT_WAIT', 'SHUTDOWN',
+        'DEREGISTER', 'SEND', 'RECEIVE_WAIT', 'RECEIVE_TRANSFER', 'RECEIVE_DECODE')
 
 
 _EVENT_PALETTE = {
         'REGISTER': '#910f33',
         'CONNECT': '#c85172',
+        'SHUTDOWN_WAIT': '#ffdddd',
         'DISCONNECT_WAIT': '#eedddd',
+        'SHUTDOWN': '#c85172',
         'DEREGISTER': '#910f33',
         'RECEIVE_WAIT': '#cccccc',
         'RECEIVE_TRANSFER': '#ff7d00',
@@ -172,6 +174,11 @@ def __init__(self, performance_file: Path) -> None:
         instances = sorted(begin_times.keys())
         self._instances = instances
 
+        if not begin_times:
+            raise RuntimeError(
+                    'This database appears to be empty. Did the simulation crash'
+                    ' before any data were generated?')
+
         # Rest of plot
         ax.set_title('Execution timeline')
         ax.set_xlabel('Wallclock time (s)')