From 758579b5708e107f571f594049de846ab8812269 Mon Sep 17 00:00:00 2001
From: Shashank Srikanth <ssrikanth@netflix.com>
Date: Thu, 17 Oct 2024 14:59:26 -0700
Subject: [PATCH] Initial commit for logging all metaflow exceptions

---
 metaflow/cli.py                   |  7 +++++++
 metaflow/exception.py             | 16 ++++++++++++++++
 metaflow/system/system_logger.py  |  7 +++++++
 metaflow/system/system_monitor.py |  6 ++++++
 4 files changed, 36 insertions(+)

diff --git a/metaflow/cli.py b/metaflow/cli.py
index 270a1bd6216..cfef27a6977 100644
--- a/metaflow/cli.py
+++ b/metaflow/cli.py
@@ -974,6 +974,13 @@ def start(
     ctx.obj.monitor.start()
     _system_monitor.init_system_monitor(ctx.obj.flow.name, ctx.obj.monitor)
 
+    current._update_env(
+        {
+            "system_logger": _system_logger,
+            "system_monitor": _system_monitor,
+        }
+    )
+
     ctx.obj.metadata = [m for m in METADATA_PROVIDERS if m.TYPE == metadata][0](
         ctx.obj.environment, ctx.obj.flow, ctx.obj.event_logger, ctx.obj.monitor
     )
diff --git a/metaflow/exception.py b/metaflow/exception.py
index 4b85a2d1b39..dc9cf5ec424 100644
--- a/metaflow/exception.py
+++ b/metaflow/exception.py
@@ -46,6 +46,22 @@ class MetaflowException(Exception):
     def __init__(self, msg="", lineno=None):
         self.message = msg
         self.line_no = lineno
+
+        # We can do this check only if a config like METAFLOW_LOG_EXCEPTIONS is set
+        from metaflow import current
+
+        if hasattr(current, "system_logger") and hasattr(current, "system_monitor"):
+            # Wait for the system logger and monitor to be initialized
+            from metaflow.system import _system_logger, _system_monitor
+
+            with _system_monitor.count("metaflow.exception"):
+                _system_logger.log_event(
+                    level="error",
+                    module="metaflow.exception",
+                    name=self.__class__.__name__,
+                    payload={"msg": msg},
+                )
+
         super(MetaflowException, self).__init__()
 
     def __str__(self):
diff --git a/metaflow/system/system_logger.py b/metaflow/system/system_logger.py
index d3d3e1e5a3f..42111a4d5d2 100644
--- a/metaflow/system/system_logger.py
+++ b/metaflow/system/system_logger.py
@@ -38,11 +38,18 @@ def _init_logger_outside_flow(self):
         from .system_utils import init_environment_outside_flow
         from metaflow.plugins import LOGGING_SIDECARS
         from metaflow.metaflow_config import DEFAULT_EVENT_LOGGER
+        from metaflow.metaflow_current import current
 
         self._flow_name = "not_a_real_flow"
         _flow = DummyFlow(self._flow_name)
         _environment = init_environment_outside_flow(_flow)
         _logger = LOGGING_SIDECARS[DEFAULT_EVENT_LOGGER](_flow, _environment)
+
+        current._update_env(
+            {
+                "system_logger": self,
+            }
+        )
         return _logger
 
     @property
diff --git a/metaflow/system/system_monitor.py b/metaflow/system/system_monitor.py
index 3701607f34d..a548bb7b299 100644
--- a/metaflow/system/system_monitor.py
+++ b/metaflow/system/system_monitor.py
@@ -49,11 +49,17 @@ def _init_system_monitor_outside_flow(self):
         from .system_utils import init_environment_outside_flow
         from metaflow.plugins import MONITOR_SIDECARS
         from metaflow.metaflow_config import DEFAULT_MONITOR
+        from metaflow.metaflow_current import current
 
         self._flow_name = "not_a_real_flow"
         _flow = DummyFlow(self._flow_name)
         _environment = init_environment_outside_flow(_flow)
         _monitor = MONITOR_SIDECARS[DEFAULT_MONITOR](_flow, _environment)
+        current._update_env(
+            {
+                "system_monitor": self,
+            }
+        )
         return _monitor
 
     @property