Properly treat DeadKernelError

nb2workflow/nbadapter.py

@@ -53,6 +53,9 @@
 import logging
 from threading import Lock
 
+from nbclient.exceptions import DeadKernelError
+
+
 logger=logging.getLogger(__name__)
 
 logstasher = logstash.LogStasher()
@@ -683,22 +686,29 @@ def _execute(self, parameters, progress_bar=True, log_output=True, inplace=False
                        log_output = True,
                        cwd = tmpdir,
                     )
-                except pm.PapermillExecutionError as e:
+                except (pm.PapermillExecutionError, DeadKernelError) as e:
                     exceptions.append([e,e.args])
                     logger.info(e)
                     logger.info(e.args)
 
-                    if e.ename == "WorkflowIncomplete":
+                    if isinstance(e, DeadKernelError):
+                        sentry.capture_exception(e)
+
+                    elif e.ename == "WorkflowIncomplete":
                         logger.info("detected incomplete workflow")
                         self.update_summary(state="incomplete dependency", dependency=repr(e))
                         raise  PapermillWorkflowIncomplete()
-
+                
                 except nbformat.reader.NotJSONError:
                     ntries -= 1
                     logger.info("retrying... %s", ntries)
                     time.sleep(2)
                     continue
-
+
+                except Exception as e:
+                    logger.error('Unexpected exception %s', e)
+                    sentry.capture_exception(e)
+
                 break
 
         if len(exceptions) == 0:

tests/test_exceptions.py

@@ -0,0 +1,56 @@
+import pytest
+import nb2workflow
+import logging
+import os
+import threading
+import time
+
+logger = logging.getLogger(__name__)
+
+@pytest.fixture
+def app():
+    testfiles_path = os.path.join(os.path.dirname(__file__), 'testfiles')
+    app = nb2workflow.service.app
+    app.notebook_adapters = nb2workflow.nbadapter.find_notebooks(testfiles_path)
+    nb2workflow.service.setup_routes(app)
+    print("creating app")
+    return app
+
+@pytest.mark.parametrize('exc_type', ['runtime', 'kernel'])
+def test_exceptions_sync(client, exc_type):
+    r = client.get(f'/api/v1.0/get/raising?exception_type={exc_type}')
+
+    assert "exceptions" in r.json
+    assert len(r.json['exceptions']) > 0 
+
+@pytest.mark.parametrize('exc_type', ['runtime', 'kernel'])
+def test_exceptions_async(client, exc_type):
+
+    r = client.get(
+        f'/api/v1.0/get/raising?exception_type={exc_type}&_async_request=True'
+        )
+    assert r.status_code == 201
+
+    from nb2workflow.service import AsyncWorker
+
+    def test_worker_run():
+        AsyncWorker('test-worker').run_one()
+
+    test_worker_thread = threading.Thread(target=test_worker_run)
+    # test_worker_thread = AsyncWorker('test-worker')
+    test_worker_thread.start()
+
+    while True:
+        r = client.get(
+            f'/api/v1.0/get/raising?exception_type={exc_type}&_async_request=True'
+            )
+        if r.status_code == 201:
+            time.sleep(0.1)
+            continue
+
+        assert r.status_code == 200
+        assert 'jobdir' in r.json['data']
+        assert len(r.json['data']['exceptions']) > 0
+        break
+
+    test_worker_thread.join()

tests/test_nbadapter.py

@@ -84,7 +84,7 @@ def test_find_notebooks(caplog):
     assert 'Ignoring pattern.' in caplog.text
 
     nbas = find_notebooks(nb_dir)
-    assert len(nbas) == 11
+    assert len(nbas) == 12
 
     nbas = find_notebooks(nb_dir, pattern=r'.*bool')
     assert len(nbas) == 1

tests/testfiles/raising.ipynb

@@ -0,0 +1,53 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import signal"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "parameters"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "dummy = 1\n",
+    "exception_type = \"runtime\" # oda:allowed_values \"runtime\", \"kernel\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if exception_type == \"runtime\":\n",
+    "    raise RuntimeError\n",
+    "elif exception_type == \"kernel\":\n",
+    "    os.kill(os.getpid(), signal.SIGKILL)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.10.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}