JobQueue reimplemented in Python

.mypy.ini

@@ -78,5 +78,8 @@ ignore_missing_imports = True
 [mypy-ruamel]
 ignore_missing_imports = True
 
+[mypy-statemachine]
+ignore_missing_imports = True
+
 [mypy-ert.callbacks]
 ignore_errors = True

pyproject.toml

@@ -54,20 +54,23 @@ dependencies=[
     "typing_extensions",
     "jinja2",
     "lark",
+    "lxml",
     "matplotlib",
     "numpy<2",
+    "PyQt5",
     "packaging",
     "pandas",
     "pluggy>=1.3.0",
     "psutil",
     "pydantic >= 1.10.8, < 2",
-    "PyQt5",
     "python-dateutil",
+    "python-statemachine",
     "pyyaml",
     "qtpy",
     "requests",
     "resfo",
     "scipy >= 1.10.1",
+    "python-statemachine",
     "uvicorn >= 0.17.0",
     "websockets",
     "httpx",
@@ -131,6 +134,7 @@ types = [
     "types-decorator",
     "types-docutils",
     "types-tqdm",
+    "types-lxml",
 ]
 
 [tool.setuptools]
@@ -161,7 +165,7 @@ write_to = "src/ert/shared/version.py"
 src = ["src"]
 select = [
   "W",   # pycodestyle
-  "I",   # isort 
+  "I",   # isort
   "B",   # flake-8-bugbear
   "SIM", # flake-8-simplify
   "F",   # pyflakes
@@ -170,15 +174,15 @@ select = [
 line-length = 88
 ignore = ["PLW2901", # redefined-loop-name
           "PLR2004", # magic-value-comparison
-          "PLR0915", # too-many-statements 
-          "PLR0912", # too-many-branches 
-          "PLR5501", # collapsible-else-if 
+          "PLR0915", # too-many-statements
+          "PLR0912", # too-many-branches
+          "PLR5501", # collapsible-else-if
           "PLR0911",  # too-many-return-statements
 ]
 
 [tool.ruff.extend-per-file-ignores]
 "tests/*" = [
-"F401", # unused-import, 
+"F401", # unused-import,
 "PLW0603" # global-statement
 ]
 

src/_ert_job_runner/job.py

@@ -105,7 +105,8 @@ def ensure_file_handles_closed():
                     f"Most likely you are missing and should add "
                     f"'#!/usr/bin/env python' to the top of the file: "
                 )
-            stderr.write(msg)
+            if stderr is not None:
+                stderr.write(msg)
             ensure_file_handles_closed()
             yield Exited(self, e.errno).with_error(msg)
             return

src/_ert_job_runner/reporting/file.py

@@ -158,8 +158,6 @@ def _add_log_line(self, job):
             time_str = time.strftime(TIME_FORMAT, time.localtime())
             f.write(f"{time_str}  Calling: {job.job_data['executable']} {args}\n")
 
-    # This file will be read by the job_queue_node_fscanf_EXIT() function
-    # in job_queue.c. Be very careful with changes in output format.
     def _dump_error_file(self, job, error_msg):
         with append(ERROR_file) as file:
             file.write("<error>\n")
@@ -176,13 +174,22 @@ def _dump_error_file(self, job, error_msg):
                         if stderr:
                             stderr_file = os.path.join(os.getcwd(), job.std_err)
                         else:
-                            stderr = f"<Not written by:{job.name()}>\n"
+                            stderr = f"Not written by:{job.name()}"
                 else:
-                    stderr = f"<stderr: Could not find file:{job.std_err}>\n"
+                    stderr = f"Could not find file:{job.std_err}"
             else:
-                stderr = "<stderr: Not redirected>\n"
+                stderr = "Not redirected"
+
+            # Escape XML characters
+            stderr = (
+                stderr.replace("&", "&amp;")
+                .replace("<", "&lt;")
+                .replace(">", "&gt;")
+                .replace('"', "&quot;")
+                .replace("'", "&apos;")
+            )
 
-            file.write(f"  <stderr>\n{stderr}</stderr>\n")
+            file.write(f"  <stderr>{stderr}</stderr>\n")
             if stderr_file:
                 file.write(f"  <stderr_file>{stderr_file}</stderr_file>\n")
 

src/clib/cmake/libres-config-version.cmake.in

@@ -4,7 +4,7 @@ set(PACKAGE_VERSION_MINOR   @ERT_VERSION_MINOR@)
 set(PACKAGE_VERSION_PATCH   @ERT_VERSION_MICRO@)
 
 set(libres_INCLUDE_DIRS @CMAKE_INSTALL_PREFIX@/include)
-set(libres_LIBRARIES -lenkf -lsched -lrms -lconfig -lanalysis -ljob_queue)
+set(libres_LIBRARIES -lenkf -lsched -lrms -lconfig -lanalysis)
 link_directories( @CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@ )
 include_directories( @CMAKE_INSTALL_PREFIX@/include )
 

src/clib/cmake/libres-config.cmake.in

@@ -1,5 +1,5 @@
 set(libres_INCLUDE_DIRS @CMAKE_INSTALL_PREFIX@/include)
-set(libres_LIBRARIES -lenkf -lsched -lrms -lconfig -lanalysis -ljob_queue)
+set(libres_LIBRARIES -lenkf -lsched -lrms -lconfig -lanalysis)
 
 link_directories( @CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@ )
 include_directories( @CMAKE_INSTALL_PREFIX@/include )

src/clib/lib/CMakeLists.txt

@@ -3,15 +3,6 @@ pybind11_add_module(
   SHARED
   python/init.cpp
   python/logging.cpp
-  job_queue/job_list.cpp
-  job_queue/job_node.cpp
-  job_queue/job_queue.cpp
-  job_queue/local_driver.cpp
-  job_queue/lsf_driver.cpp
-  job_queue/queue_driver.cpp
-  job_queue/slurm_driver.cpp
-  job_queue/torque_driver.cpp
-  job_queue/spawn.cpp
   enkf/enkf_obs.cpp
   enkf/read_summary.cpp
   enkf/row_scaling.cpp)
@@ -28,13 +19,6 @@ target_include_directories(_clib PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include"
 set_target_properties(_clib PROPERTIES CXX_VISIBILITY_PRESET "default")
 install(TARGETS _clib LIBRARY DESTINATION src/ert)
 
-# This extra deploy is only for ctest to be able to locate the script, this file
-# is otherwise deployed through setup.py.
-file(
-  COPY "${CMAKE_CURRENT_SOURCE_DIR}/../../ert/job_queue/qstat_proxy.sh"
-  DESTINATION "${CMAKE_BINARY_DIR}/tests"
-  FILE_PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ)
-
 # -----------------------------------------------------------------
 # Target: 'libert.so' for use in tests
 # -----------------------------------------------------------------