From ce065d4fbf243ce4f082b0ebc78f3ac19ced4bb2 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 29 Oct 2024 08:44:42 -0400
Subject: [PATCH 01/78] Jinja2 templating infrastructure

Functionality to render jinja2 templates to make creating batch files
easier. Also opens up the door to future use of templates like creating
config files. Associated docs+tests included.
---
 flepimop/gempyor_pkg/src/gempyor/_jinja.py    | 114 ++++++++++++++++++
 .../src/gempyor/templates/test_template.j2    |   1 +
 .../tests/_jinja/test__get_template.py        |  32 +++++
 .../tests/_jinja/test__render_template.py     |  19 +++
 .../_jinja/test__render_template_to_file.py   |  24 ++++
 .../test__render_template_to_temp_file.py     |  26 ++++
 6 files changed, 216 insertions(+)
 create mode 100644 flepimop/gempyor_pkg/src/gempyor/_jinja.py
 create mode 100644 flepimop/gempyor_pkg/src/gempyor/templates/test_template.j2
 create mode 100644 flepimop/gempyor_pkg/tests/_jinja/test__get_template.py
 create mode 100644 flepimop/gempyor_pkg/tests/_jinja/test__render_template.py
 create mode 100644 flepimop/gempyor_pkg/tests/_jinja/test__render_template_to_file.py
 create mode 100644 flepimop/gempyor_pkg/tests/_jinja/test__render_template_to_temp_file.py

diff --git a/flepimop/gempyor_pkg/src/gempyor/_jinja.py b/flepimop/gempyor_pkg/src/gempyor/_jinja.py
new file mode 100644
index 000000000..406c8f2ef
--- /dev/null
+++ b/flepimop/gempyor_pkg/src/gempyor/_jinja.py
@@ -0,0 +1,114 @@
+"""
+Internal Jinja2 template rendering utilities.
+
+This module contains utilities for finding and rendering Jinja2 templates. This module
+is tightly coupled to the organization of the package and not intended for external use.
+"""
+
+# Exports
+__all__ = []
+
+
+# Imports
+from pathlib import Path
+from tempfile import mkstemp
+from typing import Any
+
+from jinja2 import Environment, PackageLoader, Template
+
+
+# Globals
+_jinja_environment = Environment(loader=PackageLoader("gempyor", "templates"))
+
+
+# Functions
+def _get_template(name: str) -> Template:
+    """
+    Get a jinja template by name.
+
+    Args:
+        name: The name of the template to pull.
+
+    Returns:
+        A jinja template object corresponding to `name`.
+
+    Examples:
+        >>> _get_template("test_template.j2")
+        <Template 'test_template.j2'>
+    """
+    return _jinja_environment.get_template(name)
+
+
+def _render_template(name: str, data: dict[str, Any]) -> str:
+    """
+    Render a jinja template by name.
+
+    Args:
+        name: The name of the template to pull.
+        data: The data to pass to the template when rendering.
+
+    Returns:
+        The rendered template as a string.
+
+    Examples:
+        >>> _render_template("test_template.j2", {"name": "Bob"})
+        'Hello Bob!'
+    """
+    return _get_template(name).render(data)
+
+
+def _render_template_to_file(name: str, data: dict[str, Any], file: Path) -> None:
+    """
+    Render a jinja template and save to a file.
+
+    Args:
+        name: The name of the template to pull.
+        data: The data to pass to the template when rendering.
+        file: The file to save the rendered template to.
+
+    Examples:
+        >>> from pathlib import Path
+        >>> file = Path("hi.txt")
+        >>> _render_template_to_file("test_template.j2", {"name": "Jane"}, file)
+        >>> file.read_text()
+        'Hello Jane!'
+    """
+    with file.open(mode="w", encoding="utf-8") as f:
+        f.write(_render_template(name, data))
+
+
+def _render_template_to_temp_file(
+    name: str,
+    data: dict[str, Any],
+    suffix: str | None = None,
+    prefix: str | None = None,
+) -> Path:
+    """
+    Render a jinja template and save to a temporary file.
+
+    Args:
+        name: The name of the template to pull.
+        data: The data to pass to the template when rendering.
+        suffix: The suffix of the temporary file, such as an extension. Passed on to
+            `tempfile.mkstemp`.
+        prefix: The prefix of the temporary file. Passed on to `tempfile.mkstemp`.
+
+    Returns:
+        The file containing the rendered template as a `Path` object.
+
+    See Also:
+        [`tempfile.mkstemp`](https://docs.python.org/3/library/tempfile.html#tempfile.mkstemp)
+
+    Examples:
+        >>> file = _render_template_to_temp_file(
+        ...     "test_template.j2", {"name": "John"}, suffix=".txt", prefix="foo_"
+        ... )
+        >>> file
+        PosixPath('/var/folders/2z/h3pc0p7s3ng1tvxrgsw5kr680000gp/T/foo_ocaomg4k.txt')
+        >>> file.read_text()
+        'Hello John!'
+    """
+    _, tmp = mkstemp(suffix=suffix, prefix=prefix, text=True)
+    tmp = Path(tmp)
+    _render_template_to_file(name, data, tmp)
+    return tmp
diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/test_template.j2 b/flepimop/gempyor_pkg/src/gempyor/templates/test_template.j2
new file mode 100644
index 000000000..c7a43bc1f
--- /dev/null
+++ b/flepimop/gempyor_pkg/src/gempyor/templates/test_template.j2
@@ -0,0 +1 @@
+Hello {{ name }}!
\ No newline at end of file
diff --git a/flepimop/gempyor_pkg/tests/_jinja/test__get_template.py b/flepimop/gempyor_pkg/tests/_jinja/test__get_template.py
new file mode 100644
index 000000000..a8eeeaaca
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/_jinja/test__get_template.py
@@ -0,0 +1,32 @@
+from typing import Any
+
+from jinja2 import Template
+from jinja2.exceptions import TemplateNotFound
+import pytest
+
+from gempyor._jinja import _get_template
+
+
+@pytest.mark.parametrize(
+    "name", (("template_does_not_exist.j2"), ("template_does_not_exist_again.j2"))
+)
+def test__get_template_template_not_found_error(name: str) -> None:
+    with pytest.raises(TemplateNotFound, match=name):
+        _get_template(name)
+
+
+@pytest.mark.parametrize(
+    ("name", "data", "output"),
+    (
+        ("test_template.j2", {"name": "John"}, "Hello John!"),
+        ("test_template.j2", {"name": "Jake"}, "Hello Jake!"),
+        ("test_template.j2", {"name": "Jane"}, "Hello Jane!"),
+    ),
+)
+def test__get_template_renders_correctly(
+    name: str, data: dict[str, Any], output: str
+) -> None:
+    template = _get_template(name)
+    assert isinstance(template, Template)
+    rendered_template = template.render(**data)
+    assert rendered_template == output
diff --git a/flepimop/gempyor_pkg/tests/_jinja/test__render_template.py b/flepimop/gempyor_pkg/tests/_jinja/test__render_template.py
new file mode 100644
index 000000000..3d76490b4
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/_jinja/test__render_template.py
@@ -0,0 +1,19 @@
+from typing import Any
+
+import pytest
+
+from gempyor._jinja import _render_template
+
+
+@pytest.mark.parametrize(
+    ("name", "data", "output"),
+    (
+        ("test_template.j2", {"name": "John"}, "Hello John!"),
+        ("test_template.j2", {"name": "Jake"}, "Hello Jake!"),
+        ("test_template.j2", {"name": "Jane"}, "Hello Jane!"),
+    ),
+)
+def test__render_template_renders_correctly(
+    name: str, data: dict[str, Any], output: str
+) -> None:
+    assert _render_template(name, data) == output
diff --git a/flepimop/gempyor_pkg/tests/_jinja/test__render_template_to_file.py b/flepimop/gempyor_pkg/tests/_jinja/test__render_template_to_file.py
new file mode 100644
index 000000000..330bd77b1
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/_jinja/test__render_template_to_file.py
@@ -0,0 +1,24 @@
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from gempyor._jinja import _render_template_to_file
+
+
+@pytest.mark.parametrize(
+    ("name", "data", "file", "output"),
+    (
+        ("test_template.j2", {"name": "John"}, Path("foo.txt"), "Hello John!"),
+        ("test_template.j2", {"name": "Jake"}, Path("bar.txt"), "Hello Jake!"),
+        ("test_template.j2", {"name": "Jane"}, Path("fizz.md"), "Hello Jane!"),
+    ),
+)
+def test__render_template_to_file_renders_correctly(
+    tmp_path: Path, name: str, data: dict[str, Any], file: Path, output: str
+) -> None:
+    if file.is_absolute():
+        raise ValueError("The `file` argument must be relative for unit testing.")
+    file = tmp_path / file
+    _render_template_to_file(name, data, file)
+    assert file.read_text() == output
diff --git a/flepimop/gempyor_pkg/tests/_jinja/test__render_template_to_temp_file.py b/flepimop/gempyor_pkg/tests/_jinja/test__render_template_to_temp_file.py
new file mode 100644
index 000000000..280214be4
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/_jinja/test__render_template_to_temp_file.py
@@ -0,0 +1,26 @@
+from typing import Any
+
+import pytest
+
+from gempyor._jinja import _render_template_to_temp_file
+
+
+@pytest.mark.parametrize(
+    ("name", "data", "suffix", "prefix", "output"),
+    (
+        ("test_template.j2", {"name": "John"}, None, None, "Hello John!"),
+        ("test_template.j2", {"name": "Jake"}, ".txt", None, "Hello Jake!"),
+        ("test_template.j2", {"name": "Jane"}, None, "abc_", "Hello Jane!"),
+        ("test_template.j2", {"name": "Job"}, ".dat", "def_", "Hello Job!"),
+    ),
+)
+def test__render_template_to_temp_file_renders_correctly(
+    name: str, data: dict[str, Any], suffix: str | None, prefix: str | None, output: str
+) -> None:
+    temp_file = _render_template_to_temp_file(name, data, suffix=suffix, prefix=prefix)
+    assert temp_file.exists()
+    if suffix:
+        assert temp_file.name.endswith(suffix)
+    if prefix:
+        assert temp_file.name.startswith(prefix)
+    assert temp_file.read_text(encoding="utf-8") == output

From 8ecc3e026984626ed7322dbb241eefe394f67c0c Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 29 Oct 2024 16:48:45 -0400
Subject: [PATCH 02/78] Logging infrastructure for `gempyor`

Added `gempyor.logging` module with `_get_logging_level`,
`get_script_logger`, and `ClickHandler` along with corresponding
documentation and unit tests.
---
 flepimop/gempyor_pkg/src/gempyor/logging.py   | 170 ++++++++++++++++++
 .../tests/logging/test__get_logging_level.py  |  34 ++++
 .../tests/logging/test_click_handler_class.py |  71 ++++++++
 .../tests/logging/test_get_script_logger.py   |  72 ++++++++
 4 files changed, 347 insertions(+)
 create mode 100644 flepimop/gempyor_pkg/src/gempyor/logging.py
 create mode 100644 flepimop/gempyor_pkg/tests/logging/test__get_logging_level.py
 create mode 100644 flepimop/gempyor_pkg/tests/logging/test_click_handler_class.py
 create mode 100644 flepimop/gempyor_pkg/tests/logging/test_get_script_logger.py

diff --git a/flepimop/gempyor_pkg/src/gempyor/logging.py b/flepimop/gempyor_pkg/src/gempyor/logging.py
new file mode 100644
index 000000000..c6d7b3398
--- /dev/null
+++ b/flepimop/gempyor_pkg/src/gempyor/logging.py
@@ -0,0 +1,170 @@
+"""
+Logging utilities for consistent script output.
+
+This module provides functionality for creating consistent outputs from CLI tools 
+provided by this package. Currently exported are:
+- `ClickHandler`: Custom logging handler specifically designed for CLI output using 
+    click.
+- `get_script_logger`: Factory for creating a logger instance with a consistent style
+    across CLI tools.
+"""
+
+__all__ = ["ClickHandler", "get_script_logger"]
+
+
+import logging
+from typing import Any, IO
+
+import click
+
+
+DEFAULT_LOG_FORMAT = "%(asctime)s:%(levelname)s:%(name)s> %(message)s"
+
+
+class ClickHandler(logging.Handler):
+    """
+    Custom logging handler specifically for click based CLI tools.
+    """
+
+    _punctuation = (".", ",", "?", "!", ":")
+
+    def __init__(
+        self,
+        level: int | str = 0,
+        file: IO[Any] | None = None,
+        nl: bool = True,
+        err: bool = False,
+        color: bool | None = None,
+        punctuate: bool = True,
+    ) -> None:
+        """
+        Initialize an instance of the click handler.
+
+        Args:
+            level: The logging level to use for this handler.
+            file: The file to write to. Defaults to stdout.
+            nl: Print a newline after the message. Enabled by default.
+            err: Write to stderr instead of stdout.
+            color: Force showing or hiding colors and other styles. By default click
+                will remove color if the output does not look like an interactive
+                terminal.
+            punctuate: A boolean indicating if punctuation should be added to the end
+                of a log message provided if missing.
+
+        Notes:
+            For more details on the `file`, `nl`, `err`, and `color` args please refer
+            to [`click.echo`](https://click.palletsprojects.com/en/8.1.x/api/#click.echo).
+        """
+        super().__init__(level)
+        self._file = file
+        self._nl = nl
+        self._err = err
+        self._color = color
+        self._punctuate = punctuate
+
+    def emit(self, record: logging.LogRecord) -> None:
+        """
+        Emit a given log record via `click.echo`
+
+        Args:
+            record: The log record to output.
+
+        See Also:
+            [`logging.Handler.emit`](https://docs.python.org/3/library/logging.html#logging.Handler.emit)
+        """
+        msg = self.format(record)
+        msg = (
+            f"{msg}."
+            if self._punctuate and not msg.endswith(self._punctuation)
+            else msg
+        )
+        click.echo(
+            message=msg, file=self._file, nl=self._nl, err=self._err, color=self._color
+        )
+
+
+def get_script_logger(
+    name: str,
+    verbosity: int,
+    handler: logging.Handler | None = None,
+    log_format: str = DEFAULT_LOG_FORMAT,
+) -> logging.Logger:
+    """
+    Create a logger for use in scripts.
+
+    Args:
+        name: The name to display in the log message, useful for locating the source
+            of logging messages. Almost always `__name__`.
+        verbosity: A non-negative integer for the verbosity level.
+        handler: An optional logging handler to use in creating the logger returned, or
+            `None` to just use the `ClickHandler`.
+        log_format: The format to use for logged messages. Passed directly to the `fmt`
+            argument of [logging.Formatter](https://docs.python.org/3/library/logging.html#logging.Formatter).
+
+    Returns:
+        An instance of `logging.Logger` that has the appropriate level set based on
+        `verbosity` and a custom handler for outputting for CLI tools.
+
+    Examples:
+        >>> from gempyor.logging import get_script_logger
+        >>> logger = get_script_logger(__name__, 3)
+        >>> logger.info("This is a log info message")
+        2024-10-29 16:07:20,272:INFO:__main__> This is a log info message.
+    """
+    logger = logging.getLogger(name)
+    logger.setLevel(_get_logging_level(verbosity))
+    handler = ClickHandler() if handler is None else handler
+    log_formatter = logging.Formatter(log_format)
+    for old_handler in logger.handlers:
+        logger.removeHandler(old_handler)
+    handler.setFormatter(log_formatter)
+    logger.addHandler(handler)
+    logger.propagate = False
+    return logger
+
+
+def _get_logging_level(verbosity: int) -> int:
+    """
+    An internal method to convert verbosity to a logging level.
+
+    Args:
+        verbosity: A non-negative integer for the verbosity level or level from
+            `logging` that will be returned as is.
+
+    Examples:
+        >>> _get_logging_level(0)
+        40
+        >>> _get_logging_level(1)
+        30
+        >>> _get_logging_level(2)
+        20
+        >>> _get_logging_level(3)
+        10
+        >>> _get_logging_level(4)
+        10
+        >>> import logging
+        >>> _get_logging_level(logging.ERROR) == logging.ERROR
+        True
+
+    Raises:
+        ValueError: If `verbosity` is less than zero.
+
+    Returns:
+        The log level from the `logging` module corresponding to the given `verbosity`.
+    """
+    if verbosity < 0:
+        raise ValueError(f"`verbosity` must be non-negative, was given '{verbosity}'.")
+    if verbosity in (
+        logging.DEBUG,
+        logging.INFO,
+        logging.WARNING,
+        logging.ERROR,
+        logging.CRITICAL,
+    ):
+        return verbosity
+    verbosity_to_logging_level = {
+        0: logging.ERROR,
+        1: logging.WARNING,
+        2: logging.INFO,
+    }
+    return verbosity_to_logging_level.get(verbosity, logging.DEBUG)
diff --git a/flepimop/gempyor_pkg/tests/logging/test__get_logging_level.py b/flepimop/gempyor_pkg/tests/logging/test__get_logging_level.py
new file mode 100644
index 000000000..210d0bd69
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/logging/test__get_logging_level.py
@@ -0,0 +1,34 @@
+import logging
+
+import pytest
+
+from gempyor.logging import _get_logging_level
+
+
+@pytest.mark.parametrize("verbosity", (-1, -100))
+def test__get_logging_level_negative_verbosity_value_error(verbosity: int) -> None:
+    with pytest.raises(
+        ValueError, match=f"`verbosity` must be non-negative, was given '{verbosity}'."
+    ):
+        _get_logging_level(verbosity)
+
+
+@pytest.mark.parametrize(
+    ("verbosity", "expected_level"),
+    (
+        (logging.ERROR, logging.ERROR),
+        (logging.WARNING, logging.WARNING),
+        (logging.INFO, logging.INFO),
+        (logging.DEBUG, logging.DEBUG),
+        (0, logging.ERROR),
+        (1, logging.WARNING),
+        (2, logging.INFO),
+        (3, logging.DEBUG),
+        (4, logging.DEBUG),
+        (5, logging.DEBUG),
+    ),
+)
+def test__get_logging_level_output_validation(
+    verbosity: int, expected_level: int
+) -> None:
+    assert _get_logging_level(verbosity) == expected_level
diff --git a/flepimop/gempyor_pkg/tests/logging/test_click_handler_class.py b/flepimop/gempyor_pkg/tests/logging/test_click_handler_class.py
new file mode 100644
index 000000000..983acba8a
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/logging/test_click_handler_class.py
@@ -0,0 +1,71 @@
+import io
+import logging
+from typing import Any, IO
+
+import pytest
+
+from gempyor.logging import ClickHandler
+
+
+@pytest.mark.parametrize(
+    "level",
+    (logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL),
+)
+@pytest.mark.parametrize("file", (None, io.StringIO))
+@pytest.mark.parametrize("nl", (True, False))
+@pytest.mark.parametrize("err", (False, True))
+@pytest.mark.parametrize("color", (None, False))
+@pytest.mark.parametrize("punctuate", (True, False))
+def test_click_handler_init(
+    level: int | str,
+    file: IO[Any] | None,
+    nl: bool,
+    err: bool,
+    color: bool | None,
+    punctuate: bool,
+) -> None:
+    handler = ClickHandler(
+        level=level, file=file, nl=nl, err=err, color=color, punctuate=punctuate
+    )
+    assert handler.level == level
+    assert handler._file == file
+    assert handler._nl == nl
+    assert handler._err == err
+    assert handler._color == color
+    assert handler._punctuate == punctuate
+
+
+@pytest.mark.parametrize("punctuate", (True, False))
+@pytest.mark.parametrize(
+    "msg",
+    (
+        "This is a message",
+        "Another message.",
+        "Start to a list:",
+        "Middle of a sentence,",
+        "Oh-no!",
+        "Question?",
+    ),
+)
+def test_click_handler_punctuation_formatting(punctuate: bool, msg: str) -> None:
+    buffer = io.StringIO()
+    handler = ClickHandler(level=logging.DEBUG, file=buffer, punctuate=punctuate)
+    log_record = logging.LogRecord(
+        name="",
+        level=logging.DEBUG,
+        pathname="",
+        lineno=1,
+        msg=msg,
+        args=None,
+        exc_info=None,
+    )
+    handler.emit(log_record)
+    buffer.seek(0)
+    handler_msg = buffer.getvalue()
+    if not punctuate:
+        assert handler_msg == msg + "\n"
+    else:
+        assert (
+            handler_msg
+            == (msg if msg.endswith(ClickHandler._punctuation) else f"{msg}.") + "\n"
+        )
diff --git a/flepimop/gempyor_pkg/tests/logging/test_get_script_logger.py b/flepimop/gempyor_pkg/tests/logging/test_get_script_logger.py
new file mode 100644
index 000000000..98306c64b
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/logging/test_get_script_logger.py
@@ -0,0 +1,72 @@
+import logging
+import random
+import string
+
+import pytest
+
+from gempyor.logging import _get_logging_level, get_script_logger, DEFAULT_LOG_FORMAT
+
+
+@pytest.mark.parametrize("name", ("foobar", "fizzbuzz"))
+@pytest.mark.parametrize(
+    "verbosity",
+    (
+        0,
+        1,
+        2,
+        3,
+        4,
+        logging.CRITICAL,
+        logging.ERROR,
+        logging.WARNING,
+        logging.INFO,
+        logging.DEBUG,
+    ),
+)
+@pytest.mark.parametrize("handler", (None, logging.StreamHandler()))
+@pytest.mark.parametrize("log_format", (None, "%(message)s"))
+def test_get_script_logger(
+    caplog: pytest.LogCaptureFixture,
+    name: str,
+    verbosity: int,
+    handler: logging.Handler | None,
+    log_format: str | None,
+) -> None:
+    caplog.set_level(logging.DEBUG, logger=name)
+
+    if log_format is None:
+        logger = get_script_logger(name, verbosity, handler=handler)
+    else:
+        logger = get_script_logger(
+            name, verbosity, handler=handler, log_format=log_format
+        )
+
+    # pytest-dev/pytest#3697
+    logger.propagate = True
+
+    assert isinstance(logger, logging.Logger)
+    assert logger.name == name
+    assert logger.level == _get_logging_level(verbosity)
+    if handler is not None:
+        assert len(logger.handlers) == 1
+        assert logger.handlers[0] == handler
+    assert logger.handlers[0].formatter._fmt == (
+        log_format if log_format else DEFAULT_LOG_FORMAT
+    )
+
+    i = 0
+    for val, lvl in [
+        (10, "debug"),
+        (20, "info"),
+        (30, "warning"),
+        (40, "error"),
+        (50, "critical"),
+    ]:
+        msg = "".join(random.choice(string.ascii_letters) for _ in range(20))
+        getattr(logger, lvl)(msg)
+        if val < _get_logging_level(verbosity):
+            assert len(caplog.records) == i
+            continue
+        assert caplog.records[i].levelno == val
+        assert caplog.records[i].message == msg
+        i += 1

From 1afe3c8b39bb13dde9cf41b8098cbf7409caa326 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Wed, 30 Oct 2024 10:36:48 -0400
Subject: [PATCH 03/78] Add private `shutil.which` wrapper

Added private helper `gempyor.utils._shutil_which` which has a `check`
argument that behaves very similar in spirit to the same argument of
`subprocess.run`. Corresponding docs/tests included.
---
 flepimop/gempyor_pkg/src/gempyor/utils.py     | 50 +++++++++++++++++++
 .../tests/utils/test__shutil_which.py         | 41 +++++++++++++++
 2 files changed, 91 insertions(+)
 create mode 100644 flepimop/gempyor_pkg/tests/utils/test__shutil_which.py

diff --git a/flepimop/gempyor_pkg/src/gempyor/utils.py b/flepimop/gempyor_pkg/src/gempyor/utils.py
index 377dcdee5..d0f1f15e9 100644
--- a/flepimop/gempyor_pkg/src/gempyor/utils.py
+++ b/flepimop/gempyor_pkg/src/gempyor/utils.py
@@ -1,3 +1,4 @@
+from collections.abc import Iterable
 import datetime
 import functools
 import logging
@@ -1040,3 +1041,52 @@ def move_file_at_local(name_map: dict[str, str]) -> None:
     for src, dst in name_map.items():
         os.path.makedirs(os.path.dirname(dst), exist_ok=True)
         shutil.copy(src, dst)
+
+
+def _shutil_which(
+    cmd: str,
+    mode: int = os.F_OK | os.X_OK,
+    path: str | bytes | os.PathLike | None = None,
+    check: bool = True,
+) -> str | None:
+    """
+    A thin wrapper around `shutil.which` with extra validation.
+
+    Args:
+        cmd: The name of the command to search for.
+        mode: The permission mask required of possible files.
+        path: A path describing the locations to search, or `None` to use
+            the PATH environment variable.
+        check: If `True` an `OSError` will be raised if a `cmd` is not found.
+            Similar in spirit to the `check` arg of `subprocess.run`.
+
+    Returns:
+        Either the full path to the `cmd` found, or `None` if `cmd` is not
+        found and `check` is `False`.
+
+    Raises:
+        OSError: If `cmd` is not found and `check` is `True`.
+
+    Examples:
+        >>> import os
+        >>> from shutil import which
+        >>> _shutil_which("python") == which("python")
+        True
+        >>> _shutil_which("does_not_exist", check=False)
+        >>> try:
+        ...     _shutil_which("does_not_exist", check=True)
+        ... except Exception as e:
+        ...     print(type(e))
+        ...     print(str(e).replace(os.environ.get("PATH"), "..."))
+        ...
+        <class 'OSError'>
+        Did not find 'does_not_exist' on path '...'.
+
+    See Also:
+        [`shutil.which`](https://docs.python.org/3/library/shutil.html#shutil.which)
+    """
+    result = shutil.which(cmd, mode=mode, path=path)
+    if check and result is None:
+        path = os.environ.get("PATH") if path is None else path
+        raise OSError(f"Did not find '{cmd}' on path '{path}'.")
+    return result
diff --git a/flepimop/gempyor_pkg/tests/utils/test__shutil_which.py b/flepimop/gempyor_pkg/tests/utils/test__shutil_which.py
new file mode 100644
index 000000000..19cfafce7
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/utils/test__shutil_which.py
@@ -0,0 +1,41 @@
+import os
+from pathlib import Path
+from shutil import which
+
+import pytest
+
+from gempyor.utils import _shutil_which
+
+
+@pytest.fixture
+def custom_path_setup(tmp_path: Path) -> str:
+    path_spec = set()
+    files = (
+        Path("abc"),
+        Path("bin/def"),
+        Path("bin/ghi"),
+        Path("bin/jkl"),
+        Path("dir1/dir2/xyz"),
+    )
+    for file in files:
+        file = tmp_path / file
+        file.parent.mkdir(parents=True, exist_ok=True)
+        file.write_text("#!/usr/bin/env bash\necho 'Hello!'")
+        file.chmod(0o755)
+        path_spec.add(str(file.parent.absolute()))
+    return os.pathsep.join(path_spec)
+
+
+@pytest.mark.parametrize("cmd", ("abc", "def", "ghi", "jkl", "xyz"))
+def test_matches_python_stdlib_in_basic_case(custom_path_setup: str, cmd: str) -> None:
+    assert _shutil_which(cmd, path=custom_path_setup, check=False) == which(
+        cmd, path=custom_path_setup
+    )
+
+
+def test_oserror_when_check_and_cmd_not_found(custom_path_setup: Path) -> None:
+    cmd = "does_not_exist"
+    with pytest.raises(
+        OSError, match=f"^Did not find '{cmd}' on path '{custom_path_setup}'.$"
+    ):
+        _shutil_which(cmd, path=custom_path_setup, check=True)

From cd3ca2d4506406b481f600ac429d00459d0a4fb8 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Wed, 30 Oct 2024 15:32:49 -0400
Subject: [PATCH 04/78] Only propagate logger in pytest

Pytest hooks into the root logger, which breaks the use case here where
we only want a single handler to emit a log record. Create a special
carve out for pytest and put it in `get_script_logger` to make testing
logging results of any function easy.
---
 flepimop/gempyor_pkg/src/gempyor/logging.py                  | 5 ++++-
 flepimop/gempyor_pkg/tests/logging/test_get_script_logger.py | 3 ---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/logging.py b/flepimop/gempyor_pkg/src/gempyor/logging.py
index c6d7b3398..db5123823 100644
--- a/flepimop/gempyor_pkg/src/gempyor/logging.py
+++ b/flepimop/gempyor_pkg/src/gempyor/logging.py
@@ -13,6 +13,8 @@
 
 
 import logging
+import os
+import sys
 from typing import Any, IO
 
 import click
@@ -119,7 +121,8 @@ def get_script_logger(
         logger.removeHandler(old_handler)
     handler.setFormatter(log_formatter)
     logger.addHandler(handler)
-    logger.propagate = False
+    # pytest-dev/pytest#3697
+    logger.propagate = os.path.basename(sys.argv[0]) == "pytest" if sys.argv else False
     return logger
 
 
diff --git a/flepimop/gempyor_pkg/tests/logging/test_get_script_logger.py b/flepimop/gempyor_pkg/tests/logging/test_get_script_logger.py
index 98306c64b..1e0c5a2d0 100644
--- a/flepimop/gempyor_pkg/tests/logging/test_get_script_logger.py
+++ b/flepimop/gempyor_pkg/tests/logging/test_get_script_logger.py
@@ -41,9 +41,6 @@ def test_get_script_logger(
             name, verbosity, handler=handler, log_format=log_format
         )
 
-    # pytest-dev/pytest#3697
-    logger.propagate = True
-
     assert isinstance(logger, logging.Logger)
     assert logger.name == name
     assert logger.level == _get_logging_level(verbosity)

From e6c86ef8d8ee37ead941823ed0d692c3745c5d10 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Thu, 31 Oct 2024 09:56:38 -0400
Subject: [PATCH 05/78] Add `_sbatch` to `_slurm`

Internal utility to submit a batch file to slurm via `sbatch` along with
detailed logging and corresponding unit tests and docs.
---
 flepimop/gempyor_pkg/src/gempyor/_slurm.py    | 102 +++++++++++++
 .../gempyor_pkg/tests/_slurm/test__sbatch.py  | 140 ++++++++++++++++++
 2 files changed, 242 insertions(+)
 create mode 100644 flepimop/gempyor_pkg/src/gempyor/_slurm.py
 create mode 100644 flepimop/gempyor_pkg/tests/_slurm/test__sbatch.py

diff --git a/flepimop/gempyor_pkg/src/gempyor/_slurm.py b/flepimop/gempyor_pkg/src/gempyor/_slurm.py
new file mode 100644
index 000000000..7abc9c0b0
--- /dev/null
+++ b/flepimop/gempyor_pkg/src/gempyor/_slurm.py
@@ -0,0 +1,102 @@
+"""
+Internal utilities for interacting with slurm.
+"""
+
+__all__ = []
+
+
+from pathlib import Path
+from shlex import quote
+import subprocess
+from typing import Any, Literal
+
+from .logging import get_script_logger
+from .utils import _shutil_which
+
+
+def _sbatch(
+    script: Path,
+    environment_variables: dict[str, Any] | Literal["all", "nil", "none"],
+    options: dict[str, Any],
+    verbosity: int | None,
+    dry_run: bool,
+) -> None:
+    """
+    Submit a job to slurm via the `sbatch` command.
+
+    Args:
+        script: The batch file to submit to slurm.
+        environment_variables: Environment variables to pass to the job via the
+            '--export' option. Keys correspond to the variable name and the value is
+            the variable value. All values are coerced to a string and then escaped.
+            Or can be a literal for one of the '--export' option's special values.
+        options: Options to pass when calling sbatch. Keys correspond to the option
+            name and the value is the option value. Options can be provided as either
+            the long or short name, but this function will not be able to determine that
+            these are duplicates so only using long names is recommended.
+        verbosity: A integer verbosity level to enable logging or `None` for no logging.
+        dry_run: A boolean indicating if this is a dry run or not, if set to `True` this
+            function will not actually submit a job to slurm.
+
+    Raises:
+        ValueError: If 'export' is given as a key in `options` instead of using
+            `environment_variables`.
+
+    Examples:
+        >>> from pathlib import Path
+        >>> _sbatch(
+        ...     Path("my_batch_script.sbatch"),
+        ...     {"VAR1": 1, "VAR2": "true"},
+        ...     {"J": "My job name", "output": Path("out.log")},
+        ...     3,
+        ...     True,
+        ... )
+        2024-10-31 09:26:48,361:DEBUG:gempyor._slurm> Using batch script '/my_batch_script.sbatch' to submit to slurm.
+        2024-10-31 09:26:48,363:INFO:gempyor._slurm> If not dry mode would have submitted to slurm with: /usr/bin/sbatch --export=VAR1=1,VAR2=true -J='My job name' --output=out.log /my_batch_script.sbatch.
+
+    See Also:
+        [`sbatch`'s documentation](https://slurm.schedmd.com/sbatch.html)
+    """
+    if "export" in options:
+        raise ValueError(
+            "Found 'export' in `options`, please use `environment_variables` instead."
+        )
+
+    if verbosity is not None:
+        logger = get_script_logger(__name__, verbosity)
+        logger.debug("Using batch script '%s' to submit to slurm", script.absolute())
+
+    if isinstance(environment_variables, dict):
+        env_vars = [f"{k}={quote(str(v))}" for k, v in environment_variables.items()]
+        export = "--export=" + ",".join(env_vars)
+    else:
+        export = f"--export={environment_variables.upper()}"
+    options = [
+        f"{'-' if len(k) == 1 else '--'}{k}={quote(str(v))}" for k, v in options.items()
+    ]
+    sbatch_cmd = _shutil_which("sbatch")
+    if len(export) > 9:
+        cmd_args = [sbatch_cmd, export] + options + [str(script.absolute())]
+    else:
+        cmd_args = [sbatch_cmd] + options + [str(script.absolute())]
+
+    if dry_run:
+        if verbosity is not None:
+            logger.info(
+                "If not dry mode would have submitted to slurm with: %s",
+                " ".join(cmd_args),
+            )
+        return
+
+    if verbosity is not None:
+        logger.info("Submitting to slurm with: %s", " ".join(cmd_args))
+    process = subprocess.run(cmd_args, check=True, capture_output=True)
+    if verbosity is not None:
+        if process.stdout:
+            logger.debug(
+                "Captured stdout from sbatch submission: %s", process.stdout.decode()
+            )
+        if process.stderr:
+            logger.error(
+                "Captured stderr from sbatch submission: %s", process.stderr.decode()
+            )
diff --git a/flepimop/gempyor_pkg/tests/_slurm/test__sbatch.py b/flepimop/gempyor_pkg/tests/_slurm/test__sbatch.py
new file mode 100644
index 000000000..e846f5c37
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/_slurm/test__sbatch.py
@@ -0,0 +1,140 @@
+import logging
+import os
+from pathlib import Path
+from unittest.mock import patch
+import subprocess
+from typing import Any, Literal
+
+import pytest
+
+from gempyor._slurm import _sbatch
+from gempyor.logging import _get_logging_level
+from gempyor.utils import _shutil_which
+
+
+@pytest.fixture
+def sample_sbatch_script(tmp_path: Path) -> Path:
+    script = tmp_path / "example.sbatch"
+    script.write_text("#!/usr/bin/env bash\necho 'Hello sbatch!'")
+    return script
+
+
+def test_export_option_causes_value_error(sample_sbatch_script: Path) -> None:
+    with pytest.raises(
+        ValueError,
+        match="^Found 'export' in `options`, please use `environment_variables` instead.$",
+    ):
+        _sbatch(sample_sbatch_script, {}, {"export": "VAR1=1,VAR2=abc"}, None, True)
+
+
+@pytest.mark.parametrize(
+    ("environment_variables", "expected_export"),
+    (
+        ({}, None),
+        ({"ENV_VAR": "true"}, "ENV_VAR=true"),
+        ({"VAR1": 1, "VAR2": "abc", "VAR3": True}, "VAR1=1,VAR2=abc,VAR3=True"),
+        (
+            {"NEEDS_ESCAPING": "`", "ALSO_ESCAPED": "a b c"},
+            "NEEDS_ESCAPING='`',ALSO_ESCAPED='a b c'",
+        ),
+        ("all", "ALL"),
+        ("nil", "NIL"),
+        ("none", "NONE"),
+    ),
+)
+@pytest.mark.parametrize(
+    ("options", "expected_options"),
+    (
+        ({}, None),
+        ({"time": "1:00:00", "mem": "2GB"}, "--time=1:00:00 --mem=2GB"),
+        (
+            {"output": Path("/abc/def/ghi.log"), "ntasks": 2},
+            "--output=/abc/def/ghi.log --ntasks=2",
+        ),
+        (
+            {"J": "My Job Name", "e": "/path/to/error.log"},
+            "-J='My Job Name' -e=/path/to/error.log",
+        ),
+    ),
+)
+@pytest.mark.parametrize(
+    "verbosity", (None, logging.DEBUG, logging.INFO, logging.WARNING)
+)
+@pytest.mark.parametrize("dry_run", (True, False))
+def test_output_validation(
+    caplog: pytest.LogCaptureFixture,
+    sample_sbatch_script: Path,
+    environment_variables: dict[str, Any] | Literal["all", "nil", "none"],
+    expected_export: str | None,
+    options: dict[str, Any],
+    expected_options: str | None,
+    verbosity: int | None,
+    dry_run: bool,
+) -> None:
+    def shutil_which_wraps(
+        cmd: str,
+        mode: int = os.F_OK | os.X_OK,
+        path: str | bytes | os.PathLike | None = None,
+        check: bool = True,
+    ) -> str | None:
+        return (
+            "sbatch"
+            if cmd == "sbatch"
+            else _shutil_which(cmd, mode=mode, path=path, check=check)
+        )
+
+    def subprocess_run_wraps(args, **kwargs):
+        if os.path.basename(args[0]) == "sbatch":
+            return subprocess.CompletedProcess(
+                args=args, returncode=0, stdout=b"Submitted batch job 999\n", stderr=b""
+            )
+        return subprocess.run(args, **kwargs)
+
+    with patch(
+        "gempyor._slurm._shutil_which", wraps=shutil_which_wraps
+    ) as shutil_which_patch:
+        with patch(
+            "gempyor._slurm.subprocess.run", wraps=subprocess_run_wraps
+        ) as subprocess_run_patch:
+            assert (
+                _sbatch(
+                    sample_sbatch_script,
+                    environment_variables,
+                    options,
+                    verbosity,
+                    dry_run,
+                )
+                is None
+            )
+            shutil_which_patch.assert_called_once_with("sbatch")
+
+            if dry_run:
+                subprocess_run_patch.assert_not_called()
+            else:
+                subprocess_run_patch.assert_called_once()
+
+            if verbosity is None:
+                assert len(caplog.records) == 0
+            else:
+                log_messages_by_level = {
+                    hash((logging.DEBUG, True)): 2,
+                    hash((logging.DEBUG, False)): 3,
+                    hash((logging.INFO, True)): 1,
+                    hash((logging.INFO, False)): 1,
+                }
+                assert len(caplog.records) == log_messages_by_level.get(
+                    hash((_get_logging_level(verbosity), dry_run)), 0
+                )
+
+            if not dry_run:
+                args: list[str] = subprocess_run_patch.call_args.args[0]
+                assert len(args) == 2 + len(options) + min(
+                    len(environment_variables), 1
+                )
+                assert Path(args[-1]) == sample_sbatch_script
+                if environment_variables:
+                    assert args[1].startswith("--export=")
+                    assert args[1][9:] == expected_export
+                if options:
+                    start_index = 2 if environment_variables else 1
+                    assert " ".join(args[start_index:-1]) == expected_options

From 51ae598397a0e31ad6910a212fc55ff8b0b014aa Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Thu, 31 Oct 2024 10:43:26 -0400
Subject: [PATCH 06/78] Add `gempyor.utils._git_head`

Added an internal utility function to get the sha commit corresponding
to the head of a given git repository. For use in creating manifest
files.
---
 flepimop/gempyor_pkg/src/gempyor/utils.py     | 25 +++++++++
 .../gempyor_pkg/tests/utils/test__git_head.py | 54 +++++++++++++++++++
 2 files changed, 79 insertions(+)
 create mode 100644 flepimop/gempyor_pkg/tests/utils/test__git_head.py

diff --git a/flepimop/gempyor_pkg/src/gempyor/utils.py b/flepimop/gempyor_pkg/src/gempyor/utils.py
index d0f1f15e9..c39658ec0 100644
--- a/flepimop/gempyor_pkg/src/gempyor/utils.py
+++ b/flepimop/gempyor_pkg/src/gempyor/utils.py
@@ -1090,3 +1090,28 @@ def _shutil_which(
         path = os.environ.get("PATH") if path is None else path
         raise OSError(f"Did not find '{cmd}' on path '{path}'.")
     return result
+
+
+def _git_head(repository: Path) -> str:
+    """
+    Get the sha commit has for the head of a git repository.
+
+    Args:
+        repository: A directory under version control with git to get the sha commit of.
+
+    Returns:
+        The sha commit of head for `repository`.
+
+    Examples:
+        >>> from pathlib import Path
+        >>> _git_head(Path("~/Desktop/GitHub/HopkinsIDD/flepiMoP"))
+        'efe896b1a5e4f8e33667c170cd5319d6ef1e3db5'
+    """
+    git_cmd = _shutil_which("git")
+    proc = subprocess.run(
+        [git_cmd, "rev-parse", "HEAD"],
+        cwd=repository.expanduser().absolute(),
+        capture_output=True,
+        check=True,
+    )
+    return proc.stdout.decode().strip()
diff --git a/flepimop/gempyor_pkg/tests/utils/test__git_head.py b/flepimop/gempyor_pkg/tests/utils/test__git_head.py
new file mode 100644
index 000000000..52d53ffc8
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/utils/test__git_head.py
@@ -0,0 +1,54 @@
+import os
+from pathlib import Path
+from unittest.mock import patch
+import subprocess
+
+import pytest
+
+from gempyor.utils import _git_head, _shutil_which
+
+
+@pytest.mark.parametrize(
+    "repository",
+    (Path("/mock/repository"), Path("relative/repository/path"), Path("~/repo")),
+)
+@pytest.mark.parametrize(
+    "sha",
+    (
+        "59fe36d13fe34b6c1fb5c92bf8c53b83bd3ba593",
+        "bba583acf3c4b17ab3241288bff4bcad271a807c",
+    ),
+)
+def test_output_validation(repository: Path, sha: str) -> None:
+    def shutil_which_wraps(
+        cmd: str,
+        mode: int = os.F_OK | os.X_OK,
+        path: str | bytes | os.PathLike | None = None,
+        check: bool = True,
+    ) -> str | None:
+        return (
+            "git"
+            if cmd == "git"
+            else _shutil_which(cmd, mode=mode, path=path, check=check)
+        )
+
+    def subprocess_run_wraps(args, **kwargs):
+        if os.path.basename(args[0]) == "git":
+            return subprocess.CompletedProcess(
+                args=args, returncode=0, stdout=f"{sha}\n".encode(), stderr=b""
+            )
+        return subprocess.run(args, **kwargs)
+
+    with patch(
+        "gempyor.utils._shutil_which", wraps=shutil_which_wraps
+    ) as shutil_which_patch:
+        with patch(
+            "gempyor._slurm.subprocess.run", wraps=subprocess_run_wraps
+        ) as subprocess_run_patch:
+            assert _git_head(repository) == sha
+            shutil_which_patch.assert_called_once_with("git")
+            subprocess_run_patch.assert_called_once()
+            args = subprocess_run_patch.call_args.args[0]
+            assert args == ["git", "rev-parse", "HEAD"]
+            kwargs = subprocess_run_patch.call_args.kwargs
+            assert kwargs["cwd"] == repository.expanduser().absolute()

From d6427397f2fcd7fb2d00aef1b9ec3653ff134d76 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Thu, 31 Oct 2024 12:02:02 -0400
Subject: [PATCH 07/78] Add `write_manifest` for batch manifest files

Added `gempyor.manifest.write_manifest` for writing batch manifest JSON
files containing metadata about the run. Also has extended functionality
to store more metadata then the current required.
---
 flepimop/gempyor_pkg/src/gempyor/manifest.py  | 87 +++++++++++++++++++
 .../tests/manifest/test_write_manifest.py     | 78 +++++++++++++++++
 2 files changed, 165 insertions(+)
 create mode 100644 flepimop/gempyor_pkg/src/gempyor/manifest.py
 create mode 100644 flepimop/gempyor_pkg/tests/manifest/test_write_manifest.py

diff --git a/flepimop/gempyor_pkg/src/gempyor/manifest.py b/flepimop/gempyor_pkg/src/gempyor/manifest.py
new file mode 100644
index 000000000..6b84332b3
--- /dev/null
+++ b/flepimop/gempyor_pkg/src/gempyor/manifest.py
@@ -0,0 +1,87 @@
+"""
+Generate manifest files containing batch run metadata.
+
+This module provides functionality for generating manifest json files which contain
+batch run metadata. Currently exported are:
+- `write_manifest`: The main interface to this module for writing manifest json files.
+"""
+
+__all__ = ["write_manifest"]
+
+
+import json
+from pathlib import Path
+import sys
+from typing import Any
+
+from gempyor.utils import _git_head
+
+
+def write_manifest(
+    job_name: str,
+    flepi_path: Path,
+    project_path: Path,
+    destination: Path | None = None,
+    **additional_meta: Any,
+) -> Path:
+    """
+    Write job metadata to a manifest file.
+
+    This function produces a manifest metadata file for a batch run. By default the
+    json generated by this function will contain:
+    * 'cmd': The command line arguments provided to the CLI script invoked.
+    * 'job_name': A human readable unique job name.
+    * 'data_sha': The git commit of the project git repository, called 'data' for
+        legacy reasons.
+    * 'flepimop_sha': The git commit of the flepiMoP git repository.
+    Further data can be provided via `**additional_meta`, but these values are
+    overridden by the defaults described above.
+
+    Args:
+        job_name: A user specified or generated from user specified values unique name
+            for the job.
+        flepi_path: The path to the flepiMoP git repository being used.
+        project_path: The path to the project git repository being used.
+        destination: Either a path to where the json file should be written or `None` to
+            write the json file to 'manifest.json' in the current working directory.
+        additional_meta: User specified additional fields added to the manifest json.
+            Values with the name 'cmd', 'job_name', 'data_sha', or 'flepimop_sha' will
+            be overridden by the default behavior. Must be a json encodable type.
+
+    Returns:
+        The path to the written json file.
+
+    Examples:
+        >>> from pathlib import Path
+        >>> flepi_path = Path("~/Desktop/GitHub/HopkinsIDD/flepiMoP").expanduser()
+        >>> project_path = Path("~/Desktop/GitHub/HopkinsIDD/flepimop_sample").expanduser()
+        >>> manifest = write_manifest("Foobar", flepi_path, project_path)
+        >>> manifest.name
+        'manifest.json'
+        >>> print(manifest.read_text())
+        {
+            "cmd": "",
+            "job_name": "Foobar",
+            "data_sha": "59fe36d13fe34b6c1fb5c92bf8c53b83bd3ba593",
+            "flepimop_sha": "2bdfbc74e69bdd0243ef8340dda238f5504f1ad9"
+        }
+    """
+    flepimop_sha = _git_head(flepi_path)
+    data_sha = _git_head(project_path)
+
+    manifest = {
+        "cmd": " ".join(sys.argv),
+        "job_name": job_name,
+        "data_sha": data_sha,
+        "flepimop_sha": flepimop_sha,
+    }
+    if additional_meta:
+        manifest = {**additional_meta, **manifest}
+
+    destination = (
+        Path("manifest.json").absolute() if destination is None else destination
+    )
+    with destination.open(mode="w", encoding="utf-8") as f:
+        json.dump(manifest, f, indent=4)
+
+    return destination
diff --git a/flepimop/gempyor_pkg/tests/manifest/test_write_manifest.py b/flepimop/gempyor_pkg/tests/manifest/test_write_manifest.py
new file mode 100644
index 000000000..618b5ff5d
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/manifest/test_write_manifest.py
@@ -0,0 +1,78 @@
+import hashlib
+import json
+from pathlib import Path
+from unittest.mock import patch
+from typing import Any
+
+import pytest
+
+from gempyor.manifest import write_manifest
+from gempyor.utils import _git_head
+
+
+@pytest.mark.parametrize("job_name", ("my job name", "flu scenario"))
+@pytest.mark.parametrize(
+    "flepi_path", (Path("/path/to/flepiMoP"), Path("flepiMoP"), Path("~/flepiMoP"))
+)
+@pytest.mark.parametrize(
+    "project_path", (Path("/path/to/project"), Path("project"), Path("~/project"))
+)
+@pytest.mark.parametrize(
+    "destination",
+    (
+        None,
+        Path("manifest.json"),
+        Path("/absolute/manifest.json"),
+        Path("not_manifest.json"),
+    ),
+)
+@pytest.mark.parametrize(
+    "additional_meta",
+    ({}, {"var1": 1, "var2": "abc", "other": 3.14, "bool": True, "null": None}),
+)
+def test_output_validation(
+    monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
+    job_name: str,
+    flepi_path: Path,
+    project_path: Path,
+    destination: Path | None,
+    additional_meta: dict[str, Any],
+) -> None:
+    monkeypatch.chdir(tmp_path)
+    if isinstance(destination, Path) and destination.is_absolute():
+        destination = tmp_path / destination.name
+
+    def git_head_wraps(repository: Path) -> str:
+        return (
+            hashlib.sha1(str(repository).encode()).hexdigest()
+            if repository in [flepi_path, project_path]
+            else _git_head(repository)
+        )
+
+    with patch("gempyor.manifest._git_head", wraps=git_head_wraps) as git_head_patch:
+        manifest_file = write_manifest(
+            job_name,
+            flepi_path,
+            project_path,
+            destination=destination,
+            **additional_meta
+        )
+        assert (
+            manifest_file == Path("manifest.json").absolute()
+            if destination is None
+            else destination
+        )
+        with manifest_file.open(encoding="utf-8") as f:
+            manifest = json.load(f)
+
+        assert "cmd" in manifest and isinstance(manifest["cmd"], str)
+        del manifest["cmd"]
+        assert manifest == {
+            **additional_meta,
+            **{
+                "job_name": job_name,
+                "data_sha": hashlib.sha1(str(project_path).encode()).hexdigest(),
+                "flepimop_sha": hashlib.sha1(str(flepi_path).encode()).hexdigest(),
+            },
+        }

From 75304cd70cb2dd9a86a71272770f509e458255ab Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Thu, 31 Oct 2024 12:18:43 -0400
Subject: [PATCH 08/78] Move `manifest` module into `batch`

The `manifest` module will be too small to have as its own, moved into
`batch` and will start consolidating other batch functionality into as
well.
---
 flepimop/gempyor_pkg/src/gempyor/{manifest.py => batch.py} | 7 +++----
 .../tests/{manifest => batch}/test_write_manifest.py       | 4 ++--
 2 files changed, 5 insertions(+), 6 deletions(-)
 rename flepimop/gempyor_pkg/src/gempyor/{manifest.py => batch.py} (91%)
 rename flepimop/gempyor_pkg/tests/{manifest => batch}/test_write_manifest.py (94%)

diff --git a/flepimop/gempyor_pkg/src/gempyor/manifest.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
similarity index 91%
rename from flepimop/gempyor_pkg/src/gempyor/manifest.py
rename to flepimop/gempyor_pkg/src/gempyor/batch.py
index 6b84332b3..62d65db32 100644
--- a/flepimop/gempyor_pkg/src/gempyor/manifest.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -1,9 +1,8 @@
 """
-Generate manifest files containing batch run metadata.
+Functionality for creating and submitting batch jobs.
 
-This module provides functionality for generating manifest json files which contain
-batch run metadata. Currently exported are:
-- `write_manifest`: The main interface to this module for writing manifest json files.
+This module provides functionality for required for batch jobs, including creating 
+metadata and job size calculations for example.
 """
 
 __all__ = ["write_manifest"]
diff --git a/flepimop/gempyor_pkg/tests/manifest/test_write_manifest.py b/flepimop/gempyor_pkg/tests/batch/test_write_manifest.py
similarity index 94%
rename from flepimop/gempyor_pkg/tests/manifest/test_write_manifest.py
rename to flepimop/gempyor_pkg/tests/batch/test_write_manifest.py
index 618b5ff5d..5374ade71 100644
--- a/flepimop/gempyor_pkg/tests/manifest/test_write_manifest.py
+++ b/flepimop/gempyor_pkg/tests/batch/test_write_manifest.py
@@ -6,7 +6,7 @@
 
 import pytest
 
-from gempyor.manifest import write_manifest
+from gempyor.batch import write_manifest
 from gempyor.utils import _git_head
 
 
@@ -50,7 +50,7 @@ def git_head_wraps(repository: Path) -> str:
             else _git_head(repository)
         )
 
-    with patch("gempyor.manifest._git_head", wraps=git_head_wraps) as git_head_patch:
+    with patch("gempyor.batch._git_head", wraps=git_head_wraps) as git_head_patch:
         manifest_file = write_manifest(
             job_name,
             flepi_path,

From 3eb1cb270967b5fc6b05da4e369188446ef37167 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Thu, 31 Oct 2024 12:30:14 -0400
Subject: [PATCH 09/78] Moved `_slurm` module into `batch`

Moved the `_slurm` module into the new `batch` module to similar reasons
as `manifest`. There is not enough slurm functionality to justify an
individual module and slurm interactions will only occur in the context
of working with batch jobs.
---
 flepimop/gempyor_pkg/src/gempyor/_slurm.py    | 102 ------------------
 flepimop/gempyor_pkg/src/gempyor/batch.py     |  95 +++++++++++++++-
 .../tests/{_slurm => batch}/test__sbatch.py   |   6 +-
 .../gempyor_pkg/tests/utils/test__git_head.py |   2 +-
 4 files changed, 97 insertions(+), 108 deletions(-)
 delete mode 100644 flepimop/gempyor_pkg/src/gempyor/_slurm.py
 rename flepimop/gempyor_pkg/tests/{_slurm => batch}/test__sbatch.py (96%)

diff --git a/flepimop/gempyor_pkg/src/gempyor/_slurm.py b/flepimop/gempyor_pkg/src/gempyor/_slurm.py
deleted file mode 100644
index 7abc9c0b0..000000000
--- a/flepimop/gempyor_pkg/src/gempyor/_slurm.py
+++ /dev/null
@@ -1,102 +0,0 @@
-"""
-Internal utilities for interacting with slurm.
-"""
-
-__all__ = []
-
-
-from pathlib import Path
-from shlex import quote
-import subprocess
-from typing import Any, Literal
-
-from .logging import get_script_logger
-from .utils import _shutil_which
-
-
-def _sbatch(
-    script: Path,
-    environment_variables: dict[str, Any] | Literal["all", "nil", "none"],
-    options: dict[str, Any],
-    verbosity: int | None,
-    dry_run: bool,
-) -> None:
-    """
-    Submit a job to slurm via the `sbatch` command.
-
-    Args:
-        script: The batch file to submit to slurm.
-        environment_variables: Environment variables to pass to the job via the
-            '--export' option. Keys correspond to the variable name and the value is
-            the variable value. All values are coerced to a string and then escaped.
-            Or can be a literal for one of the '--export' option's special values.
-        options: Options to pass when calling sbatch. Keys correspond to the option
-            name and the value is the option value. Options can be provided as either
-            the long or short name, but this function will not be able to determine that
-            these are duplicates so only using long names is recommended.
-        verbosity: A integer verbosity level to enable logging or `None` for no logging.
-        dry_run: A boolean indicating if this is a dry run or not, if set to `True` this
-            function will not actually submit a job to slurm.
-
-    Raises:
-        ValueError: If 'export' is given as a key in `options` instead of using
-            `environment_variables`.
-
-    Examples:
-        >>> from pathlib import Path
-        >>> _sbatch(
-        ...     Path("my_batch_script.sbatch"),
-        ...     {"VAR1": 1, "VAR2": "true"},
-        ...     {"J": "My job name", "output": Path("out.log")},
-        ...     3,
-        ...     True,
-        ... )
-        2024-10-31 09:26:48,361:DEBUG:gempyor._slurm> Using batch script '/my_batch_script.sbatch' to submit to slurm.
-        2024-10-31 09:26:48,363:INFO:gempyor._slurm> If not dry mode would have submitted to slurm with: /usr/bin/sbatch --export=VAR1=1,VAR2=true -J='My job name' --output=out.log /my_batch_script.sbatch.
-
-    See Also:
-        [`sbatch`'s documentation](https://slurm.schedmd.com/sbatch.html)
-    """
-    if "export" in options:
-        raise ValueError(
-            "Found 'export' in `options`, please use `environment_variables` instead."
-        )
-
-    if verbosity is not None:
-        logger = get_script_logger(__name__, verbosity)
-        logger.debug("Using batch script '%s' to submit to slurm", script.absolute())
-
-    if isinstance(environment_variables, dict):
-        env_vars = [f"{k}={quote(str(v))}" for k, v in environment_variables.items()]
-        export = "--export=" + ",".join(env_vars)
-    else:
-        export = f"--export={environment_variables.upper()}"
-    options = [
-        f"{'-' if len(k) == 1 else '--'}{k}={quote(str(v))}" for k, v in options.items()
-    ]
-    sbatch_cmd = _shutil_which("sbatch")
-    if len(export) > 9:
-        cmd_args = [sbatch_cmd, export] + options + [str(script.absolute())]
-    else:
-        cmd_args = [sbatch_cmd] + options + [str(script.absolute())]
-
-    if dry_run:
-        if verbosity is not None:
-            logger.info(
-                "If not dry mode would have submitted to slurm with: %s",
-                " ".join(cmd_args),
-            )
-        return
-
-    if verbosity is not None:
-        logger.info("Submitting to slurm with: %s", " ".join(cmd_args))
-    process = subprocess.run(cmd_args, check=True, capture_output=True)
-    if verbosity is not None:
-        if process.stdout:
-            logger.debug(
-                "Captured stdout from sbatch submission: %s", process.stdout.decode()
-            )
-        if process.stderr:
-            logger.error(
-                "Captured stderr from sbatch submission: %s", process.stderr.decode()
-            )
diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 62d65db32..cdb739c58 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -10,10 +10,13 @@
 
 import json
 from pathlib import Path
+from shlex import quote
+import subprocess
 import sys
-from typing import Any
+from typing import Any, Literal
 
-from gempyor.utils import _git_head
+from .logging import get_script_logger
+from .utils import _git_head, _shutil_which
 
 
 def write_manifest(
@@ -84,3 +87,91 @@ def write_manifest(
         json.dump(manifest, f, indent=4)
 
     return destination
+
+
+def _sbatch(
+    script: Path,
+    environment_variables: dict[str, Any] | Literal["all", "nil", "none"],
+    options: dict[str, Any],
+    verbosity: int | None,
+    dry_run: bool,
+) -> None:
+    """
+    Submit a job to slurm via the `sbatch` command.
+
+    Args:
+        script: The batch file to submit to slurm.
+        environment_variables: Environment variables to pass to the job via the
+            '--export' option. Keys correspond to the variable name and the value is
+            the variable value. All values are coerced to a string and then escaped.
+            Or can be a literal for one of the '--export' option's special values.
+        options: Options to pass when calling sbatch. Keys correspond to the option
+            name and the value is the option value. Options can be provided as either
+            the long or short name, but this function will not be able to determine that
+            these are duplicates so only using long names is recommended.
+        verbosity: A integer verbosity level to enable logging or `None` for no logging.
+        dry_run: A boolean indicating if this is a dry run or not, if set to `True` this
+            function will not actually submit a job to slurm.
+
+    Raises:
+        ValueError: If 'export' is given as a key in `options` instead of using
+            `environment_variables`.
+
+    Examples:
+        >>> from pathlib import Path
+        >>> _sbatch(
+        ...     Path("my_batch_script.sbatch"),
+        ...     {"VAR1": 1, "VAR2": "true"},
+        ...     {"J": "My job name", "output": Path("out.log")},
+        ...     3,
+        ...     True,
+        ... )
+        2024-10-31 09:26:48,361:DEBUG:gempyor._slurm> Using batch script '/my_batch_script.sbatch' to submit to slurm.
+        2024-10-31 09:26:48,363:INFO:gempyor._slurm> If not dry mode would have submitted to slurm with: /usr/bin/sbatch --export=VAR1=1,VAR2=true -J='My job name' --output=out.log /my_batch_script.sbatch.
+
+    See Also:
+        [`sbatch`'s documentation](https://slurm.schedmd.com/sbatch.html)
+    """
+    if "export" in options:
+        raise ValueError(
+            "Found 'export' in `options`, please use `environment_variables` instead."
+        )
+
+    if verbosity is not None:
+        logger = get_script_logger(__name__, verbosity)
+        logger.debug("Using batch script '%s' to submit to slurm", script.absolute())
+
+    if isinstance(environment_variables, dict):
+        env_vars = [f"{k}={quote(str(v))}" for k, v in environment_variables.items()]
+        export = "--export=" + ",".join(env_vars)
+    else:
+        export = f"--export={environment_variables.upper()}"
+    options = [
+        f"{'-' if len(k) == 1 else '--'}{k}={quote(str(v))}" for k, v in options.items()
+    ]
+    sbatch_cmd = _shutil_which("sbatch")
+    if len(export) > 9:
+        cmd_args = [sbatch_cmd, export] + options + [str(script.absolute())]
+    else:
+        cmd_args = [sbatch_cmd] + options + [str(script.absolute())]
+
+    if dry_run:
+        if verbosity is not None:
+            logger.info(
+                "If not dry mode would have submitted to slurm with: %s",
+                " ".join(cmd_args),
+            )
+        return
+
+    if verbosity is not None:
+        logger.info("Submitting to slurm with: %s", " ".join(cmd_args))
+    process = subprocess.run(cmd_args, check=True, capture_output=True)
+    if verbosity is not None:
+        if process.stdout:
+            logger.debug(
+                "Captured stdout from sbatch submission: %s", process.stdout.decode()
+            )
+        if process.stderr:
+            logger.error(
+                "Captured stderr from sbatch submission: %s", process.stderr.decode()
+            )
diff --git a/flepimop/gempyor_pkg/tests/_slurm/test__sbatch.py b/flepimop/gempyor_pkg/tests/batch/test__sbatch.py
similarity index 96%
rename from flepimop/gempyor_pkg/tests/_slurm/test__sbatch.py
rename to flepimop/gempyor_pkg/tests/batch/test__sbatch.py
index e846f5c37..c2fab6e9f 100644
--- a/flepimop/gempyor_pkg/tests/_slurm/test__sbatch.py
+++ b/flepimop/gempyor_pkg/tests/batch/test__sbatch.py
@@ -7,7 +7,7 @@
 
 import pytest
 
-from gempyor._slurm import _sbatch
+from gempyor.batch import _sbatch
 from gempyor.logging import _get_logging_level
 from gempyor.utils import _shutil_which
 
@@ -91,10 +91,10 @@ def subprocess_run_wraps(args, **kwargs):
         return subprocess.run(args, **kwargs)
 
     with patch(
-        "gempyor._slurm._shutil_which", wraps=shutil_which_wraps
+        "gempyor.batch._shutil_which", wraps=shutil_which_wraps
     ) as shutil_which_patch:
         with patch(
-            "gempyor._slurm.subprocess.run", wraps=subprocess_run_wraps
+            "gempyor.batch.subprocess.run", wraps=subprocess_run_wraps
         ) as subprocess_run_patch:
             assert (
                 _sbatch(
diff --git a/flepimop/gempyor_pkg/tests/utils/test__git_head.py b/flepimop/gempyor_pkg/tests/utils/test__git_head.py
index 52d53ffc8..0f789bea6 100644
--- a/flepimop/gempyor_pkg/tests/utils/test__git_head.py
+++ b/flepimop/gempyor_pkg/tests/utils/test__git_head.py
@@ -43,7 +43,7 @@ def subprocess_run_wraps(args, **kwargs):
         "gempyor.utils._shutil_which", wraps=shutil_which_wraps
     ) as shutil_which_patch:
         with patch(
-            "gempyor._slurm.subprocess.run", wraps=subprocess_run_wraps
+            "gempyor.batch.subprocess.run", wraps=subprocess_run_wraps
         ) as subprocess_run_patch:
             assert _git_head(repository) == sha
             shutil_which_patch.assert_called_once_with("git")

From 06dbf0de387eedbdedc41f6a21be22edeee84245 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Thu, 31 Oct 2024 14:11:36 -0400
Subject: [PATCH 10/78] Initial `JobSize` class

Added a dataclass to represent batch job sizes, including light
validation.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py     | 32 ++++++++++++++++++-
 .../tests/batch/test_job_size_class.py        | 31 ++++++++++++++++++
 2 files changed, 62 insertions(+), 1 deletion(-)
 create mode 100644 flepimop/gempyor_pkg/tests/batch/test_job_size_class.py

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index cdb739c58..9b0fc95ee 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -5,9 +5,10 @@
 metadata and job size calculations for example.
 """
 
-__all__ = ["write_manifest"]
+__all__ = ["JobSize", "write_manifest"]
 
 
+from dataclasses import dataclass
 import json
 from pathlib import Path
 from shlex import quote
@@ -19,6 +20,35 @@
 from .utils import _git_head, _shutil_which
 
 
+@dataclass(frozen=True, slots=True)
+class JobSize:
+    """
+    A batch submission job size.
+
+    Attributes:
+        jobs: The number of jobs to use.
+        simulations: The number of simulations to run per a block.
+        blocks: The number of sequential blocks to run per a job.
+
+    Raises:
+        ValueError: If any of the attributes are less than 1.
+    """
+
+    jobs: int
+    simulations: int
+    blocks: int
+
+    def __post_init__(self) -> None:
+        for p in self.__slots__:
+            if (val := getattr(self, p)) < 1:
+                raise ValueError(
+                    (
+                        f"The '{p}' attribute must be greater than 0, "
+                        f"but instead was given '{val}'."
+                    )
+                )
+
+
 def write_manifest(
     job_name: str,
     flepi_path: Path,
diff --git a/flepimop/gempyor_pkg/tests/batch/test_job_size_class.py b/flepimop/gempyor_pkg/tests/batch/test_job_size_class.py
new file mode 100644
index 000000000..cdee832d4
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/batch/test_job_size_class.py
@@ -0,0 +1,31 @@
+from typing import Literal
+
+import pytest
+
+from gempyor.batch import JobSize
+
+
+@pytest.mark.parametrize(
+    "kwargs",
+    (
+        {"jobs": 0, "simulations": 1, "blocks": 1},
+        {"jobs": 1, "simulations": 0, "blocks": 1},
+        {"jobs": 1, "simulations": 1, "blocks": 0},
+        {"jobs": 0, "simulations": 0, "blocks": 1},
+        {"jobs": 1, "simulations": 0, "blocks": 0},
+        {"jobs": 0, "simulations": 1, "blocks": 0},
+        {"jobs": 0, "simulations": 0, "blocks": 0},
+    ),
+)
+def test_less_than_one_value_error(
+    kwargs: dict[Literal["jobs", "simulations", "blocks"], int]
+) -> None:
+    param = next(k for k, v in kwargs.items() if v < 1)
+    with pytest.raises(
+        ValueError,
+        match=(
+            f"^The '{param}' attribute must be greater than 0, "
+            f"but instead was given '{kwargs.get(param)}'.$"
+        ),
+    ):
+        JobSize(**kwargs)

From 10faf777132c0f2460db1632b440968337c6d8a8 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Thu, 31 Oct 2024 15:18:52 -0400
Subject: [PATCH 11/78] Add `JobSize.size_from_jobs_sims_blocks`

Added a class method to `gempyor.batch.JobSize` class that is slightly
more convenient to use from a CLI script. Provides a good home for
future code realted to GH-360. Corresponding tests and docs.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py     | 88 ++++++++++++++++++-
 .../tests/batch/test_job_size_class.py        | 78 +++++++++++++++-
 2 files changed, 162 insertions(+), 4 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 9b0fc95ee..8fd22d23c 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -10,6 +10,7 @@
 
 from dataclasses import dataclass
 import json
+import math
 from pathlib import Path
 from shlex import quote
 import subprocess
@@ -48,6 +49,89 @@ def __post_init__(self) -> None:
                     )
                 )
 
+    @classmethod
+    def size_from_jobs_sims_blocks(
+        cls,
+        jobs: int | None,
+        simulations: int | None,
+        blocks: int | None,
+        iterations_per_slot: int | None,
+        slots: int | None,
+        subpops: int | None,
+        batch_system: Literal["aws", "local", "slurm"],
+    ) -> "JobSize":
+        """
+        Infer a job size from several explicit and implicit parameters.
+
+        Args:
+            jobs: An explicit number of jobs.
+            simulations: An explicit number of simulations per a block.
+            blocks: An explicit number of blocks per a job.
+            iterations_per_slot: A total number of iterations per a job, which is
+                simulations times blocks. Required if `simulations` or `blocks` is
+                not given.
+            slots: An implicit number of slots to use for the job. Required if `jobs`
+                is not given.
+            subpops: The number of subpopulations being considered in this job. Affects
+                the inferred simulations per a job on AWS. Required if `simulations`
+                and `blocks` are not given.
+            batch_size: The system the job is being sized for. Affects the inferred
+                simulations per a job.
+
+        Returns:
+            A job size instance with either the explicit or inferred job sizing.
+
+        Examples:
+            >>> JobSize.size_from_jobs_sims_blocks(1, 2, 3, None, None, None, "local")
+            JobSize(jobs=1, simulations=2, blocks=3)
+            >>> JobSize.size_from_jobs_sims_blocks(
+            ...     None, None, None, 100, 10, 25, "local"
+            ... )
+            JobSize(jobs=10, simulations=100, blocks=1)
+            >>> JobSize.size_from_jobs_sims_blocks(None, None, 4, 100, 10, 25, "local")
+            JobSize(jobs=10, simulations=25, blocks=4)
+
+        Raises:
+            ValueError: If `iterations_per_slot` is `None` and either `simulations` or
+                `blocks` is `None`.
+            ValueError: If `jobs` and `slots` are both `None`.
+            ValueError: If `simulations`, `blocks`, and `subpops` are all `None`.
+        """
+        if iterations_per_slot is None and (simulations is None or blocks is None):
+            raise ValueError(
+                (
+                    "If simulations and blocks are not all explicitly "
+                    "provided then an iterations per slot must be given."
+                )
+            )
+
+        jobs = slots if jobs is None else jobs
+        if jobs is None:
+            raise ValueError(
+                "If jobs is not explicitly provided, it must be given via slots."
+            )
+
+        if simulations is None:
+            if blocks is None:
+                if subpops is None:
+                    raise ValueError(
+                        (
+                            "If simulations and blocks are not explicitly "
+                            "provided, then a subpops must be given."
+                        )
+                    )
+                if batch_system == "aws":
+                    simulations = 5 * math.ceil(max(60 - math.sqrt(subpops), 10) / 5)
+                else:
+                    simulations = iterations_per_slot
+            else:
+                simulations = math.ceil(iterations_per_slot / blocks)
+
+        if blocks is None:
+            blocks = math.ceil(iterations_per_slot / simulations)
+
+        return cls(jobs=jobs, simulations=simulations, blocks=blocks)
+
 
 def write_manifest(
     job_name: str,
@@ -110,9 +194,7 @@ def write_manifest(
     if additional_meta:
         manifest = {**additional_meta, **manifest}
 
-    destination = (
-        Path("manifest.json").absolute() if destination is None else destination
-    )
+    destination = Path("manifest.json").absolute() if destination is None else destination
     with destination.open(mode="w", encoding="utf-8") as f:
         json.dump(manifest, f, indent=4)
 
diff --git a/flepimop/gempyor_pkg/tests/batch/test_job_size_class.py b/flepimop/gempyor_pkg/tests/batch/test_job_size_class.py
index cdee832d4..7ffe4d7ac 100644
--- a/flepimop/gempyor_pkg/tests/batch/test_job_size_class.py
+++ b/flepimop/gempyor_pkg/tests/batch/test_job_size_class.py
@@ -1,4 +1,5 @@
-from typing import Literal
+from itertools import product
+from typing import Generator, Literal
 
 import pytest
 
@@ -29,3 +30,78 @@ def test_less_than_one_value_error(
         ),
     ):
         JobSize(**kwargs)
+
+
+@pytest.mark.parametrize(("simulations", "blocks"), [(None, None), (1, None), (None, 1)])
+def test_size_from_jobs_sims_blocks_iteration_value_error(
+    simulations: int | None, blocks: int | None
+) -> None:
+    with pytest.raises(
+        ValueError,
+        match=(
+            "^If simulations and blocks are not all explicitly "
+            "provided then an iterations per slot must be given.$"
+        ),
+    ):
+        JobSize.size_from_jobs_sims_blocks(1, simulations, blocks, None, 1, 1, "aws")
+
+
+def test_size_from_jobs_sims_blocks_slots_value_error() -> None:
+    with pytest.raises(
+        ValueError,
+        match="^If jobs is not explicitly provided, it must be given via slots.$",
+    ):
+        JobSize.size_from_jobs_sims_blocks(None, 1, 1, 1, None, 1, "aws")
+
+
+def test_size_from_jobs_sims_blocks_subpops_value_error() -> None:
+    with pytest.raises(
+        ValueError,
+        match=(
+            "^If simulations and blocks are not explicitly "
+            "provided, then a subpops must be given.$"
+        ),
+    ):
+        JobSize.size_from_jobs_sims_blocks(1, None, None, 1, 1, None, "aws")
+
+
+def generate_size_from_jobs_sims_blocks(
+    *args: int | None,
+) -> Generator[tuple[int | None, ...], None, None]:
+    for combo in product(args, repeat=6):
+        jobs, simulations, blocks, iterations_per_slot, slots, subpops = combo
+        if iterations_per_slot is None and (simulations is None or blocks is None):
+            continue
+        elif jobs is None and slots is None:
+            continue
+        elif simulations is None and blocks is None and subpops is None:
+            continue
+        yield combo
+
+
+@pytest.mark.parametrize("combo", generate_size_from_jobs_sims_blocks(None, 1, 10))
+def test_size_from_jobs_sims_blocks_output(combo: tuple[int | None, ...]) -> None:
+    jobs, simulations, blocks, iterations_per_slot, slots, subpops = combo
+    job_sizes_by_batch_system = {}
+    for batch_system in ("aws", "local", "slurm"):
+        job_size = JobSize.size_from_jobs_sims_blocks(
+            jobs, simulations, blocks, iterations_per_slot, slots, subpops, batch_system
+        )
+        assert (
+            job_size.jobs == jobs
+            if jobs is not None
+            else isinstance(job_size.jobs, int) and job_size.jobs > 0
+        )
+        assert (
+            job_size.simulations == simulations
+            if simulations is not None
+            else isinstance(job_size.simulations, int) and job_size.simulations > 0
+        )
+        assert (
+            job_size.blocks == blocks
+            if blocks is not None
+            else isinstance(job_size.blocks, int) and job_size.blocks > 0
+        )
+        job_sizes_by_batch_system[batch_system] = job_size
+    assert job_sizes_by_batch_system["local"] == job_sizes_by_batch_system["slurm"]
+    assert job_sizes_by_batch_system["local"].jobs == job_sizes_by_batch_system["aws"].jobs

From 498197c5e3338c37ce8aa87f17c076e860920789 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Thu, 31 Oct 2024 16:13:02 -0400
Subject: [PATCH 12/78] Reformat new files with line length 92

In anticipation of GH-358, minimize future work.
---
 flepimop/gempyor_pkg/src/gempyor/logging.py               | 6 +-----
 flepimop/gempyor_pkg/tests/batch/test__sbatch.py          | 8 ++------
 flepimop/gempyor_pkg/tests/batch/test_write_manifest.py   | 6 +-----
 .../gempyor_pkg/tests/logging/test__get_logging_level.py  | 4 +---
 .../gempyor_pkg/tests/logging/test_get_script_logger.py   | 4 +---
 5 files changed, 6 insertions(+), 22 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/logging.py b/flepimop/gempyor_pkg/src/gempyor/logging.py
index db5123823..cf31bafc3 100644
--- a/flepimop/gempyor_pkg/src/gempyor/logging.py
+++ b/flepimop/gempyor_pkg/src/gempyor/logging.py
@@ -75,11 +75,7 @@ def emit(self, record: logging.LogRecord) -> None:
             [`logging.Handler.emit`](https://docs.python.org/3/library/logging.html#logging.Handler.emit)
         """
         msg = self.format(record)
-        msg = (
-            f"{msg}."
-            if self._punctuate and not msg.endswith(self._punctuation)
-            else msg
-        )
+        msg = f"{msg}." if self._punctuate and not msg.endswith(self._punctuation) else msg
         click.echo(
             message=msg, file=self._file, nl=self._nl, err=self._err, color=self._color
         )
diff --git a/flepimop/gempyor_pkg/tests/batch/test__sbatch.py b/flepimop/gempyor_pkg/tests/batch/test__sbatch.py
index c2fab6e9f..1c69d73d2 100644
--- a/flepimop/gempyor_pkg/tests/batch/test__sbatch.py
+++ b/flepimop/gempyor_pkg/tests/batch/test__sbatch.py
@@ -57,9 +57,7 @@ def test_export_option_causes_value_error(sample_sbatch_script: Path) -> None:
         ),
     ),
 )
-@pytest.mark.parametrize(
-    "verbosity", (None, logging.DEBUG, logging.INFO, logging.WARNING)
-)
+@pytest.mark.parametrize("verbosity", (None, logging.DEBUG, logging.INFO, logging.WARNING))
 @pytest.mark.parametrize("dry_run", (True, False))
 def test_output_validation(
     caplog: pytest.LogCaptureFixture,
@@ -128,9 +126,7 @@ def subprocess_run_wraps(args, **kwargs):
 
             if not dry_run:
                 args: list[str] = subprocess_run_patch.call_args.args[0]
-                assert len(args) == 2 + len(options) + min(
-                    len(environment_variables), 1
-                )
+                assert len(args) == 2 + len(options) + min(len(environment_variables), 1)
                 assert Path(args[-1]) == sample_sbatch_script
                 if environment_variables:
                     assert args[1].startswith("--export=")
diff --git a/flepimop/gempyor_pkg/tests/batch/test_write_manifest.py b/flepimop/gempyor_pkg/tests/batch/test_write_manifest.py
index 5374ade71..69234d0bc 100644
--- a/flepimop/gempyor_pkg/tests/batch/test_write_manifest.py
+++ b/flepimop/gempyor_pkg/tests/batch/test_write_manifest.py
@@ -52,11 +52,7 @@ def git_head_wraps(repository: Path) -> str:
 
     with patch("gempyor.batch._git_head", wraps=git_head_wraps) as git_head_patch:
         manifest_file = write_manifest(
-            job_name,
-            flepi_path,
-            project_path,
-            destination=destination,
-            **additional_meta
+            job_name, flepi_path, project_path, destination=destination, **additional_meta
         )
         assert (
             manifest_file == Path("manifest.json").absolute()
diff --git a/flepimop/gempyor_pkg/tests/logging/test__get_logging_level.py b/flepimop/gempyor_pkg/tests/logging/test__get_logging_level.py
index 210d0bd69..67480442c 100644
--- a/flepimop/gempyor_pkg/tests/logging/test__get_logging_level.py
+++ b/flepimop/gempyor_pkg/tests/logging/test__get_logging_level.py
@@ -28,7 +28,5 @@ def test__get_logging_level_negative_verbosity_value_error(verbosity: int) -> No
         (5, logging.DEBUG),
     ),
 )
-def test__get_logging_level_output_validation(
-    verbosity: int, expected_level: int
-) -> None:
+def test__get_logging_level_output_validation(verbosity: int, expected_level: int) -> None:
     assert _get_logging_level(verbosity) == expected_level
diff --git a/flepimop/gempyor_pkg/tests/logging/test_get_script_logger.py b/flepimop/gempyor_pkg/tests/logging/test_get_script_logger.py
index 1e0c5a2d0..b96e69cb7 100644
--- a/flepimop/gempyor_pkg/tests/logging/test_get_script_logger.py
+++ b/flepimop/gempyor_pkg/tests/logging/test_get_script_logger.py
@@ -37,9 +37,7 @@ def test_get_script_logger(
     if log_format is None:
         logger = get_script_logger(name, verbosity, handler=handler)
     else:
-        logger = get_script_logger(
-            name, verbosity, handler=handler, log_format=log_format
-        )
+        logger = get_script_logger(name, verbosity, handler=handler, log_format=log_format)
 
     assert isinstance(logger, logging.Logger)
     assert logger.name == name

From d9b03a725f73ef32b8b6682b3e33d8504843c9b0 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Fri, 1 Nov 2024 08:50:14 -0400
Subject: [PATCH 13/78] Add `_format_cli_options` helper

Added internal utility to convert a dict of options into a list
appropriate for `subprocess.run` and similar.
---
 flepimop/gempyor_pkg/src/gempyor/utils.py     | 30 +++++++++++++++++++
 .../tests/utils/test__format_cli_options.py   | 26 ++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 flepimop/gempyor_pkg/tests/utils/test__format_cli_options.py

diff --git a/flepimop/gempyor_pkg/src/gempyor/utils.py b/flepimop/gempyor_pkg/src/gempyor/utils.py
index c39658ec0..20ed5de58 100644
--- a/flepimop/gempyor_pkg/src/gempyor/utils.py
+++ b/flepimop/gempyor_pkg/src/gempyor/utils.py
@@ -5,6 +5,7 @@
 import numbers
 import os
 from pathlib import Path
+from shlex import quote
 import shutil
 import subprocess
 import time
@@ -1115,3 +1116,32 @@ def _git_head(repository: Path) -> str:
         check=True,
     )
     return proc.stdout.decode().strip()
+
+
+def _format_cli_options(options: dict[str, Any]) -> list[str]:
+    """
+    Convert a dictionary of CLI options into a formatted list.
+
+    Args:
+        options: A dictionary where the keys correspond to the option name and the
+            values correspond to the option value. If the option name is one character
+            it's assumed to be a short name and prefixed with one dash. Values are
+            coerced to a string and then escaped for shell.
+
+    Returns:
+        A list of options that can be passed to
+        [`subprocess.run`](https://docs.python.org/3/library/subprocess.html#subprocess.run)
+        or similar functions.
+
+    Examples:
+        >>> from pathlib import Path
+        >>> _format_cli_options({"name": "foo bar fizz buzz"})
+        ["--name='foo bar fizz buzz'"]
+        >>> _format_cli_options({"o": Path("/path/to/output.log")})
+        ['-o=/path/to/output.log']
+        >>> _format_cli_options({"opt1": "```", "opt2": "$( echo 'Hello!')"})
+        ["--opt1='```'", '--opt2=\'$( echo \'"\'"\'Hello!\'"\'"\')\'']
+    """
+    return [
+        f"{'-' if len(k) == 1 else '--'}{k}={quote(str(v))}" for k, v in options.items()
+    ]
diff --git a/flepimop/gempyor_pkg/tests/utils/test__format_cli_options.py b/flepimop/gempyor_pkg/tests/utils/test__format_cli_options.py
new file mode 100644
index 000000000..3c7df10d3
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/utils/test__format_cli_options.py
@@ -0,0 +1,26 @@
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from gempyor.utils import _format_cli_options
+
+
+@pytest.mark.parametrize(
+    ("options", "formatted_options"),
+    (
+        ({"name": "foo bar fizz buzz"}, ["--name='foo bar fizz buzz'"]),
+        ({"o": Path("/path/to/output.log")}, ["-o=/path/to/output.log"]),
+        (
+            {"opt1": "```", "opt2": "$( echo 'Hello!')"},
+            ["--opt1='```'", "--opt2='$( echo '\"'\"'Hello!'\"'\"')'"],
+        ),
+        (
+            {"start": datetime(2024, 1, 1, 12, 34, 56), "J": "rm -rf ~"},
+            ["--start='2024-01-01 12:34:56'", "-J='rm -rf ~'"],
+        ),
+    ),
+)
+def test_output_validation(options: dict[str, Any], formatted_options: list[str]) -> None:
+    assert _format_cli_options(options) == formatted_options

From 23b315ed38228eb2f946734a63b6eb59cac26f7e Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Fri, 1 Nov 2024 09:08:54 -0400
Subject: [PATCH 14/78] Draft `_sbatch_template` function

Added `gempyor.batch._sbatch_template` that is a thin wrapper around
`_sbatch` that takes advantage of templating. There are docs, but no
unit tests. Still determining how this interface should be best setup.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py | 57 +++++++++++++++++++++--
 1 file changed, 53 insertions(+), 4 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 8fd22d23c..8faa19747 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -17,8 +17,9 @@
 import sys
 from typing import Any, Literal
 
+from ._jinja import _render_template_to_file, _render_template_to_temp_file
 from .logging import get_script_logger
-from .utils import _git_head, _shutil_which
+from .utils import _format_cli_options, _git_head, _shutil_which
 
 
 @dataclass(frozen=True, slots=True)
@@ -258,9 +259,7 @@ def _sbatch(
         export = "--export=" + ",".join(env_vars)
     else:
         export = f"--export={environment_variables.upper()}"
-    options = [
-        f"{'-' if len(k) == 1 else '--'}{k}={quote(str(v))}" for k, v in options.items()
-    ]
+    options = _format_cli_options(options)
     sbatch_cmd = _shutil_which("sbatch")
     if len(export) > 9:
         cmd_args = [sbatch_cmd, export] + options + [str(script.absolute())]
@@ -287,3 +286,53 @@ def _sbatch(
             logger.error(
                 "Captured stderr from sbatch submission: %s", process.stderr.decode()
             )
+
+
+def _sbatch_template(
+    template: str,
+    script: Path | None,
+    template_data: dict[str, Any],
+    environment_variables: dict[str, Any] | Literal["all", "nil", "none"],
+    options: dict[str, Any],
+    verbosity: int | None,
+    dry_run: bool,
+) -> Path:
+    """
+    Submit a job from a template to slurm via the `sbatch` command.
+
+    Args:
+        template: The name of the template to use for rendering the sbatch script.
+        script: Either the path of where to save the rendered sbatch script to or `None`
+            for a tmp file.
+        template_data: Data accessible to the template when rendering.
+        environment_variables: Environment variables to pass to the job via the
+            '--export' option. Keys correspond to the variable name and the value is
+            the variable value. All values are coerced to a string and then escaped.
+            Or can be a literal for one of the '--export' option's special values.
+        options: Options to pass when calling sbatch. Keys correspond to the option
+            name and the value is the option value. Options can be provided as either
+            the long or short name, but this function will not be able to determine that
+            these are duplicates so only using long names is recommended.
+        verbosity: A integer verbosity level to enable logging or `None` for no logging.
+        dry_run: A boolean indicating if this is a dry run or not, if set to `True` this
+            function will not actually submit a job to slurm.
+
+    Returns:
+        The rendered sbatch script, either a tmp file if `script` is None otherwise
+        `script` is returned.
+
+    Raises:
+        ValueError: If 'options' is found in `template_data` and `options` is not empty.
+    """
+    template_data["interpreter"] = template_data.get("interpreter", "bash")
+    if "options" in template_data and options:
+        raise ValueError(
+            "Found 'options' in `template_data` but `options` is not empty, can only one."
+        )
+    template_data["options"] = _format_cli_options(template_data.get("options", options))
+    if script is None:
+        script = _render_template_to_temp_file(template, template_data, suffix=".sbatch")
+    else:
+        _render_template_to_file(template, template_data, script)
+    _sbatch(script, environment_variables, {}, verbosity, dry_run)
+    return script

From ff2d1f50f9dbf287511ee095c7431ddf7a31f5b9 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Fri, 1 Nov 2024 13:11:13 -0400
Subject: [PATCH 15/78] Initial pass at generic cluster config

Draft solution to GH-342 with the ability to extend to other types of
generic config/info in the future. Add `gempyor.info` module with
exported function `get_cluster_info` which pulls and parses an
information yaml into a pydantic model.
---
 .gitignore                                    |  3 +
 flepimop/gempyor_pkg/pyproject.toml           |  2 +
 flepimop/gempyor_pkg/src/gempyor/info.py      | 70 +++++++++++++++++++
 .../gempyor_pkg/tests/info/test__get_info.py  | 56 +++++++++++++++
 .../tests/info/test_get_cluster_info.py       | 15 ++++
 info/cluster/longleaf.yml                     |  7 ++
 info/cluster/rockfish.yml                     |  9 +++
 info/cluster/schema.yml                       | 25 +++++++
 8 files changed, 187 insertions(+)
 create mode 100644 flepimop/gempyor_pkg/src/gempyor/info.py
 create mode 100644 flepimop/gempyor_pkg/tests/info/test__get_info.py
 create mode 100644 flepimop/gempyor_pkg/tests/info/test_get_cluster_info.py
 create mode 100644 info/cluster/longleaf.yml
 create mode 100644 info/cluster/rockfish.yml
 create mode 100644 info/cluster/schema.yml

diff --git a/.gitignore b/.gitignore
index 6eb84927e..1e9726360 100644
--- a/.gitignore
+++ b/.gitignore
@@ -78,3 +78,6 @@ flepimop/gempyor_pkg/.coverage
 
 # Environment variables
 .env
+
+# info/ directory
+info/**/*.json
diff --git a/flepimop/gempyor_pkg/pyproject.toml b/flepimop/gempyor_pkg/pyproject.toml
index 6f90e1053..53d7c921a 100644
--- a/flepimop/gempyor_pkg/pyproject.toml
+++ b/flepimop/gempyor_pkg/pyproject.toml
@@ -20,11 +20,13 @@ dependencies = [
     "emcee",
     "graphviz",
     "h5py",
+    "Jinja2",
     "matplotlib",
     "numba>=0.53.1",
     "numpy",
     "pandas",
     "pyarrow",
+    "pydantic",
     "scipy",
     "seaborn",
     "sympy",
diff --git a/flepimop/gempyor_pkg/src/gempyor/info.py b/flepimop/gempyor_pkg/src/gempyor/info.py
new file mode 100644
index 000000000..0b8c7055a
--- /dev/null
+++ b/flepimop/gempyor_pkg/src/gempyor/info.py
@@ -0,0 +1,70 @@
+__all__ = ["Cluster", "Module", "get_cluster_info"]
+
+
+import os
+from pathlib import Path
+from typing import TypeVar
+
+from pydantic import BaseModel
+import yaml
+
+
+class Module(BaseModel):
+    name: str
+    version: str | None = None
+
+
+class Cluster(BaseModel):
+    name: str
+    modules: list[Module]
+
+
+T = TypeVar("T", bound=BaseModel)
+
+
+def _get_info(
+    category: str, name: str, model: type[T], flepi_path: os.PathLike | None
+) -> T:
+    """
+    Get and parse an information yaml file.
+
+    This function is a light wrapper around reading and parsing yaml files located in
+    `$FLEPI_PATH/info`.
+
+    Args:
+        category: The category of info to get, corresponds to a subdirectory in
+            `$FLEPI_PATH/info`.
+        name: The name of the info to get, corresponds to the name of a yaml file and
+            is usually a human readable short name.
+        model: The pydantic class to parse the info file with, determines the return
+            type.
+        flepi_path: Either a path like determine the directory to look for the info
+            directory in or `None` to use the `FLEPI_PATH` environment variable.
+
+    Returns:
+        An instance of `model` with the contained info found and parsed.
+    """
+    flepi_path = Path(os.getenv("FLEPI_PATH") if flepi_path is None else flepi_path)
+    info = flepi_path / "info" / category / f"{name}.yml"
+    if not info.exists() or not info.is_file():
+        raise ValueError(
+            f"Was expecting an information yaml at {info.absolute()}, "
+            "but either does not exist or is not a file."
+        )
+    return model.model_validate(yaml.safe_load(info.read_text()))
+
+
+def get_cluster_info(name: str, flepi_path: os.PathLike | None = None) -> Cluster:
+    """
+    Get cluster specific info.
+
+    Args:
+        name: The name of the cluster to pull information for. Currently only 'longleaf'
+            and 'rockfish' are supported.
+        flepi_path: Either a path like determine the directory to look for the info
+            directory in or `None` to use the `FLEPI_PATH` environment variable.
+
+    Returns
+        An object containing the information about the `name` cluster.
+    """
+    return _get_info("cluster", name, Cluster, flepi_path)
diff --git a/flepimop/gempyor_pkg/tests/info/test__get_info.py b/flepimop/gempyor_pkg/tests/info/test__get_info.py
new file mode 100644
index 000000000..910c1bbe4
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/info/test__get_info.py
@@ -0,0 +1,56 @@
+from pathlib import Path
+
+from pydantic import BaseModel
+import pytest
+import yaml
+
+from gempyor.info import _get_info
+
+
+class NameOnly(BaseModel):
+    name: str
+
+
+@pytest.fixture
+def create_mock_info_directory(tmp_path: Path) -> Path:
+    for file, contents in (
+        ("abc/def.yml", {"name": "Foobar"}),
+        ("abc/ghi.yml", {"name": "Fizzbuzz"}),
+    ):
+        file = tmp_path / "info" / file
+        file.parent.mkdir(parents=True, exist_ok=True)
+        with file.open(mode="w") as f:
+            yaml.dump(contents, f)
+    return tmp_path.absolute()
+
+
+def test_file_does_not_exist_value_error(
+    monkeypatch: pytest.MonkeyPatch, create_mock_info_directory: Path
+) -> None:
+    monkeypatch.setenv("FLEPI_PATH", str(create_mock_info_directory.parent))
+    with pytest.raises(ValueError):
+        _get_info("does_not", "exist", object, None)
+
+
+@pytest.mark.parametrize(
+    ("category", "name", "model"), (("abc", "def", NameOnly), ("abc", "ghi", NameOnly))
+)
+def test_output_validation(
+    monkeypatch: pytest.MonkeyPatch,
+    create_mock_info_directory: Path,
+    category: str,
+    name: str,
+    model: type[BaseModel],
+) -> None:
+    monkeypatch.setenv("FLEPI_PATH", str(create_mock_info_directory))
+    results = []
+    for flepi_path in (
+        None,
+        create_mock_info_directory,
+        str(create_mock_info_directory),
+    ):
+        info = _get_info(category, name, model, flepi_path)
+        assert isinstance(info, model)
+        results.append(info)
+    for i in range(len(results) - 1):
+        assert results[i] == results[i + 1]
diff --git a/flepimop/gempyor_pkg/tests/info/test_get_cluster_info.py b/flepimop/gempyor_pkg/tests/info/test_get_cluster_info.py
new file mode 100644
index 000000000..b8ab64436
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/info/test_get_cluster_info.py
@@ -0,0 +1,15 @@
+import os
+
+import pytest
+
+from gempyor.info import Cluster, get_cluster_info
+
+
+@pytest.mark.parametrize("name", ("longleaf", "rockfish"))
+@pytest.mark.skipif(
+    os.getenv("FLEPI_PATH") is None,
+    reason="The $FLEPI_PATH environment variable is not set.",
+)
+def test_output_validation(name: str) -> None:
+    cluster = get_cluster_info(name)
+    assert isinstance(cluster, Cluster)
diff --git a/info/cluster/longleaf.yml b/info/cluster/longleaf.yml
new file mode 100644
index 000000000..5b6cb9857
--- /dev/null
+++ b/info/cluster/longleaf.yml
@@ -0,0 +1,7 @@
+name: longleaf
+modules:
+  - name: gcc
+    version: 9.1.0
+  - name: anaconda
+    version: "2023.03"
+  - name: git
diff --git a/info/cluster/rockfish.yml b/info/cluster/rockfish.yml
new file mode 100644
index 000000000..5231900b9
--- /dev/null
+++ b/info/cluster/rockfish.yml
@@ -0,0 +1,9 @@
+name: rockfish
+modules:
+  - name: gcc
+    version: 9.3.0
+  - name: anaconda
+    version: "2020.07"
+  - name: git
+    version: 2.42.0
+  - name: slurm
diff --git a/info/cluster/schema.yml b/info/cluster/schema.yml
new file mode 100644
index 000000000..d2f43e959
--- /dev/null
+++ b/info/cluster/schema.yml
@@ -0,0 +1,25 @@
+$schema: https://json-schema.org/draft/2020-12/schema
+$id: https://example.com/product.schema.json
+title: Cluster
+description: A generic description of an HPC cluster and its properties.
+type: object
+required:
+  - name
+properties:
+  name:
+    description: A short hand human readable unique name.
+    type: string
+  modules:
+    description: The modules to load when setting up this cluster.
+    type: array
+    items:
+      type: object
+      required:
+        - name
+      properties:
+        name:
+          description: The name of the module.
+          type: string
+        version:
+          description: The preferred version of the module if there is one.
+          type: string

From 05c119b2b22d6aaf562758643f1ad249db51ccec Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Fri, 1 Nov 2024 13:24:49 -0400
Subject: [PATCH 16/78] Initial EMCEE inference sbatch script

Very draft, needs to be better connected to a CLI to fully implement.
---
 .../templates/emcee_inference.sbatch.j2       | 40 +++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2

diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2 b/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2
new file mode 100644
index 000000000..e0cad5e80
--- /dev/null
+++ b/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2
@@ -0,0 +1,40 @@
+#!/usr/bin/env {{ interpreter }}
+{% for opt in options -%}
+#SBATCH {{ opt }}
+{%- endfor %}
+
+{% if sbatch_debug %}
+# Debugging
+set -x
+{% endif %}
+
+# Load modules
+module purge
+{% if load_slurm -%}
+{% if slurm_version is defined and slurm_version is not none -%}
+module load slurm/{{ slurm_version }}
+{%- else -%}
+module load slurm
+{%- endif %}
+{%- endif %}
+{% if load_gcc -%}
+{% if gcc_version is defined and gcc_version is not none -%}
+module load gcc/{{ gcc_version }}
+{%- else -%}
+module load gcc
+{%- endif %}
+{%- endif %}
+{% if load_anaconda -%}
+{% if anaconda_version is defined and anaconda_version is not none -%}
+module load anaconda/{{ anaconda_version }}
+{%- else -%}
+module load anaconda
+{%- endif %}
+{%- endif %}
+{% if load_git -%}
+{% if git_version is defined and git_version is not none -%}
+module load git/{{ git_version }}
+{%- else -%}
+module load git
+{%- endif %}
+{%- endif %}

From d98eb80916b9ef66d6b678c1ad556086c9301155 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Mon, 4 Nov 2024 11:03:23 -0500
Subject: [PATCH 17/78] Add path modifications to cluster info

Added specification for path modifications to the cluster info (namely
to add AWS cli on rockfish to path). Specification includes the path to
add, prepend vs append, and a flag to error if path is missing.
---
 flepimop/gempyor_pkg/src/gempyor/info.py |  9 +++++++--
 info/cluster/longleaf.yml                |  1 +
 info/cluster/rockfish.yml                |  2 ++
 info/cluster/schema.yml                  | 17 +++++++++++++++++
 4 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/info.py b/flepimop/gempyor_pkg/src/gempyor/info.py
index 0b8c7055a..09475e96c 100644
--- a/flepimop/gempyor_pkg/src/gempyor/info.py
+++ b/flepimop/gempyor_pkg/src/gempyor/info.py
@@ -1,4 +1,4 @@
-__all__ = ["Cluster", "Module", "get_cluster_info"]
+__all__ = ["Cluster", "Module", "PathExport", "get_cluster_info"]
 
 
 import os
@@ -13,10 +13,15 @@ class Module(BaseModel):
     name: str
     version: str | None = None
 
+class PathExport(BaseModel):
+    path: Path
+    prepend: bool = True
+    error_if_missing: bool = False
 
 class Cluster(BaseModel):
     name: str
-    modules: list[Module]
+    modules: list[Module] = []
+    path_exports: list[PathExport] = []
 
 
 T = TypeVar("T", bound=BaseModel)
diff --git a/info/cluster/longleaf.yml b/info/cluster/longleaf.yml
index 5b6cb9857..edfa0e378 100644
--- a/info/cluster/longleaf.yml
+++ b/info/cluster/longleaf.yml
@@ -5,3 +5,4 @@ modules:
   - name: anaconda
     version: "2023.03"
   - name: git
+  - name: aws
diff --git a/info/cluster/rockfish.yml b/info/cluster/rockfish.yml
index 5231900b9..2a7dc90ef 100644
--- a/info/cluster/rockfish.yml
+++ b/info/cluster/rockfish.yml
@@ -7,3 +7,5 @@ modules:
   - name: git
     version: 2.42.0
   - name: slurm
+path_exports:
+  - path: ~/aws-cli/bin
diff --git a/info/cluster/schema.yml b/info/cluster/schema.yml
index d2f43e959..75aef81b0 100644
--- a/info/cluster/schema.yml
+++ b/info/cluster/schema.yml
@@ -23,3 +23,20 @@ properties:
         version:
           description: The preferred version of the module if there is one.
           type: string
+  path_exports:
+    description: Modifications to the $PATH environment variable in cluster setup.
+    type: array
+    items:
+      type: object
+      required:
+        - path
+      properties:
+        path:
+          description: The path to add to the $PATH environment variable.
+          type: string
+        prepend:
+          description: If true the path will prepended and appended if false.
+          type: boolean
+        error_if_missing:
+          description: If true an error will be thrown if the path is missing and if false will be ignored if missing.
+          type: boolean

From abd92e6ff24b92ef69434b58e57db77a775e315b Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Mon, 4 Nov 2024 15:02:32 -0500
Subject: [PATCH 18/78] Skeleton of sbatch script

Not fleshed out at all, but at least contains the main sections that
will need to be filled in. Now includes dynamic modules and $PATH
modifications that mirror the objects returned from `get_cluster_info`.
---
 .../templates/emcee_inference.sbatch.j2       | 59 +++++++++++--------
 1 file changed, 33 insertions(+), 26 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2 b/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2
index e0cad5e80..6cab06a0b 100644
--- a/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2
+++ b/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2
@@ -8,33 +8,40 @@
 set -x
 {% endif %}
 
+# Set script variables
+FLEPI_SLOT_INDEX=${SLURM_ARRAY_TASK_ID}
+
 # Load modules
 module purge
-{% if load_slurm -%}
-{% if slurm_version is defined and slurm_version is not none -%}
-module load slurm/{{ slurm_version }}
-{%- else -%}
-module load slurm
-{%- endif %}
-{%- endif %}
-{% if load_gcc -%}
-{% if gcc_version is defined and gcc_version is not none -%}
-module load gcc/{{ gcc_version }}
-{%- else -%}
-module load gcc
-{%- endif %}
-{%- endif %}
-{% if load_anaconda -%}
-{% if anaconda_version is defined and anaconda_version is not none -%}
-module load anaconda/{{ anaconda_version }}
-{%- else -%}
-module load anaconda
-{%- endif %}
-{%- endif %}
-{% if load_git -%}
-{% if git_version is defined and git_version is not none -%}
-module load git/{{ git_version }}
+{% for module in cluster.modules -%}
+{% if module.version is defined and module.version is not none -%}
+module load {{ module.name }}/{{ module.version }}
 {%- else -%}
-module load git
-{%- endif %}
+module load {{ module.name }}
 {%- endif %}
+{%- endfor %}
+
+# Path modifications
+{% for path_export in cluster.path_exports -%}
+if [ -r "{{ path_export.path }}" ]; then
+    {% if path_export.prepend -%}
+    export PATH={{ path_export.path }}:$PATH
+    {%- else -%}
+    export PATH=$PATH:{{ path_export.path }}
+    {%- endif %}
+elif [ "{{ path_export.error_if_missing }}" = "True" ]; then
+    echo "The path '{{ path_export.path }}' does not exist but is required."
+    exit 1
+fi
+{%- endfor %}
+
+# Load conda env
+conda activate {{ conda_env }}
+
+# AWS setup
+
+# Fetch/setup resume files
+
+# Run inference
+
+# Upload/move results

From e3e373d74982763a01e82662164b24a13faf8442 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Mon, 4 Nov 2024 15:09:33 -0500
Subject: [PATCH 19/78] Apply black formatter

Just rebased `dev` into `GH-365/emcee-submit` which now contains black
formatting for python.
---
 flepimop/gempyor_pkg/src/gempyor/info.py           | 2 ++
 flepimop/gempyor_pkg/tests/utils/test__git_head.py | 4 +---
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/info.py b/flepimop/gempyor_pkg/src/gempyor/info.py
index 09475e96c..1048f3f8a 100644
--- a/flepimop/gempyor_pkg/src/gempyor/info.py
+++ b/flepimop/gempyor_pkg/src/gempyor/info.py
@@ -13,11 +13,13 @@ class Module(BaseModel):
     name: str
     version: str | None = None
 
+
 class PathExport(BaseModel):
     path: Path
     prepend: bool = True
     error_if_missing: bool = False
 
+
 class Cluster(BaseModel):
     name: str
     modules: list[Module] = []
diff --git a/flepimop/gempyor_pkg/tests/utils/test__git_head.py b/flepimop/gempyor_pkg/tests/utils/test__git_head.py
index 0f789bea6..5005bf86e 100644
--- a/flepimop/gempyor_pkg/tests/utils/test__git_head.py
+++ b/flepimop/gempyor_pkg/tests/utils/test__git_head.py
@@ -27,9 +27,7 @@ def shutil_which_wraps(
         check: bool = True,
     ) -> str | None:
         return (
-            "git"
-            if cmd == "git"
-            else _shutil_which(cmd, mode=mode, path=path, check=check)
+            "git" if cmd == "git" else _shutil_which(cmd, mode=mode, path=path, check=check)
         )
 
     def subprocess_run_wraps(args, **kwargs):

From dc672543ba2db68edcab57c5ce09d02298ec0fa6 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 5 Nov 2024 10:12:53 -0500
Subject: [PATCH 20/78] Add `log_cli_inputs` utility

Added a helper to log the inputs given to a CLI tool when the verbosity
is set to debug. Mostly for user/dev debugging. Also added corresponding
docs and unit tests.
---
 .../gempyor_pkg/src/gempyor/shared_cli.py     | 49 +++++++++++++++++++
 .../tests/shared_cli/test_log_cli_inputs.py   | 34 +++++++++++++
 2 files changed, 83 insertions(+)
 create mode 100644 flepimop/gempyor_pkg/tests/shared_cli/test_log_cli_inputs.py

diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py
index 3e5f0daea..6e96fbddb 100644
--- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py
+++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py
@@ -12,6 +12,7 @@
 import click
 import confuse
 
+from .logging import get_script_logger
 from .utils import config, as_list
 
 __all__ = []
@@ -265,3 +266,51 @@ def _parse_option(param: click.Parameter, value: Any) -> Any:
             cfg[option] = _parse_option(config_file_options[option], value)
 
     return cfg
+
+
+def log_cli_inputs(kwargs: dict[str, Any], verbosity: int | None = None) -> None:
+    """
+    Log CLI inputs for user debugging.
+
+    This function only logs debug messages so the verbosity has to be set quite high
+    to see the output of this function.
+
+    Args:
+        kwargs: The CLI arguments given as a dictionary of key word arguments.
+        verbosity: The verbosity level of the CLI tool being used or `None` to infer
+            from the given `kwargs`.
+
+    Examples:
+        >>> from gempyor.shared_cli import log_cli_inputs
+        >>> log_cli_inputs({"abc": 123, "def": True}, 3)
+        2024-11-05 09:27:58,884:DEBUG:gempyor.shared_cli> CLI was given 2 arguments:
+        2024-11-05 09:27:58,885:DEBUG:gempyor.shared_cli> abc = 123.
+        2024-11-05 09:27:58,885:DEBUG:gempyor.shared_cli> def = True.
+        >>> log_cli_inputs({"abc": 123, "def": True}, 2)
+        >>> from pathlib import Path
+        >>> kwargs = {
+        ...     "input_file": Path("config.in"),
+        ...     "stochastic": True,
+        ...     "cluster": "longleaf",
+        ...     "verbosity": 3,
+        ... }
+        >>> log_cli_inputs(kwargs)
+        2024-11-05 09:29:21,666:DEBUG:gempyor.shared_cli> CLI was given 4 arguments:
+        2024-11-05 09:29:21,667:DEBUG:gempyor.shared_cli> input_file = /Users/twillard/Desktop/GitHub/HopkinsIDD/flepiMoP/flepimop/gempyor_pkg/config.in.
+        2024-11-05 09:29:21,667:DEBUG:gempyor.shared_cli> stochastic = True.
+        2024-11-05 09:29:21,668:DEBUG:gempyor.shared_cli> cluster    = longleaf.
+        2024-11-05 09:29:21,668:DEBUG:gempyor.shared_cli> verbosity  = 3.
+    """
+    verbosity = kwargs.get("verbosity") if verbosity is None else verbosity
+    if verbosity is None:
+        return
+    logger = get_script_logger(__name__, verbosity)
+    longest_key = -1
+    total_keys = 0
+    for k, _ in kwargs.items():
+        longest_key = len(k) if len(k) > longest_key else longest_key
+        total_keys += 1
+    logger.debug("CLI was given %u arguments:", total_keys)
+    for k, v in kwargs.items():
+        v = v.absolute() if isinstance(v, pathlib.Path) else v
+        logger.debug("%s = %s", k.ljust(longest_key, " "), v)
diff --git a/flepimop/gempyor_pkg/tests/shared_cli/test_log_cli_inputs.py b/flepimop/gempyor_pkg/tests/shared_cli/test_log_cli_inputs.py
new file mode 100644
index 000000000..1693a5733
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/shared_cli/test_log_cli_inputs.py
@@ -0,0 +1,34 @@
+import logging
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from gempyor.logging import _get_logging_level
+from gempyor.shared_cli import log_cli_inputs
+
+
+@pytest.mark.parametrize(
+    "kwargs",
+    (
+        {"abc": 123},
+        {"abc": 123, "verbosity": 3},
+        {"str": "foobar", "int": 123, "float": 3.14, "bool": True},
+        {"str": "foobar", "int": 123, "float": 3.14, "bool": True, "verbosity": 3},
+        {"file_path": Path("fizzbuzz.log")},
+        {"file_path": Path("fizzbuzz.log"), "verbosity": 3},
+    ),
+)
+@pytest.mark.parametrize("verbosity", (None, 0, 1, 2, 3, logging.DEBUG, logging.INFO))
+def test_number_of_messages(
+    caplog: pytest.LogCaptureFixture, kwargs: dict[str, Any], verbosity: int | None
+) -> None:
+    log_cli_inputs(kwargs, verbosity=verbosity)
+    assert len(caplog.records) == (
+        len(kwargs) + 1
+        if _get_logging_level(
+            kwargs.get("verbosity", 0) if verbosity is None else verbosity
+        )
+        <= logging.DEBUG
+        else 0
+    )

From 30893588034aac156a97c37a4695c15fe3fbf365 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 5 Nov 2024 11:42:36 -0500
Subject: [PATCH 21/78] Add `gempyor.batch._job_name`

Internal utility to generate a human readable unique batch job name
along with corresponding docs and tests.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py     | 51 ++++++++++++++++++-
 .../gempyor_pkg/tests/batch/test__job_name.py | 37 ++++++++++++++
 2 files changed, 87 insertions(+), 1 deletion(-)
 create mode 100644 flepimop/gempyor_pkg/tests/batch/test__job_name.py

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 8faa19747..1136a60ac 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -8,10 +8,13 @@
 __all__ = ["JobSize", "write_manifest"]
 
 
+from click import Context, pass_context
 from dataclasses import dataclass
+from datetime import datetime, timezone
 import json
 import math
 from pathlib import Path
+import re
 from shlex import quote
 import subprocess
 import sys
@@ -19,7 +22,19 @@
 
 from ._jinja import _render_template_to_file, _render_template_to_temp_file
 from .logging import get_script_logger
-from .utils import _format_cli_options, _git_head, _shutil_which
+from .utils import _format_cli_options, _git_head, _shutil_which, config
+from .shared_cli import (
+    cli,
+    config_files_argument,
+    config_file_options,
+    log_cli_inputs,
+    mock_context,
+    parse_config_files,
+    verbosity_options,
+)
+
+
+_JOB_NAME_REGEX = re.compile(r"^[a-z]{1}([a-z0-9\_\-]+)?$", flags=re.IGNORECASE)
 
 
 @dataclass(frozen=True, slots=True)
@@ -336,3 +351,37 @@ def _sbatch_template(
         _render_template_to_file(template, template_data, script)
     _sbatch(script, environment_variables, {}, verbosity, dry_run)
     return script
+
+
+def _job_name(name: str | None, timestamp: datetime | None) -> str:
+    """
+    Generate a unique human readable job name.
+
+    Args:
+        name: The config name used as a prefix or `None` for no prefix.
+        timestamp: The timestamp used to make the job name unique or `None` to use the
+            current UTC timestamp.
+
+    Returns:
+        A job name that is unique and intended for use when submitting to slurm.
+
+    Raises:
+        ValueError: If `name` does not start with a letter and contains characters other
+            than the alphabet, numbers, underscores or dashes.
+
+    Examples:
+        >>> from gempyor.batch import _job_name
+        >>> _job_name(None, None)
+        '20241105T153818'
+        >>> _job_name("foobar", None)
+        'foobar-20241105T153831'
+        >>> from datetime import datetime, timezone
+        >>> _job_name(None, datetime(2024, 1, 1, tzinfo=timezone.utc))
+        '20240101T000000'
+    """
+    timestamp = datetime.now(timezone.utc) if timestamp is None else timestamp
+    timestamp = timestamp.strftime("%Y%m%dT%H%M%S")
+    if name is not None and not _JOB_NAME_REGEX.match(name):
+        raise ValueError(f"The given `name`, '{name}', is not a valid safe name.")
+    return f"{name}-{timestamp}" if name else timestamp
+
diff --git a/flepimop/gempyor_pkg/tests/batch/test__job_name.py b/flepimop/gempyor_pkg/tests/batch/test__job_name.py
new file mode 100644
index 000000000..bd08a9452
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/batch/test__job_name.py
@@ -0,0 +1,37 @@
+from datetime import datetime, timedelta, timezone
+import re
+
+import pytest
+
+from gempyor.batch import _job_name
+
+
+VALID_JOB_NAME = re.compile(
+    r"^([a-z]{1}([a-z0-9\_\-]+)?\-)?[0-9]{8}T[0-9]{6}$", flags=re.IGNORECASE
+)
+
+
+@pytest.mark.parametrize("name", ("123", "_abc", "", "abc!@#"))
+def test_invalid_name_value_error(name: str) -> None:
+    with pytest.raises(
+        ValueError, match=f"^The given `name`, '{name}', is not a valid safe name.$"
+    ):
+        _job_name(name, None)
+
+
+@pytest.mark.parametrize("name", (None, "abc", "flu_usa", "covid-19"))
+@pytest.mark.parametrize(
+    "timestamp",
+    (
+        None,
+        datetime(2024, 1, 1, tzinfo=timezone.utc),
+        datetime(2025, 1, 2, 3, 4, 5, tzinfo=timezone(timedelta(hours=3))),
+    ),
+)
+def test_output_validation(name: str | None, timestamp: datetime | None) -> None:
+    job_name = _job_name(name, timestamp)
+    if name is not None:
+        assert job_name.startswith(name)
+    if timestamp is not None:
+        assert job_name.endswith(timestamp.strftime("%Y%m%dT%H%M%S"))
+    assert VALID_JOB_NAME.match(job_name)

From d6773ff9d6d81f189334fb722cd878518af83f30 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 5 Nov 2024 13:56:08 -0500
Subject: [PATCH 22/78] Add `_resolve_batch_system` utility

Added internal `gempyor.batch._resolve_batch_system` utility to make
sense of batch system CLI options.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py     | 43 +++++++++++-
 .../tests/batch/test__resolve_batch_system.py | 69 +++++++++++++++++++
 2 files changed, 111 insertions(+), 1 deletion(-)
 create mode 100644 flepimop/gempyor_pkg/tests/batch/test__resolve_batch_system.py

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 1136a60ac..804c11e4a 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -8,7 +8,7 @@
 __all__ = ["JobSize", "write_manifest"]
 
 
-from click import Context, pass_context
+from click import Choice, Context, IntRange, Option, pass_context
 from dataclasses import dataclass
 from datetime import datetime, timezone
 import json
@@ -385,3 +385,44 @@ def _job_name(name: str | None, timestamp: datetime | None) -> str:
         raise ValueError(f"The given `name`, '{name}', is not a valid safe name.")
     return f"{name}-{timestamp}" if name else timestamp
 
+
+def _resolve_batch_system(
+    batch_system: Literal["aws", "local", "slurm"] | None,
+    aws: bool,
+    local: bool,
+    slurm: bool,
+) -> Literal["aws", "local", "slurm"]:
+    """
+    Resolve the batch system options.
+
+    Args:
+        batch_system: The name of the batch system to use if provided explicitly by
+            name or `None` to rely on the other flags.
+        aws: A flag indicating if the batch system should be AWS.
+        local: A flag indicating if the batch system should be local.
+        slurm: A flag indicating if the batch system should be slurm.
+
+    Returns:
+        The name of the batch system to use given the user options.
+    """
+    batch_system = batch_system.lower() if batch_system is not None else batch_system
+    if (boolean_flags := sum((aws, local, slurm))) > 1:
+        raise ValueError(
+            f"There were {boolean_flags} boolean flags given, expected either 0 or 1."
+        )
+    if batch_system is not None:
+        for name, flag in zip(("aws", "local", "slurm"), (aws, local, slurm)):
+            if flag and batch_system != name:
+                raise ValueError(
+                    "Conflicting batch systems given. The batch system name "
+                    f"is '{batch_system}' and the flags indicate '{name}'."
+                )
+    if batch_system is None:
+        if aws:
+            batch_system = "aws"
+        elif local:
+            batch_system = "local"
+        else:
+            batch_system = "slurm"
+    return batch_system
+
diff --git a/flepimop/gempyor_pkg/tests/batch/test__resolve_batch_system.py b/flepimop/gempyor_pkg/tests/batch/test__resolve_batch_system.py
new file mode 100644
index 000000000..6d5eb6420
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/batch/test__resolve_batch_system.py
@@ -0,0 +1,69 @@
+from typing import Literal
+
+import pytest
+
+from gempyor.batch import _resolve_batch_system
+
+
+@pytest.mark.parametrize(
+    ("aws", "local", "slurm"),
+    ((True, True, False), (False, True, True), (True, False, True), (True, True, True)),
+)
+def test_multiple_flags_value_error(aws: bool, local: bool, slurm: bool) -> None:
+    with pytest.raises(
+        ValueError,
+        match=(
+            f"^There were {sum((aws, local, slurm))} boolean "
+            "flags given, expected either 0 or 1.$"
+        ),
+    ):
+        _resolve_batch_system(None, aws, local, slurm)
+
+
+@pytest.mark.parametrize(
+    ("batch_system", "aws", "local", "slurm"),
+    (
+        ("aws", False, True, False),
+        ("aws", False, False, True),
+        ("slurm", True, False, False),
+        ("slurm", False, True, False),
+        ("local", True, False, False),
+        ("local", False, False, True),
+    ),
+)
+def test_batch_system_flag_mismatch_value_error(
+    batch_system: Literal["aws", "local", "slurm"], aws: bool, local: bool, slurm: bool
+) -> None:
+    name = next(n for n, f in zip(("aws", "local", "slurm"), (aws, local, slurm)) if f)
+    with pytest.raises(
+        ValueError,
+        match=(
+            "^Conflicting batch systems given. The batch system name "
+            f"is '{batch_system}' and the flags indicate '{name}'.$"
+        ),
+    ):
+        _resolve_batch_system(batch_system, aws, local, slurm)
+
+
+@pytest.mark.parametrize(
+    ("batch_system", "aws", "local", "slurm", "expected"),
+    (
+        (None, True, False, False, "aws"),
+        ("aws", False, False, False, "aws"),
+        ("aws", True, False, False, "aws"),
+        (None, False, True, False, "local"),
+        ("local", False, False, False, "local"),
+        ("local", False, True, False, "local"),
+        (None, False, False, True, "slurm"),
+        ("slurm", False, False, False, "slurm"),
+        ("slurm", False, False, True, "slurm"),
+    ),
+)
+def test_output_validation(
+    batch_system: Literal["aws", "local", "slurm"] | None,
+    aws: bool,
+    local: bool,
+    slurm: bool,
+    expected: Literal["aws", "local", "slurm"],
+) -> None:
+    assert _resolve_batch_system(batch_system, aws, local, slurm) == expected

From 42ceda3ef721f36e39d9c0f6bf9453c53ef08ef2 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 5 Nov 2024 14:50:02 -0500
Subject: [PATCH 23/78] Initial work on `flepimop batch` CLI

Very preliminary work on the `flepimop batch` CLI tool including some
light testing for the not implemented errors thrown.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py     | 99 +++++++++++++++++++
 flepimop/gempyor_pkg/src/gempyor/cli.py       |  4 +-
 .../gempyor_pkg/src/gempyor/shared_cli.py     | 15 +++
 .../tests/batch/test__click_batch.py          | 55 +++++++++++
 4 files changed, 171 insertions(+), 2 deletions(-)
 create mode 100644 flepimop/gempyor_pkg/tests/batch/test__click_batch.py

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 804c11e4a..98d9aaf25 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -426,3 +426,102 @@ def _resolve_batch_system(
             batch_system = "slurm"
     return batch_system
 
+
+@cli.command(
+    name="batch",
+    params=[config_files_argument]
+    + list(config_file_options.values())
+    + [
+        Option(
+            ["--simulations"],
+            "simulations",
+            default=None,
+            type=IntRange(min=1),
+            help="The number of simulations per a job.",
+        ),
+        Option(
+            ["--blocks"],
+            "blocks",
+            default=None,
+            type=IntRange(min=1),
+            help="The number of sequential blocks to run per a job.",
+        ),
+        Option(
+            ["--batch-system"],
+            "batch_system",
+            default=None,
+            type=Choice(("aws", "local", "slurm"), case_sensitive=False),
+            help="The name of the batch system being used.",
+        ),
+        Option(
+            ["--aws"],
+            "aws",
+            default=False,
+            type=bool,
+            is_flag=True,
+            help="A flag indicating this is being run on AWS.",
+        ),
+        Option(
+            ["--local"],
+            "local",
+            default=False,
+            type=bool,
+            is_flag=True,
+            help="A flag indicating this is being run local.",
+        ),
+        Option(
+            ["--slurm"],
+            "slurm",
+            default=False,
+            type=bool,
+            is_flag=True,
+            help="A flag indicating this is being run on slurm.",
+        ),
+    ]
+    + list(verbosity_options.values()),
+)
+@pass_context
+def _click_batch(ctx: Context = mock_context, **kwargs) -> None:
+    """Submit batch jobs"""
+    logger = get_script_logger(__name__, kwargs.get("verbosity", 0))
+    log_cli_inputs(kwargs)
+    cfg = parse_config_files(config, ctx, **kwargs)
+
+    if (
+        not cfg["inference"].exists()
+        or not cfg["inference"]["method"].exists()
+        or cfg["inference"]["method"].as_str() != "emcee"
+    ):
+        raise NotImplementedError(
+            "The `flepimop batch` CLI only supports EMCEE inference jobs."
+        )
+
+    name = cfg["name"].get(str) if cfg["name"].exists() else None
+    job_name = _job_name(name, None)
+    logger.info("Assigning job name of '%s'", job_name)
+
+    batch_system = _resolve_batch_system(
+        kwargs["batch_system"], kwargs["aws"], kwargs["local"], kwargs["slurm"]
+    )
+    if batch_system != "slurm":
+        raise NotImplementedError(
+            "The `flepimop batch` CLI only supports batch submission to slurm."
+        )
+    logger.info("Constructing a job to submit to %s", batch_system)
+
+    iterations_per_slot = (
+        cfg["inference"]["iterations_per_slot"].get(int)
+        if cfg["inference"].exists() and cfg["inference"]["iterations_per_slot"].exists()
+        else None
+    )
+    nslots = cfg["nslots"].get(int) if cfg["nslots"].exists() else None
+    job_size = JobSize.size_from_jobs_sims_blocks(
+        kwargs["jobs"],
+        kwargs["simulations"],
+        kwargs["blocks"],
+        iterations_per_slot,
+        nslots,
+        None,
+        "slurm",
+    )
+    logger.info("Preparing a job with size %s", job_size)
diff --git a/flepimop/gempyor_pkg/src/gempyor/cli.py b/flepimop/gempyor_pkg/src/gempyor/cli.py
index 8e0c0b44e..cf5149cc1 100644
--- a/flepimop/gempyor_pkg/src/gempyor/cli.py
+++ b/flepimop/gempyor_pkg/src/gempyor/cli.py
@@ -10,8 +10,8 @@
 from .utils import config
 
 # register the commands from the other modules
-from . import compartments
-from . import simulate
+from . import compartments, simulate, batch
+
 
 # Guidance for extending the CLI:
 # - to add a new small command to the CLI, add a new function with the @cli.command() decorator here (e.g. patch below)
diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py
index 6e96fbddb..bd720d562 100644
--- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py
+++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py
@@ -124,6 +124,21 @@ def cli(ctx: click.Context) -> None:
     ),
 }
 
+verbosity_options = {
+    "verbosity": click.Option(
+        ["-v", "--verbose"],
+        count=True,
+        help="The verbosity level to use for this command.",
+    ),
+    "dry_run": click.Option(
+        ["--dry-run"],
+        type=bool,
+        default=False,
+        is_flag=True,
+        help="Should this command be run using dry run?",
+    ),
+}
+
 
 def click_helpstring(
     params: click.Parameter | list[click.Parameter],
diff --git a/flepimop/gempyor_pkg/tests/batch/test__click_batch.py b/flepimop/gempyor_pkg/tests/batch/test__click_batch.py
new file mode 100644
index 000000000..2a8c6886c
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/batch/test__click_batch.py
@@ -0,0 +1,55 @@
+from pathlib import Path
+from typing import Any
+
+from click.testing import CliRunner
+import pytest
+import yaml
+
+from gempyor.batch import _click_batch
+
+
+@pytest.mark.parametrize(
+    "config",
+    (
+        {},
+        {"inference": {"abc": 123}},
+        {"inference": {"method": "legacy"}},
+    ),
+)
+def test_only_inference_emcee_supported_not_implemented_error(
+    tmp_path: Path, config: dict[str, Any]
+) -> None:
+    config_file = tmp_path / "config.yml"
+    with config_file.open(mode="w") as f:
+        yaml.dump(config, f)
+
+    runner = CliRunner()
+    result = runner.invoke(_click_batch, [str(config_file.absolute())])
+
+    assert result.exit_code == 1
+    assert isinstance(result.exception, NotImplementedError)
+    assert (
+        str(result.exception)
+        == "The `flepimop batch` CLI only supports EMCEE inference jobs."
+    )
+
+
+@pytest.mark.parametrize(
+    "args", (["--aws"], ["--batch-system", "aws"], ["--local"], ["--batch-system", "local"])
+)
+def test_only_slurm_batch_system_supported_not_implemented_error(
+    tmp_path: Path, args: list[str]
+) -> None:
+    config_file = tmp_path / "config.yml"
+    with config_file.open(mode="w") as f:
+        yaml.dump({"inference": {"method": "emcee"}}, f)
+
+    runner = CliRunner()
+    result = runner.invoke(_click_batch, args + [str(config_file.absolute())])
+
+    assert result.exit_code == 1
+    assert isinstance(result.exception, NotImplementedError)
+    assert (
+        str(result.exception)
+        == "The `flepimop batch` CLI only supports batch submission to slurm."
+    )

From ed4ddc7dd54e51002ddc0ea5cf0fcc596eae7f5e Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 5 Nov 2024 15:38:54 -0500
Subject: [PATCH 24/78] Minor additions to `flepimop batch`

* Changed to generic import of `click` since the `gempyor.batch` module
  relies on it heavily.
* Add manifest json to `flepimop batch`.
* Add subpops to the job size calculation of `flepimop batch`.
* Leftover TODO note about resume/continuation locations.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py | 60 +++++++++++++++++------
 1 file changed, 46 insertions(+), 14 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 98d9aaf25..cc01def50 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -8,7 +8,7 @@
 __all__ = ["JobSize", "write_manifest"]
 
 
-from click import Choice, Context, IntRange, Option, pass_context
+import click
 from dataclasses import dataclass
 from datetime import datetime, timezone
 import json
@@ -432,28 +432,28 @@ def _resolve_batch_system(
     params=[config_files_argument]
     + list(config_file_options.values())
     + [
-        Option(
+        click.Option(
             ["--simulations"],
             "simulations",
             default=None,
-            type=IntRange(min=1),
+            type=click.IntRange(min=1),
             help="The number of simulations per a job.",
         ),
-        Option(
+        click.Option(
             ["--blocks"],
             "blocks",
             default=None,
-            type=IntRange(min=1),
+            type=click.IntRange(min=1),
             help="The number of sequential blocks to run per a job.",
         ),
-        Option(
+        click.Option(
             ["--batch-system"],
             "batch_system",
             default=None,
-            type=Choice(("aws", "local", "slurm"), case_sensitive=False),
+            type=click.Choice(("aws", "local", "slurm"), case_sensitive=False),
             help="The name of the batch system being used.",
         ),
-        Option(
+        click.Option(
             ["--aws"],
             "aws",
             default=False,
@@ -461,7 +461,7 @@ def _resolve_batch_system(
             is_flag=True,
             help="A flag indicating this is being run on AWS.",
         ),
-        Option(
+        click.Option(
             ["--local"],
             "local",
             default=False,
@@ -469,7 +469,7 @@ def _resolve_batch_system(
             is_flag=True,
             help="A flag indicating this is being run local.",
         ),
-        Option(
+        click.Option(
             ["--slurm"],
             "slurm",
             default=False,
@@ -477,11 +477,27 @@ def _resolve_batch_system(
             is_flag=True,
             help="A flag indicating this is being run on slurm.",
         ),
+        click.Option(
+            ["--flepi-path"],
+            "flepi_path",
+            envvar="FLEPI_PATH",
+            type=click.Path(exists=True),
+            required=True,
+            help="Path to the flepiMoP directory being used.",
+        ),
+        click.Option(
+            ["--project-path"],
+            "project_path",
+            envvar="PROJECT_PATH",
+            type=click.Path(exists=True),
+            required=True,
+            help="Path to the project directory being used.",
+        ),
     ]
     + list(verbosity_options.values()),
 )
-@pass_context
-def _click_batch(ctx: Context = mock_context, **kwargs) -> None:
+@click.pass_context
+def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
     """Submit batch jobs"""
     logger = get_script_logger(__name__, kwargs.get("verbosity", 0))
     log_cli_inputs(kwargs)
@@ -515,13 +531,29 @@ def _click_batch(ctx: Context = mock_context, **kwargs) -> None:
         else None
     )
     nslots = cfg["nslots"].get(int) if cfg["nslots"].exists() else None
+    subpops: int | None = None
+    if (
+        cfg["subpop_setup"].exists()
+        and cfg["subpop_setup"]["geodata"].exists()
+        and (geodata := cfg["subpop_setup"]["geodata"].as_path()).exists()
+    ):
+        with geodata.open() as f:
+            subpops = sum(1 for _ in f)
+        subpops -= 1
     job_size = JobSize.size_from_jobs_sims_blocks(
         kwargs["jobs"],
         kwargs["simulations"],
         kwargs["blocks"],
         iterations_per_slot,
         nslots,
-        None,
-        "slurm",
+        subpops,
+        batch_system,
     )
     logger.info("Preparing a job with size %s", job_size)
+
+    # Restart/continuation location
+    # TODO: Implement this
+
+    # Manifest
+    manifest = write_manifest(job_name, kwargs["flepi_path"], kwargs["project_path"])
+    logger.info("Writing manifest metadata to '%s'", manifest.absolute())

From 382a6165141edd2e67cf51d5e67903f1342e1a6d Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 5 Nov 2024 16:44:09 -0500
Subject: [PATCH 25/78] Add `JobTimeLimit` class

Added a representation of batch job time limits, similar to `JobSize`,
along with corresponding documentation and unit tests.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py     | 58 ++++++++++++++++-
 .../tests/batch/test_job_time_limit_class.py  | 63 +++++++++++++++++++
 2 files changed, 119 insertions(+), 2 deletions(-)
 create mode 100644 flepimop/gempyor_pkg/tests/batch/test_job_time_limit_class.py

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index cc01def50..acad4235a 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -5,12 +5,12 @@
 metadata and job size calculations for example.
 """
 
-__all__ = ["JobSize", "write_manifest"]
+__all__ = ["JobSize", "JobTimeLimit", "write_manifest"]
 
 
 import click
 from dataclasses import dataclass
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 import json
 import math
 from pathlib import Path
@@ -149,6 +149,60 @@ def size_from_jobs_sims_blocks(
         return cls(jobs=jobs, simulations=simulations, blocks=blocks)
 
 
+@dataclass(frozen=True, slots=True)
+class JobTimeLimit:
+    """
+    A batch submission job time limit.
+
+    Attributes:
+        time_limit: The time limit of the batch job.
+
+    Raises:
+        ValueError: If the `time_limit` attribute is not positive.
+    """
+
+    time_limit: timedelta
+
+    def __post_init__(self) -> None:
+        if (total_seconds := self.time_limit.total_seconds()) <= 0.0:
+            raise ValueError(
+                f"The `time_limit` attribute has {math.floor(total_seconds):,} "
+                "seconds, which is less than or equal to 0."
+            )
+
+    def __str__(self) -> str:
+        return self.format()
+
+    def format(self, batch_system: Literal["aws", "local", "slurm"] | None = None) -> str:
+        """
+        Format the job time limit as a string appropriate for a given batch system.
+
+        Args:
+            batch_system: The batch system the format should be formatted for.
+
+        Returns:
+            The time limit formatted for the batch system.
+
+        Examples:
+            >>> from datetime import timedelta
+            >>> job_time_limit = JobTimeLimit(
+            ...     time_limit=timedelta(days=1, hours=2, minutes=34, seconds=5)
+            ... )
+            >>> job_time_limit.format()
+            '1595'
+            >>> job_time_limit.format(batch_system="slurm")
+            '26:34:05'
+        """
+        if batch_system == "slurm":
+            total_seconds = self.time_limit.total_seconds()
+            hours = math.floor(total_seconds / (60.0 * 60.0))
+            minutes = math.floor((total_seconds - (60.0 * 60.0 * hours)) / 60.0)
+            seconds = math.ceil(total_seconds - (60.0 * minutes) - (60.0 * 60.0 * hours))
+            return f"{hours}:{minutes:02d}:{seconds:02d}"
+        limit_in_mins = math.ceil(self.time_limit.total_seconds() / 60.0)
+        return str(limit_in_mins)
+
+
 def write_manifest(
     job_name: str,
     flepi_path: Path,
diff --git a/flepimop/gempyor_pkg/tests/batch/test_job_time_limit_class.py b/flepimop/gempyor_pkg/tests/batch/test_job_time_limit_class.py
new file mode 100644
index 000000000..6fe1a9b8d
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/batch/test_job_time_limit_class.py
@@ -0,0 +1,63 @@
+from datetime import timedelta
+import re
+from typing import Literal
+
+import pytest
+
+from gempyor.batch import JobTimeLimit
+
+
+@pytest.mark.parametrize(
+    "time_limit", (timedelta(), timedelta(hours=-1.0), timedelta(days=-3.0))
+)
+def test_time_limit_non_positive_value_error(time_limit: timedelta) -> None:
+    with pytest.raises(
+        ValueError,
+        match=(
+            r"^The \`time\_limit\` attribute has [0-9\,\-]+ seconds\, "
+            r"which is less than or equal to 0\.$"
+        ),
+    ):
+        JobTimeLimit(time_limit=time_limit)
+
+
+@pytest.mark.parametrize(
+    "time_limit",
+    (
+        timedelta(hours=1),
+        timedelta(hours=2, minutes=34, seconds=56),
+        timedelta(days=1, seconds=3),
+        timedelta(minutes=12345),
+    ),
+)
+@pytest.mark.parametrize("batch_system", (None, "aws", "local", "slurm"))
+def test_format_output_validation(
+    time_limit: timedelta, batch_system: Literal["aws", "local", "slurm"] | None
+) -> None:
+    job_time_limit = JobTimeLimit(time_limit=time_limit)
+    formatted_time_limit = job_time_limit.format(batch_system=batch_system)
+    assert isinstance(formatted_time_limit, str)
+    if batch_system == "slurm":
+        assert re.match(r"^[0-9]+\:[0-9]{2}\:[0-9]{2}$", formatted_time_limit)
+    else:
+        assert formatted_time_limit.isdigit()
+
+
+@pytest.mark.parametrize(
+    ("time_limit", "batch_system", "expected"),
+    (
+        (timedelta(hours=1), None, "60"),
+        (timedelta(seconds=20), None, "1"),
+        (timedelta(days=2, hours=3, minutes=45), None, "3105"),
+        (timedelta(hours=1), "slurm", "1:00:00"),
+        (timedelta(seconds=20), "slurm", "0:00:20"),
+        (timedelta(days=1, hours=2, minutes=34, seconds=5), "slurm", "26:34:05"),
+    ),
+)
+def test_format_exact_results(
+    time_limit: timedelta,
+    batch_system: Literal["aws", "local", "slurm"] | None,
+    expected: str,
+) -> None:
+    job_time_limit = JobTimeLimit(time_limit=time_limit)
+    assert job_time_limit.format(batch_system=batch_system) == expected

From 28965ffd3e7b05c20c9dfd881b397287c0ef983d Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Wed, 6 Nov 2024 08:24:34 -0500
Subject: [PATCH 26/78] Add `from_per_simulation_time` method

Added the `JobTimeLimit.from_per_simulation_time` class method to easily
create instances from user provided inputs. Also added comparison
methods for ease of unit testing.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py     | 70 ++++++++++++++++++-
 .../tests/batch/test_job_time_limit_class.py  | 69 ++++++++++++++++--
 2 files changed, 134 insertions(+), 5 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index acad4235a..dbb242f8b 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -18,7 +18,7 @@
 from shlex import quote
 import subprocess
 import sys
-from typing import Any, Literal
+from typing import Any, Literal, Self
 
 from ._jinja import _render_template_to_file, _render_template_to_temp_file
 from .logging import get_script_logger
@@ -173,6 +173,38 @@ def __post_init__(self) -> None:
     def __str__(self) -> str:
         return self.format()
 
+    def __hash__(self) -> int:
+        return hash(self.time_limit)
+
+    def __eq__(self, other: Self | timedelta) -> bool:
+        if isinstance(other, JobTimeLimit):
+            return self.time_limit == other.time_limit
+        if isinstance(other, timedelta):
+            return self.time_limit == other
+        raise TypeError(
+            "'==' not supported between instances of "
+            f"'JobTimeLimit' and '{type(other).__name__}'."
+        )
+
+    def __lt__(self, other: Self | timedelta) -> bool:
+        if isinstance(other, JobTimeLimit):
+            return self.time_limit < other.time_limit
+        if isinstance(other, timedelta):
+            return self.time_limit < other
+        raise TypeError(
+            "'<' not supported between instances of "
+            f"'JobTimeLimit' and '{type(other).__name__}'."
+        )
+
+    def __le__(self, other: Self | timedelta) -> bool:
+        return self.__eq__(other) or self.__lt__(other)
+
+    def __gt__(self, other: Self | timedelta) -> bool:
+        return not self.__le__(other)
+
+    def __ge__(self, other: Self | timedelta) -> bool:
+        return self.__eq__(other) or self.__gt__(other)
+
     def format(self, batch_system: Literal["aws", "local", "slurm"] | None = None) -> str:
         """
         Format the job time limit as a string appropriate for a given batch system.
@@ -202,6 +234,42 @@ def format(self, batch_system: Literal["aws", "local", "slurm"] | None = None) -
         limit_in_mins = math.ceil(self.time_limit.total_seconds() / 60.0)
         return str(limit_in_mins)
 
+    @classmethod
+    def from_per_simulation_time(
+        cls, job_size: JobSize, time_per_simulation: timedelta, initial_time: timedelta
+    ) -> "JobTimeLimit":
+        """
+        Construct a job time limit that scales with job size.
+
+        Args:
+            job_size: The job size to scale the time limit with.
+            time_per_simulation: The time per a simulation.
+            initial_time: Time required to setup per a job.
+
+        Returns:
+            A job time limit that is scaled to match `job_size`.
+
+        Raises:
+            ValueError: If `time_per_simulation` is non-positive.
+            ValueError: If `initial_time` is non-positive.
+
+        Examples:
+        """
+        if (total_seconds := time_per_simulation.total_seconds()) <= 0.0:
+            raise ValueError(
+                f"The `time_per_simulation` is '{math.floor(total_seconds):,}' "
+                "seconds, which is less than or equal to 0."
+            )
+        if (total_seconds := initial_time.total_seconds()) <= 0.0:
+            raise ValueError(
+                f"The `initial_time` is '{math.floor(total_seconds):,}' "
+                "seconds, which is less than or equal to 0."
+            )
+        time_limit = (
+            job_size.blocks * job_size.simulations * time_per_simulation
+        ) + initial_time
+        return cls(time_limit=time_limit)
+
 
 def write_manifest(
     job_name: str,
diff --git a/flepimop/gempyor_pkg/tests/batch/test_job_time_limit_class.py b/flepimop/gempyor_pkg/tests/batch/test_job_time_limit_class.py
index 6fe1a9b8d..f51be9ef7 100644
--- a/flepimop/gempyor_pkg/tests/batch/test_job_time_limit_class.py
+++ b/flepimop/gempyor_pkg/tests/batch/test_job_time_limit_class.py
@@ -4,12 +4,13 @@
 
 import pytest
 
-from gempyor.batch import JobTimeLimit
+from gempyor.batch import JobSize, JobTimeLimit
 
 
-@pytest.mark.parametrize(
-    "time_limit", (timedelta(), timedelta(hours=-1.0), timedelta(days=-3.0))
-)
+NONPOSITIVE_TIMEDELTAS = (timedelta(), timedelta(hours=-1.0), timedelta(days=-3.0))
+
+
+@pytest.mark.parametrize("time_limit", NONPOSITIVE_TIMEDELTAS)
 def test_time_limit_non_positive_value_error(time_limit: timedelta) -> None:
     with pytest.raises(
         ValueError,
@@ -61,3 +62,63 @@ def test_format_exact_results(
 ) -> None:
     job_time_limit = JobTimeLimit(time_limit=time_limit)
     assert job_time_limit.format(batch_system=batch_system) == expected
+
+
+@pytest.mark.parametrize("time_per_simulation", NONPOSITIVE_TIMEDELTAS)
+def test_from_per_simulation_time_per_simulation_nonpositive_value_error(
+    time_per_simulation: timedelta,
+) -> None:
+    job_size = JobSize(jobs=1, simulations=1, blocks=1)
+    with pytest.raises(
+        ValueError,
+        match=(
+            r"^The \`time\_per\_simulation\` is \'[0-9\,\-]+\' seconds\, "
+            r"which is less than or equal to 0\.$"
+        ),
+    ):
+        JobTimeLimit.from_per_simulation_time(
+            job_size, time_per_simulation, timedelta(minutes=10)
+        )
+
+
+@pytest.mark.parametrize("initial_time", NONPOSITIVE_TIMEDELTAS)
+def test_from_per_simulation_initial_time_nonpositive_value_error(
+    initial_time: timedelta,
+) -> None:
+    job_size = JobSize(jobs=1, simulations=1, blocks=1)
+    with pytest.raises(
+        ValueError,
+        match=(
+            r"^The \`initial\_time\` is \'[0-9\,\-]+\' seconds\, "
+            r"which is less than or equal to 0\.$"
+        ),
+    ):
+        JobTimeLimit.from_per_simulation_time(job_size, timedelta(minutes=10), initial_time)
+
+
+@pytest.mark.parametrize(
+    "job_size",
+    (
+        JobSize(jobs=1, simulations=10, blocks=1),
+        JobSize(jobs=10, simulations=25, blocks=15),
+    ),
+)
+@pytest.mark.parametrize(
+    "time_per_simulation",
+    (timedelta(minutes=5), timedelta(seconds=120), timedelta(hours=1.5)),
+)
+@pytest.mark.parametrize(
+    "initial_time", (timedelta(minutes=10), timedelta(seconds=30), timedelta(hours=2))
+)
+def test_from_per_simulation_time(
+    job_size: JobSize, time_per_simulation: timedelta, initial_time: timedelta
+) -> None:
+    job_time_limit = JobTimeLimit.from_per_simulation_time(
+        job_size, time_per_simulation, initial_time
+    )
+    assert job_time_limit.time_limit >= initial_time
+
+    double_job_time_limit = JobTimeLimit.from_per_simulation_time(
+        job_size, 2 * time_per_simulation, initial_time
+    )
+    assert double_job_time_limit > job_time_limit

From 772e872b8f13a22b1eed80c917e60d91a116629f Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Wed, 6 Nov 2024 09:26:10 -0500
Subject: [PATCH 27/78] Begin constructing sbatch call

Started work constructing a call to `_sbatch_template` in the `flepimop
batch` CLI. Incorporated `JobTimeLimit` class and added additional
comments into the `_click_batch` function.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py     | 59 +++++++++++++++++++
 .../templates/emcee_inference.sbatch.j2       |  6 +-
 2 files changed, 64 insertions(+), 1 deletion(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index dbb242f8b..5959d905a 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -615,16 +615,39 @@ def _resolve_batch_system(
             required=True,
             help="Path to the project directory being used.",
         ),
+        click.Option(
+            ["--partition"],
+            "partition",
+            type=str,
+            default=None,
+            help="The slurm partition to submit the job to.",
+        ),
+        click.Option(
+            ["--simulation-time"],
+            "simulation_time",
+            type=click.FloatRange(min=0.0),
+            default=3.0,
+            help="The time limit in minutes per a simulation.",
+        ),
+        click.Option(
+            ["--initial-time"],
+            "initial_time",
+            type=click.FloatRange(min=0.0),
+            default=20.0,
+            help="The initialization time limit in minutes.",
+        ),
     ]
     + list(verbosity_options.values()),
 )
 @click.pass_context
 def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
     """Submit batch jobs"""
+    # Generic setup
     logger = get_script_logger(__name__, kwargs.get("verbosity", 0))
     log_cli_inputs(kwargs)
     cfg = parse_config_files(config, ctx, **kwargs)
 
+    # Temporary limitation
     if (
         not cfg["inference"].exists()
         or not cfg["inference"]["method"].exists()
@@ -634,19 +657,23 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
             "The `flepimop batch` CLI only supports EMCEE inference jobs."
         )
 
+    # Job name
     name = cfg["name"].get(str) if cfg["name"].exists() else None
     job_name = _job_name(name, None)
     logger.info("Assigning job name of '%s'", job_name)
 
+    # Batch system
     batch_system = _resolve_batch_system(
         kwargs["batch_system"], kwargs["aws"], kwargs["local"], kwargs["slurm"]
     )
     if batch_system != "slurm":
+        # Temporary limitation
         raise NotImplementedError(
             "The `flepimop batch` CLI only supports batch submission to slurm."
         )
     logger.info("Constructing a job to submit to %s", batch_system)
 
+    # Job size
     iterations_per_slot = (
         cfg["inference"]["iterations_per_slot"].get(int)
         if cfg["inference"].exists() and cfg["inference"]["iterations_per_slot"].exists()
@@ -673,9 +700,41 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
     )
     logger.info("Preparing a job with size %s", job_size)
 
+    # Job time limit
+    job_time_limit = JobTimeLimit.from_per_simulation_time(
+        job_size,
+        timedelta(minutes=kwargs["simulation_time"]),
+        timedelta(minutes=kwargs["initial_time"]),
+    )
+    logger.info("Setting a total job time limit of %s minutes", job_time_limit.format())
+
     # Restart/continuation location
     # TODO: Implement this
 
     # Manifest
     manifest = write_manifest(job_name, kwargs["flepi_path"], kwargs["project_path"])
     logger.info("Writing manifest metadata to '%s'", manifest.absolute())
+
+    # Construct the sbatch call
+    template_data = {}
+    options = {
+        "chdir": kwargs["project_path"].absolute(),
+        "cpus-per-task": job_size.jobs,
+        "job-name": job_name,
+        "mem": "100GB",
+        "nodes": 1,  # EMCEE can only run on one node for now.
+        "ntasks": 1,
+        "partition": "general",
+        "time": job_time_limit.format("slurm"),
+    }
+    if kwargs["partition"] is not None:
+        options["partition"] = kwargs["partition"]
+    _sbatch_template(
+        "emcee_inference.sbatch.j2",
+        None,  # Script
+        template_data,  # Template data
+        "all",
+        options,
+        kwargs["verbosity"],
+        kwargs["dry_run"],
+    )
diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2 b/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2
index 6cab06a0b..66c00bdc4 100644
--- a/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2
+++ b/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2
@@ -9,7 +9,11 @@ set -x
 {% endif %}
 
 # Set script variables
-FLEPI_SLOT_INDEX=${SLURM_ARRAY_TASK_ID}
+if [ -z "$SLURM_ARRAY_TASK_ID" ]; then
+    FLEPI_SLOT_INDEX="1"
+else
+    FLEPI_SLOT_INDEX=$SLURM_ARRAY_TASK_ID
+fi
 
 # Load modules
 module purge

From a8ab8db3ee4d9dcd00d125b9ccb808bb93a041ab Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Wed, 6 Nov 2024 10:58:20 -0500
Subject: [PATCH 28/78] Add custom duration click param type

Added the `DurationParamType` for specifying custom durations in a user
friendly way for CLI interfaces along with corresponding unit tests.
Also incorporated the new custom param type into the `flepimop batch`
CLI.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py     | 17 +++--
 .../gempyor_pkg/src/gempyor/shared_cli.py     | 69 +++++++++++++++++--
 .../test_duration_param_type_class.py         | 40 +++++++++++
 3 files changed, 112 insertions(+), 14 deletions(-)
 create mode 100644 flepimop/gempyor_pkg/tests/shared_cli/test_duration_param_type_class.py

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 5959d905a..1ba7c9ee9 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -24,6 +24,7 @@
 from .logging import get_script_logger
 from .utils import _format_cli_options, _git_head, _shutil_which, config
 from .shared_cli import (
+    NONNEGATIVE_DURATION,
     cli,
     config_files_argument,
     config_file_options,
@@ -625,16 +626,16 @@ def _resolve_batch_system(
         click.Option(
             ["--simulation-time"],
             "simulation_time",
-            type=click.FloatRange(min=0.0),
-            default=3.0,
-            help="The time limit in minutes per a simulation.",
+            type=NONNEGATIVE_DURATION,
+            default=timedelta(minutes=3.0),
+            help="The time limit per a simulation.",
         ),
         click.Option(
             ["--initial-time"],
             "initial_time",
-            type=click.FloatRange(min=0.0),
-            default=20.0,
-            help="The initialization time limit in minutes.",
+            type=NONNEGATIVE_DURATION,
+            default=timedelta(minutes=20.0),
+            help="The initialization time limit.",
         ),
     ]
     + list(verbosity_options.values()),
@@ -702,9 +703,7 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
 
     # Job time limit
     job_time_limit = JobTimeLimit.from_per_simulation_time(
-        job_size,
-        timedelta(minutes=kwargs["simulation_time"]),
-        timedelta(minutes=kwargs["initial_time"]),
+        job_size, kwargs["simulation_time"], kwargs["initial_time"]
     )
     logger.info("Setting a total job time limit of %s minutes", job_time_limit.format())
 
diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py
index bd720d562..84b2c3897 100644
--- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py
+++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py
@@ -3,19 +3,21 @@
 supported options for config file overrides, and custom click decorators.
 """
 
+__all__ = []
+
+
+from datetime import timedelta
 import multiprocessing
 import pathlib
-import warnings
-from typing import Callable, Any
 import re
+from typing import Any, Callable, Literal
+import warnings
 
 import click
 import confuse
 
 from .logging import get_script_logger
-from .utils import config, as_list
-
-__all__ = []
+from .utils import as_list, config
 
 
 @click.group()
@@ -140,6 +142,63 @@ def cli(ctx: click.Context) -> None:
 }
 
 
+class DurationParamType(click.ParamType):
+    name = "duration"
+    _abbreviations = {
+        "s": "seconds",
+        "sec": "seconds",
+        "secs": "seconds",
+        "second": "seconds",
+        "seconds": "seconds",
+        "m": "minutes",
+        "min": "minutes",
+        "mins": "minutes",
+        "minute": "minutes",
+        "minutes": "minutes",
+        "h": "hours",
+        "hr": "hours",
+        "hrs": "hours",
+        "hour": "hours",
+        "hours": "hours",
+        "d": "days",
+        "day": "days",
+        "days": "days",
+        "w": "weeks",
+        "week": "weeks",
+        "weeks": "weeks",
+    }
+
+    def __init__(
+        self,
+        nonnegative: bool,
+        default_unit: Literal["seconds", "minutes", "hours", "days", "weeks"] = "minutes",
+    ) -> None:
+        super().__init__()
+        self._nonnegative = nonnegative
+        self._duration_regex = re.compile(
+            rf"^((-)?([0-9]+)?(\.[0-9]+)?)({'|'.join(self._abbreviations.keys())})?$",
+            flags=re.IGNORECASE,
+        )
+        self._default_unit = default_unit
+
+    def convert(
+        self, value: Any, param: click.Parameter | None, ctx: click.Context | None
+    ) -> timedelta:
+        value = str(value).strip()
+        if (m := self._duration_regex.match(value)) is None:
+            self.fail(f"{value!r} is not a valid duration", param, ctx)
+        number, posneg, _, _, unit = m.groups()
+        if self._nonnegative and posneg == "-":
+            self.fail(f"{value!r} is a negative duration", param, ctx)
+        kwargs = {}
+        kwargs[self._abbreviations.get(unit, self._default_unit)] = float(number)
+        return timedelta(**kwargs)
+
+
+DURATION = DurationParamType(nonnegative=False)
+NONNEGATIVE_DURATION = DurationParamType(nonnegative=True)
+
+
 def click_helpstring(
     params: click.Parameter | list[click.Parameter],
 ) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
diff --git a/flepimop/gempyor_pkg/tests/shared_cli/test_duration_param_type_class.py b/flepimop/gempyor_pkg/tests/shared_cli/test_duration_param_type_class.py
new file mode 100644
index 000000000..cfeffb146
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/shared_cli/test_duration_param_type_class.py
@@ -0,0 +1,40 @@
+from datetime import timedelta
+from typing import Any
+
+from click.exceptions import BadParameter
+import pytest
+
+from gempyor.shared_cli import DurationParamType
+
+
+@pytest.mark.parametrize("nonnegative", (True, False))
+@pytest.mark.parametrize("value", ("abc", "$12.34", "12..3", "12years", "12.a2"))
+def test_invalid_duration_bad_parameter(nonnegative: bool, value: Any) -> None:
+    duration = DurationParamType(nonnegative=nonnegative)
+    with pytest.raises(BadParameter, match="^'.*' is not a valid duration$"):
+        duration.convert(value, None, None)
+
+
+@pytest.mark.parametrize("value", ("-1", "-123", "-99.45", "-.9"))
+def test_negative_duration_bad_parameter(value: Any) -> None:
+    duration = DurationParamType(nonnegative=True)
+    with pytest.raises(BadParameter, match="^'.*' is a negative duration$"):
+        duration.convert(value, None, None)
+
+
+@pytest.mark.parametrize(
+    ("value", "expected"),
+    (
+        ("1", timedelta(minutes=1)),
+        ("2s", timedelta(seconds=2)),
+        ("3hrs", timedelta(hours=3)),
+        ("-4min", timedelta(minutes=-4)),
+        ("-5d", timedelta(days=-5)),
+        ("12.3", timedelta(minutes=12.3)),
+        ("-45.6h", timedelta(hours=-45.6)),
+        ("-.1w", timedelta(weeks=-0.1)),
+    ),
+)
+def test_exact_results_for_select_inputs(value: Any, expected: timedelta) -> None:
+    duration = DurationParamType(nonnegative=False)
+    assert duration.convert(value, None, None) == expected

From 8db1e4a8818e4b2c5e7a39a03f1083551e8f2e61 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Wed, 6 Nov 2024 11:30:38 -0500
Subject: [PATCH 29/78] Fix bug in time defaults, add in cluster info

* Changed default from `timedelta` to `str`. Looks like the `default`
  value is passed to the `convert` method of `type` if instance of
 `click.ParamType`.
* Began work incorporating cluster info into batch submission with
  slurm.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py | 31 ++++++++++++++++++++---
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 1ba7c9ee9..ae11986a4 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -11,6 +11,7 @@
 import click
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
+from getpass import getuser
 import json
 import math
 from pathlib import Path
@@ -21,6 +22,7 @@
 from typing import Any, Literal, Self
 
 from ._jinja import _render_template_to_file, _render_template_to_temp_file
+from .info import Cluster, get_cluster_info
 from .logging import get_script_logger
 from .utils import _format_cli_options, _git_head, _shutil_which, config
 from .shared_cli import (
@@ -627,16 +629,26 @@ def _resolve_batch_system(
             ["--simulation-time"],
             "simulation_time",
             type=NONNEGATIVE_DURATION,
-            default=timedelta(minutes=3.0),
+            default="3min",
             help="The time limit per a simulation.",
         ),
         click.Option(
             ["--initial-time"],
             "initial_time",
             type=NONNEGATIVE_DURATION,
-            default=timedelta(minutes=20.0),
+            default="20min",
             help="The initialization time limit.",
         ),
+        click.Option(
+            ["--cluster"],
+            "cluster",
+            type=str,
+            default=None,
+            help=(
+                "The name of the cluster this job is being launched on. "
+                "Only applicable to slurm submissions."
+            ),
+        ),
     ]
     + list(verbosity_options.values()),
 )
@@ -644,6 +656,7 @@ def _resolve_batch_system(
 def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
     """Submit batch jobs"""
     # Generic setup
+    now = datetime.now(timezone.utc)
     logger = get_script_logger(__name__, kwargs.get("verbosity", 0))
     log_cli_inputs(kwargs)
     cfg = parse_config_files(config, ctx, **kwargs)
@@ -660,7 +673,7 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
 
     # Job name
     name = cfg["name"].get(str) if cfg["name"].exists() else None
-    job_name = _job_name(name, None)
+    job_name = _job_name(name, now)
     logger.info("Assigning job name of '%s'", job_name)
 
     # Batch system
@@ -707,6 +720,13 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
     )
     logger.info("Setting a total job time limit of %s minutes", job_time_limit.format())
 
+    # Cluster info
+    cluster: Cluster | None = None
+    if batch_system == "slurm":
+        if kwargs["cluster"] is None:
+            raise ValueError("When submitting a batch job to slurm a cluster is required.")
+        cluster = get_cluster_info(kwargs["cluster"])
+
     # Restart/continuation location
     # TODO: Implement this
 
@@ -715,9 +735,12 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
     logger.info("Writing manifest metadata to '%s'", manifest.absolute())
 
     # Construct the sbatch call
-    template_data = {}
+    template_data = {
+        "cluster": cluster if cluster is None else cluster.model_dump(),
+    }
     options = {
         "chdir": kwargs["project_path"].absolute(),
+        "comment": f"Generated on {now:%c %Z} and submitted by {getuser()}.",
         "cpus-per-task": job_size.jobs,
         "job-name": job_name,
         "mem": "100GB",

From 5a95683986645b3b77d98caaa0c380546cbb5b31 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Wed, 6 Nov 2024 13:16:23 -0500
Subject: [PATCH 30/78] Write config for batch, work on batch script

* Final config file is now parsed out to a file, either user specified
  or temp.
* Further work on the batch script, mostly in translating variables
  given to `_click_batch` to the template.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py     | 58 +++++++++++++++++--
 .../templates/emcee_inference.sbatch.j2       | 15 ++++-
 2 files changed, 67 insertions(+), 6 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index ae11986a4..8c45d13d3 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -8,7 +8,6 @@
 __all__ = ["JobSize", "JobTimeLimit", "write_manifest"]
 
 
-import click
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 from getpass import getuser
@@ -19,8 +18,12 @@
 from shlex import quote
 import subprocess
 import sys
+from tempfile import mkstemp
 from typing import Any, Literal, Self
 
+import click
+import yaml
+
 from ._jinja import _render_template_to_file, _render_template_to_temp_file
 from .info import Cluster, get_cluster_info
 from .logging import get_script_logger
@@ -606,7 +609,7 @@ def _resolve_batch_system(
             ["--flepi-path"],
             "flepi_path",
             envvar="FLEPI_PATH",
-            type=click.Path(exists=True),
+            type=click.Path(exists=True, path_type=Path),
             required=True,
             help="Path to the flepiMoP directory being used.",
         ),
@@ -614,7 +617,7 @@ def _resolve_batch_system(
             ["--project-path"],
             "project_path",
             envvar="PROJECT_PATH",
-            type=click.Path(exists=True),
+            type=click.Path(exists=True, path_type=Path),
             required=True,
             help="Path to the project directory being used.",
         ),
@@ -649,6 +652,31 @@ def _resolve_batch_system(
                 "Only applicable to slurm submissions."
             ),
         ),
+        click.Option(
+            ["--conda-env"],
+            "conda_env",
+            type=str,
+            default="flepimop-env",
+            help="The name of the conda environment being used for this batch run.",
+        ),
+        click.Option(
+            ["--debug"],
+            "debug",
+            type=bool,
+            default=False,
+            is_flag=True,
+            help="Flag to enable debugging in batch submission scripts.",
+        ),
+        click.Option(
+            ["--config-out"],
+            "config_out",
+            type=click.Path(path_type=Path),
+            default=None,
+            help=(
+                "The location to dump the final parsed config file. "
+                "If not provided a temporary file will be used."
+            ),
+        ),
     ]
     + list(verbosity_options.values()),
 )
@@ -726,6 +754,10 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
         if kwargs["cluster"] is None:
             raise ValueError("When submitting a batch job to slurm a cluster is required.")
         cluster = get_cluster_info(kwargs["cluster"])
+    logger.info("Utilizing info for the '%s' cluster to construct this job", cluster.name)
+    logger.debug(
+        "The full settings for cluster '%s' are %s", cluster.name, cluster.model_dump()
+    )
 
     # Restart/continuation location
     # TODO: Implement this
@@ -734,9 +766,25 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
     manifest = write_manifest(job_name, kwargs["flepi_path"], kwargs["project_path"])
     logger.info("Writing manifest metadata to '%s'", manifest.absolute())
 
+    # Config out
+    if (config_out := kwargs["config_out"]) is None:
+        _, tmp = mkstemp(suffix=".yml", prefix="config", text=True)
+        config_out = Path(tmp)
+    with config_out.open(mode="w") as f:
+        f.write(cfg.dump())
+    logger.info(
+        "Dumped the final config for this batch submission to %s", config_out.absolute()
+    )
+
     # Construct the sbatch call
     template_data = {
+        "conda_env": kwargs["conda_env"],
+        "config_out": config_out.absolute(),
         "cluster": cluster if cluster is None else cluster.model_dump(),
+        "debug": kwargs["debug"],
+        "flepi_path": kwargs["flepi_path"].absolute(),
+        "jobs": job_size.jobs,
+        "project_path": kwargs["project_path"].absolute(),
     }
     options = {
         "chdir": kwargs["project_path"].absolute(),
@@ -753,8 +801,8 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
         options["partition"] = kwargs["partition"]
     _sbatch_template(
         "emcee_inference.sbatch.j2",
-        None,  # Script
-        template_data,  # Template data
+        None,
+        template_data,
         "all",
         options,
         kwargs["verbosity"],
diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2 b/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2
index 66c00bdc4..c57575fcc 100644
--- a/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2
+++ b/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2
@@ -3,7 +3,7 @@
 #SBATCH {{ opt }}
 {%- endfor %}
 
-{% if sbatch_debug %}
+{% if debug %}
 # Debugging
 set -x
 {% endif %}
@@ -14,6 +14,9 @@ if [ -z "$SLURM_ARRAY_TASK_ID" ]; then
 else
     FLEPI_SLOT_INDEX=$SLURM_ARRAY_TASK_ID
 fi
+export FLEPI_PATH={{ flepi_path }}
+export PROJECT_PATH={{ project_path }}
+export CONFIG_PATH={{ config_out }}
 
 # Load modules
 module purge
@@ -47,5 +50,15 @@ conda activate {{ conda_env }}
 # Fetch/setup resume files
 
 # Run inference
+flepimop-calibrate --config $CONFIG_PATH \
+    --project_path $PROJECT_PATH \
+    --nslots {{ nslots }} \
+    --niterations {{ niterations }} \
+    --nthin {{ nthin }} \
+    --jobs {{ jobs }} \
+    --id {{ id }} \
+    --prefix {{ prefix}} \
+    --no-resume \
+    --resume_location {{ resume_location }} > out.out 2>&1
 
 # Upload/move results

From 3fc27b9814f5d444d66ed97aa93f5b5fb348add2 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Wed, 6 Nov 2024 13:54:50 -0500
Subject: [PATCH 31/78] Implement options for `flepimop-calibrate`

Implemented the remaining options for `flepimop-calibrate` into
`flepimop batch`.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py     | 42 ++++++++++++++++++-
 .../templates/emcee_inference.sbatch.j2       |  9 ++--
 2 files changed, 45 insertions(+), 6 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 8c45d13d3..3401ba31b 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -22,9 +22,9 @@
 from typing import Any, Literal, Self
 
 import click
-import yaml
 
 from ._jinja import _render_template_to_file, _render_template_to_temp_file
+from .file_paths import run_id
 from .info import Cluster, get_cluster_info
 from .logging import get_script_logger
 from .utils import _format_cli_options, _git_head, _shutil_which, config
@@ -677,6 +677,29 @@ def _resolve_batch_system(
                 "If not provided a temporary file will be used."
             ),
         ),
+        click.Option(
+            ["--nthin"],
+            "nthin",
+            type=click.IntRange(min=5),
+            default=5,
+            help="The number of samples to thin.",
+        ),
+        click.Option(
+            ["--id", "--run-id"],
+            "run_id",
+            envvar="FLEPI_RUN_INDEX",
+            type=str,
+            default=None,
+            help="Unique identifier for this run.",
+        ),
+        click.Option(
+            ["--prefix"],
+            "prefix",
+            envvar="FLEPI_PREFIX",
+            type=str,
+            default=None,
+            help="Unique prefix for this run.",
+        ),
     ]
     + list(verbosity_options.values()),
 )
@@ -698,11 +721,15 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
         raise NotImplementedError(
             "The `flepimop batch` CLI only supports EMCEE inference jobs."
         )
+    inference_method = cfg["inference"]["method"].as_str()
 
-    # Job name
+    # Job name/run id
     name = cfg["name"].get(str) if cfg["name"].exists() else None
     job_name = _job_name(name, now)
     logger.info("Assigning job name of '%s'", job_name)
+    if kwargs["run_id"] is None:
+        kwargs["run_id"] = run_id(now)
+    logger.info("Using a run id of '%s'", kwargs["run_id"])
 
     # Batch system
     batch_system = _resolve_batch_system(
@@ -741,6 +768,11 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
         batch_system,
     )
     logger.info("Preparing a job with size %s", job_size)
+    if inference_method == "emcee" and job_size.blocks != 1:
+        logger.warning(
+            "When using EMCEE for inference the job size blocks is ignored, given %u.",
+            job_size.blocks,
+        )
 
     # Job time limit
     job_time_limit = JobTimeLimit.from_per_simulation_time(
@@ -783,8 +815,14 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
         "cluster": cluster if cluster is None else cluster.model_dump(),
         "debug": kwargs["debug"],
         "flepi_path": kwargs["flepi_path"].absolute(),
+        "job_name": job_name,
         "jobs": job_size.jobs,
+        "nslots": kwargs["nslots"],
+        "nthin": kwargs["nthin"],
+        "prefix": kwargs["prefix"],
         "project_path": kwargs["project_path"].absolute(),
+        "run_id": kwargs["run_id"],
+        "simulations": job_size.simulations,
     }
     options = {
         "chdir": kwargs["project_path"].absolute(),
diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2 b/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2
index c57575fcc..22afc93ee 100644
--- a/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2
+++ b/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2
@@ -53,12 +53,13 @@ conda activate {{ conda_env }}
 flepimop-calibrate --config $CONFIG_PATH \
     --project_path $PROJECT_PATH \
     --nslots {{ nslots }} \
-    --niterations {{ niterations }} \
+    --niterations {{ simulations }} \
     --nthin {{ nthin }} \
     --jobs {{ jobs }} \
-    --id {{ id }} \
+    --id {{ run_id }} \
+{% if prefix is defined and prefix is not none -%}
     --prefix {{ prefix}} \
-    --no-resume \
-    --resume_location {{ resume_location }} > out.out 2>&1
+{%- endif %}
+    > out_{{ job_name }}_$FLEPI_SLOT_INDEX.out 2>&1
 
 # Upload/move results

From c40aed9503c82b9b270a151d2b627c743ba3e1a6 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Wed, 6 Nov 2024 14:56:45 -0500
Subject: [PATCH 32/78] Add `--email` to `flepimop batch`

Added an email option for slurm jobs for notifying users of job status
via email if desired.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 3401ba31b..57b8f21b6 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -700,6 +700,13 @@ def _resolve_batch_system(
             default=None,
             help="Unique prefix for this run.",
         ),
+        click.Option(
+            ["--email"],
+            "email",
+            type=str,
+            default=None,
+            help="Optionally an email that can be notified on job begin and end.",
+        ),
     ]
     + list(verbosity_options.values()),
 )
@@ -741,6 +748,13 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
             "The `flepimop batch` CLI only supports batch submission to slurm."
         )
     logger.info("Constructing a job to submit to %s", batch_system)
+    if batch_system != "slurm" and kwargs["email"] is not None:
+        logger.warning(
+            "The email option, given '%s', is only used when "
+            "the batch system is slurm, but is instead %s.",
+            kwargs["email"],
+            batch_system,
+        )
 
     # Job size
     iterations_per_slot = (
@@ -837,6 +851,9 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
     }
     if kwargs["partition"] is not None:
         options["partition"] = kwargs["partition"]
+    if kwargs["email"] is not None:
+        options["mail-type"] = "BEGIN,END"
+        options["mail-user"] = kwargs["email"]
     _sbatch_template(
         "emcee_inference.sbatch.j2",
         None,

From 458ac001c307da88b63fd762c8cb5b95bcb81102 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Thu, 7 Nov 2024 12:57:31 -0500
Subject: [PATCH 33/78] Correct `param_decls` arg to `click.Option`

Previously was calling `click.Option` with `param_decls` specified
incorrectly. The option name was being provided correctly, but the name
of the option as an argument was being provided as a seperate arg
(unclear which though).
---
 flepimop/gempyor_pkg/src/gempyor/batch.py     | 59 +++++++------------
 .../gempyor_pkg/src/gempyor/shared_cli.py     |  4 +-
 2 files changed, 22 insertions(+), 41 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 57b8f21b6..9a70f5472 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -561,90 +561,78 @@ def _resolve_batch_system(
     + list(config_file_options.values())
     + [
         click.Option(
-            ["--simulations"],
-            "simulations",
+            param_decls=["--simulations", "simulations"],
             default=None,
             type=click.IntRange(min=1),
             help="The number of simulations per a job.",
         ),
         click.Option(
-            ["--blocks"],
-            "blocks",
+            param_decls=["--blocks", "blocks"],
             default=None,
             type=click.IntRange(min=1),
             help="The number of sequential blocks to run per a job.",
         ),
         click.Option(
-            ["--batch-system"],
-            "batch_system",
+            param_decls=["--batch-system", "batch_system"],
             default=None,
             type=click.Choice(("aws", "local", "slurm"), case_sensitive=False),
             help="The name of the batch system being used.",
         ),
         click.Option(
-            ["--aws"],
-            "aws",
+            param_decls=["--aws", "aws"],
             default=False,
             type=bool,
             is_flag=True,
             help="A flag indicating this is being run on AWS.",
         ),
         click.Option(
-            ["--local"],
-            "local",
+            param_decls=["--local", "local"],
             default=False,
             type=bool,
             is_flag=True,
             help="A flag indicating this is being run local.",
         ),
         click.Option(
-            ["--slurm"],
-            "slurm",
+            param_decls=["--slurm", "slurm"],
             default=False,
             type=bool,
             is_flag=True,
             help="A flag indicating this is being run on slurm.",
         ),
         click.Option(
-            ["--flepi-path"],
-            "flepi_path",
+            param_decls=["--flepi-path", "flepi_path"],
             envvar="FLEPI_PATH",
             type=click.Path(exists=True, path_type=Path),
             required=True,
             help="Path to the flepiMoP directory being used.",
         ),
         click.Option(
-            ["--project-path"],
-            "project_path",
+            param_decls=["--project-path", "project_path"],
             envvar="PROJECT_PATH",
             type=click.Path(exists=True, path_type=Path),
             required=True,
             help="Path to the project directory being used.",
         ),
         click.Option(
-            ["--partition"],
-            "partition",
+            param_decls=["--partition", "partition"],
             type=str,
             default=None,
             help="The slurm partition to submit the job to.",
         ),
         click.Option(
-            ["--simulation-time"],
-            "simulation_time",
+            param_decls=["--simulation-time", "simulation_time"],
             type=NONNEGATIVE_DURATION,
             default="3min",
             help="The time limit per a simulation.",
         ),
         click.Option(
-            ["--initial-time"],
-            "initial_time",
+            param_decls=["--initial-time", "initial_time"],
             type=NONNEGATIVE_DURATION,
             default="20min",
             help="The initialization time limit.",
         ),
         click.Option(
-            ["--cluster"],
-            "cluster",
+            param_decls=["--cluster", "cluster"],
             type=str,
             default=None,
             help=(
@@ -653,23 +641,20 @@ def _resolve_batch_system(
             ),
         ),
         click.Option(
-            ["--conda-env"],
-            "conda_env",
+            param_decls=["--conda-env", "conda_env"],
             type=str,
             default="flepimop-env",
             help="The name of the conda environment being used for this batch run.",
         ),
         click.Option(
-            ["--debug"],
-            "debug",
+            param_decls=["--debug", "debug"],
             type=bool,
             default=False,
             is_flag=True,
             help="Flag to enable debugging in batch submission scripts.",
         ),
         click.Option(
-            ["--config-out"],
-            "config_out",
+            param_decls=["--config-out", "config_out"],
             type=click.Path(path_type=Path),
             default=None,
             help=(
@@ -678,31 +663,27 @@ def _resolve_batch_system(
             ),
         ),
         click.Option(
-            ["--nthin"],
-            "nthin",
+            param_decls=["--nthin", "nthin"],
             type=click.IntRange(min=5),
             default=5,
             help="The number of samples to thin.",
         ),
         click.Option(
-            ["--id", "--run-id"],
-            "run_id",
+            param_decls=["--id", "--run-id", "run_id"],
             envvar="FLEPI_RUN_INDEX",
             type=str,
             default=None,
             help="Unique identifier for this run.",
         ),
         click.Option(
-            ["--prefix"],
-            "prefix",
+            param_decls=["--prefix", "prefix"],
             envvar="FLEPI_PREFIX",
             type=str,
             default=None,
             help="Unique prefix for this run.",
         ),
         click.Option(
-            ["--email"],
-            "email",
+            param_decls=["--email", "email"],
             type=str,
             default=None,
             help="Optionally an email that can be notified on job begin and end.",
@@ -715,7 +696,7 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
     """Submit batch jobs"""
     # Generic setup
     now = datetime.now(timezone.utc)
-    logger = get_script_logger(__name__, kwargs.get("verbosity", 0))
+    logger = get_script_logger(__name__, kwargs["verbosity"])
     log_cli_inputs(kwargs)
     cfg = parse_config_files(config, ctx, **kwargs)
 
diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py
index 84b2c3897..805de36c4 100644
--- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py
+++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py
@@ -128,12 +128,12 @@ def cli(ctx: click.Context) -> None:
 
 verbosity_options = {
     "verbosity": click.Option(
-        ["-v", "--verbose"],
+        param_decls=["-v", "--verbose", "verbosity"],
         count=True,
         help="The verbosity level to use for this command.",
     ),
     "dry_run": click.Option(
-        ["--dry-run"],
+        param_decls=["--dry-run", "dry_run"],
         type=bool,
         default=False,
         is_flag=True,

From b4c028d8adf2e466be83b2ad4b0508dee05c3034 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Thu, 7 Nov 2024 13:32:59 -0500
Subject: [PATCH 34/78] New lines in `emcee_inference` template

Ensuring the template renders correctly by adding appropriate new lines.
---
 .../templates/emcee_inference.sbatch.j2       | 20 ++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2 b/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2
index 22afc93ee..c707c73db 100644
--- a/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2
+++ b/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2
@@ -1,12 +1,12 @@
 #!/usr/bin/env {{ interpreter }}
-{% for opt in options -%}
+{%- for opt in options %}
 #SBATCH {{ opt }}
 {%- endfor %}
 
-{% if debug %}
+{% if debug -%}
 # Debugging
 set -x
-{% endif %}
+{%- endif %}
 
 # Set script variables
 if [ -z "$SLURM_ARRAY_TASK_ID" ]; then
@@ -18,18 +18,19 @@ export FLEPI_PATH={{ flepi_path }}
 export PROJECT_PATH={{ project_path }}
 export CONFIG_PATH={{ config_out }}
 
-# Load modules
+# Purge/load modules
 module purge
-{% for module in cluster.modules -%}
-{% if module.version is defined and module.version is not none -%}
+{%- for module in cluster.modules %}
+{%- if module.version is defined and module.version is not none %}
 module load {{ module.name }}/{{ module.version }}
-{%- else -%}
+{%- else %}
 module load {{ module.name }}
 {%- endif %}
 {%- endfor %}
 
+{% if cluster.path_exports|length -%}
 # Path modifications
-{% for path_export in cluster.path_exports -%}
+{%- for path_export in cluster.path_exports %}
 if [ -r "{{ path_export.path }}" ]; then
     {% if path_export.prepend -%}
     export PATH={{ path_export.path }}:$PATH
@@ -41,6 +42,7 @@ elif [ "{{ path_export.error_if_missing }}" = "True" ]; then
     exit 1
 fi
 {%- endfor %}
+{%- endif %}
 
 # Load conda env
 conda activate {{ conda_env }}
@@ -57,7 +59,7 @@ flepimop-calibrate --config $CONFIG_PATH \
     --nthin {{ nthin }} \
     --jobs {{ jobs }} \
     --id {{ run_id }} \
-{% if prefix is defined and prefix is not none -%}
+{%- if prefix is defined and prefix is not none %}
     --prefix {{ prefix}} \
 {%- endif %}
     > out_{{ job_name }}_$FLEPI_SLOT_INDEX.out 2>&1

From 1e7e1e45d0012a5cbfcbd4e05cf45af5e42c02f2 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Thu, 7 Nov 2024 15:15:41 -0500
Subject: [PATCH 35/78] Add slurm+cluster option value error unit test

Added a unit test to assert that a `ValueError` is raised when the
`--slurm` flag is provided and the `--cluster` flag is not.
---
 .../tests/batch/test__click_batch.py          | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/flepimop/gempyor_pkg/tests/batch/test__click_batch.py b/flepimop/gempyor_pkg/tests/batch/test__click_batch.py
index 2a8c6886c..bc83d8f24 100644
--- a/flepimop/gempyor_pkg/tests/batch/test__click_batch.py
+++ b/flepimop/gempyor_pkg/tests/batch/test__click_batch.py
@@ -53,3 +53,31 @@ def test_only_slurm_batch_system_supported_not_implemented_error(
         str(result.exception)
         == "The `flepimop batch` CLI only supports batch submission to slurm."
     )
+
+
+def test_cluster_required_for_slurm_value_error(tmp_path: Path) -> None:
+    config_file = tmp_path / "config.yml"
+    with config_file.open(mode="w") as f:
+        yaml.dump({"inference": {"method": "emcee"}}, f)
+
+    runner = CliRunner()
+    result = runner.invoke(
+        _click_batch,
+        [
+            "--slurm",
+            "--simulations",
+            "1",
+            "--jobs",
+            "1",
+            "--blocks",
+            "1",
+            str(config_file.absolute()),
+        ],
+    )
+
+    assert result.exit_code == 1
+    assert isinstance(result.exception, ValueError)
+    assert (
+        str(result.exception)
+        == "When submitting a batch job to slurm a cluster is required."
+    )

From a18af5fc22ee5bf141b05074d3bf8ea64b4c2c24 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Thu, 7 Nov 2024 16:05:43 -0500
Subject: [PATCH 36/78] Add `BatchSystem` enum

Added an enum for batch systems to formalize the valid systems flepiMoP
can run on.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py     | 112 ++++++++++--------
 .../tests/batch/test__click_batch.py          |  10 ++
 ...h_system.py => test_batch_system_class.py} |  26 ++--
 3 files changed, 87 insertions(+), 61 deletions(-)
 rename flepimop/gempyor_pkg/tests/batch/{test__resolve_batch_system.py => test_batch_system_class.py} (67%)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 9a70f5472..2c64bfd13 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -5,11 +5,12 @@
 metadata and job size calculations for example.
 """
 
-__all__ = ["JobSize", "JobTimeLimit", "write_manifest"]
+__all__ = ["BatchSystem", "JobSize", "JobTimeLimit", "write_manifest"]
 
 
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
+from enum import Enum, auto
 from getpass import getuser
 import json
 import math
@@ -43,6 +44,62 @@
 _JOB_NAME_REGEX = re.compile(r"^[a-z]{1}([a-z0-9\_\-]+)?$", flags=re.IGNORECASE)
 
 
+class BatchSystem(Enum):
+    """
+    Enum representing the various batch systems that flepiMoP can run on.
+    """
+
+    AWS = auto()
+    LOCAL = auto()
+    SLURM = auto()
+
+    @classmethod
+    def from_options(
+        cls,
+        batch_system: Literal["aws", "local", "slurm"] | None,
+        aws: bool,
+        local: bool,
+        slurm: bool,
+    ) -> "BatchSystem":
+        """
+        Resolve the batch system options.
+
+        Args:
+            batch_system: The name of the batch system to use if provided explicitly by
+                name or `None` to rely on the other flags.
+            aws: A flag indicating if the batch system should be AWS.
+            local: A flag indicating if the batch system should be local.
+            slurm: A flag indicating if the batch system should be slurm.
+
+        Returns:
+            The name of the batch system to use given the user options.
+        """
+        batch_system = batch_system.lower() if batch_system is not None else batch_system
+        if (boolean_flags := sum((aws, local, slurm))) > 1:
+            raise ValueError(
+                f"There were {boolean_flags} boolean flags given, expected either 0 or 1."
+            )
+        if batch_system is not None:
+            for name, flag in zip(("aws", "local", "slurm"), (aws, local, slurm)):
+                if flag and batch_system != name:
+                    raise ValueError(
+                        "Conflicting batch systems given. The batch system name "
+                        f"is '{batch_system}' and the flags indicate '{name}'."
+                    )
+        if batch_system is None:
+            if aws:
+                batch_system = "aws"
+            elif local:
+                batch_system = "local"
+            else:
+                batch_system = "slurm"
+        if batch_system == "aws":
+            return cls.AWS
+        elif batch_system == "local":
+            return cls.LOCAL
+        return cls.SLURM
+
+
 @dataclass(frozen=True, slots=True)
 class JobSize:
     """
@@ -80,7 +137,7 @@ def size_from_jobs_sims_blocks(
         iterations_per_slot: int | None,
         slots: int | None,
         subpops: int | None,
-        batch_system: Literal["aws", "local", "slurm"],
+        batch_system: BatchSystem,
     ) -> "JobSize":
         """
         Infer a job size from several explicit and implicit parameters.
@@ -142,7 +199,7 @@ def size_from_jobs_sims_blocks(
                             "provided, then a subpops must be given."
                         )
                     )
-                if batch_system == "aws":
+                if batch_system == BatchSystem.AWS:
                     simulations = 5 * math.ceil(max(60 - math.sqrt(subpops), 10) / 5)
                 else:
                     simulations = iterations_per_slot
@@ -514,47 +571,6 @@ def _job_name(name: str | None, timestamp: datetime | None) -> str:
     return f"{name}-{timestamp}" if name else timestamp
 
 
-def _resolve_batch_system(
-    batch_system: Literal["aws", "local", "slurm"] | None,
-    aws: bool,
-    local: bool,
-    slurm: bool,
-) -> Literal["aws", "local", "slurm"]:
-    """
-    Resolve the batch system options.
-
-    Args:
-        batch_system: The name of the batch system to use if provided explicitly by
-            name or `None` to rely on the other flags.
-        aws: A flag indicating if the batch system should be AWS.
-        local: A flag indicating if the batch system should be local.
-        slurm: A flag indicating if the batch system should be slurm.
-
-    Returns:
-        The name of the batch system to use given the user options.
-    """
-    batch_system = batch_system.lower() if batch_system is not None else batch_system
-    if (boolean_flags := sum((aws, local, slurm))) > 1:
-        raise ValueError(
-            f"There were {boolean_flags} boolean flags given, expected either 0 or 1."
-        )
-    if batch_system is not None:
-        for name, flag in zip(("aws", "local", "slurm"), (aws, local, slurm)):
-            if flag and batch_system != name:
-                raise ValueError(
-                    "Conflicting batch systems given. The batch system name "
-                    f"is '{batch_system}' and the flags indicate '{name}'."
-                )
-    if batch_system is None:
-        if aws:
-            batch_system = "aws"
-        elif local:
-            batch_system = "local"
-        else:
-            batch_system = "slurm"
-    return batch_system
-
-
 @cli.command(
     name="batch",
     params=[config_files_argument]
@@ -720,16 +736,16 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
     logger.info("Using a run id of '%s'", kwargs["run_id"])
 
     # Batch system
-    batch_system = _resolve_batch_system(
+    batch_system = BatchSystem.from_options(
         kwargs["batch_system"], kwargs["aws"], kwargs["local"], kwargs["slurm"]
     )
-    if batch_system != "slurm":
+    if batch_system != BatchSystem.SLURM:
         # Temporary limitation
         raise NotImplementedError(
             "The `flepimop batch` CLI only supports batch submission to slurm."
         )
     logger.info("Constructing a job to submit to %s", batch_system)
-    if batch_system != "slurm" and kwargs["email"] is not None:
+    if batch_system != BatchSystem.SLURM and kwargs["email"] is not None:
         logger.warning(
             "The email option, given '%s', is only used when "
             "the batch system is slurm, but is instead %s.",
@@ -777,7 +793,7 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
 
     # Cluster info
     cluster: Cluster | None = None
-    if batch_system == "slurm":
+    if batch_system == BatchSystem.SLURM:
         if kwargs["cluster"] is None:
             raise ValueError("When submitting a batch job to slurm a cluster is required.")
         cluster = get_cluster_info(kwargs["cluster"])
diff --git a/flepimop/gempyor_pkg/tests/batch/test__click_batch.py b/flepimop/gempyor_pkg/tests/batch/test__click_batch.py
index bc83d8f24..f0c36dac2 100644
--- a/flepimop/gempyor_pkg/tests/batch/test__click_batch.py
+++ b/flepimop/gempyor_pkg/tests/batch/test__click_batch.py
@@ -1,3 +1,4 @@
+import os
 from pathlib import Path
 from typing import Any
 
@@ -8,6 +9,15 @@
 from gempyor.batch import _click_batch
 
 
+@pytest.fixture
+def add_sbatch_to_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+    sbatch = tmp_path / "bin" / "sbatch"
+    sbatch.parent.mkdir(parents=True, exist_ok=True)
+    sbatch.touch(mode=0o755)
+    monkeypatch.setenv("PATH", str(sbatch.parent.absolute()), prepend=os.pathsep)
+    return sbatch
+
+
 @pytest.mark.parametrize(
     "config",
     (
diff --git a/flepimop/gempyor_pkg/tests/batch/test__resolve_batch_system.py b/flepimop/gempyor_pkg/tests/batch/test_batch_system_class.py
similarity index 67%
rename from flepimop/gempyor_pkg/tests/batch/test__resolve_batch_system.py
rename to flepimop/gempyor_pkg/tests/batch/test_batch_system_class.py
index 6d5eb6420..fe60b5a52 100644
--- a/flepimop/gempyor_pkg/tests/batch/test__resolve_batch_system.py
+++ b/flepimop/gempyor_pkg/tests/batch/test_batch_system_class.py
@@ -2,7 +2,7 @@
 
 import pytest
 
-from gempyor.batch import _resolve_batch_system
+from gempyor.batch import BatchSystem
 
 
 @pytest.mark.parametrize(
@@ -17,7 +17,7 @@ def test_multiple_flags_value_error(aws: bool, local: bool, slurm: bool) -> None
             "flags given, expected either 0 or 1.$"
         ),
     ):
-        _resolve_batch_system(None, aws, local, slurm)
+        BatchSystem.from_options(None, aws, local, slurm)
 
 
 @pytest.mark.parametrize(
@@ -42,21 +42,21 @@ def test_batch_system_flag_mismatch_value_error(
             f"is '{batch_system}' and the flags indicate '{name}'.$"
         ),
     ):
-        _resolve_batch_system(batch_system, aws, local, slurm)
+        BatchSystem.from_options(batch_system, aws, local, slurm)
 
 
 @pytest.mark.parametrize(
     ("batch_system", "aws", "local", "slurm", "expected"),
     (
-        (None, True, False, False, "aws"),
-        ("aws", False, False, False, "aws"),
-        ("aws", True, False, False, "aws"),
-        (None, False, True, False, "local"),
-        ("local", False, False, False, "local"),
-        ("local", False, True, False, "local"),
-        (None, False, False, True, "slurm"),
-        ("slurm", False, False, False, "slurm"),
-        ("slurm", False, False, True, "slurm"),
+        (None, True, False, False, BatchSystem.AWS),
+        ("aws", False, False, False, BatchSystem.AWS),
+        ("aws", True, False, False, BatchSystem.AWS),
+        (None, False, True, False, BatchSystem.LOCAL),
+        ("local", False, False, False, BatchSystem.LOCAL),
+        ("local", False, True, False, BatchSystem.LOCAL),
+        (None, False, False, True, BatchSystem.SLURM),
+        ("slurm", False, False, False, BatchSystem.SLURM),
+        ("slurm", False, False, True, BatchSystem.SLURM),
     ),
 )
 def test_output_validation(
@@ -66,4 +66,4 @@ def test_output_validation(
     slurm: bool,
     expected: Literal["aws", "local", "slurm"],
 ) -> None:
-    assert _resolve_batch_system(batch_system, aws, local, slurm) == expected
+    assert BatchSystem.from_options(batch_system, aws, local, slurm) == expected

From b0d1852330b3080515b88eeb5f6bae753ae1e11f Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Thu, 7 Nov 2024 16:55:52 -0500
Subject: [PATCH 37/78] Greatly simplify `JobSize` for first draft

Mainly reduced the functionality of the `size_from_jobs_sims_blocks`
class method. Best to leave that to more feedback and GH-360.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py     | 92 ++-----------------
 .../tests/batch/test_job_size_class.py        | 74 ++-------------
 2 files changed, 19 insertions(+), 147 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 2c64bfd13..ae3137964 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -21,6 +21,7 @@
 import sys
 from tempfile import mkstemp
 from typing import Any, Literal, Self
+import warnings
 
 import click
 
@@ -134,10 +135,7 @@ def size_from_jobs_sims_blocks(
         jobs: int | None,
         simulations: int | None,
         blocks: int | None,
-        iterations_per_slot: int | None,
-        slots: int | None,
-        subpops: int | None,
-        batch_system: BatchSystem,
+        inference_method: Literal["emcee"] | None,
     ) -> "JobSize":
         """
         Infer a job size from several explicit and implicit parameters.
@@ -146,69 +144,17 @@ def size_from_jobs_sims_blocks(
             jobs: An explicit number of jobs.
             simulations: An explicit number of simulations per a block.
             blocks: An explicit number of blocks per a job.
-            iterations_per_slot: A total number of iterations per a job, which is
-                simulations times blocks. Required if `simulations` or `blocks` is
-                not given.
-            slots: An implicit number of slots to use for the job. Required if `jobs`
-                is not given.
-            subpops: The number of subpopulations being considered in this job. Affects
-                the inferred simulations per a job on AWS. Required if `simulations`
-                and `blocks` are not given.
-            batch_size: The system the job is being sized for. Affects the inferred
-                simulations per a job.
+            inference_method: The inference method being used as different methods have
+                different restrictions.
 
         Returns:
             A job size instance with either the explicit or inferred job sizing.
-
-        Examples:
-            >>> JobSize.size_from_jobs_sims_blocks(1, 2, 3, None, None, None, "local")
-            JobSize(jobs=1, simulations=2, blocks=3)
-            >>> JobSize.size_from_jobs_sims_blocks(
-            ...     None, None, None, 100, 10, 25, "local"
-            ... )
-            JobSize(jobs=10, simulations=100, blocks=1)
-            >>> JobSize.size_from_jobs_sims_blocks(None, None, 4, 100, 10, 25, "local")
-            JobSize(jobs=10, simulations=25, blocks=4)
-
-        Raises:
-            ValueError: If `iterations_per_slot` is `None` and either `simulations` or
-                `blocks` is `None`.
-            ValueError: If `jobs` and `slots` are both `None`.
-            ValueError: If `simulations`, `blocks`, and `subpops` are all `None`.
         """
-        if iterations_per_slot is None and (simulations is None or blocks is None):
-            raise ValueError(
-                (
-                    "If simulations and blocks are not all explicitly "
-                    "provided then an iterations per slot must be given."
-                )
-            )
-
-        jobs = slots if jobs is None else jobs
-        if jobs is None:
-            raise ValueError(
-                "If jobs is not explicitly provided, it must be given via slots."
-            )
-
-        if simulations is None:
-            if blocks is None:
-                if subpops is None:
-                    raise ValueError(
-                        (
-                            "If simulations and blocks are not explicitly "
-                            "provided, then a subpops must be given."
-                        )
-                    )
-                if batch_system == BatchSystem.AWS:
-                    simulations = 5 * math.ceil(max(60 - math.sqrt(subpops), 10) / 5)
-                else:
-                    simulations = iterations_per_slot
-            else:
-                simulations = math.ceil(iterations_per_slot / blocks)
-
-        if blocks is None:
-            blocks = math.ceil(iterations_per_slot / simulations)
-
+        inference_method = (
+            inference_method if inference_method is None else inference_method.lower()
+        )
+        if inference_method == "emcee":
+            return cls(jobs=jobs, simulations=blocks * simulations, blocks=1)
         return cls(jobs=jobs, simulations=simulations, blocks=blocks)
 
 
@@ -754,29 +700,11 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
         )
 
     # Job size
-    iterations_per_slot = (
-        cfg["inference"]["iterations_per_slot"].get(int)
-        if cfg["inference"].exists() and cfg["inference"]["iterations_per_slot"].exists()
-        else None
-    )
-    nslots = cfg["nslots"].get(int) if cfg["nslots"].exists() else None
-    subpops: int | None = None
-    if (
-        cfg["subpop_setup"].exists()
-        and cfg["subpop_setup"]["geodata"].exists()
-        and (geodata := cfg["subpop_setup"]["geodata"].as_path()).exists()
-    ):
-        with geodata.open() as f:
-            subpops = sum(1 for _ in f)
-        subpops -= 1
     job_size = JobSize.size_from_jobs_sims_blocks(
         kwargs["jobs"],
         kwargs["simulations"],
         kwargs["blocks"],
-        iterations_per_slot,
-        nslots,
-        subpops,
-        batch_system,
+        inference_method,
     )
     logger.info("Preparing a job with size %s", job_size)
     if inference_method == "emcee" and job_size.blocks != 1:
diff --git a/flepimop/gempyor_pkg/tests/batch/test_job_size_class.py b/flepimop/gempyor_pkg/tests/batch/test_job_size_class.py
index 7ffe4d7ac..eba282dab 100644
--- a/flepimop/gempyor_pkg/tests/batch/test_job_size_class.py
+++ b/flepimop/gempyor_pkg/tests/batch/test_job_size_class.py
@@ -32,76 +32,20 @@ def test_less_than_one_value_error(
         JobSize(**kwargs)
 
 
-@pytest.mark.parametrize(("simulations", "blocks"), [(None, None), (1, None), (None, 1)])
-def test_size_from_jobs_sims_blocks_iteration_value_error(
-    simulations: int | None, blocks: int | None
-) -> None:
-    with pytest.raises(
-        ValueError,
-        match=(
-            "^If simulations and blocks are not all explicitly "
-            "provided then an iterations per slot must be given.$"
-        ),
-    ):
-        JobSize.size_from_jobs_sims_blocks(1, simulations, blocks, None, 1, 1, "aws")
-
-
-def test_size_from_jobs_sims_blocks_slots_value_error() -> None:
-    with pytest.raises(
-        ValueError,
-        match="^If jobs is not explicitly provided, it must be given via slots.$",
-    ):
-        JobSize.size_from_jobs_sims_blocks(None, 1, 1, 1, None, 1, "aws")
-
-
-def test_size_from_jobs_sims_blocks_subpops_value_error() -> None:
-    with pytest.raises(
-        ValueError,
-        match=(
-            "^If simulations and blocks are not explicitly "
-            "provided, then a subpops must be given.$"
-        ),
-    ):
-        JobSize.size_from_jobs_sims_blocks(1, None, None, 1, 1, None, "aws")
-
-
 def generate_size_from_jobs_sims_blocks(
     *args: int | None,
 ) -> Generator[tuple[int | None, ...], None, None]:
-    for combo in product(args, repeat=6):
-        jobs, simulations, blocks, iterations_per_slot, slots, subpops = combo
-        if iterations_per_slot is None and (simulations is None or blocks is None):
-            continue
-        elif jobs is None and slots is None:
-            continue
-        elif simulations is None and blocks is None and subpops is None:
-            continue
+    for combo in product(args, repeat=3):
         yield combo
 
 
-@pytest.mark.parametrize("combo", generate_size_from_jobs_sims_blocks(None, 1, 10))
-def test_size_from_jobs_sims_blocks_output(combo: tuple[int | None, ...]) -> None:
-    jobs, simulations, blocks, iterations_per_slot, slots, subpops = combo
-    job_sizes_by_batch_system = {}
-    for batch_system in ("aws", "local", "slurm"):
+@pytest.mark.parametrize("combo", generate_size_from_jobs_sims_blocks(1, 5, 10))
+def test_size_from_jobs_sims_blocks_output(combo: tuple[int, ...]) -> None:
+    jobs, simulations, blocks = combo
+    for inference_method in (None, ""):
         job_size = JobSize.size_from_jobs_sims_blocks(
-            jobs, simulations, blocks, iterations_per_slot, slots, subpops, batch_system
-        )
-        assert (
-            job_size.jobs == jobs
-            if jobs is not None
-            else isinstance(job_size.jobs, int) and job_size.jobs > 0
-        )
-        assert (
-            job_size.simulations == simulations
-            if simulations is not None
-            else isinstance(job_size.simulations, int) and job_size.simulations > 0
-        )
-        assert (
-            job_size.blocks == blocks
-            if blocks is not None
-            else isinstance(job_size.blocks, int) and job_size.blocks > 0
+            jobs, simulations, blocks, inference_method
         )
-        job_sizes_by_batch_system[batch_system] = job_size
-    assert job_sizes_by_batch_system["local"] == job_sizes_by_batch_system["slurm"]
-    assert job_sizes_by_batch_system["local"].jobs == job_sizes_by_batch_system["aws"].jobs
+        assert job_size.jobs == jobs
+        assert job_size.simulations >= simulations
+        assert job_size.blocks >= 1

From 770d6f6ccfa40bc42671569ccac7eb0d08542af5 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Thu, 7 Nov 2024 17:28:16 -0500
Subject: [PATCH 38/78] Add `JobResources` class

Added a class to represent job resources required/requested for a batch
job. Includes documentation and some unit tests, but not enough.
Incorporated into `_click_batch`.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py     | 103 ++++++++++++++++--
 .../tests/batch/test_job_resources_class.py   |  31 ++++++
 2 files changed, 122 insertions(+), 12 deletions(-)
 create mode 100644 flepimop/gempyor_pkg/tests/batch/test_job_resources_class.py

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index ae3137964..244e734cb 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -5,7 +5,7 @@
 metadata and job size calculations for example.
 """
 
-__all__ = ["BatchSystem", "JobSize", "JobTimeLimit", "write_manifest"]
+__all__ = ["BatchSystem", "JobSize", "JobResources", "JobTimeLimit", "write_manifest"]
 
 
 from dataclasses import dataclass
@@ -150,14 +150,90 @@ def size_from_jobs_sims_blocks(
         Returns:
             A job size instance with either the explicit or inferred job sizing.
         """
-        inference_method = (
-            inference_method if inference_method is None else inference_method.lower()
-        )
         if inference_method == "emcee":
             return cls(jobs=jobs, simulations=blocks * simulations, blocks=1)
         return cls(jobs=jobs, simulations=simulations, blocks=blocks)
 
 
+@dataclass(frozen=True, slots=True)
+class JobResources:
+    """
+    A batch submission job resources request.
+
+    Attributes:
+        nodes: The number of nodes to request.
+        cpus: The number of CPUs to request per a node.
+        memory: The amount of memory to request per a node in MB.
+
+    Raises:
+        ValueError: If any of the attributes are less than 1.
+    """
+
+    nodes: int
+    cpus: int
+    memory: int
+
+    def __post_init__(self) -> None:
+        for p in self.__slots__:
+            if (val := getattr(self, p)) < 1:
+                raise ValueError(
+                    (
+                        f"The '{p}' attribute must be greater than 0, "
+                        f"but instead was given '{val}'."
+                    )
+                )
+
+    @classmethod
+    def from_presets(
+        cls, job_size: JobSize, inference_method: Literal["emcee"] | None
+    ) -> "JobResources":
+        """
+        Calculate suggested job resources from presets.
+
+        Args:
+            job_size: The size of the job being ran.
+            inference_method: The inference method being used for this job.
+
+        Returns:
+            A job resources instances scaled to the job size given.
+        """
+        if inference_method == "emcee":
+            return cls(
+                nodes=1, cpus=2 * job_size.jobs, memory=2 * 1024 * job_size.simulations
+            )
+        return cls(nodes=job_size.jobs, cpus=2, memory=2 * 1024)
+
+    @property
+    def total_cpus(self) -> int:
+        """
+        Calculate the total number of CPUs.
+
+        Returns:
+            The total number of CPUs represented by this instance.
+        """
+        return self.nodes * self.cpus
+
+    @property
+    def total_memory(self) -> int:
+        """
+        Calculate the total amount of memory.
+
+        Returns:
+            The total amount of memory represented by this instance.
+        """
+        return self.nodes * self.memory
+
+    def total_resources(self) -> tuple[int, int, int]:
+        """
+        Calculate the total resources.
+
+        Returns:
+            A tuple of the nodes, total CPUs, and total memory represented by
+            this instance.
+        """
+        return (self.nodes, self.total_cpus, self.total_memory)
+
+
 @dataclass(frozen=True, slots=True)
 class JobTimeLimit:
     """
@@ -261,8 +337,6 @@ def from_per_simulation_time(
         Raises:
             ValueError: If `time_per_simulation` is non-positive.
             ValueError: If `initial_time` is non-positive.
-
-        Examples:
         """
         if (total_seconds := time_per_simulation.total_seconds()) <= 0.0:
             raise ValueError(
@@ -672,6 +746,9 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
             "The `flepimop batch` CLI only supports EMCEE inference jobs."
         )
     inference_method = cfg["inference"]["method"].as_str()
+    inference_method = (
+        inference_method if inference_method is None else inference_method.lower()
+    )
 
     # Job name/run id
     name = cfg["name"].get(str) if cfg["name"].exists() else None
@@ -719,6 +796,9 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
     )
     logger.info("Setting a total job time limit of %s minutes", job_time_limit.format())
 
+    # Job resources
+    job_resources = JobResources.from_presets(job_size, inference_method)
+
     # Cluster info
     cluster: Cluster | None = None
     if batch_system == BatchSystem.SLURM:
@@ -756,7 +836,7 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
         "flepi_path": kwargs["flepi_path"].absolute(),
         "job_name": job_name,
         "jobs": job_size.jobs,
-        "nslots": kwargs["nslots"],
+        "nslots": job_size.simulations,
         "nthin": kwargs["nthin"],
         "prefix": kwargs["prefix"],
         "project_path": kwargs["project_path"].absolute(),
@@ -766,12 +846,11 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
     options = {
         "chdir": kwargs["project_path"].absolute(),
         "comment": f"Generated on {now:%c %Z} and submitted by {getuser()}.",
-        "cpus-per-task": job_size.jobs,
+        "cpus-per-task": job_resources.cpus,
         "job-name": job_name,
-        "mem": "100GB",
-        "nodes": 1,  # EMCEE can only run on one node for now.
-        "ntasks": 1,
-        "partition": "general",
+        "mem": job_resources.memory,
+        "nodes": job_resources.nodes,
+        "ntasks-per-node": 1,
         "time": job_time_limit.format("slurm"),
     }
     if kwargs["partition"] is not None:
diff --git a/flepimop/gempyor_pkg/tests/batch/test_job_resources_class.py b/flepimop/gempyor_pkg/tests/batch/test_job_resources_class.py
new file mode 100644
index 000000000..6bb8b8551
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/batch/test_job_resources_class.py
@@ -0,0 +1,31 @@
+from typing import Literal
+
+import pytest
+
+from gempyor.batch import JobResources
+
+
+@pytest.mark.parametrize(
+    "kwargs",
+    (
+        {"nodes": 0, "cpus": 1, "memory": 1},
+        {"nodes": 1, "cpus": 0, "memory": 1},
+        {"nodes": 1, "cpus": 1, "memory": 0},
+        {"nodes": 0, "cpus": 0, "memory": 1},
+        {"nodes": 1, "cpus": 0, "memory": 0},
+        {"nodes": 0, "cpus": 1, "memory": 0},
+        {"nodes": 0, "cpus": 0, "memory": 0},
+    ),
+)
+def test_less_than_one_value_error(
+    kwargs: dict[Literal["nodes", "cpus", "memory"], int]
+) -> None:
+    param = next(k for k, v in kwargs.items() if v < 1)
+    with pytest.raises(
+        ValueError,
+        match=(
+            f"^The '{param}' attribute must be greater than 0, "
+            f"but instead was given '{kwargs.get(param)}'.$"
+        ),
+    ):
+        JobResources(**kwargs)

From e2f3e13ebcaee4e42db08e47d1ba01214197fe85 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Fri, 8 Nov 2024 08:01:28 -0500
Subject: [PATCH 39/78] Further unit tests of `JobResources`

More complete set of unit tests for the `JobResources` class. Also
incorporated the formatting methods of that class into `_click_batch`.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py     | 26 ++++++---
 .../tests/batch/test_job_resources_class.py   | 54 ++++++++++++++++++-
 2 files changed, 72 insertions(+), 8 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 244e734cb..b74bb8405 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -233,6 +233,17 @@ def total_resources(self) -> tuple[int, int, int]:
         """
         return (self.nodes, self.total_cpus, self.total_memory)
 
+    def format_nodes(self, batch_system: BatchSystem | None) -> str:
+        return str(self.nodes)
+
+    def format_cpus(self, batch_system: BatchSystem | None) -> str:
+        return str(self.cpus)
+
+    def format_memory(self, batch_system: BatchSystem | None) -> str:
+        if batch_system == BatchSystem.SLURM:
+            return f"{self.memory}MB"
+        return str(self.memory)
+
 
 @dataclass(frozen=True, slots=True)
 class JobTimeLimit:
@@ -290,7 +301,7 @@ def __gt__(self, other: Self | timedelta) -> bool:
     def __ge__(self, other: Self | timedelta) -> bool:
         return self.__eq__(other) or self.__gt__(other)
 
-    def format(self, batch_system: Literal["aws", "local", "slurm"] | None = None) -> str:
+    def format(self, batch_system: BatchSystem | None = None) -> str:
         """
         Format the job time limit as a string appropriate for a given batch system.
 
@@ -301,16 +312,17 @@ def format(self, batch_system: Literal["aws", "local", "slurm"] | None = None) -
             The time limit formatted for the batch system.
 
         Examples:
+            >>> from gempyor.batch import BatchSystem
             >>> from datetime import timedelta
             >>> job_time_limit = JobTimeLimit(
             ...     time_limit=timedelta(days=1, hours=2, minutes=34, seconds=5)
             ... )
             >>> job_time_limit.format()
             '1595'
-            >>> job_time_limit.format(batch_system="slurm")
+            >>> job_time_limit.format(batch_system=BatchSystem.SLURM)
             '26:34:05'
         """
-        if batch_system == "slurm":
+        if batch_system == BatchSystem.SLURM:
             total_seconds = self.time_limit.total_seconds()
             hours = math.floor(total_seconds / (60.0 * 60.0))
             minutes = math.floor((total_seconds - (60.0 * 60.0 * hours)) / 60.0)
@@ -846,12 +858,12 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
     options = {
         "chdir": kwargs["project_path"].absolute(),
         "comment": f"Generated on {now:%c %Z} and submitted by {getuser()}.",
-        "cpus-per-task": job_resources.cpus,
+        "cpus-per-task": job_resources.format_cpus(batch_system),
         "job-name": job_name,
-        "mem": job_resources.memory,
-        "nodes": job_resources.nodes,
+        "mem": job_resources.format_memory(batch_system),
+        "nodes": job_resources.format_nodes(batch_system),
         "ntasks-per-node": 1,
-        "time": job_time_limit.format("slurm"),
+        "time": job_time_limit.format(batch_system),
     }
     if kwargs["partition"] is not None:
         options["partition"] = kwargs["partition"]
diff --git a/flepimop/gempyor_pkg/tests/batch/test_job_resources_class.py b/flepimop/gempyor_pkg/tests/batch/test_job_resources_class.py
index 6bb8b8551..3b7d6b6fb 100644
--- a/flepimop/gempyor_pkg/tests/batch/test_job_resources_class.py
+++ b/flepimop/gempyor_pkg/tests/batch/test_job_resources_class.py
@@ -2,7 +2,7 @@
 
 import pytest
 
-from gempyor.batch import JobResources
+from gempyor.batch import BatchSystem, JobResources, JobSize
 
 
 @pytest.mark.parametrize(
@@ -29,3 +29,55 @@ def test_less_than_one_value_error(
         ),
     ):
         JobResources(**kwargs)
+
+
+@pytest.mark.parametrize("nodes", (1, 2, 4, 8))
+@pytest.mark.parametrize("cpus", (1, 2, 4, 8))
+@pytest.mark.parametrize("memory", (1024, 2 * 1024, 4 * 1024, 8 * 1024))
+def test_instance_attributes(nodes: int, cpus: int, memory: int) -> None:
+    job_resources = JobResources(nodes=nodes, cpus=cpus, memory=memory)
+    assert job_resources.total_cpus >= cpus
+    assert job_resources.total_memory >= memory
+    assert job_resources.total_resources() == (nodes, nodes * cpus, nodes * memory)
+
+
+@pytest.mark.parametrize("nodes", (1, 2, 4, 8))
+@pytest.mark.parametrize("cpus", (1, 2, 4, 8))
+@pytest.mark.parametrize("memory", (1024, 2 * 1024, 4 * 1024, 8 * 1024))
+@pytest.mark.parametrize(
+    "batch_system", (BatchSystem.AWS, BatchSystem.LOCAL, BatchSystem.SLURM, None)
+)
+def test_formatting(
+    nodes: int, cpus: int, memory: int, batch_system: BatchSystem | None
+) -> None:
+    job_resources = JobResources(nodes=nodes, cpus=cpus, memory=memory)
+
+    formatted_nodes = job_resources.format_nodes(batch_system)
+    assert isinstance(formatted_nodes, str)
+    assert str(nodes) in formatted_nodes
+
+    formatted_cpus = job_resources.format_cpus(batch_system)
+    assert isinstance(formatted_cpus, str)
+    assert str(cpus) in formatted_cpus
+
+    formatted_memory = job_resources.format_memory(batch_system)
+    assert isinstance(formatted_memory, str)
+    assert str(memory) in formatted_memory
+
+
+@pytest.mark.parametrize("jobs", (1, 4, 16, 32))
+@pytest.mark.parametrize("simulations", (250, 4 * 250, 16 * 250, 32 * 250))
+@pytest.mark.parametrize("blocks", (1, 4, 16, 32))
+@pytest.mark.parametrize("inference_method", ("emcee", None))
+def test_from_presets_for_select_inputs(
+    jobs: int, simulations: int, blocks: int, inference_method: Literal["emcee"] | None
+) -> None:
+    job_size = JobSize(jobs=jobs, simulations=simulations, blocks=blocks)
+    job_resources = JobResources.from_presets(job_size, inference_method)
+    if inference_method == "emcee":
+        assert job_resources.nodes == 1
+        assert job_resources.cpus % 2 == 0
+        assert job_resources.memory % (2 * 1024) == 0
+    else:
+        assert job_resources.cpus == 2
+        assert job_resources.memory == 2 * 1024

From c710522abea72222165829eec3c226cbe1dce052 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Fri, 8 Nov 2024 08:14:28 -0500
Subject: [PATCH 40/78] Remove unneeded `warnings` import

---
 flepimop/gempyor_pkg/src/gempyor/batch.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index b74bb8405..1abcc02d9 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -21,7 +21,6 @@
 import sys
 from tempfile import mkstemp
 from typing import Any, Literal, Self
-import warnings
 
 import click
 

From fcc1dbcd4df01f43078b716a57954a8a8c4701b0 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Fri, 8 Nov 2024 12:03:10 -0500
Subject: [PATCH 41/78] Rename `flepimop batch` to `submit`

Also corrected bugs in unit tests of `JobTimeLimit` with new
`BatchSystem` enum.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py     |  8 ++++----
 ...__click_batch.py => test__click_submit.py} | 12 +++++------
 .../tests/batch/test_job_time_limit_class.py  | 20 +++++++++----------
 3 files changed, 20 insertions(+), 20 deletions(-)
 rename flepimop/gempyor_pkg/tests/batch/{test__click_batch.py => test__click_submit.py} (85%)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 1abcc02d9..2c12085ad 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -603,7 +603,7 @@ def _job_name(name: str | None, timestamp: datetime | None) -> str:
 
 
 @cli.command(
-    name="batch",
+    name="submit",
     params=[config_files_argument]
     + list(config_file_options.values())
     + [
@@ -739,7 +739,7 @@ def _job_name(name: str | None, timestamp: datetime | None) -> str:
     + list(verbosity_options.values()),
 )
 @click.pass_context
-def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
+def _click_submit(ctx: click.Context = mock_context, **kwargs) -> None:
     """Submit batch jobs"""
     # Generic setup
     now = datetime.now(timezone.utc)
@@ -754,7 +754,7 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
         or cfg["inference"]["method"].as_str() != "emcee"
     ):
         raise NotImplementedError(
-            "The `flepimop batch` CLI only supports EMCEE inference jobs."
+            "The `flepimop submit` CLI only supports EMCEE inference jobs."
         )
     inference_method = cfg["inference"]["method"].as_str()
     inference_method = (
@@ -776,7 +776,7 @@ def _click_batch(ctx: click.Context = mock_context, **kwargs) -> None:
     if batch_system != BatchSystem.SLURM:
         # Temporary limitation
         raise NotImplementedError(
-            "The `flepimop batch` CLI only supports batch submission to slurm."
+            "The `flepimop submit` CLI only supports batch submission to slurm."
         )
     logger.info("Constructing a job to submit to %s", batch_system)
     if batch_system != BatchSystem.SLURM and kwargs["email"] is not None:
diff --git a/flepimop/gempyor_pkg/tests/batch/test__click_batch.py b/flepimop/gempyor_pkg/tests/batch/test__click_submit.py
similarity index 85%
rename from flepimop/gempyor_pkg/tests/batch/test__click_batch.py
rename to flepimop/gempyor_pkg/tests/batch/test__click_submit.py
index f0c36dac2..abe4fda61 100644
--- a/flepimop/gempyor_pkg/tests/batch/test__click_batch.py
+++ b/flepimop/gempyor_pkg/tests/batch/test__click_submit.py
@@ -6,7 +6,7 @@
 import pytest
 import yaml
 
-from gempyor.batch import _click_batch
+from gempyor.batch import _click_submit
 
 
 @pytest.fixture
@@ -34,13 +34,13 @@ def test_only_inference_emcee_supported_not_implemented_error(
         yaml.dump(config, f)
 
     runner = CliRunner()
-    result = runner.invoke(_click_batch, [str(config_file.absolute())])
+    result = runner.invoke(_click_submit, [str(config_file.absolute())])
 
     assert result.exit_code == 1
     assert isinstance(result.exception, NotImplementedError)
     assert (
         str(result.exception)
-        == "The `flepimop batch` CLI only supports EMCEE inference jobs."
+        == "The `flepimop submit` CLI only supports EMCEE inference jobs."
     )
 
 
@@ -55,13 +55,13 @@ def test_only_slurm_batch_system_supported_not_implemented_error(
         yaml.dump({"inference": {"method": "emcee"}}, f)
 
     runner = CliRunner()
-    result = runner.invoke(_click_batch, args + [str(config_file.absolute())])
+    result = runner.invoke(_click_submit, args + [str(config_file.absolute())])
 
     assert result.exit_code == 1
     assert isinstance(result.exception, NotImplementedError)
     assert (
         str(result.exception)
-        == "The `flepimop batch` CLI only supports batch submission to slurm."
+        == "The `flepimop submit` CLI only supports batch submission to slurm."
     )
 
 
@@ -72,7 +72,7 @@ def test_cluster_required_for_slurm_value_error(tmp_path: Path) -> None:
 
     runner = CliRunner()
     result = runner.invoke(
-        _click_batch,
+        _click_submit,
         [
             "--slurm",
             "--simulations",
diff --git a/flepimop/gempyor_pkg/tests/batch/test_job_time_limit_class.py b/flepimop/gempyor_pkg/tests/batch/test_job_time_limit_class.py
index f51be9ef7..fddfb7ec3 100644
--- a/flepimop/gempyor_pkg/tests/batch/test_job_time_limit_class.py
+++ b/flepimop/gempyor_pkg/tests/batch/test_job_time_limit_class.py
@@ -4,7 +4,7 @@
 
 import pytest
 
-from gempyor.batch import JobSize, JobTimeLimit
+from gempyor.batch import BatchSystem, JobSize, JobTimeLimit
 
 
 NONPOSITIVE_TIMEDELTAS = (timedelta(), timedelta(hours=-1.0), timedelta(days=-3.0))
@@ -31,14 +31,16 @@ def test_time_limit_non_positive_value_error(time_limit: timedelta) -> None:
         timedelta(minutes=12345),
     ),
 )
-@pytest.mark.parametrize("batch_system", (None, "aws", "local", "slurm"))
+@pytest.mark.parametrize(
+    "batch_system", (None, BatchSystem.AWS, BatchSystem.LOCAL, BatchSystem.SLURM)
+)
 def test_format_output_validation(
-    time_limit: timedelta, batch_system: Literal["aws", "local", "slurm"] | None
+    time_limit: timedelta, batch_system: BatchSystem | None
 ) -> None:
     job_time_limit = JobTimeLimit(time_limit=time_limit)
     formatted_time_limit = job_time_limit.format(batch_system=batch_system)
     assert isinstance(formatted_time_limit, str)
-    if batch_system == "slurm":
+    if batch_system == BatchSystem.SLURM:
         assert re.match(r"^[0-9]+\:[0-9]{2}\:[0-9]{2}$", formatted_time_limit)
     else:
         assert formatted_time_limit.isdigit()
@@ -50,15 +52,13 @@ def test_format_output_validation(
         (timedelta(hours=1), None, "60"),
         (timedelta(seconds=20), None, "1"),
         (timedelta(days=2, hours=3, minutes=45), None, "3105"),
-        (timedelta(hours=1), "slurm", "1:00:00"),
-        (timedelta(seconds=20), "slurm", "0:00:20"),
-        (timedelta(days=1, hours=2, minutes=34, seconds=5), "slurm", "26:34:05"),
+        (timedelta(hours=1), BatchSystem.SLURM, "1:00:00"),
+        (timedelta(seconds=20), BatchSystem.SLURM, "0:00:20"),
+        (timedelta(days=1, hours=2, minutes=34, seconds=5), BatchSystem.SLURM, "26:34:05"),
     ),
 )
 def test_format_exact_results(
-    time_limit: timedelta,
-    batch_system: Literal["aws", "local", "slurm"] | None,
-    expected: str,
+    time_limit: timedelta, batch_system: BatchSystem | None, expected: str
 ) -> None:
     job_time_limit = JobTimeLimit(time_limit=time_limit)
     assert job_time_limit.format(batch_system=batch_system) == expected

From a893b0039ef6391a530d97a4ad5d9f572f1b9d72 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Fri, 8 Nov 2024 12:54:49 -0500
Subject: [PATCH 42/78] Dump config to actual rather than temp file

The `/tmp` directory is not network attached so not available
everywhere. Instead write to the project directory with the job name.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 2c12085ad..37d610d64 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -19,7 +19,6 @@
 from shlex import quote
 import subprocess
 import sys
-from tempfile import mkstemp
 from typing import Any, Literal, Self
 
 import click
@@ -830,8 +829,7 @@ def _click_submit(ctx: click.Context = mock_context, **kwargs) -> None:
 
     # Config out
     if (config_out := kwargs["config_out"]) is None:
-        _, tmp = mkstemp(suffix=".yml", prefix="config", text=True)
-        config_out = Path(tmp)
+        config_out = kwargs["project_path"] / f"config_{job_name}.yml"
     with config_out.open(mode="w") as f:
         f.write(cfg.dump())
     logger.info(

From d5eab61f5ea14b15771f56e1240cfc11b88d417b Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Fri, 8 Nov 2024 14:53:21 -0500
Subject: [PATCH 43/78] Remove `--nthin` from `flepimop submit`

---
 flepimop/gempyor_pkg/src/gempyor/batch.py                  | 7 -------
 .../src/gempyor/templates/emcee_inference.sbatch.j2        | 1 -
 2 files changed, 8 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 37d610d64..e2cdd4aa7 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -708,12 +708,6 @@ def _job_name(name: str | None, timestamp: datetime | None) -> str:
                 "If not provided a temporary file will be used."
             ),
         ),
-        click.Option(
-            param_decls=["--nthin", "nthin"],
-            type=click.IntRange(min=5),
-            default=5,
-            help="The number of samples to thin.",
-        ),
         click.Option(
             param_decls=["--id", "--run-id", "run_id"],
             envvar="FLEPI_RUN_INDEX",
@@ -846,7 +840,6 @@ def _click_submit(ctx: click.Context = mock_context, **kwargs) -> None:
         "job_name": job_name,
         "jobs": job_size.jobs,
         "nslots": job_size.simulations,
-        "nthin": kwargs["nthin"],
         "prefix": kwargs["prefix"],
         "project_path": kwargs["project_path"].absolute(),
         "run_id": kwargs["run_id"],
diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2 b/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2
index c707c73db..d171aff17 100644
--- a/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2
+++ b/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2
@@ -56,7 +56,6 @@ flepimop-calibrate --config $CONFIG_PATH \
     --project_path $PROJECT_PATH \
     --nslots {{ nslots }} \
     --niterations {{ simulations }} \
-    --nthin {{ nthin }} \
     --jobs {{ jobs }} \
     --id {{ run_id }} \
 {%- if prefix is defined and prefix is not none %}

From 1316ebefd5409306a39e20bc87ea58fa57775239 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Fri, 8 Nov 2024 16:08:58 -0500
Subject: [PATCH 44/78] Optional overrides to job resource presets

---
 flepimop/gempyor_pkg/src/gempyor/batch.py     | 52 ++++++++++++++++---
 .../tests/batch/test_job_resources_class.py   | 16 ++++++
 2 files changed, 60 insertions(+), 8 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index e2cdd4aa7..9b6742f61 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -183,23 +183,35 @@ def __post_init__(self) -> None:
 
     @classmethod
     def from_presets(
-        cls, job_size: JobSize, inference_method: Literal["emcee"] | None
+        cls,
+        job_size: JobSize,
+        inference_method: Literal["emcee"] | None,
+        nodes: int | None = None,
+        cpus: int | None = None,
+        memory: int | None = None,
     ) -> "JobResources":
         """
-        Calculate suggested job resources from presets.
+        Calculate suggested job resources from presets with optional overrides.
 
         Args:
             job_size: The size of the job being ran.
             inference_method: The inference method being used for this job.
+            nodes: Optional manual override for the number of nodes.
+            cpus: Optional manual override for the number of CPUs per node.
+            memory: Optional manual override for the amount of memory per node.
 
         Returns:
             A job resources instances scaled to the job size given.
         """
         if inference_method == "emcee":
-            return cls(
-                nodes=1, cpus=2 * job_size.jobs, memory=2 * 1024 * job_size.simulations
-            )
-        return cls(nodes=job_size.jobs, cpus=2, memory=2 * 1024)
+            nodes = 1 if nodes is None else nodes
+            cpus = 2 * job_size.jobs if cpus is None else cpus
+            memory = 2 * 1024 * job_size.simulations if memory is None else memory
+        else:
+            nodes = job_size.jobs if nodes is None else nodes
+            cpus = 2 if cpus is None else cpus
+            memory = 2 * 1024 if memory is None else memory
+        return cls(nodes=nodes, cpus=cpus, memory=memory)
 
     @property
     def total_cpus(self) -> int:
@@ -728,6 +740,24 @@ def _job_name(name: str | None, timestamp: datetime | None) -> str:
             default=None,
             help="Optionally an email that can be notified on job begin and end.",
         ),
+        click.Option(
+            param_decls=["--nodes", "nodes"],
+            type=click.IntRange(min=1),
+            default=None,
+            help="Override for the number of nodes to use.",
+        ),
+        click.Option(
+            param_decls=["--cpus", "cpus"],
+            type=click.IntRange(min=1),
+            default=None,
+            help="Override for the number of CPUs per node to use.",
+        ),
+        click.Option(
+            param_decls=["--memory", "memory"],
+            type=click.IntRange(min=1),
+            default=None,
+            help="Override for the amount of memory per node to use in MB.",
+        ),
     ]
     + list(verbosity_options.values()),
 )
@@ -801,7 +831,13 @@ def _click_submit(ctx: click.Context = mock_context, **kwargs) -> None:
     logger.info("Setting a total job time limit of %s minutes", job_time_limit.format())
 
     # Job resources
-    job_resources = JobResources.from_presets(job_size, inference_method)
+    job_resources = JobResources.from_presets(
+        job_size,
+        inference_method,
+        nodes=kwargs["nodes"],
+        cpus=kwargs["cpus"],
+        memory=kwargs["memory"],
+    )
 
     # Cluster info
     cluster: Cluster | None = None
@@ -839,7 +875,7 @@ def _click_submit(ctx: click.Context = mock_context, **kwargs) -> None:
         "flepi_path": kwargs["flepi_path"].absolute(),
         "job_name": job_name,
         "jobs": job_size.jobs,
-        "nslots": job_size.simulations,
+        "nslots": job_size.simulations,  # aka nwalkers
         "prefix": kwargs["prefix"],
         "project_path": kwargs["project_path"].absolute(),
         "run_id": kwargs["run_id"],
diff --git a/flepimop/gempyor_pkg/tests/batch/test_job_resources_class.py b/flepimop/gempyor_pkg/tests/batch/test_job_resources_class.py
index 3b7d6b6fb..c5abeb040 100644
--- a/flepimop/gempyor_pkg/tests/batch/test_job_resources_class.py
+++ b/flepimop/gempyor_pkg/tests/batch/test_job_resources_class.py
@@ -81,3 +81,19 @@ def test_from_presets_for_select_inputs(
     else:
         assert job_resources.cpus == 2
         assert job_resources.memory == 2 * 1024
+
+
+@pytest.mark.parametrize("inference_method", ("emcee", None))
+@pytest.mark.parametrize("nodes", (1, 2, 4, 8))
+@pytest.mark.parametrize("cpus", (1, 2, 4, 8))
+@pytest.mark.parametrize("memory", (1024, 2 * 1024, 4 * 1024, 8 * 1024))
+def test_from_presets_overrides(
+    inference_method: Literal["emcee"] | None, nodes: int, cpus: int, memory: int
+) -> None:
+    job_size = JobSize(jobs=1, simulations=1, blocks=1)
+    job_resources = JobResources.from_presets(
+        job_size, inference_method, nodes=nodes, cpus=cpus, memory=memory
+    )
+    assert job_resources.nodes == nodes
+    assert job_resources.cpus == cpus
+    assert job_resources.memory == memory

From d4a86a984b1b3c5141d6b2348fa62cf17c4a7d77 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Mon, 11 Nov 2024 13:26:53 -0500
Subject: [PATCH 45/78] Initial pass at incorporating R inference

Untested, but put the initial command in with an if. Still blocked by a
`NotImplementedError` in `_click_submit`.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py     |  3 ++-
 ...nference.sbatch.j2 => inference.sbatch.j2} | 22 +++++++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)
 rename flepimop/gempyor_pkg/src/gempyor/templates/{emcee_inference.sbatch.j2 => inference.sbatch.j2} (67%)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 9b6742f61..157f38298 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -873,6 +873,7 @@ def _click_submit(ctx: click.Context = mock_context, **kwargs) -> None:
         "cluster": cluster if cluster is None else cluster.model_dump(),
         "debug": kwargs["debug"],
         "flepi_path": kwargs["flepi_path"].absolute(),
+        "inference_method": "" if inference_method is None else inference_method,
         "job_name": job_name,
         "jobs": job_size.jobs,
         "nslots": job_size.simulations,  # aka nwalkers
@@ -897,7 +898,7 @@ def _click_submit(ctx: click.Context = mock_context, **kwargs) -> None:
         options["mail-type"] = "BEGIN,END"
         options["mail-user"] = kwargs["email"]
     _sbatch_template(
-        "emcee_inference.sbatch.j2",
+        "inference.sbatch.j2",
         None,
         template_data,
         "all",
diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2 b/flepimop/gempyor_pkg/src/gempyor/templates/inference.sbatch.j2
similarity index 67%
rename from flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2
rename to flepimop/gempyor_pkg/src/gempyor/templates/inference.sbatch.j2
index d171aff17..4b30a187c 100644
--- a/flepimop/gempyor_pkg/src/gempyor/templates/emcee_inference.sbatch.j2
+++ b/flepimop/gempyor_pkg/src/gempyor/templates/inference.sbatch.j2
@@ -46,12 +46,15 @@ fi
 
 # Load conda env
 conda activate {{ conda_env }}
+WHICH_PYTHON=$( which python )
+WHICH_RSCRIPT=$( which Rscript )
 
 # AWS setup
 
 # Fetch/setup resume files
 
 # Run inference
+{%- if inference_method == 'emcee' %}
 flepimop-calibrate --config $CONFIG_PATH \
     --project_path $PROJECT_PATH \
     --nslots {{ nslots }} \
@@ -62,5 +65,24 @@ flepimop-calibrate --config $CONFIG_PATH \
     --prefix {{ prefix}} \
 {%- endif %}
     > out_{{ job_name }}_$FLEPI_SLOT_INDEX.out 2>&1
+{%- else %}
+flepimop-inference-slot --config $CONFIG_PATH \
+    --run_id {{ run_id }} \
+    --seir_modifiers_scenarios $FLEPI_SEIR_SCENARIOS \
+    --outcome_modifiers_scenarios $FLEPI_OUTCOME_SCENARIOS \
+    --jobs 1 \
+    --iterations_per_slot {{ simulations }} \
+    --slots {{ nslots }} \
+    --this_slot $FLEPI_SLOT_INDEX \
+    --this_block 1 \
+    --stoch_traj_flag $FLEPI_STOCHASTIC_RUN \
+    --flepi_path $FLEPI_PATH \
+    --python $WHICH_PYTHON \
+    --rpath $WHICH_RSCRIPT \
+    --is-resume $RESUME_RUN \
+    --is-interactive FALSE \
+    --reset_chimeric_on_accept $FLEPI_RESET_CHIMERICS \
+    > out_{{ job_name }}_$FLEPI_SLOT_INDEX.out 2>&1
+{%- endif %}
 
 # Upload/move results

From 82dfe8fc879a40a41dc01fc43127f89bc4b6a4b6 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Mon, 11 Nov 2024 14:14:27 -0500
Subject: [PATCH 46/78] Clean whitespace in `_sbatch` logging

---
 flepimop/gempyor_pkg/src/gempyor/batch.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 157f38298..13e8c0a78 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -519,15 +519,13 @@ def _sbatch(
     if verbosity is not None:
         logger.info("Submitting to slurm with: %s", " ".join(cmd_args))
     process = subprocess.run(cmd_args, check=True, capture_output=True)
+    stdout = process.stdout.decode().strip()
+    stderr = process.stderr.decode().strip()
     if verbosity is not None:
-        if process.stdout:
-            logger.debug(
-                "Captured stdout from sbatch submission: %s", process.stdout.decode()
-            )
-        if process.stderr:
-            logger.error(
-                "Captured stderr from sbatch submission: %s", process.stderr.decode()
-            )
+        if stdout:
+            logger.debug("Captured stdout from sbatch submission: %s", stdout)
+        if stderr:
+            logger.error("Captured stderr from sbatch submission: %s", stderr)
 
 
 def _sbatch_template(

From 3d5484bad54c88d77ab728074c838b9e7b6c8804 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Mon, 11 Nov 2024 16:41:22 -0500
Subject: [PATCH 47/78] Add `_local` internal utility

Added the local analog to `_sbatch` for slurm. Along with corresponding
tests and documentation. Sets the groundwork for the `--local` flag for
`_click_submit` for ease of (integration?) testing.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py     | 49 ++++++++++++
 .../gempyor_pkg/tests/batch/test__local.py    | 76 +++++++++++++++++++
 2 files changed, 125 insertions(+)
 create mode 100644 flepimop/gempyor_pkg/tests/batch/test__local.py

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 13e8c0a78..b67c0227e 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -17,6 +17,7 @@
 from pathlib import Path
 import re
 from shlex import quote
+from stat import S_IXUSR
 import subprocess
 import sys
 from typing import Any, Literal, Self
@@ -444,6 +445,54 @@ def write_manifest(
     return destination
 
 
+def _local(script: Path, verbosity: int | None, dry_run: bool) -> None:
+    """
+    Execute a local script.
+
+    Args:
+        script: The script to run.
+        verbosity: A integer verbosity level to enable logging or `None` for no logging.
+        dry_run: A boolean indicating if this is a dry run or not, if set to `True` this
+            function will not actually execute the script given.
+    """
+    if verbosity is not None:
+        logger = get_script_logger(__name__, verbosity)
+        logger.debug("Using script '%s' for local execution", script.absolute())
+
+    if not script.exists() or not script.is_file():
+        raise ValueError(
+            f"The script '{script.absolute()}' either does not exist or is not a file."
+        )
+    if not bool((current_perms := script.stat().st_mode) & S_IXUSR):
+        if verbosity is not None:
+            logger.warning(
+                "The script '%s' is not executable, making it executable.",
+                script.absolute(),
+            )
+        new_perms = current_perms | S_IXUSR
+        script.chmod(new_perms)
+    cmd_args = [str(script.absolute())]
+
+    if dry_run:
+        if verbosity is not None:
+            logger.info(
+                "If not dry mode would have executed script with: %s",
+                " ".join(cmd_args),
+            )
+        return
+
+    if verbosity is not None:
+        logger.info("Executing script with: %s", " ".join(cmd_args))
+    process = subprocess.run(cmd_args, check=True, capture_output=True)
+    stdout = process.stdout.decode().strip()
+    stderr = process.stderr.decode().strip()
+    if verbosity is not None:
+        if stdout:
+            logger.debug("Captured stdout from executed script: %s", stdout)
+        if stderr:
+            logger.error("Captured stderr from executed script: %s", stderr)
+
+
 def _sbatch(
     script: Path,
     environment_variables: dict[str, Any] | Literal["all", "nil", "none"],
diff --git a/flepimop/gempyor_pkg/tests/batch/test__local.py b/flepimop/gempyor_pkg/tests/batch/test__local.py
new file mode 100644
index 000000000..4e034af51
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/batch/test__local.py
@@ -0,0 +1,76 @@
+import logging
+from pathlib import Path
+from unittest.mock import patch
+from stat import S_IXUSR
+import subprocess
+
+import pytest
+
+from gempyor.batch import _local
+
+
+def sample_script(directory: Path, executable: bool) -> Path:
+    script = directory / "example"
+    script.write_text("#!/usr/bin/env bash\necho 'Hello local!'")
+    if executable:
+        script.chmod(script.stat().st_mode | S_IXUSR)
+    return script
+
+
+def test_script_does_not_exist_or_is_not_file_value_error(tmp_path: Path) -> None:
+    script = tmp_path / "does_not_exist"
+    with pytest.raises(
+        ValueError,
+        match=f"^The script '{script.absolute()}' either does not exist or is not a file.$",
+    ):
+        _local(script, None, True)
+
+    directory = tmp_path / "not_a_file"
+    directory.mkdir(parents=True, exist_ok=True)
+    with pytest.raises(
+        ValueError,
+        match=(
+            f"^The script '{directory.absolute()}' either does not exist or is not a file.$"
+        ),
+    ):
+        _local(directory, None, True)
+
+
+@pytest.mark.parametrize("executable", (True, False))
+@pytest.mark.parametrize("verbosity", (None, logging.DEBUG, logging.INFO, logging.WARNING))
+@pytest.mark.parametrize("dry_run", (True, False))
+def test_output_validation(
+    caplog: pytest.LogCaptureFixture,
+    tmp_path: Path,
+    executable: bool,
+    verbosity: int,
+    dry_run: bool,
+) -> None:
+    script = sample_script(tmp_path, executable)
+
+    def subprocess_run_wraps(args, **kwargs):
+        if args[0] == str(script.absolute()):
+            return subprocess.CompletedProcess(
+                args=args, returncode=0, stdout=b"Hello local\n", stderr=b""
+            )
+        return subprocess.run(args, **kwargs)
+
+    with patch(
+        "gempyor.batch.subprocess.run", wraps=subprocess_run_wraps
+    ) as subprocess_run_patch:
+        assert _local(script, verbosity, dry_run) is None
+
+        if dry_run:
+            subprocess_run_patch.assert_not_called()
+        else:
+            subprocess_run_patch.assert_called_once()
+
+        log_messages_by_level = {
+            hash((logging.DEBUG, True)): 2,
+            hash((logging.DEBUG, False)): 3,
+            hash((logging.INFO, True)): 1,
+            hash((logging.INFO, False)): 1,
+        }
+        assert len(caplog.records) == log_messages_by_level.get(
+            hash((verbosity, dry_run)), 0
+        ) + (1 if not executable and verbosity is not None else 0)

From a2fe5dfd3bb352ad4476b05d940ad5d0f022a007 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Mon, 11 Nov 2024 16:44:27 -0500
Subject: [PATCH 48/78] Splitting out templates

Once again moved the inference template to clarify its the slurm
inference template. Started drafting a local inference script. Broke out
the conda environment activiation into its own template to be used by
both.
---
 .../gempyor_pkg/src/gempyor/templates/conda_activate.bash.j2 | 4 ++++
 .../src/gempyor/templates/inference_local.bash.j2            | 3 +++
 .../{inference.sbatch.j2 => inference_slurm.sbatch.j2}       | 5 +----
 3 files changed, 8 insertions(+), 4 deletions(-)
 create mode 100644 flepimop/gempyor_pkg/src/gempyor/templates/conda_activate.bash.j2
 create mode 100644 flepimop/gempyor_pkg/src/gempyor/templates/inference_local.bash.j2
 rename flepimop/gempyor_pkg/src/gempyor/templates/{inference.sbatch.j2 => inference_slurm.sbatch.j2} (95%)

diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/conda_activate.bash.j2 b/flepimop/gempyor_pkg/src/gempyor/templates/conda_activate.bash.j2
new file mode 100644
index 000000000..0de429020
--- /dev/null
+++ b/flepimop/gempyor_pkg/src/gempyor/templates/conda_activate.bash.j2
@@ -0,0 +1,4 @@
+# Load conda env
+conda activate {{ conda_env }}
+WHICH_PYTHON=$( which python )
+WHICH_RSCRIPT=$( which Rscript )
\ No newline at end of file
diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/inference_local.bash.j2 b/flepimop/gempyor_pkg/src/gempyor/templates/inference_local.bash.j2
new file mode 100644
index 000000000..632a4db34
--- /dev/null
+++ b/flepimop/gempyor_pkg/src/gempyor/templates/inference_local.bash.j2
@@ -0,0 +1,3 @@
+#!/usr/bin/env {{ interpreter }}
+
+{% include "conda_activate.bash.j2" with context %}
diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/inference.sbatch.j2 b/flepimop/gempyor_pkg/src/gempyor/templates/inference_slurm.sbatch.j2
similarity index 95%
rename from flepimop/gempyor_pkg/src/gempyor/templates/inference.sbatch.j2
rename to flepimop/gempyor_pkg/src/gempyor/templates/inference_slurm.sbatch.j2
index 4b30a187c..fbcd83c92 100644
--- a/flepimop/gempyor_pkg/src/gempyor/templates/inference.sbatch.j2
+++ b/flepimop/gempyor_pkg/src/gempyor/templates/inference_slurm.sbatch.j2
@@ -44,10 +44,7 @@ fi
 {%- endfor %}
 {%- endif %}
 
-# Load conda env
-conda activate {{ conda_env }}
-WHICH_PYTHON=$( which python )
-WHICH_RSCRIPT=$( which Rscript )
+{% include "conda_activate.bash.j2" with context %}
 
 # AWS setup
 

From 641bc70ffa76b6157044432955010208d382c931 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Mon, 11 Nov 2024 17:39:50 -0500
Subject: [PATCH 49/78] Initial template testing

* Broke out cluster setup into its own include template.
* Added some unit testing for templates.
---
 .../gempyor/templates/cluster_setup.bash.j2   |  25 +++++
 .../templates/inference_slurm.sbatch.j2       |  26 +----
 .../templates/test_cluster_setup_bash.py      | 105 ++++++++++++++++++
 .../templates/test_conda_activate_bash.py     |  11 ++
 4 files changed, 142 insertions(+), 25 deletions(-)
 create mode 100644 flepimop/gempyor_pkg/src/gempyor/templates/cluster_setup.bash.j2
 create mode 100644 flepimop/gempyor_pkg/tests/templates/test_cluster_setup_bash.py
 create mode 100644 flepimop/gempyor_pkg/tests/templates/test_conda_activate_bash.py

diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/cluster_setup.bash.j2 b/flepimop/gempyor_pkg/src/gempyor/templates/cluster_setup.bash.j2
new file mode 100644
index 000000000..8d53fff5b
--- /dev/null
+++ b/flepimop/gempyor_pkg/src/gempyor/templates/cluster_setup.bash.j2
@@ -0,0 +1,25 @@
+# Purge/load modules
+module purge
+{%- for module in cluster.modules %}
+{%- if module.version is defined and module.version is not none %}
+module load {{ module.name }}/{{ module.version }}
+{%- else %}
+module load {{ module.name }}
+{%- endif %}
+{%- endfor %}
+{%- if cluster.path_exports|length %}
+
+# Path modifications
+{%- for path_export in cluster.path_exports %}
+if [ -r "{{ path_export.path }}" ]; then
+    {% if path_export.prepend -%}
+    export PATH={{ path_export.path }}:$PATH
+    {%- else -%}
+    export PATH=$PATH:{{ path_export.path }}
+    {%- endif %}
+elif [ "{{ path_export.error_if_missing }}" = "True" ]; then
+    echo "The path '{{ path_export.path }}' does not exist but is required."
+    exit 1
+fi
+{%- endfor %}
+{%- endif %}
\ No newline at end of file
diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/inference_slurm.sbatch.j2 b/flepimop/gempyor_pkg/src/gempyor/templates/inference_slurm.sbatch.j2
index fbcd83c92..211a876bc 100644
--- a/flepimop/gempyor_pkg/src/gempyor/templates/inference_slurm.sbatch.j2
+++ b/flepimop/gempyor_pkg/src/gempyor/templates/inference_slurm.sbatch.j2
@@ -18,31 +18,7 @@ export FLEPI_PATH={{ flepi_path }}
 export PROJECT_PATH={{ project_path }}
 export CONFIG_PATH={{ config_out }}
 
-# Purge/load modules
-module purge
-{%- for module in cluster.modules %}
-{%- if module.version is defined and module.version is not none %}
-module load {{ module.name }}/{{ module.version }}
-{%- else %}
-module load {{ module.name }}
-{%- endif %}
-{%- endfor %}
-
-{% if cluster.path_exports|length -%}
-# Path modifications
-{%- for path_export in cluster.path_exports %}
-if [ -r "{{ path_export.path }}" ]; then
-    {% if path_export.prepend -%}
-    export PATH={{ path_export.path }}:$PATH
-    {%- else -%}
-    export PATH=$PATH:{{ path_export.path }}
-    {%- endif %}
-elif [ "{{ path_export.error_if_missing }}" = "True" ]; then
-    echo "The path '{{ path_export.path }}' does not exist but is required."
-    exit 1
-fi
-{%- endfor %}
-{%- endif %}
+{% include "cluster_setup.bash.j2" with context %}
 
 {% include "conda_activate.bash.j2" with context %}
 
diff --git a/flepimop/gempyor_pkg/tests/templates/test_cluster_setup_bash.py b/flepimop/gempyor_pkg/tests/templates/test_cluster_setup_bash.py
new file mode 100644
index 000000000..89048444a
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/templates/test_cluster_setup_bash.py
@@ -0,0 +1,105 @@
+from typing import Any
+
+import pytest
+
+from gempyor._jinja import _render_template
+from gempyor.info import Cluster, Module, PathExport, get_cluster_info
+
+
+@pytest.mark.parametrize(
+    "cluster",
+    (
+        None,
+        get_cluster_info("longleaf").model_dump(),
+        get_cluster_info("rockfish").model_dump(),
+    ),
+)
+def test_output_validation(cluster: dict[str, Any]) -> None:
+    rendered_template = _render_template("cluster_setup.bash.j2", {"cluster": cluster})
+    lines = rendered_template.split("\n")
+    assert "module purge" in lines
+    assert len(lines) == 2 if cluster is None else len(lines) > 2
+
+
+@pytest.mark.parametrize(
+    ("cluster", "expected"),
+    (
+        (
+            Cluster(name="foobar").model_dump(),
+            [
+                "# Purge/load modules",
+                "module purge",
+            ],
+        ),
+        (
+            Cluster(name="foo", modules=[Module(name="fizz")]).model_dump(),
+            [
+                "# Purge/load modules",
+                "module purge",
+                "module load fizz",
+            ],
+        ),
+        (
+            Cluster(
+                name="bar", modules=[Module(name="buzz", version="12.34")]
+            ).model_dump(),
+            [
+                "# Purge/load modules",
+                "module purge",
+                "module load buzz/12.34",
+            ],
+        ),
+        (
+            Cluster(
+                name="fizz", path_exports=[PathExport(path="/path/to/custom/bin")]
+            ).model_dump(),
+            [
+                "# Purge/load modules",
+                "module purge",
+                "",
+                "# Path modifications",
+                'if [ -r "/path/to/custom/bin" ]; then',
+                "    export PATH=/path/to/custom/bin:$PATH",
+                'elif [ "False" = "True" ]; then',
+                "    echo \"The path '/path/to/custom/bin' does not exist but is required.\"",
+                "    exit 1",
+                "fi",
+            ],
+        ),
+        (
+            Cluster(
+                name="new_cluster",
+                modules=[Module(name="abc", version="123"), Module(name="def")],
+                path_exports=[
+                    PathExport(path="~/helpful/bin", prepend=False),
+                    PathExport(path="~/required/bin", error_if_missing=True),
+                ],
+            ).model_dump(),
+            [
+                "# Purge/load modules",
+                "module purge",
+                "module load abc/123",
+                "module load def",
+                "",
+                "# Path modifications",
+                'if [ -r "~/helpful/bin" ]; then',
+                "    export PATH=$PATH:~/helpful/bin",
+                'elif [ "False" = "True" ]; then',
+                "    echo \"The path '~/helpful/bin' does not exist but is required.\"",
+                "    exit 1",
+                "fi",
+                'if [ -r "~/required/bin" ]; then',
+                "    export PATH=~/required/bin:$PATH",
+                'elif [ "True" = "True" ]; then',
+                "    echo \"The path '~/required/bin' does not exist but is required.\"",
+                "    exit 1",
+                "fi",
+            ],
+        ),
+    ),
+)
+def test_exact_results_for_select_inputs(
+    cluster: dict[str, Any], expected: list[str]
+) -> None:
+    lines = _render_template("cluster_setup.bash.j2", {"cluster": cluster}).split("\n")
+    assert lines == expected
diff --git a/flepimop/gempyor_pkg/tests/templates/test_conda_activate_bash.py b/flepimop/gempyor_pkg/tests/templates/test_conda_activate_bash.py
new file mode 100644
index 000000000..329b294a0
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/templates/test_conda_activate_bash.py
@@ -0,0 +1,11 @@
+import pytest
+
+from gempyor._jinja import _render_template
+
+
+@pytest.mark.parametrize("conda_env", ("abc", "flepimop-env"))
+def test_output_validation(conda_env: str) -> None:
+    rendered_template = _render_template("conda_activate.bash.j2", {"conda_env": conda_env})
+    lines = rendered_template.split("\n")
+    assert len(lines) == 4
+    assert conda_env in lines[1]

From 4eecc0bfdad29f60bf527d37e458358d288fe143 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 12 Nov 2024 08:09:02 -0500
Subject: [PATCH 50/78] Add exact results for conda activate template

Unit test for exact results from the conda activate jinja template.
---
 .../templates/test_conda_activate_bash.py     | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/flepimop/gempyor_pkg/tests/templates/test_conda_activate_bash.py b/flepimop/gempyor_pkg/tests/templates/test_conda_activate_bash.py
index 329b294a0..daae1b4d2 100644
--- a/flepimop/gempyor_pkg/tests/templates/test_conda_activate_bash.py
+++ b/flepimop/gempyor_pkg/tests/templates/test_conda_activate_bash.py
@@ -9,3 +9,31 @@ def test_output_validation(conda_env: str) -> None:
     lines = rendered_template.split("\n")
     assert len(lines) == 4
     assert conda_env in lines[1]
+
+
+@pytest.mark.parametrize(
+    ("conda_env", "expected"),
+    (
+        (
+            "flepimop-env",
+            [
+                "# Load conda env",
+                "conda activate flepimop-env",
+                "WHICH_PYTHON=$( which python )",
+                "WHICH_RSCRIPT=$( which Rscript )",
+            ],
+        ),
+        (
+            "/path/to/conda/env",
+            [
+                "# Load conda env",
+                "conda activate /path/to/conda/env",
+                "WHICH_PYTHON=$( which python )",
+                "WHICH_RSCRIPT=$( which Rscript )",
+            ],
+        ),
+    ),
+)
+def test_exact_results_for_select_inputs(conda_env: str, expected: list[str]) -> None:
+    lines = _render_template("conda_activate.bash.j2", {"conda_env": conda_env}).split("\n")
+    assert lines == expected

From a84234286e90f2199adfd2c674ddb0c6a2a2e824 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 12 Nov 2024 08:42:39 -0500
Subject: [PATCH 51/78] Add `_local_template` utility

Complement to `_sbatch_template` for local execution. Includes
documentation, but no unit tests right now.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py | 33 +++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index b67c0227e..5d8c6030c 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -493,6 +493,39 @@ def _local(script: Path, verbosity: int | None, dry_run: bool) -> None:
             logger.error("Captured stderr from executed script: %s", stderr)
 
 
+def _local_template(
+    template: str,
+    script: Path | None,
+    template_data: dict[str, Any],
+    verbosity: int | None,
+    dry_run: bool,
+) -> Path:
+    """
+    Execute a local script generated from a template.
+
+    Args:
+        template: The name of the template to use for rendering the executable script.
+        script: Either the path of where to save the rendered executable script to or
+            `None` for a tmp file.
+        template_data: Data accessible to the template when rendering.
+        verbosity: A integer verbosity level to enable logging or `None` for no logging.
+        dry_run: A boolean indicating if this is a dry run or not, if set to `True` this
+            function will not actually submit a job to slurm.
+
+    Returns:
+        The rendered executable script, either a tmp file if `script` is None otherwise
+        `script` is returned.
+    """
+    if script is None:
+        script = _render_template_to_temp_file(template, template_data, suffix=".bash")
+        new_perms = script.stat().st_mode | S_IXUSR
+        script.chmod(new_perms)
+    else:
+        _render_template_to_file(template, template_data, script)
+    _local(script, verbosity, dry_run)
+    return script
+
+
 def _sbatch(
     script: Path,
     environment_variables: dict[str, Any] | Literal["all", "nil", "none"],

From 398c352f737ad7daedc0576e83d6e3758dc31c76 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 12 Nov 2024 09:43:41 -0500
Subject: [PATCH 52/78] Incorporate batch system into `JobSize`

The `JobSize.from_jobs_simulations_blocks` now considers batch system,
namely to limit the size of the job when running locally since local
inference is meant for testing.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py     |  8 +++-
 .../tests/batch/test_job_size_class.py        | 44 ++++++++++++++-----
 2 files changed, 40 insertions(+), 12 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 5d8c6030c..bdedf071d 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -129,12 +129,13 @@ def __post_init__(self) -> None:
                 )
 
     @classmethod
-    def size_from_jobs_sims_blocks(
+    def from_jobs_simulations_blocks(
         cls,
         jobs: int | None,
         simulations: int | None,
         blocks: int | None,
         inference_method: Literal["emcee"] | None,
+        batch_system: BatchSystem | None,
     ) -> "JobSize":
         """
         Infer a job size from several explicit and implicit parameters.
@@ -149,6 +150,8 @@ def size_from_jobs_sims_blocks(
         Returns:
             A job size instance with either the explicit or inferred job sizing.
         """
+        if batch_system == BatchSystem.LOCAL:
+            return cls(jobs=1, simulations=min(blocks * simulations, 10), blocks=1)
         if inference_method == "emcee":
             return cls(jobs=jobs, simulations=blocks * simulations, blocks=1)
         return cls(jobs=jobs, simulations=simulations, blocks=blocks)
@@ -891,11 +894,12 @@ def _click_submit(ctx: click.Context = mock_context, **kwargs) -> None:
         )
 
     # Job size
-    job_size = JobSize.size_from_jobs_sims_blocks(
+    job_size = JobSize.from_jobs_simulations_blocks(
         kwargs["jobs"],
         kwargs["simulations"],
         kwargs["blocks"],
         inference_method,
+        batch_system,
     )
     logger.info("Preparing a job with size %s", job_size)
     if inference_method == "emcee" and job_size.blocks != 1:
diff --git a/flepimop/gempyor_pkg/tests/batch/test_job_size_class.py b/flepimop/gempyor_pkg/tests/batch/test_job_size_class.py
index eba282dab..738aaea85 100644
--- a/flepimop/gempyor_pkg/tests/batch/test_job_size_class.py
+++ b/flepimop/gempyor_pkg/tests/batch/test_job_size_class.py
@@ -3,7 +3,7 @@
 
 import pytest
 
-from gempyor.batch import JobSize
+from gempyor.batch import BatchSystem, JobSize
 
 
 @pytest.mark.parametrize(
@@ -32,20 +32,44 @@ def test_less_than_one_value_error(
         JobSize(**kwargs)
 
 
-def generate_size_from_jobs_sims_blocks(
+def generate_from_jobs_simulations_blocks(
     *args: int | None,
 ) -> Generator[tuple[int | None, ...], None, None]:
     for combo in product(args, repeat=3):
         yield combo
 
 
-@pytest.mark.parametrize("combo", generate_size_from_jobs_sims_blocks(1, 5, 10))
-def test_size_from_jobs_sims_blocks_output(combo: tuple[int, ...]) -> None:
+@pytest.mark.parametrize("combo", generate_from_jobs_simulations_blocks(1, 5, 10))
+def test_from_jobs_simulations_blocks_output(combo: tuple[int, ...]) -> None:
     jobs, simulations, blocks = combo
     for inference_method in (None, ""):
-        job_size = JobSize.size_from_jobs_sims_blocks(
-            jobs, simulations, blocks, inference_method
-        )
-        assert job_size.jobs == jobs
-        assert job_size.simulations >= simulations
-        assert job_size.blocks >= 1
+        for batch_system in (None, BatchSystem.AWS, BatchSystem.LOCAL, BatchSystem.SLURM):
+            job_size = JobSize.from_jobs_simulations_blocks(
+                jobs, simulations, blocks, inference_method, batch_system
+            )
+            assert job_size.jobs >= 1
+            assert job_size.simulations >= simulations
+            assert job_size.blocks >= 1
+
+
+@pytest.mark.parametrize(
+    ("jobs", "simulations", "blocks", "inference_method", "batch_system", "expected"),
+    (
+        (4, 32, 8, None, None, JobSize(jobs=4, simulations=32, blocks=8)),
+        (4, 32, 8, None, BatchSystem.LOCAL, JobSize(jobs=1, simulations=10, blocks=1)),
+        (4, 4, 1, "emcee", BatchSystem.LOCAL, JobSize(jobs=1, simulations=4, blocks=1)),
+        (4, 16, 8, "emcee", None, JobSize(jobs=4, simulations=8 * 16, blocks=1)),
+    ),
+)
+def test_from_jobs_simulations_blocks_exact_results_for_select_inputs(
+    jobs: int | None,
+    simulations: int | None,
+    blocks: int | None,
+    inference_method: Literal["emcee"] | None,
+    batch_system: BatchSystem,
+    expected: JobSize,
+) -> None:
+    job_size = JobSize.from_jobs_simulations_blocks(
+        jobs, simulations, blocks, inference_method, batch_system
+    )
+    assert job_size == expected

From 75448f7d57ff7fb0f1df3c8a9587e4641762bc96 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 12 Nov 2024 10:58:54 -0500
Subject: [PATCH 53/78] Add `conda init` shim

A way to emulate `conda init` without restarting the shell (requirement
for running locally) is to run `eval "$(conda shell.bash hook)"`. Should
have no downside if conda is already initialized.
---
 .../src/gempyor/templates/conda_activate.bash.j2       |  1 +
 .../tests/templates/test_conda_activate_bash.py        | 10 ++--------
 2 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/conda_activate.bash.j2 b/flepimop/gempyor_pkg/src/gempyor/templates/conda_activate.bash.j2
index 0de429020..7338f6b01 100644
--- a/flepimop/gempyor_pkg/src/gempyor/templates/conda_activate.bash.j2
+++ b/flepimop/gempyor_pkg/src/gempyor/templates/conda_activate.bash.j2
@@ -1,4 +1,5 @@
 # Load conda env
+eval "$( conda shell.bash hook )"
 conda activate {{ conda_env }}
 WHICH_PYTHON=$( which python )
 WHICH_RSCRIPT=$( which Rscript )
\ No newline at end of file
diff --git a/flepimop/gempyor_pkg/tests/templates/test_conda_activate_bash.py b/flepimop/gempyor_pkg/tests/templates/test_conda_activate_bash.py
index daae1b4d2..a72b01682 100644
--- a/flepimop/gempyor_pkg/tests/templates/test_conda_activate_bash.py
+++ b/flepimop/gempyor_pkg/tests/templates/test_conda_activate_bash.py
@@ -3,14 +3,6 @@
 from gempyor._jinja import _render_template
 
 
-@pytest.mark.parametrize("conda_env", ("abc", "flepimop-env"))
-def test_output_validation(conda_env: str) -> None:
-    rendered_template = _render_template("conda_activate.bash.j2", {"conda_env": conda_env})
-    lines = rendered_template.split("\n")
-    assert len(lines) == 4
-    assert conda_env in lines[1]
-
-
 @pytest.mark.parametrize(
     ("conda_env", "expected"),
     (
@@ -18,6 +10,7 @@ def test_output_validation(conda_env: str) -> None:
             "flepimop-env",
             [
                 "# Load conda env",
+                'eval "$( conda shell.bash hook )"',
                 "conda activate flepimop-env",
                 "WHICH_PYTHON=$( which python )",
                 "WHICH_RSCRIPT=$( which Rscript )",
@@ -27,6 +20,7 @@ def test_output_validation(conda_env: str) -> None:
             "/path/to/conda/env",
             [
                 "# Load conda env",
+                'eval "$( conda shell.bash hook )"',
                 "conda activate /path/to/conda/env",
                 "WHICH_PYTHON=$( which python )",
                 "WHICH_RSCRIPT=$( which Rscript )",

From e3e4adcf2cca99ec0bc8f10a715bfc58dc077df6 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 12 Nov 2024 11:38:34 -0500
Subject: [PATCH 54/78] Split env vars out into its own template

---
 .../inference_environment_variables.bash.j2   |  9 +++
 .../test_inference_environment_variables.py   | 59 +++++++++++++++++++
 2 files changed, 68 insertions(+)
 create mode 100644 flepimop/gempyor_pkg/src/gempyor/templates/inference_environment_variables.bash.j2
 create mode 100644 flepimop/gempyor_pkg/tests/templates/test_inference_environment_variables.py

diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/inference_environment_variables.bash.j2 b/flepimop/gempyor_pkg/src/gempyor/templates/inference_environment_variables.bash.j2
new file mode 100644
index 000000000..8da1a7016
--- /dev/null
+++ b/flepimop/gempyor_pkg/src/gempyor/templates/inference_environment_variables.bash.j2
@@ -0,0 +1,9 @@
+# Set environment variables
+if [ -z "$SLURM_ARRAY_TASK_ID" ]; then
+    FLEPI_SLOT_INDEX="1"
+else
+    FLEPI_SLOT_INDEX=$SLURM_ARRAY_TASK_ID
+fi
+export FLEPI_PATH={{ flepi_path }}
+export PROJECT_PATH={{ project_path }}
+export CONFIG_PATH={{ config_path }}
\ No newline at end of file
diff --git a/flepimop/gempyor_pkg/tests/templates/test_inference_environment_variables.py b/flepimop/gempyor_pkg/tests/templates/test_inference_environment_variables.py
new file mode 100644
index 000000000..99c05fbb0
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/templates/test_inference_environment_variables.py
@@ -0,0 +1,59 @@
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from gempyor._jinja import _render_template
+
+
+@pytest.mark.parametrize(
+    ("template_data", "expected"),
+    (
+        (
+            {
+                "flepi_path": "/home/foobar/flepiMoP",
+                "project_path": "/home/foobar/project",
+                "config_path": "/home/foobar/project/config.yml",
+            },
+            [
+                "# Set environment variables",
+                'if [ -z "$SLURM_ARRAY_TASK_ID" ]; then',
+                '    FLEPI_SLOT_INDEX="1"',
+                "else",
+                "    FLEPI_SLOT_INDEX=$SLURM_ARRAY_TASK_ID",
+                "fi",
+                "export FLEPI_PATH=/home/foobar/flepiMoP",
+                "export PROJECT_PATH=/home/foobar/project",
+                "export CONFIG_PATH=/home/foobar/project/config.yml",
+            ],
+        ),
+        (
+            {
+                "flepi_path": Path("/users/f/o/foobar/flepiMoP"),
+                "project_path": Path("/users/f/o/foobar/flepimop_sample"),
+                "config_path": Path(
+                    "/users/f/o/foobar/flepimop_sample/config_sample_2pop_inference.yml"
+                ),
+            },
+            [
+                "# Set environment variables",
+                'if [ -z "$SLURM_ARRAY_TASK_ID" ]; then',
+                '    FLEPI_SLOT_INDEX="1"',
+                "else",
+                "    FLEPI_SLOT_INDEX=$SLURM_ARRAY_TASK_ID",
+                "fi",
+                "export FLEPI_PATH=/users/f/o/foobar/flepiMoP",
+                "export PROJECT_PATH=/users/f/o/foobar/flepimop_sample",
+                "export CONFIG_PATH="
+                "/users/f/o/foobar/flepimop_sample/config_sample_2pop_inference.yml",
+            ],
+        ),
+    ),
+)
+def test_exact_results_for_select_inputs(
+    template_data: dict[str, Any], expected: list[str]
+) -> None:
+    lines = _render_template(
+        "inference_environment_variables.bash.j2", template_data
+    ).split("\n")
+    assert lines == expected

From e9bcb23a468fa30a383c84ce74e030ed5cd1ee18 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 12 Nov 2024 15:44:50 -0500
Subject: [PATCH 55/78] Now supports `--local` and non-EMCEE inference

* Add support for non-EMCEE inference which required extracting out
  submission logic into `_submit_scenario_job`.
* Added suport for local runs for testing.
* Added `--skip-manifest` flag to skip writing the manifest, mostly for
  testing.
* Further modularization of the templates, shared components between
  slurm and local runs.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py     | 289 +++++++++++++-----
 .../templates/inference_command.bash.j2       |  27 ++
 .../inference_environment_variables.bash.j2   |  11 +-
 .../gempyor/templates/inference_local.bash.j2 |   8 +-
 .../templates/inference_slurm.sbatch.j2       |  42 +--
 .../tests/batch/test__click_submit.py         |  60 +---
 .../test_inference_environment_variables.py   |  40 ++-
 7 files changed, 302 insertions(+), 175 deletions(-)
 create mode 100644 flepimop/gempyor_pkg/src/gempyor/templates/inference_command.bash.j2

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index bdedf071d..1dd2d9531 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -12,6 +12,7 @@
 from datetime import datetime, timedelta, timezone
 from enum import Enum, auto
 from getpass import getuser
+from itertools import product
 import json
 import math
 from pathlib import Path
@@ -151,6 +152,8 @@ def from_jobs_simulations_blocks(
             A job size instance with either the explicit or inferred job sizing.
         """
         if batch_system == BatchSystem.LOCAL:
+            blocks = 11 if blocks is None else blocks
+            simulations = 11 if simulations is None else simulations
             return cls(jobs=1, simulations=min(blocks * simulations, 10), blocks=1)
         if inference_method == "emcee":
             return cls(jobs=jobs, simulations=blocks * simulations, blocks=1)
@@ -486,10 +489,14 @@ def _local(script: Path, verbosity: int | None, dry_run: bool) -> None:
 
     if verbosity is not None:
         logger.info("Executing script with: %s", " ".join(cmd_args))
-    process = subprocess.run(cmd_args, check=True, capture_output=True)
+    process = subprocess.run(cmd_args, capture_output=True)
     stdout = process.stdout.decode().strip()
     stderr = process.stderr.decode().strip()
     if verbosity is not None:
+        if process.returncode != 0:
+            logger.critical(
+                "Received non-zero exit code, %u, from executed script.", process.returncode
+            )
         if stdout:
             logger.debug("Captured stdout from executed script: %s", stdout)
         if stderr:
@@ -603,10 +610,14 @@ def _sbatch(
 
     if verbosity is not None:
         logger.info("Submitting to slurm with: %s", " ".join(cmd_args))
-    process = subprocess.run(cmd_args, check=True, capture_output=True)
+    process = subprocess.run(cmd_args, capture_output=True)
     stdout = process.stdout.decode().strip()
     stderr = process.stderr.decode().strip()
     if verbosity is not None:
+        if process.returncode != 0:
+            logger.critical(
+                "Received non-zero exit code, %u, from executed script.", process.returncode
+            )
         if stdout:
             logger.debug("Captured stdout from sbatch submission: %s", stdout)
         if stderr:
@@ -696,6 +707,126 @@ def _job_name(name: str | None, timestamp: datetime | None) -> str:
     return f"{name}-{timestamp}" if name else timestamp
 
 
+def _submit_scenario_job(
+    outcome_modifiers_scenario: str | None,
+    seir_modifiers_scenario: str | None,
+    name: str,
+    batch_system: BatchSystem,
+    inference_method: Literal["emcee"] | None,
+    config_out: Path,
+    job_name: str,
+    job_size: JobSize,
+    job_time_limit: JobTimeLimit,
+    job_resources: JobResources,
+    kwargs: dict[str, Any],
+    verbosity: int | None,
+    dry_run: bool,
+    now: datetime,
+) -> None:
+    """
+    Submit a job for a scenario.
+
+    Args:
+        ...
+    """
+    # Get logger
+    if verbosity is not None:
+        logger = get_script_logger(__name__, verbosity)
+        if outcome_modifiers_scenario is None:
+            logger.warning(
+                "The outcome modifiers scenario is `None`, may lead to "
+                "unintended consequences in output file/directory names."
+            )
+        if seir_modifiers_scenario is None:
+            logger.warning(
+                "The seir modifiers scenario is `None`, may lead to "
+                "unintended consequences in output file/directory names."
+            )
+
+    # Modify the job for the given scenario info
+    job_name += f"_{seir_modifiers_scenario}_{outcome_modifiers_scenario}"
+    prefix = f"{name}_{seir_modifiers_scenario}_{outcome_modifiers_scenario}"
+    if verbosity is not None:
+        logger.info(
+            "Preparing a job for outcome and seir modifiers scenarios "
+            "'%s' and '%s', respectively, with job name '%s'.",
+            outcome_modifiers_scenario,
+            seir_modifiers_scenario,
+            job_name,
+        )
+
+    # Generically useful template data, regardless of batch system
+    template_data = {
+        "conda_env": kwargs["conda_env"],
+        "config_path": config_out.absolute(),
+        "debug": kwargs["debug"],
+        "flepi_path": kwargs["flepi_path"].absolute(),
+        "inference_method": "" if inference_method is None else inference_method,
+        "job_name": job_name,
+        "jobs": job_size.jobs,
+        "outcome_modifiers_scenario": outcome_modifiers_scenario,
+        "nslots": job_size.simulations,  # aka nwalkers
+        "prefix": prefix,
+        "project_path": kwargs["project_path"].absolute(),
+        "reset_chimerics": kwargs["reset_chimerics"],
+        "run_id": kwargs["run_id"],
+        "seir_modifiers_scenario": seir_modifiers_scenario,
+        "simulations": job_size.simulations,
+        "stoch_traj_flag": kwargs["stoch_traj_flag"],
+    }
+
+    # Branch on the specific batch system
+    if batch_system == BatchSystem.LOCAL:
+        # Construct the local execution
+        _local_template(
+            "inference_local.bash.j2",
+            None,
+            template_data,
+            verbosity,
+            dry_run,
+        )
+    elif batch_system == BatchSystem.SLURM:
+        # Cluster info
+        if kwargs["cluster"] is None:
+            raise ValueError("When submitting a batch job to slurm a cluster is required.")
+        cluster = get_cluster_info(kwargs["cluster"])
+        if verbosity is not None:
+            logger.info(
+                "Utilizing info for the '%s' cluster to construct this job", cluster.name
+            )
+            logger.debug(
+                "The full settings for cluster '%s' are %s",
+                cluster.name,
+                cluster.model_dump(),
+            )
+        template_data["cluster"] = cluster.model_dump()
+        # Construct the sbatch call
+        options = {
+            "chdir": kwargs["project_path"].absolute(),
+            "comment": f"Generated on {now:%c %Z} and submitted by {getuser()}.",
+            "cpus-per-task": job_resources.format_cpus(batch_system),
+            "job-name": job_name,
+            "mem": job_resources.format_memory(batch_system),
+            "nodes": job_resources.format_nodes(batch_system),
+            "ntasks-per-node": 1,
+            "time": job_time_limit.format(batch_system),
+        }
+        if kwargs["partition"] is not None:
+            options["partition"] = kwargs["partition"]
+        if kwargs["email"] is not None:
+            options["mail-type"] = "BEGIN,END"
+            options["mail-user"] = kwargs["email"]
+        _sbatch_template(
+            "inference_slurm.sbatch.j2",
+            None,
+            template_data,
+            "all",
+            options,
+            verbosity,
+            dry_run,
+        )
+
+
 @cli.command(
     name="submit",
     params=[config_files_argument]
@@ -810,13 +941,6 @@ def _job_name(name: str | None, timestamp: datetime | None) -> str:
             default=None,
             help="Unique identifier for this run.",
         ),
-        click.Option(
-            param_decls=["--prefix", "prefix"],
-            envvar="FLEPI_PREFIX",
-            type=str,
-            default=None,
-            help="Unique prefix for this run.",
-        ),
         click.Option(
             param_decls=["--email", "email"],
             type=str,
@@ -841,11 +965,27 @@ def _job_name(name: str | None, timestamp: datetime | None) -> str:
             default=None,
             help="Override for the amount of memory per node to use in MB.",
         ),
+        click.Option(
+            param_decls=["--skip-manifest", "skip_manifest"],
+            type=bool,
+            default=False,
+            is_flag=True,
+            help="Flag to skip writing a 'manifest.json' file to the project path.",
+        ),
+        click.Option(
+            param_decls=[
+                "--reset-chimerics-on-global-accept/--no-reset-chimerics-on-global-accept",
+                "reset_chimerics",
+            ],
+            type=bool,
+            default=False,
+            help="Flag to reset chimerics on global accept.",
+        ),
     ]
     + list(verbosity_options.values()),
 )
 @click.pass_context
-def _click_submit(ctx: click.Context = mock_context, **kwargs) -> None:
+def _click_submit(ctx: click.Context = mock_context, **kwargs: Any) -> None:
     """Submit batch jobs"""
     # Generic setup
     now = datetime.now(timezone.utc)
@@ -853,20 +993,6 @@ def _click_submit(ctx: click.Context = mock_context, **kwargs) -> None:
     log_cli_inputs(kwargs)
     cfg = parse_config_files(config, ctx, **kwargs)
 
-    # Temporary limitation
-    if (
-        not cfg["inference"].exists()
-        or not cfg["inference"]["method"].exists()
-        or cfg["inference"]["method"].as_str() != "emcee"
-    ):
-        raise NotImplementedError(
-            "The `flepimop submit` CLI only supports EMCEE inference jobs."
-        )
-    inference_method = cfg["inference"]["method"].as_str()
-    inference_method = (
-        inference_method if inference_method is None else inference_method.lower()
-    )
-
     # Job name/run id
     name = cfg["name"].get(str) if cfg["name"].exists() else None
     job_name = _job_name(name, now)
@@ -875,14 +1001,30 @@ def _click_submit(ctx: click.Context = mock_context, **kwargs) -> None:
         kwargs["run_id"] = run_id(now)
     logger.info("Using a run id of '%s'", kwargs["run_id"])
 
+    # Determine inference method
+    inference_method = (
+        cfg["inference"]["method"].as_str()
+        if cfg["inference"].exists() and cfg["inference"]["method"].exists()
+        else None
+    )
+    inference_method = (
+        inference_method if inference_method is None else inference_method.lower()
+    )
+    if kwargs["reset_chimerics"] and inference_method is not None:
+        logger.warning(
+            "The inference method is '%s', which does not support "
+            "the reset chimerics on global accept flag.",
+            inference_method,
+        )
+
     # Batch system
     batch_system = BatchSystem.from_options(
         kwargs["batch_system"], kwargs["aws"], kwargs["local"], kwargs["slurm"]
     )
-    if batch_system != BatchSystem.SLURM:
+    if batch_system == BatchSystem.AWS:
         # Temporary limitation
         raise NotImplementedError(
-            "The `flepimop submit` CLI only supports batch submission to slurm."
+            "The `flepimop submit` CLI does not support batch submission to AWS yet."
         )
     logger.info("Constructing a job to submit to %s", batch_system)
     if batch_system != BatchSystem.SLURM and kwargs["email"] is not None:
@@ -922,24 +1064,35 @@ def _click_submit(ctx: click.Context = mock_context, **kwargs) -> None:
         cpus=kwargs["cpus"],
         memory=kwargs["memory"],
     )
-
-    # Cluster info
-    cluster: Cluster | None = None
-    if batch_system == BatchSystem.SLURM:
-        if kwargs["cluster"] is None:
-            raise ValueError("When submitting a batch job to slurm a cluster is required.")
-        cluster = get_cluster_info(kwargs["cluster"])
-    logger.info("Utilizing info for the '%s' cluster to construct this job", cluster.name)
-    logger.debug(
-        "The full settings for cluster '%s' are %s", cluster.name, cluster.model_dump()
+    logger.info("Requesting the resources %s for this job.", job_resources)
+
+    # Outcome/seir modifier scenarios
+    outcome_modifiers_scenarios = (
+        cfg["outcome_modifiers"]["scenarios"].as_str_seq()
+        if cfg["outcome_modifiers"].exists()
+        and cfg["outcome_modifiers"]["scenarios"].exists()
+        else [None]
+    )
+    seir_modifiers_scenarios = (
+        cfg["seir_modifiers"]["scenarios"].as_str_seq()
+        if cfg["seir_modifiers"].exists() and cfg["seir_modifiers"]["scenarios"].exists()
+        else [None]
     )
 
     # Restart/continuation location
     # TODO: Implement this
 
     # Manifest
-    manifest = write_manifest(job_name, kwargs["flepi_path"], kwargs["project_path"])
-    logger.info("Writing manifest metadata to '%s'", manifest.absolute())
+    if kwargs["skip_manifest"]:
+        manifest = write_manifest(
+            job_name,
+            kwargs["flepi_path"],
+            kwargs["project_path"],
+            destination=kwargs["project_path"] / "manifest.json",
+        )
+        logger.info("Writing manifest metadata to '%s'", manifest.absolute())
+    else:
+        logger.debug("Skipped writing manifest metadata.")
 
     # Config out
     if (config_out := kwargs["config_out"]) is None:
@@ -950,43 +1103,23 @@ def _click_submit(ctx: click.Context = mock_context, **kwargs) -> None:
         "Dumped the final config for this batch submission to %s", config_out.absolute()
     )
 
-    # Construct the sbatch call
-    template_data = {
-        "conda_env": kwargs["conda_env"],
-        "config_out": config_out.absolute(),
-        "cluster": cluster if cluster is None else cluster.model_dump(),
-        "debug": kwargs["debug"],
-        "flepi_path": kwargs["flepi_path"].absolute(),
-        "inference_method": "" if inference_method is None else inference_method,
-        "job_name": job_name,
-        "jobs": job_size.jobs,
-        "nslots": job_size.simulations,  # aka nwalkers
-        "prefix": kwargs["prefix"],
-        "project_path": kwargs["project_path"].absolute(),
-        "run_id": kwargs["run_id"],
-        "simulations": job_size.simulations,
-    }
-    options = {
-        "chdir": kwargs["project_path"].absolute(),
-        "comment": f"Generated on {now:%c %Z} and submitted by {getuser()}.",
-        "cpus-per-task": job_resources.format_cpus(batch_system),
-        "job-name": job_name,
-        "mem": job_resources.format_memory(batch_system),
-        "nodes": job_resources.format_nodes(batch_system),
-        "ntasks-per-node": 1,
-        "time": job_time_limit.format(batch_system),
-    }
-    if kwargs["partition"] is not None:
-        options["partition"] = kwargs["partition"]
-    if kwargs["email"] is not None:
-        options["mail-type"] = "BEGIN,END"
-        options["mail-user"] = kwargs["email"]
-    _sbatch_template(
-        "inference.sbatch.j2",
-        None,
-        template_data,
-        "all",
-        options,
-        kwargs["verbosity"],
-        kwargs["dry_run"],
-    )
+    # Loop over the scenarios
+    for outcome_modifiers_scenario, seir_modifiers_scenario in product(
+        outcome_modifiers_scenarios, seir_modifiers_scenarios
+    ):
+        _submit_scenario_job(
+            outcome_modifiers_scenario,
+            seir_modifiers_scenario,
+            name,
+            batch_system,
+            inference_method,
+            config_out,
+            job_name,
+            job_size,
+            job_time_limit,
+            job_resources,
+            kwargs,
+            kwargs["verbosity"],
+            kwargs["dry_run"],
+            now,
+        )
diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/inference_command.bash.j2 b/flepimop/gempyor_pkg/src/gempyor/templates/inference_command.bash.j2
new file mode 100644
index 000000000..9a348f259
--- /dev/null
+++ b/flepimop/gempyor_pkg/src/gempyor/templates/inference_command.bash.j2
@@ -0,0 +1,27 @@
+# Run inference
+{%- if inference_method == 'emcee' %}
+flepimop-calibrate --config $CONFIG_PATH \
+    --project_path $PROJECT_PATH \
+    --nslots {{ nslots }} \
+    --niterations {{ simulations }} \
+    --jobs {{ jobs }} \
+    --id {{ run_id }} \
+    --prefix {{ prefix}} \
+    > $LOG_FILE 2>&1
+{%- else %}
+flepimop-inference-slot --config $CONFIG_PATH \
+    --run_id {{ run_id }} \
+    --seir_modifiers_scenarios $FLEPI_SEIR_SCENARIOS \
+    --outcome_modifiers_scenarios $FLEPI_OUTCOME_SCENARIOS \
+    --jobs 1 \
+    --iterations_per_slot {{ simulations }} \
+    --this_slot $FLEPI_SLOT_INDEX \
+    --this_block 1 \
+    --stoch_traj_flag $FLEPI_STOCHASTIC_RUN \
+    --flepi_path $FLEPI_PATH \
+    --python $WHICH_PYTHON \
+    --rpath $WHICH_RSCRIPT \
+    --is-interactive FALSE \
+    --reset_chimeric_on_accept $FLEPI_RESET_CHIMERICS \
+    > $LOG_FILE 2>&1
+{%- endif %}
\ No newline at end of file
diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/inference_environment_variables.bash.j2 b/flepimop/gempyor_pkg/src/gempyor/templates/inference_environment_variables.bash.j2
index 8da1a7016..611efee41 100644
--- a/flepimop/gempyor_pkg/src/gempyor/templates/inference_environment_variables.bash.j2
+++ b/flepimop/gempyor_pkg/src/gempyor/templates/inference_environment_variables.bash.j2
@@ -4,6 +4,11 @@ if [ -z "$SLURM_ARRAY_TASK_ID" ]; then
 else
     FLEPI_SLOT_INDEX=$SLURM_ARRAY_TASK_ID
 fi
-export FLEPI_PATH={{ flepi_path }}
-export PROJECT_PATH={{ project_path }}
-export CONFIG_PATH={{ config_path }}
\ No newline at end of file
+export FLEPI_PATH="{{ flepi_path }}"
+export PROJECT_PATH="{{ project_path }}"
+export CONFIG_PATH="{{ config_path }}"
+export FLEPI_STOCHASTIC_RUN="{{ stoch_traj_flag | string | upper }}"
+export FLEPI_OUTCOME_SCENARIOS="{{ outcome_modifiers_scenario }}"
+export FLEPI_SEIR_SCENARIOS="{{ seir_modifiers_scenario }}"
+export FLEPI_RESET_CHIMERICS="{{ reset_chimerics | string | upper }}"
+export LOG_FILE="out_{{ job_name }}_$FLEPI_SLOT_INDEX.out"
\ No newline at end of file
diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/inference_local.bash.j2 b/flepimop/gempyor_pkg/src/gempyor/templates/inference_local.bash.j2
index 632a4db34..b9dda9f07 100644
--- a/flepimop/gempyor_pkg/src/gempyor/templates/inference_local.bash.j2
+++ b/flepimop/gempyor_pkg/src/gempyor/templates/inference_local.bash.j2
@@ -1,3 +1,9 @@
-#!/usr/bin/env {{ interpreter }}
+#!/usr/bin/env bash
+
+{% include "inference_environment_variables.bash.j2" with context %}
+
+cd $PROJECT_PATH
 
 {% include "conda_activate.bash.j2" with context %}
+
+{% include "inference_command.bash.j2" with context %}
diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/inference_slurm.sbatch.j2 b/flepimop/gempyor_pkg/src/gempyor/templates/inference_slurm.sbatch.j2
index 211a876bc..5b69562d4 100644
--- a/flepimop/gempyor_pkg/src/gempyor/templates/inference_slurm.sbatch.j2
+++ b/flepimop/gempyor_pkg/src/gempyor/templates/inference_slurm.sbatch.j2
@@ -8,15 +8,7 @@
 set -x
 {%- endif %}
 
-# Set script variables
-if [ -z "$SLURM_ARRAY_TASK_ID" ]; then
-    FLEPI_SLOT_INDEX="1"
-else
-    FLEPI_SLOT_INDEX=$SLURM_ARRAY_TASK_ID
-fi
-export FLEPI_PATH={{ flepi_path }}
-export PROJECT_PATH={{ project_path }}
-export CONFIG_PATH={{ config_out }}
+{% include "inference_environment_variables.bash.j2" with context %}
 
 {% include "cluster_setup.bash.j2" with context %}
 
@@ -26,36 +18,6 @@ export CONFIG_PATH={{ config_out }}
 
 # Fetch/setup resume files
 
-# Run inference
-{%- if inference_method == 'emcee' %}
-flepimop-calibrate --config $CONFIG_PATH \
-    --project_path $PROJECT_PATH \
-    --nslots {{ nslots }} \
-    --niterations {{ simulations }} \
-    --jobs {{ jobs }} \
-    --id {{ run_id }} \
-{%- if prefix is defined and prefix is not none %}
-    --prefix {{ prefix}} \
-{%- endif %}
-    > out_{{ job_name }}_$FLEPI_SLOT_INDEX.out 2>&1
-{%- else %}
-flepimop-inference-slot --config $CONFIG_PATH \
-    --run_id {{ run_id }} \
-    --seir_modifiers_scenarios $FLEPI_SEIR_SCENARIOS \
-    --outcome_modifiers_scenarios $FLEPI_OUTCOME_SCENARIOS \
-    --jobs 1 \
-    --iterations_per_slot {{ simulations }} \
-    --slots {{ nslots }} \
-    --this_slot $FLEPI_SLOT_INDEX \
-    --this_block 1 \
-    --stoch_traj_flag $FLEPI_STOCHASTIC_RUN \
-    --flepi_path $FLEPI_PATH \
-    --python $WHICH_PYTHON \
-    --rpath $WHICH_RSCRIPT \
-    --is-resume $RESUME_RUN \
-    --is-interactive FALSE \
-    --reset_chimeric_on_accept $FLEPI_RESET_CHIMERICS \
-    > out_{{ job_name }}_$FLEPI_SLOT_INDEX.out 2>&1
-{%- endif %}
+{% include "inference_command.bash.j2" with context %}
 
 # Upload/move results
diff --git a/flepimop/gempyor_pkg/tests/batch/test__click_submit.py b/flepimop/gempyor_pkg/tests/batch/test__click_submit.py
index abe4fda61..31582c9f6 100644
--- a/flepimop/gempyor_pkg/tests/batch/test__click_submit.py
+++ b/flepimop/gempyor_pkg/tests/batch/test__click_submit.py
@@ -9,66 +9,38 @@
 from gempyor.batch import _click_submit
 
 
-@pytest.fixture
-def add_sbatch_to_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
-    sbatch = tmp_path / "bin" / "sbatch"
-    sbatch.parent.mkdir(parents=True, exist_ok=True)
-    sbatch.touch(mode=0o755)
-    monkeypatch.setenv("PATH", str(sbatch.parent.absolute()), prepend=os.pathsep)
-    return sbatch
-
-
-@pytest.mark.parametrize(
-    "config",
-    (
-        {},
-        {"inference": {"abc": 123}},
-        {"inference": {"method": "legacy"}},
-    ),
-)
-def test_only_inference_emcee_supported_not_implemented_error(
-    tmp_path: Path, config: dict[str, Any]
-) -> None:
+def test_batch_system_aws_not_implemented_error(tmp_path: Path) -> None:
     config_file = tmp_path / "config.yml"
     with config_file.open(mode="w") as f:
-        yaml.dump(config, f)
+        yaml.dump({"name": "foobar", "inference": {"method": "emcee"}}, f)
 
     runner = CliRunner()
-    result = runner.invoke(_click_submit, [str(config_file.absolute())])
-
-    assert result.exit_code == 1
-    assert isinstance(result.exception, NotImplementedError)
-    assert (
-        str(result.exception)
-        == "The `flepimop submit` CLI only supports EMCEE inference jobs."
+    result = runner.invoke(
+        _click_submit,
+        [
+            "--aws",
+            "--simulations",
+            "1",
+            "--jobs",
+            "1",
+            "--blocks",
+            "1",
+            str(config_file.absolute()),
+        ],
     )
 
-
-@pytest.mark.parametrize(
-    "args", (["--aws"], ["--batch-system", "aws"], ["--local"], ["--batch-system", "local"])
-)
-def test_only_slurm_batch_system_supported_not_implemented_error(
-    tmp_path: Path, args: list[str]
-) -> None:
-    config_file = tmp_path / "config.yml"
-    with config_file.open(mode="w") as f:
-        yaml.dump({"inference": {"method": "emcee"}}, f)
-
-    runner = CliRunner()
-    result = runner.invoke(_click_submit, args + [str(config_file.absolute())])
-
     assert result.exit_code == 1
     assert isinstance(result.exception, NotImplementedError)
     assert (
         str(result.exception)
-        == "The `flepimop submit` CLI only supports batch submission to slurm."
+        == "The `flepimop submit` CLI does not support batch submission to AWS yet."
     )
 
 
 def test_cluster_required_for_slurm_value_error(tmp_path: Path) -> None:
     config_file = tmp_path / "config.yml"
     with config_file.open(mode="w") as f:
-        yaml.dump({"inference": {"method": "emcee"}}, f)
+        yaml.dump({"name": "foobar", "inference": {"method": "emcee"}}, f)
 
     runner = CliRunner()
     result = runner.invoke(
diff --git a/flepimop/gempyor_pkg/tests/templates/test_inference_environment_variables.py b/flepimop/gempyor_pkg/tests/templates/test_inference_environment_variables.py
index 99c05fbb0..61afebe28 100644
--- a/flepimop/gempyor_pkg/tests/templates/test_inference_environment_variables.py
+++ b/flepimop/gempyor_pkg/tests/templates/test_inference_environment_variables.py
@@ -11,9 +11,14 @@
     (
         (
             {
+                "config_path": "/home/foobar/project/config.yml",
                 "flepi_path": "/home/foobar/flepiMoP",
+                "job_name": "sample_2pop-20240101T0000_None_None",
+                "outcome_modifiers_scenario": None,
                 "project_path": "/home/foobar/project",
-                "config_path": "/home/foobar/project/config.yml",
+                "reset_chimerics": False,
+                "seir_modifiers_scenario": None,
+                "stoch_traj_flag": True,
             },
             [
                 "# Set environment variables",
@@ -22,18 +27,29 @@
                 "else",
                 "    FLEPI_SLOT_INDEX=$SLURM_ARRAY_TASK_ID",
                 "fi",
-                "export FLEPI_PATH=/home/foobar/flepiMoP",
-                "export PROJECT_PATH=/home/foobar/project",
-                "export CONFIG_PATH=/home/foobar/project/config.yml",
+                'export FLEPI_PATH="/home/foobar/flepiMoP"',
+                'export PROJECT_PATH="/home/foobar/project"',
+                'export CONFIG_PATH="/home/foobar/project/config.yml"',
+                'export FLEPI_STOCHASTIC_RUN="TRUE"',
+                'export FLEPI_OUTCOME_SCENARIOS="None"',
+                'export FLEPI_SEIR_SCENARIOS="None"',
+                'export FLEPI_RESET_CHIMERICS="FALSE"',
+                "export LOG_FILE="
+                '"out_sample_2pop-20240101T0000_None_None_$FLEPI_SLOT_INDEX.out"',
             ],
         ),
         (
             {
-                "flepi_path": Path("/users/f/o/foobar/flepiMoP"),
-                "project_path": Path("/users/f/o/foobar/flepimop_sample"),
                 "config_path": Path(
                     "/users/f/o/foobar/flepimop_sample/config_sample_2pop_inference.yml"
                 ),
+                "flepi_path": Path("/users/f/o/foobar/flepiMoP"),
+                "job_name": "sample_2pop-20240101T0000_Ro_all_test_limits",
+                "outcome_modifiers_scenario": "test_limits",
+                "project_path": Path("/users/f/o/foobar/flepimop_sample"),
+                "reset_chimerics": True,
+                "seir_modifiers_scenario": "Ro_all",
+                "stoch_traj_flag": False,
             },
             [
                 "# Set environment variables",
@@ -42,10 +58,16 @@
                 "else",
                 "    FLEPI_SLOT_INDEX=$SLURM_ARRAY_TASK_ID",
                 "fi",
-                "export FLEPI_PATH=/users/f/o/foobar/flepiMoP",
-                "export PROJECT_PATH=/users/f/o/foobar/flepimop_sample",
+                'export FLEPI_PATH="/users/f/o/foobar/flepiMoP"',
+                'export PROJECT_PATH="/users/f/o/foobar/flepimop_sample"',
                 "export CONFIG_PATH="
-                "/users/f/o/foobar/flepimop_sample/config_sample_2pop_inference.yml",
+                '"/users/f/o/foobar/flepimop_sample/config_sample_2pop_inference.yml"',
+                'export FLEPI_STOCHASTIC_RUN="FALSE"',
+                'export FLEPI_OUTCOME_SCENARIOS="test_limits"',
+                'export FLEPI_SEIR_SCENARIOS="Ro_all"',
+                'export FLEPI_RESET_CHIMERICS="TRUE"',
+                "export LOG_FILE="
+                '"out_sample_2pop-20240101T0000_Ro_all_test_limits_$FLEPI_SLOT_INDEX.out"',
             ],
         ),
     ),

From 08ec61e96209a405a2b25a21d6d75a4532264ec1 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 12 Nov 2024 16:02:21 -0500
Subject: [PATCH 56/78] Fix bug in `--skip-manifest` flag

The `--skip-manifest` flag had the opposite behaivor of what the name
and help would suggest.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 1dd2d9531..2510dfc33 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -1084,6 +1084,8 @@ def _click_submit(ctx: click.Context = mock_context, **kwargs: Any) -> None:
 
     # Manifest
     if kwargs["skip_manifest"]:
+        logger.debug("Skipped writing manifest metadata.")
+    else:
         manifest = write_manifest(
             job_name,
             kwargs["flepi_path"],
@@ -1091,8 +1093,6 @@ def _click_submit(ctx: click.Context = mock_context, **kwargs: Any) -> None:
             destination=kwargs["project_path"] / "manifest.json",
         )
         logger.info("Writing manifest metadata to '%s'", manifest.absolute())
-    else:
-        logger.debug("Skipped writing manifest metadata.")
 
     # Config out
     if (config_out := kwargs["config_out"]) is None:

From b6c95ca051bc51ef776ac97b386f941121390061 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 12 Nov 2024 16:26:05 -0500
Subject: [PATCH 57/78] Add git branch checkout to preserve run.

Added internal `_git_checkout` utility to checkout a new branch. The
`_click_submit` tool now checks out a new branch based on the name of
the run at the end of the tool. Still need to add unit tests for
`_git_checkout`.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py | 21 ++++++++++++++++++++-
 flepimop/gempyor_pkg/src/gempyor/utils.py | 23 +++++++++++++++++++++++
 2 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 2510dfc33..7fccea7c9 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -29,7 +29,7 @@
 from .file_paths import run_id
 from .info import Cluster, get_cluster_info
 from .logging import get_script_logger
-from .utils import _format_cli_options, _git_head, _shutil_which, config
+from .utils import _format_cli_options, _git_checkout, _git_head, _shutil_which, config
 from .shared_cli import (
     NONNEGATIVE_DURATION,
     cli,
@@ -981,6 +981,13 @@ def _submit_scenario_job(
             default=False,
             help="Flag to reset chimerics on global accept.",
         ),
+        click.Option(
+            param_decls=["--skip-checkout", "skip_checkout"],
+            type=bool,
+            default=False,
+            is_flag=True,
+            help="Flag to skip checking out a new branch in the project git directory.",
+        ),
     ]
     + list(verbosity_options.values()),
 )
@@ -1123,3 +1130,15 @@ def _click_submit(ctx: click.Context = mock_context, **kwargs: Any) -> None:
             kwargs["dry_run"],
             now,
         )
+
+    # Checkout a new branch to preserve run information
+    if kwargs["skip_checkout"]:
+        logger.debug("Skipped checking out a new branch to preserve run.")
+    else:
+        branch = f"run_{job_name}"
+        _git_checkout(kwargs["project_path"], branch)
+        logger.info(
+            "Checked out a new branch, '%s', in the project "
+            "repository to preserve run details.",
+            branch,
+        )
diff --git a/flepimop/gempyor_pkg/src/gempyor/utils.py b/flepimop/gempyor_pkg/src/gempyor/utils.py
index 20ed5de58..5b9b75fa1 100644
--- a/flepimop/gempyor_pkg/src/gempyor/utils.py
+++ b/flepimop/gempyor_pkg/src/gempyor/utils.py
@@ -1118,6 +1118,29 @@ def _git_head(repository: Path) -> str:
     return proc.stdout.decode().strip()
 
 
+def _git_checkout(repository: Path, branch: str) -> None:
+    """
+    Checkout a new branch from a given git repository.
+
+    Args:
+        repository: A directory under version control with git to
+            checkout a new branch in.
+        branch: The name of the new branch to checkout.
+
+    Examples:
+        >>> from pathlib import Path
+        >>> _git_checkout(Path("~/Desktop/GitHub/HopkinsIDD/flepiMoP"), "my-new-branch")
+    """
+    git_cmd = _shutil_which("git")
+    subprocess.run(
+        [git_cmd, "checkout", "-b", branch],
+        cwd=repository.expanduser().absolute(),
+        check=True,
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
+
+
 def _format_cli_options(options: dict[str, Any]) -> list[str]:
     """
     Convert a dictionary of CLI options into a formatted list.

From 3fb4dbe5eaaa9132288a1e6c1d7787615c192fc4 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 12 Nov 2024 16:38:30 -0500
Subject: [PATCH 58/78] Remove custom interpreter for sbatch

Making the assumption that sbatch submission scripts are written in
bash. Too complicated to make them work for multiple shells.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py                       | 1 -
 .../gempyor_pkg/src/gempyor/templates/inference_slurm.sbatch.j2 | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 7fccea7c9..ce0ca7e54 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -660,7 +660,6 @@ def _sbatch_template(
     Raises:
         ValueError: If 'options' is found in `template_data` and `options` is not empty.
     """
-    template_data["interpreter"] = template_data.get("interpreter", "bash")
     if "options" in template_data and options:
         raise ValueError(
             "Found 'options' in `template_data` but `options` is not empty, can only one."
diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/inference_slurm.sbatch.j2 b/flepimop/gempyor_pkg/src/gempyor/templates/inference_slurm.sbatch.j2
index 5b69562d4..62d346323 100644
--- a/flepimop/gempyor_pkg/src/gempyor/templates/inference_slurm.sbatch.j2
+++ b/flepimop/gempyor_pkg/src/gempyor/templates/inference_slurm.sbatch.j2
@@ -1,4 +1,4 @@
-#!/usr/bin/env {{ interpreter }}
+#!/usr/bin/env bash
 {%- for opt in options %}
 #SBATCH {{ opt }}
 {%- endfor %}

From 93ab6602de4a6010ba76872f62a7bfa7836bcb94 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Thu, 14 Nov 2024 08:59:25 -0500
Subject: [PATCH 59/78] Legacy inference use slurm array jobs

---
 flepimop/gempyor_pkg/src/gempyor/batch.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index ce0ca7e54..ca139bb13 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -815,6 +815,8 @@ def _submit_scenario_job(
         if kwargs["email"] is not None:
             options["mail-type"] = "BEGIN,END"
             options["mail-user"] = kwargs["email"]
+        if inference_method is None:
+            options["array"] = f"1-{job_size.jobs}"
         _sbatch_template(
             "inference_slurm.sbatch.j2",
             None,

From 049089544dafa4abe2e37d9a4f6a41588d00c397 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Thu, 14 Nov 2024 12:01:58 -0500
Subject: [PATCH 60/78] Override nodes for legacy inference on slurm

---
 flepimop/gempyor_pkg/src/gempyor/batch.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index ca139bb13..7df377582 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -816,6 +816,7 @@ def _submit_scenario_job(
             options["mail-type"] = "BEGIN,END"
             options["mail-user"] = kwargs["email"]
         if inference_method is None:
+            options["nodes"] = "1"
             options["array"] = f"1-{job_size.jobs}"
         _sbatch_template(
             "inference_slurm.sbatch.j2",

From 7e7efd5511a0a13b4fea9e4e523d5053032e2f27 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Thu, 14 Nov 2024 13:49:46 -0500
Subject: [PATCH 61/78] Infer cluster from FQDN

* Added `gempyor.info._infer_cluster_from_fqdn` internal utility.
* `gempyor.info.get_cluster_info` now accepts `None` for `name`.
* `gempyor.batch._click_submit` accepts `None` for `--cluster` when
  using `--slurm`.
* Minor testing fixes.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py     |  2 -
 flepimop/gempyor_pkg/src/gempyor/info.py      | 32 ++++++++++++++--
 .../tests/batch/test__click_submit.py         | 32 ++--------------
 .../info/test__infer_cluster_from_fqdn.py     | 37 +++++++++++++++++++
 .../tests/info/test_get_cluster_info.py       | 20 +++++++++-
 .../templates/test_cluster_setup_bash.py      | 15 ++++----
 6 files changed, 96 insertions(+), 42 deletions(-)
 create mode 100644 flepimop/gempyor_pkg/tests/info/test__infer_cluster_from_fqdn.py

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 7df377582..b2d56c99f 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -786,8 +786,6 @@ def _submit_scenario_job(
         )
     elif batch_system == BatchSystem.SLURM:
         # Cluster info
-        if kwargs["cluster"] is None:
-            raise ValueError("When submitting a batch job to slurm a cluster is required.")
         cluster = get_cluster_info(kwargs["cluster"])
         if verbosity is not None:
             logger.info(
diff --git a/flepimop/gempyor_pkg/src/gempyor/info.py b/flepimop/gempyor_pkg/src/gempyor/info.py
index 1048f3f8a..00d094c09 100644
--- a/flepimop/gempyor_pkg/src/gempyor/info.py
+++ b/flepimop/gempyor_pkg/src/gempyor/info.py
@@ -3,7 +3,9 @@
 
 import os
 from pathlib import Path
-from typing import TypeVar
+import re
+from socket import getfqdn
+from typing import Pattern, TypeVar
 
 from pydantic import BaseModel
 import yaml
@@ -29,6 +31,12 @@ class Cluster(BaseModel):
 T = TypeVar("T", bound=BaseModel)
 
 
+_CLUSTER_FQDN_REGEXES: tuple[tuple[str, Pattern], ...] = (
+    ("longleaf", re.compile(r"^longleaf\-login[0-9]+\.its\.unc\.edu$")),
+    ("rockfish", re.compile(r"^login[0-9]+\.cm\.cluster$")),
+)
+
+
 def _get_info(
     category: str, name: str, model: type[T], flepi_path: os.PathLike | None
 ) -> T:
@@ -61,17 +69,35 @@ def _get_info(
     return model.model_validate(yaml.safe_load(info.read_text()))
 
 
-def get_cluster_info(name: str, flepi_path: os.PathLike | None = None) -> Cluster:
+def get_cluster_info(name: str | None, flepi_path: os.PathLike | None = None) -> Cluster:
     """
     Get cluster specific info.
 
     Args:
         name: The name of the cluster to pull information for. Currently only 'longleaf'
-            and 'rockfish' are supported.
+            and 'rockfish' are supported or `None` to infer from the FQDN.
         flepi_path: Either a path like determine the directory to look for the info
             directory in or `None` to use the `FLEPI_PATH` environment variable.
 
     Returns
         An object containing the information about the `name` cluster.
     """
+    name = _infer_cluster_from_fqdn() if name is None else name
     return _get_info("cluster", name, Cluster, flepi_path)
+
+
+def _infer_cluster_from_fqdn() -> str:
+    """
+    Infer the cluster name from the FQDN.
+
+    Returns:
+        The name of the cluster inferred from the FQDN.
+
+    Raises:
+        ValueError: If the value of `socket.getfqdn()` does not match an expected regex.
+    """
+    fqdn = getfqdn()
+    for cluster, regex in _CLUSTER_FQDN_REGEXES:
+        if regex.match(fqdn):
+            return cluster
+    raise ValueError(f"The fqdn, '{fqdn}', does not match any of the expected clusters.")
diff --git a/flepimop/gempyor_pkg/tests/batch/test__click_submit.py b/flepimop/gempyor_pkg/tests/batch/test__click_submit.py
index 31582c9f6..993c0a8c1 100644
--- a/flepimop/gempyor_pkg/tests/batch/test__click_submit.py
+++ b/flepimop/gempyor_pkg/tests/batch/test__click_submit.py
@@ -25,6 +25,10 @@ def test_batch_system_aws_not_implemented_error(tmp_path: Path) -> None:
             "1",
             "--blocks",
             "1",
+            "--flepi-path",
+            str(tmp_path.absolute()),
+            "--project-path",
+            str(tmp_path.absolute()),
             str(config_file.absolute()),
         ],
     )
@@ -35,31 +39,3 @@ def test_batch_system_aws_not_implemented_error(tmp_path: Path) -> None:
         str(result.exception)
         == "The `flepimop submit` CLI does not support batch submission to AWS yet."
     )
-
-
-def test_cluster_required_for_slurm_value_error(tmp_path: Path) -> None:
-    config_file = tmp_path / "config.yml"
-    with config_file.open(mode="w") as f:
-        yaml.dump({"name": "foobar", "inference": {"method": "emcee"}}, f)
-
-    runner = CliRunner()
-    result = runner.invoke(
-        _click_submit,
-        [
-            "--slurm",
-            "--simulations",
-            "1",
-            "--jobs",
-            "1",
-            "--blocks",
-            "1",
-            str(config_file.absolute()),
-        ],
-    )
-
-    assert result.exit_code == 1
-    assert isinstance(result.exception, ValueError)
-    assert (
-        str(result.exception)
-        == "When submitting a batch job to slurm a cluster is required."
-    )
diff --git a/flepimop/gempyor_pkg/tests/info/test__infer_cluster_from_fqdn.py b/flepimop/gempyor_pkg/tests/info/test__infer_cluster_from_fqdn.py
new file mode 100644
index 000000000..a26705503
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/info/test__infer_cluster_from_fqdn.py
@@ -0,0 +1,37 @@
+from unittest.mock import patch
+
+import pytest
+
+from gempyor.info import _infer_cluster_from_fqdn
+
+
+@pytest.mark.parametrize("fqdn", ("new.cluster.com", "unsupported.cluster"))
+def test_no_matching_fqdn_found_value_error(fqdn: str) -> None:
+    def socket_fqdn_wraps() -> str:
+        return fqdn
+
+    with patch("gempyor.info.getfqdn", wraps=socket_fqdn_wraps) as socket_fqdn_patch:
+        with pytest.raises(
+            ValueError,
+            match=f"^The fqdn, '{fqdn}', does not match any of the expected clusters.$",
+        ):
+            _infer_cluster_from_fqdn()
+        socket_fqdn_patch.assert_called_once()
+
+
+@pytest.mark.parametrize(
+    ("fqdn", "expected"),
+    (
+        ("login01.cm.cluster", "rockfish"),
+        ("login3.cm.cluster", "rockfish"),
+        ("longleaf-login1.its.unc.edu", "longleaf"),
+        ("longleaf-login07.its.unc.edu", "longleaf"),
+    ),
+)
+def test_exact_results_for_select_values(fqdn: str, expected: str) -> None:
+    def socket_fqdn_wraps() -> str:
+        return fqdn
+
+    with patch("gempyor.info.getfqdn", wraps=socket_fqdn_wraps) as socket_fqdn_patch:
+        assert _infer_cluster_from_fqdn() == expected
+        socket_fqdn_patch.assert_called_once()
diff --git a/flepimop/gempyor_pkg/tests/info/test_get_cluster_info.py b/flepimop/gempyor_pkg/tests/info/test_get_cluster_info.py
index b8ab64436..66fcd9836 100644
--- a/flepimop/gempyor_pkg/tests/info/test_get_cluster_info.py
+++ b/flepimop/gempyor_pkg/tests/info/test_get_cluster_info.py
@@ -1,4 +1,5 @@
 import os
+from unittest.mock import patch
 
 import pytest
 
@@ -10,6 +11,23 @@
     os.getenv("FLEPI_PATH") is None,
     reason="The $FLEPI_PATH environment variable is not set.",
 )
-def test_output_validation(name: str) -> None:
+def test_exact_results_given_cluster_name(name: str) -> None:
     cluster = get_cluster_info(name)
     assert isinstance(cluster, Cluster)
+
+
+@pytest.mark.parametrize("name", ("longleaf", "rockfish"))
+@pytest.mark.skipif(
+    os.getenv("FLEPI_PATH") is None,
+    reason="The $FLEPI_PATH environment variable is not set.",
+)
+def test_exact_results_when_inferred_from_fqdn(name: str) -> None:
+    def infer_cluster_from_fqdn_wraps() -> str:
+        return name
+
+    with patch(
+        "gempyor.info._infer_cluster_from_fqdn", wraps=infer_cluster_from_fqdn_wraps
+    ) as infer_cluster_from_fqdn_patch:
+        cluster = get_cluster_info(None)
+        assert isinstance(cluster, Cluster)
+        infer_cluster_from_fqdn_patch.assert_called_once()
diff --git a/flepimop/gempyor_pkg/tests/templates/test_cluster_setup_bash.py b/flepimop/gempyor_pkg/tests/templates/test_cluster_setup_bash.py
index 89048444a..90e1e4cc2 100644
--- a/flepimop/gempyor_pkg/tests/templates/test_cluster_setup_bash.py
+++ b/flepimop/gempyor_pkg/tests/templates/test_cluster_setup_bash.py
@@ -1,3 +1,4 @@
+import os
 from typing import Any
 
 import pytest
@@ -6,15 +7,13 @@
 from gempyor.info import Cluster, Module, PathExport, get_cluster_info
 
 
-@pytest.mark.parametrize(
-    "cluster",
-    (
-        None,
-        get_cluster_info("longleaf").model_dump(),
-        get_cluster_info("rockfish").model_dump(),
-    ),
+@pytest.mark.parametrize("cluster", (None, "longleaf", "rockfish"))
+@pytest.mark.skipif(
+    os.getenv("FLEPI_PATH") is None,
+    reason="The $FLEPI_PATH environment variable is not set.",
 )
-def test_output_validation(cluster: dict[str, Any]) -> None:
+def test_output_validation(cluster: str | None) -> None:
+    cluster = cluster if cluster is None else get_cluster_info(cluster).model_dump()
     rendered_template = _render_template("cluster_setup.bash.j2", {"cluster": cluster})
     lines = rendered_template.split("\n")
     assert "module purge" in lines

From 821210ddf245b7ac8f17721dec65a50bcd98d8db Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Thu, 14 Nov 2024 14:41:42 -0500
Subject: [PATCH 62/78] Add `--samples` option to `flepimop submit`

---
 flepimop/gempyor_pkg/src/gempyor/batch.py             | 11 ++++++++++-
 .../src/gempyor/templates/inference_command.bash.j2   |  1 +
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index b2d56c99f..5d4b7ca3c 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -27,7 +27,7 @@
 
 from ._jinja import _render_template_to_file, _render_template_to_temp_file
 from .file_paths import run_id
-from .info import Cluster, get_cluster_info
+from .info import get_cluster_info
 from .logging import get_script_logger
 from .utils import _format_cli_options, _git_checkout, _git_head, _shutil_which, config
 from .shared_cli import (
@@ -764,6 +764,9 @@ def _submit_scenario_job(
         "job_name": job_name,
         "jobs": job_size.jobs,
         "outcome_modifiers_scenario": outcome_modifiers_scenario,
+        "nsamples": (
+            job_size.simulations if kwargs["samples"] is None else kwargs["samples"]
+        ),
         "nslots": job_size.simulations,  # aka nwalkers
         "prefix": prefix,
         "project_path": kwargs["project_path"].absolute(),
@@ -844,6 +847,12 @@ def _submit_scenario_job(
             type=click.IntRange(min=1),
             help="The number of sequential blocks to run per a job.",
         ),
+        click.Option(
+            param_decls=["--samples", "samples"],
+            default=None,
+            type=click.IntRange(min=1),
+            help="The number of samples to produce. Defaults to all samples.",
+        ),
         click.Option(
             param_decls=["--batch-system", "batch_system"],
             default=None,
diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/inference_command.bash.j2 b/flepimop/gempyor_pkg/src/gempyor/templates/inference_command.bash.j2
index 9a348f259..8763df704 100644
--- a/flepimop/gempyor_pkg/src/gempyor/templates/inference_command.bash.j2
+++ b/flepimop/gempyor_pkg/src/gempyor/templates/inference_command.bash.j2
@@ -4,6 +4,7 @@ flepimop-calibrate --config $CONFIG_PATH \
     --project_path $PROJECT_PATH \
     --nslots {{ nslots }} \
     --niterations {{ simulations }} \
+    --nsamples {{ nsamples }} \
     --jobs {{ jobs }} \
     --id {{ run_id }} \
     --prefix {{ prefix}} \

From 9353b665db64535870b4b1090d2a545ae340c7aa Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Thu, 14 Nov 2024 16:41:20 -0500
Subject: [PATCH 63/78] Add detailed documentation for `flepimop submit`

---
 flepimop/gempyor_pkg/src/gempyor/batch.py | 84 +++++++++++++++++++----
 1 file changed, 70 insertions(+), 14 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 5d4b7ca3c..bac7dc41f 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -1002,7 +1002,63 @@ def _submit_scenario_job(
 )
 @click.pass_context
 def _click_submit(ctx: click.Context = mock_context, **kwargs: Any) -> None:
-    """Submit batch jobs"""
+    """
+    Submit batch inference jobs.
+
+    This CLI tool makes it a breeze to submit batch inference jobs. In general, this
+    tool does:
+
+    1. Determine information about the job such as the inference method, batch system,
+        job size/time limit, and resources.
+    2. Write metadata about the batch job to a 'manifest.json' file.
+    3. Loop over the outcome/seir scenario modifiers and generate/submit batch jobs for
+        each of those.
+    4. Checkout a new branch in the project directory to preserve the outputs.
+
+    To get a better understanding of this tool you can use the `--dry-run` flag which
+    will complete all of steps described above except for submitting the jobs. Or if you
+    would like to test run the batch scripts without submitting to slurm or AWS you can
+    use the `--local` flag which will run the "batch" job locally (only use for small
+    test jobs).
+
+    Here are some common examples to get started:
+
+    1. Assuming you are in your clone of the `HopkinsIDD/flepimop_sample` repository
+    this command will do a dry run of the sample inference configuration with two
+    populations.
+
+    \b
+    ```bash
+    $ flepimop submit --jobs 4 \ # With legacy inference jobs refers to chains
+        --simulations 20 \       # The number of iterations per a chain
+        --blocks 1 \             # The number of consecutive blocks to run
+        --slurm \                # Use slurm, shorthand for `--batch-system slurm`
+        --project-path $(pwd) \  # Manually specify the project path
+        --email me@college.edu \ # Use slurm's built-in email alerts for the job
+        --skip-checkout \        # Do not create a new branch in the project repo
+        -vvv \                   # Use the max verbosity
+        --dry-run \              # Do not actually submit the job via `sbatch`.
+        config_sample_2pop_inference.yml
+    ```
+
+    2. Extending on example (1) let's make some small modifications to submit a "real"
+    job.
+
+    \b
+    ```bash
+    $ flepimop submit --jobs 10 \               # See before
+        --simulations 400 \                     # See before
+        --blocks 1 \                            # See before
+        --slurm \                               # See before
+        --partition dedicated_slurm_partition \ # Submit to a particular slurm partition
+        --simulation-time 30s \                 # Specify the time limit per simulation
+        --initial-time 5min \                   # Specify the initial time limit
+        --conda-env custom-flepimop-env \       # Specify a custom conda env to use
+        --cpus 4 \                              # Use 4 CPUs per a job
+        -vvv \                                  # Use the max verbosity
+        config_sample_2pop_inference.yml
+    ```
+    """
     # Generic setup
     now = datetime.now(timezone.utc)
     logger = get_script_logger(__name__, kwargs["verbosity"])
@@ -1082,19 +1138,6 @@ def _click_submit(ctx: click.Context = mock_context, **kwargs: Any) -> None:
     )
     logger.info("Requesting the resources %s for this job.", job_resources)
 
-    # Outcome/seir modifier scenarios
-    outcome_modifiers_scenarios = (
-        cfg["outcome_modifiers"]["scenarios"].as_str_seq()
-        if cfg["outcome_modifiers"].exists()
-        and cfg["outcome_modifiers"]["scenarios"].exists()
-        else [None]
-    )
-    seir_modifiers_scenarios = (
-        cfg["seir_modifiers"]["scenarios"].as_str_seq()
-        if cfg["seir_modifiers"].exists() and cfg["seir_modifiers"]["scenarios"].exists()
-        else [None]
-    )
-
     # Restart/continuation location
     # TODO: Implement this
 
@@ -1119,6 +1162,19 @@ def _click_submit(ctx: click.Context = mock_context, **kwargs: Any) -> None:
         "Dumped the final config for this batch submission to %s", config_out.absolute()
     )
 
+    # Outcome/seir modifier scenarios
+    outcome_modifiers_scenarios = (
+        cfg["outcome_modifiers"]["scenarios"].as_str_seq()
+        if cfg["outcome_modifiers"].exists()
+        and cfg["outcome_modifiers"]["scenarios"].exists()
+        else [None]
+    )
+    seir_modifiers_scenarios = (
+        cfg["seir_modifiers"]["scenarios"].as_str_seq()
+        if cfg["seir_modifiers"].exists() and cfg["seir_modifiers"]["scenarios"].exists()
+        else [None]
+    )
+
     # Loop over the scenarios
     for outcome_modifiers_scenario, seir_modifiers_scenario in product(
         outcome_modifiers_scenarios, seir_modifiers_scenarios

From a56a4a8984b34bdc63f4fb171131fdcb94eeb852 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Thu, 14 Nov 2024 17:29:27 -0500
Subject: [PATCH 64/78] Add custom click param type for memory

Also added unit tests, but still need to add documentation.
---
 .../gempyor_pkg/src/gempyor/shared_cli.py     | 47 ++++++++++++++++
 .../test_memory_param_type_class.py           | 56 +++++++++++++++++++
 2 files changed, 103 insertions(+)
 create mode 100644 flepimop/gempyor_pkg/tests/shared_cli/test_memory_param_type_class.py

diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py
index 805de36c4..d47d6de96 100644
--- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py
+++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py
@@ -199,6 +199,53 @@ def convert(
 NONNEGATIVE_DURATION = DurationParamType(nonnegative=True)
 
 
+class MemoryParamType(click.ParamType):
+    name = "memory"
+    _units = {
+        "kb": 1024.0**1.0,
+        "k": 1024.0**1.0,
+        "mb": 1024.0**2.0,
+        "m": 1024.0**2.0,
+        "gb": 1024.0**3.0,
+        "g": 1024.0**3.0,
+        "t": 1024.0**4.0,
+        "tb": 1024.0**4.0,
+    }
+
+    def __init__(self, unit: str) -> None:
+        super().__init__()
+        if (unit := unit.lower()) not in self._units.keys():
+            raise ValueError(
+                f"The `unit` given is not valid, given '{unit}' and "
+                "must be one of: {', '.join(self._units.keys())}."
+            )
+        self._unit = unit
+        self._regex = re.compile(
+            rf"^(([0-9]+)?(\.[0-9]+)?)({'|'.join(self._units.keys())})?$",
+            flags=re.IGNORECASE,
+        )
+
+    def convert(
+        self, value: Any, param: click.Parameter | None, ctx: click.Context | None
+    ) -> float:
+        value = str(value).strip()
+        if (m := self._regex.match(value)) is None:
+            self.fail(f"{value!r} is not a valid memory size.", param, ctx)
+        number, _, _, unit = m.groups()
+        unit = unit.lower()
+        if unit == self._unit:
+            return float(number)
+        return (self._units.get(unit, self._unit) * float(number)) / (
+            self._units.get(self._unit)
+        )
+
+
+MEMORY_KB = MemoryParamType("kb")
+MEMORY_MB = MemoryParamType("mb")
+MEMORY_GB = MemoryParamType("gb")
+MEMORY_TB = MemoryParamType("tb")
+
+
 def click_helpstring(
     params: click.Parameter | list[click.Parameter],
 ) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
diff --git a/flepimop/gempyor_pkg/tests/shared_cli/test_memory_param_type_class.py b/flepimop/gempyor_pkg/tests/shared_cli/test_memory_param_type_class.py
new file mode 100644
index 000000000..1db89a5a7
--- /dev/null
+++ b/flepimop/gempyor_pkg/tests/shared_cli/test_memory_param_type_class.py
@@ -0,0 +1,56 @@
+import random
+from typing import Any
+
+from click.exceptions import BadParameter
+import pytest
+
+from gempyor.shared_cli import MemoryParamType
+
+
+@pytest.mark.parametrize("unit", ("Nope", "NO CHANCE", "wrong", "bb"))
+def test_invalid_unit_value_error(unit: str) -> None:
+    with pytest.raises(
+        ValueError,
+        match=(
+            "^The `unit` given is not valid, given "
+            f"'{unit.lower()}' and must be one of:.*.$"
+        ),
+    ):
+        MemoryParamType(unit)
+
+
+@pytest.mark.parametrize("value", ("1..2MB", "3.4cb", "56.abc", "-1GB"))
+def test_invalid_value_bad_parameter(value: Any) -> None:
+    memory = MemoryParamType("mb")
+    with pytest.raises(BadParameter, match="^.* is not a valid memory size.$"):
+        memory.convert(value, None, None)
+
+
+@pytest.mark.parametrize("unit", MemoryParamType._units.keys())
+@pytest.mark.parametrize(
+    "number",
+    [random.randint(1, 1000) for _ in range(3)]  # int
+    + [random.random() for _ in range(3)]  # float without numbers left of decimal
+    + [
+        random.randint(1, 25) + random.random() for _ in range(3)
+    ],  # float with numbers left of the decimal
+)
+def test_convert_acts_as_identity(unit: str, number: int) -> None:
+    memory = MemoryParamType(unit)
+    assert memory.convert(f"{number}{unit}".lstrip("0"), None, None) == number
+    assert memory.convert(f"{number}{unit.upper()}".lstrip("0"), None, None) == number
+
+
+@pytest.mark.parametrize(
+    ("unit", "value", "expected"),
+    (
+        ("gb", "1.2gb", 1.2),
+        ("kb", "1mb", 1024.0),
+        ("gb", "30mb", 30.0 / 1024.0),
+        ("kb", "2tb", 2.0 * (1024.0**3.0)),
+        ("mb", "0.1gb", 0.1 * 1024.0),
+    ),
+)
+def test_exact_results_for_select_inputs(unit: str, value: Any, expected: float) -> None:
+    memory = MemoryParamType(unit)
+    assert memory.convert(value, None, None) == expected

From b3961f631e354e14e19b27eff35d51c2d0e70b86 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Thu, 14 Nov 2024 17:35:14 -0500
Subject: [PATCH 65/78] Replace memory option `IntRange` with `MEMORY_MB`

---
 flepimop/gempyor_pkg/src/gempyor/batch.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index bac7dc41f..ac90d7692 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -31,6 +31,7 @@
 from .logging import get_script_logger
 from .utils import _format_cli_options, _git_checkout, _git_head, _shutil_which, config
 from .shared_cli import (
+    MEMORY_MB,
     NONNEGATIVE_DURATION,
     cli,
     config_files_argument,
@@ -970,7 +971,7 @@ def _submit_scenario_job(
         ),
         click.Option(
             param_decls=["--memory", "memory"],
-            type=click.IntRange(min=1),
+            type=MEMORY_MB,
             default=None,
             help="Override for the amount of memory per node to use in MB.",
         ),
@@ -1129,12 +1130,19 @@ def _click_submit(ctx: click.Context = mock_context, **kwargs: Any) -> None:
     logger.info("Setting a total job time limit of %s minutes", job_time_limit.format())
 
     # Job resources
+    memory = None if kwargs["memory"] is None else math.ceil(kwargs["memory"])
+    if memory != kwargs["memory"]:
+        logger.warning(
+            "The requested memory of %.3fMB has been rounded up to %uMB for submission",
+            kwargs["memory"],
+            memory,
+        )
     job_resources = JobResources.from_presets(
         job_size,
         inference_method,
         nodes=kwargs["nodes"],
         cpus=kwargs["cpus"],
-        memory=kwargs["memory"],
+        memory=memory,
     )
     logger.info("Requesting the resources %s for this job.", job_resources)
 

From 0fa1f9b71cca8d9a4a75023bd8a3c392bb1beb31 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Fri, 15 Nov 2024 09:23:02 -0500
Subject: [PATCH 66/78] Document info/shared_cli, MemoryParamType int

* Fleshed out documentation for the `gempyor.info/shared_cli` modules.
* Added the ability to optionally return the converted memory as an int
  instead of as a float to `MemoryParamType`.
* Removed pre-instantiated custom click param types.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py     |  21 ++--
 flepimop/gempyor_pkg/src/gempyor/info.py      |  74 +++++++++++-
 .../gempyor_pkg/src/gempyor/shared_cli.py     | 114 +++++++++++++++---
 .../test_memory_param_type_class.py           |  34 ++++--
 4 files changed, 197 insertions(+), 46 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index ac90d7692..701828662 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -31,11 +31,11 @@
 from .logging import get_script_logger
 from .utils import _format_cli_options, _git_checkout, _git_head, _shutil_which, config
 from .shared_cli import (
-    MEMORY_MB,
-    NONNEGATIVE_DURATION,
+    DurationParamType,
+    MemoryParamType,
     cli,
-    config_files_argument,
     config_file_options,
+    config_files_argument,
     log_cli_inputs,
     mock_context,
     parse_config_files,
@@ -903,13 +903,13 @@ def _submit_scenario_job(
         ),
         click.Option(
             param_decls=["--simulation-time", "simulation_time"],
-            type=NONNEGATIVE_DURATION,
+            type=DurationParamType(True),
             default="3min",
             help="The time limit per a simulation.",
         ),
         click.Option(
             param_decls=["--initial-time", "initial_time"],
-            type=NONNEGATIVE_DURATION,
+            type=DurationParamType(True),
             default="20min",
             help="The initialization time limit.",
         ),
@@ -971,7 +971,7 @@ def _submit_scenario_job(
         ),
         click.Option(
             param_decls=["--memory", "memory"],
-            type=MEMORY_MB,
+            type=MemoryParamType("mb", as_int=True),
             default=None,
             help="Override for the amount of memory per node to use in MB.",
         ),
@@ -1130,19 +1130,12 @@ def _click_submit(ctx: click.Context = mock_context, **kwargs: Any) -> None:
     logger.info("Setting a total job time limit of %s minutes", job_time_limit.format())
 
     # Job resources
-    memory = None if kwargs["memory"] is None else math.ceil(kwargs["memory"])
-    if memory != kwargs["memory"]:
-        logger.warning(
-            "The requested memory of %.3fMB has been rounded up to %uMB for submission",
-            kwargs["memory"],
-            memory,
-        )
     job_resources = JobResources.from_presets(
         job_size,
         inference_method,
         nodes=kwargs["nodes"],
         cpus=kwargs["cpus"],
-        memory=memory,
+        memory=kwargs["memory"],
     )
     logger.info("Requesting the resources %s for this job.", job_resources)
 
diff --git a/flepimop/gempyor_pkg/src/gempyor/info.py b/flepimop/gempyor_pkg/src/gempyor/info.py
index 00d094c09..bdcdc7b36 100644
--- a/flepimop/gempyor_pkg/src/gempyor/info.py
+++ b/flepimop/gempyor_pkg/src/gempyor/info.py
@@ -1,3 +1,31 @@
+"""
+Retrieving static information from developer managed yaml files.
+
+Currently, it includes utilities for handling cluster-specific information, but it can 
+be extended to other categories as needed.
+
+Classes:
+    Module: Represents a software module with a name and optional version.
+    PathExport: Represents a path export with a path, prepend flag, and error handling.
+    Cluster: Represents a cluster with a name, list of modules, and list of path 
+        exports.
+
+Functions:
+    get_cluster_info: Retrieves cluster-specific information.
+
+Examples:
+    >>> from pprint import pprint
+    >>> from gempyor.info import get_cluster_info
+    >>> cluster_info = get_cluster_info("longleaf")
+    >>> cluster_info.name
+    'longleaf'
+    >>> pprint(cluster_info.modules)
+    [Module(name='gcc', version='9.1.0'),
+    Module(name='anaconda', version='2023.03'),
+    Module(name='git', version=None),
+    Module(name='aws', version=None)]
+"""
+
 __all__ = ["Cluster", "Module", "PathExport", "get_cluster_info"]
 
 
@@ -12,23 +40,55 @@
 
 
 class Module(BaseModel):
+    """
+    A model representing a module to load.
+
+    Attributes:
+        name: The name of the module to load.
+        version: The specific version of the module to load if there is one.
+
+    See Also:
+        [Lmod](https://lmod.readthedocs.io/en/latest/)
+    """
+
     name: str
     version: str | None = None
 
 
 class PathExport(BaseModel):
+    """
+    A model representing the export path configuration.
+
+    Attributes:
+        path: The file system path of the path to add to the `$PATH` environment
+            variable.
+        prepend: A flag indicating whether to prepend additional information to the
+            `$PATH` environment variable.
+        error_if_missing: A flag indicating whether to raise an error if the path is
+            missing.
+    """
+
     path: Path
     prepend: bool = True
     error_if_missing: bool = False
 
 
 class Cluster(BaseModel):
+    """
+    A model representing a cluster configuration.
+
+    Attributes:
+        name: The name of the cluster.
+        modules: A list of modules associated with the cluster.
+        path_exports: A list of path exports for the cluster.
+    """
+
     name: str
     modules: list[Module] = []
     path_exports: list[PathExport] = []
 
 
-T = TypeVar("T", bound=BaseModel)
+_BASE_MODEL_TYPE = TypeVar("T", bound=BaseModel)
 
 
 _CLUSTER_FQDN_REGEXES: tuple[tuple[str, Pattern], ...] = (
@@ -38,8 +98,8 @@ class Cluster(BaseModel):
 
 
 def _get_info(
-    category: str, name: str, model: type[T], flepi_path: os.PathLike | None
-) -> T:
+    category: str, name: str, model: type[_BASE_MODEL_TYPE], flepi_path: os.PathLike | None
+) -> _BASE_MODEL_TYPE:
     """
     Get and parse an information yaml file.
 
@@ -79,8 +139,14 @@ def get_cluster_info(name: str | None, flepi_path: os.PathLike | None = None) ->
         flepi_path: Either a path like determine the directory to look for the info
             directory in or `None` to use the `FLEPI_PATH` environment variable.
 
-    Returns
+    Returns:
         An object containing the information about the `name` cluster.
+
+    Examples:
+        >>> from gempyor.info import get_cluster_info
+        >>> cluster_info = get_cluster_info("longleaf")
+        >>> cluster_info.name
+        'longleaf'
     """
     name = _infer_cluster_from_fqdn() if name is None else name
     return _get_info("cluster", name, Cluster, flepi_path)
diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py
index d47d6de96..230839784 100644
--- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py
+++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py
@@ -7,6 +7,7 @@
 
 
 from datetime import timedelta
+from math import ceil
 import multiprocessing
 import pathlib
 import re
@@ -143,6 +144,23 @@ def cli(ctx: click.Context) -> None:
 
 
 class DurationParamType(click.ParamType):
+    """
+    A custom Click parameter type for parsing duration strings into `timedelta` objects.
+
+    Attributes:
+        name: The name of the parameter type.
+
+    Examples:
+        >>> from gempyor.shared_cli import DurationParamType
+        >>> duration_param_type = DurationParamType(False)
+        >>> duration_param_type.convert("23min", None, None)
+        datetime.timedelta(seconds=1380)
+        >>> duration_param_type.convert("2.5hr", None, None)
+        datetime.timedelta(seconds=9000)
+        >>> duration_param_type.convert("-2", None, None)
+        datetime.timedelta(days=-1, seconds=86280)
+    """
+
     name = "duration"
     _abbreviations = {
         "s": "seconds",
@@ -173,6 +191,14 @@ def __init__(
         nonnegative: bool,
         default_unit: Literal["seconds", "minutes", "hours", "days", "weeks"] = "minutes",
     ) -> None:
+        """
+        Initialize the instance based on parameter settings.
+
+        Args:
+            nonnegative: If `True` negative durations are not allowed.
+            default_unit: The default unit to use if no unit is specified in the input
+                string.
+        """
         super().__init__()
         self._nonnegative = nonnegative
         self._duration_regex = re.compile(
@@ -184,6 +210,24 @@ def __init__(
     def convert(
         self, value: Any, param: click.Parameter | None, ctx: click.Context | None
     ) -> timedelta:
+        """
+        Converts a string representation of a duration into a `timedelta` object.
+
+        Args:
+            value: The value to convert, expected to be a string like representation of
+                a duration.
+            param: The Click parameter object for context in errors.
+            ctx: The Click context object for context in errors.
+
+        Returns:
+            The converted duration as a `timedelta` object.
+
+        Raises:
+            click.BadParameter: If the value is not a valid duration based on the
+                format.
+            click.BadParameter: If the duration is negative and the class was
+                initialized with `nonnegative` set to `True`.
+        """
         value = str(value).strip()
         if (m := self._duration_regex.match(value)) is None:
             self.fail(f"{value!r} is not a valid duration", param, ctx)
@@ -195,11 +239,24 @@ def convert(
         return timedelta(**kwargs)
 
 
-DURATION = DurationParamType(nonnegative=False)
-NONNEGATIVE_DURATION = DurationParamType(nonnegative=True)
+class MemoryParamType(click.ParamType):
+    """
+    A custom Click parameter type for parsing duration strings into `timedelta` objects.
 
+    Attributes:
+        name: The name of the parameter type.
+
+    Examples:
+        >>> from gempyor.shared_cli import DurationParamType
+        >>> duration_param_type = DurationParamType(False)
+        >>> duration_param_type.convert("23min", None, None)
+        datetime.timedelta(seconds=1380)
+        >>> duration_param_type.convert("2.5hr", None, None)
+        datetime.timedelta(seconds=9000)
+        >>> duration_param_type.convert("-2", None, None)
+        datetime.timedelta(days=-1, seconds=86280)
+    """
 
-class MemoryParamType(click.ParamType):
     name = "memory"
     _units = {
         "kb": 1024.0**1.0,
@@ -212,38 +269,63 @@ class MemoryParamType(click.ParamType):
         "tb": 1024.0**4.0,
     }
 
-    def __init__(self, unit: str) -> None:
+    def __init__(self, unit: str, as_int: bool = False) -> None:
+        """
+        Initialize the instance based on parameter settings.
+
+        Args:
+            unit: The output unit to use in the `convert` method.
+            as_int: if `True` the `convert` method returns an integer instead of a
+                float.
+
+        Raises:
+            ValueError: If `unit` is not a valid memory unit size.
+        """
         super().__init__()
         if (unit := unit.lower()) not in self._units.keys():
             raise ValueError(
                 f"The `unit` given is not valid, given '{unit}' and "
-                "must be one of: {', '.join(self._units.keys())}."
+                f"must be one of: {', '.join(self._units.keys())}."
             )
         self._unit = unit
         self._regex = re.compile(
             rf"^(([0-9]+)?(\.[0-9]+)?)({'|'.join(self._units.keys())})?$",
             flags=re.IGNORECASE,
         )
+        self._as_int = as_int
 
     def convert(
         self, value: Any, param: click.Parameter | None, ctx: click.Context | None
-    ) -> float:
+    ) -> float | int:
+        """
+        Converts a string representation of a memory size into a numeric.
+
+        Args:
+            value: The value to convert, expected to be a string like representation of
+                memory size.
+            param: The Click parameter object for context in errors.
+            ctx: The Click context object for context in errors.
+
+        Returns:
+            The converted memory size as a numeric. Specifically an integer if the
+            `as_int` attribute is `True` and float otherwise.
+
+        Raises:
+            click.BadParameter: If the value is not a valid memory size based on the
+                format.
+        """
         value = str(value).strip()
         if (m := self._regex.match(value)) is None:
             self.fail(f"{value!r} is not a valid memory size.", param, ctx)
         number, _, _, unit = m.groups()
         unit = unit.lower()
         if unit == self._unit:
-            return float(number)
-        return (self._units.get(unit, self._unit) * float(number)) / (
-            self._units.get(self._unit)
-        )
-
-
-MEMORY_KB = MemoryParamType("kb")
-MEMORY_MB = MemoryParamType("mb")
-MEMORY_GB = MemoryParamType("gb")
-MEMORY_TB = MemoryParamType("tb")
+            result = float(number)
+        else:
+            result = (self._units.get(unit, self._unit) * float(number)) / (
+                self._units.get(self._unit)
+            )
+        return ceil(result) if self._as_int else result
 
 
 def click_helpstring(
diff --git a/flepimop/gempyor_pkg/tests/shared_cli/test_memory_param_type_class.py b/flepimop/gempyor_pkg/tests/shared_cli/test_memory_param_type_class.py
index 1db89a5a7..f3c60e504 100644
--- a/flepimop/gempyor_pkg/tests/shared_cli/test_memory_param_type_class.py
+++ b/flepimop/gempyor_pkg/tests/shared_cli/test_memory_param_type_class.py
@@ -27,6 +27,7 @@ def test_invalid_value_bad_parameter(value: Any) -> None:
 
 
 @pytest.mark.parametrize("unit", MemoryParamType._units.keys())
+@pytest.mark.parametrize("as_int", (True, False))
 @pytest.mark.parametrize(
     "number",
     [random.randint(1, 1000) for _ in range(3)]  # int
@@ -35,22 +36,31 @@ def test_invalid_value_bad_parameter(value: Any) -> None:
         random.randint(1, 25) + random.random() for _ in range(3)
     ],  # float with numbers left of the decimal
 )
-def test_convert_acts_as_identity(unit: str, number: int) -> None:
-    memory = MemoryParamType(unit)
-    assert memory.convert(f"{number}{unit}".lstrip("0"), None, None) == number
-    assert memory.convert(f"{number}{unit.upper()}".lstrip("0"), None, None) == number
+def test_convert_acts_as_identity(unit: str, as_int: bool, number: int | float) -> None:
+    memory = MemoryParamType(unit, as_int=as_int)
+    for u in (unit, unit.upper()):
+        result = memory.convert(f"{number}{u}".lstrip("0"), None, None)
+        assert isinstance(result, int if as_int else float)
+        assert abs(result - number) <= 1 if as_int else result == number
 
 
 @pytest.mark.parametrize(
-    ("unit", "value", "expected"),
+    ("unit", "as_int", "value", "expected"),
     (
-        ("gb", "1.2gb", 1.2),
-        ("kb", "1mb", 1024.0),
-        ("gb", "30mb", 30.0 / 1024.0),
-        ("kb", "2tb", 2.0 * (1024.0**3.0)),
-        ("mb", "0.1gb", 0.1 * 1024.0),
+        ("gb", False, "1.2gb", 1.2),
+        ("gb", True, "1.2gb", 2),
+        ("kb", False, "1mb", 1024.0),
+        ("kb", True, "1mb", 1024),
+        ("gb", False, "30mb", 30.0 / 1024.0),
+        ("gb", True, "30mb", 1),
+        ("kb", False, "2tb", 2.0 * (1024.0**3.0)),
+        ("kb", True, "2tb", 2147483648),
+        ("mb", False, "0.1gb", 0.1 * 1024.0),
+        ("mb", True, "0.1gb", 103),
     ),
 )
-def test_exact_results_for_select_inputs(unit: str, value: Any, expected: float) -> None:
-    memory = MemoryParamType(unit)
+def test_exact_results_for_select_inputs(
+    unit: str, as_int: bool, value: Any, expected: float | int
+) -> None:
+    memory = MemoryParamType(unit, as_int=as_int)
     assert memory.convert(value, None, None) == expected

From 6e38dd2aeedaedfdfb3992ca40be6362788e33dc Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Mon, 25 Nov 2024 13:56:24 -0500
Subject: [PATCH 67/78] Assume unitless numbers are in default unit

Bug in `MemoryParamType.convert` was failing if a unit was not given
with a value. Similar to `DurationParamType` it assumes that numbers
without a unit are given in the default unit of the type.
---
 flepimop/gempyor_pkg/src/gempyor/shared_cli.py                | 2 +-
 .../tests/shared_cli/test_memory_param_type_class.py          | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py
index 230839784..4d613496e 100644
--- a/flepimop/gempyor_pkg/src/gempyor/shared_cli.py
+++ b/flepimop/gempyor_pkg/src/gempyor/shared_cli.py
@@ -318,7 +318,7 @@ def convert(
         if (m := self._regex.match(value)) is None:
             self.fail(f"{value!r} is not a valid memory size.", param, ctx)
         number, _, _, unit = m.groups()
-        unit = unit.lower()
+        unit = self._unit if unit is None else unit.lower()
         if unit == self._unit:
             result = float(number)
         else:
diff --git a/flepimop/gempyor_pkg/tests/shared_cli/test_memory_param_type_class.py b/flepimop/gempyor_pkg/tests/shared_cli/test_memory_param_type_class.py
index f3c60e504..53c3558ce 100644
--- a/flepimop/gempyor_pkg/tests/shared_cli/test_memory_param_type_class.py
+++ b/flepimop/gempyor_pkg/tests/shared_cli/test_memory_param_type_class.py
@@ -57,6 +57,10 @@ def test_convert_acts_as_identity(unit: str, as_int: bool, number: int | float)
         ("kb", True, "2tb", 2147483648),
         ("mb", False, "0.1gb", 0.1 * 1024.0),
         ("mb", True, "0.1gb", 103),
+        ("gb", False, "4", 4.0),
+        ("gb", True, "4", 4),
+        ("mb", False, "1234.56", 1234.56),
+        ("mb", True, "1234.56", 1235),
     ),
 )
 def test_exact_results_for_select_inputs(

From 4f936db20d7870ae5ae0070e85a18bba5e38be87 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Mon, 25 Nov 2024 16:29:50 -0500
Subject: [PATCH 68/78] Move `get_cluster_info` call into `_click_submit`

Moved from the slurm specific code path in `_submit_scenario_job`,
should allow for using the `--local` flag on the cluster.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py          | 14 +++++++++++---
 .../src/gempyor/templates/inference_local.bash.j2  |  4 ++++
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 701828662..5a9ba1f5d 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -27,7 +27,7 @@
 
 from ._jinja import _render_template_to_file, _render_template_to_temp_file
 from .file_paths import run_id
-from .info import get_cluster_info
+from .info import Cluster, get_cluster_info
 from .logging import get_script_logger
 from .utils import _format_cli_options, _git_checkout, _git_head, _shutil_which, config
 from .shared_cli import (
@@ -718,6 +718,7 @@ def _submit_scenario_job(
     job_size: JobSize,
     job_time_limit: JobTimeLimit,
     job_resources: JobResources,
+    cluster: Cluster | None,
     kwargs: dict[str, Any],
     verbosity: int | None,
     dry_run: bool,
@@ -757,6 +758,7 @@ def _submit_scenario_job(
 
     # Generically useful template data, regardless of batch system
     template_data = {
+        "cluster": None if cluster is None else cluster.model_dump(),
         "conda_env": kwargs["conda_env"],
         "config_path": config_out.absolute(),
         "debug": kwargs["debug"],
@@ -790,7 +792,6 @@ def _submit_scenario_job(
         )
     elif batch_system == BatchSystem.SLURM:
         # Cluster info
-        cluster = get_cluster_info(kwargs["cluster"])
         if verbosity is not None:
             logger.info(
                 "Utilizing info for the '%s' cluster to construct this job", cluster.name
@@ -800,7 +801,6 @@ def _submit_scenario_job(
                 cluster.name,
                 cluster.model_dump(),
             )
-        template_data["cluster"] = cluster.model_dump()
         # Construct the sbatch call
         options = {
             "chdir": kwargs["project_path"].absolute(),
@@ -1139,6 +1139,13 @@ def _click_submit(ctx: click.Context = mock_context, **kwargs: Any) -> None:
     )
     logger.info("Requesting the resources %s for this job.", job_resources)
 
+    # Cluster info
+    cluster = (
+        None
+        if kwargs["cluster"] is None and batch_system != BatchSystem.SLURM
+        else get_cluster_info(kwargs["cluster"])
+    )
+
     # Restart/continuation location
     # TODO: Implement this
 
@@ -1191,6 +1198,7 @@ def _click_submit(ctx: click.Context = mock_context, **kwargs: Any) -> None:
             job_size,
             job_time_limit,
             job_resources,
+            cluster,
             kwargs,
             kwargs["verbosity"],
             kwargs["dry_run"],
diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/inference_local.bash.j2 b/flepimop/gempyor_pkg/src/gempyor/templates/inference_local.bash.j2
index b9dda9f07..66e6ce6dc 100644
--- a/flepimop/gempyor_pkg/src/gempyor/templates/inference_local.bash.j2
+++ b/flepimop/gempyor_pkg/src/gempyor/templates/inference_local.bash.j2
@@ -2,6 +2,10 @@
 
 {% include "inference_environment_variables.bash.j2" with context %}
 
+{%- if cluster is defined and cluster is not none %}
+{% include "cluster_setup.bash.j2" with context %}
+{%- endif %}
+
 cd $PROJECT_PATH
 
 {% include "conda_activate.bash.j2" with context %}

From 8d7b3d8721da87de428e7311a28b65c7608601f9 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Mon, 25 Nov 2024 16:59:34 -0500
Subject: [PATCH 69/78] Helpful logging if using `--local` on a cluster

Made the `ValueError` from `_infer_cluster_from_fqdn` optional, but
default to `True` for current behavior.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py     | 11 ++++++++-
 flepimop/gempyor_pkg/src/gempyor/info.py      | 15 +++++++++---
 .../info/test__infer_cluster_from_fqdn.py     | 23 +++++++++++++------
 3 files changed, 38 insertions(+), 11 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 5a9ba1f5d..d12f60c63 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -27,7 +27,7 @@
 
 from ._jinja import _render_template_to_file, _render_template_to_temp_file
 from .file_paths import run_id
-from .info import Cluster, get_cluster_info
+from .info import Cluster, _infer_cluster_from_fqdn, get_cluster_info
 from .logging import get_script_logger
 from .utils import _format_cli_options, _git_checkout, _git_head, _shutil_which, config
 from .shared_cli import (
@@ -1107,6 +1107,15 @@ def _click_submit(ctx: click.Context = mock_context, **kwargs: Any) -> None:
             kwargs["email"],
             batch_system,
         )
+    if (
+        batch_system == BatchSystem.LOCAL
+        and (cluster_name := _infer_cluster_from_fqdn(raise_error=False)) is not None
+    ):
+        logger.critical(
+            "The batch system is local but detected a cluster, '%s'. "
+            "This will likely result in a vague error.",
+            cluster_name,
+        )
 
     # Job size
     job_size = JobSize.from_jobs_simulations_blocks(
diff --git a/flepimop/gempyor_pkg/src/gempyor/info.py b/flepimop/gempyor_pkg/src/gempyor/info.py
index bdcdc7b36..958b29168 100644
--- a/flepimop/gempyor_pkg/src/gempyor/info.py
+++ b/flepimop/gempyor_pkg/src/gempyor/info.py
@@ -152,18 +152,27 @@ def get_cluster_info(name: str | None, flepi_path: os.PathLike | None = None) ->
     return _get_info("cluster", name, Cluster, flepi_path)
 
 
-def _infer_cluster_from_fqdn() -> str:
+def _infer_cluster_from_fqdn(raise_error: bool = True) -> str | None:
     """
     Infer the cluster name from the FQDN.
 
+    Args:
+        raise_error: A flag indicating whether to raise an error if the FQDN does not
+            match any of the expected regexes.
+
     Returns:
         The name of the cluster inferred from the FQDN.
 
     Raises:
-        ValueError: If the value of `socket.getfqdn()` does not match an expected regex.
+        ValueError: If the value of `socket.getfqdn()` does not match an expected regex
+            and `raise_error` is `True`.
     """
     fqdn = getfqdn()
     for cluster, regex in _CLUSTER_FQDN_REGEXES:
         if regex.match(fqdn):
             return cluster
-    raise ValueError(f"The fqdn, '{fqdn}', does not match any of the expected clusters.")
+    if raise_error:
+        raise ValueError(
+            f"The fqdn, '{fqdn}', does not match any of the expected clusters."
+        )
+    return None
diff --git a/flepimop/gempyor_pkg/tests/info/test__infer_cluster_from_fqdn.py b/flepimop/gempyor_pkg/tests/info/test__infer_cluster_from_fqdn.py
index a26705503..0502cdf9f 100644
--- a/flepimop/gempyor_pkg/tests/info/test__infer_cluster_from_fqdn.py
+++ b/flepimop/gempyor_pkg/tests/info/test__infer_cluster_from_fqdn.py
@@ -20,18 +20,27 @@ def socket_fqdn_wraps() -> str:
 
 
 @pytest.mark.parametrize(
-    ("fqdn", "expected"),
+    ("fqdn", "raise_error", "expected"),
     (
-        ("login01.cm.cluster", "rockfish"),
-        ("login3.cm.cluster", "rockfish"),
-        ("longleaf-login1.its.unc.edu", "longleaf"),
-        ("longleaf-login07.its.unc.edu", "longleaf"),
+        ("login01.cm.cluster", True, "rockfish"),
+        ("login01.cm.cluster", False, "rockfish"),
+        ("login3.cm.cluster", True, "rockfish"),
+        ("login3.cm.cluster", False, "rockfish"),
+        ("longleaf-login1.its.unc.edu", True, "longleaf"),
+        ("longleaf-login1.its.unc.edu", False, "longleaf"),
+        ("longleaf-login07.its.unc.edu", True, "longleaf"),
+        ("longleaf-login07.its.unc.edu", False, "longleaf"),
+        ("longleaf-login07.its.unc.edu", True, "longleaf"),
+        ("longleaf-login07.its.unc.edu", False, "longleaf"),
+        ("epid-iss-MacBook-Pro.local", False, None),
     ),
 )
-def test_exact_results_for_select_values(fqdn: str, expected: str) -> None:
+def test_exact_results_for_select_values(
+    fqdn: str, raise_error: bool, expected: str
+) -> None:
     def socket_fqdn_wraps() -> str:
         return fqdn
 
     with patch("gempyor.info.getfqdn", wraps=socket_fqdn_wraps) as socket_fqdn_patch:
-        assert _infer_cluster_from_fqdn() == expected
+        assert _infer_cluster_from_fqdn(raise_error=raise_error) == expected
         socket_fqdn_patch.assert_called_once()

From 502749991cf1c9a235b98b4ef4816c952889e3f8 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 26 Nov 2024 15:30:29 -0500
Subject: [PATCH 70/78] Add args documentation for `_submit_scenario_job`

---
 flepimop/gempyor_pkg/src/gempyor/batch.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index d12f60c63..24f225daf 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -728,7 +728,22 @@ def _submit_scenario_job(
     Submit a job for a scenario.
 
     Args:
-        ...
+        outcome_modifiers_scenario: The outcome modifiers scenario to use.
+        seir_modifiers_scenario: The seir modifiers scenario to use.
+        name: The name of the config file used as a prefix for the job name.
+        batch_system: The batch system to submit the job to.
+        inference_method: The inference method being used.
+        config_out: The path to the config file to use.
+        job_name: The name of the job to submit.
+        job_size: The size of the job to submit.
+        job_time_limit: The time limit of the job to submit.
+        job_resources: The resources required for the job to submit.
+        cluster: The cluster information to use for submitting the job.
+        kwargs: Additional options provided to the submit job CLI as keyword arguments.
+        verbosity: A integer verbosity level to enable logging or `None` for no logging.
+        dry_run: A boolean indicating if this is a dry run or not, if set to `True` this
+            function will not actually submit/run a job.
+        now: The current UTC timestamp.
     """
     # Get logger
     if verbosity is not None:

From d549728b56ec2606d5f7e33d905c4eb1aa526d03 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 26 Nov 2024 15:54:06 -0500
Subject: [PATCH 71/78] Remove temporary comments

---
 flepimop/gempyor_pkg/src/gempyor/batch.py                   | 3 ---
 .../src/gempyor/templates/inference_slurm.sbatch.j2         | 6 ------
 2 files changed, 9 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index 24f225daf..b0f33eba9 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -1170,9 +1170,6 @@ def _click_submit(ctx: click.Context = mock_context, **kwargs: Any) -> None:
         else get_cluster_info(kwargs["cluster"])
     )
 
-    # Restart/continuation location
-    # TODO: Implement this
-
     # Manifest
     if kwargs["skip_manifest"]:
         logger.debug("Skipped writing manifest metadata.")
diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/inference_slurm.sbatch.j2 b/flepimop/gempyor_pkg/src/gempyor/templates/inference_slurm.sbatch.j2
index 62d346323..1398a2f8b 100644
--- a/flepimop/gempyor_pkg/src/gempyor/templates/inference_slurm.sbatch.j2
+++ b/flepimop/gempyor_pkg/src/gempyor/templates/inference_slurm.sbatch.j2
@@ -14,10 +14,4 @@ set -x
 
 {% include "conda_activate.bash.j2" with context %}
 
-# AWS setup
-
-# Fetch/setup resume files
-
 {% include "inference_command.bash.j2" with context %}
-
-# Upload/move results

From b76bb3b6b3bab347d7064605c67791b5a7dc4084 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 26 Nov 2024 16:20:20 -0500
Subject: [PATCH 72/78] Add `FLEPI_PATH` to GH action, fix Jinja2 loader

* Export `$FLEPI_PATH` in the GH action before running tests to run the
  unit tests for `get_cluster_info`.
* Attempt to fix Jinja2 lader issue by falling back to a file system
  loader if a package loader fails.
---
 .github/workflows/gempyor-ci.yml                      |  1 +
 flepimop/gempyor_pkg/src/gempyor/_jinja.py            | 11 +++++++++--
 .../gempyor_pkg/src/gempyor/templates/__init__.py     |  0
 3 files changed, 10 insertions(+), 2 deletions(-)
 create mode 100644 flepimop/gempyor_pkg/src/gempyor/templates/__init__.py

diff --git a/.github/workflows/gempyor-ci.yml b/.github/workflows/gempyor-ci.yml
index d70d19071..451235d8d 100644
--- a/.github/workflows/gempyor-ci.yml
+++ b/.github/workflows/gempyor-ci.yml
@@ -43,6 +43,7 @@ jobs:
         shell: bash
       - name: Run gempyor tests
         run: |
+          export FLEPI_PATH=$(pwd)
           cd flepimop/gempyor_pkg
           pytest --exitfirst
         shell: bash
diff --git a/flepimop/gempyor_pkg/src/gempyor/_jinja.py b/flepimop/gempyor_pkg/src/gempyor/_jinja.py
index 406c8f2ef..9495996f1 100644
--- a/flepimop/gempyor_pkg/src/gempyor/_jinja.py
+++ b/flepimop/gempyor_pkg/src/gempyor/_jinja.py
@@ -14,11 +14,18 @@
 from tempfile import mkstemp
 from typing import Any
 
-from jinja2 import Environment, PackageLoader, Template
+from jinja2 import Environment, FileSystemLoader, PackageLoader, Template
 
 
 # Globals
-_jinja_environment = Environment(loader=PackageLoader("gempyor", "templates"))
+try:
+    _jinja_environment = Environment(loader=PackageLoader("gempyor", "templates"))
+except ValueError:
+    from os.path import dirname
+
+    _jinja_environment = Environment(
+        loader=FileSystemLoader(dirname(__file__) + "/templates")
+    )
 
 
 # Functions
diff --git a/flepimop/gempyor_pkg/src/gempyor/templates/__init__.py b/flepimop/gempyor_pkg/src/gempyor/templates/__init__.py
new file mode 100644
index 000000000..e69de29bb

From 2f46391e173e98b5964327fc072509c024dcdbe3 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 26 Nov 2024 16:50:04 -0500
Subject: [PATCH 73/78] Add `Self` shim for python 3.10

---
 flepimop/gempyor_pkg/src/gempyor/batch.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index b0f33eba9..a550e2b72 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -21,7 +21,7 @@
 from stat import S_IXUSR
 import subprocess
 import sys
-from typing import Any, Literal, Self
+from typing import Any, Literal
 
 import click
 
@@ -43,6 +43,12 @@
 )
 
 
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    Self = Any
+
+
 _JOB_NAME_REGEX = re.compile(r"^[a-z]{1}([a-z0-9\_\-]+)?$", flags=re.IGNORECASE)
 
 

From c14552e17abf3637da15539a84517cf9a9552fa1 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 26 Nov 2024 16:51:38 -0500
Subject: [PATCH 74/78] Escape `\` in `flepimop submit --help` man page

To deal with a `DeprecationWarning` that python keeps producing.
---
 flepimop/gempyor_pkg/src/gempyor/batch.py | 38 +++++++++++------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/batch.py b/flepimop/gempyor_pkg/src/gempyor/batch.py
index a550e2b72..e8bbc78a9 100644
--- a/flepimop/gempyor_pkg/src/gempyor/batch.py
+++ b/flepimop/gempyor_pkg/src/gempyor/batch.py
@@ -1051,15 +1051,15 @@ def _click_submit(ctx: click.Context = mock_context, **kwargs: Any) -> None:
 
     \b
     ```bash
-    $ flepimop submit --jobs 4 \ # With legacy inference jobs refers to chains
-        --simulations 20 \       # The number of iterations per a chain
-        --blocks 1 \             # The number of consecutive blocks to run
-        --slurm \                # Use slurm, shorthand for `--batch-system slurm`
-        --project-path $(pwd) \  # Manually specify the project path
-        --email me@college.edu \ # Use slurm's built-in email alerts for the job
-        --skip-checkout \        # Do not create a new branch in the project repo
-        -vvv \                   # Use the max verbosity
-        --dry-run \              # Do not actually submit the job via `sbatch`.
+    $ flepimop submit --jobs 4 \\ # With legacy inference jobs refers to chains
+        --simulations 20 \\       # The number of iterations per a chain
+        --blocks 1 \\             # The number of consecutive blocks to run
+        --slurm \\                # Use slurm, shorthand for `--batch-system slurm`
+        --project-path $(pwd) \\  # Manually specify the project path
+        --email me@college.edu \\ # Use slurm's built-in email alerts for the job
+        --skip-checkout \\        # Do not create a new branch in the project repo
+        -vvv \\                   # Use the max verbosity
+        --dry-run \\              # Do not actually submit the job via `sbatch`.
         config_sample_2pop_inference.yml
     ```
 
@@ -1068,16 +1068,16 @@ def _click_submit(ctx: click.Context = mock_context, **kwargs: Any) -> None:
 
     \b
     ```bash
-    $ flepimop submit --jobs 10 \               # See before
-        --simulations 400 \                     # See before
-        --blocks 1 \                            # See before
-        --slurm \                               # See before
-        --partition dedicated_slurm_partition \ # Submit to a particular slurm partition
-        --simulation-time 30s \                 # Specify the time limit per simulation
-        --initial-time 5min \                   # Specify the initial time limit
-        --conda-env custom-flepimop-env \       # Specify a custom conda env to use
-        --cpus 4 \                              # Use 4 CPUs per a job
-        -vvv \                                  # Use the max verbosity
+    $ flepimop submit --jobs 10 \\               # See before
+        --simulations 400 \\                     # See before
+        --blocks 1 \\                            # See before
+        --slurm \\                               # See before
+        --partition dedicated_slurm_partition \\ # Submit to a particular slurm partition
+        --simulation-time 30s \\                 # Specify the time limit per simulation
+        --initial-time 5min \\                   # Specify the initial time limit
+        --conda-env custom-flepimop-env \\       # Specify a custom conda env to use
+        --cpus 4 \\                              # Use 4 CPUs per a job
+        -vvv \\                                  # Use the max verbosity
         config_sample_2pop_inference.yml
     ```
     """

From a64ca7e91ab64611476fce7ea16a04a62d766fdc Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 26 Nov 2024 17:35:09 -0500
Subject: [PATCH 75/78] Provide only pkg name to Jinja2 `PackageLoader`

---
 flepimop/gempyor_pkg/src/gempyor/_jinja.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/_jinja.py b/flepimop/gempyor_pkg/src/gempyor/_jinja.py
index 9495996f1..ce086e76b 100644
--- a/flepimop/gempyor_pkg/src/gempyor/_jinja.py
+++ b/flepimop/gempyor_pkg/src/gempyor/_jinja.py
@@ -19,12 +19,12 @@
 
 # Globals
 try:
-    _jinja_environment = Environment(loader=PackageLoader("gempyor", "templates"))
+    _jinja_environment = Environment(loader=PackageLoader("gempyor"))
 except ValueError:
     from os.path import dirname
 
     _jinja_environment = Environment(
-        loader=FileSystemLoader(dirname(__file__) + "/templates")
+        loader=FileSystemLoader(dirname(__file__).replace("\\", "/") + "/templates")
     )
 
 

From 7c03736b211595455751c980e966c867785614fb Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 26 Nov 2024 17:39:40 -0500
Subject: [PATCH 76/78] Prefer `FileSystemLoader` over `PackageLoader`

---
 flepimop/gempyor_pkg/src/gempyor/_jinja.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/flepimop/gempyor_pkg/src/gempyor/_jinja.py b/flepimop/gempyor_pkg/src/gempyor/_jinja.py
index ce086e76b..bae0e9aae 100644
--- a/flepimop/gempyor_pkg/src/gempyor/_jinja.py
+++ b/flepimop/gempyor_pkg/src/gempyor/_jinja.py
@@ -10,6 +10,7 @@
 
 
 # Imports
+from os.path import dirname
 from pathlib import Path
 from tempfile import mkstemp
 from typing import Any
@@ -19,13 +20,11 @@
 
 # Globals
 try:
-    _jinja_environment = Environment(loader=PackageLoader("gempyor"))
-except ValueError:
-    from os.path import dirname
-
     _jinja_environment = Environment(
         loader=FileSystemLoader(dirname(__file__).replace("\\", "/") + "/templates")
     )
+except ValueError:
+    _jinja_environment = Environment(loader=PackageLoader("gempyor"))
 
 
 # Functions

From 0149bc73f5440237c637e0fac232dc3aa30e7f19 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Tue, 26 Nov 2024 17:56:19 -0500
Subject: [PATCH 77/78] Set `include-package-data` to `true`

---
 flepimop/gempyor_pkg/pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flepimop/gempyor_pkg/pyproject.toml b/flepimop/gempyor_pkg/pyproject.toml
index 53d7c921a..008de866f 100644
--- a/flepimop/gempyor_pkg/pyproject.toml
+++ b/flepimop/gempyor_pkg/pyproject.toml
@@ -53,7 +53,7 @@ gempyor-simulate = "gempyor.simulate:_deprecated_simulate"
 
 [tool.setuptools]
 package-dir = {"" = "src"}
-include-package-data = false
+include-package-data = true
 
 [tool.setuptools.packages.find]
 where = ["src"]

From ad11faea053045f6e61fab546ef9335b518ff726 Mon Sep 17 00:00:00 2001
From: Timothy Willard <9395586+TimothyWillard@users.noreply.github.com>
Date: Mon, 2 Dec 2024 09:33:33 -0500
Subject: [PATCH 78/78] Include templates with `include-data` directive

---
 flepimop/gempyor_pkg/pyproject.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flepimop/gempyor_pkg/pyproject.toml b/flepimop/gempyor_pkg/pyproject.toml
index 008de866f..dfb6b28bd 100644
--- a/flepimop/gempyor_pkg/pyproject.toml
+++ b/flepimop/gempyor_pkg/pyproject.toml
@@ -54,6 +54,7 @@ gempyor-simulate = "gempyor.simulate:_deprecated_simulate"
 [tool.setuptools]
 package-dir = {"" = "src"}
 include-package-data = true
+package-data = { "gempyor" = ["templates/*.j2"] }
 
 [tool.setuptools.packages.find]
 where = ["src"]