Upload Non-DICOM primary files and resources independently

.flake8

@@ -11,6 +11,6 @@ exclude =
     docs/source/conf.py
 max-line-length = 88
 select = C,E,F,W,B,B950
-extend-ignore = E203,E501,E129,W503
+extend-ignore = E203,E501,E129,W503,E701
 per-file-ignores =
     setup.py:F401

conftest.py

@@ -3,8 +3,10 @@
 import logging
 import typing as ty
 import tempfile
-from logging.handlers import SMTPHandler
+
+# from logging.handlers import SMTPHandler
 import pytest
+import click.testing
 from click.testing import CliRunner
 import xnat4tests  # type: ignore[import-untyped]
 from datetime import datetime
@@ -31,11 +33,12 @@
 if os.getenv("_PYTEST_RAISE", "0") != "0":
 
     @pytest.hookimpl(tryfirst=True)
-    def pytest_exception_interact(call):
-        raise call.excinfo.value
+    def pytest_exception_interact(call: pytest.CallInfo[ty.Any]) -> None:
+        if call.excinfo is not None:
+            raise call.excinfo.value
 
     @pytest.hookimpl(tryfirst=True)
-    def pytest_internalerror(excinfo):
+    def pytest_internalerror(excinfo: pytest.ExceptionInfo[BaseException]) -> None:
         raise excinfo.value
 
     CATCH_CLI_EXCEPTIONS = False
@@ -44,82 +47,88 @@ def pytest_internalerror(excinfo):
 
 
 @pytest.fixture
-def catch_cli_exceptions():
+def catch_cli_exceptions() -> bool:
     return CATCH_CLI_EXCEPTIONS
 
 
 @pytest.fixture(scope="session")
-def run_prefix():
+def run_prefix() -> str:
     "A datetime string used to avoid stale data left over from previous tests"
     return datetime.strftime(datetime.now(), "%Y%m%d%H%M%S")
 
 
 @pytest.fixture(scope="session")
-def xnat_repository():
+def xnat_repository() -> None:
     xnat4tests.start_xnat()
 
 
 @pytest.fixture(scope="session")
-def xnat_archive_dir(xnat_repository):
-    return xnat4tests.Config().xnat_root_dir / "archive"
+def xnat_archive_dir(xnat_repository: None) -> Path:
+    return xnat4tests.Config().xnat_root_dir / "archive"  # type: ignore[no-any-return]
 
 
 @pytest.fixture(scope="session")
-def tmp_gen_dir():
+def tmp_gen_dir() -> Path:
     # tmp_gen_dir = Path("~").expanduser() / ".xnat-ingest-work3"
     # tmp_gen_dir.mkdir(exist_ok=True)
     # return tmp_gen_dir
     return Path(tempfile.mkdtemp())
 
 
 @pytest.fixture(scope="session")
-def xnat_login(xnat_repository):
+def xnat_login(xnat_repository: str) -> ty.Any:
     return xnat4tests.connect()
 
 
 @pytest.fixture(scope="session")
-def xnat_project(xnat_login, run_prefix):
+def xnat_project(xnat_login: ty.Any, run_prefix: str) -> ty.Any:
     project_id = f"INGESTUPLOAD{run_prefix}"
     with xnat4tests.connect() as xnat_login:
         xnat_login.put(f"/data/archive/projects/{project_id}")
     return project_id
 
 
 @pytest.fixture(scope="session")
-def xnat_server(xnat_config):
-    return xnat_config.xnat_uri
+def xnat_server(xnat_config: xnat4tests.Config) -> str:
+    return xnat_config.xnat_uri  # type: ignore[no-any-return]
 
 
 @pytest.fixture(scope="session")
-def xnat_config(xnat_repository):
+def xnat_config(xnat_repository: str) -> xnat4tests.Config:
     return xnat4tests.Config()
 
 
 @pytest.fixture
-def cli_runner(catch_cli_exceptions):
-    def invoke(*args, catch_exceptions=catch_cli_exceptions, **kwargs):
+def cli_runner(catch_cli_exceptions: bool) -> ty.Callable[..., ty.Any]:
+    def invoke(
+        *args: ty.Any, catch_exceptions: bool = catch_cli_exceptions, **kwargs: ty.Any
+    ) -> click.testing.Result:
         runner = CliRunner()
-        result = runner.invoke(*args, catch_exceptions=catch_exceptions, **kwargs)
+        result = runner.invoke(*args, catch_exceptions=catch_exceptions, **kwargs)  # type: ignore[misc]
         return result
 
     return invoke
 
 
-# Create a custom handler that captures email messages for testing
-class TestSMTPHandler(SMTPHandler):
-    def __init__(
-        self, mailhost, fromaddr, toaddrs, subject, credentials=None, secure=None
-    ):
-        super().__init__(mailhost, fromaddr, toaddrs, subject, credentials, secure)
-        self.emails = []  # A list to store captured email messages
+# # Create a custom handler that captures email messages for testing
+# class TestSMTPHandler(SMTPHandler):
+#     def __init__(
+#         self, mailhost, fromaddr, toaddrs, subject, credentials=None, secure=None
+#     ):
+#         super().__init__(mailhost, fromaddr, toaddrs, subject, credentials, secure)
+#         self.emails = []  # A list to store captured email messages
 
-    def emit(self, record):
-        # Capture the email message and append it to the list
-        msg = self.format(record)
-        self.emails.append(msg)
+#     def emit(self, record):
+#         # Capture the email message and append it to the list
+#         msg = self.format(record)
+#         self.emails.append(msg)
 
 
-def get_raw_data_files(out_dir: ty.Optional[Path] = None, **kwargs) -> ty.List[Path]:
+def get_raw_data_files(
+    out_dir: ty.Optional[Path] = None, **kwargs: ty.Any
+) -> ty.List[Path]:
     if out_dir is None:
         out_dir = Path(tempfile.mkdtemp())
-    return get_listmode_data(out_dir, **kwargs) + get_countrate_data(out_dir, **kwargs)
+    return get_listmode_data(out_dir, skip_unknown=True, **kwargs) + get_countrate_data(  # type: ignore[no-any-return]
+        out_dir, skip_unknown=True, **kwargs
+    )

pyproject.toml

@@ -9,6 +9,7 @@ readme = "README.md"
 requires-python = ">=3.8"
 dependencies = [
     "click >=8.1",
+    "discord",
     "fileformats-medimage-extras",
     "pydicom >=2.3.1",
     "tqdm >=4.64.1",
@@ -84,4 +85,19 @@ doctests = true
 per-file-ignores = ["__init__.py:F401"]
 max-line-length = 88
 select = "C,E,F,W,B,B950"
-extend-ignore = ['E203', 'E501', 'E129', "W503"]
+extend-ignore = ['E203', 'E501', 'E129', "W503", 'E701']
+
+
+[tool.mypy]
+python_version = "3.10"
+ignore_missing_imports = true
+strict = true
+explicit_package_bases = true
+exclude = [
+    "tests",
+    "scripts",
+    "docs",
+    "build",
+    "dist",
+    "xnat_ingest/_version.py",
+]

real-tests/usyd_stage.py

@@ -9,15 +9,15 @@
     stage,
     [],
     env={
-        "XNAT_INGEST_STAGE_DICOMS_PATH": "/vol/vmhost/kubernetes/<path>/<to>/<dicom>/<store>/**/*.IMA",
-        "XNAT_INGEST_STAGE_DIR": "/vol/vmhost/usyd-data-export/STAGING",
-        "XNAT_INGEST_STAGE_PROJECT": "ProtocolName",
-        "XNAT_INGEST_STAGE_SUBJECT": "PatientID",
-        "XNAT_INGEST_STAGE_VISIT": "AccessionNumber",
-        "XNAT_INGEST_STAGE_ASSOCIATED": '"/vol/vmhost/usyd-data-export/RAW-DATA-EXPORT/{PatientName.family_name}_{PatientName.given_name}/.ptd","./[^\\.]+.[^\\.]+.[^\\.]+.(?P\\d+).[A-Z]+_(?P[^\\.]+)."',
-        "XNAT_INGEST_STAGE_DELETE": "0",
-        "XNAT_INGEST_STAGE_LOGFILE": "<somewhere-sensible>,INFO",
-        "XNAT_INGEST_STAGE_DEIDENTIFY": "1",
+        "XINGEST_DICOMS_PATH": "/vol/vmhost/kubernetes/<path>/<to>/<dicom>/<store>/**/*.IMA",
+        "XINGEST_DIR": "/vol/vmhost/usyd-data-export/STAGING",
+        "XINGEST_PROJECT": "ProtocolName",
+        "XINGEST_SUBJECT": "PatientID",
+        "XINGEST_VISIT": "AccessionNumber",
+        "XINGEST_ASSOCIATED": '"/vol/vmhost/usyd-data-export/RAW-DATA-EXPORT/{PatientName.family_name}_{PatientName.given_name}/.ptd","./[^\\.]+.[^\\.]+.[^\\.]+.(?P\\d+).[A-Z]+_(?P[^\\.]+)."',
+        "XINGEST_DELETE": "0",
+        "XINGEST_LOGFILE": "<somewhere-sensible>,INFO",
+        "XINGEST_DEIDENTIFY": "1",
     },
     catch_exceptions=False,
 )

real-tests/usyd_transfer.py

@@ -9,7 +9,7 @@
     transfer,
     [],
     env={
-        "XNAT_INGEST_STAGE_DIR": "/Users/tclose/Data/testing/staging-test/",
+        "XINGEST_DIR": "/Users/tclose/Data/testing/staging-test/",
         "XNAT_INGEST_TRANSFER_LOGFILE": "/Users/tclose/Desktop/test-log.log,INFO",
         "XNAT_INGEST_TRANSFER_DELETE": "0",
     },

real-tests/usyd_upload.py

@@ -8,17 +8,17 @@
     upload,
     [],
     env={
-        "XNAT_INGEST_UPLOAD_STAGED": "<s3-bucket>",
-        "XNAT_INGEST_UPLOAD_HOST": "https://xnat.sydney.edu.au",
-        "XNAT_INGEST_UPLOAD_USER": "<role-account-user>",
-        "XNAT_INGEST_UPLOAD_PASS": "<role-account-pass>",
-        "XNAT_INGEST_UPLOAD_ALWAYSINCLUDE": "medimage/dicom-series",
-        "XNAT_INGEST_UPLOAD_STORE_CREDENTIALS": "<s3-bucket-access-key>,<s3-bucket-access-secret>",
-        "XNAT_INGEST_UPLOAD_LOGFILE": "<somewhere-sensible>,INFO",
-        "XNAT_INGEST_UPLOAD_DELETE": "0",
-        "XNAT_INGEST_UPLOAD_TEMPDIR": "<somewhere-else-sensible>",
-        "XNAT_INGEST_UPLOAD_REQUIRE_MANIFEST": "1",
-        "XNAT_INGEST_UPLOAD_CLEANUP_OLDER_THAN": "30",
+        "XINGEST_STAGED": "<s3-bucket>",
+        "XINGEST_HOST": "https://xnat.sydney.edu.au",
+        "XINGEST_USER": "<role-account-user>",
+        "XINGEST_PASS": "<role-account-pass>",
+        "XINGEST_ALWAYSINCLUDE": "medimage/dicom-series",
+        "XINGEST_STORE_CREDENTIALS": "<s3-bucket-access-key>,<s3-bucket-access-secret>",
+        "XINGEST_LOGFILE": "<somewhere-sensible>,INFO",
+        "XINGEST_DELETE": "0",
+        "XINGEST_TEMPDIR": "<somewhere-else-sensible>",
+        "XINGEST_REQUIRE_MANIFEST": "1",
+        "XINGEST_CLEANUP_OLDER_THAN": "30",
     },
     catch_exceptions=False,
 )

xnat_ingest/__init__.py

@@ -1 +1,3 @@
 from ._version import __version__
+
+__all__ = ["__version__"]

xnat_ingest/cli/base.py

@@ -4,5 +4,5 @@
 
 @click.group(help="Checks and uploads scans exported from scanner consoles to XNAT")
 @click.version_option(version=__version__)
-def cli():
+def cli() -> None:
     pass