Refactoring of IO helper functions (bidscoin.bids -> `bidscoin.plug…

…ins`)

bidscoin/bidscoiner.py

@@ -21,6 +21,7 @@
 if find_spec('bidscoin') is None:
     sys.path.append(str(Path(__file__).parents[1]))
 from bidscoin import bcoin, bids, lsdirs, bidsversion, trackusage, __version__, DEBUG
+from bidscoin.plugins import unpack
 
 
 def bidscoiner(sourcefolder: str, bidsfolder: str, participant: list=(), force: bool=False, bidsmap: str='bidsmap.yaml', cluster: str='') -> None:
@@ -253,7 +254,7 @@ def bidscoiner(sourcefolder: str, bidsfolder: str, participant: list=(), force:
             for session in sessions:
 
                 # Unpack the data in a temporary folder if it is tarballed/zipped and/or contains a DICOMDIR file
-                sesfolders, unpacked = bids.unpack(session, bidsmap.options.get('unzip',''))
+                sesfolders, unpacked = unpack(session, bidsmap.options.get('unzip',''))
                 for sesfolder in sesfolders:
 
                     # Check if we should skip the session-folder

bidscoin/bidsmapper.py

@@ -18,8 +18,9 @@
 from importlib.util import find_spec
 if find_spec('bidscoin') is None:
     sys.path.append(str(Path(__file__).parents[1]))
-from bidscoin import bcoin, bids, lsdirs, trackusage, check_version, __version__
+from bidscoin import bcoin, lsdirs, trackusage, check_version, __version__
 from bidscoin.bids import BidsMap
+from bidscoin.plugins import unpack
 
 _, uptodate, versionmessage = check_version()
 
@@ -122,7 +123,7 @@ def bidsmapper(sourcefolder: str, bidsfolder: str, bidsmap: str, template: str,
                 LOGGER.info(f"Mapping: {session} (subject {n}/{len(subjects)})")
 
                 # Unpack the data in a temporary folder if it is tarballed/zipped and/or contains a DICOMDIR file
-                sesfolders, unpacked = bids.unpack(session, unzip)
+                sesfolders, unpacked = unpack(session, unzip)
                 for sesfolder in sesfolders:
                     if store:
                         bidsmap_new.store = {'source': sesfolder.parent.parent.parent.parent if unpacked else rawfolder.parent,

bidscoin/plugins/dcm2niix2bids.py

@@ -16,7 +16,7 @@
 from bidscoin import bids, run_command, lsdirs, due, Doi
 from bidscoin.utilities import physio
 from bidscoin.bids import BidsMap, DataFormat, Plugin, Plugins
-from bidscoin.plugins import PluginInterface
+from bidscoin.plugins import PluginInterface, is_dicomfile, is_parfile, get_dicomfield, get_parfield, get_dicomfile, get_parfiles
 try:
     from nibabel.testing import data_path
 except ImportError:
@@ -84,10 +84,10 @@ def has_support(self, file: Path, dataformat: Union[DataFormat, str]='') -> str:
         if dataformat and dataformat not in ('DICOM', 'PAR'):
             return ''
 
-        if bids.is_dicomfile(file):     # To support pet2bids add: and bids.get_dicomfield('Modality', file) != 'PT'
+        if is_dicomfile(file):     # To support pet2bids add: and get_dicomfield('Modality', file) != 'PT'
             return 'DICOM'
 
-        if bids.is_parfile(file):
+        if is_parfile(file):
             return 'PAR'
 
         return ''
@@ -104,10 +104,10 @@ def get_attribute(self, dataformat: Union[DataFormat, str], sourcefile: Path, at
         :return:            The retrieved attribute value
         """
         if dataformat == 'DICOM':
-            return bids.get_dicomfield(attribute, sourcefile)
+            return get_dicomfield(attribute, sourcefile)
 
         if dataformat == 'PAR':
-            return bids.get_parfield(attribute, sourcefile)
+            return get_parfield(attribute, sourcefile)
 
 
     def bidsmapper(self, session: Path, bidsmap_new: BidsMap, bidsmap_old: BidsMap, template: BidsMap) -> None:
@@ -133,11 +133,11 @@ def bidsmapper(self, session: Path, bidsmap_new: BidsMap, bidsmap_old: BidsMap,
         if dataformat == 'DICOM':
             for sourcedir in lsdirs(session, '**/*'):
                 for n in range(1):      # Option: Use range(2) to scan two files and catch e.g. magnitude1/2 fieldmap files that are stored in one Series folder (but bidscoiner sees only the first file anyhow and it makes bidsmapper 2x slower :-()
-                    sourcefile = bids.get_dicomfile(sourcedir, n)
+                    sourcefile = get_dicomfile(sourcedir, n)
                     if sourcefile.name:
                         sourcefiles.append(sourcefile)
         elif dataformat == 'PAR':
-            sourcefiles = bids.get_parfiles(session)
+            sourcefiles = get_parfiles(session)
         else:
             LOGGER.error(f"Unsupported dataformat '{dataformat}'")
 
@@ -227,7 +227,7 @@ def bidscoiner(self, session: Path, bidsmap: BidsMap, bidsses: Path) -> Union[No
             sources      = lsdirs(session, '**/*')
             manufacturer = datasource.attributes('Manufacturer')
         elif dataformat == 'PAR':
-            sources      = bids.get_parfiles(session)
+            sources      = get_parfiles(session)
             manufacturer = 'Philips Medical Systems'
         else:
             LOGGER.error(f"Unsupported dataformat '{dataformat}'")
@@ -245,7 +245,7 @@ def bidscoiner(self, session: Path, bidsmap: BidsMap, bidsses: Path) -> Union[No
 
             # Get a sourcefile
             if dataformat == 'DICOM':
-                sourcefile = bids.get_dicomfile(source)
+                sourcefile = get_dicomfile(source)
             else:
                 sourcefile = source
             if not sourcefile.name or not self.has_support(sourcefile):
@@ -302,7 +302,7 @@ def bidscoiner(self, session: Path, bidsmap: BidsMap, bidsses: Path) -> Union[No
             # Convert physiological log files (dcm2niix can't handle these)
             if suffix == 'physio':
                 target = (outfolder/bidsname).with_suffix('.tsv.gz')
-                if bids.get_dicomfile(source, 2).name:                  # TODO: issue warning or support PAR
+                if get_dicomfile(source, 2).name:                  # TODO: issue warning or support PAR
                     LOGGER.warning(f"Found > 1 DICOM file in {source}, using: {sourcefile}")
                 try:
                     physiodata = physio.readphysio(sourcefile)

bidscoin/plugins/spec2nii2bids.py

@@ -12,7 +12,7 @@
 from pathlib import Path
 from bidscoin import run_command, is_hidden, bids, due, Doi
 from bidscoin.bids import BidsMap, DataFormat, Plugin
-from bidscoin.plugins import PluginInterface
+from bidscoin.plugins import PluginInterface, is_dicomfile, get_twixfield, get_sparfield, get_p7field
 
 LOGGER = logging.getLogger(__name__)
 
@@ -36,9 +36,6 @@ def test(self, options: Plugin=OPTIONS) -> int:
 
         LOGGER.info('Testing the spec2nii2bids installation:')
 
-        if not all(hasattr(bids, name) for name in ('get_twixfield', 'get_sparfield', 'get_p7field')):
-            LOGGER.error("Could not import the expected 'get_twixfield', 'get_sparfield' and/or 'get_p7field' from the bids.py library")
-            return 1
         if 'command' not in {**OPTIONS, **options}:
             LOGGER.error(f"The expected 'command' key is not defined in the spec2nii2bids options")
             return 1
@@ -66,7 +63,7 @@ def has_support(self, file: Path, dataformat: Union[DataFormat, str]='') -> str:
             return 'Twix'
         elif suffix == '.spar':
             return 'SPAR'
-        elif suffix == '.7' and not bids.is_dicomfile(file):
+        elif suffix == '.7' and not is_dicomfile(file):
             return 'Pfile'
 
         return ''
@@ -92,15 +89,15 @@ def get_attribute(self, dataformat: Union[DataFormat, str], sourcefile: Path, at
 
         if dataformat == 'Twix':
 
-            return bids.get_twixfield(attribute, sourcefile, options.get('multiraid', OPTIONS['multiraid']))
+            return get_twixfield(attribute, sourcefile, options.get('multiraid', OPTIONS['multiraid']))
 
         if dataformat == 'SPAR':
 
-            return bids.get_sparfield(attribute, sourcefile)
+            return get_sparfield(attribute, sourcefile)
 
         if dataformat == 'Pfile':
 
-            return bids.get_p7field(attribute, sourcefile)
+            return get_p7field(attribute, sourcefile)
 
         LOGGER.error(f"Unsupported MRS data-format: {dataformat}")
 

bidscoin/utilities/bidsparticipants.py

@@ -12,6 +12,7 @@
     sys.path.append(str(Path(__file__).parents[2]))
 from bidscoin import bcoin, bids, lsdirs, trackusage, __version__
 from bidscoin.bids import BidsMap
+from bidscoin.plugins import unpack
 
 
 def scanpersonals(bidsmap: BidsMap, session: Path, personals: dict, keys: list) -> bool:
@@ -123,7 +124,7 @@ def bidsparticipants(sourcefolder: str, bidsfolder: str, keys: list, bidsmap: st
                 subid, sesid = bids.DataSource(session/'dum.my', bidsmap.plugins, '', bidsmap.options).subid_sesid()
 
                 # Unpack the data in a temporary folder if it is tarballed/zipped and/or contains a DICOMDIR file
-                sesfolders, unpacked = bids.unpack(session, bidsmap.options.get('unzip',''))
+                sesfolders, unpacked = unpack(session, bidsmap.options.get('unzip',''))
                 for sesfolder in sesfolders:
 
                     # Update/append the personal source data

bidscoin/utilities/dicomsort.py

@@ -11,7 +11,7 @@
 if find_spec('bidscoin') is None:
     import sys
     sys.path.append(str(Path(__file__).parents[2]))
-from bidscoin import bcoin, bids, lsdirs, trackusage
+from bidscoin import bcoin, lsdirs, trackusage
 
 LOGGER = logging.getLogger(__name__)
 
@@ -27,15 +27,18 @@ def construct_name(scheme: str, dicomfile: Path, force: bool) -> str:
     :return:            The new name constructed from the scheme
     """
 
+    # Avoid circular import from bidscoin.plugins
+    from bidscoin.plugins import get_dicomfield
+
     # Alternative field names based on earlier DICOM versions or on other reasons
     alternatives = {'PatientName':'PatientsName', 'SeriesDescription':'ProtocolName', 'InstanceNumber':'ImageNumber', 'SeriesNumber':'SeriesInstanceUID',
                     'PatientsName':'PatientName', 'ProtocolName':'SeriesDescription', 'ImageNumber':'InstanceNumber'}
 
     schemevalues = {}
     for field in re.findall(r'(?<={)([a-zA-Z0-9]+)(?::\d+[d-gD-Gn])?(?=})', scheme):
-        value = cleanup(bids.get_dicomfield(field, dicomfile))
+        value = cleanup(get_dicomfield(field, dicomfile))
         if not value and value != 0 and field in alternatives.keys():
-            value = cleanup(bids.get_dicomfield(alternatives[field], dicomfile))
+            value = cleanup(get_dicomfield(alternatives[field], dicomfile))
             if field == 'SeriesNumber':
                 value = int(value.replace('.',''))      # Convert the SeriesInstanceUID to an int
         if not value and value != 0 and not force:

bidscoin/utilities/rawmapper.py

@@ -9,6 +9,7 @@
     import sys
     sys.path.append(str(Path(__file__).parents[2]))
 from bidscoin import lsdirs, bids, trackusage
+from bidscoin.plugins import get_dicomfield, get_dicomfile
 
 
 def rawmapper(sourcefolder, outfolder: str='', sessions: tuple=(), rename: bool=False, clobber: bool=False, field: tuple=('PatientComments',), wildcard: str= '*', subprefix: str= 'sub-', sesprefix: str= 'ses-', dryrun: bool=False) -> None:
@@ -75,14 +76,14 @@ def rawmapper(sourcefolder, outfolder: str='', sessions: tuple=(), rename: bool=
         series = lsdirs(session, wildcard)
         if not series:
             series    = Path()
-            dicomfile = bids.get_dicomfile(session)     # Try and see if there is a DICOM file in the root of the session folder
+            dicomfile = get_dicomfile(session)     # Try and see if there is a DICOM file in the root of the session folder
         else:
             series    = series[0]                       # NB: Assumes the first folder contains a dicom file and that all folders give the same info
-            dicomfile = bids.get_dicomfile(series)      # Try and see if there is a DICOM file in the root of the session folder
+            dicomfile = get_dicomfile(series)      # Try and see if there is a DICOM file in the root of the session folder
         if not dicomfile.name:
             print(f"No DICOM files found in: {session}")
             continue
-        dicomval = [str(bids.get_dicomfield(dcmfield, dicomfile)) for dcmfield in field]
+        dicomval = [str(get_dicomfield(dcmfield, dicomfile)) for dcmfield in field]
 
         # Rename the session subfolder in the sourcefolder and print & save this info
         if rename:

docs/bidsmap.rst

@@ -11,7 +11,7 @@ A central concept in BIDScoin is the so-called bidsmap. Generally speaking, a bi
 3. **The attributes dictionary** contains attributes from the source data itself, such as the 'ProtocolName' from the DICOM header. The source attributes are a very rich source of information of which a minimal subset is normally sufficient to identify the different data types in your source data repository. The attributes are read from (the header of) the source file itself or, if present, from an accompanying sidecar file. This sidecar file transparently extends (or overrule) the available source attributes, as if that data would have been written to (the header of) the source data file itself. The name of the sidecar file should be the same as the name of the first associated source file and have a ``.json`` file extension. For instance, the ``001.dcm``, ``002.dcm``, ``003.dcm``, [..], DICOM source images can have a sidecar file in the same directory named ``001.json`` (e.g. containing metadata that is not available in the DICOM header or that must be overruled). It should be noted that BIDScoin `plugins <./plugins.html>`__ will copy the extended attribute data over to the json sidecar files in your BIDS output folder, giving you additional control to generate your BIDS sidecar files (in addition to the meta dictionary described in point 5 below).
 4. **The bids dictionary** contains the BIDS data type and entities that determine the filename of the BIDS output data. The values in this dictionary are encouraged to be edited by the user
 5. **The meta dictionary** contains custom key-value pairs that are added to the json sidecar file by the BIDScoin plugins. Meta data may well vary from session to session, hence this dictionary often contains dynamic attribute values that are evaluated during bidscoiner runtime (see the `special features <#special-bidsmap-features>`__ below)
-6. **The events dictionary** contains settings for parsing Events data (WIP))
+6. **The events dictionary** contains settings for parsing Events data (experimental))
 
 In sum, a run-item contains a single bids-mapping, which links the input dictionaries (2) and (3) to the output dictionaries (4) and (5).
 

tests/test_bids.py

@@ -5,9 +5,7 @@
 import re
 import json
 from datetime import datetime, timedelta
-from importlib.util import find_spec
 from pathlib import Path
-from nibabel.testing import data_path
 from pydicom.data import get_testdata_file
 from bidscoin import bcoin, bids, bidsmap_template, bidscoinroot
 from bidscoin.bids import BidsMap, RunItem, DataSource, Plugin, Meta
@@ -20,21 +18,11 @@ def dcm_file():
     return Path(get_testdata_file('MR_small.dcm'))
 
 
-@pytest.fixture(scope='module')
-def dcm_file_csa():
-    return Path(data_path)/'1.dcm'
-
-
 @pytest.fixture(scope='module')
 def dicomdir():
     return Path(get_testdata_file('DICOMDIR'))
 
 
-@pytest.fixture(scope='module')
-def par_file():
-    return Path(data_path)/'phantom_EPI_asc_CLEAR_2_1.PAR'
-
-
 @pytest.fixture(scope='module')
 def study_bidsmap():
     """The path to the study bidsmap `test_data/bidsmap.yaml`"""
@@ -487,76 +475,12 @@ def test_exist_run(self, study_bidsmap):
         assert bidsmap.exist_run(runitem) is False
 
 
-def test_unpack(dicomdir, tmp_path):
-    sessions, unpacked = bids.unpack(dicomdir.parent, '', tmp_path, None)   # None -> simulate commandline usage of dicomsort()
-    assert unpacked
-    assert len(sessions) == 6
-    for session in sessions:
-        assert 'Doe^Archibald' in session.parts or 'Doe^Peter' in session.parts
-
-
-def test_is_dicomfile(dcm_file):
-    assert bids.is_dicomfile(dcm_file)
-
-
-def test_is_parfile(par_file):
-    assert bids.is_parfile(par_file)
-
-
-def test_get_dicomfile(dcm_file, dicomdir):
-    assert bids.get_dicomfile(dcm_file.parent).name == '693_J2KI.dcm'
-    assert bids.get_dicomfile(dicomdir.parent).name == '6154'
-
-
 def test_get_datasource(dicomdir):
     datasource = bids.get_datasource(dicomdir.parent, {'dcm2niix2bids': {}})
     assert datasource.has_support()
     assert datasource.dataformat == 'DICOM'
 
 
-def test_get_dicomfield(dcm_file_csa):
-
-    # -> Standard DICOM
-    value = bids.get_dicomfield('SeriesDescription', dcm_file_csa)
-    assert value == 'CBU_DTI_64D_1A'
-
-    # -> The pydicom-style tag number
-    value = bids.get_dicomfield('SeriesNumber', dcm_file_csa)
-    assert value == 12
-    assert value == bids.get_dicomfield('0x00200011', dcm_file_csa)
-    assert value == bids.get_dicomfield('(0x20,0x11)', dcm_file_csa)
-    assert value == bids.get_dicomfield('(0020,0011)', dcm_file_csa)
-
-    # -> The special PhaseEncodingDirection tag
-    value = bids.get_dicomfield('PhaseEncodingDirection', dcm_file_csa)
-    assert value == 'AP'
-
-    # -> CSA Series header
-    value = bids.get_dicomfield('PhaseGradientAmplitude', dcm_file_csa)
-    assert value == '0.0'
-
-    # -> CSA Image header
-    value = bids.get_dicomfield('ImaCoilString', dcm_file_csa)
-    assert value == 'T:HEA;HEP'
-
-    value = bids.get_dicomfield('B_matrix', dcm_file_csa)
-    assert value == ''
-
-    value = bids.get_dicomfield('NonExistingTag', dcm_file_csa)
-    assert value == ''
-
-    # -> CSA MrPhoenixProtocol
-    if find_spec('dicom_parser'):
-        value = bids.get_dicomfield('MrPhoenixProtocol.tProtocolName', dcm_file_csa)
-        assert value == 'CBU+AF8-DTI+AF8-64D+AF8-1A'
-
-        value = bids.get_dicomfield('MrPhoenixProtocol.sDiffusion', dcm_file_csa)
-        assert value == "{'lDiffWeightings': 2, 'alBValue': [None, 1000], 'lNoiseLevel': 40, 'lDiffDirections': 64, 'ulMode': 256}"
-
-        value = bids.get_dicomfield('MrPhoenixProtocol.sProtConsistencyInfo.tBaselineString', dcm_file_csa)
-        assert value == 'N4_VB17A_LATEST_20090307'
-
-
 def test_match_runvalue():
     assert bids.match_runvalue('my_pulse_sequence_name', '_name')      is False
     assert bids.match_runvalue('my_pulse_sequence_name', '^my.*name$') is True