Merge pull request #30 from BCDA-APS/14-PersistentDict-alternative

StoredDict is alternative to  PersistentDict

.gitignore

@@ -172,3 +172,6 @@ qserver/existing_plans_and_devices.yaml
 .logs/
 *.dat
 *.hdf
+
+# Local Run Engine metadata dictionary
+.re_md_dict.yml

docs/source/api/utils.rst

@@ -15,10 +15,12 @@ configs, core, devices, or plans.
     ~instrument.utils.config_loaders
     ~instrument.utils.logging_setup
     ~instrument.utils.metadata
+    ~instrument.utils.stored_dict
 
 .. automodule:: instrument.utils.aps_functions
 .. automodule:: instrument.utils.controls_setup
 .. automodule:: instrument.utils.helper_functions
 .. automodule:: instrument.utils.config_loaders
 .. automodule:: instrument.utils.logging_setup
 .. automodule:: instrument.utils.metadata
+.. automodule:: instrument.utils.stored_dict

docs/source/index.rst

@@ -31,9 +31,9 @@ console, a Jupyter notebook, the queueserver, or even a Python script:
 About ...
 -----------
 
-:home: https://prjemian.github.io/model_instrument/
-:bug tracker: https://github.com/prjemian/model_instrument/issues
-:source: https://github.com/prjemian/model_instrument
+:home: https://BCDA-APS.github.io/bs_model_instrument/
+:bug tracker: https://github.com/BCDA-APS/bs_model_instrument/issues
+:source: https://github.com/BCDA-APS/bs_model_instrument
 :license: :ref:`license`
 :full version: |version|
 :published: |today|

src/instrument/configs/iconfig.yml

@@ -20,9 +20,10 @@ RUN_ENGINE:
     ### Default: `RE.md["scan_id"]` (not using an EPICS PV)
     # SCAN_ID_PV: f"{IOC}bluesky_scan_id"
 
-    ### Directory to "autosave" the RE.md dictionary (uses PersistentDict).
-    ### Default: HOME/.config/Bluesky_RunEngine_md'
-    # MD_PATH: /home/beams/USERNAME/.config/Bluesky_RunEngine_md
+    ### Where to "autosave" the RE.md dictionary.
+    ### Defaults:
+    MD_STORAGE_HANDLER: StoredDict
+    MD_PATH: .re_md_dict.yml
 
     ### The progress bar is nice to see,
     ### except when it clutters the output in Jupyter notebooks.

src/instrument/core/run_engine_init.py

@@ -18,6 +18,7 @@
 from ..utils.controls_setup import set_timeouts
 from ..utils.metadata import MD_PATH
 from ..utils.metadata import re_metadata
+from ..utils.stored_dict import StoredDict
 from .best_effort_init import bec
 from .catalog_init import cat
 
@@ -31,13 +32,24 @@
 
 # Save/restore RE.md dictionary, in this precise order.
 if MD_PATH is not None:
+    handler_name = re_config.get("MD_STORAGE_HANDLER", "StoredDict")
+    logger.debug(
+        "Select %r to store 'RE.md' dictionary in %s.",
+        handler_name,
+        MD_PATH,
+    )
     try:
-        RE.md = bluesky.utils.PersistentDict(MD_PATH)
-    except Exception as e:
+        if handler_name == "PersistentDict":
+            RE.md = bluesky.utils.PersistentDict(MD_PATH)
+        else:
+            RE.md = StoredDict(MD_PATH)
+    except Exception as error:
         print(
-            f"\n Could not create PersistentDict for RE metadata. Continuing without "
-            f"saving metadata to disk. The error is: {e} \n"
+            "\n"
+            f"Could not create {handler_name} for RE metadata. Continuing"
+            f" without saving metadata to disk. {error=}\n"
         )
+        logger.warning("%s('%s') error:%s", handler_name, MD_PATH, error)
 
 RE.md.update(re_metadata(cat))  # programmatic metadata
 RE.md.update(re_config.get("DEFAULT_METADATA", {}))

src/instrument/tests/test_stored_dict.py

@@ -0,0 +1,138 @@
+"""
+Test the utils.stored_dict module.
+"""
+
+import pathlib
+import tempfile
+import time
+from contextlib import nullcontext as does_not_raise
+
+import pytest
+
+from ..utils.config_loaders import load_config_yaml
+from ..utils.stored_dict import StoredDict
+
+
+def luftpause(delay=0.01):
+    """A brief wait for content to flush to storage."""
+    time.sleep(max(0, delay))
+
+
+@pytest.fixture
+def md_file():
+    """Provide a temporary file (deleted on close)."""
+    tfile = tempfile.NamedTemporaryFile(
+        prefix="re_md_",
+        suffix=".yml",
+        delete=False,
+    )
+    path = pathlib.Path(tfile.name)
+    yield pathlib.Path(tfile.name)
+
+    if path.exists():
+        path.unlink()  # delete the file
+
+
+def test_StoredDict(md_file):
+    """Test the StoredDict class."""
+    assert md_file.exists()
+    assert len(open(md_file).read().splitlines()) == 0  # empty
+
+    sdict = StoredDict(md_file, delay=0.2, title="unit testing")
+    assert sdict is not None
+    assert len(sdict) == 0
+    assert sdict._delay == 0.2
+    assert sdict._title == "unit testing"
+    assert len(open(md_file).read().splitlines()) == 0  # still empty
+    assert sdict._sync_key == f"sync_agent_{id(sdict):x}"
+    assert not sdict.sync_in_progress
+
+    # Write an empty dictionary.
+    sdict.flush()
+    luftpause()
+    buf = open(md_file).read().splitlines()
+    assert len(buf) == 4, f"{buf=}"
+    assert buf[-1] == "{}"  # The empty dict.
+    assert buf[0].startswith("# ")
+    assert buf[1].startswith("# ")
+    assert "unit testing" in buf[0]
+
+    # Add a new {key: value} pair.
+    assert not sdict.sync_in_progress
+    sdict["a"] = 1
+    assert sdict.sync_in_progress
+    sdict.flush()
+    assert time.time() > sdict._sync_deadline
+    luftpause()
+    assert not sdict.sync_in_progress
+    assert len(open(md_file).read().splitlines()) == 4
+
+    # Change the only value.
+    sdict["a"] = 2
+    sdict.flush()
+    luftpause()
+    assert len(open(md_file).read().splitlines()) == 4  # Still.
+
+    # Add another key.
+    sdict["bee"] = "bumble"
+    sdict.flush()
+    luftpause()
+    assert len(open(md_file).read().splitlines()) == 5
+
+    # Test _delayed_sync_to_storage.
+    sdict["bee"] = "queen"
+    md = load_config_yaml(md_file)
+    assert len(md) == 2  # a & bee
+    assert "a" in md
+    assert md["bee"] == "bumble"  # The old value.
+
+    time.sleep(sdict._delay / 2)
+    # Still not written ...
+    assert load_config_yaml(md_file)["bee"] == "bumble"
+
+    time.sleep(sdict._delay)
+    # Should be written by now.
+    assert load_config_yaml(md_file)["bee"] == "queen"
+
+    del sdict["bee"]  # __delitem__
+    assert "bee" not in sdict  # __getitem__
+
+
+@pytest.mark.parametrize(
+    "md, xcept, text",
+    [
+        [{"a": 1}, None, str(None)],  # int value is ok
+        [{"a": 2.2}, None, str(None)],  # float value is ok
+        [{"a": "3"}, None, str(None)],  # str value is ok
+        [{"a": [4, 5, 6]}, None, str(None)],  # list value is ok
+        [{"a": {"bb": [4, 5, 6]}}, None, str(None)],  # nested value is ok
+        [{1: 1}, None, str(None)],  # int key is ok
+        [{"a": object()}, TypeError, "not JSON serializable"],
+        [{object(): 1}, TypeError, "keys must be str, int, float, "],
+        [{"a": [4, object(), 6]}, TypeError, "not JSON serializable"],
+        [{"a": {object(): [4, 5, 6]}}, TypeError, "keys must be str, int, "],
+    ],
+)
+def test_set_exceptions(md, xcept, text, md_file):
+    """Cases that might raise an exception."""
+    sdict = StoredDict(md_file, delay=0.2, title="unit testing")
+    context = does_not_raise() if xcept is None else pytest.raises(xcept)
+    with context as reason:
+        sdict.update(md)
+    assert text in str(reason), f"{reason=}"
+
+
+def test_popitem(md_file):
+    """Can't popitem from empty dict."""
+    sdict = StoredDict(md_file, delay=0.2, title="unit testing")
+    with pytest.raises(KeyError) as reason:
+        sdict.popitem()
+    assert "dictionary is empty" in str(reason), f"{reason=}"
+
+
+def test_repr(md_file):
+    """__repr__"""
+    sdict = StoredDict(md_file, delay=0.1, title="unit testing")
+    sdict["a"] = 1
+    assert repr(sdict) == "<StoredDict {'a': 1}>"
+    assert str(sdict) == "<StoredDict {'a': 1}>"

src/instrument/utils/metadata.py

@@ -53,11 +53,26 @@
     pysumreg=pysumreg.__version__,
     spec2nexus=spec2nexus.__version__,
 )
+RE_CONFIG = iconfig.get("RUN_ENGINE", {})
 
 
 def get_md_path():
-    """Get PersistentDict directory for RE metadata."""
-    path = pathlib.Path(iconfig.get("MD_PATH", DEFAULT_MD_PATH))
+    """
+    Get path for RE metadata.
+
+    ==============  ==============================================
+    support         path
+    ==============  ==============================================
+    PersistentDict  Directory where dictionary keys are stored in separate files.
+    StoredDict      File where dictionary is stored as YAML.
+    ==============  ==============================================
+
+    In either case, the 'path' can be relative or absolute.  Relative
+    paths are with respect to the present working directory when the
+    bluesky session is started.
+    """
+    md_path_name = RE_CONFIG.get("MD_PATH", DEFAULT_MD_PATH)
+    path = pathlib.Path(md_path_name)
     logger.info("RunEngine metadata saved in directory: %s", str(path))
     return str(path)