Overhaul configuration system (#729)

* The current configuration system suffers from some challenges, including
fragile / special-case parsing of container traits.  This work aims to
make this area of the code robust and more future-proof:

* Move config tools into a submodule, extract some things from traits.py.

* Support arbitrary nested containers, including parsing of those and None
  values from the commandline.  Expand unit tests to included nested
  containers.

* Dedicated set of routines for translating back and forth from trait
  values and strings.

* Separate file-format code more cleanly from code that just works with the
  internal config dictionary format.

* Add support for YAML config files, along with existing JSON and TOML
  formats.

* Expand the `toast_config_verify` script to support loading and migrating
  old config files to new ones (or for converting between supported formats).

* Keep support for loading old config files, while printing warnings telling
  the user to update their configs.

* Address review comments.  Run format_source.sh

setup.py

@@ -297,7 +297,7 @@ def readme():
     "psutil",
     "h5py",
     "pshmem>=0.2.10",
-    "pyyaml",
+    "ruamel.yaml",
     "astropy",
     "healpy",
     "ephem",
@@ -310,7 +310,7 @@ def readme():
         "ipywidgets",
         "plotly",
         "plotly-resampler",
-    ]
+    ],
 }
 conf["packages"] = find_packages(
     "src",

src/toast/CMakeLists.txt

@@ -119,7 +119,7 @@ install(FILES
     mpi.py
     timing.py
     traits.py
-    config.py
+    trait_utils.py
     job.py
     pixels.py
     pixels_io_healpix.py
@@ -163,6 +163,7 @@ install(DIRECTORY
 )
 
 # Process the sub directories
+add_subdirectory(config)
 add_subdirectory(io)
 add_subdirectory(accelerator)
 add_subdirectory(tests)

src/toast/__init__.py

@@ -85,7 +85,7 @@
         raise ImportError("Cannot read RELEASE file")
 
 
-from .config import create_from_config, load_config, parse_config
+from .config import load_config, parse_config
 from .data import Data
 from .instrument import Focalplane, GroundSite, SpaceSite, Telescope
 from .instrument_sim import fake_hexagon_focalplane
@@ -95,4 +95,5 @@
 from .observation import Observation
 from .pixels import PixelData, PixelDistribution
 from .timing import GlobalTimers, Timer
+from .traits import create_from_config
 from .weather import Weather