diff --git a/setup.py b/setup.py index 0c8cabd6b..b0abdf0d3 100644 --- a/setup.py +++ b/setup.py @@ -297,7 +297,7 @@ def readme(): "psutil", "h5py", "pshmem>=0.2.10", - "pyyaml", + "ruamel.yaml", "astropy", "healpy", "ephem", @@ -310,7 +310,7 @@ def readme(): "ipywidgets", "plotly", "plotly-resampler", - ] + ], } conf["packages"] = find_packages( "src", diff --git a/src/toast/CMakeLists.txt b/src/toast/CMakeLists.txt index 7e302ad53..1967731a1 100644 --- a/src/toast/CMakeLists.txt +++ b/src/toast/CMakeLists.txt @@ -119,7 +119,7 @@ install(FILES mpi.py timing.py traits.py - config.py + trait_utils.py job.py pixels.py pixels_io_healpix.py @@ -163,6 +163,7 @@ install(DIRECTORY ) # Process the sub directories +add_subdirectory(config) add_subdirectory(io) add_subdirectory(accelerator) add_subdirectory(tests) diff --git a/src/toast/__init__.py b/src/toast/__init__.py index 19ae022a0..5ebe077f3 100644 --- a/src/toast/__init__.py +++ b/src/toast/__init__.py @@ -85,7 +85,7 @@ raise ImportError("Cannot read RELEASE file") -from .config import create_from_config, load_config, parse_config +from .config import load_config, parse_config from .data import Data from .instrument import Focalplane, GroundSite, SpaceSite, Telescope from .instrument_sim import fake_hexagon_focalplane @@ -95,4 +95,5 @@ from .observation import Observation from .pixels import PixelData, PixelDistribution from .timing import GlobalTimers, Timer +from .traits import create_from_config from .weather import Weather diff --git a/src/toast/config.py b/src/toast/config.py deleted file mode 100644 index d72e5c391..000000000 --- a/src/toast/config.py +++ /dev/null @@ -1,1364 +0,0 @@ -# Copyright (c) 2015-2020 by the parties listed in the AUTHORS file. -# All rights reserved. Use of this source code is governed by -# a BSD-style license that can be found in the LICENSE file. - -import argparse -import ast -import copy -import json -import re -import sys -import types -from collections import OrderedDict -from collections.abc import MutableMapping - -import numpy as np -import tomlkit -from astropy import units as u -from tomlkit import comment, document, dumps, loads, nl, table - -from . import instrument -from .instrument import Focalplane, Telescope -from .traits import TraitConfig, trait_scalar_to_string, trait_string_to_scalar -from .utils import Environment, Logger - - -def build_config(objects): - """Build a configuration of current values. - - Args: - objects (list): A list of class instances to add to the config. These objects - must inherit from the TraitConfig base class. - - Returns: - (dict): The configuration. - - """ - conf = OrderedDict() - for o in objects: - if not isinstance(o, TraitConfig): - raise RuntimeError("The object list should contain TraitConfig instances") - if o.name is None: - raise RuntimeError("Cannot buid config from objects without a name") - conf = o.get_config(input=conf) - return conf - - -class TraitAction(argparse.Action): - """Custom argparse action to check for valid use of None. - - Some traits support a None value even though they have a specific - type. This custom action checks for that None value and validates - that it is an acceptable value. - - """ - - def __init__( - self, - option_strings, - dest, - nargs=None, - const=None, - default=None, - type=None, - choices=None, - required=False, - help=None, - metavar=None, - ): - super(TraitAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=nargs, - const=const, - default=default, - type=None, - choices=choices, - required=required, - help=help, - metavar=metavar, - ) - self.trait_type = type - return - - def __call__(self, parser, namespace, values, option_string=None): - if isinstance(values, list): - realval = list() - for v in values: - if v == "None" or v is None: - realval.append(None) - else: - realval.append(self.trait_type(v)) - else: - if values == "None" or values is None: - realval = None - else: - realval = self.trait_type(values) - setattr(namespace, self.dest, realval) - - -def add_config_args(parser, conf, section, ignore=list(), prefix="", separator="."): - """Add arguments to an argparser for each parameter in a config dictionary. - - Using a previously created config dictionary, add a commandline argument for each - object parameter in a section of the config. The type, units, and help string for - the commandline argument come from the config, which is in turn built from the - class traits of the object. Boolean parameters are converted to store_true or - store_false actions depending on their current value. - - Containers in the config (list, set, tuple, dict) are parsed as a string to - support easy passing on the commandline, and then later converted to the actual - type. - - Args: - parser (ArgumentParser): The parser to append to. - conf (dict): The configuration dictionary. - section (str): Process objects in this section of the config. - ignore (list): List of object parameters to ignore when adding args. - prefix (str): Prepend this to the beginning of all options. - separator (str): Use this character between the class name and parameter. - - Returns: - None - - """ - parent = conf - if section is not None: - path = section.split("/") - for p in path: - if p not in parent: - msg = "section {} does not exist in config".format(section) - raise RuntimeError(msg) - parent = parent[p] - for obj, props in parent.items(): - for name, info in props.items(): - # print("examine options for {} = {}".format(name, info)) - if name in ignore: - # Skip this as requested - # print(" ignoring") - continue - if name == "class": - # This is not a user-configurable parameter. - # print(" skipping") - continue - if info["type"] not in [ - "bool", - "int", - "float", - "str", - "Quantity", - "Unit", - "list", - "dict", - "tuple", - "set", - ]: - # This is not something that we can get from parsing commandline - # options. Skip it. - # print(" no type- skipping") - continue - if info["type"] == "bool": - # Special case for boolean - if name == "enabled": - # Special handling of the TraitConfig "enabled" trait. We provide - # both an enable and disable option for every instance which later - # toggles that trait. - df = True - if info["value"] == "False": - df = False - parser.add_argument( - "--{}{}{}enable".format(prefix, obj, separator), - required=False, - default=df, - action="store_true", - help="Enable use of {}".format(obj), - ) - parser.add_argument( - "--{}{}{}disable".format(prefix, obj, separator), - required=False, - default=(not df), - action="store_false", - help="Disable use of {}".format(obj), - dest="{}{}{}enable".format(prefix, obj, separator), - ) - else: - # General bool option - option = "--{}{}{}{}".format(prefix, obj, separator, name) - act = "store_true" - dflt = False - if info["value"] == "True": - act = "store_false" - dflt = True - option = "--{}{}{}no_{}".format(prefix, obj, separator, name) - # print(" add bool argument {}".format(option)) - parser.add_argument( - option, - required=False, - default=dflt, - action=act, - help=info["help"], - dest="{}{}{}{}".format(prefix, obj, separator, name), - ) - else: - option = "--{}{}{}{}".format(prefix, obj, separator, name) - default = None - typ = None - hlp = info["help"] - if info["type"] == "int": - typ = int - if info["value"] != "None": - default = int(info["value"]) - elif info["type"] == "float": - typ = float - if info["value"] != "None": - default = float(info["value"]) - elif info["type"] == "str": - typ = str - if info["value"] != "None": - default = info["value"] - elif info["type"] == "Unit": - typ = u.Unit - # The "value" for a Unit is always "unit" - if info["unit"] != "None": - default = u.Unit(info["unit"]) - elif info["type"] == "Quantity": - typ = u.Quantity - if info["value"] != "None": - default = u.Quantity( - "{} {}".format(info["value"], info["unit"]) - ) - elif info["type"] == "list": - typ = str - if info["value"] != "None": - if len(info["value"]) == 0: - default = "[]" - else: - formatted = [ - trait_scalar_to_string(x) for x in info["value"] - ] - default = f"{formatted}" - elif info["type"] == "set": - typ = str - if info["value"] != "None": - if len(info["value"]) == 0: - default = "{}" - else: - formatted = set( - [trait_scalar_to_string(x) for x in info["value"]] - ) - default = f"{formatted}" - elif info["type"] == "tuple": - typ = str - if info["value"] != "None": - if len(info["value"]) == 0: - default = "()" - else: - formatted = tuple( - [trait_scalar_to_string(x) for x in info["value"]] - ) - default = f"{formatted}" - elif info["type"] == "dict": - typ = str - if info["value"] != "None": - if len(info["value"]) == 0: - default = "{}" - else: - formatted = { - x: trait_scalar_to_string(y) - for x, y in info["value"].items() - } - # print(f"dict formatted = {formatted}") - default = f"{info['value']}" - else: - raise RuntimeError("Invalid trait type, should never get here!") - # print( - # f" add_argument: {option}, type={typ}, default={default} ({type(default)})" - # ) - parser.add_argument( - option, - action=TraitAction, - required=False, - default=default, - type=typ, - help=hlp, - ) - return - - -def args_update_config(args, conf, useropts, section, prefix="", separator="\."): - """Override options in a config dictionary from args namespace. - - Args: - args (namespace): The args namespace returned by ArgumentParser.parse_args() - conf (dict): The configuration to update. - useropts (list): The list of options actually specified by the user. - section (str): Process objects in this section of the config. - prefix (str): Prepend this to the beginning of all options. - separator (str): Use this character between the class name and parameter. - - Returns: - (namespace): The un-parsed remaining arg vars. - - """ - # Build the regex match of option names - obj_pat = re.compile("{}(.*?){}(.*)".format(prefix, separator)) - user_pat = re.compile("--{}(.*?){}(.*)".format(prefix, separator)) - no_pat = re.compile("^no_(.*)") - - # Regex patterns for containers - list_pat = re.compile(r"\[(.*)\]") - dictset_pat = re.compile(r"\{(.*)\}") - tuple_pat = re.compile(r"\((.*)\)") - dictelem_pat = re.compile(r"(.*):(.*)") - - def _strip_quote(s): - if s == "": - return s - else: - return s.strip(" '\"") - - def _parse_list(parg): - if parg is None: - return "None" - else: - mat = list_pat.match(parg) - if mat is None: - msg = f"Argparse value '{parg}' is not a list" - raise ValueError(msg) - value = list() - inner = mat.group(1) - if inner == "[]" or inner == "": - return value - else: - elems = inner.split(",") - # print(f"_parse_list elems split = {elems}") - for elem in elems: - # value.append(trait_string_to_scalar(_strip_quote(elem))) - value.append(_strip_quote(elem)) - return value - - def _parse_tuple(parg): - if parg is None: - return "None" - else: - mat = tuple_pat.match(parg) - if mat is None: - msg = f"Argparse value '{parg}' is not a tuple" - raise ValueError(msg) - value = list() - inner = mat.group(1) - if inner == "()" or inner == "": - return tuple(value) - else: - elems = inner.split(",") - for elem in elems: - value.append(_strip_quote(elem)) - # value.append(trait_string_to_scalar(_strip_quote(elem))) - return tuple(value) - - def _parse_set(parg): - if parg is None: - return "None" - else: - mat = dictset_pat.match(parg) - if mat is None: - msg = f"Argparse value '{parg}' is not a set" - raise ValueError(msg) - value = set() - inner = mat.group(1) - if inner == "{}" or inner == "": - return value - else: - elems = inner.split(",") - for elem in elems: - value.add(_strip_quote(elem)) - # value.add(trait_string_to_scalar(_strip_quote(elem))) - return value - - def _parse_dict(parg): - if parg is None: - return "None" - else: - mat = dictset_pat.match(parg) - if mat is None: - msg = f"Argparse value '{parg}' is not a dict" - raise ValueError(msg) - dstr = mat.group(1) - value = dict() - if dstr == "{}" or dstr == "": - return value - else: - elems = dstr.split(",") - for elem in elems: - el = _strip_quote(elem) - elem_mat = dictelem_pat.match(el) - if elem_mat is None: - msg = f"Argparse value '{parg}', element '{el}' " - msg += f" is not a dictionary key / value " - raise ValueError(msg) - elem_key = _strip_quote(elem_mat.group(1)) - elem_val = _strip_quote(elem_mat.group(2)) - value[elem_key] = elem_val - # value[elem_key] = trait_string_to_scalar(elem_val) - return value - - # Parse the list of user options - user = dict() - for uo in useropts: - user_mat = user_pat.match(uo) - if user_mat is not None: - name = user_mat.group(1) - optname = user_mat.group(2) - if name not in user: - user[name] = set() - # Handle mapping of option names - if optname == "enable" or optname == "disable": - optname = "enabled" - no_mat = no_pat.match(optname) - if no_mat is not None: - optname = no_mat.group(1) - user[name].add(optname) - - remain = copy.deepcopy(args) - parent = conf - if section is not None: - path = section.split("/") - for p in path: - if p not in parent: - msg = "section {} does not exist in config".format(section) - raise RuntimeError(msg) - parent = parent[p] - - # print(f"PARSER start config = {parent}") - # print("PARSER = ", args) - for arg, val in vars(args).items(): - obj_mat = obj_pat.match(arg) - if obj_mat is not None: - name = obj_mat.group(1) - optname = obj_mat.group(2) - if name not in parent: - # This command line option is not part of this section - continue - - if optname == "enable": - # The actual trait name is "enabled" - optname = "enabled" - - # For each option, convert the parsed value and compare with - # the default config value. If it differs, and the user - # actively set this option, then change it. - - if parent[name][optname]["type"] == "bool": - if isinstance(val, bool): - # Convert to str - if val: - val = "True" - else: - val = "False" - elif val is None: - val = "None" - else: - raise ValueError(f"value '{val}' is not bool or None") - if (val != parent[name][optname]["value"]) and ( - name in user and optname in user[name] - ): - parent[name][optname]["value"] = val - elif parent[name][optname]["type"] == "Unit": - # print(f"Parsing Unit: {val} ({type(val)})") - if isinstance(val, u.UnitBase): - unit = str(val) - elif val is None: - unit = "None" - else: - raise ValueError(f"value '{val}' is not a Unit or None") - if (unit != parent[name][optname]["unit"]) and ( - name in user and optname in user[name] - ): - parent[name][optname]["value"] = "unit" - parent[name][optname]["unit"] = unit - elif parent[name][optname]["type"] == "Quantity": - # print(f"Parsing Quantity: {val} ({type(val)})") - if isinstance(val, u.Quantity): - value = f"{val.value:0.14e}" - unit = str(val.unit) - elif val is None: - value = "None" - unit = parent[name][optname]["unit"] - else: - raise ValueError(f"value '{val}' is not a Quantity or None") - if ( - value != parent[name][optname]["value"] - or unit != parent[name][optname]["unit"] - ) and (name in user and optname in user[name]): - parent[name][optname]["value"] = value - parent[name][optname]["unit"] = unit - elif parent[name][optname]["type"] == "float": - if val is None: - val = "None" - else: - val = f"{val:0.14e}" - if (val != parent[name][optname]["value"]) and ( - name in user and optname in user[name] - ): - parent[name][optname]["value"] = val - elif parent[name][optname]["type"] == "int": - if val is None: - val = "None" - else: - val = f"{val}" - if (val != parent[name][optname]["value"]) and ( - name in user and optname in user[name] - ): - parent[name][optname]["value"] = val - elif parent[name][optname]["type"] == "list": - # For configs that are constructed from TOML, all sequence - # containers are parsed as a list. So try several types. - try: - value = _parse_set(val) - except ValueError: - try: - value = _parse_tuple(val) - except ValueError: - value = _parse_list(val) - if (value != parent[name][optname]["value"]) and ( - name in user and optname in user[name] - ): - parent[name][optname]["value"] = value - elif parent[name][optname]["type"] == "tuple": - value = _parse_tuple(val) - if (value != parent[name][optname]["value"]) and ( - name in user and optname in user[name] - ): - parent[name][optname]["value"] = value - elif parent[name][optname]["type"] == "set": - value = _parse_set(val) - if (value != parent[name][optname]["value"]) and ( - name in user and optname in user[name] - ): - parent[name][optname]["value"] = value - elif parent[name][optname]["type"] == "dict": - value = _parse_dict(val) - if (value != parent[name][optname]["value"]) and ( - name in user and optname in user[name] - ): - parent[name][optname]["value"] = value - else: - # This is a plain string - if val is None: - val = "None" - else: - val = str(val) - if (val != parent[name][optname]["value"]) and ( - name in user and optname in user[name] - ): - parent[name][optname]["value"] = val - delattr(remain, arg) - return remain - - -def parse_config( - parser, - operators=list(), - templates=list(), - prefix="", - opts=None, -): - """Load command line arguments associated with object properties. - - This function: - - * Adds a "--config" option to the parser which accepts multiple config file - paths to load. - - * Adds "--default_toml" and "--default_json" options to dump the default config - options to files. - - * Adds a option "--job_group_size" to provide the commonly used option for - setting the group size of the TOAST communicator. - - * Adds arguments for all object parameters in the defaults for the specified - classes. - - * Builds a config dictionary starting from the defaults, updating these using - values from any config files, and then applies any overrides from the - commandline. - - Args: - parser (ArgumentParser): The argparse parser. - operators (list): The operator instances to configure from files or - commandline. These instances should have their "name" attribute set to - something meaningful, since that name is used to construct the commandline - options. An enable / disable option is created for each, which toggles the - TraitConfig base class "disabled" trait. - templates (list): The template instances to add to the commandline. - These instances should have their "name" attribute set to something - meaningful, since that name is used to construct the commandline options. - An enable / disable option is created for each, which toggles the - TraitConfig base class "disabled" trait. - prefix (str): Optional string to prefix all options by. - opts (list): If not None, parse arguments from this list instead of sys.argv. - - Returns: - (tuple): The (config dictionary, args). The args namespace contains all the - remaining parameters after extracting the operator and template options. - - """ - - # The default configuration - defaults_op = build_config(operators) - defaults_tmpl = build_config(templates) - - # Add commandline overrides - if len(operators) > 0: - add_config_args( - parser, - defaults_op, - "operators", - ignore=["API"], - prefix=prefix, - ) - if len(templates) > 0: - add_config_args( - parser, - defaults_tmpl, - "templates", - ignore=["API"], - prefix=prefix, - ) - - # Combine all the defaults - defaults = OrderedDict() - defaults["operators"] = OrderedDict() - defaults["templates"] = OrderedDict() - if "operators" in defaults_op: - defaults["operators"].update(defaults_op["operators"]) - if "templates" in defaults_tmpl: - defaults["templates"].update(defaults_tmpl["templates"]) - - # Add an option to load one or more config files. These should have compatible - # names for the operators used in defaults. - parser.add_argument( - "--config", - type=str, - required=False, - nargs="+", - help="One or more input config files.", - ) - - # Add options to dump default values - parser.add_argument( - "--defaults_toml", - type=str, - required=False, - default=None, - help="Dump default config values to a TOML file", - ) - parser.add_argument( - "--defaults_json", - type=str, - required=False, - default=None, - help="Dump default config values to a JSON file", - ) - - parser.add_argument( - "--job_group_size", - required=False, - type=int, - default=None, - help="(Advanced) Size of the process groups", - ) - - parser.add_argument( - "--job_node_mem", - required=False, - type=int, - default=None, - help="(Advanced) Override the detected memory per node in bytes", - ) - - # Parse commandline or list of options. - if opts is None: - opts = sys.argv[1:] - args = parser.parse_args(args=opts) - - # print(f"DBG conf defaults = {defaults}") - # print(f"DBG after parse_args: {args}") - - # Dump default config values. - if args.defaults_toml is not None: - dump_toml(args.defaults_toml, defaults) - if args.defaults_json is not None: - dump_json(args.defaults_json, defaults) - - # Parse job args - jobargs = types.SimpleNamespace( - node_mem=args.job_node_mem, - group_size=args.job_group_size, - ) - del args.job_node_mem - del args.job_group_size - - # Load any config files. This overrides default values with config file contents. - config = copy.deepcopy(defaults) - - # print(f"DBG conf before load = {config}") - - if args.config is not None: - for conf in args.config: - config = load_config(conf, input=config) - - # print(f"DBG conf after load = {config}") - - # Parse operator commandline options. These override any config file or default - # values. - op_remaining = args_update_config(args, config, opts, "operators", prefix=prefix) - remaining = args_update_config( - op_remaining, config, opts, "templates", prefix=prefix - ) - - # Remove the options we created in this function - del remaining.config - del remaining.defaults_toml - del remaining.defaults_json - - return config, remaining, jobargs - - -def _merge_config(loaded, original): - log = Logger.get() - for section, objs in loaded.items(): - if section in original.keys(): - # We have this section - for objname, objprops in objs.items(): - if objname not in original[section]: - # This is a new object - original[section][objname] = objprops - else: - # Only update the value and unit, while preserving - # any pre-existing type information. - for k, v in objprops.items(): - if k == "class": - continue - if k in original[section][objname]: - # This key exists in the original object traits - cursor = original[section][objname][k] - cursor["value"] = v["value"] - cursor["unit"] = v["unit"] - if "type" not in cursor: - cursor["type"] = v["type"] - else: - # We have a config option that does not exist - # in the current object. Warn user that this may - # indicate a stale config file. - msg = f"Object {objname} currently has no configuration" - msg += f" trait '{k}'. This might be handled by the " - msg += f"class through API translation, but your config " - msg += f"file may be out of date." - log.warning(msg) - original[section][objname][k] = v - else: - # A new section - original[section] = objs - - -def _load_toml_traits(tbl): - # print("LOAD TraitConfig object {}".format(tbl), flush=True) - result = OrderedDict() - for k in list(tbl.keys()): - # print(f"LOAD examine key {k}: {type(tbl[k])}") - if k == "class": - # print(f" found trait class '{tbl[k]}'") - result[k] = tbl[k] - elif isinstance(tbl[k], tomlkit.items.Table): - # This is a dictionary trait - # print(f" found trait Dict '{tbl[k]}'") - result[k] = OrderedDict() - result[k]["value"] = OrderedDict() - result[k]["type"] = "dict" - result[k]["unit"] = "None" - for tk, tv in tbl[k].items(): - result[k]["value"][str(tk)] = str(tv) - elif isinstance(tbl[k], tomlkit.items.Array): - # This is a list - result[k] = OrderedDict() - result[k]["value"] = list() - result[k]["type"] = "list" - result[k]["unit"] = "None" - for it in tbl[k]: - # print(f" Array element '{str(it)}'") - result[k]["value"].append(str(it)) - elif isinstance(tbl[k], str): - # print(f" Checking string '{tbl[k]}'") - if len(tbl[k]) == 0: - # Special handling for empty string - # print(f" found trait str empty '{tbl[k]}'") - result[k] = OrderedDict() - result[k]["value"] = "" - result[k]["type"] = "str" - result[k]["unit"] = "None" - elif tbl[k] == "None": - # Copy None values. There is no way to determine the type in this case. - # This is just a feature of how we are constructing the TOML files for - # ease of use, rather than dumping the full trait info as sub-tables. - # In practice these parameters will be ignored when constructing - # TraitConfig objects and the defaults will be used anyway. - # print(f" found trait str None '{tbl[k]}'") - result[k] = OrderedDict() - result[k]["value"] = tbl[k] - result[k]["type"] = "unknown" - result[k]["unit"] = "None" - else: - # print(" Not empty or None") - result[k] = OrderedDict() - # Is this string actually a Quantity or Unit? - try: - parts = tbl[k].split() - vstr = parts.pop(0) - ustr = " ".join(parts) - if vstr == "unit": - # this is a Unit - result[k]["value"] = "unit" - result[k]["type"] = "Unit" - if ustr == "None": - result[k]["unit"] = "None" - else: - result[k]["unit"] = str(u.Unit(ustr)) - # print(f" found trait Unit '{tbl[k]}'") - elif ustr == "": - raise ValueError("No units, just a string") - else: - result[k]["type"] = "Quantity" - if vstr == "None": - result[k]["value"] = "None" - else: - v = float(vstr) - result[k]["value"] = f"{v:0.14e}" - result[k]["unit"] = str(u.Unit(ustr)) - # print(f" found trait Quantity '{tbl[k]}'") - except Exception as e: - # print(" failed... just a string") - # Just a regular string - # print(f" found trait str '{tbl[k]}'") - result[k]["value"] = tbl[k] - result[k]["type"] = "str" - result[k]["unit"] = "None" - elif isinstance(tbl[k], bool): - # print(f" found trait bool '{tbl[k]}'") - result[k] = OrderedDict() - if tbl[k]: - result[k]["value"] = "True" - else: - result[k]["value"] = "False" - result[k]["type"] = "bool" - result[k]["unit"] = "None" - elif isinstance(tbl[k], int): - # print(f" found trait int '{tbl[k]}'") - result[k] = OrderedDict() - result[k]["value"] = "{}".format(tbl[k]) - result[k]["type"] = "int" - result[k]["unit"] = "None" - elif isinstance(tbl[k], float): - # print(f" found trait float '{tbl[k]}'") - result[k] = OrderedDict() - result[k]["value"] = "{:0.14e}".format(tbl[k]) - result[k]["type"] = "float" - result[k]["unit"] = "None" - # print(f" parsed as: {result[k]}") - # print("LOAD toml result = {}".format(result)) - return result - - -def load_toml(file, input=None, comm=None): - """Load a TOML config file. - - This loads the document into a config dictionary. If input is specified, the file - contents are merged into this dictionary. - - Args: - file (str): The file to load. - input (dict): Append to this dictionary. - comm (MPI.Comm): Optional communicator to broadcast across. - - Returns: - (dict): The result. - - """ - raw = None - if comm is None or comm.rank == 0: - with open(file, "r") as f: - raw = loads(f.read()) - if comm is not None: - raw = comm.bcast(raw, root=0) - - # Convert this TOML document into a dictionary - - def convert_node(raw_root, conf_root): - """Helper function to recursively convert tables""" - if isinstance( - raw_root, (tomlkit.toml_document.TOMLDocument, tomlkit.items.Table) - ): - for k in list(raw_root.keys()): - try: - subkeys = list(raw_root[k].keys()) - # This element is table-like. - if "class" in subkeys: - # print("LOAD found traitconfig {}".format(k), flush=True) - conf_root[k] = _load_toml_traits(raw_root[k]) - else: - # This is just a dictionary - conf_root[k] = OrderedDict() - convert_node(raw_root[k], conf_root[k]) - except Exception as e: - # This element is not a sub-table, just copy. - conf_root[k] = raw_root[k] - raise - else: - raise RuntimeError("Cannot convert TOML node {}".format(raw_root)) - - raw_config = OrderedDict() - convert_node(raw, raw_config) - - if input is None: - return raw_config - - # We need to merge results. - _merge_config(raw_config, input) - - return input - - -def _dump_toml_trait(tbl, indent, name, value, unit, typ, help): - if typ == "bool": - # Bools seem to have an issue adding comments. To workaround, we - # add the value as a string, add the comment, and then set it to - # a real bool. - tbl.add(name, "temp") - if help is not None: - tbl[name].comment(help) - tbl[name].indent(indent) - if value == "None": - tbl[name] = "None" - elif value == "True": - tbl[name] = True - else: - tbl[name] = False - else: - if typ == "Quantity": - qval = "None" - if value != "None": - qval = f"{value} {unit}" - tbl.add(name, qval) - elif typ == "Unit": - uval = "unit None" - if unit != "None": - uval = f"unit {unit}" - tbl.add(name, uval) - elif typ in ["list", "set", "tuple"]: - val = "None" - if value != "None": - if isinstance(value, str): - val = ast.literal_eval(value) - else: - val = list() - for elem in value: - if isinstance(elem, tuple): - # This is a quantity / unit - val.append(" ".join(elem)) - else: - val.append(elem) - tbl.add(name, val) - elif typ == "dict": - val = "None" - if value != "None": - if isinstance(value, str): - dval = ast.literal_eval(value) - else: - dval = dict(value) - val = table() - subindent = indent + 2 - for k, v in dval.items(): - if isinstance(v, tuple): - # This is a quantity - val.add(k, " ".join(v)) - else: - val.add(k, v) - val[k].indent(subindent) - tbl.add(name, val) - elif typ == "int": - val = "None" - if value != "None": - val = int(value) - tbl.add(name, val) - elif typ == "float": - val = "None" - if value != "None": - val = float(value) - tbl.add(name, val) - elif typ == "callable": - # Just add None - tbl.add(name, "None") - else: - # This must be a custom class or a str - tbl.add(name, value) - if help is not None: - tbl[name].comment(help) - tbl[name].indent(indent) - - -def dump_toml(file, conf, comm=None): - """Dump a configuration to a TOML file. - - This writes a config dictionary to a TOML file. - - Args: - file (str): The file to write. - conf (dict): The configuration to dump. - comm (MPI.Comm): Optional communicator to control which process writes. - - Returns: - None - - """ - if comm is None or comm.rank == 0: - env = Environment.get() - doc = document() - - doc.add(comment("TOAST config")) - doc.add(comment("Generated with version {}".format(env.version()))) - - def convert_node(conf_root, table_root, indent_size): - """Helper function to recursively convert dictionaries to tables""" - if isinstance(conf_root, (dict, OrderedDict)): - # print("{}found dict".format(" " * indent_size)) - for k in list(conf_root.keys()): - # print("{} examine key {}".format(" " * indent_size, k)) - if isinstance(conf_root[k], (dict, OrderedDict)): - # print("{} key is a dict".format(" " * indent_size)) - if "value" in conf_root[k] and "type" in conf_root[k]: - # this is a trait - unit = None - if "unit" in conf_root[k]: - unit = conf_root[k]["unit"] - help = None - if "help" in conf_root[k]: - help = conf_root[k]["help"] - # print( - # "{} trait {}, type {}, ({}): {}".format( - # " " * indent_size, - # conf_root[k]["value"], - # conf_root[k]["type"], - # unit, - # help, - # ) - # ) - _dump_toml_trait( - table_root, - indent_size, - k, - conf_root[k]["value"], - unit, - conf_root[k]["type"], - help, - ) - else: - # descend tree - # print( - # "{} {} not a trait, descending".format( - # " " * indent_size, k - # ) - # ) - table_root[k] = table() - convert_node(conf_root[k], table_root[k], indent_size + 2) - else: - # print( - # "{} key {} not dict".format(" " * indent_size, k), - # flush=True, - # ) - table_root.add(k, conf_root[k]) - table_root[k].indent(indent_size) - else: - raise RuntimeError("Cannot convert config node {}".format(conf_root)) - - # Convert all top-level sections from the config dictionary into a TOML table. - convert_node(conf, doc, 0) - - with open(file, "w") as f: - f.write(dumps(doc)) - - -def load_json(file, input=None, comm=None): - """Load a JSON config file. - - This loads the document into a config dictionary. If input is specified, the file - contents are merged into this dictionary. - - Args: - file (str): The file to load. - input (dict): Append to this dictionary. - comm (MPI.Comm): Optional communicator to broadcast across. - - Returns: - (dict): The result. - - """ - raw = None - if comm is None or comm.rank == 0: - with open(file, "r") as f: - raw = json.load(f) - if comm is not None: - raw = comm.bcast(raw, root=0) - - if input is None: - return raw - - # We need to merge results. - _merge_config(raw, input) - - return input - - -def dump_json(file, conf, comm=None): - """Dump a configuration to a JSON file. - - This writes a config dictionary to a JSON file. - - Args: - file (str): The file to write. - conf (dict): The configuration to dump. - comm (MPI.Comm): Optional communicator to control which process writes. - - Returns: - None - - """ - if comm is None or comm.rank == 0: - env = Environment.get() - versioned = OrderedDict() - versioned["version"] = env.version() - versioned.update(conf) - - with open(file, "w") as f: - json.dump(versioned, f, indent=2) - - -def load_config(file, input=None, comm=None): - """Load a config file in a supported format. - - This loads the document into a config dictionary. If input is specified, the file - contents are merged into this dictionary. - - Args: - file (str): The file to load. - input (dict): Append to this dictionary. - - Returns: - (dict): The result. - - """ - ret = None - try: - ret = load_json(file, input=input, comm=comm) - except Exception: - ret = load_toml(file, input=input, comm=comm) - return ret - - -def create_from_config(conf): - """Instantiate classes in a configuration. - - This iteratively instantiates classes defined in the configuration, replacing - object names with references to those objects. References to other objects in the - config are specified with the string '@config:' followed by a UNIX-style "path" - where each element of the path is a dictionary key in the config. For example: - - @config:/operators/pointing - - Would reference an object at conf["operators"]["pointing"]. Object references like - this only work if the target of the reference is a built-in type (str, float, int, - etc) or a class derived from TraitConfig. - - Args: - conf (dict): the configuration - - Returns: - (SimpleNamespace): A namespace containing the sections and instantiated - objects specified in the original config structure. - - """ - log = Logger.get() - ref_prefix = "@config:" - ref_pat = re.compile("^{}/(.*)".format(ref_prefix)) - - # Helper functions - - def get_node(tree, cursor): - node = None - try: - node = tree - for c in cursor: - parent = node - node = parent[c] - # We found it! - except: - node = None - return node - - def find_object_ref(top, name): - """ - Return same string if no match, None if matched but nonexistant, or - the object itself. - """ - # print(f"OBJREF get {name}", flush=True) - if not isinstance(name, str): - return name - found = name - mat = ref_pat.match(name) - if mat is not None: - # See if the referenced object exists - path = mat.group(1) - path_keys = path.split("/") - # print("OBJREF checking {}".format(path_keys)) - found = get_node(top, path_keys) - if found is not None: - # It exists, but is this a TraitConfig object that has not yet been - # created? - if isinstance(found, (dict, OrderedDict)) and "class" in found: - # Yes... - found = None - # print("OBJREF found = {}".format(found)) - return found - - def parse_tree(tree, cursor): - unresolved = 0 - # print("PARSE ------------------------") - - # The node at this cursor location - # print("PARSE fetching node at cursor {}".format(cursor)) - node = get_node(tree, cursor) - - # print("PARSE at cursor {} got node {}".format(cursor, node)) - - # The output parent node - parent_cursor = list(cursor) - node_name = parent_cursor.pop() - parent = get_node(tree, parent_cursor) - # print("PARSE at parent {} got node {}".format(parent_cursor, parent)) - - # In terms of this function, "nodes" are always dictionary-like - for child_key in list(node.keys()): - # We are modifying the tree in place, so we get a new reference to our - # node each time. - child_cursor = list(cursor) - child_cursor.append(child_key) - child_val = get_node(tree, child_cursor) - # print(f"PARSE child_key {child_key} = {child_val}") - - if isinstance(child_val, TraitConfig): - # This is an already-created object - continue - elif isinstance(child_val, str): - # print("PARSE child value {} is a string".format(child_val)) - # See if this string is an object reference and try to resolve it. - check = find_object_ref(tree, child_val) - if check is None: - unresolved += 1 - else: - parent[node_name][child_key] = check - else: - is_dict = None - try: - subkeys = child_val.keys() - # Ok, this child is like a dictionary - is_dict = True - except: - is_dict = False - if is_dict: - # print( - # "PARSE child value {} is a dict, descend with cursor {}".format( - # child_val, child_cursor - # ) - # ) - unresolved += parse_tree(tree, child_cursor) - else: - # Not a dictionary - is_list = None - try: - _ = len(child_val) - # It is a list, tuple, or set - is_list = True - except: - is_list = False - if is_list: - parent[node_name][child_key] = list(child_val) - for elem in range(len(child_val)): - test_val = parent[node_name][child_key][elem] - found = find_object_ref(tree, test_val) - # print("find_object {} --> {}".format(test_val, found)) - if found is None: - unresolved += 1 - else: - parent[node_name][child_key][elem] = found - # print("PARSE child value {} is a list".format(child_val)) - else: - # print("PARSE not modifying {}".format(child_val)) - pass - - # If this node is an object and all refs exist, then create it. Otherwise - # leave it alone. - # print( - # "PARSE unresolved = {}, parent[{}] has class? {}".format( - # unresolved, node_name, ("class" in parent[node_name]) - # ) - # ) - if unresolved == 0 and "class" in parent[node_name]: - # We have a TraitConfig object with all references resolved. - # Instantiate it. - # print("PARSE creating TraitConfig {}".format(node_name)) - obj = TraitConfig.from_config(node_name, parent[node_name]) - # print("PARSE instantiated {}".format(obj)) - parent[node_name] = obj - - # print("PARSE VERIFY parent[{}] = {}".format(node_name, parent[node_name])) - # print("PARSE tree now:\n", tree, "\n--------------") - return unresolved - - # Iteratively instantiate objects - - out = copy.deepcopy(conf) - - done = False - last_unresolved = None - - it = 0 - while not done: - # print("PARSE iter ", it) - done = True - unresolved = 0 - for sect in list(out.keys()): - # print("PARSE examine ", sect, "-->", type(out[sect])) - if not isinstance(out[sect], (dict, OrderedDict)): - continue - # print("PARSE section ", sect) - unresolved += parse_tree(out, [sect]) - - if last_unresolved is not None: - if unresolved == last_unresolved: - msg = "Cannot resolve all references in the configuration" - log.error(msg) - raise RuntimeError(msg) - last_unresolved = unresolved - if unresolved > 0: - done = False - it += 1 - - # Convert this recursively into a namespace for easy use - - root_temp = dict() - for sect in list(out.keys()): - sect_ns = types.SimpleNamespace(**out[sect]) - root_temp[sect] = sect_ns - - out_ns = types.SimpleNamespace(**root_temp) - - return out_ns diff --git a/src/toast/config/CMakeLists.txt b/src/toast/config/CMakeLists.txt new file mode 100644 index 000000000..e5e446638 --- /dev/null +++ b/src/toast/config/CMakeLists.txt @@ -0,0 +1,12 @@ + +# Install the python files + +install(FILES + __init__.py + cli.py + utils.py + json.py + toml.py + yaml.py + DESTINATION ${PYTHON_SITE}/toast/config +) diff --git a/src/toast/config/__init__.py b/src/toast/config/__init__.py new file mode 100644 index 000000000..9f162877d --- /dev/null +++ b/src/toast/config/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2015-2024 by the parties listed in the AUTHORS file. +# All rights reserved. Use of this source code is governed by +# a BSD-style license that can be found in the LICENSE file. + +from .cli import ( + add_config_args, + args_update_config, + build_config, + dump_config, + load_config, + parse_config, +) +from .json import dump_json +from .toml import dump_toml +from .yaml import dump_yaml diff --git a/src/toast/config/cli.py b/src/toast/config/cli.py new file mode 100644 index 000000000..5813047b7 --- /dev/null +++ b/src/toast/config/cli.py @@ -0,0 +1,530 @@ +# Copyright (c) 2015-2024 by the parties listed in the AUTHORS file. +# All rights reserved. Use of this source code is governed by +# a BSD-style license that can be found in the LICENSE file. + +import argparse +import ast +import copy +import re +import sys +import types +from collections import OrderedDict +from collections.abc import MutableMapping + +import numpy as np +from astropy import units as u +from tomlkit.exceptions import TOMLKitError + +from ..trait_utils import scalar_to_string as trait_scalar_to_string +from ..trait_utils import string_to_scalar as trait_string_to_scalar +from ..trait_utils import string_to_trait, trait_to_string +from ..utils import Environment, Logger +from .json import dump_json, load_json +from .toml import dump_toml, load_toml +from .yaml import dump_yaml, load_yaml + + +def build_config(objects): + """Build a configuration of current values. + + Args: + objects (list): A list of class instances to add to the config. These objects + must inherit from the TraitConfig base class. + + Returns: + (dict): The configuration. + + """ + conf = OrderedDict() + for o in objects: + if not hasattr(o, "get_config"): + raise RuntimeError("The object list should contain TraitConfig instances") + if o.name is None: + raise RuntimeError("Cannot buid config from objects without a name") + conf = o.get_config(input=conf) + return conf + + +def load_config(file, input=None, comm=None): + """Load a config file in a supported format. + + This loads the document into a config dictionary. If input is specified, the file + contents are merged into this dictionary. + + Args: + file (str): The file to load. + input (dict): Append to this dictionary. + + Returns: + (dict): The result. + + """ + ret = None + try: + ret = load_toml(file, input=input, comm=comm) + except TOMLKitError: + try: + ret = load_json(file, input=input, comm=comm) + except ValueError: + ret = load_yaml(file, input=input, comm=comm) + return ret + + +def dump_config(file, conf, format="toml", comm=None): + """Dump a config file to a supported format. + + Writes the configuration to a file in the specified format. + + Args: + file (str): The file to write. + conf (dict): The configuration to dump. + format (str): The format ("toml", "json", "yaml") + comm (MPI.Comm): Optional communicator to control which process writes. + + Returns: + None + + """ + if format == "toml": + dump_toml(file, conf, comm=comm) + elif format == "json": + dump_json(file, conf, comm=comm) + elif format == "yaml": + dump_yaml(file, conf, comm=comm) + else: + msg = "Unknown config format '{format}'" + raise ValueError(msg) + + +class TraitAction(argparse.Action): + """Custom argparse action to check for valid use of None. + + Some traits support a None value even though they have a specific + type. This custom action checks for that None value and validates + that it is an acceptable value. + + """ + + def __init__( + self, + option_strings, + dest, + nargs=None, + const=None, + default=None, + argtype=None, + choices=None, + required=False, + help=None, + metavar=None, + ): + super(TraitAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=nargs, + const=const, + default=default, + type=None, + choices=choices, + required=required, + help=help, + metavar=metavar, + ) + self.trait_type = argtype + return + + def __call__(self, parser, namespace, values, option_string=None): + if isinstance(values, list): + realval = list() + for v in values: + if v == "None" or v is None: + realval.append(None) + else: + realval.append(v) + else: + if values == "None" or values is None: + realval = None + else: + realval = values + setattr(namespace, self.dest, realval) + + +def add_config_args(parser, conf, section, ignore=list(), prefix="", separator="."): + """Add arguments to an argparser for each parameter in a config dictionary. + + Using a previously created config dictionary, add a commandline argument for each + object parameter in a section of the config. The type, units, and help string for + the commandline argument come from the config, which is in turn built from the + class traits of the object. Boolean parameters are converted to store_true or + store_false actions depending on their current value. + + Containers in the config (list, set, tuple, dict) are parsed as a string to + support easy passing on the commandline, and then later converted to the actual + type. + + Args: + parser (ArgumentParser): The parser to append to. + conf (dict): The configuration dictionary. + section (str): Process objects in this section of the config. + ignore (list): List of object parameters to ignore when adding args. + prefix (str): Prepend this to the beginning of all options. + separator (str): Use this character between the class name and parameter. + + Returns: + None + + """ + parent = conf + if section is not None: + path = section.split("/") + for p in path: + if p not in parent: + msg = "section {} does not exist in config".format(section) + raise RuntimeError(msg) + parent = parent[p] + scalar_types = { + "bool": bool, + "int": int, + "float": float, + "str": str, + "Quantity": u.Quantity, + "Unit": u.Unit, + } + parsable_types = { + "list": list, + "dict": dict, + "tuple": tuple, + "set": set, + } + parsable_types.update(scalar_types) + + for obj, props in parent.items(): + for name, info in props.items(): + if name in ignore: + # Skip this as requested + continue + if name == "class": + # This is not a user-configurable parameter. + continue + if info["type"] not in parsable_types: + # This is not something that we can get from parsing commandline + # options. Skip it. + continue + if info["type"] == "bool": + # Special case for boolean + if name == "enabled": + # Special handling of the TraitConfig "enabled" trait. We provide + # both an enable and disable option for every instance which later + # toggles that trait. + df = True + if info["value"] == "False": + df = False + parser.add_argument( + "--{}{}{}enable".format(prefix, obj, separator), + required=False, + default=df, + action="store_true", + help="Enable use of {}".format(obj), + ) + parser.add_argument( + "--{}{}{}disable".format(prefix, obj, separator), + required=False, + default=(not df), + action="store_false", + help="Disable use of {}".format(obj), + dest="{}{}{}enable".format(prefix, obj, separator), + ) + else: + # General bool option + option = "--{}{}{}{}".format(prefix, obj, separator, name) + act = "store_true" + dflt = False + if info["value"] == "True": + act = "store_false" + dflt = True + option = "--{}{}{}no_{}".format(prefix, obj, separator, name) + parser.add_argument( + option, + required=False, + default=dflt, + action=act, + help=info["help"], + dest="{}{}{}{}".format(prefix, obj, separator, name), + ) + else: + option = "--{}{}{}{}".format(prefix, obj, separator, name) + typ = str + hlp = info["help"] + + # Scalar types are parsed directly. Containers are left + # as strings. + if info["type"] in scalar_types: + typ = scalar_types[info["type"]] + default = trait_to_string(info["value"]) + parser.add_argument( + option, + action=TraitAction, + required=False, + default=default, + argtype=typ, + help=hlp, + ) + return + + +def args_update_config(args, conf, useropts, section, prefix="", separator="\."): + """Override options in a config dictionary from args namespace. + + Args: + args (namespace): The args namespace returned by ArgumentParser.parse_args() + conf (dict): The configuration to update. + useropts (list): The list of options actually specified by the user. + section (str): Process objects in this section of the config. + prefix (str): Prepend this to the beginning of all options. + separator (str): Use this character between the class name and parameter. + + Returns: + (namespace): The un-parsed remaining arg vars. + + """ + # Build the regex match of option names + obj_pat = re.compile("{}(.*?){}(.*)".format(prefix, separator)) + user_pat = re.compile("--{}(.*?){}(.*)".format(prefix, separator)) + no_pat = re.compile("^no_(.*)") + + # Parse the list of user options + user = dict() + for uo in useropts: + user_mat = user_pat.match(uo) + if user_mat is not None: + name = user_mat.group(1) + optname = user_mat.group(2) + if name not in user: + user[name] = set() + # Handle mapping of option names + if optname == "enable" or optname == "disable": + optname = "enabled" + no_mat = no_pat.match(optname) + if no_mat is not None: + optname = no_mat.group(1) + user[name].add(optname) + + remain = copy.deepcopy(args) + parent = conf + if section is not None: + path = section.split("/") + for p in path: + if p not in parent: + msg = "section {} does not exist in config".format(section) + raise RuntimeError(msg) + parent = parent[p] + + for arg, val in vars(args).items(): + obj_mat = obj_pat.match(arg) + if obj_mat is not None: + name = obj_mat.group(1) + optname = obj_mat.group(2) + if name not in parent: + # This command line option is not part of this section + continue + if optname == "enable": + # The actual trait name is "enabled" + optname = "enabled" + + # For each option, convert the parsed value and compare with + # the default config value. If it differs, and the user + # actively set this option, then change it. + # + # NOTE: The TraitAction class correctly assigns empty set defaults to + # the string "set()", but argparse seems to convert that to "{}". So + # here we fix that. + if parent[name][optname]["type"] == "set" and val == "{}": + val = "set()" + + value = trait_to_string(val) + if (value != parent[name][optname]["value"]) and ( + name in user and optname in user[name] + ): + parent[name][optname]["value"] = value + delattr(remain, arg) + return remain + + +def parse_config( + parser, + operators=list(), + templates=list(), + prefix="", + opts=None, +): + """Load command line arguments associated with object properties. + + This function: + + * Adds a "--config" option to the parser which accepts multiple config file + paths to load. + + * Adds "--default_toml" and "--default_json" options to dump the default config + options to files. + + * Adds a option "--job_group_size" to provide the commonly used option for + setting the group size of the TOAST communicator. + + * Adds arguments for all object parameters in the defaults for the specified + classes. + + * Builds a config dictionary starting from the defaults, updating these using + values from any config files, and then applies any overrides from the + commandline. + + Args: + parser (ArgumentParser): The argparse parser. + operators (list): The operator instances to configure from files or + commandline. These instances should have their "name" attribute set to + something meaningful, since that name is used to construct the commandline + options. An enable / disable option is created for each, which toggles the + TraitConfig base class "disabled" trait. + templates (list): The template instances to add to the commandline. + These instances should have their "name" attribute set to something + meaningful, since that name is used to construct the commandline options. + An enable / disable option is created for each, which toggles the + TraitConfig base class "disabled" trait. + prefix (str): Optional string to prefix all options by. + opts (list): If not None, parse arguments from this list instead of sys.argv. + + Returns: + (tuple): The (config dictionary, args). The args namespace contains all the + remaining parameters after extracting the operator and template options. + + """ + + # The default configuration + defaults_op = build_config(operators) + defaults_tmpl = build_config(templates) + + # Add commandline overrides + if len(operators) > 0: + add_config_args( + parser, + defaults_op, + "operators", + ignore=["API"], + prefix=prefix, + ) + if len(templates) > 0: + add_config_args( + parser, + defaults_tmpl, + "templates", + ignore=["API"], + prefix=prefix, + ) + + # Combine all the defaults + defaults = OrderedDict() + defaults["operators"] = OrderedDict() + defaults["templates"] = OrderedDict() + if "operators" in defaults_op: + defaults["operators"].update(defaults_op["operators"]) + if "templates" in defaults_tmpl: + defaults["templates"].update(defaults_tmpl["templates"]) + + # Add an option to load one or more config files. These should have compatible + # names for the operators used in defaults. + parser.add_argument( + "--config", + type=str, + required=False, + nargs="+", + help="One or more input config files.", + ) + + # Add options to dump default values + parser.add_argument( + "--defaults_toml", + type=str, + required=False, + default=None, + help="Dump default config values to a TOML file", + ) + parser.add_argument( + "--defaults_json", + type=str, + required=False, + default=None, + help="Dump default config values to a JSON file", + ) + parser.add_argument( + "--defaults_yaml", + type=str, + required=False, + default=None, + help="Dump default config values to a YAML file", + ) + + parser.add_argument( + "--job_group_size", + required=False, + type=int, + default=None, + help="(Advanced) Size of the process groups", + ) + + parser.add_argument( + "--job_node_mem", + required=False, + type=int, + default=None, + help="(Advanced) Override the detected memory per node in bytes", + ) + + # Parse commandline or list of options. + if opts is None: + opts = sys.argv[1:] + args = parser.parse_args(args=opts) + + # Dump default config values. + if args.defaults_toml is not None: + dump_toml(args.defaults_toml, defaults) + if args.defaults_json is not None: + dump_json(args.defaults_json, defaults) + if args.defaults_yaml is not None: + dump_yaml(args.defaults_yaml, defaults) + + # Parse job args + jobargs = types.SimpleNamespace( + node_mem=args.job_node_mem, + group_size=args.job_group_size, + ) + del args.job_node_mem + del args.job_group_size + + # Load any config files. This overrides default values with config file contents. + config = copy.deepcopy(defaults) + + if args.config is not None: + for conf in args.config: + config = load_config(conf, input=config) + + # Parse operator and template commandline options. These override any config + # file or default values. + if len(operators) > 0: + op_remaining = args_update_config( + args, config, opts, "operators", prefix=prefix + ) + else: + op_remaining = args + if len(templates) > 0: + remaining = args_update_config( + op_remaining, config, opts, "templates", prefix=prefix + ) + else: + remaining = op_remaining + + # Remove the options we created in this function + del remaining.config + del remaining.defaults_toml + del remaining.defaults_json + del remaining.defaults_yaml + + return config, remaining, jobargs diff --git a/src/toast/config/json.py b/src/toast/config/json.py new file mode 100644 index 000000000..d0cd0f648 --- /dev/null +++ b/src/toast/config/json.py @@ -0,0 +1,86 @@ +# Copyright (c) 2015-2024 by the parties listed in the AUTHORS file. +# All rights reserved. Use of this source code is governed by +# a BSD-style license that can be found in the LICENSE file. +"""Tools for reading and writing a configuration dictionary to JSON. + +The internal config dictionary stores all values as strings. For +JSON format config documents we serialize the config dictionary +directly and keep all values as strings. This makes it a trivial +format to dump, load, send, and archive. + +""" + +import json +from collections import OrderedDict + +from ..utils import Environment +from .utils import merge_config + + +def _load_json_traits(tbl): + """Load traits for a single TraitConfig object.""" + result = OrderedDict() + for k in list(tbl.keys()): + if k == "class": + result[k] = tbl[k] + else: + # This is a trait + result[k] = OrderedDict() + result[k]["value"] = tbl[k]["value"] + result[k]["type"] = tbl[k]["type"] + return result + + +def load_json(file, input=None, comm=None): + """Load a JSON config file. + + This loads the document into a config dictionary. If input is specified, the file + contents are merged into this dictionary. + + Args: + file (str): The file to load. + input (dict): Append to this dictionary. + comm (MPI.Comm): Optional communicator to broadcast across. + + Returns: + (dict): The result. + + """ + raw = None + if comm is None or comm.rank == 0: + with open(file, "r") as f: + raw = json.load(f, object_pairs_hook=OrderedDict) + if comm is not None: + raw = comm.bcast(raw, root=0) + + if input is None: + return raw + + # We need to merge results. + merge_config(raw, input) + + return input + + +def dump_json(file, conf, comm=None): + """Dump a configuration to a JSON file. + + This writes a config dictionary to a JSON file. + + Args: + file (str): The file to write. + conf (dict): The configuration to dump. + comm (MPI.Comm): Optional communicator to control which process writes. + + Returns: + None + + """ + if comm is None or comm.rank == 0: + env = Environment.get() + versioned = OrderedDict() + versioned["version"] = env.version() + versioned.update(conf) + + with open(file, "w") as f: + json.dump(versioned, f, indent=2) diff --git a/src/toast/config/toml.py b/src/toast/config/toml.py new file mode 100644 index 000000000..a7994a7b8 --- /dev/null +++ b/src/toast/config/toml.py @@ -0,0 +1,253 @@ +# Copyright (c) 2015-2024 by the parties listed in the AUTHORS file. +# All rights reserved. Use of this source code is governed by +# a BSD-style license that can be found in the LICENSE file. +"""Tools for reading and writing a configuration dictionary to TOML. + +The internal config dictionary stores all values as strings. For +data types supported by the TOML standard, we convert these strings +back to their native values for ease of use and editing. + +""" + +from collections import OrderedDict + +import tomlkit +from astropy import units as u +from tomlkit import comment, document, dumps, loads, nl, table + +from ..trait_utils import string_to_trait, trait_to_string +from ..utils import Environment, Logger +from .utils import merge_config + + +def _load_toml_element(elem): + log = Logger.get() + # See if we are loading one of the TOML supported types + if isinstance(elem, bool): + if elem: + return ("True", "bool") + else: + return ("False", "bool") + if isinstance(elem, int): + return (trait_to_string(elem), "int") + if isinstance(elem, float): + return (trait_to_string(elem), "float") + + # In old TOML files, lists were stored as TOMLKit arrays. + # Here we cast those cases into strings to be parsed like + # modern containers (which are stored as strings in the + # the TOML files). Remove this code after a suitable + # deprecation period. + if isinstance(elem, list): + msg = f"Storing trait '{elem}' as a TOML array is deprecated. " + msg += f"All containers should be stored as a string representation." + msg += f"You can use toast_config_verify to update your config files." + log.warning(msg) + elist = "[" + for el in elem: + if isinstance(el, str): + el = string_to_trait(el) + if isinstance(el, str) and len(el) > 0: + elist += f"'{trait_to_string(el)}'," + else: + elist += f"{trait_to_string(el)}," + else: + elist += f"{trait_to_string(el)}," + elist += "]" + elem = elist + + # This is a string, which might represent a quantity or + # some complicated container. Convert to a value that + # we can use to determine the type. + val = string_to_trait(elem) + if val is None: + return ("None", "unknown") + if isinstance(val, u.UnitBase): + return (trait_to_string(val), "Unit") + if isinstance(val, u.Quantity): + return (trait_to_string(val), "Quantity") + if isinstance(val, set): + return (trait_to_string(val), "set") + if isinstance(val, list): + return (trait_to_string(val), "list") + if isinstance(val, tuple): + return (trait_to_string(val), "tuple") + if isinstance(val, dict): + return (trait_to_string(val), "dict") + + # This is just a string + return (elem, "str") + + +def _load_toml_traits(tbl): + """Load traits for a single TraitConfig object from the TOML.""" + # print("LOAD TraitConfig object {}".format(tbl), flush=True) + result = OrderedDict() + for k in list(tbl.keys()): + if k == "class": + # print(f" found trait class '{tbl[k]}'") + result[k] = tbl[k] + else: + # This is a trait + result[k] = OrderedDict() + result[k]["value"], result[k]["type"] = _load_toml_element(tbl[k]) + # print(f" parsed as: {result[k]}") + # print("LOAD toml result = {}".format(result)) + return result + + +def load_toml(file, input=None, comm=None): + """Load a TOML config file. + + This loads the document into a config dictionary. If input is specified, the file + contents are merged into this dictionary. + + Args: + file (str): The file to load. + input (dict): Append to this dictionary. + comm (MPI.Comm): Optional communicator to broadcast across. + + Returns: + (dict): The result. + + """ + raw = None + if comm is None or comm.rank == 0: + with open(file, "r") as f: + raw = loads(f.read()) + if comm is not None: + raw = comm.bcast(raw, root=0) + + # Convert this TOML document into a dictionary + + def convert_node(raw_root, conf_root): + """Helper function to recursively convert tables""" + if isinstance( + raw_root, (tomlkit.toml_document.TOMLDocument, tomlkit.items.Table) + ): + for k in list(raw_root.keys()): + try: + subkeys = list(raw_root[k].keys()) + # This element is table-like. + if "class" in subkeys: + # print("LOAD found traitconfig {}".format(k), flush=True) + conf_root[k] = _load_toml_traits(raw_root[k]) + else: + # This is just a dictionary + conf_root[k] = OrderedDict() + convert_node(raw_root[k], conf_root[k]) + except Exception as e: + # This element is not a sub-table, just copy. + conf_root[k] = raw_root[k] + raise + else: + raise RuntimeError("Cannot convert TOML node {}".format(raw_root)) + + raw_config = OrderedDict() + convert_node(raw, raw_config) + + if input is None: + return raw_config + + # We need to merge results. + merge_config(raw_config, input) + + return input + + +def _dump_toml_element(elem): + # Convert the config string into a value so that + # we can encode it properly. + val = string_to_trait(elem) + if isinstance(val, bool): + return val + if isinstance(val, int): + return val + if isinstance(val, float): + return val + else: + # Leave this as the string representation + return elem + + +def _dump_toml_trait(tbl, indent, name, value, typ, help): + # Get the TOML-compatible value + val = _dump_toml_element(value) + # print(f"dump[{indent}] {name} ({typ}): '{value}' --> |{val}|", flush=True) + if typ == "bool": + # Bools seem to have an issue adding comments. To workaround, we + # add the value as a string, add the comment, and then set it to + # a real bool. + tbl.add(name, "temp") + if help is not None: + tbl[name].comment(help) + tbl[name].indent(indent) + if val == "None": + tbl[name] = "None" + elif val: + tbl[name] = True + else: + tbl[name] = False + else: + tbl.add(name, val) + if help is not None: + tbl[name].comment(help) + tbl[name].indent(indent) + + +def dump_toml(file, conf, comm=None): + """Dump a configuration to a TOML file. + + This writes a config dictionary to a TOML file. + + Args: + file (str): The file to write. + conf (dict): The configuration to dump. + comm (MPI.Comm): Optional communicator to control which process writes. + + Returns: + None + + """ + + def convert_node(conf_root, table_root, indent_size): + """Helper function to recursively convert dictionaries to tables""" + if isinstance(conf_root, (dict, OrderedDict)): + # print("{}found dict".format(" " * indent_size)) + for k in list(conf_root.keys()): + # print("{} examine key {}".format(" " * indent_size, k)) + if isinstance(conf_root[k], (dict, OrderedDict)): + # print("{} key is a dict".format(" " * indent_size)) + if "value" in conf_root[k] and "type" in conf_root[k]: + # this is a trait + help = None + if "help" in conf_root[k]: + help = conf_root[k]["help"] + _dump_toml_trait( + table_root, + indent_size, + k, + conf_root[k]["value"], + conf_root[k]["type"], + help, + ) + else: + table_root[k] = table() + convert_node(conf_root[k], table_root[k], indent_size + 2) + else: + table_root.add(k, conf_root[k]) + table_root[k].indent(indent_size) + else: + raise RuntimeError("Cannot convert config node {}".format(conf_root)) + + if comm is None or comm.rank == 0: + env = Environment.get() + doc = document() + doc.add(comment("TOAST config")) + doc.add(comment("Generated with version {}".format(env.version()))) + + # Convert all top-level sections from the config dictionary into a TOML table. + convert_node(conf, doc, 0) + + with open(file, "w") as f: + f.write(dumps(doc)) diff --git a/src/toast/config/utils.py b/src/toast/config/utils.py new file mode 100644 index 000000000..c1ffb5aca --- /dev/null +++ b/src/toast/config/utils.py @@ -0,0 +1,57 @@ +# Copyright (c) 2015-2024 by the parties listed in the AUTHORS file. +# All rights reserved. Use of this source code is governed by +# a BSD-style license that can be found in the LICENSE file. + +from ..utils import Environment, Logger + + +def merge_config(loaded, original): + log = Logger.get() + for section, objs in loaded.items(): + if section in original.keys(): + # We have this section + if isinstance(original[section], str): + # This is not a real section, but rather a string value of + # top-level metadata. Just copy it into place, overriding + # the existing value. + original[section] = objs + continue + for objname, objprops in objs.items(): + if objname not in original[section]: + # This is a new object + original[section][objname] = objprops + else: + # Only update the value, while preserving + # any pre-existing type information. + for k, v in objprops.items(): + if k == "class": + continue + if k in original[section][objname]: + # This key exists in the original object traits + cursor = original[section][objname][k] + cursor["value"] = v["value"] + if "type" not in cursor: + cursor["type"] = v["type"] + elif ( + v["type"] is not None + and v["type"] != "unknown" + and cursor["type"] != "enum" + ): + # The loaded data has at type, check for consistency + if v["type"] != cursor["type"]: + msg = f"Loaded trait {v} has different type than " + msg += f"existing trait {cursor}" + raise ValueError(msg) + else: + # We have a config option that does not exist + # in the current object. Warn user that this may + # indicate a stale config file. + msg = f"Object {objname} currently has no configuration" + msg += f" trait '{k}'. This might be handled by the " + msg += f"class through API translation, but your config " + msg += f"file may be out of date." + log.warning(msg) + original[section][objname][k] = v + else: + # A new section + original[section] = objs diff --git a/src/toast/config/yaml.py b/src/toast/config/yaml.py new file mode 100644 index 000000000..a6449fbdb --- /dev/null +++ b/src/toast/config/yaml.py @@ -0,0 +1,210 @@ +# Copyright (c) 2015-2024 by the parties listed in the AUTHORS file. +# All rights reserved. Use of this source code is governed by +# a BSD-style license that can be found in the LICENSE file. +"""Tools for reading and writing a configuration dictionary to YAML. + +The internal config dictionary stores all values as strings. For +data types supported by the YAML standard, we convert these strings +back to their native values for ease of use and editing. + +""" + +from collections import OrderedDict + +from astropy import units as u +from ruamel.yaml import YAML, CommentedMap + +from ..trait_utils import string_to_trait, trait_to_string +from ..utils import Environment, Logger +from .utils import merge_config + +yaml = YAML() + + +def _dump_yaml_element(elem): + # Convert the config string into a value so that + # we can encode it properly. + val = string_to_trait(elem) + if val is None: + return None + if isinstance(val, bool): + return val + if isinstance(val, int): + return val + if isinstance(val, float): + return val + else: + # Leave this as the string representation + return elem + + +def _dump_yaml_trait(cm, name, value, typ, help): + # Get the YAML-compatible value and insert into the commented map. + val = _dump_yaml_element(value) + # print(f"YAML dump {name}: {val} ({typ}) # {help}", flush=True) + cm[name] = val + if help is None or help == "None" or help == "": + cm.yaml_add_eol_comment("(help string missing)", key=name) + else: + cm.yaml_add_eol_comment(help, key=name) + + +def dump_yaml(file, conf, comm=None): + """Dump a configuration to a YAML file. + + This writes a config dictionary to a YAML file. + + Args: + file (str): The file to write. + conf (dict): The configuration to dump. + comm (MPI.Comm): Optional communicator to control which process writes. + + Returns: + None + + """ + + def convert_node(conf_root, cm_root): + """Helper function to recursively convert dictionaries.""" + if isinstance(conf_root, (dict, OrderedDict)): + # print("{}found dict".format(" " * indent_size)) + for k in list(conf_root.keys()): + # print("{} examine key {}".format(" " * indent_size, k)) + if isinstance(conf_root[k], (dict, OrderedDict)): + # print("{} key is a dict".format(" " * indent_size)) + if "value" in conf_root[k] and "type" in conf_root[k]: + # this is a trait + help = None + if "help" in conf_root[k]: + help = conf_root[k]["help"] + _dump_yaml_trait( + cm_root, + k, + conf_root[k]["value"], + conf_root[k]["type"], + help, + ) + else: + cm_root[k] = CommentedMap() + convert_node(conf_root[k], cm_root[k]) + else: + cm_root[k] = conf_root[k] + else: + raise RuntimeError(f"Cannot convert config node {conf_root}") + + if comm is None or comm.rank == 0: + env = Environment.get() + doc = CommentedMap() + doc.yaml_set_start_comment( + f"TOAST config generated with version {env.version()}" + ) + convert_node(conf, doc) + # print(f"YAML dump final = {doc}", flush=True) + with open(file, "w") as f: + yaml.dump(doc, f) + + +def _load_yaml_element(elem): + # See if we are loading one of the YAML supported scalar types + if isinstance(elem, bool): + if elem: + return ("True", "bool") + else: + return ("False", "bool") + if isinstance(elem, int): + return (trait_to_string(elem), "int") + if isinstance(elem, float): + return (trait_to_string(elem), "float") + if isinstance(elem, list): + return (trait_to_string(elem), "list") + + # This is a string, which might represent a quantity or + # some complicated container. Convert to a value that + # we can use to determine the type. + val = string_to_trait(elem) + if val is None: + return ("None", "unknown") + if isinstance(val, u.UnitBase): + return (trait_to_string(val), "Unit") + if isinstance(val, u.Quantity): + return (trait_to_string(val), "Quantity") + if isinstance(val, set): + return (trait_to_string(val), "set") + if isinstance(val, list): + return (trait_to_string(val), "list") + if isinstance(val, tuple): + return (trait_to_string(val), "tuple") + if isinstance(val, dict): + return (trait_to_string(val), "dict") + + # This is just a string + return (elem, "str") + + +def _load_yaml_traits(tbl): + """Load traits for a single TraitConfig object.""" + result = OrderedDict() + for k in list(tbl.keys()): + if k == "class": + # print(f" found trait class '{tbl[k]}'") + result[k] = tbl[k] + else: + # This is a trait + result[k] = OrderedDict() + result[k]["value"], result[k]["type"] = _load_yaml_element(tbl[k]) + return result + + +def load_yaml(file, input=None, comm=None): + """Load a YAML config file. + + This loads the document into a config dictionary. If input is specified, the file + contents are merged into this dictionary. + + Args: + file (str): The file to load. + input (dict): Append to this dictionary. + comm (MPI.Comm): Optional communicator to broadcast across. + + Returns: + (dict): The result. + + """ + raw = None + if comm is None or comm.rank == 0: + with open(file, "r") as f: + raw = yaml.load(f) + if comm is not None: + raw = comm.bcast(raw, root=0) + + # Parse the doc into a config dictionary + def convert_node(raw_root, conf_root): + """Helper function to recursively convert tables""" + if isinstance(raw_root, dict): + for k in list(raw_root.keys()): + try: + subkeys = list(raw_root[k].keys()) + # This element is table-like. + if "class" in subkeys: + conf_root[k] = _load_yaml_traits(raw_root[k]) + else: + # This is just a dictionary + conf_root[k] = OrderedDict() + convert_node(raw_root[k], conf_root[k]) + except Exception as e: + # This element is not a sub-table, just copy. + conf_root[k] = raw_root[k] + raise + else: + raise RuntimeError("Cannot convert YAML node {}".format(raw_root)) + + raw_config = OrderedDict() + convert_node(raw, raw_config) + + if input is None: + return raw_config + + # We need to merge results. + merge_config(raw_config, input) + + return input diff --git a/src/toast/jax/intervals.py b/src/toast/jax/intervals.py index abc01d066..e96e43b69 100644 --- a/src/toast/jax/intervals.py +++ b/src/toast/jax/intervals.py @@ -1,5 +1,6 @@ import jax import numpy as np + from ..timing import function_timer diff --git a/src/toast/jax/maps.py b/src/toast/jax/maps.py index 928f30096..099f0e8b4 100644 --- a/src/toast/jax/maps.py +++ b/src/toast/jax/maps.py @@ -1,10 +1,12 @@ -import jax -from jax import vmap, lax -from jax import numpy as jnp +import itertools from copy import deepcopy +from inspect import Parameter, Signature from types import EllipsisType -from inspect import Signature, Parameter -import itertools + +import jax +from jax import lax +from jax import numpy as jnp +from jax import vmap # ---------------------------------------------------------------------------------------- # PYTREE FUNCTIONS diff --git a/src/toast/ops/azimuth_intervals.py b/src/toast/ops/azimuth_intervals.py index 21f65dd4d..8bf9025ba 100644 --- a/src/toast/ops/azimuth_intervals.py +++ b/src/toast/ops/azimuth_intervals.py @@ -14,9 +14,8 @@ from ..data import Data from ..mpi import MPI from ..observation import default_values as defaults -from ..timing import function_timer +from ..timing import Timer, function_timer from ..traits import Bool, Float, Int, Unicode, trait_docs -from ..timing import Timer from ..utils import Environment, Logger, rate_from_times from ..vis import set_matplotlib_backend from .operator import Operator diff --git a/src/toast/ops/cadence_map.py b/src/toast/ops/cadence_map.py index 56b83818a..4645c0447 100644 --- a/src/toast/ops/cadence_map.py +++ b/src/toast/ops/cadence_map.py @@ -16,7 +16,7 @@ from ..data import Data from ..mpi import MPI, Comm, MPI_Comm, use_mpi from ..observation import default_values as defaults -from ..timing import function_timer, Timer, GlobalTimers +from ..timing import GlobalTimers, Timer, function_timer from ..traits import Bool, Instance, Int, Unicode, trait_docs from ..utils import Environment, Logger, dtype_to_aligned from .operator import Operator diff --git a/src/toast/ops/conviqt.py b/src/toast/ops/conviqt.py index 86b0f2f72..4f13916de 100644 --- a/src/toast/ops/conviqt.py +++ b/src/toast/ops/conviqt.py @@ -14,14 +14,9 @@ from .. import qarray as qa from ..mpi import MPI, Comm, MPI_Comm, use_mpi from ..observation import default_values as defaults -from ..timing import function_timer, Timer +from ..timing import Timer, function_timer from ..traits import Bool, Dict, Instance, Int, Quantity, Unicode, Unit, trait_docs -from ..utils import ( - Environment, - Logger, - dtype_to_aligned, - unit_conversion, -) +from ..utils import Environment, Logger, dtype_to_aligned, unit_conversion from .operator import Operator conviqt = None diff --git a/src/toast/ops/demodulation.py b/src/toast/ops/demodulation.py index 274805dcc..52b53d301 100644 --- a/src/toast/ops/demodulation.py +++ b/src/toast/ops/demodulation.py @@ -20,7 +20,7 @@ from ..noise import Noise from ..observation import Observation from ..observation import default_values as defaults -from ..timing import function_timer, Timer +from ..timing import Timer, function_timer from ..traits import Bool, Instance, Int, Quantity, Unicode, trait_docs from ..utils import Logger, dtype_to_aligned, name_UID from .operator import Operator diff --git a/src/toast/ops/flag_sso.py b/src/toast/ops/flag_sso.py index e0d95ffa1..877e5cfe3 100644 --- a/src/toast/ops/flag_sso.py +++ b/src/toast/ops/flag_sso.py @@ -15,7 +15,7 @@ from ..data import Data from ..mpi import MPI from ..observation import default_values as defaults -from ..timing import function_timer, Timer +from ..timing import Timer, function_timer from ..traits import Bool, Float, Instance, Int, List, Quantity, Unicode, trait_docs from ..utils import Environment, Logger from .operator import Operator diff --git a/src/toast/ops/groundfilter.py b/src/toast/ops/groundfilter.py index cd60f3b46..569782a86 100644 --- a/src/toast/ops/groundfilter.py +++ b/src/toast/ops/groundfilter.py @@ -13,7 +13,7 @@ from ..data import Data from ..mpi import MPI from ..observation import default_values as defaults -from ..timing import function_timer, Timer +from ..timing import Timer, function_timer from ..traits import Bool, Int, Unicode, trait_docs from ..utils import Environment, Logger from .operator import Operator diff --git a/src/toast/ops/hwpfilter.py b/src/toast/ops/hwpfilter.py index 189eaca00..8de9a906b 100644 --- a/src/toast/ops/hwpfilter.py +++ b/src/toast/ops/hwpfilter.py @@ -12,7 +12,7 @@ from ..data import Data from ..mpi import MPI from ..observation import default_values as defaults -from ..timing import function_timer, Timer +from ..timing import Timer, function_timer from ..traits import Bool, Int, Unicode, trait_docs from ..utils import Environment, Logger from .operator import Operator diff --git a/src/toast/ops/madam.py b/src/toast/ops/madam.py index 03b36449c..89088c291 100644 --- a/src/toast/ops/madam.py +++ b/src/toast/ops/madam.py @@ -12,7 +12,7 @@ from ..mpi import MPI, use_mpi from ..observation import default_values as defaults from ..templates import Offset -from ..timing import function_timer, Timer +from ..timing import Timer, function_timer from ..traits import Bool, Dict, Instance, Int, Unicode, trait_docs from ..utils import Environment, Logger, dtype_to_aligned from .delete import Delete diff --git a/src/toast/ops/noise_estimation.py b/src/toast/ops/noise_estimation.py index 45f1abf31..dab194bed 100644 --- a/src/toast/ops/noise_estimation.py +++ b/src/toast/ops/noise_estimation.py @@ -13,7 +13,7 @@ from ..noise import Noise from ..observation import default_values as defaults -from ..timing import function_timer, Timer +from ..timing import Timer, function_timer from ..traits import Bool, Instance, Int, List, Quantity, Tuple, Unicode, trait_docs from ..utils import Logger from .arithmetic import Combine diff --git a/src/toast/ops/pointing_detector/kernels_jax.py b/src/toast/ops/pointing_detector/kernels_jax.py index 51ffb3be0..c594c5d8b 100644 --- a/src/toast/ops/pointing_detector/kernels_jax.py +++ b/src/toast/ops/pointing_detector/kernels_jax.py @@ -4,6 +4,7 @@ import jax import jax.numpy as jnp + from ...accelerator import ImplementationType, kernel from ...jax.intervals import INTERVALS_JAX from ...jax.maps import imap diff --git a/src/toast/ops/polyfilter/polyfilter.py b/src/toast/ops/polyfilter/polyfilter.py index 2aa89cbcf..4979f3a47 100644 --- a/src/toast/ops/polyfilter/polyfilter.py +++ b/src/toast/ops/polyfilter/polyfilter.py @@ -16,15 +16,9 @@ from ...accelerator import ImplementationType from ...mpi import MPI, Comm, MPI_Comm, use_mpi from ...observation import default_values as defaults -from ...timing import function_timer, GlobalTimers, Timer +from ...timing import GlobalTimers, Timer, function_timer from ...traits import Bool, Dict, Instance, Int, Quantity, Unicode, UseEnum, trait_docs -from ...utils import ( - AlignedF64, - AlignedU8, - Environment, - Logger, - dtype_to_aligned, -) +from ...utils import AlignedF64, AlignedU8, Environment, Logger, dtype_to_aligned from ..operator import Operator from .kernels import filter_poly2D, filter_polynomial diff --git a/src/toast/ops/sim_tod_atm.py b/src/toast/ops/sim_tod_atm.py index d1e654cdd..f744dd0df 100644 --- a/src/toast/ops/sim_tod_atm.py +++ b/src/toast/ops/sim_tod_atm.py @@ -14,7 +14,7 @@ from ..data import Data from ..mpi import MPI from ..observation import default_values as defaults -from ..timing import function_timer, Timer +from ..timing import Timer, function_timer from ..traits import Bool, Float, Instance, Int, Quantity, Unicode, Unit, trait_docs from ..utils import Environment, Logger from .operator import Operator diff --git a/src/toast/ops/sim_tod_atm_observe.py b/src/toast/ops/sim_tod_atm_observe.py index 28552c520..1490101a6 100644 --- a/src/toast/ops/sim_tod_atm_observe.py +++ b/src/toast/ops/sim_tod_atm_observe.py @@ -15,7 +15,7 @@ from ..mpi import MPI from ..observation import default_values as defaults from ..observation_dist import global_interval_times -from ..timing import GlobalTimers, function_timer, Timer +from ..timing import GlobalTimers, Timer, function_timer from ..traits import Bool, Float, Int, Quantity, Unicode, Unit, trait_docs from ..utils import Environment, Logger, unit_conversion from .operator import Operator diff --git a/src/toast/ops/stokes_weights/kernels_jax.py b/src/toast/ops/stokes_weights/kernels_jax.py index 143d377e2..5311bc401 100644 --- a/src/toast/ops/stokes_weights/kernels_jax.py +++ b/src/toast/ops/stokes_weights/kernels_jax.py @@ -12,7 +12,6 @@ from ...jax.mutableArray import MutableJaxArray from ...utils import Logger - # ---------------------------------------------------------------------------------------- # IQU diff --git a/src/toast/ops/totalconvolve.py b/src/toast/ops/totalconvolve.py index f5a91e200..97fa8dd7e 100644 --- a/src/toast/ops/totalconvolve.py +++ b/src/toast/ops/totalconvolve.py @@ -13,7 +13,7 @@ from .. import qarray as qa from ..mpi import MPI, Comm, MPI_Comm, use_mpi from ..observation import default_values as defaults -from ..timing import function_timer, Timer +from ..timing import Timer, function_timer from ..traits import ( Bool, Dict, diff --git a/src/toast/scripts/toast_config_verify.py b/src/toast/scripts/toast_config_verify.py index b88fd9f16..4c045433d 100644 --- a/src/toast/scripts/toast_config_verify.py +++ b/src/toast/scripts/toast_config_verify.py @@ -11,9 +11,10 @@ import re import sys import traceback +from collections import OrderedDict import toast -from toast.config import dump_toml, load_config +from toast.config import dump_json, dump_toml, dump_yaml, load_config from toast.mpi import Comm, get_world from toast.utils import Environment, Logger, import_from_name, object_fullname @@ -36,7 +37,17 @@ def main(): in_class = False in_conf = False while iop < len(opts): - if opts[iop] == "--out": + if opts[iop] == "--out_toml": + user_opts.append(opts[iop]) + iop += 1 + user_opts.append(opts[iop]) + iop += 1 + if opts[iop] == "--out_json": + user_opts.append(opts[iop]) + iop += 1 + user_opts.append(opts[iop]) + iop += 1 + if opts[iop] == "--out_yaml": user_opts.append(opts[iop]) iop += 1 user_opts.append(opts[iop]) @@ -76,11 +87,25 @@ def main(): parser = argparse.ArgumentParser(description=help) parser.add_argument( - "--out", + "--out_toml", required=False, - default="out.toml", + default=None, type=str, - help="The output config file to write", + help="The output TOML config file to write", + ) + parser.add_argument( + "--out_yaml", + required=False, + default=None, + type=str, + help="The output YAML config file to write", + ) + parser.add_argument( + "--out_json", + required=False, + default=None, + type=str, + help="The output JSON config file to write", ) parser.add_argument( "--class", @@ -139,8 +164,23 @@ def main(): opts=conf_opts, ) + # Instantiate everything and then convert back to a config for dumping. + # This will automatically prune stale traits, etc. + run = toast.create_from_config(config) + out_config = OrderedDict() + for sect_key, sect_val in vars(run).items(): + obj_list = list() + for obj_name, obj in vars(sect_val).items(): + obj_list.append(obj) + out_config[sect_key] = toast.config.build_config(obj_list) + # Write the final config out - dump_toml(user_args.out, config) + if user_args.out_toml is not None: + dump_toml(user_args.out_toml, out_config, comm=mpiworld) + if user_args.out_json is not None: + dump_json(user_args.out_json, out_config, comm=mpiworld) + if user_args.out_yaml is not None: + dump_yaml(user_args.out_yaml, out_config, comm=mpiworld) return diff --git a/src/toast/tests/accelerator.py b/src/toast/tests/accelerator.py index 305e44135..fc75a4b71 100644 --- a/src/toast/tests/accelerator.py +++ b/src/toast/tests/accelerator.py @@ -34,6 +34,7 @@ if use_accel_jax: import jax + from ..jax.mutableArray import MutableJaxArray, _zero_out_jitted diff --git a/src/toast/tests/config.py b/src/toast/tests/config.py index aaa03f988..4c4b2769f 100644 --- a/src/toast/tests/config.py +++ b/src/toast/tests/config.py @@ -16,8 +16,10 @@ from .. import ops from ..config import ( build_config, - create_from_config, + dump_config, + dump_json, dump_toml, + dump_yaml, load_config, parse_config, ) @@ -25,6 +27,7 @@ from ..instrument import Focalplane, Telescope from ..schedule_sim_satellite import create_satellite_schedule from ..templates import Offset, SubHarmonic +from ..trait_utils import string_to_trait, trait_to_string from ..traits import ( Bool, Dict, @@ -37,8 +40,8 @@ Tuple, Unicode, Unit, + create_from_config, trait_docs, - trait_scalar_to_string, ) from ..utils import Environment, Logger from ._helpers import close_data, create_comm, create_outdir, create_space_telescope @@ -126,6 +129,19 @@ class ConfigOperator(ops.Operator): (None, True, "", "foo", 4.56, 7.89 * u.meter), help="Tuple mixed" ) + list_of_tuples = List( + [(None, True), ("foo", 1.23), (4.56, 7.89 * u.meter)], help="list of tuples" + ) + + dict_of_lists_of_tuples = Dict( + { + "a": (None, True), + "b": ("foo", 1.23), + "c": (4.56, 7.89 * u.meter), + }, + help="dict of list of tuples", + ) + def __init__(self, **kwargs): super().__init__(**kwargs) @@ -203,34 +219,26 @@ def args_test(self): targs.append("None") elif isinstance(v, set): if len(v) == 0: - targs.append("{}") + targs.append("set()") else: - formatted = set([trait_scalar_to_string(x) for x in v]) - targs.append(str(formatted)) + targs.append(trait_to_string(v)) elif isinstance(v, tuple): if len(v) == 0: targs.append("()") else: - formatted = tuple([trait_scalar_to_string(x) for x in v]) - targs.append(str(formatted)) + targs.append(trait_to_string(v)) elif isinstance(v, dict): if len(v) == 0: targs.append("{}") else: - formatted = {x: trait_scalar_to_string(y) for x, y in v.items()} - targs.append(str(formatted)) + targs.append(trait_to_string(v)) elif isinstance(v, list): if len(v) == 0: targs.append("[]") else: - formatted = [trait_scalar_to_string(x) for x in v] - targs.append(str(formatted)) - elif isinstance(v, u.Unit): - targs.append(str(v)) - elif isinstance(v, u.Quantity): - targs.append(f"{v.value:0.14e} {v.unit}") + targs.append(trait_to_string(v)) else: - targs.append(str(v)) + targs.append(trait_to_string(v)) return targs def _exec(self, data, detectors=None, **kwargs): @@ -308,6 +316,24 @@ def create_templates(self): tmpls = [Offset(name="baselines"), SubHarmonic(name="subharmonic")] return {x.name: x for x in tmpls} + def test_trait_utils(self): + fake = ConfigOperator(name="fake") + tdict = dict() + for trait_name, trait in fake.traits().items(): + tdict[trait_name] = trait_to_string(trait.get(fake)) + + check_fake = ConfigOperator(name="check_fake") + for trait_name, trait in check_fake.traits().items(): + trait.set(check_fake, string_to_trait(tdict[trait_name])) + + # Compare + for tname, trait in check_fake.traits().items(): + oval = trait.get(fake) + tval = trait.get(check_fake) + if not self.compare_trait(tval, oval): + print(f"{tval} != {oval}") + self.assertTrue(False) + def test_trait_types(self): fake = ConfigOperator(name="fake") @@ -336,7 +362,6 @@ def test_trait_types(self): def test_trait_types_argparse(self): fake = ConfigOperator(name="fake") - test_args = fake.args_test() parser = argparse.ArgumentParser(description="Test") @@ -364,105 +389,123 @@ def test_trait_types_argparse(self): self.assertTrue(False) def test_config_multi(self): - testops = self.create_operators() - testops["fake"] = ConfigOperator(name="fake") - testops["scan_map"] = ops.ScanHealpixMap(name="scan_map") + for case in ["toml", "json", "yaml"]: + testops = self.create_operators() + testops["fake"] = ConfigOperator(name="fake") + testops["scan_map"] = ops.ScanHealpixMap(name="scan_map") - objs = [y for x, y in testops.items()] - defaults = build_config(objs) + objs = [y for x, y in testops.items()] + defaults = build_config(objs) - conf_defaults_file = os.path.join(self.outdir, "multi_defaults.toml") - if self.toastcomm.world_rank == 0: - dump_toml(conf_defaults_file, defaults) - if self.toastcomm.comm_world is not None: - self.toastcomm.comm_world.barrier() - - # Now change some values - testops["mem_count"].prefix = "newpref" - testops["mem_count"].enabled = False - testops["sim_noise"].serial = False - testops["sim_satellite"].hwp_rpm = 8.0 - testops["sim_satellite"].distribute_time = True - testops["sim_satellite"].shared_flags = None - testops["scan_map"].file = "blat" - testops["fake"].unicode_none = "foo" - - # Dump these to 2 disjoint configs - testops_fake = {"fake": testops["fake"]} - testops_notfake = dict(testops) - del testops_notfake["fake"] - - conf_fake = build_config([y for x, y in testops_fake.items()]) - conf_notfake = build_config([y for x, y in testops_notfake.items()]) - - conf_notfake_file = os.path.join(self.outdir, "multi_notfake.toml") - if self.toastcomm.world_rank == 0: - dump_toml(conf_notfake_file, conf_notfake) - if self.toastcomm.comm_world is not None: - self.toastcomm.comm_world.barrier() + conf_defaults_file = os.path.join(self.outdir, f"multi_defaults.{case}") + if self.toastcomm.world_rank == 0: + dump_config( + conf_defaults_file, + defaults, + format=case, + comm=self.toastcomm.comm_world, + ) + if self.toastcomm.comm_world is not None: + self.toastcomm.comm_world.barrier() - conf_fake_file = os.path.join(self.outdir, "multi_fake.toml") - if self.toastcomm.world_rank == 0: - dump_toml(conf_fake_file, conf_fake) - if self.toastcomm.comm_world is not None: - self.toastcomm.comm_world.barrier() + # Now change some values + testops["mem_count"].prefix = "newpref" + testops["mem_count"].enabled = False + testops["sim_noise"].serial = False + testops["sim_satellite"].hwp_rpm = 8.0 + testops["sim_satellite"].distribute_time = True + testops["sim_satellite"].shared_flags = None + testops["scan_map"].file = "blat" + testops["fake"].unicode_none = "foo" + + # Dump these to 2 disjoint configs + testops_fake = {"fake": testops["fake"]} + testops_notfake = dict(testops) + del testops_notfake["fake"] + + conf_fake = build_config([y for x, y in testops_fake.items()]) + conf_notfake = build_config([y for x, y in testops_notfake.items()]) + + conf_notfake_file = os.path.join(self.outdir, f"multi_notfake.{case}") + if self.toastcomm.world_rank == 0: + dump_config( + conf_notfake_file, + conf_notfake, + format=case, + comm=self.toastcomm.comm_world, + ) + if self.toastcomm.comm_world is not None: + self.toastcomm.comm_world.barrier() - # Load the configs in either order (since they are disjoint) - # and verify that the final result is the same - - iter = 1 - for conf_order in ( - [conf_notfake_file, conf_fake_file], - [conf_fake_file, conf_notfake_file], - ): - # Options for testing - arg_opts = [ - "--mem_count.prefix", - "altpref", - "--mem_count.enable", - "--sim_noise.serial", - "--sim_satellite.hwp_rpm", - "3.0", - "--sim_satellite.no_distribute_time", - "--pixels.resolution", - "(0.05 deg, 0.05 deg)", - "--scan_map.file", - "foobar", - "--fake.unicode_none", - "None", - "--config", - ] - arg_opts.extend(conf_order) - - parser = argparse.ArgumentParser(description="Test") - config, remaining, jobargs = parse_config( - parser, - operators=[y for x, y in testops.items()], - templates=list(), - prefix="", - opts=arg_opts, - ) - debug_file = os.path.join(self.outdir, f"debug_{iter}.toml") + conf_fake_file = os.path.join(self.outdir, f"multi_fake.{case}") if self.toastcomm.world_rank == 0: - dump_toml(debug_file, config) + dump_config( + conf_fake_file, + conf_fake, + format=case, + comm=self.toastcomm.comm_world, + ) if self.toastcomm.comm_world is not None: self.toastcomm.comm_world.barrier() - iter += 1 - - # Instantiate - run = create_from_config(config) - runops = run.operators - - # Check - self.assertTrue(runops.mem_count.prefix == "altpref") - self.assertTrue(runops.mem_count.enabled == True) - self.assertTrue(runops.sim_noise.serial == True) - self.assertTrue(runops.sim_satellite.distribute_time == False) - self.assertTrue(runops.sim_satellite.hwp_rpm == 3.0) - self.assertTrue(runops.sim_satellite.shared_flags is None) - self.assertTrue(runops.fake.unicode_none is None) - self.assertTrue(runops.scan_map.file == "foobar") + # Load the configs in either order (since they are disjoint) + # and verify that the final result is the same + + iter = 1 + for conf_order in ( + [conf_notfake_file, conf_fake_file], + [conf_fake_file, conf_notfake_file], + ): + # Options for testing + arg_opts = [ + "--mem_count.prefix", + "altpref", + "--mem_count.enable", + "--sim_noise.serial", + "--sim_satellite.hwp_rpm", + "3.0", + "--sim_satellite.no_distribute_time", + "--pixels.resolution", + "(Quantity('0.05 deg'), Quantity('0.05 deg'))", + "--scan_map.file", + "foobar", + "--fake.unicode_none", + "None", + "--config", + ] + arg_opts.extend(conf_order) + + parser = argparse.ArgumentParser(description="Test") + config, remaining, jobargs = parse_config( + parser, + operators=[y for x, y in testops.items()], + templates=list(), + prefix="", + opts=arg_opts, + ) + debug_file = os.path.join(self.outdir, f"debug_{iter}.{case}") + if self.toastcomm.world_rank == 0: + dump_config( + debug_file, config, format=case, comm=self.toastcomm.comm_world + ) + if self.toastcomm.comm_world is not None: + self.toastcomm.comm_world.barrier() + + iter += 1 + + # Instantiate + run = create_from_config(config) + runops = run.operators + + # Check + self.assertTrue(runops.mem_count.prefix == "altpref") + self.assertTrue(runops.mem_count.enabled == True) + self.assertTrue(runops.sim_noise.serial == True) + self.assertTrue(runops.sim_satellite.distribute_time == False) + self.assertTrue(runops.sim_satellite.hwp_rpm == 3.0) + self.assertTrue(runops.sim_satellite.shared_flags is None) + self.assertTrue(runops.fake.unicode_none is None) + self.assertTrue(runops.scan_map.file == "foobar") def test_roundtrip(self): testops = self.create_operators() diff --git a/src/toast/tests/ops_mapmaker.py b/src/toast/tests/ops_mapmaker.py index 9dbfb0290..718dc1836 100644 --- a/src/toast/tests/ops_mapmaker.py +++ b/src/toast/tests/ops_mapmaker.py @@ -16,8 +16,9 @@ from ..observation import default_values as defaults from ..pixels import PixelData, PixelDistribution from ..pixels_io_healpix import write_healpix_fits -from ..timing import GlobalTimers, gather_timers +from ..timing import GlobalTimers from ..timing import dump as dump_timers +from ..timing import gather_timers from ..vis import set_matplotlib_backend from ._helpers import close_data, create_fake_sky, create_outdir, create_satellite_data from .mpi import MPITestCase diff --git a/src/toast/trait_utils.py b/src/toast/trait_utils.py new file mode 100644 index 000000000..e48299d11 --- /dev/null +++ b/src/toast/trait_utils.py @@ -0,0 +1,308 @@ +# Copyright (c) 2023-2024 by the parties listed in the AUTHORS file. +# All rights reserved. Use of this source code is governed by +# a BSD-style license that can be found in the LICENSE file. + +import re + +import astropy.units as u +from astropy.units import Quantity, Unit + +from .utils import Logger + + +def fix_quotes(s, force=False): + clean = s.strip(" '\"") + if len(s) == 0 or force: + return f"'{clean}'" + else: + return clean + + +def string_to_scalar(val): + """Attempt to convert a string to supported scalar types. + + This handles the special case of Quantities and Units expressed as a string + with a space separating the value and unit. + + Args: + val (str): The input. + + Returns: + (scalar): The converted value. + + """ + if not isinstance(val, str): + # This is an already-instantiated object + return val + if val == "None": + return None + elif val == "True": + return True + elif val == "False": + return False + elif val == "": + return val + elif re.match(r"^Quantity.*", val) is not None: + return eval(val) + elif re.match(r"^Unit.*", val) is not None: + return eval(val) + else: + # See if we have a legacy Quantity or Unit string representation. + # Remove next few lines after sufficient deprecation period. + try: + qval = convert_legacy_quantity(val) + return qval + except ValueError: + # No. Try int next + try: + ival = int(val) + # Yes + return ival + except ValueError: + # No. Try float + try: + fval = float(val) + # Yes + return fval + except ValueError: + # String or some other object + return fix_quotes(val) + + +def scalar_to_string(val, force=False): + """Convert a scalar value into a string. + + This converts the value into a string representation which can be + reversed with the `eval()` function. + + Args: + val (object): A python scalar + + Returns: + (str): The string version. + + """ + if val is None: + return "None" + elif isinstance(val, u.UnitBase): + return f"Unit('{str(val)}')" + elif isinstance(val, u.Quantity): + return f"Quantity('{val.value:0.14e} {str(val.unit)}')" + elif isinstance(val, bool): + if val: + return "True" + else: + return "False" + elif isinstance(val, float): + return f"{val:0.14e}" + elif isinstance(val, int): + return f"{val}" + elif hasattr(val, "name") and hasattr(val, "enabled"): + # This trait value is a reference to another TraitConfig + return f"'@config:{val.get_config_path()}'" + else: + # Must be a string or other object + if isinstance(val, str): + val = fix_quotes(val, force=force) + return val + + +def string_to_trait(val): + """Attempt to convert a string to an arbitrary trait. + + Args: + val (str): The input. + + Returns: + (scalar): The converted value. + + """ + list_pat = re.compile(r"\[.*\]") + set_pat = re.compile(r"\{.*\}") + set_alt_pat = re.compile(r"set\(.*\)") + tuple_pat = re.compile(r"\(.*\)") + dict_pat = re.compile(r"\{.*:.*\}") + if not isinstance(val, str): + # This is an already-instantiated object + return val + bareval = fix_quotes(val) + if ( + list_pat.match(bareval) is not None + or set_pat.match(bareval) is not None + or set_alt_pat.match(bareval) is not None + or dict_pat.match(bareval) is not None + or tuple_pat.match(bareval) is not None + ): + # print(f"DBG calling eval on container {bareval}", flush=True) + # The string is a container. Just eval it. + container = eval(bareval) + # FIXME: Remove this call after sufficient deprecation period. + return parse_deprecated_quantities(container) + # It must be a scalar + # print(f"DBG calling string_to_scalar {bareval}", flush=True) + return string_to_scalar(bareval) + + +def trait_to_string(val): + """Convert a trait into a string. + + This creates a string which can be passed to `eval()` to re-create + the original container. + + Args: + val (object): A python scalar + + Returns: + (str): The string version. + + """ + + def _convert_elem(v, nest): + if isinstance(v, dict): + s = _convert_dict(v, nest + 1) + elif isinstance(v, set): + s = _convert_set(v, nest + 1) + elif isinstance(v, list): + s = _convert_list(v, nest + 1) + elif isinstance(v, tuple): + s = _convert_tuple(v, nest + 1) + else: + s = scalar_to_string(v, force=(nest > 0)) + return s + + def _convert_dict(t, nest): + out = "{" + for k, v in t.items(): + s = _convert_elem(v, nest + 1) + out += f"'{k}':{s}," + out += "}" + return out + + def _convert_set(t, nest): + if len(t) == 0: + return "set()" + out = "{" + for v in t: + s = _convert_elem(v, nest + 1) + out += f"{s}," + out += "}" + return out + + def _convert_list(t, nest): + out = "[" + for v in t: + s = _convert_elem(v, nest + 1) + out += f"{s}," + out += "]" + return out + + def _convert_tuple(t, nest): + out = "(" + for v in t: + s = _convert_elem(v, nest + 1) + out += f"{s}," + out += ")" + return out + + out = _convert_elem(val, 0) + # print( + # f"DBG converted {val} to str '{out}'", + # flush=True, + # ) + return out + + +def convert_legacy_quantity(qstring): + """Convert and return old-style quantity string.""" + log = Logger.get() + try: + parts = qstring.split() + vstr = parts.pop(0) + ustr = " ".join(parts) + if vstr == "unit": + # This string is a unit. See if there is anything + # following, and if not assume dimensionless_unscaled. + if ustr == "" or ustr == "None": + out = u.dimensionless_unscaled + else: + out = u.Unit(ustr) + elif ustr == "": + raise ValueError("Empty unit string") + else: + # See if we have a quantity + value = float(vstr) + unit = u.Unit(ustr) + out = u.Quantity(value, unit=unit) + # We have one of these, raise warning + if isinstance(out, u.Unit): + msg = f"Legacy Unit string '{qstring}' is deprecated. " + msg += f"Use 'Unit(\"{ustr}\")' instead." + else: + msg = f"Legacy Quantity string '{qstring}' is deprecated. " + msg += f"Use 'Quantity(\"{qstring}\")' instead." + log.warning(msg) + return out + except (IndexError, ValueError, TypeError): + # Nope, not a legacy quantity string + raise ValueError("Not a legacy quantity string") + + +def parse_deprecated_quantities(container): + """Attempt to parse container values with deprecated Quantity strings. + + Old config files stored Quantities as a string with just the value and + units (rather than a string which can be eval'd directly into the object). + This function attempts to handle that case and also print a warning. + + Args: + container (object): One of the supported containers + + Returns: + (object): The input container with quantity strings instantiated. + + """ + + def _parse_obj(c): + if isinstance(c, list): + return _parse_list(c) + elif isinstance(c, tuple): + return _parse_tuple(c) + elif isinstance(c, set): + return _parse_set(c) + elif isinstance(c, dict): + return _parse_dict(c) + else: + if isinstance(c, str): + try: + out = convert_legacy_quantity(c) + return out + except ValueError: + return c + else: + return c + + def _parse_list(c): + out = list() + for it in c: + out.append(_parse_obj(it)) + return out + + def _parse_tuple(c): + out = list() + for it in c: + out.append(_parse_obj(it)) + return tuple(out) + + def _parse_set(c): + out = set() + for it in c: + out.add(_parse_obj(it)) + return out + + def _parse_dict(c): + out = dict() + for k, v in c.items(): + out[k] = _parse_obj(v) + return out + + return _parse_obj(container) diff --git a/src/toast/traits.py b/src/toast/traits.py index 76ab65fc9..35bdc45b0 100644 --- a/src/toast/traits.py +++ b/src/toast/traits.py @@ -5,6 +5,7 @@ import collections import copy import re +import types from collections import OrderedDict import traitlets @@ -29,120 +30,15 @@ ) from .accelerator import ImplementationType, use_accel_jax, use_accel_omp +from .trait_utils import fix_quotes, string_to_trait, trait_to_string from .utils import Logger, import_from_name, object_fullname - -def trait_string_to_scalar(val): - """Attempt to convert a string to other basic python types. - - Trait containers support arbitrary objects, and there are situations where - we just have a string value and need to determine the actual python type. - This arises when parsing a config dictionary or when converting the config - elements of a container. - - Args: - val (str): The input. - - Returns: - (scalar): The converted value. - - """ - if not isinstance(val, str): - # This is an already-instantiated object - return val - if val == "None": - return None - elif val == "True": - return True - elif val == "False": - return False - elif val == "": - return val - else: - # See if we have a Quantity or Unit string representation - try: - parts = val.split() - vstr = parts.pop(0) - ustr = " ".join(parts) - if vstr == "unit": - # This string is a unit. See if there is anything - # following, and if not assume dimensionless_unscaled. - if ustr == "": - return u.dimensionless_unscaled - else: - return u.Unit(ustr) - elif ustr == "": - raise ValueError("Empty unit string") - else: - # See if we have a quantity - value = float(vstr) - unit = u.Unit(ustr) - return u.Quantity(value, unit=unit) - except (IndexError, ValueError): - # No. Try int next - try: - ival = int(val) - # Yes - return ival - except ValueError: - # No. Try float - try: - fval = float(val) - # Yes - return fval - except ValueError: - # String or some other object - return val - - -def trait_scalar_to_string(val): - """Convert a scalar value into a string. - - This is needed to stringify both scalar traits and also container - elements. - - Args: - val (object): A python scalar - - Returns: - (str): The string version. - - """ - if val is None: - return "None" - elif isinstance(val, u.UnitBase): - return f"unit {str(val)}" - elif isinstance(val, u.Quantity): - return f"{val.value:0.14e} {str(val.unit)}" - elif isinstance(val, bool): - if val: - return "True" - else: - return "False" - elif isinstance(val, float): - return f"{val:0.14e}" - elif isinstance(val, int): - return f"{val}" - if isinstance(val, TraitConfig): - # This trait value is a reference to another TraitConfig - return "@config:{}".format(val.get_config_path()) - else: - # Must be a string - return val - - # Add mixin methods to built-in Trait types # Scalar base types -Bool.py_type = lambda self: bool -UseEnum.py_type = lambda self: self.enum_class -Float.py_type = lambda self: float -Int.py_type = lambda self: int -Unicode.py_type = lambda self: str - -def _create_scalar_trait_get_conf(conf_type): +def _create_scalar_trait_get_conf(conf_type, py_type): # Create a class method that gets a config entry for a scalar # trait with no units. def _get_conf(self, obj=None): @@ -152,135 +48,62 @@ def _get_conf(self, obj=None): v = self.default_value else: v = self.get(obj) - cf["value"] = trait_scalar_to_string(v) - cf["unit"] = "None" + if v is None or v == traitlets.Undefined or v == traitlets.Sentinel: + vpy = None + elif py_type is None: + vpy = v + else: + vpy = py_type(v) + cf["value"] = trait_to_string(vpy) cf["help"] = str(self.help) return cf return _get_conf -Bool.get_conf = _create_scalar_trait_get_conf("bool") -UseEnum.get_conf = _create_scalar_trait_get_conf("enum") -Float.get_conf = _create_scalar_trait_get_conf("float") -Int.get_conf = _create_scalar_trait_get_conf("int") -Unicode.get_conf = _create_scalar_trait_get_conf("str") - -# Container types. These need specialized get_conf() methods. - -List.py_type = lambda self: list +Bool.get_conf = _create_scalar_trait_get_conf("bool", bool) +UseEnum.get_conf = _create_scalar_trait_get_conf("enum", None) +Float.get_conf = _create_scalar_trait_get_conf("float", float) +Int.get_conf = _create_scalar_trait_get_conf("int", int) +Unicode.get_conf = _create_scalar_trait_get_conf("str", str) +# Container types. -def list_get_conf(self, obj=None): - cf = dict() - cf["type"] = "list" - if obj is None: - val = self.default_value - else: - v = self.get(obj) - if v is None: - msg = ( - f"The toast config system does not support None values for " - f"List traits. " - f"Failed to parse '{self.name}' : '{self.help}'" - ) - raise ValueError(msg) - # val = "None" - else: - val = list() - for item in v: - val.append(trait_scalar_to_string(item)) - cf["value"] = val - cf["unit"] = "None" - cf["help"] = str(self.help) - return cf - -List.get_conf = list_get_conf - -Set.py_type = lambda self: set - - -def set_get_conf(self, obj=None): - cf = dict() - cf["type"] = "set" - if obj is None: - val = self.default_value - else: - v = self.get(obj) - if v is None: - raise ValueError( - "The toast config system does not support None values for Set traits." - ) - # val = "None" +def _create_container_trait_get_conf(conf_type, py_type): + # Create a class method that gets a config entry for a container + # trait and checks for None. + def _get_conf(self, obj=None): + cf = dict() + cf["type"] = conf_type + if obj is None: + v = self.default_value else: - val = set() - for item in v: - val.add(trait_scalar_to_string(item)) - cf["value"] = val - cf["unit"] = "None" - cf["help"] = str(self.help) - return cf - - -Set.get_conf = set_get_conf - -Dict.py_type = lambda self: dict - - -def dict_get_conf(self, obj=None): - cf = dict() - cf["type"] = "dict" - if obj is None: - val = self.default_value - else: - v = self.get(obj) - if v is None: - raise ValueError( - "The toast config system does not support None values for Dict traits." - ) - # val = "None" + v = self.get(obj) + if v is None: + msg = "The toast config system does not support None values for " + msg += f"{conf_type} traits. Failed to parse '{self.name}' :" + msg += f" '{self.help}'" + raise ValueError(msg) + if v == traitlets.Undefined or v == traitlets.Sentinel: + vpy = py_type() else: - val = dict() - for k, v in v.items(): - val[k] = trait_scalar_to_string(v) - cf["value"] = val - cf["unit"] = "None" - cf["help"] = str(self.help) - return cf - - -Dict.get_conf = dict_get_conf - -Tuple.py_type = lambda self: tuple + vpy = py_type(v) + # print(f"DBG {conf_type} get_conf({v} -> {vpy})") + cf["value"] = trait_to_string(vpy) + cf["help"] = str(self.help) + return cf + return _get_conf -def tuple_get_conf(self, obj=None): - cf = dict() - cf["type"] = "tuple" - if obj is None: - val = self.default_value - else: - v = self.get(obj) - if v is None: - raise ValueError( - "The toast config system does not support None values for Tuple traits." - ) - # val = "None" - else: - val = list() - for item in v: - val.append(trait_scalar_to_string(item)) - val = tuple(val) - cf["value"] = val - cf["unit"] = "None" - cf["help"] = str(self.help) - return cf +List.get_conf = _create_container_trait_get_conf("list", list) +Set.get_conf = _create_container_trait_get_conf("set", set) +Dict.get_conf = _create_container_trait_get_conf("dict", dict) +Tuple.get_conf = _create_container_trait_get_conf("tuple", tuple) -Tuple.get_conf = tuple_get_conf -Instance.py_type = lambda self: self.klass +# Special case for Instance and Callable traits def instance_get_conf(self, obj=None): @@ -293,20 +116,17 @@ def instance_get_conf(self, obj=None): if v is None: val = "None" elif isinstance(v, TraitConfig): - val = trait_scalar_to_string(v) + val = trait_to_string(v) else: # There is nothing we can do with this val = "None" cf["value"] = val - cf["unit"] = "None" cf["help"] = str(self.help) return cf Instance.get_conf = instance_get_conf -Callable.py_type = lambda self: collections.abc.Callable - def callable_get_conf(self, obj=None): cf = dict() @@ -319,16 +139,17 @@ def callable_get_conf(self, obj=None): val = "None" else: # There is no way of serializing a generic callable - # into a string. Just set it to None. + # into a string. Just set it to None for now. val = "None" cf["value"] = val - cf["unit"] = "None" cf["help"] = str(self.help) return cf Callable.get_conf = callable_get_conf +# Unit / Quantity traits + class Unit(TraitType): """A trait representing an astropy Unit.""" @@ -339,30 +160,14 @@ class Unit(TraitType): def __init__(self, default_value=Undefined, **kwargs): super().__init__(default_value=default_value, **kwargs) - def py_type(self): - return u.Unit - - def get_conf(self, obj=None): - cf = dict() - cf["type"] = "Unit" - if obj is None: - val = self.default_value - else: - val = self.get(obj) - cf["value"] = "unit" - if val is None: - cf["unit"] = "None" - else: - cf["unit"] = str(val) - cf["help"] = str(self.help) - return cf - def validate(self, obj, value): if value is None: if self.allow_none: return None else: - raise TraitError("Attempt to set trait to None, while allow_none=False") + raise TraitError( + f"Attempt to set trait {self.name} to None, while allow_none=False" + ) try: # Can we construct a unit from this? return u.Unit(value) @@ -376,6 +181,9 @@ def from_string(self, s): return u.Unit(s) +Unit.get_conf = _create_scalar_trait_get_conf("Unit", u.Unit) + + class Quantity(Float): """A Quantity trait with units.""" @@ -385,23 +193,14 @@ class Quantity(Float): def __init__(self, default_value=Undefined, **kwargs): super().__init__(default_value=default_value, **kwargs) - def get_conf(self, obj=None): - cf = dict() - cf["type"] = "Quantity" - if obj is None: - v = self.default_value - else: - v = self.get(obj) - if v is None: - cf["value"] = "None" - cf["unit"] = "None" - else: - cf["value"] = trait_scalar_to_string(v.value) - cf["unit"] = str(v.unit) - cf["help"] = str(self.help) - return cf - def validate(self, obj, value): + if value is None: + if self.allow_none: + return None + else: + raise TraitError( + f"Attempt to set trait {self.name} to None, while allow_none=False" + ) if not isinstance(value, u.Quantity): # We can't read minds- force the user to specify the units msg = "Value '{}' does not have units".format(value) @@ -416,6 +215,9 @@ def from_string(self, s): return u.Quantity(s) +Quantity.get_conf = _create_scalar_trait_get_conf("Quantity", u.Quantity) + + def trait_docs(cls): """Decorator which adds trait properties to signature and docstring for a class. @@ -430,10 +232,7 @@ def trait_docs(cls): # doc += "Attributes:\n" for trait_name, trait in cls.class_traits().items(): cf = trait.get_conf() - if cf["type"] == "Unit": - default = cf["unit"] - else: - default = cf["value"] + default = cf["value"] doc += "\t{} ({}): {} (default = {})\n".format( trait_name, cf["type"], cf["help"], default ) @@ -451,8 +250,10 @@ class TraitConfig(HasTraits): * Traitlet info and help string added to the docstring (cls.__doc__) for the class constructor. - * Dump / Load of a named INSTANCE (not just a class) to a configuration file. - This differs from the traitlets.Configuration package. + * Dump / Load of a named INSTANCE (not just a class) to a configuration + dictionary in memory. This configuration dictionary serves as an + intermediate representation which can then be translated into several + configuration file formats. * Creation and parsing of commandline options to set the traits on a named instance of the class. @@ -538,9 +339,6 @@ def select_kernels(self, use_accel=None): def __eq__(self, other): if len(self.traits()) != len(other.traits()): - # print( - # f"DBG self has {len(self.traits())} traits, other has {len(other.traits())}" - # ) return False # Now we know that both objects have the same number of traits- compare the # types and values. @@ -550,17 +348,14 @@ def __eq__(self, other): tset = {x: x for x in trait.get(self)} oset = {x: x for x in trother.get(other)} if tset != oset: - # print(f"DBG {tset} != {oset}") return False elif isinstance(trait, Dict): tdict = dict(trait.get(self)) odict = dict(trother.get(other)) if tdict != odict: - # print(f"DBG {tdict} != {odict}") return False else: if trait.get(self) != trother.get(other): - # print(f"DBG trait {trait.get(self)} != {trother.get(other)}") return False return True @@ -625,12 +420,8 @@ def get_class_config(cls, section=None, input=None): cf = trait.get_conf() parent[name][trait_name] = OrderedDict() parent[name][trait_name]["value"] = cf["value"] - parent[name][trait_name]["unit"] = cf["unit"] parent[name][trait_name]["type"] = cf["type"] parent[name][trait_name]["help"] = cf["help"] - # print( - # f"{name} class conf {trait_name}: {cf}" - # ) return input def get_config(self, section=None, input=None): @@ -663,10 +454,8 @@ def get_config(self, section=None, input=None): cf = trait.get_conf(obj=self) parent[name][trait_name] = OrderedDict() parent[name][trait_name]["value"] = cf["value"] - parent[name][trait_name]["unit"] = cf["unit"] parent[name][trait_name]["type"] = cf["type"] parent[name][trait_name]["help"] = cf["help"] - # print(f"{name} instance conf {trait_name}: {cf}") return input @classmethod @@ -705,6 +494,7 @@ def from_config(name, props): (TraitConfig): The instantiated derived class. """ + log = Logger.get() if "class" not in props: msg = "Property dictionary does not contain 'class' key" raise RuntimeError(msg) @@ -716,79 +506,337 @@ def from_config(name, props): # Parse all the parameter type information and create values we will pass to # the constructor. + parsable = set( + [ + "Unit", + "Quantity", + "set", + "dict", + "tuple", + "list", + "float", + "int", + "str", + "bool", + "enum", + ] + ) kw = dict() kw["name"] = name + avail_traits = set(cls.class_trait_names()) for k, v in props.items(): - if v["type"] == "Unit": - if v["value"] != "unit": - raise RuntimeError( - f"Unit trait does not have 'unit' as the conf value" - ) - if v["unit"] == "None": - kw[k] = None - else: - kw[k] = u.Unit(v["unit"]) - # print(f"from_config {name}: {k} = {kw[k]}") - elif v["type"] == "Quantity": - # print(f"from_config {name}: {v}") - if v["value"] == "None": - kw[k] = None - else: - kw[k] = u.Quantity(float(v["value"]), u.Unit(v["unit"])) - # print(f"from_config {name}: {k} = {kw[k]}") - elif v["type"] == "set": - if v["value"] == "None": - kw[k] = None - elif v["value"] == "{}": - kw[k] = set() - else: - kw[k] = set([trait_string_to_scalar(x) for x in v["value"]]) - # print(f"from_config {name}: {k} = {kw[k]}") - elif v["type"] == "list": - if v["value"] == "None": - kw[k] = None - elif v["value"] == "[]": - kw[k] = list() - else: - kw[k] = list([trait_string_to_scalar(x) for x in v["value"]]) - # print(f"from_config {name}: {k} = {kw[k]}") + if k == "class": continue - elif v["type"] == "tuple": - if v["value"] == "None": - kw[k] = None - elif v["value"] == "()": - kw[k] = tuple() - else: - kw[k] = tuple([trait_string_to_scalar(x) for x in v["value"]]) - # print(f"from_config {name}: {k} = {kw[k]}") - elif v["type"] == "dict": - if v["value"] == "None": - kw[k] = None - elif v["value"] == "{}": - kw[k] = dict() - else: - # print(f"from_config input dict = {v['value']}") - kw[k] = { - x: trait_string_to_scalar(y) for x, y in v["value"].items() - } - # print(f"from_config {name}: {k} = {kw[k]}") - elif v["value"] == "None": + if k not in avail_traits: + msg = f"Class {cls_path} currently has no configuration" + msg += f" trait '{k}'. This will be ignored, and your config " + msg += f"file is likely out of date." + log.warning(msg) + continue + # print(f"from_config: parsing {v}", flush=True) + if v["value"] == "None": # Regardless of type, we set this to None kw[k] = None - elif ( - v["type"] == "float" - or v["type"] == "int" - or v["type"] == "str" - or v["type"] == "bool" - ): - kw[k] = trait_string_to_scalar(v["value"]) - # print(f"from_config {name}: {k} = {kw[k]}") elif v["type"] == "unknown": # This was a None value in the TOML or similar unknown object pass + elif v["type"] in parsable: + kw[k] = string_to_trait(v["value"]) else: # This is either a class instance of some arbitrary type, # or a callable. pass # Instantiate class and return + # print(f"Instantiate class with {kw}", flush=True) return cls(**kw) + + +def create_from_config(conf): + """Instantiate classes in a configuration. + + This iteratively instantiates classes defined in the configuration, replacing + object names with references to those objects. References to other objects in the + config are specified with the string '@config:' followed by a UNIX-style "path" + where each element of the path is a dictionary key in the config. For example: + + @config:/operators/pointing + + Would reference an object at conf["operators"]["pointing"]. Object references like + this only work if the target of the reference is a built-in type (str, float, int, + etc) or a class derived from TraitConfig. + + Args: + conf (dict): the configuration + + Returns: + (SimpleNamespace): A namespace containing the sections and instantiated + objects specified in the original config structure. + + """ + log = Logger.get() + ref_prefix = "@config:" + ref_pat = re.compile("^{}/(.*)".format(ref_prefix)) + + def _get_node(path, tree): + """Given the path as a list of keys, descend tree and return the node.""" + node = tree + for elem in path: + if isinstance(node, list): + if not isinstance(elem, int): + msg = f"Node path {path}, element {elem} is not an integer." + msg += f" Cannot index list node {node}." + raise RuntimeError(msg) + if elem >= len(node): + # We hit the end of our path, node does not yet exist. + return None + node = node[elem] + elif isinstance(node, dict): + if elem not in node: + # We hit the end of our path, node does not yet exist. + return None + node = node[elem] + else: + # We have hit a leaf node without getting to the end of our + # path. This means the node does not exist yet. + return None + return node + + def _dereference(value, tree): + """If the object is a string with a path reference, return the object at + that location in the tree. Otherwise return the object. + """ + if not isinstance(value, str): + return value + ref_mat = ref_pat.match(value) + if ref_mat is not None: + # This string contains a reference + ref_path = ref_mat.group(1).split("/") + return _get_node(ref_path, tree) + else: + # No reference, it is just a string + return value + + def _insert_element(obj, parent_path, key, tree): + """Insert object as a child of the parent path.""" + parent = _get_node(parent_path, tree) + # print(f"{parent_path}: insert at {parent}: {obj}", flush=True) + if parent is None: + msg = f"Node {parent} at {parent_path} does not exist, cannot" + msg += f" insert {obj}" + raise RuntimeError(msg) + elif isinstance(parent, list): + parent.append(obj) + elif isinstance(parent, dict): + if key is None: + msg = f"Node {parent} at {parent_path} cannot add {obj} without key" + raise RuntimeError(msg) + parent[key] = obj + else: + msg = f"Node {parent} at {parent_path} is not a list or dict" + raise RuntimeError(msg) + + def _parse_string(value, parent_path, key, out): + """Add a string to the output tree if it resolves.""" + obj = _dereference(value, out) + # print(f"parse_string DEREF str {value} -> {obj}", flush=True) + if obj is None: + # Does not yet exist + return 1 + # Add to output + _insert_element(obj, parent_path, key, out) + return 0 + + def _parse_trait_value(obj, tree): + """Recursively check trait value for references. + Note that the input has already been tested for None values, + so returning None from this function indicates that the + trait value contains undefined references. + """ + if isinstance(obj, str): + temp = _dereference(obj, tree) + # print(f"parse_trait DEREF str {obj} -> {temp}") + return temp + if isinstance(obj, list): + ret = list() + for it in obj: + if it is None: + ret.append(None) + else: + check = _parse_trait_value(it, tree) + if check is None: + return None + else: + ret.append(check) + return ret + if isinstance(obj, tuple): + ret = list() + for it in obj: + if it is None: + ret.append(None) + else: + check = _parse_trait_value(it, tree) + if check is None: + return None + else: + ret.append(check) + return tuple(ret) + if isinstance(obj, set): + ret = set() + for it in obj: + if it is None: + ret.add(None) + else: + check = _parse_trait_value(it, tree) + if check is None: + return None + else: + ret.add(check) + return ret + if isinstance(obj, dict): + ret = dict() + for k, v in obj.items(): + if v is None: + ret[k] = None + else: + check = _parse_trait_value(v, tree) + if check is None: + return None + else: + ret[k] = check + return ret + # This must be some other scalar trait with no references + return obj + + def _parse_traitconfig(value, parent_path, key, out): + instance_name = None + ctor = dict() + for tname, tprops in value.items(): + if tname == "class": + ctor["class"] = tprops + continue + ctor[tname] = dict() + ctor[tname]["type"] = tprops["type"] + tstring = tprops["value"] + trait = string_to_trait(tstring) + # print(f"{key} trait {tname} = {trait}", flush=True) + if trait is None: + ctor[tname]["value"] = None + # print(f"{key} trait {tname} value = None", flush=True) + else: + check = _parse_trait_value(trait, out) + # print(f"{key} trait {tname} value check = {check}", flush=True) + if check is None: + # This trait contained unresolved references + # print(f"{key} trait {tname} value unresolved", flush=True) + return 1 + ctor[tname]["value"] = check + # If we got this far, it means that we parsed all traits and can + # instantiate the class. + # print(f"{parent_path}|{key}: parse_tc ctor = {ctor}", flush=True) + obj = TraitConfig.from_config(instance_name, ctor) + # print(f"{parent_path}|{key}: parse_tc {obj}", flush=True) + _insert_element(obj, parent_path, key, out) + return 0 + + def _parse_list(value, parent_path, key, out): + parent = _get_node(parent_path, out) + # print(f"{parent_path}: parse_list parent = {parent}", flush=True) + _insert_element(list(), parent_path, key, out) + child_path = list(parent_path) + child_path.append(key) + # print(f"{parent_path}: parse_list child = {child_path}", flush=True) + unresolved = 0 + for val in value: + if isinstance(val, list): + unresolved += _parse_list(val, child_path, None, out) + # print(f"parse_list: after {val} unresolved = {unresolved}", flush=True) + elif isinstance(val, dict): + if "class" in val: + # This is a TraitConfig instance + unresolved += _parse_traitconfig(val, child_path, None, out) + # print( + # f"parse_list: after {val} unresolved = {unresolved}", flush=True + # ) + else: + # Just a normal dictionary + unresolved += _parse_dict(val, child_path, None, out) + # print( + # f"parse_list: after {val} unresolved = {unresolved}", flush=True + # ) + else: + unresolved += _parse_string(val, child_path, None, out) + # print(f"parse_list: after {val} unresolved = {unresolved}", flush=True) + return unresolved + + def _parse_dict(value, parent_path, key, out): + parent = _get_node(parent_path, out) + # print(f"{parent_path}: parse_dict parent = {parent}", flush=True) + _insert_element(OrderedDict(), parent_path, key, out) + child_path = list(parent_path) + child_path.append(key) + # print(f"{parent_path}: parse_dict child = {child_path}", flush=True) + unresolved = 0 + for k, val in value.items(): + if isinstance(val, list): + unresolved += _parse_list(val, child_path, k, out) + # print(f"parse_dict: after {k} unresolved = {unresolved}", flush=True) + elif isinstance(val, dict): + if "class" in val: + # This is a TraitConfig instance + unresolved += _parse_traitconfig(val, child_path, k, out) + # print( + # f"parse_dict: after {k} unresolved = {unresolved}", flush=True + # ) + else: + # Just a normal dictionary + unresolved += _parse_dict(val, child_path, k, out) + # print( + # f"parse_dict: after {k} unresolved = {unresolved}", flush=True + # ) + else: + unresolved += _parse_string(val, child_path, k, out) + # print(f"parse_dict: after {k} unresolved = {unresolved}", flush=True) + return unresolved + + # Iteratively instantiate objects + + out = OrderedDict() + done = False + last_unresolved = None + + it = 0 + while not done: + # print("PARSE iter ", it) + done = True + unresolved = 0 + + # Go through the top-level dictionary + for topkey in list(conf.keys()): + if isinstance(conf[topkey], str): + out[topkey] = conf[topkey] + continue + if len(conf[topkey]) > 0: + unresolved += _parse_dict(conf[topkey], list(), topkey, out) + # print(f"PARSE {it}: {topkey} unresolved now {unresolved}", flush=True) + + if last_unresolved is not None: + if unresolved == last_unresolved: + msg = f"Cannot resolve all references ({unresolved} remaining)" + msg += f" in the configuration" + log.error(msg) + raise RuntimeError(msg) + last_unresolved = unresolved + if unresolved > 0: + done = False + it += 1 + + # Convert this recursively into a namespace for easy use + root_temp = dict() + for sect in list(out.keys()): + if isinstance(out[sect], str): + root_temp[sect] = out[sect] + else: + sect_ns = types.SimpleNamespace(**out[sect]) + root_temp[sect] = sect_ns + out_ns = types.SimpleNamespace(**root_temp) + return out_ns