Including

.github/workflows/tests.yml

@@ -22,6 +22,7 @@ jobs:
           - "3.9"
           - "3.10"
           - "3.11"
+          - "3.12"
         os:
           - ubuntu-latest
           - windows-latest

CHANGES.md

@@ -1,8 +1,9 @@
 ## Changes
 
-### 3.0.1 (unreleased)
+### 3.1.0 (unreleased)
 
 - Provide `directory` default setting [rnix]
+- Feature: Include other INI config files [jensens]
 
 ### 3.0.0 (2023-05-08)
 

MANIFEST.in

@@ -7,5 +7,6 @@ recursive-include example *.ini
 recursive-include example *.txt
 recursive-include src *.md
 recursive-exclude example *-outfile.txt
+recursive-include src *.ini
 recursive-exclude .vscode *
 graft mxdev

README.md

@@ -7,7 +7,7 @@ It builds on top of the idea to have stable version constraints and then develop
 As part of the above use-case sometimes versions of the stable constraints need an override with a different (i.e. newer) version.
 Other software follow the same idea are [mr.developer](https://pypi.org/project/mr.developer)  for Python's *zc.buildout* or [mrs-developer](https://www.npmjs.com/package/mrs-developer) for NPM packages.
 
-**mxdev 2.0 needs pip version 22 at minimum to work properly**
+**mxdev >=2.0 needs pip version 22 at minimum to work properly**
 
 
 ### Overview
@@ -53,6 +53,22 @@ Output of the combined constraints.
 
 Default: `constraints-mxdev.txt`
 
+#### `include`
+
+Include one or more other INI files.
+
+The included file is read before the main file, so the main file overrides included settings.
+Included files may include other files.
+Innermost inclusions are read first.
+
+If an included file is an HTTP-URL, it is loaded from there.
+
+If the included file is a relative path, it is loaded relative to the parents directory or URL.
+
+The feature utilizes the [ConfigParser feature to read multiple files at once](https://docs.python.org/3/library/configparser.html#configparser.ConfigParser.read).
+
+Default: empty
+
 #### `default-target`
 
 Target directory for sources from VCS. Default: `./sources`

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "mxdev"
 description = "Enable to work with Python projects containing lots of packages, of which you only want to develop some."
-version = "3.0.1.dev0"
+version = "3.1.0.dev0"
 keywords = ["pip", "vcs", "git", "development"]
 authors = [
   {name = "MX Stack Developers", email = "[email protected]" }
@@ -20,6 +20,7 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
 ]
 dependencies = []
 dynamic = ["readme"]
@@ -33,6 +34,7 @@ test = [
     "pytest",
     "pytest-cov",
     "pytest-mock",
+    "httpretty",
     "types-setuptools",
 ]
 

src/mxdev/config.py

@@ -1,6 +1,6 @@
+from .including import read_with_included
 from .logging import logger
 
-import configparser
 import os
 import pkg_resources
 import typing
@@ -13,9 +13,7 @@
 def to_bool(value):
     if not isinstance(value, str):
         return bool(value)
-    if value.lower() in ("true", "on", "yes", "1"):
-        return True
-    return False
+    return value.lower() in ("true", "on", "yes", "1")
 
 
 class Configuration:
@@ -27,22 +25,13 @@ class Configuration:
 
     def __init__(
         self,
-        tio: typing.TextIO,
+        mxini: str,
         override_args: typing.Dict = {},
         hooks: typing.List["Hook"] = [],
     ) -> None:
         logger.debug("Read configuration")
-        data = configparser.ConfigParser(
-            default_section="settings",
-            interpolation=configparser.ExtendedInterpolation(),
-        )
-        data.optionxform = str  # type: ignore
+        data = read_with_included(mxini)
 
-        # default settings to be used in mx.ini config file
-        data["settings"]["directory"] = os.getcwd()
-        # end default settings
-
-        data.read_file(tio)
         settings = self.settings = dict(data["settings"].items())
 
         logger.debug(f"infile={self.infile}")

src/mxdev/including.py

@@ -0,0 +1,80 @@
+from configparser import ConfigParser
+from configparser import ExtendedInterpolation
+from pathlib import Path
+from urllib import parse
+from urllib import request
+
+import os
+import tempfile
+import typing
+
+
+def resolve_dependencies(
+    file_or_url: typing.Union[str, Path],
+    tmpdir: str,
+    http_parent=None,
+) -> typing.List[Path]:
+    """Resolve dependencies of a file or url
+
+    The result is a list of Path objects, starting with the
+    given file_or_url and followed by all file_or_urls referenced from it.
+
+    The file_or_url is assumed to be a ini file or url to such, with an option key "include"
+    under the "[settings]" section.
+    """
+    if isinstance(file_or_url, str):
+        if http_parent:
+            file_or_url = parse.urljoin(http_parent, file_or_url)
+        parsed = parse.urlparse(str(file_or_url))
+        if parsed.scheme:
+            with request.urlopen(str(file_or_url)) as fio:
+                tf = tempfile.NamedTemporaryFile(
+                    suffix=".ini",
+                    dir=str(tmpdir),
+                    delete=False,
+                )
+                tf.write(fio.read())
+                tf.flush()
+                file = Path(tf.name)
+            parts = list(parsed)
+            parts[2] = str(Path(parts[2]).parent)
+            http_parent = parse.urlunparse(parts)
+        else:
+            file = Path(file_or_url)
+    else:
+        file = file_or_url
+    if not file.exists():
+        raise FileNotFoundError(file)
+    cfg = ConfigParser()
+    cfg.read(file)
+    if not ("settings" in cfg and "include" in cfg["settings"]):
+        return [file]
+    file_list = []
+    for include in cfg["settings"]["include"].split("\n"):
+        include = include.strip()
+        if not include:
+            continue
+        if http_parent or parse.urlparse(include).scheme:
+            file_list += resolve_dependencies(include, tmpdir, http_parent)
+        else:
+            file_list += resolve_dependencies(file.parent / include, tmpdir)
+
+    file_list.append(file)
+    return file_list
+
+
+def read_with_included(file_or_url: typing.Union[str, Path]) -> ConfigParser:
+    """Read a file or url and include all referenced files,
+
+    Parse the result as a ConfigParser and return it.
+    """
+    cfg = ConfigParser(
+        default_section="settings",
+        interpolation=ExtendedInterpolation(),
+    )
+    cfg.optionxform = str  # type: ignore
+    cfg["settings"]["directory"] = os.getcwd()
+    with tempfile.TemporaryDirectory() as tmpdir:
+        resolved = resolve_dependencies(file_or_url, tmpdir)
+        cfg.read(resolved)
+    return cfg

src/mxdev/main.py

@@ -22,8 +22,7 @@
     "-c",
     "--configuration",
     help="configuration file in INI format",
-    nargs="?",
-    type=argparse.FileType("r"),
+    type=str,
     default="mx.ini",
 )
 parser.add_argument(
@@ -62,7 +61,9 @@ def main() -> None:
     if args.threads:
         override_args["threads"] = args.threads
     configuration = Configuration(
-        tio=args.configuration, override_args=override_args, hooks=hooks
+        mxini=args.configuration,
+        override_args=override_args,
+        hooks=hooks,
     )
     state = State(configuration=configuration)
     logger.info("#" * 79)

src/mxdev/tests/fixtures.py → src/mxdev/tests/conftest.py

@@ -40,3 +40,13 @@ def develop(src):
     develop = MockDevelop()
     develop.sources_dir = src
     return develop
+
+
+@pytest.fixture
+def httpretty():
+    import httpretty
+
+    httpretty.enable()
+    yield httpretty
+    httpretty.disable()
+    httpretty.reset()

src/mxdev/tests/data/file01.ini

@@ -0,0 +1,6 @@
+[settings]
+include =
+    file02.ini
+    file04.ini
+test = 1
+unique_1 = 1

src/mxdev/tests/data/file02.ini

@@ -0,0 +1,6 @@
+[settings]
+include =
+    file03.ini
+
+test = 2
+unique_2 = 2

src/mxdev/tests/data/file03.ini

@@ -0,0 +1,3 @@
+[settings]
+test = 3
+unique_3 = 3

src/mxdev/tests/data/file04.ini

@@ -0,0 +1,3 @@
+[settings]
+test = 4
+unique_4 = 4

src/mxdev/tests/data/file_with_http_include01.ini

@@ -0,0 +1,6 @@
+[settings]
+include =
+    http://www.example.com/file_with_http_include02.ini
+    file_with_http_include04.ini
+test = 1
+unique_1 = 2

src/mxdev/tests/data/file_with_http_include02.ini

@@ -0,0 +1,6 @@
+[settings]
+include =
+    file_with_http_include03.ini
+
+test = 2
+unique_2 = true

src/mxdev/tests/data/file_with_http_include03.ini

@@ -0,0 +1,3 @@
+[settings]
+test = 3
+unique_3 = true

src/mxdev/tests/data/file_with_http_include04.ini

@@ -0,0 +1,3 @@
+[settings]
+test = 4
+unique_4 = true

src/mxdev/tests/test_common.py

@@ -1,6 +1,5 @@
 from .. import vcs
 from ..vcs import common
-from unittest import mock
 
 import logging
 import os
@@ -28,16 +27,16 @@ def test_BaseWorkingCopy():
         common.BaseWorkingCopy(source={})
 
     class TestWorkingCopy(common.BaseWorkingCopy):
-        def checkout(self, **kwargs) -> typing.Union[str, None]:
+        def checkout(self, **kwargs) -> typing.Union[str, None]:  # type: ignore
             ...
 
-        def status(self, **kwargs) -> typing.Union[typing.Tuple[str, str], str]:
+        def status(self, **kwargs) -> typing.Union[typing.Tuple[str, str], str]:  # type: ignore
             ...
 
-        def matches(self) -> bool:
+        def matches(self) -> bool:  # type: ignore
             ...
 
-        def update(self, **kwargs) -> typing.Union[str, None]:
+        def update(self, **kwargs) -> typing.Union[str, None]:  # type: ignore
             ...
 
     bwc = TestWorkingCopy(source=dict(url="https://tld.com/repo.git"))
@@ -163,10 +162,10 @@ def checkout(self, **kwargs) -> typing.Union[str, None]:
         def status(self, **kwargs) -> typing.Union[typing.Tuple[str, str], str]:
             return self.package_status
 
-        def matches(self) -> bool:
+        def matches(self) -> bool:  # type: ignore
             ...
 
-        def update(self, **kwargs) -> typing.Union[str, None]:
+        def update(self, **kwargs) -> typing.Union[str, None]:  # type: ignore
             ...
 
     class WCT(dict):

src/mxdev/tests/test_git.py

@@ -1,7 +1,4 @@
 # pylint: disable=redefined-outer-name
-from .fixtures import mkgitrepo
-from .fixtures import src
-from .fixtures import tempdir
 from logging import getLogger
 from logging import Logger
 from mxdev.tests.utils import vcs_checkout

src/mxdev/tests/test_git_submodules.py

@@ -1,7 +1,3 @@
-# pylint: disable=redefined-outer-name
-from .fixtures import mkgitrepo
-from .fixtures import src
-from .fixtures import tempdir
 from mxdev.tests.utils import GitRepo
 from mxdev.tests.utils import vcs_checkout
 from mxdev.tests.utils import vcs_update