diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 93fe076..354e757 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,7 +3,7 @@ on: release: types: [published] jobs: - publish-una: + publish-all: environment: release runs-on: ubuntu-latest permissions: @@ -14,20 +14,5 @@ jobs: id: setup-rye with: version: '0.38.0' - - run: rye build - - uses: pypa/gh-action-pypi-publish@release/v1 - - publish-hatch-una: - environment: release - runs-on: ubuntu-latest - permissions: - id-token: write - steps: - - uses: actions/checkout@v4 - - uses: eifinger/setup-rye@v3 - id: setup-rye - with: - version: '0.38.0' - - run: rye build - working-directory: plugins/hatch + - run: rye build --all - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/README.md b/README.md index 92d40d8..a975bcf 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# una +# Una
@@ -20,25 +20,34 @@
-una is a tool to make Python monorepos with Rye easier. It is a CLI tool and a Hatch plugin that does the following things: +Una is a tool to make Python monorepos with Rye easier. It is a CLI tool and a build plugin that does the following things: + 1. Enable builds of individual apps or projects within a monorepo. 2. Ensure that internal and external dependencies are correctly specified. -una is inspired by [python-polylith](https://github.com/DavidVujic/python-polylith) and is based on that codebase. -But I find the [Polylith](https://polylith.gitbook.io/polylith) architecture to be quite intimidating for many, so wanted to create a lighter touch alternative that doesn't require too much re-thinking. This project has very limited ambitions and doesn't try to do everything a proper build system such as [Bazel](https://bazel.build/) or [Pants](https://www.pantsbuild.org/) does. +Una is inspired by [python-polylith](https://github.com/DavidVujic/python-polylith) and borrows extensively from that codebase. +But I find the [Polylith](https://polylith.gitbook.io/polylith) architecture to be quite intimidating for many, so wanted to create a lighter touch alternative that doesn't require too much re-thinking. +This project has very limited ambitions and doesn't try to do everything a proper build system such as [Bazel](https://bazel.build/) or [Pants](https://www.pantsbuild.org/) does. It just tries to make a simple monorepo build just about possible. -una allows two directory structures or styles: +Una allows two directory structures or styles: - `packages`: this is the default style, that is just some extra build help on top of a Rye workspace. - `modules`: a more novel approach with just a single pyproject.toml, arguably better DevX and doesn't require a Rye workspace. Within this context, we use the following words frequently: + - `lib`: a module or package that will be imported but not run. - `app`: a module or package that will be run but never imported. - `project`: a package with no code but only dependencies (only used in the `modules` style) +Currently it works with the following build backends, but more will follow: + +- [Hatch](https://hatch.pypa.io) (used by default and and in all documentation) +- [PDM](https://pdm-project.org/) + ## Examples You can see examples for each of the two styles here: + - [una-example-packages](https://github.com/carderne/una-example-packages) - [una-example-modules](https://github.com/carderne/una-example-modules) @@ -52,7 +61,7 @@ cd unarepo rye add --dev una ``` -Then setup the una workspace. This will generate a structure and an example lib and app. +Then setup the Una workspace. This will generate a structure and an example lib and app. ``` rye run una create workspace rye sync @@ -66,7 +75,7 @@ tree Have a look at the generated `__init__.py` files in the `apps/printer` and `libs/greeter` packages. An external dependency ([cowsay-python](https://pypi.org/project/cowsay-python/)) has also been added to the latter's `pyproject.toml`. -The magic of `una` then comes in to resolve the graph of direct and transitive dependencies, which looks like this: +The magic of Una then comes in to resolve the graph of direct and transitive dependencies, which looks like this: ```elm printer --> greeter --> cowsay-python ``` @@ -155,4 +164,4 @@ It covers additional things like: - and more! ## License -una is distributed under the terms of the MIT license. +Una is distributed under the terms of the MIT license. diff --git a/docs/build.md b/docs/build.md index 548970c..7df9a85 100644 --- a/docs/build.md +++ b/docs/build.md @@ -1,6 +1,6 @@ # Build -At build-time, `una` itself does nothing. +At build-time, Una itself does nothing. This is when `hatch-una`, the plugin for [Hatch](https://hatch.pypa.io/) steps in and resolves the graph of dependencies. Assuming your `pyproject.toml` is correctly configured, and your `[tool.una.libs]` section includes all necessary dependencies (_and_ transitive dependencies!), @@ -17,4 +17,5 @@ rye build --wheel rye run build ``` -You'll get some `*.whl` files, which you can then deploy with Docker or whatever you prefer. They are fully self-contained, so you don't need Rye or `una` or anything else wherever you want to install them. +You'll get some `*.whl` files, which you can then deploy with Docker or whatever you prefer. +They are fully self-contained, so you don't need Rye or Una or anything else wherever you want to install them. diff --git a/docs/index.md b/docs/index.md index 74d7d61..ae708f6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,4 +1,4 @@ -# una +# Una
@@ -20,18 +20,18 @@
-una is a tool to make Python monorepos with Rye easier. It is a CLI tool and a Hatch plugin that does the following things: +Una is a tool to make Python monorepos with Rye easier. It is a CLI tool and a build plugin that does the following things: 1. Enable builds of individual apps or projects within a monorepo. 2. Ensure that internal and external dependencies are correctly specified. -una is inspired by [python-polylith](https://github.com/DavidVujic/python-polylith) and is based on that codebase. -But I find the [Polylith](https://polylith.gitbook.io/polylith) architecture to be quite intimidating for many, so wanted to create a lighter touch alternative that doesn't require too much re-thinking. This project has very limited ambitions and doesn't try to do everything a proper build system such as [Bazel](https://bazel.build/) or [Pants](https://www.pantsbuild.org/) does. +Una is inspired by [python-polylith](https://github.com/DavidVujic/python-polylith) and borrows extensively from that codebase. +But I find the [Polylith](https://polylith.gitbook.io/polylith) architecture to be quite intimidating for many, so wanted to create a lighter touch alternative that doesn't require too much re-thinking. +This project has very limited ambitions and doesn't try to do everything a proper build system such as [Bazel](https://bazel.build/) or [Pants](https://www.pantsbuild.org/) does. It just tries to make a simple monorepo build just about possible. -una allows two directory structures or styles: - -- `packages`: this is the lightest-touch approach, that is just some extra build help on top of a Rye workspace. +Una allows two directory structures or styles: +- `packages`: this is the default style, that is just some extra build help on top of a Rye workspace. - `modules`: a more novel approach with just a single pyproject.toml, arguably better DevX and doesn't require a Rye workspace. Within this context, we use the following words frequently: @@ -40,6 +40,11 @@ Within this context, we use the following words frequently: - `app`: a module or package that will be run but never imported. - `project`: a package with no code but only dependencies (only used in the `modules` style) +Currently it works with the following build backends, but more will follow: + +- [Hatch](https://hatch.pypa.io) (used by default and and in all documentation) +- [PDM](https://pdm-project.org/) + ## Examples You can see examples for each of the two styles here: diff --git a/docs/quickstart.md b/docs/quickstart.md index ae0c6db..6d00495 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -8,7 +8,7 @@ cd unarepo rye add --dev una ``` -Then setup the una workspace. This will generate a structure and an example lib and app. +Then setup the Una workspace. This will generate a structure and an example lib and app. ``` rye run una create workspace rye sync @@ -22,7 +22,7 @@ tree Have a look at the generated `__init__.py` files in the `apps/printer` and `libs/greeter` packages. An external dependency ([cowsay-python](https://pypi.org/project/cowsay-python/)) has also been added to the latter's `pyproject.toml`. -The magic of `una` then comes in to resolve the graph of direct and transitive dependencies, which looks like this: +The magic of Una then comes in to resolve the graph of direct and transitive dependencies, which looks like this: ```elm printer --> greeter --> cowsay-python ``` diff --git a/docs/style-modules.md b/docs/style-modules.md index 4e51fb7..6318539 100644 --- a/docs/style-modules.md +++ b/docs/style-modules.md @@ -34,9 +34,9 @@ The key differences are as follows: So we add `projects/` directory. This contains no code, just a pyproject.toml and whatever else is needed to deploy the built project. The pyproject will specify which internal dependencies are used in the project: exactly one app, and zero or more libs. 4. It must also specify all external dependencies that are used, including the transitive dependencies of internal libs that it uses. -But the una CLI will help with this! +But the Una CLI will help with this! And there's one more benefit: 5. You can run pyright and pytest from the root directory! This gives you a true monorepo benefit of having a single static analysis of the entire codebase. -But don't worry, una will help you to only test the bits that are needed. +But don't worry, Una will help you to only test the bits that are needed. diff --git a/docs/style-packages.md b/docs/style-packages.md index 7a55094..e107e9a 100644 --- a/docs/style-packages.md +++ b/docs/style-packages.md @@ -33,11 +33,11 @@ This means: 3. Type-checking and testing should be done on a per-package level. That is, you should run `pyright` and `pytest` from `apps/server` or `libs/mylib`, _not_ from the root. -In the example above, the only build artifact will be for `apps/server`. At build-time, una will do the following: +In the example above, the only build artifact will be for `apps/server`. At build-time, Una will do the following: 1. Read the list of internal dependencies (more on this shortly) and inject them into the build. 2. Read all externel requirements of those dependencies, and add them to the dependency table. -You can then use the `una` CLI tool to ensure that all internal dependencies are kept in sync. What are the key steps? +You can then use the Una CLI tool to ensure that all internal dependencies are kept in sync. What are the key steps? 1. Use a Rye workspace: ```toml # /pyproject.toml @@ -64,7 +64,7 @@ build-backend = "hatchling.build" [tool.una.libs] "../../libs/mylib/example/mylib" = "example/mylib" ``` -4. Then you can run `rye build --wheel` from that package directory and una will inject everything that is needed. +4. Then you can run `rye build --wheel` from that package directory and Una will inject everything that is needed. Once you have your built `.whl` file, all you need in your Dockerfile is: ```Dockerfile FROM python diff --git a/plugins/hatch/hatch_una/hatch_build.py b/plugins/hatch/hatch_una/hatch_build.py index b751f02..b6c0447 100644 --- a/plugins/hatch/hatch_una/hatch_build.py +++ b/plugins/hatch/hatch_una/hatch_build.py @@ -54,7 +54,9 @@ def initialize(self, version: str, build_data: dict[str, Any]) -> None: # need to add the root workspace pyproject.toml so that in src -> sdist -> wheel builds, # we can still determine the style (for packages style) add_root_pyproj = { - str(root_path / util.PYPROJ): str(util.EXTRA_PYPROJ / "root" / util.PYPROJ) + str(root_path / util.PYPROJ): str( + util.EXTRA_PYPROJ / util.ROOT_PYPROJ_SUBDIR / util.PYPROJ + ) } if style == "packages": add_packages_pyproj = { diff --git a/plugins/hatch/hatch_una/py.typed b/plugins/hatch/hatch_una/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/plugins/hatch/hatch_una/util.py b/plugins/hatch/hatch_una/util.py index d100ee3..69aabd5 100644 --- a/plugins/hatch/hatch_una/util.py +++ b/plugins/hatch/hatch_una/util.py @@ -3,7 +3,8 @@ from typing import Any, Literal, TypeAlias PYPROJ = "pyproject.toml" -EXTRA_PYPROJ = Path("extra_pyproj") +EXTRA_PYPROJ = Path("_extra_pyproj") +ROOT_PYPROJ_SUBDIR = Path("_root") def load_conf(path: Path) -> dict[str, Any]: @@ -24,7 +25,7 @@ def get_workspace_style(root_path: Path) -> Style: # In builds that do src -> sdist -> wheel, the root pyproject.toml file will # have been copied into the sdist so available for the wheel build. # Here we check for both in order. - extra_root_path = EXTRA_PYPROJ / "root" + extra_root_path = EXTRA_PYPROJ / ROOT_PYPROJ_SUBDIR if (root_path / PYPROJ).exists(): use_root_path = root_path elif (extra_root_path / PYPROJ).exists(): diff --git a/plugins/hatch/pyproject.toml b/plugins/hatch/pyproject.toml index 285259f..8d03104 100644 --- a/plugins/hatch/pyproject.toml +++ b/plugins/hatch/pyproject.toml @@ -11,7 +11,7 @@ authors = [ readme = "README.md" license = {text = "MIT License"} requires-python = ">= 3.11" -keywords = ["rye", "monorepo", "build", "python"] +keywords = ["rye", "hatch", "monorepo", "build", "python"] classifiers = [ "Environment :: Console", diff --git a/plugins/pdm/README.md b/plugins/pdm/README.md new file mode 100644 index 0000000..dce2894 --- /dev/null +++ b/plugins/pdm/README.md @@ -0,0 +1,5 @@ +# pdm-una + +This is the PDM plugin for [una](https://github.com/carderne/una). + +Read the full README there. diff --git a/plugins/pdm/pdm_una/__init__.py b/plugins/pdm/pdm_una/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/pdm/pdm_una/hook.py b/plugins/pdm/pdm_una/hook.py new file mode 100644 index 0000000..1201dfd --- /dev/null +++ b/plugins/pdm/pdm_una/hook.py @@ -0,0 +1,9 @@ +from pdm.backend.hooks import Context + +from pdm_una.include import force_include +from pdm_una.meta import add_dependencies + + +def pdm_build_initialize(context: Context): + add_dependencies(context) + force_include(context) diff --git a/plugins/pdm/pdm_una/include.py b/plugins/pdm/pdm_una/include.py new file mode 100644 index 0000000..f197420 --- /dev/null +++ b/plugins/pdm/pdm_una/include.py @@ -0,0 +1,70 @@ +from pathlib import Path + +from pdm.backend.hooks import Context + +from pdm_una import util + + +def force_include(context: Context) -> None: + """ + Force-include all needed internal monorepo dependencies. + """ + print("una: Injecting internal dependencies") + context.ensure_build_dir() + + build_dir = context.build_dir + + # load the config for this app/project + path = Path(context.root) + conf = util.load_conf(path) + name: str = conf["project"]["name"] + + try: + int_deps: dict[str, str] = conf["tool"]["una"]["libs"] + except KeyError as e: + raise KeyError( + f"App/project '{name}' is missing '[tool.una.libs]' in pyproject.toml" + ) from e + + # need to determine workspace style (packages or modules) + # as packages style needs dependencies' pyproject.tomls to be included + # so that they're available in src -> sdist -> wheel builds + via_sdist = (util.EXTRA_PYPROJ / util.ROOT_PYPROJ_SUBDIR / util.PYPROJ).exists() + if via_sdist: + # nothing to do as everything should already be included in sdist... + return + + root_path = path.parents[1] + style = util.get_workspace_style(root_path) + + if not int_deps: + if style == "packages": + # this is fine, the app doesn't import anything internally + return + else: + # this is an empty project, useless and accidental + raise ValueError(f"Project '{name}' has no dependencies") + + # make sure all int_deps exist + found = [Path(k) for k in int_deps if (path / k).exists()] + missing = set(int_deps) - set(str(p) for p in found) + if len(missing) > 0: + missing_str = ", ".join(missing) + raise ValueError(f"Could not find these paths: {missing_str}") + + # need to add the root workspace pyproject.toml so that in src -> sdist -> wheel builds, + # we can still determine the style (for packages style) + util.copy_file( + root_path / util.PYPROJ, + build_dir / util.EXTRA_PYPROJ / util.ROOT_PYPROJ_SUBDIR / util.PYPROJ, + ) + for src_str, dst_str in int_deps.items(): + src = Path(src_str) + destination = build_dir / dst_str + util.copy_tree(src, destination) + if style == "packages": + # need these so src->sdist->wheel builds can access them for external deps + util.copy_file( + src.parents[1] / util.PYPROJ, + build_dir / util.EXTRA_PYPROJ / src.name / util.PYPROJ, + ) diff --git a/plugins/pdm/pdm_una/meta.py b/plugins/pdm/pdm_una/meta.py new file mode 100644 index 0000000..fb68d99 --- /dev/null +++ b/plugins/pdm/pdm_una/meta.py @@ -0,0 +1,64 @@ +from pathlib import Path + +from pdm.backend.hooks import Context + +from pdm_una import util + + +def add_dependencies(context: Context): + """ + Inject needed third-party dependencies into project.dependencies. + + This hook is only needed for Packages style una workspaces. + Modules style should have all dependencies specified in pyproject.toml. + """ + print("una: Injecting transitive external dependencies") + metadata = context.config.metadata + path = context.root + + conf = util.load_conf(path) + name: str = conf["project"]["name"] + + root_path = path.parents[1] + style = util.get_workspace_style(root_path) + if style == "modules": + # in Hatch, we raise here (should we?) + # but in PDM the hooks are not configured separately + # so just return + return + + try: + int_deps: dict[str, str] = conf["tool"]["una"]["libs"] + except KeyError as e: + raise KeyError( + f"App/project '{name}' is missing '[tool.una.libs]' in pyproject.toml" + ) from e + + project_deps: list[str] = metadata.get("dependencies", []) + project_deps = [d.strip().replace(" ", "") for d in project_deps] + + add_deps: list[str] = [] + for dep_path in int_deps: + # In builds that do src -> sdist -> wheel, the needed pyproject.toml files + # will have been copied into the sdist so they're available for the wheel build. + # Here we check for both in order. + dep_project_path = Path(dep_path).parents[1] + extra_path = util.EXTRA_PYPROJ / Path(dep_path).name + if dep_project_path.exists(): + use_path = dep_project_path + elif extra_path.exists(): + use_path = extra_path + else: + raise ValueError(f"Could not find internal dependency at '{dep_path}'") + + # load all third-party dependencies from this internal dependency into the + # project.dependencies table + dep_conf = util.load_conf(use_path) + try: + dep_deps: list[str] = dep_conf["project"]["dependencies"] + except KeyError as e: + raise KeyError(f"No project.dependcies table for '{use_path}'") + dep_deps = [d.strip().replace(" ", "") for d in dep_deps] + add_deps.extend(dep_deps) + + metadata["dependencies"] = list(set(project_deps + add_deps)) diff --git a/plugins/pdm/pdm_una/py.typed b/plugins/pdm/pdm_una/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/plugins/pdm/pdm_una/util.py b/plugins/pdm/pdm_una/util.py new file mode 100644 index 0000000..2f9e7d6 --- /dev/null +++ b/plugins/pdm/pdm_una/util.py @@ -0,0 +1,69 @@ +import shutil +import tomllib +from pathlib import Path +from typing import Any, Literal, TypeAlias + +PYPROJ = "pyproject.toml" +EXTRA_PYPROJ = Path("_extra_pyproj") +ROOT_PYPROJ_SUBDIR = Path("_root") + + +def load_conf(path: Path) -> dict[str, Any]: + with (path / PYPROJ).open("rb") as fp: + return tomllib.load(fp) + + +Style: TypeAlias = Literal["packages", "modules"] + + +def get_workspace_style(root_path: Path) -> Style: + """ + Get the root workspace style + + Param `path` should be the path to the ap/project, + NOT to the root workspace. + """ + # In builds that do src -> sdist -> wheel, the root pyproject.toml file will + # have been copied into the sdist so available for the wheel build. + # Here we check for both in order. + extra_root_path = EXTRA_PYPROJ / ROOT_PYPROJ_SUBDIR + if (root_path / PYPROJ).exists(): + use_root_path = root_path + elif (extra_root_path / PYPROJ).exists(): + use_root_path = extra_root_path + else: + raise ValueError("No root pyproject to determine workspace style") + root_conf = load_conf(use_root_path) + try: + style: Style = root_conf["tool"]["una"]["style"] + return style + except KeyError as e: + raise KeyError( + "Root workspace pyproject.toml needs '[tool.una]' with style specified" + ) from e + + +def copy_file(src: Path, dst: Path) -> Path: + dst.parents[0].mkdir(parents=True, exist_ok=True) + return shutil.copyfile(src, dst) + + +def copy_tree(src: Path, dst: Path) -> Path: + ignore = shutil.ignore_patterns( + "*.pyc", + "__pycache__", + ".venv", + "__pypackages__", + ".mypy_cache", + ".pytest_cache", + "node_modules", + ".git", + ) + + res: Path = shutil.copytree( # return might not actually be a Path + src, + dst, + ignore=ignore, + dirs_exist_ok=True, + ) + return Path(res) diff --git a/plugins/pdm/pyproject.toml b/plugins/pdm/pyproject.toml new file mode 100644 index 0000000..9473e7b --- /dev/null +++ b/plugins/pdm/pyproject.toml @@ -0,0 +1,56 @@ +[tool.rye.scripts] +check = "basedpyright" + +[project] +name = "pdm-una" +dynamic = ["version"] +description = "Python monorepo tooling" +authors = [ + { name = "Chris Arderne", email = "chris@rdrn.me" } +] +readme = "README.md" +license = {text = "MIT License"} +requires-python = ">= 3.11" +keywords = ["rye", "pdm", "monorepo", "build", "python"] + +classifiers = [ + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: Unix", + "Programming Language :: Python", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] + +dependencies = [] + +[project.urls] +homepage = "https://github.com/carderne/una" +repository = "https://github.com/carderne/una" + +[project.entry-points."pdm.build.hook"] +una = "pdm_una.hook" + +[tool.rye] +managed = true +dev-dependencies = ["pdm-backend"] + +[build-system] +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" + +[tool.hatch.version] +source = "vcs" +raw-options = { root = "../.." } + +[tool.basedpyright] +venvPath = "../.." +venv = ".venv" +pythonVersion = "3.11" +strict = ["**/*.py"] +reportUnnecessaryTypeIgnoreComment = true +reportImplicitOverride = false +reportUnusedCallResult = false +enableTypeIgnoreComments = true diff --git a/pyproject.toml b/pyproject.toml index 934cf61..064e9f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ dev-dependencies = [ ] [tool.rye.workspace] -members = [".", "plugins/hatch"] +members = [".", "plugins/*"] [build-system] requires = ["hatchling", "hatch-vcs"] diff --git a/requirements-dev.lock b/requirements-dev.lock index 958a1af..9f69242 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -11,6 +11,7 @@ -e file:. -e file:plugins/hatch +-e file:plugins/pdm babel==2.16.0 # via mkdocs-material basedpyright==1.15.2 @@ -75,6 +76,7 @@ paginate==0.5.6 pathspec==0.12.1 # via hatchling # via mkdocs +pdm-backend==2.3.3 platformdirs==4.2.2 # via mkdocs-get-deps pluggy==1.5.0 diff --git a/requirements.lock b/requirements.lock index c6054a3..1c83999 100644 --- a/requirements.lock +++ b/requirements.lock @@ -11,6 +11,7 @@ -e file:. -e file:plugins/hatch +-e file:plugins/pdm click==8.1.7 # via typer dataclasses-json==0.6.7