diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 0000000..77ef355 --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,51 @@ +name: pr + +on: + pull_request: + types: [opened, reopened, synchronize] + +concurrency: + group: pr-${{ github.event.number }} + cancel-in-progress: true + +jobs: + changes: + runs-on: ubuntu-latest + permissions: + pull-requests: read + outputs: + python: ${{ steps.filter.outputs.python }} + steps: + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + python: + - '**/*.py' + - 'pyproject.toml' + + test-python: + runs-on: ubuntu-latest + needs: changes + if: needs.changes.outputs.python == 'true' + steps: + - uses: actions/checkout@v4 + - uses: eifinger/setup-rye@v3 + id: setup-rye + with: + version: '0.38.0' + - run: rye pin 3.12.3 + - name: Sync + run: | + rye sync + if [[ $(git diff --stat requirements.lock) != '' ]]; then + echo 'Rye lockfile not up-to-date' + git diff requirements.lock + exit 1 + fi + - run: rye fmt --check + - run: rye lint + - run: rye run check + - run: rye run test + - run: rye run check + working-directory: plugins/hatch diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..93fe076 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,33 @@ +name: release +on: + release: + types: [published] +jobs: + publish-una: + environment: release + runs-on: ubuntu-latest + permissions: + id-token: write + steps: + - uses: actions/checkout@v4 + - uses: eifinger/setup-rye@v3 + id: setup-rye + with: + version: '0.38.0' + - run: rye build + - uses: pypa/gh-action-pypi-publish@release/v1 + + publish-hatch-una: + environment: release + runs-on: ubuntu-latest + permissions: + id-token: write + steps: + - uses: actions/checkout@v4 + - uses: eifinger/setup-rye@v3 + id: setup-rye + with: + version: '0.38.0' + - run: rye build + working-directory: plugins/hatch + - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.gitignore b/.gitignore index 6c7c332..8f460c4 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ __pycache__ .mypy_cache dist .coverage +.hypothesis +.obsidian diff --git a/LICENSE b/LICENSE index 543cbf8..6ff48a8 100644 --- a/LICENSE +++ b/LICENSE @@ -2,6 +2,9 @@ MIT License Copyright (c) 2024 Chris Arderne +Significant parts of this codebase come from python-polylith +Copyright (c) 2022 David Vujic + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights diff --git a/README.md b/README.md index 730dfb0..694890a 100644 --- a/README.md +++ b/README.md @@ -1 +1,120 @@ -# mono +# una +**Warning: this is pre-alpha and probably doesn't work at all. You'll probably just get frustrated if you even try to use it.** + +una is a tool to make Python monorepos with Hatch and/or Rye easier. It is a CLI tool and a Hatch plugin that does the following things: +1. Enable builds of individual apps or projects within a monorepo. +2. Ensure that internal and external dependencies are correctly specified. + +una is inspired by [python-polylith](https://github.com/DavidVujic/python-polylith) and is based on that codebase. But I find the [Polylith](https://polylith.gitbook.io/polylith) architecture to be quite intimidating for many, so wanted to create a lighter touch alternative that doesn't require too much re-thinking. This project has very limited ambitions and doesn't try to do everything a proper build system such as [Bazel](https://bazel.build/) or [Pants](https://www.pantsbuild.org/) does. It just tries to make a simple monorepo build just about possible. + +una allows two directory structures or styles: +- `packages`: this is the lightest-touch approach, that is just some extra build help on top of a Rye workspace. +- modules: a more novel approach with just a single pyproject.toml, arguably better DevX and compatible with Rye or Hatch alone. + +Within this context, we use the following words frequently: +- `lib`: a module or package that will be imported but not run. +- `app`: a module or package that will be run but never imported. +- `project`: a package with no code but only dependencies (only used in the `modules` style) + +## Style: Packages +In this setup, we use Rye's built-in workspace support. The structure will look something like this: +```bash +. +├── pyproject.toml +├── requirements.lock +├── apps +│   └── server +│   ├── pyproject.toml +│   ├── your_ns +│   │   └── server +│   │   ├── __init__.py +│   └── tests +│   └── test_server.py +└── libs +    └── mylib +       ├── pyproject.toml +    ├── your_ns +    │   └── mylib +    │   ├── __init__.py +    │   └── py.typed +    └── tests +    └── test_mylib.py +``` + +This means: +1. Each `app` or `lib` (collectively, internal dependencies) is it's own Python package with a `pyproject.toml`. +2. You must specify the workspace members in `tool.rye.workspace.members`. +3. Type-checking and testing should be done on a per-package level. That is, you should run `pyright` and `pytest` from `apps/server` or `libs/mylib`, _not_ from the root. + +In the example above, the only build artifact will be for `apps/server`. At build-time, una will do the following: +1. Read the list of internal dependencies (more on this shortly) and inject them into the build. +2. Read all externel requirements of those dependencies, and add them to the dependency table. + +You can then use the `una` CLI tool to ensure that all internal dependencies are kept in sync. What are the key steps? +1. Use a Rye workspace: +```toml +# /pyproject.toml +[tool.rye] +managed = true +virtual = true + +[tool.rye.workspace] +members = ["apps/*", "libs/*"] +``` + +2. Create your apps and your libs as you would, ensuring that app code is never imported. Ensure that you choose a good namespace and always use it in your package structures (check `your_ns` in the example structure above.) +3. Add external dependencies to your libs and apps as normal. Then, to add an internal dependency to an app, we do the following in its pyproject.toml: +```toml +# /apps/server/pyproject.toml +[build-system] +requires = ["hatchling", "hatch-una"] +build-backend = "hatchling.build" + +[tool.hatch.build.hooks.una-build] +[tool.hatch.build.hooks.una-meta] +[tool.una.libs] +"../../libs/mylib/example/mylib" = "example/mylib" +``` +4. Then you can run `rye build --wheel` from that package directory and una will inject everything that is needed. Once you have your built `.whl` file, all you need in your Dockerfile is: +```Dockerfile +FROM python +COPY dist . +RUN pip install dist/*.whl +``` +### Historical note +What is needed to get root-level pyright and pytest to work? +1. Delete apps/lib pyproject.toml as they cause pyright/basedpyright to get lost. +2. Add pythonpath to pytest settings. + +## Style: Modules +This approach is inspired by [Polylith](https://davidvujic.github.io/python-polylith-docs/). You don't use a Rye workspace (and indeed this approach will work with just Hatch), and there's only a single `pyproject.toml`. + +The structure looks like this: +```bash +. +├── pyproject.toml +├── requirements.lock +├── apps +│   └── your_ns +│   └── server +│   ├── __init__.py +│   └── test_server.py +├── libs +│   └── your_ns +│   └── mylib +│   ├── __init__.py +│   ├── core.py +│   └── test_core.py +└── projects +    └── server +    └── pyproject.toml +``` + +The key differences are as follows: +1. `apps/` and `libs/` contain only pure Python code, structured into modules under a common namespace. +2. Tests are colocated with Python code (this will be familiar to those coming from Go or Rust). +3. Because `apps/` is just pure Python code, we need somewhere else to convert this into deployable artifacts (Docker images and the like). So we add `projects/` directory. This contains no code, just a pyproject.toml and whatever else is needed to deploy the built project. The pyproject will specify which internal dependencies are used in the project: exactly one app, and zero or more libs. +4. It must also specify all external dependencies that are used, including the transitive dependencies of internal libs that it uses. But the una CLI will help with this! + +And there's one more benefit: +5. You can run pyright and pytest from the root directory! This gives you a true monorepo benefit of having a single static analysis of the entire codebase. But don't worry, una will help you to only test the bits that are needed. diff --git a/plugins/hatch/README.md b/plugins/hatch/README.md new file mode 100644 index 0000000..5b9b589 --- /dev/null +++ b/plugins/hatch/README.md @@ -0,0 +1,5 @@ +# hatch-una + +This is the Hatch plugin for [una](https://github.com/carderne/una). + +Read the full README there. diff --git a/plugins/hatch/hatch_una/__init__.py b/plugins/hatch/hatch_una/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/hatch/hatch_una/hatch_build.py b/plugins/hatch/hatch_una/hatch_build.py new file mode 100644 index 0000000..e927f70 --- /dev/null +++ b/plugins/hatch/hatch_una/hatch_build.py @@ -0,0 +1,32 @@ +import tomllib +from pathlib import Path +from typing import Any + +from hatchling.builders.config import BuilderConfig +from hatchling.builders.hooks.plugin.interface import BuildHookInterface +from hatchling.plugin import hookimpl + + +class BuildConfig(BuilderConfig): + pass + + +class UnaHook(BuildHookInterface[BuildConfig]): + PLUGIN_NAME = "una-build" + + def initialize(self, version: str, build_data: dict[str, Any]) -> None: + print("una: Injecting internal dependencies") + root = Path(self.root) + with (root / "pyproject.toml").open("rb") as fp: + conf = tomllib.load(fp) + + int_deps: dict[str, str] = conf["tool"]["una"]["libs"] + found = {k: v for k, v in int_deps.items() if (root / k).exists()} + if not int_deps or not found: + return + build_data["force_include"] = {**build_data["force_include"], **int_deps} + + +@hookimpl +def hatch_register_build_hook(): + return UnaHook diff --git a/plugins/hatch/hatch_una/hatch_meta.py b/plugins/hatch/hatch_una/hatch_meta.py new file mode 100644 index 0000000..4763503 --- /dev/null +++ b/plugins/hatch/hatch_una/hatch_meta.py @@ -0,0 +1,43 @@ +import tomllib +from pathlib import Path +from typing import Any + +from hatchling.metadata.plugin.interface import MetadataHookInterface +from hatchling.plugin import hookimpl + + +def load_conf(path: Path) -> dict[str, Any]: + with (path / "pyproject.toml").open("rb") as fp: + return tomllib.load(fp) + + +class UnaMetaHook(MetadataHookInterface): + PLUGIN_NAME = "una-meta" + + def update(self, metadata: dict[str, Any]) -> None: + root = Path(self.root) + conf = load_conf(root) + int_deps: dict[str, str] = conf["tool"]["una"]["libs"] + + project_deps: list[str] = metadata.get("dependencies", {}) + project_deps = [d.strip().replace(" ", "") for d in project_deps] + + add_deps: list[str] = [] + for dep_path in int_deps: + dep_project_path = Path(dep_path).parents[1] + try: + dep_conf = load_conf(dep_project_path) + except FileNotFoundError: + continue + dep_deps: list[str] = dep_conf["project"]["dependencies"] + dep_deps = [d.strip().replace(" ", "") for d in dep_deps] + add_deps.extend(dep_deps) + + all_deps = list(set(project_deps + add_deps)) + print(all_deps) + metadata["dependencies"] = all_deps + + +@hookimpl +def hatch_register_metadata_hook(): + return UnaMetaHook diff --git a/plugins/hatch/pyproject.toml b/plugins/hatch/pyproject.toml new file mode 100644 index 0000000..285259f --- /dev/null +++ b/plugins/hatch/pyproject.toml @@ -0,0 +1,57 @@ +[tool.rye.scripts] +check = "basedpyright" + +[project] +name = "hatch-una" +dynamic = ["version"] +description = "Python monorepo tooling" +authors = [ + { name = "Chris Arderne", email = "chris@rdrn.me" } +] +readme = "README.md" +license = {text = "MIT License"} +requires-python = ">= 3.11" +keywords = ["rye", "monorepo", "build", "python"] + +classifiers = [ + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: Unix", + "Programming Language :: Python", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] + +dependencies = ["hatchling"] + +[project.urls] +homepage = "https://github.com/carderne/una" +repository = "https://github.com/carderne/una" + +[project.entry-points.hatch] +una-build = "hatch_una.hatch_build" +una-meta = "hatch_una.hatch_meta" + +[tool.rye] +managed = true +dev-dependencies = [] + +[build-system] +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" + +[tool.hatch.version] +source = "vcs" +raw-options = { root = "../.." } + +[tool.basedpyright] +venvPath = "../.." +venv = ".venv" +pythonVersion = "3.11" +strict = ["**/*.py"] +reportUnnecessaryTypeIgnoreComment = true +reportImplicitOverride = false +reportUnusedCallResult = false +enableTypeIgnoreComments = true diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9e84a15 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,88 @@ +[tool.rye.scripts] +fmt = "rye fmt" +lint = "rye lint --fix" +check = "basedpyright" +test = "rye test" +all = { chain = ["fmt", "lint", "check", "test"] } + +[project] +name = "una" +dynamic = ["version"] +description = "Python monorepo tooling" +authors = [ + { name = "Chris Arderne", email = "chris@rdrn.me" } +] +readme = "README.md" +license = {text = "MIT License"} +requires-python = ">= 3.11" +keywords = ["rye", "monorepo", "build", "python"] + +classifiers = [ + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: Unix", + "Programming Language :: Python", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] + +dependencies = [ + "dataclasses-json ~= 0.6.7", + "rich ~= 13.1", + "tomlkit ~= 0.10", + "typer ~= 0.8", +] + +[project.urls] +homepage = "https://github.com/carderne/una" +repository = "https://github.com/carderne/una" + +[project.scripts] +una = "una.cli:app" + +[tool.rye] +managed = true +dev-dependencies = [ + "pytest~=8.3.1", + "basedpyright~=1.15.2", +] + +[tool.rye.workspace] +members = [".", "plugins/hatch"] + +[build-system] +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["una"] + +[tool.hatch.version] +source = "vcs" + +[tool.ruff] +target-version = "py311" +line-length = 120 + +[tool.ruff.lint] +select = ["A", "E", "F", "I", "N", "T100", "UP", "ANN401"] +ignore = ["F841"] # pyright does this + +[tool.ruff.lint.isort] +known-first-party = ["una"] + +[tool.basedpyright] +venvPath = "." +venv = ".venv" +pythonVersion = "3.11" +strict = ["**/*.py"] +extraPaths = ["plugins/hatch"] +reportUnnecessaryTypeIgnoreComment = true +reportImplicitOverride = false +reportUnusedCallResult = false +enableTypeIgnoreComments = true + +[tool.pytest.ini_options] +addopts = "-sv" diff --git a/requirements-dev.lock b/requirements-dev.lock new file mode 100644 index 0000000..d88cc6b --- /dev/null +++ b/requirements-dev.lock @@ -0,0 +1,62 @@ +# generated by rye +# use `rye lock` or `rye sync` to update this lockfile +# +# last locked with the following flags: +# pre: false +# features: [] +# all-features: false +# with-sources: false +# generate-hashes: false +# universal: false + +-e file:. +-e file:plugins/hatch +basedpyright==1.15.2 +click==8.1.7 + # via typer +dataclasses-json==0.6.7 + # via una +editables==0.5 + # via hatchling +hatchling==1.21.1 + # via hatch-una +iniconfig==2.0.0 + # via pytest +markdown-it-py==3.0.0 + # via rich +marshmallow==3.21.3 + # via dataclasses-json +mdurl==0.1.2 + # via markdown-it-py +mypy-extensions==1.0.0 + # via typing-inspect +nodejs-wheel-binaries==20.16.0 + # via basedpyright +packaging==24.1 + # via hatchling + # via marshmallow + # via pytest +pathspec==0.12.1 + # via hatchling +pluggy==1.5.0 + # via hatchling + # via pytest +pygments==2.18.0 + # via rich +pytest==8.3.1 +rich==13.7.1 + # via typer + # via una +shellingham==1.5.4 + # via typer +tomlkit==0.13.0 + # via una +trove-classifiers==2024.7.2 + # via hatchling +typer==0.12.3 + # via una +typing-extensions==4.12.2 + # via typer + # via typing-inspect +typing-inspect==0.9.0 + # via dataclasses-json diff --git a/requirements.lock b/requirements.lock new file mode 100644 index 0000000..c6054a3 --- /dev/null +++ b/requirements.lock @@ -0,0 +1,54 @@ +# generated by rye +# use `rye lock` or `rye sync` to update this lockfile +# +# last locked with the following flags: +# pre: false +# features: [] +# all-features: false +# with-sources: false +# generate-hashes: false +# universal: false + +-e file:. +-e file:plugins/hatch +click==8.1.7 + # via typer +dataclasses-json==0.6.7 + # via una +editables==0.5 + # via hatchling +hatchling==1.21.1 + # via hatch-una +markdown-it-py==3.0.0 + # via rich +marshmallow==3.21.3 + # via dataclasses-json +mdurl==0.1.2 + # via markdown-it-py +mypy-extensions==1.0.0 + # via typing-inspect +packaging==24.1 + # via hatchling + # via marshmallow +pathspec==0.12.1 + # via hatchling +pluggy==1.5.0 + # via hatchling +pygments==2.18.0 + # via rich +rich==13.7.1 + # via typer + # via una +shellingham==1.5.4 + # via typer +tomlkit==0.13.0 + # via una +trove-classifiers==2024.7.2 + # via hatchling +typer==0.12.3 + # via una +typing-extensions==4.12.2 + # via typer + # via typing-inspect +typing-inspect==0.9.0 + # via dataclasses-json diff --git a/tests/test_alias.py b/tests/test_alias.py new file mode 100644 index 0000000..5a7ce29 --- /dev/null +++ b/tests/test_alias.py @@ -0,0 +1,41 @@ +from una import alias + + +def test_parse_one_key_one_value_alias(): + res = alias.parse(["opencv-python=cv2"]) + assert res["opencv-python"] == ["cv2"] + assert len(res.keys()) == 1 + + +def test_parse_one_key_many_values_alias(): + res = alias.parse(["matplotlib=matplotlib, mpl_toolkits"]) + assert res["matplotlib"] == ["matplotlib", "mpl_toolkits"] + assert len(res.keys()) == 1 + + +def test_parse_many_keys_many_values_alias(): + res = alias.parse(["matplotlib=matplotlib, mpl_toolkits", "opencv-python=cv2"]) + assert res["matplotlib"] == ["matplotlib", "mpl_toolkits"] + assert res["opencv-python"] == ["cv2"] + assert len(res.keys()) == 2 + + +def test_pick_alias_by_key(): + aliases = {"opencv-python": ["cv2"]} + keys = {"one", "two", "opencv-python", "three"} + res = alias.pick(aliases, keys) + assert res == {"cv2"} + + +def test_pick_aliases_by_keys(): + aliases = {"opencv-python": ["cv2"], "matplotlib": ["mpl_toolkits", "matplotlib"]} + keys = {"one", "two", "opencv-python", "matplotlib", "three"} + res = alias.pick(aliases, keys) + assert res == {"cv2", "mpl_toolkits", "matplotlib"} + + +def test_pick_empty_alias_by_keys(): + aliases: dict[str, list[str]] = {} + keys = {"one", "two", "opencv-python", "matplotlib", "three"} + res = alias.pick(aliases, keys) + assert res == set() diff --git a/tests/test_check.py b/tests/test_check.py new file mode 100644 index 0000000..adc85fe --- /dev/null +++ b/tests/test_check.py @@ -0,0 +1,21 @@ +from pathlib import Path + +from una import check +from una.types import ExtDeps, Options, Proj + + +def test_collect_known_aliases_and_sub_dependencies(): + fake_project_data = Proj( + name="", + packages=[], + path=Path(), + ext_deps=ExtDeps( + items={"typer": "1", "hello-world-library": "2"}, + source="unit-test", + ), + ) + fake_options = Options(alias=["hello-world-library=hello"]) + res = check.collect_known_aliases(fake_project_data, fake_options) + assert "typer" in res + assert "typing-extensions" in res + assert "hello" in res diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..abfab88 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,70 @@ +from pathlib import Path + +import pytest +from pytest import MonkeyPatch + +from una import config +from una.types import Include + +BASE_PYPROJECT = """ +[project] +name = "" +version = "" +description = "" +authors = [] +readme = "" +requires-python = "" +dependencies = ["fastapi~=0.109.2", "uvicorn~=0.25.0", "tomlkit"] +[tool.hatch.build] +""" + + +@pytest.fixture +def use_fake(monkeypatch: MonkeyPatch): + def patch(): + monkeypatch.setattr(config, "load_conf", lambda _: config.load_conf_from_str(BASE_PYPROJECT)) # type: ignore[reportUnknownArgumentType] + + return patch + + +fake_path = Path.cwd() + + +namespace = "unittest" +hatch_toml = f""" +{BASE_PYPROJECT} +[tool.una.libs] +"../../apps/unittest/one" = "unittest/one" +"../../libs/unittest/two" = "unittest/two" +""" + +pep_621_toml_deps = f""" +{BASE_PYPROJECT} +[project.optional-dependencies] +dev = ["an-optional-lib==1.2.3", "another"] +local = ["awsglue-local-dev==1.0.0"] +""" +expected_packages = [ + Include(src="../../apps/unittest/one", dst="unittest/one"), + Include(src="../../libs/unittest/two", dst="unittest/two"), +] + + +def test_get_hatch_package_includes(): + conf = config.load_conf_from_str(hatch_toml) + res = config.get_project_package_includes(namespace, conf) + assert res == expected_packages + + +def test_parse_pep_621_project_dependencies(): + expected_dependencies = { + "fastapi": "~=0.109.2", + "uvicorn": "~=0.25.0", + "tomlkit": "", + "an-optional-lib": "==1.2.3", + "another": "", + "awsglue-local-dev": "==1.0.0", + } + conf = config.load_conf_from_str(pep_621_toml_deps) + res = config.parse_project_dependencies(conf) + assert res == expected_dependencies diff --git a/tests/test_diff.py b/tests/test_diff.py new file mode 100644 index 0000000..98e4fe0 --- /dev/null +++ b/tests/test_diff.py @@ -0,0 +1,23 @@ +from pathlib import Path + +from una import differ + +root = Path.cwd() +ns = "my_namespace" +changed_files = [ + Path(f"libs/{ns}/a/core.py"), + Path(f"some/other/{ns}/file.py"), + Path(f"apps/{ns}/b/core.py"), + Path(f"libs/{ns}/b/core.py"), + Path(f"libs/{ns}/c/nested/subfolder/core.py"), +] + + +def test_get_changed_libs(): + res = differ.get_changed_libs(root, changed_files, ns) + assert res == ["a", "b", "c"] + + +def test_get_changed_apps(): + res = differ.get_changed_apps(root, changed_files, ns) + assert res == ["b"] diff --git a/tests/test_distributions.py b/tests/test_distributions.py new file mode 100644 index 0000000..23a048e --- /dev/null +++ b/tests/test_distributions.py @@ -0,0 +1,71 @@ +import importlib.metadata + +from una import distributions +from una.types import ExtDeps + + +class FakeDist: + def __init__(self, name: str, data: str): + self.data = data + self.metadata = {"name": name} + + def read_text(self, *args: object) -> str: + return self.data + + +def test_distribution_packages_parse_contents_of_top_level_txt(): + dists = [FakeDist("python-jose", "jose\njose/backends\n")] + res = distributions.distributions_packages(dists) # type: ignore[reportArgumentType] + expected_dist = "python-jose" + expected_packages = ["jose", "jose.backends"] + assert res.get(expected_dist) is not None + assert res[expected_dist] == expected_packages + + +def test_parse_package_name_from_dist_requires(): + expected = { + "greenlet": "greenlet !=0.4.17", + "mysqlclient": "mysqlclient >=1.4.0 ; extra == 'mysql'", + "typing-extensions": "typing-extensions>=4.6.0", + "pymysql": "pymysql ; extra == 'pymysql'", + "one": "one<=0.4.17", + "two": "two^=0.4.17", + "three": "three~=0.4.17", + } + for k, v in expected.items(): + assert k == distributions.parse_sub_package_name(v) + + +def test_distribution_sub_packages(): + dists = list(importlib.metadata.distributions()) + res = distributions.distributions_sub_packages(dists) + expected_dist = "typer" + expected_sub_package = "typing-extensions" + assert res.get(expected_dist) is not None + assert expected_sub_package in res[expected_dist] + + +def test_parse_third_party_library_name(): + fake_project_deps = ExtDeps( + items={ + "python": "^3.10", + "fastapi": "^0.110.0", + "uvicorn[standard]": "^0.27.1", + "python-jose[cryptography]": "^3.3.0", + "hello[world, something]": "^3.3.0", + }, + source="pyproject.toml", + ) + expected = { + "python", + "fastapi", + "uvicorn", + "standard", + "python-jose", + "cryptography", + "hello", + "world", + "something", + } + res = distributions.extract_library_names(fake_project_deps) + assert res == expected diff --git a/tests/test_external_deps.py b/tests/test_external_deps.py new file mode 100644 index 0000000..6ea9054 --- /dev/null +++ b/tests/test_external_deps.py @@ -0,0 +1,71 @@ +from una import external_deps +from una.types import OrgImports + + +def test_calculate_diff_reports_no_diff(): + int_dep_imports = OrgImports( + apps={"my_app": {"rich"}}, + libs={ + "one": {"rich"}, + "two": {"rich", "foo"}, + "thre": {"tomlkit"}, + }, + ) + third_party_libs = { + "tomlkit", + "foo", + "requests", + "rich", + } + res = external_deps.calculate_diff(int_dep_imports, third_party_libs, True) + assert len(res) == 0 + + +def test_calculate_diff_should_report_missing_dependency(): + expected_missing = "aws-lambda-powertools" + int_dep_imports = OrgImports( + apps={"my_app": {"foo"}}, + libs={ + "one": {"tomlkit"}, + "two": {"tomlkit", expected_missing, "rich"}, + "three": {"rich"}, + }, + ) + third_party_libs = { + "tomlkit", + "foo", + "mypy-extensions", + "rich", + } + res = external_deps.calculate_diff(int_dep_imports, third_party_libs, True) + assert res == {expected_missing} + + +def test_calculate_diff_should_identify_close_match(): + int_dep_imports = OrgImports( + apps={"my_app": {"foo"}}, + libs={ + "one": {"tomlkit"}, + "two": {"tomlkit", "aws_lambda_powertools", "rich"}, + "three": {"rich", "pyyoutube"}, + }, + ) + third_party_libs = { + "tomlkit", + "python-youtube", + "foo", + "aws-lambda-powertools", + "rich", + } + res = external_deps.calculate_diff(int_dep_imports, third_party_libs, True) + assert len(res) == 0 + + +def test_calculate_diff_should_identify_close_match_case_insensitive(): + int_dep_imports = OrgImports( + apps={"my_app": set()}, + libs={"one": {"PIL"}}, + ) + third_party_libs = {"pillow"} + res = external_deps.calculate_diff(int_dep_imports, third_party_libs, True) + assert len(res) == 0 diff --git a/tests/test_internal_deps.py b/tests/test_internal_deps.py new file mode 100644 index 0000000..7a2d335 --- /dev/null +++ b/tests/test_internal_deps.py @@ -0,0 +1,19 @@ +from una import internal_deps + +apps_in_ws = {"x"} +libs_in_ws = {"a", "b"} + + +def test_to_row_returns_columns_for_all_int_deps(): + expected_length = len(apps_in_ws) + len(libs_in_ws) + collected_import_data = {"x": {"a"}, "a": {"b"}} + flattened_imports: set[str] = set().union(*collected_import_data.values()) + rows = internal_deps.create_rows( + apps_in_ws, + libs_in_ws, + collected_import_data, + flattened_imports, # type: ignore[reportArgumentType] + ) + assert len(rows) == expected_length + for columns in rows: + assert len(columns) == expected_length diff --git a/tests/test_stdlib.py b/tests/test_stdlib.py new file mode 100644 index 0000000..94df3fa --- /dev/null +++ b/tests/test_stdlib.py @@ -0,0 +1,22 @@ +from una import stdlib + + +def test_stdlib_3_11(): + assert stdlib.py310.difference(stdlib.py311) == {"binhex"} + assert stdlib.py311.difference(stdlib.py310) == { + "tomllib", + "_tkinter", + "sitecustomize", + "usercustomize", + } + + +def test_stdlib_3_12(): + assert stdlib.py311.difference(stdlib.py312) == { + "asynchat", + "asyncore", + "distutils", + "imp", + "smtpd", + } + assert stdlib.py312.difference(stdlib.py311) == set() diff --git a/tests/test_sync.py b/tests/test_sync.py new file mode 100644 index 0000000..b5165cf --- /dev/null +++ b/tests/test_sync.py @@ -0,0 +1,86 @@ +from pathlib import Path + +from una import config, sync +from una.types import Diff, Include, OrgImports, Style + + +def test_int_dep_to_pyproject_package(): + ns = "unit_test" + int_dep = "greet" + root = "../../libs" + expected_proj = Include(src=f"{root}/{ns}/{int_dep}", dst=f"{ns}/{int_dep}") + style = Style.modules + res_proj = sync.to_package(ns, int_dep, root, style) + assert res_proj == expected_proj + + +def test_int_deps_to_pyproject_packages(): + ns = "unit_test" + app_name = "hello" + lib_name = "world" + expected = [ + Include(src=f"../../apps/{app_name}/{ns}/{app_name}", dst=f"{ns}/{app_name}"), + Include(src=f"../../libs/{lib_name}/{ns}/{lib_name}", dst=f"{ns}/{lib_name}"), + ] + diff = Diff( + name="unit-test", + path=Path.cwd(), + apps={app_name}, + libs={lib_name}, + int_dep_imports=OrgImports(), + ) + res = sync.to_packages(ns, diff) + assert res == expected + + +packages = [ + Include(src="apps/hello/first", dst="hello/first"), + Include(src="libs/hello/second", dst="hello/second"), + Include(src="libs/hello/third", dst="hello/third"), +] +expected_hatch_packages = { + "apps/hello/first": "hello/first", + "libs/hello/second": "hello/second", + "libs/hello/third": "hello/third", +} + +BASE_PYPROJECT = """ +[project] +name = "" +version = "" +description = "" +authors = [] +dependencies = [] +readme = "" +requires-python = "" +""" + + +def test_generate_updated_hatch_project_with_existing_una_sections(): + pyproj = f""" +{BASE_PYPROJECT} +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" +[tool.hatch.build] +[tool.una.libs] +"apps/hello/first" = "hello/first" +""" + conf = config.load_conf_from_str(pyproj) + updated = str(sync.generate_updated_project(conf, packages[1:])) + res = config.load_conf_from_str(updated).tool.una.libs + assert res == expected_hatch_packages + + +def test_generate_updated_hatch_project_with_missing_int_dep_config(): + pyproj = f""" +{BASE_PYPROJECT} +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" +[tool.hatch.build] + """ + conf = config.load_conf_from_str(pyproj) + updated = str(sync.generate_updated_project(conf, packages)) + res = config.load_conf_from_str(updated).tool.una.libs + assert res == expected_hatch_packages diff --git a/una/__init__.py b/una/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/una/__main__.py b/una/__main__.py new file mode 100644 index 0000000..51cb001 --- /dev/null +++ b/una/__main__.py @@ -0,0 +1,3 @@ +from una.cli import app + +app() diff --git a/una/alias.py b/una/alias.py new file mode 100644 index 0000000..1897aa0 --- /dev/null +++ b/una/alias.py @@ -0,0 +1,35 @@ +from functools import reduce +from typing import cast + + +def _to_key_with_values(acc: dict[str, list[str]], alias: str) -> dict[str, list[str]]: + k, v = str.split(alias, "=") + values = [str.strip(val) for val in str.split(v, ",")] + return {**acc, **{k: values}} + + +def parse(aliases: list[str]) -> dict[str, list[str]]: + """Parse a list of aliases defined as key=value(s) into a dictionary""" + return reduce(_to_key_with_values, aliases, {}) + + +def pick(aliases: dict[str, list[str]], keys: set[str]) -> set[str]: + matrix = [v for k, v in aliases.items() if k in keys] + flattened = sum(matrix, cast(list[str], [])) + return set(flattened) + + +KNOWN_ALIASES = { + "beautifulsoup4": ["bs4"], + "pillow": ["PIL"], + "scikit-learn": ["sklearn"], + "scikit-image": ["skimage"], + "opencv-python": ["cv2"], + "python-ffmpeg": ["ffmpeg"], + "pycryptodome": ["Crypto"], + "pycryptodomex": ["Cryptodome"], + "pyserial": ["serial"], + "python-multipart": ["multipart"], + "pyusb": ["usb"], + "pyyaml": ["yaml"], +} diff --git a/una/check.py b/una/check.py new file mode 100644 index 0000000..ff862d6 --- /dev/null +++ b/una/check.py @@ -0,0 +1,163 @@ +from copy import deepcopy +from pathlib import Path + +from rich.console import Console + +from una import config, defaults, distributions, external_deps, files, lock_files, parse +from una.types import CheckReport, ExtDeps, Imports, Options, OrgImports, Proj, Style + + +def check_int_ext_deps(root: Path, ns: str, project: Proj, options: Options) -> bool: + name = project.name + int_dep_imports, ext_dep_imports = collect_all_imports(root, ns, project) + collected_libs = collect_known_aliases(project, options) + details = create_report( + project, + int_dep_imports, + ext_dep_imports, + collected_libs, + ) + num_apps = len(project.int_deps.apps) + res = all([num_apps == 1, not details.int_dep_diff, not details.ext_dep_diff]) + if not options.quiet: + print_one_app(num_apps, name) + print_missing_deps(details.int_dep_diff, name) + print_missing_deps(details.ext_dep_diff, name) + if options.verbose: + print_int_dep_imports(details.int_dep_imports) + print_int_dep_imports(details.ext_dep_imports) + return res + + +def with_ext_deps_from_lock_file(project: Proj) -> Proj: + lock_file_path = lock_files.pick_lock_file(project) + if not lock_file_path: + return project + ext_deps = lock_files.extract_libs(lock_file_path) + project = deepcopy(project) + project.ext_deps = ExtDeps(source=str(lock_file_path), items=ext_deps) + return project + + +def collect_known_aliases(project: Proj, options: Options) -> set[str]: + return distributions.known_aliases_and_sub_dependencies(project.ext_deps, options.alias) + + +def only_int_dep_imports(imports: set[str], top_ns: str) -> set[str]: + return {i for i in imports if i.startswith(top_ns)} + + +def only_int_dep_name(int_dep_imports: set[str]) -> set[str]: + res = [i.split(".") for i in int_dep_imports] + return {i[1] for i in res if len(i) > 1} + + +def extract_int_dep_imports(all_imports: Imports, top_ns: str) -> Imports: + only_int = {k: only_int_dep_imports(v, top_ns) for k, v in all_imports.items()} + return {k: only_int_dep_name(v) for k, v in only_int.items() if v} + + +def extract_int_deps(paths: set[Path], ns: str) -> Imports: + all_imports = parse.fetch_all_imports(paths) + return extract_int_dep_imports(all_imports, ns) + + +def with_unknown_libs(root: Path, ns: str, int_dep_imports: Imports) -> Imports: + keys = set(int_dep_imports.keys()) + values: set[str] = set().union(*int_dep_imports.values()) + unknowns = values.difference(keys) + if not unknowns: + return int_dep_imports + paths = files.collect_libs_paths(root, ns, unknowns) + extracted = extract_int_deps(paths, ns) + if not extracted: + return int_dep_imports + collected = {**int_dep_imports, **extracted} + return with_unknown_libs(root, ns, collected) + + +def diff(known_int_deps: set[str], apps: set[str], libs: set[str]) -> set[str]: + int_deps: set[str] = set().union(apps, libs) + return known_int_deps.difference(int_deps) + + +def imports_diff(int_dep_imports: OrgImports, apps: set[str], libs: set[str]) -> set[str]: + flattened_apps: set[str] = set().union(*int_dep_imports.apps.values()) + flattened_libs: set[str] = set().union(*int_dep_imports.libs.values()) + flattened_imports: set[str] = set().union(flattened_apps, flattened_libs) + return diff(flattened_imports, apps, libs) + + +def fetch_int_dep_imports(root: Path, ns: str, all_imports: Imports) -> Imports: + extracted = extract_int_dep_imports(all_imports, ns) + res = with_unknown_libs(root, ns, extracted) + return res + + +def print_int_dep_imports(int_dep_imports: OrgImports) -> None: + console = Console(theme=defaults.una_theme) + apps_imports = int_dep_imports.apps + libs_imports = int_dep_imports.libs + int_deps = {**apps_imports, **libs_imports} + for key, values in int_deps.items(): + imports_in_int_dep = values.difference({key}) + if not imports_in_int_dep: + continue + joined = ", ".join(imports_in_int_dep) + message = f":information: [data]{key}[/] is importing [data]{joined}[/]" + console.print(message) + + +def print_one_app(num_apps: int, project_name: str) -> None: + if num_apps == 1: + return + console = Console(theme=defaults.una_theme) + console.print(f"Projects must include exactly ONE app, but {project_name} has {num_apps}") + + +def print_missing_deps(diff: set[str], project_name: str) -> None: + if not diff: + return + console = Console(theme=defaults.una_theme) + missing = ", ".join(sorted(diff)) + console.print(f":thinking_face: Cannot locate {missing} in {project_name}") + + +def collect_all_imports(root: Path, ns: str, project: Proj) -> tuple[OrgImports, OrgImports]: + app_pkgs = {b for b in project.int_deps.apps} + libs_pkgs = {c for c in project.int_deps.libs} + apps_paths = files.collect_apps_paths(root, ns, app_pkgs) + libs_paths = files.collect_libs_paths(root, ns, libs_pkgs) + all_imports_in_apps = parse.fetch_all_imports(apps_paths) + all_imports_in_libs = parse.fetch_all_imports(libs_paths) + int_dep_imports = OrgImports( + apps=fetch_int_dep_imports(root, ns, all_imports_in_apps), + libs=fetch_int_dep_imports(root, ns, all_imports_in_libs), + ) + ext_dep_imports = OrgImports( + apps=external_deps.extract_ext_dep_imports(all_imports_in_apps, ns), + libs=external_deps.extract_ext_dep_imports(all_imports_in_libs, ns), + ) + return int_dep_imports, ext_dep_imports + + +def create_report( + project: Proj, + int_dep_imports: OrgImports, + ext_dep_imports: OrgImports, + third_party_libs: set[str], +) -> CheckReport: + app_pkgs = {b for b in project.int_deps.apps} + lib_pkgs = {c for c in project.int_deps.libs} + int_dep_diff = imports_diff(int_dep_imports, app_pkgs, lib_pkgs) + + style = config.get_style() + include_libs = style == Style.modules + ext_dep_diff = external_deps.calculate_diff(ext_dep_imports, third_party_libs, include_libs) + + return CheckReport( + int_dep_imports=int_dep_imports, + ext_dep_imports=ext_dep_imports, + int_dep_diff=int_dep_diff, + ext_dep_diff=ext_dep_diff, + ) diff --git a/una/cli.py b/una/cli.py new file mode 100644 index 0000000..70e1b38 --- /dev/null +++ b/una/cli.py @@ -0,0 +1,138 @@ +from pathlib import Path +from typing import Annotated + +from rich.console import Console +from typer import Argument, Exit, Option, Typer + +from una import check, config, defaults, differ, external_deps, files, internal_deps, sync +from una.types import Options, Proj, Style + +app = Typer(name="una", no_args_is_help=True, add_completion=False) +create = Typer(no_args_is_help=True) +app.add_typer( + create, + name="create", + help="Commands for creating a workspace, apps, libs and projects.", +) + +option_alias = Option(help="alias for third-party libraries, useful when an import differ from the library name") # type: ignore[reportAny] +option_verbose = Option(help="More verbose output.") # type: ignore[reportAny] +option_quiet = Option(help="Do not output any messages.") # type: ignore[reportAny] + + +def filtered_projects_data(projects: list[Proj]) -> list[Proj]: + dir_path = Path.cwd().name + return [p for p in projects if dir_path in p.path.as_posix()] + + +def enriched_with_lock_file_data(project: Proj, is_verbose: bool) -> Proj: + try: + return check.with_ext_deps_from_lock_file(project) + except ValueError as e: + if is_verbose: + print(f"{project.name}: {e}") + return project + + +def enriched_with_lock_files_data(projects: list[Proj], is_verbose: bool) -> list[Proj]: + return [enriched_with_lock_file_data(p, is_verbose) for p in projects] + + +@app.command("info") +def info_command( + alias: Annotated[str, option_alias] = "", +): + """Info about the Una workspace.""" + root = config.get_workspace_root() + ns = config.get_ns() + options = Options(alias=str.split(alias, ",") if alias else []) + + internal_deps.int_deps_from_projects(root, ns) + + internal_deps.int_cross_deps(root, ns) + + # Deps on external libraries + projects = internal_deps.get_projects_data(root, ns) + filtered_projects = filtered_projects_data(projects) + merged_projects_data = enriched_with_lock_files_data(filtered_projects, False) + results = external_deps.external_deps_from_all(root, ns, merged_projects_data, options) + external_deps.print_libs_in_projects(filtered_projects, options) + if not all(results): + raise Exit(code=1) + + +@app.command("diff") +def diff_command( + since: Annotated[str, Option(help="Changed since a specific tag.")] = "", + int_deps: Annotated[bool, Option(help="Print changed int_deps.")] = False, +): + """Shows changed int_deps compared to the latest git tag.""" + differ.calc_diff(since, int_deps) + + +@app.command("sync") +def sync_command( + check_only: Annotated[bool, Option(help="Only check, make no changes")] = False, + quiet: Annotated[bool, option_quiet] = False, + verbose: Annotated[bool, option_verbose] = False, + alias: Annotated[str, option_alias] = "", +): + """Update pyproject.toml with missing int_deps.""" + root = config.get_workspace_root() + ns = config.get_ns() + projects = internal_deps.get_projects_data(root, ns) + options = Options( + quiet=quiet, + verbose=verbose, + alias=str.split(alias, ",") if alias else [], + ) + filtered_projects = filtered_projects_data(projects) + enriched_projects = enriched_with_lock_files_data(filtered_projects, verbose) + + if not check_only: + for p in filtered_projects: + sync.sync_project_int_deps(root, ns, p, options) + + results = {check.check_int_ext_deps(root, ns, p, options) for p in enriched_projects} + if not all(results): + raise Exit(code=1) + + +@create.command("lib") +def lib_command( + name: Annotated[str, Argument(help="Name of the lib.")], +): + """Creates an Una lib.""" + root = config.get_workspace_root() + style = config.get_style() + files.create_app_or_lib(root, name, "lib", style) + console = Console(theme=defaults.una_theme) + console.print("Success!") + console.print(f"Created lib {name}") + + +@create.command("app") +def app_command( + name: Annotated[str, Argument(help="Name of the app.")], +): + """Creates an Una app.""" + root = config.get_workspace_root() + style = config.get_style() + files.create_app_or_lib(root, name, "app", style) + console = Console(theme=defaults.una_theme) + console.print("Success!") + console.print(f"Created app {name}") + + +@create.command("workspace") +def workspace_command( + style: Annotated[Style, Option(help="Workspace style")] = "packages", # type:ignore[reportArgumentType] +): + """Creates an Una workspace in the current directory.""" + path = Path.cwd() + ns = config.get_ns() + files.create_workspace(path, ns, style) + console = Console(theme=defaults.una_theme) + console.print("Success!") + console.print("Set up workspace in current directory.") + console.print("Remember to delete the src/ directory") diff --git a/una/config.py b/una/config.py new file mode 100644 index 0000000..4c56869 --- /dev/null +++ b/una/config.py @@ -0,0 +1,113 @@ +import re +from functools import lru_cache +from pathlib import Path +from typing import cast + +from una import defaults +from una.types import Conf, ExtDeps, Include, Style + + +def load_conf_from_str(s: str) -> Conf: + return Conf.from_str(s) + + +@lru_cache +def _load_conf(path: Path) -> Conf: + # made this private as pyright doesn't seem to like the cache decorator + with path.open(encoding="utf-8", errors="ignore") as f: + return load_conf_from_str(f.read()) + + +def load_conf(path: Path) -> Conf: + fullpath = (path / defaults.pyproj).resolve() + return _load_conf(fullpath) + + +def get_project_package_includes(namespace: str, conf: Conf) -> list[Include]: + includes = conf.tool.una.libs + return [Include(src=k, dst=v) for k, v in includes.items()] + + +def parse_pep_621_dependency(dep: str) -> dict[str, str]: + parts = re.split(r"[\^~=!<>]", dep) + name, *_ = parts if parts else [""] + version = str.replace(dep, name, "") + return {name: version} if name else {} + + +def get_pep_621_optional_dependencies(conf: Conf) -> list[str]: + groups = conf.project.optional_dependencies + matrix = [v for v in groups.values()] if isinstance(groups, dict) else [] + return sum(matrix, cast(list[str], [])) + + +def parse_project_dependencies(conf: Conf) -> dict[str, str]: + deps = conf.project.dependencies + optional_deps = get_pep_621_optional_dependencies(conf) + all_deps = deps + optional_deps + return {k: v for dep in all_deps for k, v in parse_pep_621_dependency(dep).items()} + + +def get_project_dependencies(data: Conf) -> ExtDeps: + items = parse_project_dependencies(data) + return ExtDeps(items=items, source=defaults.pyproj) + + +def get_ns() -> str: + path = get_workspace_root() + return load_conf(path).project.name + + +def get_style() -> Style: + path = get_workspace_root() + return load_conf(path).tool.una.style + + +def get_tag_pattern(key: str | None) -> str: + return "v*" + + +def get_tag_sort_options() -> list[str]: + return ["-committerdate"] + + +def get_int_dep_structure(root: Path) -> str: + root_conf = load_conf(root) + style = root_conf.tool.una.style + if style == Style.packages: + return "{int_dep}/{package}" + else: + return "{int_dep}/{ns}/{package}" + + +def is_drive_root(cwd: Path) -> bool: + return cwd == Path(cwd.root) or cwd == cwd.parent + + +def is_repo_root(cwd: Path) -> bool: + fullpath = cwd / defaults.root_file + return fullpath.exists() + + +def find_upwards(cwd: Path, name: str) -> Path | None: + if is_drive_root(cwd): + return None + fullpath = cwd / name + if fullpath.exists(): + return fullpath + if is_repo_root(cwd): + return None + return find_upwards(cwd.parent, name) + + +def find_upwards_dir(cwd: Path, name: str) -> Path | None: + fullpath = find_upwards(cwd, name) + return fullpath.parent if fullpath else None + + +def get_workspace_root() -> Path: + cwd = Path.cwd() + root = find_upwards_dir(cwd, defaults.root_file) + if not root: + raise ValueError("Didn't find the workspace root. Expected to find a .git directory.") + return root diff --git a/una/defaults.py b/una/defaults.py new file mode 100644 index 0000000..9bbc050 --- /dev/null +++ b/una/defaults.py @@ -0,0 +1,48 @@ +from rich.theme import Theme + +una_theme = Theme( + { + "data": "#999966", + "proj": "#8A2BE2", + "lib": "#32CD32", + "app": "#6495ED", + } +) + +keep_file_name = ".keep" +lock_file = "requirements.lock" +root_file = ".git" +pyproj = "pyproject.toml" + +libs_dir = "libs" +apps_dir = "apps" +proj_dir = "projects" + +packages_pyproj = """ +[project] +name = "{name}" +version = "0.1.0" +description = "" +authors = [] +dependencies = [] +requires-python = "{python_version}" + +[build-system] +requires = ["hatchling", "hatch-una"] +build-backend = "hatchling.build" + +[tool.rye] +managed = true +dev-dependencies = [] + +[tool.hatch.build.hooks.una-build] +[tool.hatch.metadata.hooks.una-meta] + +[tool.una.libs] +""" + +test_template = """\ +from {ns} import {name} +def test_import(): + assert {name} +""" diff --git a/una/differ.py b/una/differ.py new file mode 100644 index 0000000..bdb720c --- /dev/null +++ b/una/differ.py @@ -0,0 +1,118 @@ +import re +import subprocess +from pathlib import Path + +from rich.console import Console +from rich.padding import Padding + +from una import config, defaults, internal_deps +from una.types import Proj + + +def calc_diff(tag_name: str | None, only_int_deps: bool): + root = config.get_workspace_root() + tag = get_latest_tag(tag_name) + if not tag: + print("No tags found in repository.") + return + + ns = config.get_ns() + files = get_files(tag) + apps_paths = get_changed_apps(root, files, ns) + libs_paths = get_changed_libs(root, files, ns) + projects = get_changed_projects(files) + all_projects_data = internal_deps.get_int_deps_in_projects(root, libs_paths, apps_paths, ns) + projects_data = [p for p in all_projects_data] + if only_int_deps: + print_detected_changes_in_int_deps(apps_paths, libs_paths) + return + print_diff_summary(tag, apps_paths, libs_paths) + print_detected_changes(projects, "proj") + print_diff_details(projects_data, apps_paths, libs_paths) + + +def parse_folder_parts(pattern: str, changed_file: Path) -> str: + parts = re.split(pattern, changed_file.as_posix()) + remainder = parts[-1] + file_path = Path(remainder) + return next(p for p in file_path.parts if p != file_path.root) + + +def get_changed(pattern: str, changed_files: list[Path]) -> set[str]: + return {parse_folder_parts(pattern, f) for f in changed_files if re.match(pattern, f.as_posix())} + + +def parse_path_pattern(_: Path, top_dir: str, namespace: str) -> str: + return f"{top_dir}/{namespace}/" + + +def get_changed_int_deps(root: Path, top_dir: str, changed_files: list[Path], namespace: str) -> list[str]: + pattern = parse_path_pattern(root, top_dir, namespace) + return sorted(get_changed(pattern, changed_files)) + + +def get_changed_libs(root: Path, changed_files: list[Path], namespace: str) -> list[str]: + return get_changed_int_deps(root, defaults.libs_dir, changed_files, namespace) + + +def get_changed_apps(root: Path, changed_files: list[Path], namespace: str) -> list[str]: + return get_changed_int_deps(root, defaults.apps_dir, changed_files, namespace) + + +def get_changed_projects(changed_files: list[Path]) -> list[str]: + res = get_changed(defaults.apps_dir, changed_files) + filtered = {p for p in res if p != defaults.apps_dir} + return sorted(filtered) + + +def get_latest_tag(key: str | None) -> str | None: + tag_pattern = config.get_tag_pattern(key) + sorting_options = [f"--sort={option}" for option in config.get_tag_sort_options()] + res = subprocess.run( + ["git", "tag", "-l"] + sorting_options + [f"{tag_pattern}"], + capture_output=True, + ) + return next((tag for tag in res.stdout.decode("utf-8").split()), None) + + +def get_files(tag: str) -> list[Path]: + res = subprocess.run( + ["git", "diff", tag, "--stat", "--name-only"], + capture_output=True, + ) + return [Path(p) for p in res.stdout.decode("utf-8").split()] + + +def print_diff_details(projects_data: list[Proj], apps: list[str], libs: list[str]) -> None: + if not apps and not libs: + return + console = Console(theme=defaults.una_theme) + table = internal_deps.build_int_deps_in_projects_table(projects_data, apps, libs, for_info=False) + console.print(table, overflow="ellipsis") + + +def print_detected_changes(changes: list[str], markup: str) -> None: + if not changes: + return + console = Console(theme=defaults.una_theme) + for int_dep in changes: + console.print(f"[data]:gear: Changes found in [/][{markup}]{int_dep}[/]") + + +def print_detected_changes_in_int_deps(apps: list[str], libs: list[str]) -> None: + sorted_apps = sorted(apps) + sorted_libs = sorted(libs) + print_detected_changes(sorted_libs, "lib") + print_detected_changes(sorted_apps, "app") + + +def print_diff_summary(tag: str, apps: list[str], libs: list[str]) -> None: + console = Console(theme=defaults.una_theme) + console.print(Padding(f"[data]Diff: based on the {tag} tag[/]", (1, 0, 1, 0))) + if not apps and not libs: + console.print("[data]No int_dep changes found.[/]") + return + if libs: + console.print(f"[lib]Changed libs[/]: [data]{len(libs)}[/]") + if apps: + console.print(f"[app]Changed apps[/]: [data]{len(apps)}[/]") diff --git a/una/distributions.py b/una/distributions.py new file mode 100644 index 0000000..7394070 --- /dev/null +++ b/una/distributions.py @@ -0,0 +1,108 @@ +import importlib.metadata +import re +from functools import lru_cache, reduce +from importlib.metadata import Distribution + +from una import alias +from una.types import ExtDeps + +SUB_DEP_SEPARATORS = r"[\s!=;><\^~]" + + +def extract_extras(name: str) -> set[str]: + chars = ["[", "]"] + replacement = "," + res = reduce(lambda acc, char: str.replace(acc, char, replacement), chars, name) + parts = str.split(res, replacement) + return {str.strip(p) for p in parts if p} + + +def extract_library_names(deps: ExtDeps) -> set[str]: + names = {k for k in deps.items} + with_extras = [extract_extras(n) for n in names] + res: set[str] = set().union(*with_extras) + return res + + +def known_aliases_and_sub_dependencies(deps: ExtDeps, library_alias: list[str]) -> set[str]: + """Collect known aliases (packages) for third-party libraries. + When the library origin is not from a lock-file: + collect sub-dependencies and distribution top-namespace for each library, and append to the result. + """ + lock_file = any(str.endswith(deps.source, s) for s in {".lock", ".txt"}) + third_party_libs = extract_library_names(deps) + dists = get_distributions() + dist_packages = distributions_packages(dists) + custom_aliases = alias.parse(library_alias) + sub_deps = distributions_sub_packages(dists) if not lock_file else {} + a = alias.pick(dist_packages, third_party_libs) + b = alias.pick(custom_aliases, third_party_libs) + c = alias.pick(alias.KNOWN_ALIASES, third_party_libs) + d = alias.pick(sub_deps, third_party_libs) + e = get_packages_distributions(third_party_libs) + return third_party_libs.union(a, b, c, d, e) + + +def parse_sub_package_name(dependency: str) -> str: + parts = re.split(SUB_DEP_SEPARATORS, dependency) + return str(parts[0]) + + +def dist_subpackages(dist: Distribution) -> dict[str, list[str]]: + name = dist.metadata["name"] + dependencies = importlib.metadata.requires(name) or [] + parsed_package_names = list({parse_sub_package_name(d) for d in dependencies}) + return {name: parsed_package_names} if dependencies else {} + + +def map_sub_packages(acc: dict[str, list[str]], dist: Distribution) -> dict[str, list[str]]: + return {**acc, **dist_subpackages(dist)} + + +def parsed_top_level_namespace(namespaces: list[str]) -> list[str]: + return [str.replace(ns, "/", ".") for ns in namespaces] + + +def top_level_packages(dist: Distribution) -> list[str]: + top_level = dist.read_text("top_level.txt") + namespaces = str.split(top_level or "") + return parsed_top_level_namespace(namespaces) + + +def mapped_packages(dist: Distribution) -> dict[str, list[str]]: + packages = top_level_packages(dist) + name = dist.metadata["name"] + return {name: packages} if packages else {} + + +def map_packages(acc: dict[str, list[str]], dist: Distribution) -> dict[str, list[str]]: + return {**acc, **mapped_packages(dist)} + + +def distributions_packages(dists: list[Distribution]) -> dict[str, list[str]]: + """Return a mapping of top-level packages to their distributions.""" + return reduce(map_packages, dists, {}) + + +def distributions_sub_packages(dists: list[Distribution]) -> dict[str, list[str]]: + """Return the dependencies of each distribution.""" + init: dict[str, list[str]] = {} + return reduce(map_sub_packages, dists, init) + + +@lru_cache +def get_distributions() -> list[Distribution]: + return list(importlib.metadata.distributions()) + + +def get_packages_distributions(project_dependencies: set[str]) -> set[str]: + """Return the mapped top namespace from an import + Example: + A third-party library, such as opentelemetry-instrumentation-fastapi. + The return value would be the mapped top namespace: opentelemetry + Note: available for Python >= 3.10 + """ + # added in Python 3.10 + dists = importlib.metadata.packages_distributions() + common = {k for k, v in dists.items() if project_dependencies.intersection(set(v))} + return common.difference(project_dependencies) diff --git a/una/external_deps.py b/una/external_deps.py new file mode 100644 index 0000000..a6ede83 --- /dev/null +++ b/una/external_deps.py @@ -0,0 +1,178 @@ +import difflib +from functools import reduce +from operator import itemgetter +from pathlib import Path + +from rich import box, markup +from rich.console import Console +from rich.padding import Padding +from rich.table import Table + +from una import config, defaults, distributions, files, parse, stdlib +from una.types import Imports, Options, OrgImports, Proj, Style + + +def external_deps_from_all( + root: Path, + ns: str, + projects: list[Proj], + options: Options, +) -> set[bool]: + imports = {p.name: get_third_party_imports(root, ns, p) for p in projects} + flattened = reduce(flatten_imports, imports.values(), OrgImports()) + print_libs_summary() + print_libs_in_int_deps(flattened) + return {missing_libs(p, imports, options) for p in projects} + + +def missing_libs(project: Proj, imports: dict[str, OrgImports], options: Options) -> bool: + name = project.name + deps = project.ext_deps + int_dep_imports = imports[name] + libs = distributions.known_aliases_and_sub_dependencies(deps, options.alias) + return print_missing_installed_libs( + int_dep_imports, + libs, + name, + ) + + +def flatten_imports(acc: OrgImports, item: OrgImports) -> OrgImports: + apps_i = item.apps + libs_i = item.libs + return OrgImports( + apps={**acc.apps, **apps_i}, + libs={**acc.libs, **libs_i}, + ) + + +def extract_top_ns_from_imports(imports: set[str]) -> set[str]: + return {imp.split(".")[0] for imp in imports} + + +def extract_ext_dep_imports(all_imports: Imports, top_ns: str) -> Imports: + std_libs = stdlib.get_stdlibs() + top_level_imports = {k: extract_top_ns_from_imports(v) for k, v in all_imports.items()} + to_exclude = std_libs.union({top_ns}) + with_third_party = {k: v - to_exclude for k, v in top_level_imports.items()} + return {k: v for k, v in with_third_party.items() if v} + + +def _get_third_party_imports(root: Path, paths: set[Path]) -> Imports: + top_ns = config.get_ns() + all_imports = parse.fetch_all_imports(paths) + return extract_ext_dep_imports(all_imports, top_ns) + + +def get_third_party_imports(root: Path, ns: str, project: Proj) -> OrgImports: + apps_d = {b for b in project.int_deps.apps} + libs_d = {c for c in project.int_deps.libs} + apps_paths = files.collect_apps_paths(root, ns, apps_d) + libs_paths = files.collect_libs_paths(root, ns, libs_d) + apps_imports = _get_third_party_imports(root, apps_paths) + libs_imports = _get_third_party_imports(root, libs_paths) + return OrgImports(apps=apps_imports, libs=libs_imports) + + +def flatten(imports: dict[str, set[str]]) -> set[str]: + res: set[str] = set().union(*imports.values()) + return res + + +def calculate_diff(imports: OrgImports, deps: set[str], include_libs: bool) -> set[str]: + apps_imports = flatten(imports.apps) + # workspace style doesn't require libs imports to be declared in the app deps + libs_imports: set[str] = flatten(imports.libs) if include_libs else set() + flat: set[str] = set().union(apps_imports, libs_imports) + unknown_imports = flat.difference(deps) + cutoff = 0.6 + + unknowns = {str.lower(u) for u in unknown_imports} + deps_norm = {str.lower(d).replace("-", "_") for d in deps} + filtered = {u for u in unknowns if not difflib.get_close_matches(u, deps_norm, cutoff=cutoff)} + return filtered + + +def print_libs_summary() -> None: + console = Console(theme=defaults.una_theme) + console.print(Padding("[data]Libraries in apps and libs[/]", (1, 0, 0, 0))) + + +def print_libs_in_int_deps(int_dep_imports: OrgImports) -> None: + apps_imports = flatten(int_dep_imports.apps) + libs_imports = flatten(int_dep_imports.libs) + if not apps_imports and not libs_imports: + return + console = Console(theme=defaults.una_theme) + table = Table(box=box.SIMPLE_HEAD) + apps_i = int_dep_imports.apps + libs_i = int_dep_imports.libs + table.add_column("[data]int_dep[/]") + table.add_column("[data]library[/]") + for int_dep, imports in sorted(libs_i.items(), key=itemgetter(0)): + table.add_row(f"[lib]{int_dep}[/]", ", ".join(sorted(imports))) + for int_dep, imports in sorted(apps_i.items(), key=itemgetter(0)): + table.add_row(f"[app]{int_dep}[/]", ", ".join(sorted(imports))) + console.print(table, overflow="ellipsis") + + +def print_missing_installed_libs( + int_dep_imports: OrgImports, + third_party_libs: set[str], + project_name: str, +) -> bool: + style = config.get_style() + include_libs = style == Style.modules + diff = calculate_diff(int_dep_imports, third_party_libs, include_libs) + if not diff: + return True + console = Console(theme=defaults.una_theme) + missing = ", ".join(sorted(diff)) + console.print( + f"[data]Could not locate all libraries in [/][proj]{project_name}[/]. [data]Caused by missing dependencies?[/]" + ) + console.print(f":thinking_face: {missing}") + return False + + +def printable_version(version: str | None) -> str: + if version is None: + ver = "-" + elif version == "": + ver = "✔" + else: + ver = version + return f"[data]{ver}[/]" + + +def libs_in_projects_table( + projects: list[Proj], + libraries: set[str], + options: Options, +) -> Table: + table = Table(box=box.SIMPLE_HEAD) + projects = sorted(projects, key=lambda p: p.name) + project_headers = [f"[proj]{p.name}[/]" for p in projects] + headers = ["[data]library[/]"] + project_headers + for header in headers: + table.add_column(header) + for lib in sorted(libraries): + proj_versions = [p.ext_deps.items.get(lib) for p in projects] + printable_proj_versions = [printable_version(v) for v in proj_versions] + cols = [markup.escape(lib)] + printable_proj_versions + table.add_row(*cols) + return table + + +def flattened_lib_names(projects: list[Proj]) -> set[str]: + return {k for proj in projects for k in proj.ext_deps.items} + + +def print_libs_in_projects(projects: list[Proj], options: Options) -> None: + flattened = flattened_lib_names(projects) + if not flattened: + return + table = libs_in_projects_table(projects, flattened, options) + console = Console(theme=defaults.una_theme) + console.print(Padding("[data]Libraries in projects[/]", (1, 0, 0, 0))) + console.print(table, overflow="ellipsis") diff --git a/una/files.py b/una/files.py new file mode 100644 index 0000000..7a6f2f7 --- /dev/null +++ b/una/files.py @@ -0,0 +1,181 @@ +from pathlib import Path +from typing import Literal + +import tomlkit + +from una import config, defaults +from una.types import ConfWrapper, DepKind, Include, Proj, Style + + +def create_file(path: Path, name: str) -> Path: + fullpath = path / name + fullpath.touch() + return fullpath + + +def create_dir(path: Path, dir_name: str, keep: bool = False) -> Path: + d = path / dir_name + d.mkdir(parents=True) + if keep: + create_file(d, defaults.keep_file_name) + return d + + +def is_int_dep_dir(p: Path) -> bool: + return p.is_dir() and p.name not in {"__pycache__", ".venv", ".mypy_cache"} + + +def get_libs_dirs(root: Path, top_dir: str, ns: str) -> list[Path]: + style = config.get_style() + sub = "" if style == Style.packages else ns + lib_dir = root / top_dir / sub + if not lib_dir.exists(): + return [] + return [f for f in lib_dir.iterdir() if is_int_dep_dir(f)] + + +def get_apps_libs_names(root: Path, ns: str, top_dir: str) -> list[str]: + dirs = get_libs_dirs(root, top_dir, ns) + return [d.name for d in dirs] + + +def get_libs(root: Path, ns: str) -> list[str]: + return get_apps_libs_names(root, ns, top_dir=defaults.libs_dir) + + +def get_apps(root: Path, ns: str) -> list[str]: + return get_apps_libs_names(root, ns, defaults.apps_dir) + + +def collect_paths(root: Path, ns: str, int_dep: str, packages: set[str]) -> set[Path]: + p_template = config.get_int_dep_structure(root) + paths = {p_template.format(int_dep=int_dep, ns=ns, package=p) for p in packages} + return {Path(root / p) for p in paths} + + +def collect_apps_paths(root: Path, ns: str, libs: set[str]) -> set[Path]: + return collect_paths(root, ns, defaults.apps_dir, libs) + + +def collect_libs_paths(root: Path, ns: str, libs: set[str]) -> set[Path]: + return collect_paths(root, ns, defaults.libs_dir, libs) + + +def update_workspace_config(path: Path, ns: str, style: Style) -> None: + pyproj = path / defaults.pyproj + if style == Style.modules: + with pyproj.open() as f: + toml = tomlkit.parse(f.read()) + toml["tool"]["hatch"]["build"].add("dev-mode-dirs", ["libs", "apps"]) # type:ignore[reportIndexIssues] + toml["tool"]["una"] = {"style": "modules"} # type:ignore[reportIndexIssues] + with pyproj.open("w", encoding="utf-8") as f: + f.write(tomlkit.dumps(toml)) # type:ignore[reportUnknownMemberType] + else: + with pyproj.open() as f: + toml = tomlkit.parse(f.read()) + toml["tool"]["rye"]["virtual"] = True # type:ignore[reportIndexIssues] + toml["tool"]["rye"]["workspace"] = {"member": ["apps/*", "libs/*"]} # type:ignore[reportIndexIssues] + toml["tool"]["una"] = {"style": "packages"} # type:ignore[reportIndexIssues] + with pyproj.open("w") as f: + f.write(tomlkit.dumps(toml)) # type:ignore[reportUnknownMemberType] + + +def create_workspace(path: Path, ns: str, style: Style) -> None: + update_workspace_config(path, ns, style) + + if style == Style.modules: + create_project(path, "example_project") + + create_app_or_lib(path, "example_app", "app", style) + create_app_or_lib(path, "example_lib", "lib", style) + + +def parse_package_paths(packages: list[Include]) -> list[Path]: + sorted_packages = sorted(packages, key=lambda p: p.src) + return [Path(p.src) for p in sorted_packages] + + +def create_project(path: Path, name: str) -> None: + conf = config.load_conf(path) + python_version = conf.project.requires_python + + proj_dir = create_dir(path, f"projects/{name}") + pyproject_path = proj_dir / defaults.pyproj + newconf_text = defaults.packages_pyproj.format(name=name, python_version=python_version) + with pyproject_path.open("w", encoding="utf-8") as f: + f.write(newconf_text) + + +def create_package(path: Path, name: str, kind: Literal["app", "lib"]) -> None: + conf = config.load_conf(path) + python_version = conf.project.requires_python + ns = conf.project.name + + top_dir = defaults.apps_dir if kind == "app" else defaults.libs_dir + app_dir = create_dir(path, f"{top_dir}/{name}") + code_dir = create_dir(path, f"{top_dir}/{name}/{ns}/{name}") + test_dir = create_dir(path, f"{top_dir}/{name}/tests") + + create_file(code_dir, "__init__.py") + create_file(code_dir, "py.typed") + test_path = test_dir / "test_import.py" + with test_path.open("w", encoding="utf-8") as f: + content = defaults.test_template.format(ns=ns, name=name) + f.write(content) + + pyproject_path = app_dir / defaults.pyproj + newconf_text = defaults.packages_pyproj.format(name=name, python_version=python_version) + with pyproject_path.open("w", encoding="utf-8") as f: + f.write(newconf_text) + + +def create_module(path: Path, name: str, kind: Literal["app", "lib"]) -> None: + conf = config.load_conf(path) + ns = conf.project.name + + top_dir = defaults.apps_dir if kind == "app" else defaults.libs_dir + code_dir = create_dir(path, f"{top_dir}/{ns}/{name}") + create_file(code_dir, "__init__.py") + + test_path = code_dir / f"test_{name}.py" + with test_path.open("w", encoding="utf-8") as f: + content = defaults.test_template.format(ns=ns, name=name) + f.write(content) + + +def create_app_or_lib(path: Path, name: str, kind: DepKind, style: Style) -> None: + if style == Style.packages: + create_package(path, name, kind) + else: + create_module(path, name, kind) + + +def get_project_roots(root: Path) -> list[Path]: + style = config.get_style() + prefix = "projects" if style == Style.modules else "apps" + return sorted(root.glob(f"{prefix}/*/")) + + +def toml_data(path: Path) -> ConfWrapper: + return ConfWrapper(conf=config.load_conf(path), path=path) + + +def get_toml_files(root: Path) -> list[ConfWrapper]: + project_files = get_project_roots(root) + proj = [toml_data(p) for p in project_files] + return proj + + +def get_projects(root: Path) -> list[Proj]: + root_conf = config.load_conf(root) + ns = root_conf.project.name + confs = get_toml_files(root) + return [ + Proj( + name=c.conf.project.name, + packages=config.get_project_package_includes(ns, c.conf), + path=c.path, + ext_deps=config.get_project_dependencies(c.conf), + ) + for c in confs + ] diff --git a/una/internal_deps.py b/una/internal_deps.py new file mode 100644 index 0000000..1b093e1 --- /dev/null +++ b/una/internal_deps.py @@ -0,0 +1,175 @@ +from functools import reduce # noqa: I001 +from pathlib import Path + +from rich import box +from rich.console import Console +from rich.padding import Padding +from rich.table import Table + +from una import check, config, files, defaults +from una.types import Imports, Include, IntDeps, OrgImports, Proj, Style + + +def int_cross_deps(root: Path, ns: str) -> None: + apps = set(files.get_apps(root, ns)) + libs = set(files.get_libs(root, ns)) + int_dep_imports = get_int_dep_imports(root, ns, apps, libs) + imports = {**int_dep_imports.apps, **int_dep_imports.libs} + + flattened = flatten_imports(imports) + imported_apps = sorted({b for b in flattened if b in apps}) + imported_libs = sorted({c for c in flattened if c in libs}) + imported_int_deps = imported_libs + imported_apps + table = Table(box=box.SIMPLE_HEAD) + table.add_column("[data]int_dep[/]") + cols = create_columns(imported_apps, imported_libs) + rows = create_rows(apps, libs, imports, imported_int_deps) + for col in cols: + table.add_column(col, justify="center") + for row in rows: + table.add_row(*row) + console = Console(theme=defaults.una_theme) + console.print(Padding("[data]Internal libs in libs and apps[/]", (1, 0, 0, 0))) + console.print(table, overflow="ellipsis") + + +def get_int_dep_imports(root: Path, ns: str, apps: set[str], libs: set[str]) -> OrgImports: + apps_paths = files.collect_apps_paths(root, ns, apps) + comp_paths = files.collect_libs_paths(root, ns, libs) + int_dep_imports_in_apps = check.extract_int_deps(apps_paths, ns) + int_dep_imports_in_libs = check.extract_int_deps(comp_paths, ns) + return OrgImports( + apps=check.with_unknown_libs(root, ns, int_dep_imports_in_apps), + libs=check.with_unknown_libs(root, ns, int_dep_imports_in_libs), + ) + + +def to_col(int_dep: str, tag: str) -> str: + return f"[{tag}]{int_dep}[/]" + + +def int_dep_status_matrix(int_deps: set[str], int_dep_name: str, imported: str) -> str: + status = ":heavy_check_mark:" if imported in int_deps and imported != int_dep_name else "-" + return f"[data]{status}[/]" + + +def to_row(name: str, tag: str, int_dep_imports: Imports, imported: list[str]) -> list[str]: + int_deps = int_dep_imports.get(name) or set() + statuses = [int_dep_status_matrix(int_deps, name, i) for i in imported] + return [f"[{tag}]{name}[/]"] + statuses + + +def flatten_import(acc: set[str], kv: tuple[str, set[str]]) -> set[str]: + key = kv[0] + values = kv[1] + res: set[str] = set().union(acc, values.difference({key})) + return res + + +def flatten_imports(int_dep_imports: Imports) -> set[str]: + """Flatten the dict into a set of imports, with the actual int_dep filtered away when existing as an import""" + return reduce(flatten_import, int_dep_imports.items(), set()) + + +def create_columns(imported_apps: list[str], imported_libs: list[str]) -> list[str]: + app_cols = [to_col(int_dep, "app") for int_dep in imported_apps] + lib_cols = [to_col(int_dep, "lib") for int_dep in imported_libs] + return lib_cols + app_cols + + +def create_rows(apps: set[str], libs: set[str], import_data: Imports, imported: list[str]) -> list[list[str]]: + app_rows = [to_row(b, "app", import_data, imported) for b in sorted(apps)] + lib_rows = [to_row(c, "lib", import_data, imported) for c in sorted(libs)] + return lib_rows + app_rows + + +def int_deps_from_projects(root: Path, ns: str) -> None: + apps = files.get_apps(root, ns) + libs = files.get_libs(root, ns) + if not libs and not apps: + return + projects = get_int_deps_in_projects(root, libs, apps, ns) + table = build_int_deps_in_projects_table(projects, apps, libs) + console = Console(theme=defaults.una_theme) + console.print(Padding("[data]Internal deps in projects[/]", (1, 0, 0, 0))) + console.print(table, overflow="ellipsis") + + +def get_matching_int_deps(paths: list[Path], int_deps: list[str], namespace: str) -> list[str]: + paths_in_namespace = [p.name for p in paths if p.parent.name == namespace] + res = set(int_deps).intersection(paths_in_namespace) + return sorted(list(res)) + + +def get_project_int_deps( + project_packages: list[Include], + libs_paths: list[str], + apps_paths: list[str], + namespace: str, + self_name: str, + add_self: bool, +) -> IntDeps: + paths = files.parse_package_paths(project_packages) + libs_in_project = get_matching_int_deps(paths, libs_paths, namespace) + apps_in_project = get_matching_int_deps(paths, apps_paths, namespace) + if add_self: + apps_in_project.append(self_name) + return IntDeps(libs=libs_in_project, apps=apps_in_project) + + +def get_int_deps_in_projects(root: Path, libs_paths: list[str], apps_paths: list[str], namespace: str) -> list[Proj]: + packages = files.get_projects(root) + style = config.get_style() + add_self = style == Style.packages + res = [ + Proj( + name=p.name, + packages=p.packages, + path=p.path, + ext_deps=p.ext_deps, + int_deps=get_project_int_deps( + p.packages, + libs_paths, + apps_paths, + namespace, + p.name, + add_self, + ), + ) + for p in packages + ] + return res + + +def get_projects_data(root: Path, ns: str) -> list[Proj]: + app_names = files.get_apps(root, ns) + lib_names = files.get_libs(root, ns) + return get_int_deps_in_projects(root, lib_names, app_names, ns) + + +def int_dep_status_projects(int_dep: str, int_deps: list[str], for_info: bool) -> str: + emoji = ":heavy_check_mark:" if for_info else ":gear:" + status = emoji if int_dep in int_deps else "-" + return f"[data]{status}[/]" + + +def build_int_deps_in_projects_table( + projects_data: list[Proj], + apps: list[str], + libs: list[str], + for_info: bool = True, +) -> Table: + table = Table(box=box.SIMPLE_HEAD) + table.add_column("[data]int_dep[/]") + proj_cols = [f"[proj]{p.name}[/]" for p in projects_data] + for col in proj_cols: + table.add_column(col, justify="center") + for int_dep in sorted(libs): + statuses = [int_dep_status_projects(int_dep, p.int_deps.libs, for_info) for p in projects_data] + cols = [f"[lib]{int_dep}[/]"] + statuses + table.add_row(*cols) + for int_dep in sorted(apps): + statuses = [int_dep_status_projects(int_dep, p.int_deps.apps, for_info) for p in projects_data] + cols = [f"[app]{int_dep}[/]"] + statuses + table.add_row(*cols) + return table diff --git a/una/lock_files.py b/una/lock_files.py new file mode 100644 index 0000000..540827c --- /dev/null +++ b/una/lock_files.py @@ -0,0 +1,39 @@ +from pathlib import Path + +from una import defaults +from una.types import Proj + + +def pick_lock_file(project: Proj) -> Path | None: + lock = Path(project.path / defaults.lock_file) + if not lock.exists(): + return None + return lock + + +def parse_name(row: str) -> str: + parts = str.split(row, "==") + return parts[0] + + +def parse_version(row: str) -> str: + parts = str.split(row, "==")[1] + res = str.split(parts, " ") + return res[0] + + +def extract_lib_names_from_txt(path: Path) -> dict[str, str]: + with open(path) as f: + data = f.readlines() + rows = (str.strip(line) for line in data) + filtered = (row for row in rows if row and not row.startswith(("#", "-"))) + return {parse_name(row): parse_version(row) for row in filtered} + + +def extract_libs(path: Path) -> dict[str, str]: + if not path.exists(): + return {} + try: + return extract_lib_names_from_txt(path) + except (IndexError, KeyError, ValueError) as e: + raise ValueError(f"Failed reading {path}: {repr(e)}") from e diff --git a/una/parse.py b/una/parse.py new file mode 100644 index 0000000..2036d30 --- /dev/null +++ b/una/parse.py @@ -0,0 +1,49 @@ +import ast +from functools import lru_cache +from pathlib import Path + +from una.types import Imports + + +def parse_import(node: ast.Import) -> list[str | None]: + return [name.name for name in node.names] + + +def extract_import_from(node: ast.ImportFrom) -> list[str | None]: + return [f"{node.module}.{alias.name}" for alias in node.names] if node.names else [node.module] + + +def parse_import_from(node: ast.ImportFrom) -> list[str | None]: + return extract_import_from(node) if node.module and node.level == 0 else [] + + +def parse_imports(node: ast.AST) -> list[str | None]: + if isinstance(node, ast.Import): + return parse_import(node) + if isinstance(node, ast.ImportFrom): + return parse_import_from(node) + return [] + + +@lru_cache +def parse_module(path: Path) -> ast.AST: + with open(path.as_posix(), encoding="utf-8", errors="ignore") as f: + tree = ast.parse(f.read(), path.name) + return tree + + +def extract_imports(path: Path) -> list[str]: + tree = parse_module(path) + return [i for node in ast.walk(tree) for i in parse_imports(node) if i is not None] + + +def list_imports(path: Path) -> set[str]: + py_modules = path.rglob("*.py") + extracted = (extract_imports(m) for m in py_modules) + flattened = (i for imports in extracted for i in imports) + return set(flattened) + + +def fetch_all_imports(paths: set[Path]) -> Imports: + rows = [{p.name: list_imports(p)} for p in paths] + return {k: v for row in rows for k, v in row.items()} diff --git a/una/py.typed b/una/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/una/stdlib.py b/una/stdlib.py new file mode 100644 index 0000000..10f82dc --- /dev/null +++ b/una/stdlib.py @@ -0,0 +1,270 @@ +"""Borrowed from github.com/PyCQA/isort library. Thank you!""" + +import sys + +stdlib_python_3_8 = { + "_ast", + "_dummy_thread", + "_thread", + "abc", + "aifc", + "argparse", + "array", + "ast", + "asynchat", + "asyncio", + "asyncore", + "atexit", + "audioop", + "base64", + "bdb", + "binascii", + "binhex", + "bisect", + "builtins", + "bz2", + "cProfile", + "calendar", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "contextvars", + "copy", + "copyreg", + "crypt", + "csv", + "ctypes", + "curses", + "dataclasses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "distutils", + "doctest", + "dummy_threading", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "formatter", + "fractions", + "ftplib", + "functools", + "gc", + "getopt", + "getpass", + "gettext", + "glob", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "imaplib", + "imghdr", + "imp", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "lzma", + "mailbox", + "mailcap", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multiprocessing", + "netrc", + "nis", + "nntplib", + "ntpath", + "numbers", + "operator", + "optparse", + "os", + "ossaudiodev", + "parser", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "select", + "selectors", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "socketserver", + "spwd", + "sqlite3", + "sre", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "symbol", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "turtledemo", + "types", + "typing", + "unicodedata", + "unittest", + "urllib", + "uu", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "zlib", +} + + +def omit(data: set[str], keys: set[str]) -> set[str]: + return {k for k in data if k not in keys} + + +def union(stdlib: set[str], news: set[str], removed: set[str]) -> set[str]: + return omit(stdlib.union(news), removed) + + +def with_extras(stdlib: set[str]) -> set[str]: + extras = {"__future__", "pkg_resources"} + return stdlib.union(extras) + + +def to_py39(stdlib: set[str]) -> set[str]: + news = {"graphlib", "zoneinfo"} + removed = {"_dummy_thread", "dummy_threading"} + return union(stdlib, news, removed) + + +def to_py310(stdlib: set[str]) -> set[str]: + news = {"idlelib"} + removed = {"formatter", "parser", "symbol"} + return union(stdlib, news, removed) + + +def to_py311(stdlib: set[str]) -> set[str]: + news = {"tomllib", "_tkinter", "sitecustomize", "usercustomize"} + removed = {"binhex"} + return union(stdlib, news, removed) + + +def to_py312(stdlib: set[str]) -> set[str]: + news: set[str] = set() + removed = {"asynchat", "asyncore", "distutils", "imp", "smtpd"} + return union(stdlib, news, removed) + + +py38 = with_extras(stdlib_python_3_8) +py39 = to_py39(py38) +py310 = to_py310(py39) +py311 = to_py311(py310) +py312 = to_py312(py311) + + +def get_stdlibs() -> set[str]: + libs = {10: py310, 11: py311, 12: py312} + return libs.get(sys.version_info.minor, py312) diff --git a/una/sync.py b/una/sync.py new file mode 100644 index 0000000..872bd93 --- /dev/null +++ b/una/sync.py @@ -0,0 +1,104 @@ +from collections.abc import Sequence +from pathlib import Path + +from rich.console import Console + +from una import check, config, defaults, files, internal_deps +from una.config import load_conf +from una.types import Conf, Diff, Include, Options, Proj, Style + + +def sync_project_int_deps(root: Path, ns: str, project: Proj, options: Options): + apps_pkgs = files.get_apps(root, ns) + libs_pkgs = files.get_libs(root, ns) + diff = calculate_diff(root, ns, project, apps_pkgs, libs_pkgs) + update_project(ns, diff) + if options.quiet: + return + print_summary(diff) + if options.verbose: + print_int_dep_imports(diff) + + +def calculate_diff( + root: Path, + ns: str, + project: Proj, + apps_pkgs: Sequence[str], + libs_pkgs: Sequence[str], +) -> Diff: + proj_apps = set(project.int_deps.apps) + proj_libs = set(project.int_deps.libs) + all_apps = set(apps_pkgs) + all_libs = set(libs_pkgs) + int_dep_imports = internal_deps.get_int_dep_imports(root, ns, proj_apps, proj_libs) + int_dep_diff = check.imports_diff(int_dep_imports, proj_apps, proj_libs) + apps_diff = {b for b in int_dep_diff if b in all_apps} + libs_diff = {b for b in int_dep_diff if b in all_libs} + return Diff( + name=project.name, + path=project.path, + apps=apps_diff, + libs=libs_diff, + int_dep_imports=int_dep_imports, + ) + + +def print_int_dep_imports(diff: Diff) -> None: + int_dep_imports = diff.int_dep_imports + check.print_int_dep_imports(int_dep_imports) + + +def print_summary(diff: Diff) -> None: + console = Console(theme=defaults.una_theme) + name = diff.name + apps_pkgs = diff.apps + libs_pkgs = diff.libs + anything_to_sync = apps_pkgs or libs_pkgs + emoji = ":point_right:" if anything_to_sync else ":heavy_check_mark:" + printable_name = f"[proj]{name}[/]" + console.print(f"{emoji} {printable_name}") + for b in apps_pkgs: + console.print(f"adding [app]{b}[/] app to [proj]{name}[/]") + for c in libs_pkgs: + console.print(f"adding [lib]{c}[/] lib to [proj]{name}[/]") + if anything_to_sync: + console.print("") + + +def to_package(ns: str, name: str, int_dep_root: str, style: Style) -> Include: + root = Path(int_dep_root) + src = root / ns / name if style == Style.modules else root / name / ns / name + dst = Path(ns) / name + return Include(src=str(src), dst=str(dst)) + + +def generate_updated_project(conf: Conf, packages: list[Include]) -> str | None: + for inc in packages: + conf.tool.una.libs[inc.src] = inc.dst + return conf.to_str() + + +def to_packages(ns: str, diff: Diff) -> list[Include]: + style = config.get_style() + apps_path = "../../apps" + libs_path = "../../libs" + a = [to_package(ns, b, apps_path, style) for b in diff.apps] + b = [to_package(ns, c, libs_path, style) for c in diff.libs] + return a + b + + +def rewrite_project_file(path: Path, packages: list[Include]): + conf = load_conf(path) + generated = generate_updated_project(conf, packages) + if not generated: + return + fullpath = path / defaults.pyproj + with fullpath.open("w", encoding="utf-8") as f: + f.write(generated) + + +def update_project(ns: str, diff: Diff): + packages = to_packages(ns, diff) + if packages: + rewrite_project_file(diff.path, packages) diff --git a/una/types.py b/una/types.py new file mode 100644 index 0000000..c630646 --- /dev/null +++ b/una/types.py @@ -0,0 +1,194 @@ +from collections.abc import Callable +from copy import deepcopy +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path +from typing import TYPE_CHECKING, Literal, TypeAlias, TypeVar + +import tomlkit +from dataclasses_json import dataclass_json + +Json: TypeAlias = dict[str, "Json"] | list["Json"] | str | int | float | bool | None +Imports: TypeAlias = dict[str, set[str]] +DepKind: TypeAlias = Literal["app", "lib"] + + +@dataclass +class OrgImports: + apps: Imports = field(default_factory=dict) + libs: Imports = field(default_factory=dict) + + +@dataclass +class Include: + src: str + dst: str + # path: str + # include: str + # root: str + + +@dataclass(frozen=True) +class CheckReport: + int_dep_imports: OrgImports + ext_dep_imports: OrgImports + int_dep_diff: set[str] + ext_dep_diff: set[str] + + +@dataclass(frozen=True) +class Diff: + name: str + path: Path + apps: set[str] + libs: set[str] + int_dep_imports: OrgImports + + +@dataclass +class Options: + quiet: bool = False + verbose: bool = False + alias: list[str] = field(default_factory=list) + + +def rename_keys(old: str, new: str) -> Callable[[Json], None]: + def rename_hyphens(d: Json) -> None: + if isinstance(d, dict): + for k, v in list(d.items()): + rename_hyphens(v) + if old in k: + d[k.replace(old, new)] = d.pop(k) + + return rename_hyphens + + +@dataclass_json +@dataclass(frozen=True) +class Project: + name: str + dependencies: list[str] + version: str | None = None + optional_dependencies: dict[str, list[str]] | None = None + requires_python: str = ">= 3.8" + + +class Style(Enum): + packages = "packages" + modules = "modules" + + +@dataclass_json +@dataclass(frozen=True) +class Una: + style: Style = Style.packages + libs: dict[str, str] = field(default_factory=dict) + + +@dataclass_json +@dataclass(frozen=False) +class Tool: + una: Una = field(default_factory=Una) + + +Self = TypeVar("Self", bound="Conf") + + +@dataclass_json +@dataclass() +class Conf: + """ + Conf object. + + Should never be created manually, only loaded from a toml file. + See the caveats on `to_str()`. + """ + + project: Project + tool: Tool + _tomldoc: tomlkit.TOMLDocument | None = field(default=None) + + if TYPE_CHECKING: + # these are just here becaue dataclass_json doesn't + # seem to play well with pyright? + @classmethod + def from_dict(cls: type[Self], _: Json) -> Self: + raise + + @classmethod + def from_tomldoc(cls: type[Self], tomldoc: tomlkit.TOMLDocument) -> Self: + orig = deepcopy(tomldoc) + rename_keys("-", "_")(tomldoc) + res = cls.from_dict(tomldoc) + res._tomldoc = orig + return res + + def to_tomldoc(self) -> tomlkit.TOMLDocument: + tomldoc = self._tomldoc + if not tomldoc: + raise ValueError("This Conf has no _tomldoc member. This should not happen") + + # impossible for project.dependencies to be unset as validated on load + orig_deps: list[str] = tomldoc["project"]["dependencies"] # type: ignore[reportIndexIssues] + new_deps = set(self.project.dependencies) - set(orig_deps) + for dep in new_deps: + tomldoc["project"]["dependencies"].add_line(dep) # type: ignore[reportIndexIssues] + + # deal with a a non-existent tool.una.libs + try: + tomldoc["tool"]["una"]["libs"].update(self.tool.una.libs) # type: ignore[reportIndexIssues] + except KeyError: + una = tomlkit.table(True) + libs = tomlkit.table() + libs.update(self.tool.una.libs) # type: ignore[reportUnknownMemberType] + una.append("libs", libs) + tomldoc["tool"].append("una", una) # type: ignore[reportIndexIssues] + return tomldoc + + @classmethod + def from_str(cls: type[Self], s: str) -> Self: + tomldoc = tomlkit.loads(s) + return cls.from_tomldoc(tomldoc) + + def to_str(self) -> str: + """ + Dump the config to a string. + + To preserve the original formatting and make my life easy, this function + will currently only modify the following fields: + - project.dependencies + - tool.una.libs + - tool.hatch.build.hooks.una-build + - tool.hatch.meta.hooks.una-meta + + All others will be written from the original toml file. + """ + tomldoc = self.to_tomldoc() + return tomlkit.dumps(tomldoc) # type: ignore[reportUnknownMemberType] + + +@dataclass(frozen=True) +class ConfWrapper: + conf: Conf + path: Path + + +@dataclass(frozen=True) +class ExtDeps: + source: str + items: dict[str, str] + + +@dataclass(frozen=True) +class IntDeps: + libs: list[str] = field(default_factory=list) + apps: list[str] = field(default_factory=list) + + +@dataclass(frozen=False) +class Proj: + name: str + packages: list[Include] + path: Path + ext_deps: ExtDeps + int_deps: IntDeps = field(default_factory=IntDeps)