From 734dd9e0a772e86af559a327b8145dfd40a7e500 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Br=C3=A9nainn=20Woodsend?= Date: Sun, 1 Oct 2023 22:57:48 +0100 Subject: [PATCH] Record built artifacts information in .polycotylus/artifacts.json. --- docs/source/example-library.rst | 8 ++++ polycotylus/__main__.py | 1 + polycotylus/_alpine.py | 4 ++ polycotylus/_arch.py | 1 + polycotylus/_base.py | 27 +++++++++++ polycotylus/_fedora.py | 8 +++- polycotylus/_opensuse.py | 5 +- polycotylus/_void.py | 4 ++ pyproject.toml | 1 + tests/test_alpine.py | 84 +++++++++++++++++++++++++++++++++ tests/test_fedora.py | 29 +++++++++++- tests/test_manjaro.py | 4 +- tests/test_opensuse.py | 1 + tests/test_void.py | 4 +- 14 files changed, 175 insertions(+), 6 deletions(-) diff --git a/docs/source/example-library.rst b/docs/source/example-library.rst index 9f1bd6b..2c5b79c 100644 --- a/docs/source/example-library.rst +++ b/docs/source/example-library.rst @@ -377,6 +377,14 @@ You'll notice that this time, there are three packages produced. The one labelled ``main`` is the one you'd distribute. See :ref:`building for Fedora ` for information about the other two. +.. note:: + + If you want to locate the packages that `polycotylus` builds + programmatically, then please use the ``.polycotylus/artifacts.json`` file. + Neither parsing the last few lines of `polycotylus`\ 's console output nor + trying to guess the paths are likely to survive future changes to + `polycotylus`. + The next distribution and beyond ................................ diff --git a/polycotylus/__main__.py b/polycotylus/__main__.py index 5fea76b..68a6f51 100644 --- a/polycotylus/__main__.py +++ b/polycotylus/__main__.py @@ -94,6 +94,7 @@ def cli(argv=None): self.generate() artifacts = self.build() self.test(artifacts["main"]) + self.update_artifacts_json(artifacts) except polycotylus.PolycotylusUsageError as ex: raise SystemExit("Error: " + str(ex)) print(f"Built {len(artifacts)} artifact{'s' if len(artifacts) != 1 else ''}:") diff --git a/polycotylus/_alpine.py b/polycotylus/_alpine.py index b1b2a7a..95b2bca 100644 --- a/polycotylus/_alpine.py +++ b/polycotylus/_alpine.py @@ -42,6 +42,10 @@ class Alpine(BaseDistribution): "font": "ttf-dejavu", } + @_misc.classproperty + def tag(_, cls): + return cls.version + @classmethod @lru_cache() def _package_manager_queries(cls): diff --git a/polycotylus/_arch.py b/polycotylus/_arch.py index ae1ba73..4e1b353 100644 --- a/polycotylus/_arch.py +++ b/polycotylus/_arch.py @@ -31,6 +31,7 @@ class Arch(BaseDistribution): supported_architectures = { "x86_64": "x86_64", } + tag = "" @classmethod @lru_cache() diff --git a/polycotylus/_base.py b/polycotylus/_base.py index fd8c683..26d70e3 100644 --- a/polycotylus/_base.py +++ b/polycotylus/_base.py @@ -3,6 +3,7 @@ import re import os import platform +import json from functools import lru_cache from packaging.requirements import Requirement @@ -17,6 +18,7 @@ class BaseDistribution(abc.ABC): _formatter = abc.abstractproperty() supported_architectures = abc.abstractproperty() _packages = abc.abstractproperty() + tag = abc.abstractproperty() def __init__(self, project, architecture=None): self.project = project @@ -287,6 +289,31 @@ def build_test_image(self): def test(self, package): pass + def update_artifacts_json(self, packages): + import portalocker + json_path = self.project.root / ".polycotylus/artifacts.json" + with portalocker.Lock(json_path.with_name(".artifacts.lock")): + try: + artifacts = json.loads(json_path.read_bytes()) + except: + artifacts = [] + grouped = { + (i["distribution"], i["tag"], i["architecture"], i["variant"]): i["path"] + for i in artifacts + } + for (variant, path) in packages.items(): + grouped[(self.name, self.tag, self.architecture, variant)] = \ + str(path.relative_to(self.project.root)) + artifacts = [] + for ((name, tag, architecture, variant), path) in sorted(grouped.items()): + if (self.project.root / path).exists(): + artifacts.append({ + "distribution": name, "tag": tag, + "architecture": architecture, "variant": variant, + "path": path, + }) + _misc.unix_write(json_path, json.dumps(artifacts, indent=" ")) + def _deduplicate(array): """Remove duplicates, preserving order of first appearance.""" diff --git a/polycotylus/_fedora.py b/polycotylus/_fedora.py index 0a5607f..99c12ea 100644 --- a/polycotylus/_fedora.py +++ b/polycotylus/_fedora.py @@ -47,9 +47,15 @@ def __init__(self, project, architecture=None): raise _exceptions.PolycotylusUsageError( "Building for Fedora is not supported on Windows.") super().__init__(project, architecture) + if self.project.architecture == "none": + self.architecture = "noarch" available_packages = NotImplemented + @_misc.classproperty + def tag(_, cls): + return cls.version + @staticmethod def fix_package_name(name): return re.sub(r"[^a-z0-9]+", "-", name.lower()) @@ -277,7 +283,7 @@ def build(self): volumes=[(self.distro_root, "/io")] + self._mounted_caches, architecture=self.docker_architecture, post_mortem=True) rpms = {} - machine = "noarch" if self.project.architecture == "none" else self.architecture + machine = self.architecture pattern = re.compile( fr"{re.escape(self.package_name)}(?:-([^-]+))?-{self.project.version}.*\.{machine}\.rpm") for path in (self.distro_root / machine).glob(f"*.fc{self.version}.*.rpm"): diff --git a/polycotylus/_opensuse.py b/polycotylus/_opensuse.py index f0efa6e..3807b90 100644 --- a/polycotylus/_opensuse.py +++ b/polycotylus/_opensuse.py @@ -20,6 +20,7 @@ class OpenSUSE(BaseDistribution): image = "docker.io/opensuse/tumbleweed" + tag = "tumbleweed" python_extras = { "tkinter": ["python3-tk"], "curses": ["python3-curses"], @@ -48,6 +49,8 @@ def __init__(self, project, architecture=None): raise _exceptions.PolycotylusUsageError( "Building for OpenSUSE is not supported with podman.") super().__init__(project, architecture) + if self.project.architecture == "none": + self.architecture = "noarch" @classmethod @lru_cache() @@ -351,7 +354,7 @@ def build(self): architecture=self.docker_architecture) rpms = {} for python in ["python3"] if self.project.frontend else self.active_python_abis(): - arch = "noarch" if self.project.architecture == "none" else self.architecture + arch = self.architecture name = f"{python}-{self.fix_package_name(self.project.name)}-{self.project.version}-0.{arch}.rpm" rpm = self.distro_root / "RPMS" / arch / name rpms["main" if self.project.frontend else python] = rpm diff --git a/polycotylus/_void.py b/polycotylus/_void.py index 4f7704a..99884b8 100644 --- a/polycotylus/_void.py +++ b/polycotylus/_void.py @@ -41,6 +41,10 @@ def image(self, cls): architecture = cls.preferred_architecture if self is None else self.architecture return f"ghcr.io/void-linux/void-linux:latest-mini-{architecture}{cls.libc_tag}" + @_misc.classproperty + def tag(_, cls): + return cls.libc + def _build_image(self, target): base_packages = [re.sub("^chroot-", "", i) for i in self.build_base_packages() if i != "base-files"] with self.mirror: diff --git a/pyproject.toml b/pyproject.toml index dffffe5..c3541f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ dependencies = [ "strictyaml", "appdirs", "toml", + "portalocker", ] description = """\ A CLI tool to convert Python packages into a wide range of Linux distributions \ diff --git a/tests/test_alpine.py b/tests/test_alpine.py index 73198d4..fdf351a 100644 --- a/tests/test_alpine.py +++ b/tests/test_alpine.py @@ -4,6 +4,7 @@ import shutil import re import platform +import json import toml import pytest @@ -238,6 +239,30 @@ def _write_trove(trove): def test_kitchen_sink(monkeypatch): + (shared.kitchen_sink / ".polycotylus/fedora/noarch").mkdir(exist_ok=True, parents=True) + (shared.kitchen_sink / ".polycotylus/fedora/noarch/python3-99-s1lly-name-packag3-x-y-z-1.2.3-1.fc38.noarch.rpm").touch() + (shared.kitchen_sink / ".polycotylus/artifacts.json").write_text(json.dumps([ + { + "distribution": "fedora", + "tag": "38", + "architecture": "noarch", + "variant": "main", + "path": ".polycotylus/fedora/noarch/python3-99-s1lly-name-packag3-x-y-z-1.2.3-1.fc38.noarch.rpm" + }, { + "distribution": "alpine", + "tag": "3.17", + "architecture": Alpine.preferred_architecture, + "variant": "main", + "path": ".polycotylus/alpine/3.17/x86_64/py3-99---s1lly---name---packag3--x--y--z-1.2.3-r1.apk" + }, { + "distribution": "fedora", + "tag": "39", + "architecture": "noarch", + "variant": "main", + "path": ".polycotylus/fedora/noarch/python3-99-s1lly-name-packag3-x-y-z-1.2.3-1.fc39.noarch.rpm" + }, + ])) + monkeypatch.setenv("SETUPTOOLS_SCM_PRETEND_VERSION", "1.2.3") all_apks = [] for _Alpine in (Alpine, Alpine317, AlpineEdge): @@ -253,6 +278,7 @@ def test_kitchen_sink(monkeypatch): assert "license:\ncustom" in container.output assert ("pyc" in apks) is (_Alpine is not Alpine317) all_apks.extend(apks.values()) + self.update_artifacts_json(apks) with tarfile.open(apks["doc"]) as tar: path = "usr/share/licenses/py3-99---s1lly---name---packag3--x--y--z/The license file" @@ -264,6 +290,64 @@ def test_kitchen_sink(monkeypatch): assert apk.exists() assert len(set(all_apks)) == 8 + assert json.loads((shared.kitchen_sink / ".polycotylus/artifacts.json").read_bytes()) == [ + { + "distribution": "alpine", + "tag": "3.17", + "architecture": "x86_64", + "variant": "doc", + "path": ".polycotylus/alpine/3.17/x86_64/py3-99---s1lly---name---packag3--x--y--z-doc-1.2.3-r1.apk" + }, { + "distribution": "alpine", + "tag": "3.17", + "architecture": "x86_64", + "variant": "main", + "path": ".polycotylus/alpine/3.17/x86_64/py3-99---s1lly---name---packag3--x--y--z-1.2.3-r1.apk" + }, { + "distribution": "alpine", + "tag": "3.18", + "architecture": "x86_64", + "variant": "doc", + "path": ".polycotylus/alpine/3.18/x86_64/py3-99---s1lly---name---packag3--x--y--z-doc-1.2.3-r1.apk" + }, { + "distribution": "alpine", + "tag": "3.18", + "architecture": "x86_64", + "variant": "main", + "path": ".polycotylus/alpine/3.18/x86_64/py3-99---s1lly---name---packag3--x--y--z-1.2.3-r1.apk" + }, { + "distribution": "alpine", + "tag": "3.18", + "architecture": "x86_64", + "variant": "pyc", + "path": ".polycotylus/alpine/3.18/x86_64/py3-99---s1lly---name---packag3--x--y--z-pyc-1.2.3-r1.apk" + }, { + "distribution": "alpine", + "tag": "edge", + "architecture": "x86_64", + "variant": "doc", + "path": ".polycotylus/alpine/edge/x86_64/py3-99---s1lly---name---packag3--x--y--z-doc-1.2.3-r1.apk" + }, { + "distribution": "alpine", + "tag": "edge", + "architecture": "x86_64", + "variant": "main", + "path": ".polycotylus/alpine/edge/x86_64/py3-99---s1lly---name---packag3--x--y--z-1.2.3-r1.apk" + }, { + "distribution": "alpine", + "tag": "edge", + "architecture": "x86_64", + "variant": "pyc", + "path": ".polycotylus/alpine/edge/x86_64/py3-99---s1lly---name---packag3--x--y--z-pyc-1.2.3-r1.apk" + }, { + "distribution": "fedora", + "tag": "38", + "architecture": "noarch", + "variant": "main", + "path": ".polycotylus/fedora/noarch/python3-99-s1lly-name-packag3-x-y-z-1.2.3-1.fc38.noarch.rpm" + } + ] + test_multiarch = shared.qemu(Alpine) diff --git a/tests/test_fedora.py b/tests/test_fedora.py index 81ca535..26869f3 100644 --- a/tests/test_fedora.py +++ b/tests/test_fedora.py @@ -5,6 +5,7 @@ import platform import tarfile import io +import contextlib import toml import pytest @@ -93,22 +94,46 @@ def test_png_source_icon(tmp_path, polycotylus_yaml): def test_kitchen_sink(monkeypatch): + with contextlib.suppress(FileNotFoundError): + (shared.kitchen_sink / ".polycotylus/artifacts.json").unlink() monkeypatch.setenv("SETUPTOOLS_SCM_PRETEND_VERSION", "1.2.3") self = Fedora(Project.from_root(shared.kitchen_sink)) self.generate() assert "certifi" not in self.spec() assert "setuptools" not in self.spec() assert "colorama" not in self.spec() - rpm = self.build()["main"] + rpms = self.build() + rpm = rpms["main"] self.test(rpm) + self.update_artifacts_json(rpms) self = Fedora37(Project.from_root(shared.kitchen_sink)) - rpm37 = self.build()["main"] + rpms37 = self.build() + rpm37 = rpms37["main"] self.test(rpm37) + self.update_artifacts_json(rpms37) assert rpm.exists() assert rpm != rpm37 + assert (shared.kitchen_sink / ".polycotylus/artifacts.json").read_bytes() == b"""\ +[ + { + "distribution": "fedora", + "tag": "37", + "architecture": "noarch", + "variant": "main", + "path": ".polycotylus/fedora/noarch/python3-99-s1lly-name-packag3-x-y-z-1.2.3-1.fc37.noarch.rpm" + }, + { + "distribution": "fedora", + "tag": "38", + "architecture": "noarch", + "variant": "main", + "path": ".polycotylus/fedora/noarch/python3-99-s1lly-name-packag3-x-y-z-1.2.3-1.fc38.noarch.rpm" + } +]""" + def test_test_command(polycotylus_yaml): dependencies = "dependencies:\n test:\n pip: pytest\n" diff --git a/tests/test_manjaro.py b/tests/test_manjaro.py index b3494c2..dbe8f6d 100644 --- a/tests/test_manjaro.py +++ b/tests/test_manjaro.py @@ -15,7 +15,9 @@ class TestCommon(shared.Base): def test_dumb_text_viewer(): self = Manjaro(Project.from_root(shared.dumb_text_viewer)) self.generate() - self.test(self.build()["main"]) + packages = self.build() + self.test(packages["main"]) + self.update_artifacts_json(packages) def test_mirror_detection(monkeypatch): diff --git a/tests/test_opensuse.py b/tests/test_opensuse.py index 640fab8..ceb9b51 100644 --- a/tests/test_opensuse.py +++ b/tests/test_opensuse.py @@ -70,6 +70,7 @@ def test_ubrotli(): rpms = self.build() assert len(rpms) == 4 self.test(rpms["main"]) + self.update_artifacts_json(rpms) def test_kitchen_sink(monkeypatch): diff --git a/tests/test_void.py b/tests/test_void.py index f51d4be..51e2320 100644 --- a/tests/test_void.py +++ b/tests/test_void.py @@ -63,7 +63,9 @@ def test_kitchen_sink(monkeypatch): monkeypatch.setenv("SETUPTOOLS_SCM_PRETEND_VERSION", "1.2.3") self = VoidMusl(Project.from_root(shared.kitchen_sink)) self.generate() - self.test(self.build()["main"]) + packages = self.build() + self.test(packages["main"]) + self.update_artifacts_json(packages) def test_poetry():