From 7c7b2f75f18034d74baaa6bbf46c6bc50e5ad489 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Nosek?= Date: Thu, 11 Apr 2024 04:48:42 +0200 Subject: [PATCH] rpm: Implement inject-files support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `inject-files` do extra steps for rpm package manager: * Repository metadata is created for every repoid directory. Metadata is created by executing the `createrepo_c` command on the particular directory. * Generate 'cachi2.repo' files for all arches and save them into their own subdirectory 'repos.d'. Signed-off-by: Ondřej Nosek --- cachi2/core/package_managers/rpm/__init__.py | 4 +- cachi2/core/package_managers/rpm/main.py | 77 ++++++++++++++++++++ cachi2/core/resolver.py | 10 ++- cachi2/interface/cli.py | 4 +- pyproject.toml | 1 + tests/unit/package_managers/test_rpm.py | 56 +++++++++++++- 6 files changed, 147 insertions(+), 5 deletions(-) diff --git a/cachi2/core/package_managers/rpm/__init__.py b/cachi2/core/package_managers/rpm/__init__.py index e504e43c7..ec5026cce 100644 --- a/cachi2/core/package_managers/rpm/__init__.py +++ b/cachi2/core/package_managers/rpm/__init__.py @@ -1,3 +1,3 @@ -from cachi2.core.package_managers.rpm.main import fetch_rpm_source +from cachi2.core.package_managers.rpm.main import fetch_rpm_source, inject_files_post -__all__ = ["fetch_rpm_source"] +__all__ = ["fetch_rpm_source", "inject_files_post"] diff --git a/cachi2/core/package_managers/rpm/main.py b/cachi2/core/package_managers/rpm/main.py index e80dccf1c..e214ef702 100644 --- a/cachi2/core/package_managers/rpm/main.py +++ b/cachi2/core/package_managers/rpm/main.py @@ -1,6 +1,7 @@ import asyncio import hashlib import logging +import shlex from os import PathLike from pathlib import Path from typing import Any, Union @@ -223,3 +224,79 @@ def _generate_sbom_components(files_metadata: dict[Path, Any]) -> list[Component ) ) return components + + +def inject_files_post(*args: Any, **kwargs: Any) -> None: + """Run extra tasks for the RPM package manager (callback method) within `inject-files` cmd.""" + if "from_output_dir" in kwargs and "for_output_dir" in kwargs: + from_output_dir = kwargs["from_output_dir"] + for_output_dir = kwargs["for_output_dir"] + + if Path.exists(from_output_dir.joinpath(DEFAULT_PACKAGE_DIR)): + _generate_repos(from_output_dir) + _generate_repofiles(from_output_dir, for_output_dir) + + +def _generate_repos(from_output_dir: Path) -> None: + """Search structure for all repoid dirs and create repository metadata \ + out of its RPMs (and SRPMs).""" + package_dir = from_output_dir.joinpath(DEFAULT_PACKAGE_DIR) + for arch in package_dir.iterdir(): + if not arch.is_dir(): + continue + for entry in arch.iterdir(): + if not entry.is_dir() or entry.name == "repos.d": + continue + repoid = entry.name + _createrepo(repoid, entry) + + +def _createrepo(reponame: str, repodir: Path) -> None: + """Execute the createrepo utility.""" + log.info(f"Creating repository metadata for repoid '{reponame}': {repodir}") + cmd = ["createrepo_c", str(repodir)] + log.debug("$ " + shlex.join(cmd)) + stdout = run_cmd(cmd, params={}) + log.debug(stdout) + + +def _generate_repofiles(from_output_dir: Path, for_output_dir: Path) -> None: + """ + Generate templates of repofiles for all arches. + + Search structure at 'path' and generate one repofile content for each arch. + Each repofile contains all arch's repoids (including repoids with source RPMs). + Repofile (cachi2.repo) for particular arch will be stored in arch's dir in 'repos.d' subdir. + Repofiles are not directly created from the templates in this method - templates are stored + in the project file. + """ + package_dir = from_output_dir.joinpath(DEFAULT_PACKAGE_DIR) + for arch in package_dir.iterdir(): + if not arch.is_dir(): + continue + log.debug(f"Preparing repofile content for arch '{arch.name}'") + content = "" + for entry in arch.iterdir(): + if not entry.is_dir() or entry.name == "repos.d": + continue + repoid = entry.name + localpath = for_output_dir.joinpath(DEFAULT_PACKAGE_DIR, arch.name, repoid) + content += f"[{repoid}]\n" + content += f"baseurl=file://{localpath}\n" + content += "gpgcheck=1\n" + # repoid directory matches the internal repoid + if repoid.startswith("cachi2-"): + content += ( + "name=Generated repository containing all packages unaffiliated " + "with any official repository\n" + ) + if content: + repo_file_path = arch.joinpath("repos.d", "cachi2.repo") + if repo_file_path.exists(): + log.warning(f"Overwriting {repo_file_path}") + else: + Path.mkdir(arch.joinpath("repos.d"), parents=True, exist_ok=True) + log.info(f"Creating {repo_file_path}") + + with open(repo_file_path, "w") as repo_file: + repo_file.write(content) diff --git a/cachi2/core/resolver.py b/cachi2/core/resolver.py index eca90178c..49c35bc54 100644 --- a/cachi2/core/resolver.py +++ b/cachi2/core/resolver.py @@ -1,7 +1,7 @@ from collections.abc import Iterable from pathlib import Path from tempfile import TemporaryDirectory -from typing import Callable +from typing import Any, Callable from cachi2.core.errors import UnsupportedFeature from cachi2.core.models.input import PackageManagerType, Request @@ -84,3 +84,11 @@ def _merge_outputs(outputs: Iterable[RequestOutput]) -> RequestOutput: environment_variables=env_vars, project_files=project_files, ) + + +def inject_files_post(*args: Any, **kwargs: Any) -> None: + """Do extra steps for package manager.""" + # if there is a callback method defined within the particular package manager, run it + if hasattr(rpm, "inject_files_post"): + callback_method = getattr(rpm, "inject_files_post") + callback_method(*args, **kwargs) diff --git a/cachi2/interface/cli.py b/cachi2/interface/cli.py index 62717b4e9..5f4eb0954 100644 --- a/cachi2/interface/cli.py +++ b/cachi2/interface/cli.py @@ -15,7 +15,7 @@ from cachi2.core.extras.envfile import EnvFormat, generate_envfile from cachi2.core.models.input import Flag, PackageInput, Request, parse_user_input from cachi2.core.models.output import BuildConfig -from cachi2.core.resolver import resolve_packages, supported_package_managers +from cachi2.core.resolver import inject_files_post, resolve_packages, supported_package_managers from cachi2.core.rooted_path import RootedPath from cachi2.interface.logging import LogLevel, setup_logging @@ -339,6 +339,8 @@ def inject_files( content = project_file.resolve_content(output_dir=for_output_dir) project_file.abspath.write_text(content) + inject_files_post(from_output_dir=from_output_dir, for_output_dir=for_output_dir) + def _get_build_config(output_dir: Path) -> BuildConfig: build_config_json = RootedPath(output_dir).join_within_root(".build-config.json").path diff --git a/pyproject.toml b/pyproject.toml index ed131212f..88e3faa23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ dependencies = [ "aiohttp-retry", "backoff", "beautifulsoup4", + "createrepo_c", "gitpython", "packageurl-python", "packaging", diff --git a/tests/unit/package_managers/test_rpm.py b/tests/unit/package_managers/test_rpm.py index eb3bd7d25..c9d2a18bc 100644 --- a/tests/unit/package_managers/test_rpm.py +++ b/tests/unit/package_managers/test_rpm.py @@ -7,10 +7,14 @@ from cachi2.core.errors import PackageManagerError, PackageRejected from cachi2.core.models.sbom import Component -from cachi2.core.package_managers.rpm import fetch_rpm_source +from cachi2.core.package_managers.rpm import fetch_rpm_source, inject_files_post from cachi2.core.package_managers.rpm.main import ( DEFAULT_LOCKFILE_NAME, + DEFAULT_PACKAGE_DIR, + _createrepo, _download, + _generate_repofiles, + _generate_repos, _generate_sbom_components, _resolve_rpm_project, _verify_downloaded, @@ -291,6 +295,42 @@ def test_resolve_rpm_project( mock_generate_sbom_components.assert_called_once_with({}) +@mock.patch("cachi2.core.package_managers.rpm.main.run_cmd") +def test_createrepo(mock_run_cmd: mock.Mock, rooted_tmp_path: RootedPath) -> None: + repodir = rooted_tmp_path + repoid = "repo1" + _createrepo(repoid, repodir.path) + mock_run_cmd.assert_called_once_with(["createrepo_c", str(repodir)], params={}) + + +@mock.patch("cachi2.core.package_managers.rpm.main._createrepo") +def test_generate_repos(mock_createrepo: mock.Mock, rooted_tmp_path: RootedPath) -> None: + package_dir = rooted_tmp_path.join_within_root(DEFAULT_PACKAGE_DIR) + arch_dir = package_dir.path.joinpath("x86_64") + arch_dir.joinpath("repo1").mkdir(parents=True) + arch_dir.joinpath("repos.d").mkdir(parents=True) + _generate_repos(rooted_tmp_path.path) + mock_createrepo.assert_called_once_with("repo1", arch_dir.joinpath("repo1")) + + +def test_generate_repofiles(rooted_tmp_path: RootedPath) -> None: + package_dir = rooted_tmp_path.join_within_root(DEFAULT_PACKAGE_DIR) + arch_dir = package_dir.path.joinpath("x86_64") + arch_dir.joinpath("repo1").mkdir(parents=True) + arch_dir.joinpath("cachi2-repo2").mkdir(parents=True) + arch_dir.joinpath("repos.d").mkdir(parents=True) + repopath = arch_dir.joinpath("repos.d", "cachi2.repo") + output_dir = f"{package_dir}/x86_64" + name = ( + "name=Generated repository containing all packages unaffiliated " + "with any official repository" + ) + template = f"[repo1]\nbaseurl=file://{output_dir}/repo1\ngpgcheck=1\n[cachi2-repo2]\nbaseurl=file://{output_dir}/cachi2-repo2\ngpgcheck=1\n{name}\n" + _generate_repofiles(rooted_tmp_path.path, rooted_tmp_path.path) + with open(repopath) as f: + assert template == f.read() + + @mock.patch("cachi2.core.package_managers.rpm.main.run_cmd") def test_generate_sbom_components(mock_run_cmd: mock.Mock) -> None: vender = "redhat" @@ -321,6 +361,20 @@ def test_generate_sbom_components(mock_run_cmd: mock.Mock) -> None: ] +@mock.patch("cachi2.core.package_managers.rpm.main.Path") +@mock.patch("cachi2.core.package_managers.rpm.main._generate_repofiles") +@mock.patch("cachi2.core.package_managers.rpm.main._generate_repos") +def test_inject_files_post( + mock_generate_repos: mock.Mock, + mock_generate_repofiles: mock.Mock, + mock_path: mock.Mock, + rooted_tmp_path: RootedPath, +) -> None: + inject_files_post(from_output_dir=rooted_tmp_path.path, for_output_dir=rooted_tmp_path.path) + mock_generate_repos.assert_called_once_with(rooted_tmp_path.path) + mock_generate_repofiles.assert_called_with(rooted_tmp_path.path, rooted_tmp_path.path) + + @mock.patch("cachi2.core.package_managers.rpm.main.asyncio.run") @mock.patch("cachi2.core.package_managers.rpm.main.async_download_files") def test_download(