rpm: Implement inject-files support

`inject-files` do extra steps for rpm package manager: * Repository metadata is created for every repoid directory. Metadata is created by executing the `createrepo_c` command on the particular directory. * Generate 'cachi2.repo' files for all arches and save them into their own subdirectory 'repos.d'. Signed-off-by: Ondřej Nosek <[email protected]>
containerbuildsystem · Apr 12, 2024 · 6597967 · 6597967
1 parent fd6ce2d
commit 6597967
Show file tree

Hide file tree

Showing 6 changed files with 148 additions and 5 deletions.
diff --git a/cachi2/core/package_managers/rpm/__init__.py b/cachi2/core/package_managers/rpm/__init__.py
@@ -1,3 +1,3 @@
-from cachi2.core.package_managers.rpm.main import fetch_rpm_source
+from cachi2.core.package_managers.rpm.main import fetch_rpm_source, inject_files_post
 
-__all__ = ["fetch_rpm_source"]
+__all__ = ["fetch_rpm_source", "inject_files_post"]
diff --git a/cachi2/core/package_managers/rpm/main.py b/cachi2/core/package_managers/rpm/main.py
@@ -1,6 +1,7 @@
 import asyncio
 import hashlib
 import logging
+import shlex
 from os import PathLike
 from pathlib import Path
 from typing import Any, Union
@@ -221,3 +222,79 @@ def _generate_sbom_components(files_metadata: dict[Path, Any]) -> list[Component
             )
         )
     return components
+
+
+def inject_files_post(*args: Any, **kwargs: Any) -> None:
+    """Run extra tasks for the RPM package manager (callback method) within `inject-files` cmd."""
+    if "from_output_dir" in kwargs and "for_output_dir" in kwargs:
+        from_output_dir = kwargs["from_output_dir"]
+        for_output_dir = kwargs["for_output_dir"]
+
+        if Path.exists(from_output_dir.joinpath(DEFAULT_PACKAGE_DIR)):
+            _generate_repos(from_output_dir)
+            _generate_repofiles(from_output_dir, for_output_dir)
+
+
+def _generate_repos(from_output_dir: Path) -> None:
+    """Search structure for all repoid dirs and create repository metadata \
+    out of its RPMs (and SRPMs)."""
+    package_dir = from_output_dir.joinpath(DEFAULT_PACKAGE_DIR)
+    for arch in package_dir.iterdir():
+        if not arch.is_dir():
+            continue
+        for entry in arch.iterdir():
+            if not entry.is_dir() or entry.name == "repos.d":
+                continue
+            repoid = entry.name
+            _createrepo(repoid, entry)
+
+
+def _createrepo(reponame: str, repodir: Path) -> None:
+    """Execute the createrepo utility."""
+    log.info(f"Creating repository metadata for repoid '{reponame}': {repodir}")
+    cmd = ["createrepo_c", str(repodir)]
+    log.debug("$ " + shlex.join(cmd))
+    stdout = run_cmd(cmd, params={})
+    log.debug(stdout)
+
+
+def _generate_repofiles(from_output_dir: Path, for_output_dir: Path) -> None:
+    """
+    Generate templates of repofiles for all arches.
+
+    Search structure at 'path' and generate one repofile content for each arch.
+    Each repofile contains all arch's repoids (including repoids with source RPMs).
+    Repofile (cachi2.repo) for particular arch will be stored in arch's dir in 'repos.d' subdir.
+    Repofiles are not directly created from the templates in this method - templates are stored
+    in the project file.
+    """
+    package_dir = from_output_dir.joinpath(DEFAULT_PACKAGE_DIR)
+    for arch in package_dir.iterdir():
+        if not arch.is_dir():
+            continue
+        log.debug(f"Preparing repofile content for arch '{arch.name}'")
+        content = ""
+        for entry in sorted(arch.iterdir()):
+            if not entry.is_dir() or entry.name == "repos.d":
+                continue
+            repoid = entry.name
+            localpath = for_output_dir.joinpath(DEFAULT_PACKAGE_DIR, arch.name, repoid)
+            content += f"[{repoid}]\n"
+            content += f"baseurl=file://{localpath}\n"
+            content += "gpgcheck=1\n"
+            # repoid directory matches the internal repoid
+            if repoid.startswith("cachi2-"):
+                content += (
+                    "name=Generated repository containing all packages unaffiliated "
+                    "with any official repository\n"
+                )
+        if content:
+            repo_file_path = arch.joinpath("repos.d", "cachi2.repo")
+            if repo_file_path.exists():
+                log.warning(f"Overwriting {repo_file_path}")
+            else:
+                Path.mkdir(arch.joinpath("repos.d"), parents=True, exist_ok=True)
+                log.info(f"Creating {repo_file_path}")
+
+            with open(repo_file_path, "w") as repo_file:
+                repo_file.write(content)
diff --git a/cachi2/core/resolver.py b/cachi2/core/resolver.py
@@ -1,7 +1,7 @@
 from collections.abc import Iterable
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Callable
+from typing import Any, Callable
 
 from cachi2.core.errors import UnsupportedFeature
 from cachi2.core.models.input import PackageManagerType, Request
@@ -84,3 +84,11 @@ def _merge_outputs(outputs: Iterable[RequestOutput]) -> RequestOutput:
         environment_variables=env_vars,
         project_files=project_files,
     )
+
+
+def inject_files_post(*args: Any, **kwargs: Any) -> None:
+    """Do extra steps for package manager."""
+    # if there is a callback method defined within the particular package manager, run it
+    if hasattr(rpm, "inject_files_post"):
+        callback_method = getattr(rpm, "inject_files_post")
+        callback_method(*args, **kwargs)
diff --git a/cachi2/interface/cli.py b/cachi2/interface/cli.py
@@ -15,7 +15,7 @@
 from cachi2.core.extras.envfile import EnvFormat, generate_envfile
 from cachi2.core.models.input import Flag, PackageInput, Request, parse_user_input
 from cachi2.core.models.output import BuildConfig
-from cachi2.core.resolver import resolve_packages, supported_package_managers
+from cachi2.core.resolver import inject_files_post, resolve_packages, supported_package_managers
 from cachi2.core.rooted_path import RootedPath
 from cachi2.interface.logging import LogLevel, setup_logging
 
@@ -339,6 +339,8 @@ def inject_files(
         content = project_file.resolve_content(output_dir=for_output_dir)
         project_file.abspath.write_text(content)
 
+    inject_files_post(from_output_dir=from_output_dir, for_output_dir=for_output_dir)
+
 
 def _get_build_config(output_dir: Path) -> BuildConfig:
     build_config_json = RootedPath(output_dir).join_within_root(".build-config.json").path

diff --git a/pyproject.toml b/pyproject.toml
@@ -24,6 +24,7 @@ dependencies = [
   "aiohttp-retry",
   "backoff",
   "beautifulsoup4",
+  "createrepo_c",
   "gitpython",
   "packageurl-python",
   "packaging",

diff --git a/tests/unit/package_managers/test_rpm.py b/tests/unit/package_managers/test_rpm.py
@@ -7,10 +7,14 @@
 
 from cachi2.core.errors import PackageManagerError, PackageRejected
 from cachi2.core.models.sbom import Component, Property
-from cachi2.core.package_managers.rpm import fetch_rpm_source
+from cachi2.core.package_managers.rpm import fetch_rpm_source, inject_files_post
 from cachi2.core.package_managers.rpm.main import (
     DEFAULT_LOCKFILE_NAME,
+    DEFAULT_PACKAGE_DIR,
+    _createrepo,
     _download,
+    _generate_repofiles,
+    _generate_repos,
     _generate_sbom_components,
     _resolve_rpm_project,
     _verify_downloaded,
@@ -285,6 +289,43 @@ def test_resolve_rpm_project(
     mock_generate_sbom_components.assert_called_once_with({})
 
 
+@mock.patch("cachi2.core.package_managers.rpm.main.run_cmd")
+def test_createrepo(mock_run_cmd: mock.Mock, rooted_tmp_path: RootedPath) -> None:
+    repodir = rooted_tmp_path
+    repoid = "repo1"
+    _createrepo(repoid, repodir.path)
+    mock_run_cmd.assert_called_once_with(["createrepo_c", str(repodir)], params={})
+
+
+@mock.patch("cachi2.core.package_managers.rpm.main._createrepo")
+def test_generate_repos(mock_createrepo: mock.Mock, rooted_tmp_path: RootedPath) -> None:
+    package_dir = rooted_tmp_path.join_within_root(DEFAULT_PACKAGE_DIR)
+    arch_dir = package_dir.path.joinpath("x86_64")
+    arch_dir.joinpath("repo1").mkdir(parents=True)
+    arch_dir.joinpath("repos.d").mkdir(parents=True)
+    _generate_repos(rooted_tmp_path.path)
+    mock_createrepo.assert_called_once_with("repo1", arch_dir.joinpath("repo1"))
+
+
+def test_generate_repofiles(rooted_tmp_path: RootedPath) -> None:
+    package_dir = rooted_tmp_path.join_within_root(DEFAULT_PACKAGE_DIR)
+    arch_dir = package_dir.path.joinpath("x86_64")
+    arch_dir.joinpath("repo1").mkdir(parents=True)
+    arch_dir.joinpath("cachi2-repo2").mkdir(parents=True)
+    arch_dir.joinpath("repos.d").mkdir(parents=True)
+    repopath = arch_dir.joinpath("repos.d", "cachi2.repo")
+    output_dir = f"{package_dir}/x86_64"
+    name = (
+        "name=Generated repository containing all packages unaffiliated "
+        "with any official repository"
+    )
+    # repo items need to be sorted to match with the repofile
+    template = f"[cachi2-repo2]\nbaseurl=file://{output_dir}/cachi2-repo2\ngpgcheck=1\n{name}\n[repo1]\nbaseurl=file://{output_dir}/repo1\ngpgcheck=1\n"
+    _generate_repofiles(rooted_tmp_path.path, rooted_tmp_path.path)
+    with open(repopath) as f:
+        assert template == f.read()
+
+
 @mock.patch("cachi2.core.package_managers.rpm.main.run_cmd")
 def test_generate_sbom_components(mock_run_cmd: mock.Mock) -> None:
     name = "foo"
@@ -346,6 +387,20 @@ def test_generate_sbom_components_missing_checksum(mock_run_cmd: mock.Mock) -> N
     ]
 
 
+@mock.patch("cachi2.core.package_managers.rpm.main.Path")
+@mock.patch("cachi2.core.package_managers.rpm.main._generate_repofiles")
+@mock.patch("cachi2.core.package_managers.rpm.main._generate_repos")
+def test_inject_files_post(
+    mock_generate_repos: mock.Mock,
+    mock_generate_repofiles: mock.Mock,
+    mock_path: mock.Mock,
+    rooted_tmp_path: RootedPath,
+) -> None:
+    inject_files_post(from_output_dir=rooted_tmp_path.path, for_output_dir=rooted_tmp_path.path)
+    mock_generate_repos.assert_called_once_with(rooted_tmp_path.path)
+    mock_generate_repofiles.assert_called_with(rooted_tmp_path.path, rooted_tmp_path.path)
+
+
 @mock.patch("cachi2.core.package_managers.rpm.main.asyncio.run")
 @mock.patch("cachi2.core.package_managers.rpm.main.async_download_files")
 def test_download(