From 5a89096ebe3f9442359a201844a37d9f49cdffda Mon Sep 17 00:00:00 2001 From: Alexey Ovchinnikov Date: Wed, 11 Dec 2024 20:26:39 -0600 Subject: [PATCH] yarn_v1: connecting metadispatcher to the code Now it is the sole entrypoint for both Yarn v1 and v2. This change graduates Yarn Classic from dev to fully supported package manager. Signed-off-by: Alexey Ovchinnikov --- cachi2/core/config.py | 2 ++ cachi2/core/package_managers/metayarn.py | 12 +++++--- .../package_managers/yarn_classic/main.py | 2 +- cachi2/core/resolver.py | 29 +++---------------- cachi2/core/utils.py | 22 +++++++++++++- tests/integration/test_yarn_classic.py | 20 ++++++------- .../yarn_classic/test_main.py | 4 +-- 7 files changed, 47 insertions(+), 44 deletions(-) diff --git a/cachi2/core/config.py b/cachi2/core/config.py index b6f9cfeb5..a01720b68 100644 --- a/cachi2/core/config.py +++ b/cachi2/core/config.py @@ -25,6 +25,8 @@ class Config(BaseModel, extra="forbid"): requests_timeout: int = 300 concurrency_limit: int = 5 + allow_yarnberry_processing: bool = True + @model_validator(mode="before") @classmethod def _print_deprecation_warning(cls, data: Any) -> Any: diff --git a/cachi2/core/package_managers/metayarn.py b/cachi2/core/package_managers/metayarn.py index 8ceb0ef61..a65922610 100644 --- a/cachi2/core/package_managers/metayarn.py +++ b/cachi2/core/package_managers/metayarn.py @@ -1,8 +1,9 @@ +from cachi2.core.config import get_config from cachi2.core.models.input import Request from cachi2.core.models.output import RequestOutput from cachi2.core.package_managers import yarn, yarn_classic from cachi2.core.package_managers.yarn_classic.main import MissingLockfile, NotV1Lockfile -from cachi2.core.resolver import _merge_outputs +from cachi2.core.utils import merge_outputs def fetch_yarn_source(request: Request) -> RequestOutput: @@ -15,7 +16,10 @@ def fetch_yarn_source(request: Request) -> RequestOutput: new_request = request.model_copy(update={"packages": [package]}) try: fetched_packages.append(yarn_classic.fetch_yarn_source(new_request)) - except (MissingLockfile, NotV1Lockfile): + except (MissingLockfile, NotV1Lockfile) as e: # It is assumed that if a package is not v1 then it is probably v2. - fetched_packages.append(yarn.fetch_yarn_source(new_request)) - return _merge_outputs(fetched_packages) + if get_config().allow_yarnberry_processing: + fetched_packages.append(yarn.fetch_yarn_source(new_request)) + else: + raise e + return merge_outputs(fetched_packages) diff --git a/cachi2/core/package_managers/yarn_classic/main.py b/cachi2/core/package_managers/yarn_classic/main.py index aa4481de4..d73d48c22 100644 --- a/cachi2/core/package_managers/yarn_classic/main.py +++ b/cachi2/core/package_managers/yarn_classic/main.py @@ -56,7 +56,7 @@ def fetch_yarn_source(request: Request) -> RequestOutput: def _ensure_mirror_dir_exists(output_dir: RootedPath) -> None: output_dir.join_within_root(MIRROR_DIR).path.mkdir(parents=True, exist_ok=True) - for package in request.yarn_classic_packages: + for package in request.yarn_packages: package_path = request.source_dir.join_within_root(package.path) _ensure_mirror_dir_exists(request.output_dir) _resolve_yarn_project(Project.from_source_dir(package_path), request.output_dir) diff --git a/cachi2/core/resolver.py b/cachi2/core/resolver.py index 435def23e..d28210f74 100644 --- a/cachi2/core/resolver.py +++ b/cachi2/core/resolver.py @@ -1,4 +1,3 @@ -from collections.abc import Iterable from pathlib import Path from tempfile import TemporaryDirectory from typing import Any, Callable @@ -6,9 +5,9 @@ from cachi2.core.errors import UnsupportedFeature from cachi2.core.models.input import PackageManagerType, Request from cachi2.core.models.output import RequestOutput -from cachi2.core.package_managers import bundler, generic, gomod, npm, pip, rpm, yarn, yarn_classic +from cachi2.core.package_managers import bundler, generic, gomod, metayarn, npm, pip, rpm from cachi2.core.rooted_path import RootedPath -from cachi2.core.utils import copy_directory +from cachi2.core.utils import copy_directory, merge_outputs Handler = Callable[[Request], RequestOutput] @@ -17,7 +16,7 @@ "gomod": gomod.fetch_gomod_source, "npm": npm.fetch_npm_source, "pip": pip.fetch_pip_source, - "yarn": yarn.fetch_yarn_source, + "yarn": metayarn.fetch_yarn_source, "generic": generic.fetch_generic_source, } @@ -25,7 +24,6 @@ # invoke them via CLI _dev_package_managers: dict[PackageManagerType, Handler] = { "rpm": rpm.fetch_rpm_source, - "yarn-classic": yarn_classic.fetch_yarn_source, } # This is *only* used to provide a list for `cachi2 --version` @@ -68,26 +66,7 @@ def _resolve_packages(request: Request) -> RequestOutput: solution="But the good news is that we're already working on it!", ) pkg_managers = [_supported_package_managers[type_] for type_ in sorted(requested_types)] - return _merge_outputs(pkg_manager(request) for pkg_manager in pkg_managers) - - -def _merge_outputs(outputs: Iterable[RequestOutput]) -> RequestOutput: - """Merge RequestOutput instances.""" - components = [] - env_vars = [] - project_files = [] - - for output in outputs: - components.extend(output.components) - env_vars.extend(output.build_config.environment_variables) - project_files.extend(output.build_config.project_files) - - return RequestOutput.from_obj_list( - components=components, - environment_variables=env_vars, - project_files=project_files, - options=output.build_config.options if output.build_config.options else None, - ) + return merge_outputs(pkg_manager(request) for pkg_manager in pkg_managers) def inject_files_post(from_output_dir: Path, for_output_dir: Path, **kwargs: Any) -> None: diff --git a/cachi2/core/utils.py b/cachi2/core/utils.py index bdf121eee..64e36eed4 100644 --- a/cachi2/core/utils.py +++ b/cachi2/core/utils.py @@ -8,10 +8,11 @@ import sys from functools import cache from pathlib import Path -from typing import Callable, Iterator, Optional, Sequence +from typing import Callable, Iterable, Iterator, Optional, Sequence from cachi2.core.config import get_config from cachi2.core.errors import Cachi2Error +from cachi2.core.models.output import RequestOutput log = logging.getLogger(__name__) @@ -195,3 +196,22 @@ def get_cache_dir() -> Path: except KeyError: cache_dir = Path.home().joinpath(".cache") return cache_dir.joinpath("cachi2") + + +def merge_outputs(outputs: Iterable[RequestOutput]) -> RequestOutput: + """Merge RequestOutput instances.""" + components = [] + env_vars = [] + project_files = [] + + for output in outputs: + components.extend(output.components) + env_vars.extend(output.build_config.environment_variables) + project_files.extend(output.build_config.project_files) + + return RequestOutput.from_obj_list( + components=components, + environment_variables=env_vars, + project_files=project_files, + options=output.build_config.options if output.build_config.options else None, + ) diff --git a/tests/integration/test_yarn_classic.py b/tests/integration/test_yarn_classic.py index 745da5e56..186f2a658 100644 --- a/tests/integration/test_yarn_classic.py +++ b/tests/integration/test_yarn_classic.py @@ -15,7 +15,7 @@ utils.TestParameters( repo="https://github.com/cachito-testing/cachi2-yarn.git", ref="corepack_packagemanager_ignored", - packages=({"path": ".", "type": "yarn-classic"},), + packages=({"path": ".", "type": "yarn"},), flags=["--dev-package-managers"], check_output=False, check_deps_checksums=False, @@ -29,7 +29,7 @@ utils.TestParameters( repo="https://github.com/cachito-testing/cachi2-yarn.git", ref="yarnpath_ignored", - packages=({"path": ".", "type": "yarn-classic"},), + packages=({"path": ".", "type": "yarn"},), flags=["--dev-package-managers"], check_output=False, check_deps_checksums=False, @@ -43,7 +43,7 @@ utils.TestParameters( repo="https://github.com/cachito-testing/cachi2-yarn.git", ref="invalid_checksum", - packages=({"path": ".", "type": "yarn-classic"},), + packages=({"path": ".", "type": "yarn"},), flags=["--dev-package-managers"], check_output=False, check_deps_checksums=False, @@ -57,7 +57,7 @@ utils.TestParameters( repo="https://github.com/cachito-testing/cachi2-yarn.git", ref="invalid_frozen_lockfile_add_dependency", - packages=({"path": ".", "type": "yarn-classic"},), + packages=({"path": ".", "type": "yarn"},), flags=["--dev-package-managers"], check_output=False, check_deps_checksums=False, @@ -71,7 +71,7 @@ utils.TestParameters( repo="https://github.com/cachito-testing/cachi2-yarn.git", ref="lifecycle_scripts", - packages=({"path": ".", "type": "yarn-classic"},), + packages=({"path": ".", "type": "yarn"},), flags=["--dev-package-managers"], check_output=False, check_deps_checksums=False, @@ -85,7 +85,7 @@ utils.TestParameters( repo="https://github.com/cachito-testing/cachi2-yarn.git", ref="offline-mirror-collision", - packages=({"path": ".", "type": "yarn-classic"},), + packages=({"path": ".", "type": "yarn"},), flags=["--dev-package-managers"], check_output=False, check_deps_checksums=False, @@ -128,8 +128,7 @@ def test_yarn_classic_packages( utils.TestParameters( repo="https://github.com/cachito-testing/cachi2-yarn.git", ref="valid_yarn_all_dependency_types", - packages=({"path": ".", "type": "yarn-classic"},), - flags=["--dev-package-managers"], + packages=({"path": ".", "type": "yarn"},), check_vendor_checksums=False, expected_exit_code=0, expected_output="All dependencies fetched successfully", @@ -143,10 +142,9 @@ def test_yarn_classic_packages( repo="https://github.com/cachito-testing/cachi2-yarn.git", ref="valid_multiple_packages", packages=( - {"path": "first-pkg", "type": "yarn-classic"}, - {"path": "second-pkg", "type": "yarn-classic"}, + {"path": "first-pkg", "type": "yarn"}, + {"path": "second-pkg", "type": "yarn"}, ), - flags=["--dev-package-managers"], check_vendor_checksums=False, expected_exit_code=0, expected_output="All dependencies fetched successfully", diff --git a/tests/unit/package_managers/yarn_classic/test_main.py b/tests/unit/package_managers/yarn_classic/test_main.py index 39e10d311..edf54fdbb 100644 --- a/tests/unit/package_managers/yarn_classic/test_main.py +++ b/tests/unit/package_managers/yarn_classic/test_main.py @@ -56,12 +56,12 @@ def test_generate_build_environment_variables( "input_request, components", [ pytest.param( - [{"type": "yarn-classic", "path": "."}], + [{"type": "yarn", "path": "."}], [], id="single_input_package", ), pytest.param( - [{"type": "yarn-classic", "path": "."}, {"type": "yarn-classic", "path": "./path"}], + [{"type": "yarn", "path": "."}, {"type": "yarn", "path": "./path"}], [], id="multiple_input_packages", ),