From 86e777ec4aa893c795dc42afc2779e1fd4fb49b2 Mon Sep 17 00:00:00 2001 From: Taylor Madore Date: Fri, 10 Jan 2025 07:56:47 -0500 Subject: [PATCH 1/4] exclude_none=True for pydantic model_dump in tests Signed-off-by: Taylor Madore --- tests/unit/package_managers/test_generic.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/unit/package_managers/test_generic.py b/tests/unit/package_managers/test_generic.py index a20b47e0b..87fb94a0d 100644 --- a/tests/unit/package_managers/test_generic.py +++ b/tests/unit/package_managers/test_generic.py @@ -255,7 +255,6 @@ def test_resolve_generic_lockfile_invalid( "properties": [{"name": "cachi2:found_by", "value": "cachi2"}], "purl": "pkg:generic/archive.zip?checksum=md5:3a18656e1cea70504b905836dee14db0&download_url=https://example.com/artifact", "type": "file", - "version": None, }, { "external_references": [ @@ -268,7 +267,6 @@ def test_resolve_generic_lockfile_invalid( "properties": [{"name": "cachi2:found_by", "value": "cachi2"}], "purl": "pkg:generic/file.tar.gz?checksum=md5:32112bed1914cfe3799600f962750b1d&download_url=https://example.com/more/complex/path/file.tar.gz%3Ffoo%3Dbar%23fragment", "type": "file", - "version": None, }, ], id="valid_lockfile", @@ -324,7 +322,8 @@ def test_resolve_generic_lockfile_valid( f.write(lockfile_content) assert [ - c.model_dump() for c in _resolve_generic_lockfile(lockfile_path.path, rooted_tmp_path) + c.model_dump(exclude_none=True) + for c in _resolve_generic_lockfile(lockfile_path.path, rooted_tmp_path) ] == expected_components mock_checksums.assert_called() From daf24d798d0763d1135eae4d12d831cdb8cb394d Mon Sep 17 00:00:00 2001 From: Taylor Madore Date: Fri, 10 Jan 2025 08:03:02 -0500 Subject: [PATCH 2/4] handle optional, builtin patches for yarn v4 In yarn v4, optional, builtin patches are now denoted with the prefix `optional!`. Handle this in addition to the yarn v3 prefix for the same Signed-off-by: Taylor Madore --- cachi2/core/package_managers/yarn/locators.py | 8 +++- .../package_managers/yarn/test_locators.py | 42 +++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/cachi2/core/package_managers/yarn/locators.py b/cachi2/core/package_managers/yarn/locators.py index 53a74ffd2..26ad6469a 100644 --- a/cachi2/core/package_managers/yarn/locators.py +++ b/cachi2/core/package_managers/yarn/locators.py @@ -205,10 +205,14 @@ def _parse_patch_locator(locator: "_ParsedLocator") -> PatchLocator: original_package = parse_locator(reference.source) - # https://github.com/yarnpkg/berry/blob/b6026842dfec4b012571b5982bb74420c7682a73/packages/plugin-patch/sources/patchUtils.ts#L92 def process_patch_path(patch: str) -> Union[str, Path]: - # '~' denotes an optional patch (failing to apply the patch is not fatal, only a warning) + # Yarn patches can be optional, where failing to apply the patch is not fatal, only a warning + # '~' denotes an optional patch in Yarn v3 + # https://github.com/yarnpkg/berry/blob/b6026842dfec4b012571b5982bb74420c7682a73/packages/plugin-patch/sources/patchUtils.ts#L92 patch = patch.removeprefix("~") + # `optional!' denotes an optional patch in Yarn v4 + # https://github.com/yarnpkg/berry/blob/93a56643ba3c813a87920dcf75c644eaf3b38e6f/packages/plugin-patch/sources/patchUtils.ts#L147 + patch = patch.removeprefix("optional!") if re.match(r"^builtin<([^>]+)>$", patch): return patch else: diff --git a/tests/unit/package_managers/yarn/test_locators.py b/tests/unit/package_managers/yarn/test_locators.py index 4d2cc68f5..d1103936e 100644 --- a/tests/unit/package_managers/yarn/test_locators.py +++ b/tests/unit/package_managers/yarn/test_locators.py @@ -42,9 +42,11 @@ # optional custom patch for a registry dep "left-pad@npm:1.3.0", "left-pad@patch:left-pad@npm%3A1.3.0#~./my-patches/left-pad.patch::version=1.3.0&hash=629bda&locator=berryscary%40workspace%3A.", + "left-pad@patch:left-pad@npm%3A1.3.0#optional!./my-patches/left-pad.patch::version=1.3.0&hash=629bda&locator=berryscary%40workspace%3A.", # optional builtin patch "fsevents@npm:2.3.2", "fsevents@patch:fsevents@npm%3A2.3.2#~builtin::version=2.3.2&hash=df0bf1", + "fsevents@patch:fsevents@npm%3A2.3.2#optional!builtin::version=2.3.2&hash=df0bf1", # patched patch dependency "fsevents@patch:fsevents@patch%3Afsevents@npm%253A2.3.2%23./my-patches/fsevents.patch%3A%3Aversion=2.3.2&hash=cf0bf0&locator=berryscary%2540workspace%253A.#~builtin::version=2.3.2&hash=df0bf1", # non-optional builtin patch (in reality, the typescript patch is optional) @@ -201,6 +203,23 @@ }, ), ), + ( + _ParsedLocator( + scope=None, + name="left-pad", + raw_reference="patch:left-pad@npm%3A1.3.0#optional!./my-patches/left-pad.patch::version=1.3.0&hash=629bda&locator=berryscary%40workspace%3A.", + ), + _ParsedReference( + protocol="patch:", + source="left-pad@npm:1.3.0", + selector="optional!./my-patches/left-pad.patch", + params={ + "version": ["1.3.0"], + "hash": ["629bda"], + "locator": ["berryscary@workspace:."], + }, + ), + ), ( _ParsedLocator(scope=None, name="fsevents", raw_reference="npm:2.3.2"), _ParsedReference(protocol="npm:", source=None, selector="2.3.2", params=None), @@ -218,6 +237,19 @@ params={"version": ["2.3.2"], "hash": ["df0bf1"]}, ), ), + ( + _ParsedLocator( + scope=None, + name="fsevents", + raw_reference="patch:fsevents@npm%3A2.3.2#optional!builtin::version=2.3.2&hash=df0bf1", + ), + _ParsedReference( + protocol="patch:", + source="fsevents@npm:2.3.2", + selector="optional!builtin", + params={"version": ["2.3.2"], "hash": ["df0bf1"]}, + ), + ), ( _ParsedLocator( scope=None, @@ -376,12 +408,22 @@ patches=(Path("my-patches/left-pad.patch"),), locator=WorkspaceLocator(scope=None, name="berryscary", relpath=Path(".")), ), + PatchLocator( + package=NpmLocator(scope=None, name="left-pad", version="1.3.0"), + patches=(Path("my-patches/left-pad.patch"),), + locator=WorkspaceLocator(scope=None, name="berryscary", relpath=Path(".")), + ), NpmLocator(scope=None, name="fsevents", version="2.3.2"), PatchLocator( package=NpmLocator(scope=None, name="fsevents", version="2.3.2"), patches=("builtin",), locator=None, ), + PatchLocator( + package=NpmLocator(scope=None, name="fsevents", version="2.3.2"), + patches=("builtin",), + locator=None, + ), PatchLocator( package=PatchLocator( package=NpmLocator(scope=None, name="fsevents", version="2.3.2"), From 3e304c137e4933abc885fd755e6caeef670e0922 Mon Sep 17 00:00:00 2001 From: Taylor Madore Date: Tue, 17 Dec 2024 14:38:16 -0500 Subject: [PATCH 3/4] add Component Pedigree models Adds an optional Pedigree for the Component model according to: https://cyclonedx.org/docs/1.6/json/#components_items_pedigree_patches For the PatchDiff model, provide a URL but not a text diff in the SBOM since it is not required by the schema. Signed-off-by: Taylor Madore --- cachi2/core/models/sbom.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/cachi2/core/models/sbom.py b/cachi2/core/models/sbom.py index 041218f88..cd3974684 100644 --- a/cachi2/core/models/sbom.py +++ b/cachi2/core/models/sbom.py @@ -27,6 +27,25 @@ class ExternalReference(pydantic.BaseModel): type: Literal["distribution"] = "distribution" +class PatchDiff(pydantic.BaseModel): + """A Diff inside a Patch.""" + + url: str + + +class Patch(pydantic.BaseModel): + """A Patch inside a SBOM Component Pedigree.""" + + type: Literal["backport", "cherry-pick", "monkey", "unofficial"] = "unofficial" + diff: PatchDiff + + +class Pedigree(pydantic.BaseModel): + """A Pedigree inside a SBOM component.""" + + patches: list[Patch] + + FOUND_BY_CACHI2_PROPERTY: Property = Property(name="cachi2:found_by", value="cachi2") @@ -45,6 +64,7 @@ class Component(pydantic.BaseModel): external_references: Optional[list[ExternalReference]] = pydantic.Field( serialization_alias="externalReferences", default=None ) + pedigree: Optional[Pedigree] = None def key(self) -> str: """Uniquely identifies a package. From f9f4502fda8b474f2da779652ddea7698f93d70d Mon Sep 17 00:00:00 2001 From: Taylor Madore Date: Thu, 9 Jan 2025 06:32:03 -0500 Subject: [PATCH 4/4] report yarn patches as pedigree not components Instead of reporting yarn patches as independent Components in the SBOM, report them instead as Pedigree for the parent, non-patch Component. This uses the Pedigree model of SBOM Components, which was implemented in accordance with: https://cyclonedx.org/docs/1.6/json/#components_items_pedigree_patches Yarn has the concept of "builtin" patches that are applied by yarn itself to make certain features of yarn work. These are reported out of the Yarn source repository for currently known patches from the compat plugin. Signed-off-by: Taylor Madore --- cachi2/core/package_managers/yarn/resolver.py | 122 ++++++-- .../test_data/yarn_e2e_test/bom.json | 64 ++++- tests/integration/test_data/yarn_v4/bom.json | 64 ++++- .../package_managers/yarn/test_resolver.py | 264 ++++++++++++++---- 4 files changed, 413 insertions(+), 101 deletions(-) diff --git a/cachi2/core/package_managers/yarn/resolver.py b/cachi2/core/package_managers/yarn/resolver.py index ee1e43c8b..57f2a1402 100644 --- a/cachi2/core/package_managers/yarn/resolver.py +++ b/cachi2/core/package_managers/yarn/resolver.py @@ -7,18 +7,22 @@ import json import logging +import re import zipfile +from collections import defaultdict from dataclasses import dataclass from functools import cached_property from pathlib import Path from textwrap import dedent from typing import TYPE_CHECKING, Any, Mapping, Union +from urllib.parse import quote import pydantic from packageurl import PackageURL +from semver import Version from cachi2.core.errors import PackageManagerError, PackageRejected, UnsupportedFeature -from cachi2.core.models.sbom import Component +from cachi2.core.models.sbom import Component, Patch, PatchDiff, Pedigree from cachi2.core.package_managers.yarn.locators import ( FileLocator, HttpsLocator, @@ -31,7 +35,7 @@ parse_locator, ) from cachi2.core.package_managers.yarn.project import Optional, Project -from cachi2.core.package_managers.yarn.utils import run_yarn_cmd +from cachi2.core.package_managers.yarn.utils import extract_yarn_version_from_env, run_yarn_cmd from cachi2.core.rooted_path import RootedPath from cachi2.core.scm import get_repo_id @@ -43,6 +47,10 @@ log = logging.getLogger(__name__) +COMPAT_PATCHES_SUBPATH = "packages/plugin-compat/sources/patches" +COMPAT_PATCHES_REGEX = re.compile(r"builtin]+)>") +YARN_REPO_URL = "https://github.com/yarnpkg/berry" + @dataclass(frozen=True) class Package: @@ -165,8 +173,18 @@ def create_components( packages: list[Package], project: Project, output_dir: RootedPath ) -> list[Component]: """Create SBOM components for all the packages parsed from the 'yarn info' output.""" - package_mapping = {package.parsed_locator: package for package in packages} - component_resolver = _ComponentResolver(package_mapping, project, output_dir) + package_mapping: dict[Locator, Package] = {} + patch_locators: list[PatchLocator] = [] + + # Patches are not components themselves, but they are necessary to + # resolve pedigree for their non-patch parent package components + for package in packages: + if isinstance(package.parsed_locator, PatchLocator): + patch_locators.append(package.parsed_locator) + else: + package_mapping[package.parsed_locator] = package + + component_resolver = _ComponentResolver(package_mapping, patch_locators, project, output_dir) return [component_resolver.get_component(package) for package in package_mapping.values()] @@ -192,11 +210,51 @@ class _CouldNotResolve(ValueError): class _ComponentResolver: def __init__( - self, package_mapping: Mapping[Locator, Package], project: Project, output_dir: RootedPath + self, + package_mapping: Mapping[Locator, Package], + patch_locators: list[PatchLocator], + project: Project, + output_dir: RootedPath, ) -> None: self._project = project self._output_dir = output_dir self._package_mapping = package_mapping + self._pedigree_mapping = self._get_pedigree_mapping(patch_locators) + + def _get_pedigree_mapping(self, patch_locators: list[PatchLocator]) -> dict[Locator, Pedigree]: + """Map locators for dependencies that get patched to their Pedigree.""" + pedigree_mapping: defaultdict[Locator, Pedigree] = defaultdict(lambda: Pedigree(patches=[])) + + if patch_locators: + # Builtin patches are included with the version of yarn being used + yarn_version = extract_yarn_version_from_env(self._project.source_dir) + + for patch_locator in patch_locators: + # Patches can patch other patches, so find the true parent Component + patched_package = self._get_patched_package(patch_locator) + pedigree = pedigree_mapping[patched_package] + + for patch in patch_locator.patches: + patch_url = self._get_patch_url(patch_locator, patch, yarn_version) + pedigree.patches.append(Patch(type="unofficial", diff=PatchDiff(url=patch_url))) + + return dict(pedigree_mapping) + + def _get_patch_url( + self, patch_locator: PatchLocator, patch: Union[Path, str], yarn_version: Version + ) -> str: + if isinstance(patch, Path): + return self._get_path_patch_url(patch_locator, patch) + + return self._get_builtin_patch_url(patch, yarn_version) + + def _get_patched_package(self, patch_locator: PatchLocator) -> Locator: + """Return the non-patch parent package for a given patch locator.""" + patched_locator = patch_locator.package + while isinstance(patched_locator, PatchLocator): + patched_locator = patched_locator.package + + return patched_locator def get_component(self, package: Package) -> Component: """Create an SBOM component for a yarn Package.""" @@ -217,6 +275,7 @@ def get_component(self, package: Package) -> Component: name=resolved_package.name, version=resolved_package.version, purl=purl, + pedigree=self._pedigree_mapping.get(package.parsed_locator), ) @staticmethod @@ -262,10 +321,7 @@ def _generate_purl_for_package(package: _ResolvedPackage, project: Project) -> s subpath = str(normalized.subpath_from_root) elif isinstance(package.locator, PatchLocator): - # ignore patch locators - # the actual dependency that is patched is reported separately - # the patch itself will be reported via SBOM pedigree patches - pass + raise _CouldNotResolve("Patches cannot be resolved into Components") else: assert_never(package.locator) @@ -329,23 +385,7 @@ def log_for_locator(msg: str, *args: Any, level: int = logging.DEBUG) -> None: ) name, version = self._read_name_version_from_packjson(packjson) elif isinstance(locator, PatchLocator): - if ( - package.cache_path - # yarn info seems to always report the cache path for patch dependencies, - # but the path doesn't always exist - and (cache_path := self._cache_path_as_rooted(package.cache_path)).path.exists() - ): - log_for_locator("reading package name from %s", cache_path.subpath_from_root) - name = self._read_name_from_cache(cache_path) - elif orig_package := self._package_mapping.get(locator.package): - log_for_locator("resolving the name of the original package") - name = self._resolve_package(orig_package).name - else: - raise _CouldNotResolve( - "the 'yarn info' output does not include either an existing zip archive " - "or the original unpatched package", - ) - version = package.version + raise _CouldNotResolve("Patches cannot be resolved into Components") else: # This line can never be reached assuming type-checker checks are passing # https://typing.readthedocs.io/en/latest/source/unreachable.html#assert-never-and-exhaustiveness-checking @@ -412,3 +452,33 @@ def _cache_path_as_rooted(self, cache_path: str) -> RootedPath: return self._project_subpath(cache_path) else: return self._output_dir.join_within_root(cache_path) + + def _get_path_patch_url(self, patch_locator: PatchLocator, patch_path: Path) -> str: + """Return a PURL-style VCS URL qualifier with subpath for a Patch.""" + if patch_locator.locator is None: + raise UnsupportedFeature( + ( + f"{patch_locator} is missing an associated workspace locator " + "and Cachi2 expects all non-builtin yarn patches to have one" + ) + ) + + project_path = self._project.source_dir + workspace_path = patch_locator.locator.relpath + normalized = self._project.source_dir.join_within_root(workspace_path, patch_path) + repo_url = get_repo_id(project_path.root).as_vcs_url_qualifier() + subpath_from_root = str(normalized.subpath_from_root) + + return f"{repo_url}#{subpath_from_root}" + + def _get_builtin_patch_url(self, patch: str, yarn_version: Version) -> str: + """Return a PURL-style VCS URL qualifier with subpath for a builtin Patch.""" + match = re.match(COMPAT_PATCHES_REGEX, patch) + if not match: + raise UnsupportedFeature(f"{patch} is not a builtin patch from plugin-compat") + + patch_filename = f"{match.group(1)}.patch.ts" + patch_subpath = Path(COMPAT_PATCHES_SUBPATH, patch_filename) + yarn_git_tag = quote(f"@yarnpkg/cli/{yarn_version}") + + return f"git+{YARN_REPO_URL}@{yarn_git_tag}#{patch_subpath}" diff --git a/tests/integration/test_data/yarn_e2e_test/bom.json b/tests/integration/test_data/yarn_e2e_test/bom.json index ace6704a6..462ef725b 100644 --- a/tests/integration/test_data/yarn_e2e_test/bom.json +++ b/tests/integration/test_data/yarn_e2e_test/bom.json @@ -314,18 +314,16 @@ }, { "name": "cachito-npm-without-deps", - "properties": [ - { - "name": "cachi2:found_by", - "value": "cachi2" - } - ], - "purl": "pkg:npm/cachito-npm-without-deps@1.0.0", - "type": "library", - "version": "1.0.0" - }, - { - "name": "cachito-npm-without-deps", + "pedigree": { + "patches": [ + { + "diff": { + "url": "git+https://github.com/cachito-testing/cachi2-yarn-berry.git@70515793108df42547d3320c7ea4cd6b6e505c46#.yarn/patches/ccto-wo-deps-git@github.com-e0fce8c89c.patch" + }, + "type": "unofficial" + } + ] + }, "properties": [ { "name": "cachi2:found_by", @@ -662,6 +660,22 @@ }, { "name": "fsevents", + "pedigree": { + "patches": [ + { + "diff": { + "url": "git+https://github.com/cachito-testing/cachi2-yarn-berry.git@70515793108df42547d3320c7ea4cd6b6e505c46#my-patches/fsevents.patch" + }, + "type": "unofficial" + }, + { + "diff": { + "url": "git+https://github.com/yarnpkg/berry@%40yarnpkg/cli/3.6.1#packages/plugin-compat/sources/patches/fsevents.patch.ts" + }, + "type": "unofficial" + } + ] + }, "properties": [ { "name": "cachi2:found_by", @@ -961,6 +975,22 @@ }, { "name": "left-pad", + "pedigree": { + "patches": [ + { + "diff": { + "url": "git+https://github.com/cachito-testing/cachi2-yarn-berry.git@70515793108df42547d3320c7ea4cd6b6e505c46#my-patches/left-pad.patch" + }, + "type": "unofficial" + }, + { + "diff": { + "url": "git+https://github.com/cachito-testing/cachi2-yarn-berry.git@70515793108df42547d3320c7ea4cd6b6e505c46#my-patches/left-pad-2.patch" + }, + "type": "unofficial" + } + ] + }, "properties": [ { "name": "cachi2:found_by", @@ -1691,6 +1721,16 @@ }, { "name": "typescript", + "pedigree": { + "patches": [ + { + "diff": { + "url": "git+https://github.com/yarnpkg/berry@%40yarnpkg/cli/3.6.1#packages/plugin-compat/sources/patches/typescript.patch.ts" + }, + "type": "unofficial" + } + ] + }, "properties": [ { "name": "cachi2:found_by", diff --git a/tests/integration/test_data/yarn_v4/bom.json b/tests/integration/test_data/yarn_v4/bom.json index e41656c00..1ab29f68f 100644 --- a/tests/integration/test_data/yarn_v4/bom.json +++ b/tests/integration/test_data/yarn_v4/bom.json @@ -314,18 +314,16 @@ }, { "name": "cachito-npm-without-deps", - "properties": [ - { - "name": "cachi2:found_by", - "value": "cachi2" - } - ], - "purl": "pkg:npm/cachito-npm-without-deps@1.0.0", - "type": "library", - "version": "1.0.0" - }, - { - "name": "cachito-npm-without-deps", + "pedigree": { + "patches": [ + { + "diff": { + "url": "git+https://github.com/cachito-testing/cachi2-yarn-berry.git@53a2bfe8d5ee7ed9c2f752fe75831a881d54895f#.yarn/patches/ccto-wo-deps-git@github.com-e0fce8c89c.patch" + }, + "type": "unofficial" + } + ] + }, "properties": [ { "name": "cachi2:found_by", @@ -662,6 +660,22 @@ }, { "name": "fsevents", + "pedigree": { + "patches": [ + { + "diff": { + "url": "git+https://github.com/cachito-testing/cachi2-yarn-berry.git@53a2bfe8d5ee7ed9c2f752fe75831a881d54895f#my-patches/fsevents.patch" + }, + "type": "unofficial" + }, + { + "diff": { + "url": "git+https://github.com/yarnpkg/berry@%40yarnpkg/cli/4.5.2#packages/plugin-compat/sources/patches/fsevents.patch.ts" + }, + "type": "unofficial" + } + ] + }, "properties": [ { "name": "cachi2:found_by", @@ -961,6 +975,22 @@ }, { "name": "left-pad", + "pedigree": { + "patches": [ + { + "diff": { + "url": "git+https://github.com/cachito-testing/cachi2-yarn-berry.git@53a2bfe8d5ee7ed9c2f752fe75831a881d54895f#my-patches/left-pad.patch" + }, + "type": "unofficial" + }, + { + "diff": { + "url": "git+https://github.com/cachito-testing/cachi2-yarn-berry.git@53a2bfe8d5ee7ed9c2f752fe75831a881d54895f#my-patches/left-pad-2.patch" + }, + "type": "unofficial" + } + ] + }, "properties": [ { "name": "cachi2:found_by", @@ -1691,6 +1721,16 @@ }, { "name": "typescript", + "pedigree": { + "patches": [ + { + "diff": { + "url": "git+https://github.com/yarnpkg/berry@%40yarnpkg/cli/4.5.2#packages/plugin-compat/sources/patches/typescript.patch.ts" + }, + "type": "unofficial" + } + ] + }, "properties": [ { "name": "cachi2:found_by", diff --git a/tests/unit/package_managers/yarn/test_resolver.py b/tests/unit/package_managers/yarn/test_resolver.py index c38766eaa..73e43372d 100644 --- a/tests/unit/package_managers/yarn/test_resolver.py +++ b/tests/unit/package_managers/yarn/test_resolver.py @@ -2,17 +2,28 @@ import re import zipfile from pathlib import Path -from typing import Any, NamedTuple, Optional +from typing import Any, NamedTuple, Optional, Union from unittest import mock from urllib.parse import quote import pytest +from semver import Version from cachi2.core.errors import PackageRejected, UnsupportedFeature -from cachi2.core.models.sbom import Component -from cachi2.core.package_managers.yarn.locators import parse_locator +from cachi2.core.models.sbom import Component, Patch, PatchDiff, Pedigree +from cachi2.core.package_managers.yarn.locators import ( + NpmLocator, + PatchLocator, + WorkspaceLocator, + parse_locator, +) from cachi2.core.package_managers.yarn.project import PackageJson, Project, YarnRc -from cachi2.core.package_managers.yarn.resolver import Package, create_components, resolve_packages +from cachi2.core.package_managers.yarn.resolver import ( + Package, + _ComponentResolver, + create_components, + resolve_packages, +) from cachi2.core.rooted_path import RootedPath from cachi2.core.scm import RepoID @@ -549,13 +560,29 @@ def test_create_components_single_package( assert caplog.messages == expect_logs +@mock.patch("cachi2.core.package_managers.yarn.resolver.get_repo_id") +@mock.patch("cachi2.core.package_managers.yarn.resolver.extract_yarn_version_from_env") def test_create_components_patched_packages( + mock_get_yarn_version: mock.Mock, + mock_get_repo_id: mock.Mock, rooted_tmp_path: RootedPath, - caplog: pytest.LogCaptureFixture, ) -> None: + mock_get_yarn_version.return_value = Version(3, 0, 0) + mock_get_repo_id.return_value = MOCK_REPO_ID project_dir = rooted_tmp_path mocked_packages = [ + MockedPackage( + Package( + raw_locator="fsevents@npm:2.3.2", + version="2.3.2", + checksum="97ade64e75091afee5265e6956cb72ba34db7819b4c3e94c431d4be2b19b8bb7a2d4116da417950c3425f17c8fe693d25e20212cac583ac1521ad066b77ae31f", + cache_path=project_dir.join_within_root( + ".yarn/cache/fsevents-npm-2.3.2-a881d6ac9f-97ade64e75.zip" + ).path.as_posix(), + ), + is_hardlink=True, + ), MockedPackage( Package( raw_locator="fsevents@patch:fsevents@npm%3A2.3.2#./my-patches/fsevents.patch::version=2.3.2&hash=cf0bf0&locator=berryscary%40workspace%3A.", @@ -566,16 +593,12 @@ def test_create_components_patched_packages( ).path.as_posix(), ), is_hardlink=True, - packjson_path="node_modules/fsevents/package.json", - packjson_content=json.dumps({"name": "@patch1/fsevents"}), ), MockedPackage( Package( # Note: this package patches the patched package above - raw_locator="fsevents@patch:fsevents@patch%3Afsevents@npm%253A2.3.2%23./my-patches/fsevents.patch%3A%3Aversion=2.3.2&hash=cf0bf0&locator=berryscary%2540workspace%253A.#~builtin::version=2.3.2&hash=df0bf1", - # normally, the versions would almost certainly be the same, but we need something - # to tell the two packages apart - version="2.3.2-patch2", + raw_locator="fsevents@patch:fsevents@patch%3Afsevents@npm%253A2.3.2%23./my-patches/fsevents.patch%3A%3Aversion=2.3.2&hash=cf0bf0&locator=berryscary%2540workspace%253A.#./my-patches/fsevents-2.patch::version=2.3.2&hash=df0bf1&locator=berryscary%40workspace%3A.", + version="2.3.2", checksum=None, cache_path=project_dir.join_within_root( ".yarn/cache/fsevents-patch-e4409ad759-8.zip" @@ -585,11 +608,6 @@ def test_create_components_patched_packages( ), ] - # the first package has a zip archive in the cache - mock_package_json(mocked_packages[0], project_dir) - # the second one does not - # ~~mock_package_json(mocked_packages[1], project_dir)~~ - components = create_components( [mocked_package.package for mocked_package in mocked_packages], mock_project(project_dir), @@ -598,32 +616,98 @@ def test_create_components_patched_packages( expect_components = [ Component( - name="@patch1/fsevents", + name="fsevents", version="2.3.2", - purl=f"pkg:npm/{quote('@patch1')}/fsevents@2.3.2", - ), - Component( - name="@patch1/fsevents", - version="2.3.2-patch2", - purl=f"pkg:npm/{quote('@patch1')}/fsevents@2.3.2-patch2", + purl="pkg:npm/fsevents@2.3.2", + pedigree=Pedigree( + patches=[ + Patch( + type="unofficial", + diff=PatchDiff( + url="git+https://github.com/org/project.git@fffffff#my-patches/fsevents.patch" + ), + ), + Patch( + type="unofficial", + diff=PatchDiff( + url="git+https://github.com/org/project.git@fffffff#my-patches/fsevents-2.patch" + ), + ), + ], + ), ), ] assert components == expect_components - patch_locator = "fsevents@patch:fsevents@npm%3A2.3.2#./my-patches/fsevents.patch::version=2.3.2&hash=cf0bf0&locator=berryscary%40workspace%3A." - patchpatch_locator = "fsevents@patch:fsevents@patch%3Afsevents@npm%253A2.3.2%23./my-patches/fsevents.patch%3A%3Aversion=2.3.2&hash=cf0bf0&locator=berryscary%2540workspace%253A.#~builtin::version=2.3.2&hash=df0bf1" - expect_logs = [ - # the first package has an archive in the cache - f"{patch_locator}: reading package name from .yarn/cache/fsevents-patch-9d1204d729-f73215b04b.zip", - # the second one does not, so we fall back to the original package - f"{patchpatch_locator}: resolving the name of the original package", - # ...which is the first package - f"{patch_locator}: reading package name from .yarn/cache/fsevents-patch-9d1204d729-f73215b04b.zip", +@mock.patch("cachi2.core.package_managers.yarn.resolver.get_repo_id") +@mock.patch("cachi2.core.package_managers.yarn.resolver.extract_yarn_version_from_env") +def test_create_components_patched_packages_with_multiple_paths( + mock_get_yarn_version: mock.Mock, + mock_get_repo_id: mock.Mock, + rooted_tmp_path: RootedPath, +) -> None: + mock_get_yarn_version.return_value = Version(3, 0, 0) + mock_get_repo_id.return_value = MOCK_REPO_ID + project_dir = rooted_tmp_path + + mocked_packages = [ + MockedPackage( + Package( + raw_locator="fsevents@npm:2.3.2", + version="2.3.2", + checksum="97ade64e75091afee5265e6956cb72ba34db7819b4c3e94c431d4be2b19b8bb7a2d4116da417950c3425f17c8fe693d25e20212cac583ac1521ad066b77ae31f", + cache_path=project_dir.join_within_root( + ".yarn/cache/fsevents-npm-2.3.2-a881d6ac9f-97ade64e75.zip" + ).path.as_posix(), + ), + is_hardlink=True, + ), + MockedPackage( + Package( + raw_locator="fsevents@patch:fsevents@npm%3A2.3.2#./my-patches/fsevents.patch&./my-patches/fsevents-2.patch::version=2.3.2&hash=cf0bf0&locator=berryscary%40workspace%3A.", + version="2.3.2", + checksum="f73215b04b52395389a612af4d30f7f412752cdfba1580c9e32c7ec259e448b57b464a4d0474427d6142f5ed9a6260fc1841d61834caf44706d77874fba6f17f", + cache_path=project_dir.join_within_root( + ".yarn/cache/fsevents-patch-9d1204d729-f73215b04b.zip" + ).path.as_posix(), + ), + is_hardlink=True, + ), ] - assert caplog.messages == expect_logs + components = create_components( + [mocked_package.package for mocked_package in mocked_packages], + mock_project(project_dir), + output_dir=RootedPath("/unused"), + ) + + expect_components = [ + Component( + name="fsevents", + version="2.3.2", + purl="pkg:npm/fsevents@2.3.2", + pedigree=Pedigree( + patches=[ + Patch( + type="unofficial", + diff=PatchDiff( + url="git+https://github.com/org/project.git@fffffff#my-patches/fsevents.patch" + ), + ), + Patch( + type="unofficial", + diff=PatchDiff( + url="git+https://github.com/org/project.git@fffffff#my-patches/fsevents-2.patch" + ), + ), + ], + ), + ), + ] + + assert components == expect_components @pytest.mark.parametrize( @@ -802,24 +886,6 @@ def test_create_components_patched_packages( ), id="https_no_cache_path", ), - # No cache_path for a Patch package, missing original package - pytest.param( - MockedPackage( - Package( - raw_locator="fsevents@patch:fsevents@npm%3A2.3.2#./my-patches/fsevents.patch::version=2.3.2&hash=cf0bf0&locator=berryscary%40workspace%3A.", - version="2.3.2", - checksum="f73215b04b52395389a612af4d30f7f412752cdfba1580c9e32c7ec259e448b57b464a4d0474427d6142f5ed9a6260fc1841d61834caf44706d77874fba6f17f", - cache_path=None, - ), - is_hardlink=True, - ), - ( - "Failed to resolve the name and version for " - "fsevents@patch:fsevents@npm%3A2.3.2#./my-patches/fsevents.patch::version=2.3.2&hash=cf0bf0&locator=berryscary%40workspace%3A.: " - "the 'yarn info' output does not include either an existing zip archive or the original unpatched package" - ), - id="patch_no_cache_path_no_orig_package", - ), ], ) def test_create_components_failed_to_resolve( @@ -861,3 +927,99 @@ def test_create_components_cache_path_reported_but_missing(rooted_tmp_path: Root mock_project(rooted_tmp_path), output_dir=RootedPath("/unused"), ) + + +@mock.patch("cachi2.core.package_managers.yarn.resolver.get_repo_id") +@mock.patch("cachi2.core.package_managers.yarn.resolver.extract_yarn_version_from_env") +def test_get_pedigree( + mock_get_yarn_version: mock.Mock, mock_get_repo_id: mock.Mock, rooted_tmp_path: RootedPath +) -> None: + mock_get_yarn_version.return_value = Version(3, 0, 0) + mock_get_repo_id.return_value = MOCK_REPO_ID + + project_workspace = WorkspaceLocator(None, "foo-project", Path(".")) + patched_package = NpmLocator(None, "fsevents", "1.0.0") + + first_patch_locator = PatchLocator( + patched_package, + [Path("./my-patches/fsevents.patch"), Path("./my-patches/fsevents-2.patch")], + project_workspace, + ) + second_patch_locator = PatchLocator( + first_patch_locator, [Path("./my-patches/fsevents-3.patch")], project_workspace + ) + third_patch_locator = PatchLocator(second_patch_locator, ["builtin"], None) + patch_locators = [ + first_patch_locator, + second_patch_locator, + third_patch_locator, + ] + + expected_pedigree = { + patched_package: Pedigree( + patches=[ + Patch( + type="unofficial", + diff=PatchDiff( + url="git+https://github.com/org/project.git@fffffff#my-patches/fsevents.patch" + ), + ), + Patch( + type="unofficial", + diff=PatchDiff( + url="git+https://github.com/org/project.git@fffffff#my-patches/fsevents-2.patch" + ), + ), + Patch( + type="unofficial", + diff=PatchDiff( + url="git+https://github.com/org/project.git@fffffff#my-patches/fsevents-3.patch" + ), + ), + Patch( + type="unofficial", + diff=PatchDiff( + url="git+https://github.com/yarnpkg/berry@%40yarnpkg/cli/3.0.0#packages/plugin-compat/sources/patches/fsevents.patch.ts" + ), + ), + ] + ), + } + + mock_project = mock.Mock(source_dir=rooted_tmp_path.re_root("source")) + resolver = _ComponentResolver( + {}, patch_locators, mock_project, rooted_tmp_path.re_root("output") + ) + + assert resolver._pedigree_mapping == expected_pedigree + + +@pytest.mark.parametrize( + "patch", + [ + pytest.param( + Path("foo.patch"), + id="path_patch_without_workspace", + ), + pytest.param( + "builtin", + id="builtin_patch_from_unknown_plugin", + ), + ], +) +@mock.patch("cachi2.core.package_managers.yarn.resolver.get_repo_id") +@mock.patch("cachi2.core.package_managers.yarn.resolver.extract_yarn_version_from_env") +def test_get_pedigree_with_unsupported_locators( + mock_get_yarn_version: mock.Mock, + mock_get_repo_id: mock.Mock, + patch: Union[Path, str], + rooted_tmp_path: RootedPath, +) -> None: + mock_get_yarn_version.return_value = Version(3, 0, 0) + mock_get_repo_id.return_value = MOCK_REPO_ID + + patch_locators = [PatchLocator(NpmLocator(None, "foo", "1.0.0"), [patch], None)] + mock_project = mock.Mock(source_dir=rooted_tmp_path.re_root("source")) + + with pytest.raises(UnsupportedFeature): + _ComponentResolver({}, patch_locators, mock_project, rooted_tmp_path.re_root("output"))