From 81f0c072a78b15b858c91e8b490f5160a3dafddf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20=C5=A0oltis?= Date: Thu, 21 Nov 2024 13:37:02 +0100 Subject: [PATCH] yarn v1: Add utilities for dependency resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add utility functions to identify runtime dependencies in Yarn Classic projects: - Implement find_runtime_deps() to identify production dependencies across workspaces - Support compound key expansion in yarn.lock files - Use BFS algorithm for efficient transitive dependency resolution This enables proper classification of runtime dependencies based on their usage in package.json files and transitive relationships. Signed-off-by: Michal Šoltis --- .../package_managers/yarn_classic/utils.py | 103 ++++++++++++++++++ .../yarn_classic/test_utils.py | 69 ++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 cachi2/core/package_managers/yarn_classic/utils.py create mode 100644 tests/unit/package_managers/yarn_classic/test_utils.py diff --git a/cachi2/core/package_managers/yarn_classic/utils.py b/cachi2/core/package_managers/yarn_classic/utils.py new file mode 100644 index 000000000..e21289bf7 --- /dev/null +++ b/cachi2/core/package_managers/yarn_classic/utils.py @@ -0,0 +1,103 @@ +from collections import deque +from typing import Any + +from pyarn.lockfile import Package as PYarnPackage + +from cachi2.core.package_managers.yarn_classic.project import PackageJson, YarnLock +from cachi2.core.package_managers.yarn_classic.workspaces import Workspace +from cachi2.core.rooted_path import RootedPath + + +def find_runtime_deps( + main_package_json: PackageJson, + yarn_lock: YarnLock, + workspaces: list[Workspace], +) -> set[str]: + """ + Identify all runtime dependencies in the root package and its workspaces. + + A dependency is classified as runtime if: + - It is listed in `dependencies`, `peerDependencies`, or `optionalDependencies` + of any `package.json` file. + - It is a transitive dependency of another runtime dependency. + + A dependency is classified for development if: + - It is listed in the `devDependencies` of any `package.json` file. + - It is a transitive dependency of a dev dependency. + + Note: If a dependency is marked as runtime dependency somewhere + and as a development dependency somewhere else, it is classified as runtime. + """ + # TODO: PackageJson object should be an attribute of the Workspace object + ws_package_jsons = [ + PackageJson.from_file(RootedPath(ws.path / "package.json")) for ws in workspaces + ] + all_package_jsons = [main_package_json] + ws_package_jsons + + root_dep_ids = [] + for package_json in all_package_jsons: + for dep_type in ["dependencies", "peerDependencies", "optionalDependencies"]: + for name, version in package_json.data.get(dep_type, {}).items(): + root_dep_id = f"{name}@{version}" + root_dep_ids.append(root_dep_id) + + expanded_yarn_lock = _expand_yarn_lock_keys(yarn_lock) + + all_dep_ids = set() + for root_dep_id in root_dep_ids: + transitive_dep_ids = _find_transitive_deps(root_dep_id, expanded_yarn_lock) + all_dep_ids.update(transitive_dep_ids) + + return all_dep_ids + + +def _expand_yarn_lock_keys(yarn_lock: YarnLock) -> dict[str, dict[str, Any]]: + """ + Expand compound keys in the yarn.lock dictionary into individual keys. + + In the original yarn.lock dictionary, a single key may represent multiple package names, + separated by commas (e.g., "package-a@^1.0.0, package-b@^2.0.0"). These are referred to + as compound keys, where multiple keys share the same value (N:1 mapping). + + This function splits such compound keys into individual keys, creating a new dictionary + where each key maps directly to the same shared value as in the original dictionary. + The result is a dictionary with only one-to-one (1:1) key-value mappings. + + Note: This function does not copy the values. The newly created individual keys will + all reference the same original value object. + """ + + def split_multi_key(dep_id: str) -> list[str]: + return dep_id.replace('"', "").split(", ") + + result = {} + for dep_id in yarn_lock.data.keys(): + for key in split_multi_key(dep_id): + result[key] = yarn_lock.data[dep_id] + + return result + + +def _find_transitive_deps( + root_dep_id: str, + expanded_yarn_lock: dict[str, dict[str, Any]], +) -> set[str]: + """Perform a breadth-first search (BFS) to find all transitive dependencies of a given root dependency.""" + bfs_queue = deque([root_dep_id]) + visited = set() + + while bfs_queue: + current_dep_id = bfs_queue.popleft() + visited.add(current_dep_id) + + dep_info = expanded_yarn_lock.get(current_dep_id) + if dep_info is None: + continue + + package = PYarnPackage.from_dict(current_dep_id, dep_info) + for name, version in package.dependencies.items(): + new_dep_id = f"{name}@{version}" + if new_dep_id not in visited: + bfs_queue.append(new_dep_id) + + return visited diff --git a/tests/unit/package_managers/yarn_classic/test_utils.py b/tests/unit/package_managers/yarn_classic/test_utils.py new file mode 100644 index 000000000..99f723b38 --- /dev/null +++ b/tests/unit/package_managers/yarn_classic/test_utils.py @@ -0,0 +1,69 @@ +from unittest import mock + +from cachi2.core.package_managers.yarn_classic.project import PackageJson +from cachi2.core.package_managers.yarn_classic.utils import find_runtime_deps +from cachi2.core.rooted_path import RootedPath + +PACKAGE_JSON_CONTENT = """ +{ + "name": "project", + "dependencies": { + "main-dep1": "^1.0.0" + }, + "optionalDependencies": { + "optional-dep1": "^2.0.0" + }, + "peerDependencies": { + "peer-dep1": "^3.0.0" + }, + "devDependencies": { + "dev-dep1": "^4.0.0" + } +} +""" + + +@mock.patch("cachi2.core.package_managers.yarn_classic.project.YarnLock") +@mock.patch("cachi2.core.package_managers.yarn_classic.utils._expand_yarn_lock_keys") +def test_find_runtime_deps( + mock_expand_yarn_lock_keys: mock.Mock, + mock_yarn_lock: mock.Mock, + rooted_tmp_path: RootedPath, +) -> None: + rooted_tmp_path.join_within_root("package.json").path.write_text(PACKAGE_JSON_CONTENT) + package_json = PackageJson.from_file(rooted_tmp_path.join_within_root("package.json")) + + mock_expand_yarn_lock_keys.return_value = { + # dependencies + "main-dep1@^1.0.0": { + "version": "1.0.0", + "dependencies": {"sub-dep1": "^1.1.0"}, + }, + # optional dependencies + "optional-dep1@^2.0.0": { + "version": "2.0.0", + "dependencies": {"multi-dep1": "^4.0.0", "multi-dep2": "^4.0.0"}, + }, + "multi-dep1@^4.0.0, multi-dep2@^4.0.0": {"version": "5.0.0", "dependencies": {}}, + # peer dependencies + "peer-dep1@^3.0.0": {"version": "3.0.0", "dependencies": {}}, + # transitive dependencies + "sub-dep1@^1.1.0": {"version": "1.1.0", "dependencies": {"sub-dep11": "^1.1.1"}}, + "sub-dep11@^1.1.1": {"version": "1.1.1", "dependencies": {}}, + # dev dependencies + "dev-dep1@^4.0.0": {"version": "4.0.0", "dependencies": {}}, + } + + result = find_runtime_deps(package_json, mock_yarn_lock, []) + + mock_expand_yarn_lock_keys.assert_called_once_with(mock_yarn_lock) + assert result == { + "main-dep1@^1.0.0", + "optional-dep1@^2.0.0", + "multi-dep1@^4.0.0", + "multi-dep2@^4.0.0", + "peer-dep1@^3.0.0", + "sub-dep1@^1.1.0", + "sub-dep11@^1.1.1", + # no dev dependencies + }