yarn v1: Add utilities for dependency resolution

Add utility functions to identify runtime dependencies in Yarn Classic projects: - Implement find_runtime_deps() to identify production dependencies across workspaces - Support compound key expansion in yarn.lock files - Use BFS algorithm for efficient transitive dependency resolution This enables proper classification of runtime dependencies based on their usage in package.json files and transitive relationships. Signed-off-by: Michal Šoltis <[email protected]>
containerbuildsystem · Dec 3, 2024 · 43bcb3d · 43bcb3d
1 parent 9b7afff
commit 43bcb3d
Show file tree

Hide file tree

Showing 2 changed files with 235 additions and 0 deletions.
diff --git a/cachi2/core/package_managers/yarn_classic/utils.py b/cachi2/core/package_managers/yarn_classic/utils.py
@@ -0,0 +1,105 @@
+from collections import deque
+from typing import Any
+
+from pyarn.lockfile import Package as PYarnPackage
+
+from cachi2.core.package_managers.yarn_classic.project import PackageJson, YarnLock
+from cachi2.core.package_managers.yarn_classic.workspaces import Workspace
+
+
+def find_runtime_deps(
+    main_package_json: PackageJson,
+    yarn_lock: YarnLock,
+    workspaces: list[Workspace],
+) -> set[str]:
+    """
+    Identify all runtime dependencies in the root package and its workspaces.
+
+    A dependency is classified as runtime if:
+    - It is listed in `dependencies`, `peerDependencies`, or `optionalDependencies`
+      of any `package.json` file.
+    - It is a transitive dependency of another runtime dependency.
+
+    A dependency is classified for development if:
+    - It is listed in the `devDependencies` of any `package.json` file.
+    - It is a transitive dependency of a dev dependency.
+
+    Note: If a dependency is marked as runtime dependency somewhere
+    and as a development dependency somewhere else, it is classified as runtime.
+    """
+    all_package_jsons = [main_package_json] + [ws.package_json for ws in workspaces]
+    expanded_yarn_lock = _expand_yarn_lock_keys(yarn_lock)
+
+    root_deps: list[PYarnPackage] = []
+    for package_json in all_package_jsons:
+        for dep_type in ["dependencies", "peerDependencies", "optionalDependencies"]:
+            for name, version in package_json.data.get(dep_type, {}).items():
+                raw_name = f"{name}@{version}"
+                data = expanded_yarn_lock.get(raw_name)
+
+                if not data:
+                    # peerDependencies are not always present in the yarn.lock
+                    continue
+
+                root_dep = PYarnPackage.from_dict(raw_name, data)
+                root_deps.append(root_dep)
+
+    all_dep_ids: set[str] = set()
+    for root_dep in root_deps:
+        transitive_dep_ids = _find_transitive_deps(root_dep, expanded_yarn_lock)
+        all_dep_ids.update(transitive_dep_ids)
+
+    return all_dep_ids
+
+
+def _expand_yarn_lock_keys(yarn_lock: YarnLock) -> dict[str, dict[str, Any]]:
+    """
+    Expand compound keys in the yarn.lock dictionary into individual keys.
+
+    In the original yarn.lock dictionary, a single key may represent multiple package names,
+    separated by commas (e.g., "package-a@^1.0.0, package-b@^2.0.0"). These are referred to
+    as compound keys, where multiple keys share the same value (N:1 mapping).
+
+    This function splits such compound keys into individual keys, creating a new dictionary
+    where each key maps directly to the same shared value as in the original dictionary.
+    The result is a dictionary with only one-to-one (1:1) key-value mappings.
+
+    Note: This function does not copy the values. The newly created individual keys will
+    all reference the same original value object.
+    """
+
+    def split_multi_key(dep_id: str) -> list[str]:
+        return dep_id.replace('"', "").split(", ")
+
+    result = {}
+    for dep_id in yarn_lock.data.keys():
+        for key in split_multi_key(dep_id):
+            result[key] = yarn_lock.data[dep_id]
+
+    return result
+
+
+def _find_transitive_deps(
+    root_dep: PYarnPackage,
+    expanded_yarn_lock: dict[str, dict[str, Any]],
+) -> set[str]:
+    """Perform a breadth-first search (BFS) to find all transitive dependencies of a given root dependency."""
+    bfs_queue: deque[PYarnPackage] = deque([root_dep])
+    visited: set[str] = set()
+
+    while bfs_queue:
+        current = bfs_queue.popleft()
+        dep_id = f"{current.name}@{current.version}"
+        visited.add(dep_id)
+
+        for name, version in current.dependencies.items():
+            raw_name = f"{name}@{version}"
+            data = expanded_yarn_lock.get(raw_name)
+
+            new_dep = PYarnPackage.from_dict(raw_name, data)
+            new_dep_id = f"{new_dep.name}@{new_dep.version}"
+
+            if new_dep_id not in visited:
+                bfs_queue.append(new_dep)
+
+    return visited
diff --git a/tests/unit/package_managers/yarn_classic/test_utils.py b/tests/unit/package_managers/yarn_classic/test_utils.py
@@ -0,0 +1,130 @@
+from unittest import mock
+
+from cachi2.core.package_managers.yarn_classic.project import PackageJson
+from cachi2.core.package_managers.yarn_classic.utils import find_runtime_deps
+from cachi2.core.package_managers.yarn_classic.workspaces import Workspace
+from cachi2.core.rooted_path import RootedPath
+
+PACKAGE_JSON = """
+{
+  "name": "main",
+  "dependencies": {
+    "main-dep1": "^1.2.0"
+  },
+  "optionalDependencies": {
+    "optional-dep1": "^2.3.0"
+  },
+  "peerDependencies": {
+    "peer-dep1": "^3.4.0"
+  },
+  "devDependencies": {
+    "dev-dep1": "^4.5.0"
+  }
+}
+"""
+
+
+@mock.patch("cachi2.core.package_managers.yarn_classic.project.YarnLock")
+def test_find_runtime_deps(
+    mock_yarn_lock: mock.Mock,
+    rooted_tmp_path: RootedPath,
+) -> None:
+    package_json_path = rooted_tmp_path.join_within_root("package.json")
+    package_json_path.path.write_text(PACKAGE_JSON)
+    package_json = PackageJson.from_file(package_json_path)
+
+    mock_yarn_lock_instance = mock_yarn_lock.return_value
+    mock_yarn_lock_instance.data = {
+        # dependencies
+        "main-dep1@^1.2.0": {
+            "version": "1.2.0",
+            "dependencies": {"sub-dep1": "^1.3.0"},
+        },
+        # optional dependencies
+        "optional-dep1@^2.3.0": {
+            "version": "2.3.0",
+            "dependencies": {"compound-multi-dep1": "^4.5.0", "compound-multi-dep2": "^4.6.0"},
+        },
+        "compound-multi-dep1@^4.5.0, compound-multi-dep2@^4.6.0": {
+            "version": "5.7.0",
+            "dependencies": {},
+        },
+        # peer dependencies
+        "peer-dep1@^3.4.0": {"version": "3.4.0", "dependencies": {}},
+        # transitive dependencies
+        "sub-dep1@^1.3.0": {"version": "1.3.0", "dependencies": {"sub-dep11": "^1.4.0"}},
+        "sub-dep11@^1.4.0": {"version": "1.4.0", "dependencies": {}},
+        # dev dependencies
+        "dev-dep1@^4.5.0": {"version": "4.5.0", "dependencies": {}},
+    }
+
+    result = find_runtime_deps(package_json, mock_yarn_lock_instance, [])
+    expected_result = {
+        "[email protected]",
+        "[email protected]",
+        "[email protected]",
+        "[email protected]",
+        "[email protected]",
+        "[email protected]",
+        "[email protected]",
+        # no dev dependencies
+    }
+
+    assert result == expected_result
+
+
+MAIN_PACKAGE_JSON = """
+{
+  "name": "main",
+  "workspaces": ["packages/*"],
+  "dependencies": {
+    "foo": "^2.1.0"
+  },
+  "devDependencies": {
+    "bar": "^3.2.0"
+  }
+}
+"""
+
+WORKSPACE_PACKAGE_JSON = """
+{
+  "name": "workspace",
+  "dependencies": {
+    "bar": "^3.2.0"
+  }
+}
+"""
+
+
+@mock.patch("cachi2.core.package_managers.yarn_classic.project.YarnLock")
+def test_find_runtime_deps_with_workspace(
+    mock_yarn_lock: mock.Mock,
+    rooted_tmp_path: RootedPath,
+) -> None:
+    package_json_path = rooted_tmp_path.join_within_root("package.json")
+    package_json_path.path.write_text(MAIN_PACKAGE_JSON)
+    package_json = PackageJson.from_file(package_json_path)
+
+    workspace_dir = rooted_tmp_path.join_within_root("packages/workspace")
+    workspace_dir.path.mkdir(parents=True)
+
+    workspace_package_json = workspace_dir.join_within_root("package.json")
+    workspace_package_json.path.write_text(WORKSPACE_PACKAGE_JSON)
+
+    w = Workspace(
+        path=workspace_dir.path,
+        package_json=PackageJson.from_file(workspace_package_json),
+    )
+
+    mock_yarn_lock_instance = mock_yarn_lock.return_value
+    mock_yarn_lock_instance.data = {
+        # dependencies from main package.json
+        "foo@^2.1.0": {"version": "2.1.0", "dependencies": {}},
+        # dependencies from workspace package.json
+        "bar@^3.2.0": {"version": "3.2.0", "dependencies": {}},
+    }
+
+    result = find_runtime_deps(package_json, mock_yarn_lock_instance, [w])
+    expected_result = {"[email protected]", "[email protected]"}
+
+    assert result == expected_result