yarn: Resolve dev dependencies

cachi2/core/package_managers/yarn_classic/resolver.py

@@ -10,6 +10,7 @@
 from cachi2.core.errors import PackageRejected, UnexpectedFormat
 from cachi2.core.package_managers.npm import NPM_REGISTRY_CNAMES
 from cachi2.core.package_managers.yarn_classic.project import PackageJson, Project, YarnLock
+from cachi2.core.package_managers.yarn_classic.utils import find_runtime_deps
 from cachi2.core.package_managers.yarn_classic.workspaces import (
     Workspace,
     extract_workspace_metadata,
@@ -79,8 +80,9 @@ class LinkPackage(_BasePackage, _RelpathMixin):
 
 
 class _YarnClassicPackageFactory:
-    def __init__(self, source_dir: RootedPath):
+    def __init__(self, source_dir: RootedPath, runtime_deps: set[str]) -> None:
         self._source_dir = source_dir
+        self._runtime_deps = runtime_deps
 
     def create_package_from_pyarn_package(self, package: PYarnPackage) -> YarnClassicPackage:
         def assert_package_has_relative_path(package: PYarnPackage) -> None:
@@ -93,12 +95,16 @@ def assert_package_has_relative_path(package: PYarnPackage) -> None:
                     solution="Ensure that file/link packages in yarn.lock do not have absolute paths.",
                 )
 
+        package_id = f"{package.name}@{package.version}"
+        dev = package_id not in self._runtime_deps
+
         if _is_from_npm_registry(package.url):
             return RegistryPackage(
                 name=package.name,
                 version=package.version,
                 integrity=package.checksum,
                 url=package.url,
+                dev=dev,
             )
         elif package.path is not None:
             # Ensure path is not absolute
@@ -112,24 +118,28 @@ def assert_package_has_relative_path(package: PYarnPackage) -> None:
                     version=package.version,
                     relpath=path.subpath_from_root,
                     integrity=package.checksum,
+                    dev=dev,
                 )
             return LinkPackage(
                 name=package.name,
                 version=package.version,
                 relpath=path.subpath_from_root,
+                dev=dev,
             )
         elif _is_git_url(package.url):
             return GitPackage(
                 name=package.name,
                 version=package.version,
                 url=package.url,
+                dev=dev,
             )
         elif _is_tarball_url(package.url):
             return UrlPackage(
                 name=package.name,
                 version=package.version,
                 url=package.url,
                 integrity=package.checksum,
+                dev=dev,
             )
         else:
             raise UnexpectedFormat(
@@ -187,11 +197,11 @@ def _is_from_npm_registry(url: str) -> bool:
 
 
 def _get_packages_from_lockfile(
-    source_dir: RootedPath, yarn_lock: YarnLock
+    source_dir: RootedPath, yarn_lock: YarnLock, runtime_deps: set[str]
 ) -> list[YarnClassicPackage]:
     """Return a list of Packages for all dependencies in yarn.lock."""
     pyarn_packages: list[PYarnPackage] = yarn_lock.yarn_lockfile.packages()
-    package_factory = _YarnClassicPackageFactory(source_dir)
+    package_factory = _YarnClassicPackageFactory(source_dir, runtime_deps)
 
     return [
         package_factory.create_package_from_pyarn_package(package) for package in pyarn_packages
@@ -230,8 +240,10 @@ def resolve_packages(project: Project) -> Iterable[YarnClassicPackage]:
     """Return a list of Packages corresponding to all project dependencies."""
     workspaces = extract_workspace_metadata(project.source_dir)
     yarn_lock = YarnLock.from_file(project.source_dir.join_within_root("yarn.lock"))
+    runtime_deps = find_runtime_deps(project.package_json, yarn_lock, workspaces)
+
     return chain(
         [_get_main_package(project.package_json)],
         _get_workspace_packages(project.source_dir, workspaces),
-        _get_packages_from_lockfile(project.source_dir, yarn_lock),
+        _get_packages_from_lockfile(project.source_dir, yarn_lock, runtime_deps),
     )

cachi2/core/package_managers/yarn_classic/utils.py

@@ -0,0 +1,112 @@
+from collections import deque
+from typing import Any
+
+from pyarn.lockfile import Package as PYarnPackage
+
+from cachi2.core.package_managers.yarn_classic.project import PackageJson, YarnLock
+from cachi2.core.package_managers.yarn_classic.workspaces import Workspace
+
+
+def find_runtime_deps(
+    main_package_json: PackageJson,
+    yarn_lock: YarnLock,
+    workspaces: list[Workspace],
+) -> set[str]:
+    """
+    Identify all runtime dependencies in the root package and its workspaces.
+
+    A dependency is classified as runtime if:
+    - It is listed in `dependencies`, `peerDependencies`, or `optionalDependencies`
+      of any `package.json` file.
+    - It is a transitive dependency of another runtime dependency.
+
+    A dependency is classified for development if:
+    - It is listed in the `devDependencies` of any `package.json` file.
+    - It is a transitive dependency of a dev dependency.
+
+    Note: If a dependency is marked as runtime dependency somewhere
+    and as a development dependency somewhere else, it is classified as runtime.
+    """
+    all_package_jsons = [main_package_json] + [ws.package_json for ws in workspaces]
+    expanded_yarn_lock = _expand_yarn_lock_keys(yarn_lock)
+
+    root_deps: list[PYarnPackage] = []
+    for package_json in all_package_jsons:
+        for dep_type in ["dependencies", "peerDependencies", "optionalDependencies"]:
+            for name, version_specifier in package_json.data.get(dep_type, {}).items():
+                key = f"{name}@{version_specifier}"
+                data = expanded_yarn_lock.get(key)
+
+                if not data:
+                    # peerDependencies are not always present in the yarn.lock
+                    continue
+
+                root_dep = PYarnPackage.from_dict(key, data)
+                root_deps.append(root_dep)
+
+    all_dep_ids: set[str] = set()
+    for root_dep in root_deps:
+        transitive_dep_ids = _find_transitive_deps(root_dep, expanded_yarn_lock)
+        all_dep_ids.update(transitive_dep_ids)
+
+    return all_dep_ids
+
+
+def _expand_yarn_lock_keys(yarn_lock: YarnLock) -> dict[str, dict[str, Any]]:
+    """
+    Expand compound keys in the yarn.lock dictionary into individual keys.
+
+    In the original yarn.lock dictionary, a single key may represent multiple package names,
+    separated by commas (e.g., "package-a@^1.0.0, package-b@^2.0.0"). These are referred to
+    as compound keys, where multiple keys share the same value (N:1 mapping).
+
+    This function splits such compound keys into individual keys, creating a new dictionary
+    where each key maps directly to the same shared value as in the original dictionary.
+    The result is a dictionary with only one-to-one (1:1) key-value mappings.
+
+    Note: This function does not copy the values. The newly created individual keys will
+    all reference the same original value object.
+    """
+
+    def split_multi_key(dep_id: str) -> list[str]:
+        return dep_id.replace('"', "").split(", ")
+
+    result = {}
+    for dep_id in yarn_lock.data.keys():
+        for key in split_multi_key(dep_id):
+            result[key] = yarn_lock.data[dep_id]
+
+    return result
+
+
+def _find_transitive_deps(
+    root_dep: PYarnPackage,
+    expanded_yarn_lock: dict[str, dict[str, Any]],
+) -> set[str]:
+    """Perform a breadth-first search (BFS) algorithm to find all transitive dependencies of a given root dependency.
+
+    The search is performed on the expanded yarn.lock dictionary, which contains individual keys for each package.
+    Keys in the expanded yarn.lock dictionary contains version specifiers, not the resolved version of the package.
+
+    If expanded_yarn_lock contains a key "foo@^1.0.0", the actual resolved version of "foo" may be for example "1.1.0".
+    The result of this function is a set of strings in the format "package-name@-resolved-version" of all transitive dependencies.
+    """
+    bfs_queue: deque[PYarnPackage] = deque([root_dep])
+    visited: set[str] = set()
+
+    while bfs_queue:
+        current = bfs_queue.popleft()
+        dep_id = f"{current.name}@{current.version}"
+        visited.add(dep_id)
+
+        for name, version_specifier in current.dependencies.items():
+            key = f"{name}@{version_specifier}"
+            data = expanded_yarn_lock.get(key)
+
+            new_dep = PYarnPackage.from_dict(key, data)
+            new_dep_id = f"{new_dep.name}@{new_dep.version}"
+
+            if new_dep_id not in visited:
+                bfs_queue.append(new_dep)
+
+    return visited

tests/unit/package_managers/yarn_classic/test_resolver.py

@@ -157,7 +157,7 @@ def test__is_from_npm_registry_can_parse_incorrect_registry_urls() -> None:
             GitPackage(
                 name="foo",
                 version="1.0.0",
-                dev=False,
+                dev=True,
                 url="https://github.com/org/foo.git#fffffff",
             ),
         ),
@@ -170,7 +170,7 @@ def test__is_from_npm_registry_can_parse_incorrect_registry_urls() -> None:
             UrlPackage(
                 name="foo",
                 version="1.0.0",
-                dev=False,
+                dev=True,
                 url="https://example.com/foo-1.0.0.tgz",
             ),
         ),
@@ -179,7 +179,11 @@ def test__is_from_npm_registry_can_parse_incorrect_registry_urls() -> None:
 def test_create_package_from_pyarn_package(
     pyarn_package: PYarnPackage, expected_package: YarnClassicPackage, rooted_tmp_path: RootedPath
 ) -> None:
-    package_factory = _YarnClassicPackageFactory(rooted_tmp_path)
+    runtime_deps = (
+        set() if expected_package.dev else set({f"{pyarn_package.name}@{pyarn_package.version}"})
+    )
+
+    package_factory = _YarnClassicPackageFactory(rooted_tmp_path, runtime_deps)
     assert package_factory.create_package_from_pyarn_package(pyarn_package) == expected_package
 
 
@@ -194,7 +198,7 @@ def test_create_package_from_pyarn_package_fail_absolute_path(rooted_tmp_path: R
         f"({pyarn_package.path}), which is not permitted."
     )
 
-    package_factory = _YarnClassicPackageFactory(rooted_tmp_path)
+    package_factory = _YarnClassicPackageFactory(rooted_tmp_path, set())
     with pytest.raises(PackageRejected, match=re.escape(error_msg)):
         package_factory.create_package_from_pyarn_package(pyarn_package)
 
@@ -208,7 +212,7 @@ def test_create_package_from_pyarn_package_fail_path_outside_root(
         path="../path/outside/root",
     )
 
-    package_factory = _YarnClassicPackageFactory(rooted_tmp_path)
+    package_factory = _YarnClassicPackageFactory(rooted_tmp_path, set())
     with pytest.raises(PathOutsideRoot):
         package_factory.create_package_from_pyarn_package(pyarn_package)
 
@@ -222,7 +226,7 @@ def test_create_package_from_pyarn_package_fail_unexpected_format(
         url="ftp://some-tarball.tgz",
     )
 
-    package_factory = _YarnClassicPackageFactory(rooted_tmp_path)
+    package_factory = _YarnClassicPackageFactory(rooted_tmp_path, set())
     with pytest.raises(UnexpectedFormat):
         package_factory.create_package_from_pyarn_package(pyarn_package)
 
@@ -233,7 +237,6 @@ def test_create_package_from_pyarn_package_fail_unexpected_format(
 def test__get_packages_from_lockfile(
     mock_create_package: mock.Mock, rooted_tmp_path: RootedPath
 ) -> None:
-
     # Setup lockfile instance
     mock_pyarn_lockfile = mock.Mock()
     mock_yarn_lock = mock.Mock(yarn_lockfile=mock_pyarn_lockfile)
@@ -250,7 +253,7 @@ def test__get_packages_from_lockfile(
         mock.call(mock_pyarn_package_2),
     ]
 
-    output = _get_packages_from_lockfile(rooted_tmp_path, mock_yarn_lock)
+    output = _get_packages_from_lockfile(rooted_tmp_path, mock_yarn_lock, set())
 
     mock_pyarn_lockfile.packages.assert_called_once()
     mock_create_package.assert_has_calls(create_package_expected_calls)
@@ -262,7 +265,9 @@ def test__get_packages_from_lockfile(
 @mock.patch("cachi2.core.package_managers.yarn_classic.resolver.extract_workspace_metadata")
 @mock.patch("cachi2.core.package_managers.yarn_classic.resolver._get_packages_from_lockfile")
 @mock.patch("cachi2.core.package_managers.yarn_classic.resolver._get_main_package")
+@mock.patch("cachi2.core.package_managers.yarn_classic.resolver.find_runtime_deps")
 def test_resolve_packages(
+    find_runtime_deps: mock.Mock,
     mock_get_main_package: mock.Mock,
     mock_get_lockfile_packages: mock.Mock,
     mock_extract_workspaces: mock.Mock,
@@ -278,6 +283,7 @@ def test_resolve_packages(
     lockfile_packages = [mock.Mock(), mock.Mock()]
     expected_output = [main_package, *workspace_packages, *lockfile_packages]
 
+    find_runtime_deps.return_value = set()
     mock_get_main_package.return_value = main_package
     mock_get_lockfile_packages.return_value = lockfile_packages
     mock_get_workspace_packages.return_value = workspace_packages
@@ -290,7 +296,9 @@ def test_resolve_packages(
         rooted_tmp_path, mock_extract_workspaces.return_value
     )
     mock_get_lockfile_packages.assert_called_once_with(
-        rooted_tmp_path, mock_get_yarn_lock.return_value
+        rooted_tmp_path,
+        mock_get_yarn_lock.return_value,
+        find_runtime_deps.return_value,
     )
     assert list(output) == expected_output