diff --git a/cachi2/core/package_managers/gomod.py b/cachi2/core/package_managers/gomod.py index 8d40dd4b6..f83ff1f42 100644 --- a/cachi2/core/package_managers/gomod.py +++ b/cachi2/core/package_managers/gomod.py @@ -50,6 +50,8 @@ GOMOD_INPUT_DOC = f"{GOMOD_DOC}#specifying-modules-to-process" VENDORING_DOC = f"{GOMOD_DOC}#vendoring" +ModuleDict = dict[str, Any] + class _ParsedModel(pydantic.BaseModel): """Attributes automatically get PascalCase aliases to make parsing Golang JSON easier. @@ -860,11 +862,8 @@ def _resolve_gomod( # Make Go ignore the vendor dir even if there is one go_list.extend(["-mod", "readonly"]) - main_module_name = go([*go_list, "-m"], run_params).rstrip() - main_module = ParsedModule( - path=main_module_name, - version=version_resolver.get_golang_version(main_module_name, app_dir), - main=True, + main_module, workspace_modules = _parse_local_modules( + go, go_list, run_params, app_dir, version_resolver ) def go_list_deps(pattern: Literal["./...", "all"]) -> Iterator[ParsedPackage]: @@ -878,10 +877,10 @@ def go_list_deps(pattern: Literal["./...", "all"]) -> Iterator[ParsedPackage]: cmd = [*go_list, "-deps", "-json=ImportPath,Module,Standard,Deps", pattern] return map(ParsedPackage.model_validate, load_json_stream(go(cmd, run_params))) - package_modules = ( + package_modules = [ module for pkg in go_list_deps("all") if (module := pkg.module) and not module.main - ) - + ] + package_modules.extend(workspace_modules) all_modules = _deduplicate_resolved_modules(package_modules, downloaded_modules) log.info("Retrieving the list of packages") @@ -892,6 +891,93 @@ def go_list_deps(pattern: Literal["./...", "all"]) -> Iterator[ParsedPackage]: return ResolvedGoModule(main_module, all_modules, all_packages, modules_in_go_sum) +def _parse_local_modules( + go: Go, + go_list: list[str], + run_params: dict[str, Any], + app_dir: RootedPath, + version_resolver: "ModuleVersionResolver", +) -> tuple[ParsedModule, list[ParsedModule]]: + """ + Identify and parse the main module and all workspace modules, if they exist. + + :return: A tuple containing the main module and a list of workspaces + """ + modules_json_stream = go([*go_list, "-m", "-json"], run_params).rstrip() + main_module_dict, workspace_dict_list = _process_modules_json_stream( + app_dir, modules_json_stream + ) + + main_module_path = main_module_dict["Path"] + main_module_version = version_resolver.get_golang_version(main_module_path, app_dir) + + main_module = ParsedModule( + path=main_module_path, + version=main_module_version, + main=True, + ) + + workspace_modules = [ + _parse_workspace_module(app_dir, workspace_dict, main_module_version) + for workspace_dict in workspace_dict_list + ] + + return main_module, workspace_modules + + +def _process_modules_json_stream( + app_dir: RootedPath, modules_json_stream: str +) -> tuple[ModuleDict, list[ModuleDict]]: + """Process the json stream returned by "go list -m -json". + + The stream will contain the module currently being processed, or a list of all workspaces in + case a go.work file is present in the repository. + + :param app_dir: the path to the module directory + :param modules_json_stream: the json stream returned by "go list -m -json" + :return: A tuple containing the main module and a list of workspaces + """ + module_list = [] + main_module = None + + for module in load_json_stream(modules_json_stream): + if module["Dir"] == str(app_dir): + main_module = module + else: + module_list.append(module) + + # should never happen, since the main module will always be a part of the json stream + if not main_module: + raise RuntimeError('Failed to find the main module info in the "go list -m -json" output.') + + return main_module, module_list + + +def _parse_workspace_module( + app_dir: RootedPath, module: ModuleDict, main_module_version: str +) -> ParsedModule: + """Create a ParsedModule from a listed workspace. + + The replacement info returned will always be relative to the module currently being processed. + """ + # We can't use pathlib since corresponding method PurePath.relative_to('foo', walk_up=True) + # is available only in Python 3.12 and later. + relative_dir = os.path.relpath(module["Dir"], app_dir) + + # We need to prepend "./" to a workspace that is direct child of app_dir so it can be treated + # the same way as a locally replaced module when converting it into a Module. + if not relative_dir.startswith("."): + relative_dir = f"./{relative_dir}" + + replaced_module = ParsedModule(path=relative_dir) + + return ParsedModule( + path=module["Path"], + version=main_module_version, + replace=replaced_module, + ) + + def _parse_go_sum(module_dir: RootedPath) -> frozenset[ModuleID]: """Return the set of modules present in the go.sum file in the specified directory. @@ -899,6 +985,7 @@ def _parse_go_sum(module_dir: RootedPath) -> frozenset[ModuleID]: checksums are not relevant for our purposes. """ go_sum = module_dir.join_within_root("go.sum") + if not go_sum.path.exists(): return frozenset() diff --git a/hack/mock-unittest-data/gomod.sh b/hack/mock-unittest-data/gomod.sh index a3935d633..1539c950d 100755 --- a/hack/mock-unittest-data/gomod.sh +++ b/hack/mock-unittest-data/gomod.sh @@ -25,6 +25,10 @@ $( cd "$tmpdir/gomod-pandemonium" export GOMODCACHE="$tmpdir/cachi2-mock-gomodcache" + echo "generating $mocked_data_dir/non-vendored/go_list_modules.json" + go list -m -json > \ + "$mocked_data_dir_abspath/non-vendored/go_list_modules.json" + echo "generating $mocked_data_dir/non-vendored/go_mod_download.json" go mod download -json > \ "$mocked_data_dir_abspath/non-vendored/go_mod_download.json" diff --git a/tests/unit/data/gomod-mocks/non-vendored/go_list_modules.json b/tests/unit/data/gomod-mocks/non-vendored/go_list_modules.json new file mode 100644 index 000000000..33886c18c --- /dev/null +++ b/tests/unit/data/gomod-mocks/non-vendored/go_list_modules.json @@ -0,0 +1,7 @@ +{ + "Path": "github.com/cachito-testing/gomod-pandemonium", + "Main": true, + "Dir": "{repo_dir}", + "GoMod": "{repo_dir}/go.mod", + "GoVersion": "1.19" +} diff --git a/tests/unit/package_managers/test_gomod.py b/tests/unit/package_managers/test_gomod.py index 2c65fb2a4..3bd78a803 100644 --- a/tests/unit/package_managers/test_gomod.py +++ b/tests/unit/package_managers/test_gomod.py @@ -5,6 +5,7 @@ import subprocess import textwrap from pathlib import Path +from string import Template from textwrap import dedent from typing import Any, Iterator, Optional, Tuple, Union from unittest import mock @@ -21,6 +22,7 @@ from cachi2.core.package_managers.gomod import ( Go, Module, + ModuleDict, ModuleID, ModuleVersionResolver, Package, @@ -34,7 +36,10 @@ _get_gomod_version, _get_repository_name, _parse_go_sum, + _parse_local_modules, _parse_vendor, + _parse_workspace_module, + _process_modules_json_stream, _resolve_gomod, _setup_go_toolchain, _should_vendor_deps, @@ -147,6 +152,8 @@ def test_resolve_gomod( data_dir: Path, gomod_request: Request, ) -> None: + module_dir = gomod_request.source_dir.join_within_root("path/to/module") + # Mock the "subprocess.run" calls run_side_effects = [] run_side_effects.append( @@ -162,7 +169,9 @@ def test_resolve_gomod( proc_mock( "go list -e -mod readonly -m", returncode=0, - stdout="github.com/cachito-testing/gomod-pandemonium", + stdout=get_mocked_data(data_dir, "non-vendored/go_list_modules.json").replace( + "{repo_dir}", str(module_dir) + ), ) ) run_side_effects.append( @@ -253,6 +262,8 @@ def test_resolve_gomod_vendor_dependencies( data_dir: Path, gomod_request: Request, ) -> None: + module_dir = gomod_request.source_dir.join_within_root("path/to/module") + # Mock the "subprocess.run" calls run_side_effects = [] run_side_effects.append(proc_mock("go mod vendor", returncode=0, stdout=None)) @@ -262,7 +273,9 @@ def test_resolve_gomod_vendor_dependencies( proc_mock( "go list -e -m", returncode=0, - stdout="github.com/cachito-testing/gomod-pandemonium", + stdout=get_mocked_data(data_dir, "non-vendored/go_list_modules.json").replace( + "{repo_dir}", str(module_dir) + ), ) ) run_side_effects.append( @@ -291,7 +304,6 @@ def test_resolve_gomod_vendor_dependencies( gomod_request.flags = frozenset(flags) - module_dir = gomod_request.source_dir.join_within_root("path/to/module") module_dir.join_within_root("vendor").path.mkdir(parents=True) module_dir.join_within_root("vendor/modules.txt").path.write_text( get_mocked_data(data_dir, "vendored/modules.txt") @@ -361,6 +373,8 @@ def test_resolve_gomod_no_deps( tmp_path: Path, gomod_request: Request, ) -> None: + module_path = gomod_request.source_dir.join_within_root("path/to/module") + mock_pkg_deps_no_deps = dedent( """ { @@ -373,6 +387,18 @@ def test_resolve_gomod_no_deps( """ ) + mock_go_list_modules = Template( + """ + { + "Path": "github.com/release-engineering/retrodep/v2", + "Main": true, + "Dir": "$repo_dir", + "GoMod": "$repo_dir/go.mod", + "GoVersion": "1.19" + } + """ + ).substitute({"repo_dir": str(module_path)}) + # Mock the "subprocess.run" calls run_side_effects = [] run_side_effects.append(proc_mock("go mod download -json", returncode=0, stdout="")) @@ -382,7 +408,7 @@ def test_resolve_gomod_no_deps( proc_mock( "go list -e -mod readonly -m", returncode=0, - stdout="github.com/release-engineering/retrodep/v2", + stdout=mock_go_list_modules, ) ) run_side_effects.append( @@ -405,7 +431,6 @@ def test_resolve_gomod_no_deps( if force_gomod_tidy: gomod_request.flags = frozenset({"force-gomod-tidy"}) - module_path = gomod_request.source_dir.join_within_root("path/to/module") main_module, modules, packages, _ = _resolve_gomod( module_path, gomod_request, tmp_path, mock_version_resolver ) @@ -517,6 +542,169 @@ def test_parse_broken_go_sum(rooted_tmp_path: RootedPath, caplog: pytest.LogCapt ] +@mock.patch("cachi2.core.package_managers.gomod.Go") +@mock.patch("cachi2.core.package_managers.gomod.ModuleVersionResolver") +def test_parse_local_modules(go: mock.Mock, version_resolver: mock.Mock) -> None: + go.return_value = """ + { + "Path": "myorg.com/my-project", + "Main": true, + "Dir": "/path/to/project" + } + { + "Path": "myorg.com/my-project/workspace", + "Main": true, + "Dir": "/path/to/project/workspace" + } + """ + + app_dir = RootedPath("/path/to/project") + version_resolver.get_golang_version = lambda _, __: "1.0.0" + + main_module, workspace_modules = _parse_local_modules(go, [], {}, app_dir, version_resolver) + + assert main_module == ParsedModule( + path="myorg.com/my-project", + version="1.0.0", + main=True, + ) + + assert workspace_modules[0] == ParsedModule( + path="myorg.com/my-project/workspace", + version="1.0.0", + replace=ParsedModule(path="./workspace"), + ) + + +@pytest.mark.parametrize( + "project_path, stream, expected_modules", + ( + pytest.param( + "/home/my-projects/simple-project", + dedent( + """ + { + "Path": "github.com/my-org/simple-project", + "Main": true, + "Dir": "/home/my-projects/simple-project", + "GoMod": "/home/my-projects/simple-project/go.mod", + "GoVersion": "1.19" + } + """ + ), + ( + { + "Path": "github.com/my-org/simple-project", + "Main": True, + "Dir": "/home/my-projects/simple-project", + "GoMod": "/home/my-projects/simple-project/go.mod", + "GoVersion": "1.19", + }, + [], + ), + id="no_workspaces", + ), + pytest.param( + "/home/my-projects/project-with-workspaces", + dedent( + """ + { + "Path": "github.com/my-org/project-with-workspaces", + "Main": true, + "Dir": "/home/my-projects/project-with-workspaces", + "GoMod": "/home/my-projects/project-with-workspaces/go.mod", + "GoVersion": "1.19" + } + { + "Path": "github.com/my-org/work", + "Main": true, + "Dir": "/home/my-projects/project-with-workspaces/work", + "GoMod": "/home/my-projects/project-with-workspaces/work/go.mod" + } + { + "Path": "github.com/my-org/space", + "Main": true, + "Dir": "/home/my-projects/project-with-workspaces/space", + "GoMod": "/home/my-projects/project-with-workspaces/space/go.mod" + } + """ + ), + ( + { + "Path": "github.com/my-org/project-with-workspaces", + "Main": True, + "Dir": "/home/my-projects/project-with-workspaces", + "GoMod": "/home/my-projects/project-with-workspaces/go.mod", + "GoVersion": "1.19", + }, + [ + { + "Path": "github.com/my-org/work", + "Main": True, + "Dir": "/home/my-projects/project-with-workspaces/work", + "GoMod": "/home/my-projects/project-with-workspaces/work/go.mod", + }, + { + "Path": "github.com/my-org/space", + "Main": True, + "Dir": "/home/my-projects/project-with-workspaces/space", + "GoMod": "/home/my-projects/project-with-workspaces/space/go.mod", + }, + ], + ), + id="with_workspaces", + ), + ), +) +def test_process_modules_json_stream( + project_path: str, + stream: str, + expected_modules: tuple[ModuleDict, list[ModuleDict]], +) -> None: + app_dir = RootedPath(project_path) + result = _process_modules_json_stream(app_dir, stream) + + assert result == expected_modules + + +@pytest.mark.parametrize( + "relative_app_dir, module, expected_module", + ( + # main module is also the workspace root: + pytest.param( + ".", + {"Dir": "workspace", "Path": "example.com/myproject/workspace"}, + ParsedModule( + path="example.com/myproject/workspace", + version="0.0.1", + replace=ParsedModule(path="./workspace"), + ), + id="workspace_root_is_a_go_module", + ), + # main module and workspace are inside the workspace root: + pytest.param( + "mainmod", + {"Dir": "workspace", "Path": "example.com/myproject/workspace"}, + ParsedModule( + path="example.com/myproject/workspace", + version="0.0.1", + replace=ParsedModule(path="../workspace"), + ), + id="only_nested_workspaces", + ), + ), +) +def test_parse_workspace_modules( + relative_app_dir: str, module: dict[str, Any], expected_module: ParsedModule, tmp_path: Path +) -> None: + app_dir = RootedPath(tmp_path).join_within_root(relative_app_dir) + # makes Dir an absolute path based on tmp_path + module["Dir"] = str(tmp_path.joinpath(module["Dir"])) + + parsed_workspace = _parse_workspace_module(app_dir, module, "0.0.1") + assert parsed_workspace == expected_module + + @mock.patch("cachi2.core.package_managers.gomod.ModuleVersionResolver") def test_create_modules_from_parsed_data(mock_version_resolver: mock.Mock, tmp_path: Path) -> None: main_module_dir = RootedPath(tmp_path).join_within_root("target-module") @@ -821,7 +1009,7 @@ def test_go_list_cmd_failure( expect_error = "Go execution failed: " if go_mod_rc == 0: - expect_error += "`go list -e -mod readonly -m` failed with rc=1" + expect_error += "`go list -e -mod readonly -m -json` failed with rc=1" else: expect_error += "Cachi2 re-tried running `go mod download -json` command 1 times."