From 4ba91fc726a9352303c4e0a82ddad5fcc448d9b4 Mon Sep 17 00:00:00 2001 From: Daniel Goldman Date: Wed, 13 Nov 2024 14:49:17 -0500 Subject: [PATCH 1/4] do not expose resolves for tools not using their default resolve for example, `black` exposes the resolve "black" so that `pants export --resolve=black` works. But if `[black].install_from_resolve` is set to "my_resolve", we shouldn't expose the resolve "black" --- src/python/pants/backend/python/goals/lockfile.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/python/pants/backend/python/goals/lockfile.py b/src/python/pants/backend/python/goals/lockfile.py index c38b91dfa96..2c70ce0996f 100644 --- a/src/python/pants/backend/python/goals/lockfile.py +++ b/src/python/pants/backend/python/goals/lockfile.py @@ -211,7 +211,7 @@ class KnownPythonUserResolveNamesRequest(KnownUserResolveNamesRequest): @rule -def determine_python_user_resolves( +async def determine_python_user_resolves( _: KnownPythonUserResolveNamesRequest, python_setup: PythonSetup, union_membership: UnionMembership, @@ -219,10 +219,16 @@ def determine_python_user_resolves( """Find all know Python resolves, from both user-created resolves and internal tools.""" python_tool_resolves = ExportableTool.filter_for_subclasses(union_membership, PythonToolBase) + tools_using_default_resolve = [ + resolve_name + for resolve_name, subsystem_cls in python_tool_resolves.items() + if (await _construct_subsystem(subsystem_cls)).install_from_resolve is None + ] + return KnownUserResolveNames( names=( *python_setup.resolves.keys(), - *python_tool_resolves.keys(), + *tools_using_default_resolve, ), # the order of the keys doesn't matter since shadowing is done in `setup_user_lockfile_requests` option_name="[python].resolves", requested_resolve_names_cls=RequestedPythonUserResolveNames, From 591a00fe2affa611d33f1cc056958e9391f4b7a5 Mon Sep 17 00:00:00 2001 From: Daniel Goldman Date: Wed, 13 Nov 2024 17:52:11 -0500 Subject: [PATCH 2/4] deduplicate identifying matching Python requests the request is selected twice, but they're identical because they select by name. We can just use a set to deduplicate them --- src/python/pants/backend/python/goals/lockfile.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/python/pants/backend/python/goals/lockfile.py b/src/python/pants/backend/python/goals/lockfile.py index 2c70ce0996f..d25a6004a50 100644 --- a/src/python/pants/backend/python/goals/lockfile.py +++ b/src/python/pants/backend/python/goals/lockfile.py @@ -261,10 +261,10 @@ async def setup_user_lockfile_requests( tools = ExportableTool.filter_for_subclasses(union_membership, PythonToolBase) - out = [] + out = set() for resolve in requested: if resolve in python_setup.resolves: - out.append( + out.add( GeneratePythonLockfile( requirements=PexRequirements.req_strings_from_requirement_fields( resolve_to_requirements_fields[resolve] @@ -291,7 +291,7 @@ async def setup_user_lockfile_requests( else: ic = InterpreterConstraints(tool.default_interpreter_constraints) - out.append( + out.add( GeneratePythonLockfile( requirements=FrozenOrderedSet(sorted(tool.requirements)), find_links=FrozenOrderedSet(find_links), From a9cfbaf765c6d882684d0dc0a1464d1fda738785 Mon Sep 17 00:00:00 2001 From: Daniel Goldman Date: Wed, 13 Nov 2024 17:56:41 -0500 Subject: [PATCH 3/4] use deduplicated collection to deduplicate requests A resolve name could be provided by both users and tools, so it was possible to have the same requested name twice. This fixes that by deduplicating them in the request. --- src/python/pants/core/goals/generate_lockfiles.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/python/pants/core/goals/generate_lockfiles.py b/src/python/pants/core/goals/generate_lockfiles.py index db6193fdaed..850217e7c0d 100644 --- a/src/python/pants/core/goals/generate_lockfiles.py +++ b/src/python/pants/core/goals/generate_lockfiles.py @@ -11,7 +11,7 @@ from typing import Callable, Iterable, Iterator, Mapping, Protocol, Sequence, Tuple, Type, cast from pants.core.goals.resolves import ExportableTool -from pants.engine.collection import Collection +from pants.engine.collection import Collection, DeduplicatedCollection from pants.engine.console import Console from pants.engine.environment import ChosenLocalEnvironmentName, EnvironmentName from pants.engine.fs import Digest, MergeDigests, Workspace @@ -105,13 +105,15 @@ class KnownUserResolveNames: @union(in_scope_types=[EnvironmentName]) -class RequestedUserResolveNames(Collection[str]): +class RequestedUserResolveNames(DeduplicatedCollection[str]): """The user resolves requested for a particular language ecosystem. Each language ecosystem should set up a subclass and register it with a UnionRule. Implement a rule that goes from the subclass -> UserGenerateLockfiles. """ + sort_input = True + class PackageVersion(Protocol): """Protocol for backend specific implementations, to support language-ecosystem-specific version From d6b9bb1a06391d767cb726ce6c48fe5efa5b4393 Mon Sep 17 00:00:00 2001 From: Daniel Goldman Date: Wed, 13 Nov 2024 18:31:35 -0500 Subject: [PATCH 4/4] add changelogs --- docs/notes/2.24.x.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/notes/2.24.x.md b/docs/notes/2.24.x.md index 1c795cd127f..c18c3fcabe5 100644 --- a/docs/notes/2.24.x.md +++ b/docs/notes/2.24.x.md @@ -40,6 +40,10 @@ The "legacy" system will be removed in the 2.25.x series. Many tools that Pants downloads can now be exported using [the new `export --bin` option](https://www.pantsbuild.org/2.24/reference/goals/export#bin). For example, `pants export --bin="helm"` will export the `helm` binary to `dist/export/bin/helm`. For each tool, all the files are exported to a subfolder in `dist/export/bins/`, and the main executable is linked to `dist/export/bin/`. +#### `generate-lockfiles` + +Generating all lockfiles properly deduplicates requests, preventing generating the same lockfile twice for resolves that shadow a tool's builtin resolve name. ([#21642](https://github.com/pantsbuild/pants/pull/21642)) + ### Backends #### JVM @@ -88,6 +92,8 @@ A new experimental [Python Provider](https://www.pantsbuild.org/blog/2023/03/31/ Mypy will now typecheck previously-ignored python sources without a `.py` or `.pyi` extension. +Tools that are installed from a user resolve will not present their default resolve for `generate-lockfiles` and `export`. + #### S3 The `pants.backend.url_handlers.s3` backend now correctly passes along query parameters such as `versionId` for `s3://` urls.