Fix lockfile generation for shadowed resolves

docs/notes/2.24.x.md

@@ -40,6 +40,10 @@ The "legacy" system will be removed in the 2.25.x series.
 
 Many tools that Pants downloads can now be exported using [the new `export --bin` option](https://www.pantsbuild.org/2.24/reference/goals/export#bin). For example, `pants export --bin="helm"` will export the `helm` binary to `dist/export/bin/helm`. For each tool, all the files are exported to a subfolder in `dist/export/bins/`, and the main executable is linked to `dist/export/bin/`.
 
+#### `generate-lockfiles`
+
+Generating all lockfiles properly deduplicates requests, preventing generating the same lockfile twice for resolves that shadow a tool's builtin resolve name. ([#21642](https://github.com/pantsbuild/pants/pull/21642))
+
 ### Backends
 
 #### JVM
@@ -88,6 +92,8 @@ A new experimental [Python Provider](https://www.pantsbuild.org/blog/2023/03/31/
 
 Mypy will now typecheck previously-ignored python sources without a `.py` or `.pyi` extension.
 
+Tools that are installed from a user resolve will not present their default resolve for `generate-lockfiles` and `export`.
+
 #### S3
 
 The `pants.backend.url_handlers.s3` backend now correctly passes along query parameters such as `versionId` for `s3://` urls.

src/python/pants/backend/python/goals/lockfile.py

@@ -211,18 +211,24 @@ class KnownPythonUserResolveNamesRequest(KnownUserResolveNamesRequest):
 
 
 @rule
-def determine_python_user_resolves(
+async def determine_python_user_resolves(
     _: KnownPythonUserResolveNamesRequest,
     python_setup: PythonSetup,
     union_membership: UnionMembership,
 ) -> KnownUserResolveNames:
     """Find all know Python resolves, from both user-created resolves and internal tools."""
     python_tool_resolves = ExportableTool.filter_for_subclasses(union_membership, PythonToolBase)
 
+    tools_using_default_resolve = [
+        resolve_name
+        for resolve_name, subsystem_cls in python_tool_resolves.items()
+        if (await _construct_subsystem(subsystem_cls)).install_from_resolve is None
+    ]
+
     return KnownUserResolveNames(
         names=(
             *python_setup.resolves.keys(),
-            *python_tool_resolves.keys(),
+            *tools_using_default_resolve,
         ),  # the order of the keys doesn't matter since shadowing is done in `setup_user_lockfile_requests`
         option_name="[python].resolves",
         requested_resolve_names_cls=RequestedPythonUserResolveNames,
@@ -255,10 +261,10 @@ async def setup_user_lockfile_requests(
 
     tools = ExportableTool.filter_for_subclasses(union_membership, PythonToolBase)
 
-    out = []
+    out = set()
     for resolve in requested:
         if resolve in python_setup.resolves:
-            out.append(
+            out.add(
                 GeneratePythonLockfile(
                     requirements=PexRequirements.req_strings_from_requirement_fields(
                         resolve_to_requirements_fields[resolve]
@@ -285,7 +291,7 @@ async def setup_user_lockfile_requests(
             else:
                 ic = InterpreterConstraints(tool.default_interpreter_constraints)
 
-            out.append(
+            out.add(
                 GeneratePythonLockfile(
                     requirements=FrozenOrderedSet(sorted(tool.requirements)),
                     find_links=FrozenOrderedSet(find_links),

src/python/pants/core/goals/generate_lockfiles.py

@@ -11,7 +11,7 @@
 from typing import Callable, Iterable, Iterator, Mapping, Protocol, Sequence, Tuple, Type, cast
 
 from pants.core.goals.resolves import ExportableTool
-from pants.engine.collection import Collection
+from pants.engine.collection import Collection, DeduplicatedCollection
 from pants.engine.console import Console
 from pants.engine.environment import ChosenLocalEnvironmentName, EnvironmentName
 from pants.engine.fs import Digest, MergeDigests, Workspace
@@ -105,13 +105,15 @@ class KnownUserResolveNames:
 
 
 @union(in_scope_types=[EnvironmentName])
-class RequestedUserResolveNames(Collection[str]):
+class RequestedUserResolveNames(DeduplicatedCollection[str]):
     """The user resolves requested for a particular language ecosystem.
 
     Each language ecosystem should set up a subclass and register it with a UnionRule. Implement a
     rule that goes from the subclass -> UserGenerateLockfiles.
     """
 
+    sort_input = True
+
 
 class PackageVersion(Protocol):
     """Protocol for backend specific implementations, to support language-ecosystem-specific version