diff --git a/docs/notes/2.34.x.md b/docs/notes/2.34.x.md index 161da74f163..d037fab9e1b 100644 --- a/docs/notes/2.34.x.md +++ b/docs/notes/2.34.x.md @@ -115,6 +115,8 @@ Fixed a bug that could cause tool execution to fail when resolving the tool from Required named indexes to be referenced explicitly in requirement sources when using the uv resolver, to match the pex resolver's behavior. +Local `[python-repos].find_links` directories under the buildroot no longer tie `uv` lockfiles to one checkout: they are now recorded as buildroot-relative paths. Builds using wheels from such directories also now get the same cache keys across checkouts and machines. + Sidecar lockfile metadata files (`*.lock.metadata`, written when `[python].separate_lockfile_metadata_file` is enabled) now end with a trailing newline. The bundled tool metadata files have been updated accordingly, so regenerating them no longer produces a spurious diff. Interpreter constraints can now select a specific CPython ABI, to distinguish the free-threaded (no-GIL) build from the standard build. Qualify the interpreter type using either a PEP 508 extra, `CPython[free-threaded]` or `CPython[gil]`, or the equivalent `CPython+t` / `CPython-t` spellings. Both spellings are adopted from Pex. For example, `interpreter_constraints=["CPython[free-threaded]==3.14.*"]` matches only free-threaded 3.14 interpreters, while `CPython==3.14.*` continues to match either ABI. diff --git a/src/python/pants/backend/python/goals/lockfile.py b/src/python/pants/backend/python/goals/lockfile.py index 7e81a948fb8..28f6a2fa918 100644 --- a/src/python/pants/backend/python/goals/lockfile.py +++ b/src/python/pants/backend/python/goals/lockfile.py @@ -6,7 +6,7 @@ import itertools import os.path from collections import defaultdict -from dataclasses import dataclass +from dataclasses import dataclass, replace from operator import itemgetter from typing import cast @@ -43,7 +43,12 @@ load_lockfile, strip_comments_from_pex_json_lockfile, ) -from pants.backend.python.util_rules.uv import UvEnvironment, generate_pyproject_toml +from pants.backend.python.util_rules.uv import ( + UvEnvironment, + generate_pyproject_toml, + relativize_local_find_links, +) +from pants.base.build_root import BuildRoot from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior from pants.core.goals.generate_lockfiles import ( DEFAULT_TOOL_LOCKFILE, @@ -351,6 +356,7 @@ async def generate_uv_lockfile( generate_lockfiles_subsystem: GenerateLockfilesSubsystem, downloaded_uv: DownloadedUv, uv_env: UvEnvironment, + buildroot: BuildRoot, level: LogLevel, ) -> GenerateLockfileResult: if not req.interpreter_constraints: @@ -364,12 +370,18 @@ async def generate_uv_lockfile( ) resolve_config.validate_for_uv(req.resolve_name) + resolve_find_links, resolve_local_find_links_globs = relativize_local_find_links( + resolve_config.find_links, buildroot.path + ) + extra_find_links, extra_local_find_links_globs = relativize_local_find_links( + req.find_links, buildroot.path + ) pyproject_content = generate_pyproject_toml( req.resolve_name, req.interpreter_constraints, req.requirements, indexes=resolve_config.indexes, - find_links=(*resolve_config.find_links, *req.find_links), + find_links=(*resolve_find_links, *extra_find_links), sources=resolve_config.sources, ) @@ -408,7 +420,7 @@ async def generate_uv_lockfile( # is generated. pass - uv_config = resolve_config.uv_config() + uv_config = replace(resolve_config, find_links=resolve_find_links).uv_config() uv_config_digest = await create_digest( CreateDigest( @@ -418,9 +430,13 @@ async def generate_uv_lockfile( ] ) ) - + find_links_digest = await path_globs_to_digest( + PathGlobs((*resolve_local_find_links_globs.globs, *extra_local_find_links_globs.globs)) + ) input_digest = await merge_digests( - MergeDigests([downloaded_uv.digest, uv_config_digest, existing_uv_lock_digest]) + MergeDigests( + [downloaded_uv.digest, uv_config_digest, existing_uv_lock_digest, find_links_digest] + ) ) result = await execute_process_or_raise( diff --git a/src/python/pants/backend/python/util_rules/pex.py b/src/python/pants/backend/python/util_rules/pex.py index 6a9bf23ac41..837d1661020 100644 --- a/src/python/pants/backend/python/util_rules/pex.py +++ b/src/python/pants/backend/python/util_rules/pex.py @@ -724,6 +724,9 @@ async def _setup_pex_requirements( concurrency_available, ) + if not reqs_info.req_strings: + return _BuildPexRequirementsSetup([], [], concurrency_available) + # We use pip to perform a normal resolve. digests = [] argv = [ diff --git a/src/python/pants/backend/python/util_rules/pex_test.py b/src/python/pants/backend/python/util_rules/pex_test.py index 97234d7d1d9..9caefae6ab2 100644 --- a/src/python/pants/backend/python/util_rules/pex_test.py +++ b/src/python/pants/backend/python/util_rules/pex_test.py @@ -950,6 +950,9 @@ def assert_setup( ), ) + # Nothing to resolve, so no repository args that could make the cache key path-dependent. + assert_setup(PexRequirements(), _BuildPexRequirementsSetup([], [], 0)) + # Pex lockfile. assert_setup( EntireLockfile(lockfile_obj, complete_req_strings=reqs), diff --git a/src/python/pants/backend/python/util_rules/uv.py b/src/python/pants/backend/python/util_rules/uv.py index f9e695ec108..814d59b5f6a 100644 --- a/src/python/pants/backend/python/util_rules/uv.py +++ b/src/python/pants/backend/python/util_rules/uv.py @@ -32,6 +32,7 @@ determine_resolve_config, generate_uv_index_config, ) +from pants.base.build_root import BuildRoot from pants.core.util_rules import system_binaries from pants.core.util_rules.env_vars import environment_vars_subset from pants.core.util_rules.subprocess_environment import SubprocessEnvironmentVars @@ -47,11 +48,13 @@ CreateDigest, FileContent, MergeDigests, + PathGlobs, ) from pants.engine.intrinsics import ( create_digest, get_digest_contents, merge_digests, + path_globs_to_digest, ) from pants.engine.pants_lock import pants_lock_bin from pants.engine.rules import collect_rules, concurrently, implicitly, rule @@ -93,6 +96,35 @@ def append_only_caches(cls) -> FrozenDict[str, str]: return FrozenDict({cls.cache_name: cls.cache_dir}) +def relativize_local_find_links( + find_links: Iterable[str], buildroot: str +) -> tuple[tuple[str, ...], PathGlobs]: + """Rewrite find-links dirs under the buildroot to buildroot-relative paths. + + uv records find-links paths verbatim in uv.lock, so absolute paths would tie the lockfile (and + every process that consumes it) to one checkout location. Returns the rewritten find-links and + the globs for the files of those dirs, which must be in the uv process sandbox. + """ + rewritten: list[str] = [] + local_globs: list[str] = [] + for find_link in find_links: + if find_link.startswith("file://"): + path = find_link[len("file://") :] + elif os.path.isabs(find_link): + path = find_link + else: + rewritten.append(find_link) + continue + path = os.path.normpath(path) + if os.path.isabs(path) and os.path.commonpath((path, buildroot)) == buildroot: + relative_dir = os.path.relpath(path, buildroot) + rewritten.append(relative_dir) + local_globs.append(os.path.join(relative_dir, "*")) + else: + rewritten.append(find_link) + return tuple(rewritten), PathGlobs(local_globs) + + @dataclass(frozen=True) class UvEnvironment: env: FrozenDict[str, str] @@ -189,6 +221,7 @@ async def create_venv_repository_from_uv_lockfile( realpath_binary: RealpathBinary, rm_binary: RmBinary, maybe_flock_binary: MaybeFlockBinary, + buildroot: BuildRoot, level: LogLevel, ) -> VenvRepository: """Install all packages from a uv lockfile into a virtualenv.""" @@ -209,6 +242,9 @@ async def create_venv_repository_from_uv_lockfile( ResolveConfigRequest(metadata.resolve), **implicitly() ) + find_links, local_find_links_globs = relativize_local_find_links( + resolve_config.find_links, buildroot.path + ) pyproject_content = generate_pyproject_toml( metadata.resolve, metadata.valid_for_interpreter_constraints, @@ -222,7 +258,7 @@ async def create_venv_repository_from_uv_lockfile( # they don't involve auth, so they are not strictly needed after lockfile generation. # TODO: Get rid of "extra find_links" entirely, and simply hard-code an explicit # source for pants_requirements. - find_links=resolve_config.find_links, + find_links=find_links, sources=resolve_config.sources, ) @@ -238,8 +274,9 @@ async def create_venv_repository_from_uv_lockfile( ), get_digest_contents(request.lockfile.lockfile_digest), ) - uv_lock_digest = await create_digest( - CreateDigest([FileContent("uv.lock", uv_lock_contents[0].content)]) + uv_lock_digest, find_links_digest = await concurrently( + create_digest(CreateDigest([FileContent("uv.lock", uv_lock_contents[0].content)])), + path_globs_to_digest(local_find_links_globs), ) input_digest = await merge_digests( @@ -248,6 +285,7 @@ async def create_venv_repository_from_uv_lockfile( downloaded_uv.digest, uv_config_digest, uv_lock_digest, + find_links_digest, ) ) ) @@ -265,6 +303,7 @@ async def create_venv_repository_from_uv_lockfile( entropy.update((request.uv_platform or "").encode()) entropy.update(b"\0") entropy.update(input_digest.fingerprint.encode()) + entropy.update(b"\0UV_NO_INSTALLER_METADATA=1") venv_path_suffix = entropy.hexdigest() uv_cmd = shlex.join( @@ -343,7 +382,7 @@ async def create_venv_repository_from_uv_lockfile( creation_subprocess=Subprocess( command=command, input_digest=input_digest, - env=uv_env.env, + env=FrozenDict({**uv_env.env, "UV_NO_INSTALLER_METADATA": "1"}), append_only_caches={ **downloaded_uv.append_only_caches(), **VenvRepository.append_only_caches(), diff --git a/src/python/pants/backend/python/util_rules/uv_test.py b/src/python/pants/backend/python/util_rules/uv_test.py index a75455495dc..6bb7edc93aa 100644 --- a/src/python/pants/backend/python/util_rules/uv_test.py +++ b/src/python/pants/backend/python/util_rules/uv_test.py @@ -3,7 +3,9 @@ from __future__ import annotations +import io import sys +import zipfile from pathlib import Path from pants.backend.python.goals.lockfile import GenerateUvLockfile @@ -21,6 +23,7 @@ VenvFromUvLockfileRequest, VenvRepository, generate_pyproject_toml, + relativize_local_find_links, ) from pants.core.environments.target_types import DockerEnvironmentTarget, LocalEnvironmentTarget from pants.core.goals.generate_lockfiles import GenerateLockfileResult @@ -29,15 +32,20 @@ from pants.engine import composite_process from pants.engine.composite_process import CompositeProcess from pants.engine.environment import EnvironmentName -from pants.engine.fs import Digest, DigestContents +from pants.engine.fs import Digest, DigestContents, PathGlobs from pants.engine.internals.native_engine import FrozenOrderedSet from pants.engine.process import FallibleProcessResult, Process, ProcessCacheScope from pants.testutil.rule_runner import QueryRule, RuleRunner +from pants.util.pip_requirement import PipRequirement RESOLVE_NAME = "test-resolve" -def _make_rule_runner(docker_image: str | None, named_caches_dir: Path | None = None) -> RuleRunner: +def _make_rule_runner( + docker_image: str | None, + named_caches_dir: Path | None = None, + local_find_links: tuple[str, ...] = (), +) -> RuleRunner: rule_runner = RuleRunner( bootstrap_args=[f"--named-caches-dir={named_caches_dir}"] if named_caches_dir else [], rules=[ @@ -58,6 +66,9 @@ def _make_rule_runner(docker_image: str | None, named_caches_dir: Path | None = inherent_environment=EnvironmentName("test-env") if docker_image else EnvironmentName(None), ) args = [f"--python-resolves={{'{RESOLVE_NAME}': 'uv.lock'}}"] + if local_find_links: + find_links = [f"file://{rule_runner.build_root}/{path}" for path in local_find_links] + args.append(f"--python-repos-find-links={find_links}") if docker_image: rule_runner.write_files( {"BUILD": f"docker_environment(name='test-env', image='{docker_image}')"} @@ -71,14 +82,32 @@ def _make_rule_runner(docker_image: str | None, named_caches_dir: Path | None = return rule_runner +def _generate_uv_lockfile(rule_runner: RuleRunner, requirements: tuple[str, ...] = ()) -> bytes: + lock_result = rule_runner.request( + GenerateLockfileResult, + [ + GenerateUvLockfile( + resolve_name=RESOLVE_NAME, + lockfile_dest="test.lock", + diff=False, + requirements=FrozenOrderedSet(requirements), + find_links=FrozenOrderedSet(), + interpreter_constraints=InterpreterConstraints.for_fixed_python_version("3.14.*"), + ) + ], + ) + return rule_runner.request(DigestContents, [lock_result.digest])[0].content + + def _create_venv_repository( rule_runner: RuleRunner, python_path: str, cache_scope: ProcessCacheScope = ProcessCacheScope.SUCCESSFUL, + requirements: tuple[str, ...] = (), ) -> tuple[VenvRepository, FallibleProcessResult]: metadata = PythonLockfileMetadataV8( valid_for_interpreter_constraints=InterpreterConstraints(["CPython==3.14.*"]), - requirements=set(), + requirements={PipRequirement.parse(requirement) for requirement in requirements}, manylinux=None, requirement_constraints=set(), only_binary=set(), @@ -94,23 +123,10 @@ def _create_venv_repository( ) pyproject = generate_pyproject_toml( - RESOLVE_NAME, metadata.valid_for_interpreter_constraints, () + RESOLVE_NAME, metadata.valid_for_interpreter_constraints, requirements ) rule_runner.write_files({"pyproject.toml": pyproject, "uv.toml": ""}) - lock_result = rule_runner.request( - GenerateLockfileResult, - [ - GenerateUvLockfile( - resolve_name=RESOLVE_NAME, - lockfile_dest="test.lock", - diff=False, - requirements=FrozenOrderedSet(), - find_links=FrozenOrderedSet(), - interpreter_constraints=InterpreterConstraints.for_fixed_python_version("3.14.*"), - ) - ], - ) - uv_lock_content = rule_runner.request(DigestContents, [lock_result.digest])[0].content + uv_lock_content = _generate_uv_lockfile(rule_runner, requirements) lockfile_digest = rule_runner.make_snapshot({"test.lock": uv_lock_content.decode()}).digest loaded_lockfile = LoadedLockfile( lockfile_digest, @@ -189,3 +205,65 @@ def test_existing_venv_is_reused(tmp_path: Path) -> None: ) assert result.exit_code == 0, result.stderr.decode() assert sentinel.exists(), "The venv was recreated instead of being reused." + + +def test_relativize_local_find_links() -> None: + assert relativize_local_find_links( + [ + "file:///repo/wheels", + "/repo/vendored/wheels", + "file:///repository/wheels", + "/elsewhere/wheels", + "https://example.com/wheels/", + ], + "/repo", + ) == ( + ( + "wheels", + "vendored/wheels", + "file:///repository/wheels", + "/elsewhere/wheels", + "https://example.com/wheels/", + ), + PathGlobs(["wheels/*", "vendored/wheels/*"]), + ) + + +def _write_local_wheel(rule_runner: RuleRunner, directory: str) -> None: + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as wheel: + wheel.writestr("my_wheel/__init__.py", "") + wheel.writestr( + "my_wheel-0.1.0.dist-info/METADATA", + "Metadata-Version: 2.1\nName: my-wheel\nVersion: 0.1.0\n", + ) + wheel.writestr( + "my_wheel-0.1.0.dist-info/WHEEL", + "Wheel-Version: 1.0\nGenerator: pants-test\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + ) + wheel.writestr("my_wheel-0.1.0.dist-info/RECORD", "") + rule_runner.write_files({f"{directory}/my_wheel-0.1.0-py3-none-any.whl": buffer.getvalue()}) + + +def test_local_find_links_are_portable(tmp_path: Path) -> None: + named_caches_dir = tmp_path / "named_caches" + rule_runner = _make_rule_runner( + docker_image=None, named_caches_dir=named_caches_dir, local_find_links=("wheels",) + ) + _write_local_wheel(rule_runner, "wheels") + requirements = ("my-wheel==0.1.0",) + + uv_lock_content = _generate_uv_lockfile(rule_runner, requirements).decode() + assert 'source = { registry = "wheels" }' in uv_lock_content + assert rule_runner.build_root not in uv_lock_content + + venv_repo, result = _create_venv_repository( + rule_runner, + sys.executable, + cache_scope=ProcessCacheScope.PER_SESSION, + requirements=requirements, + ) + assert result.exit_code == 0, result.stderr.decode() + venv_dir = (named_caches_dir / VenvRepository.cache_name / venv_repo.venv_path_suffix).resolve() + assert list(venv_dir.glob("lib/python*/site-packages/my_wheel-0.1.0.dist-info")) + assert not list(venv_dir.rglob("uv_cache.json"))