Add support for -r pylock.toml

Author: sbidoul

src/pip/_internal/cli/req_command.py

@@ -11,8 +11,11 @@
 import os
 from functools import partial
 from optparse import Values
+from pathlib import Path
 from typing import Any, Callable, TypeVar
 
+from pip._vendor.packaging import pylock
+
 from pip._internal.build_env import (
     BuildEnvironmentInstaller,
     InprocessBuildEnvironmentInstaller,
@@ -39,6 +42,7 @@
     install_req_from_editable,
     install_req_from_line,
     install_req_from_parsed_requirement,
+    install_req_from_pylock_package,
     install_req_from_req_string,
 )
 from pip._internal.req.pep723 import PEP723Exception, pep723_metadata
@@ -47,6 +51,7 @@
 from pip._internal.req.req_install import InstallRequirement
 from pip._internal.resolution.base import BaseResolver
 from pip._internal.utils.packaging import check_requires_python
+from pip._internal.utils.pylock import select_from_pylock_path_or_url
 from pip._internal.utils.temp_dir import (
     TempDirectory,
     TempDirectoryTypeRegistry,
@@ -332,6 +337,29 @@ def get_requirements(
 
         # NOTE: options.require_hashes may be set if --require-hashes is True
         for filename in options.requirements:
+            if pylock.is_valid_pylock_path(Path(filename)):
+                if any(
+                    [
+                        options.python_version,
+                        options.platforms,
+                        options.abis,
+                        options.implementation,
+                    ]
+                ):
+                    raise CommandError(
+                        "Patform and interpreter constraints using "
+                        "--python-version, --platform, --abi, or --implementation, "
+                        f"are not supported when installing from {filename!r}"
+                    )
+                for package, package_dist in select_from_pylock_path_or_url(
+                    filename, session=session
+                ):
+                    requirements.append(
+                        install_req_from_pylock_package(
+                            package, package_dist, filename, options.format_control
+                        )
+                    )
+                continue
             for parsed_req in parse_requirements(
                 filename, finder=finder, options=options, session=session
             ):

src/pip/_internal/req/constructors.py

@@ -14,13 +14,16 @@
 import logging
 import os
 import re
-from collections.abc import Collection
+from collections.abc import Collection, Mapping
 from dataclasses import dataclass
 
+from pip._vendor.packaging import pylock
 from pip._vendor.packaging.markers import Marker
 from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
+from pip._vendor.packaging.utils import parse_sdist_filename, parse_wheel_filename
 
 from pip._internal.exceptions import InstallationError
+from pip._internal.models.format_control import FormatControl
 from pip._internal.models.index import PyPI, TestPyPI
 from pip._internal.models.link import Link
 from pip._internal.models.wheel import Wheel
@@ -29,6 +32,13 @@
 from pip._internal.utils.filetypes import is_archive_file
 from pip._internal.utils.misc import is_installable_dir
 from pip._internal.utils.packaging import get_requirement
+from pip._internal.utils.pylock import (
+    package_archive_requirement_url,
+    package_directory_requirement_url,
+    package_sdist_requirement_url,
+    package_vcs_requirement_url,
+    package_wheel_requirement_url,
+)
 from pip._internal.utils.urls import path_to_url
 from pip._internal.vcs import is_url, vcs
 
@@ -568,3 +578,83 @@ def install_req_extend_extras(
         else None
     )
     return result
+
+
+def _pylock_hashes_to_hash_options(hashes: Mapping[str, str]) -> dict[str, list[str]]:
+    return {k: [v] for k, v in hashes.items()}
+
+
+def install_req_from_pylock_package(
+    package: pylock.Package,
+    package_dist: (
+        pylock.PackageVcs
+        | pylock.PackageArchive
+        | pylock.PackageDirectory
+        | pylock.PackageSdist
+        | pylock.PackageWheel
+    ),
+    pylock_path_or_url: str,
+    format_control: FormatControl,
+) -> InstallRequirement:
+    pass
+    # TODO: user_supplied
+    # TODO: validate file size
+    if isinstance(package_dist, pylock.PackageVcs):
+        return InstallRequirement(
+            req=Requirement(
+                f"{package.name} @ "
+                f"{package_vcs_requirement_url(pylock_path_or_url, package_dist)}"
+            ),
+            comes_from=pylock_path_or_url,
+        )
+    elif isinstance(package_dist, pylock.PackageArchive):
+        return InstallRequirement(
+            req=Requirement(
+                f"{package.name} @ "
+                f"{package_archive_requirement_url(pylock_path_or_url, package_dist)}"
+            ),
+            comes_from=pylock_path_or_url,
+            hash_options=_pylock_hashes_to_hash_options(package_dist.hashes),
+        )
+    elif isinstance(package_dist, pylock.PackageDirectory):
+        if package_dist.editable:
+            return install_req_from_editable(
+                package_directory_requirement_url(pylock_path_or_url, package_dist),
+                comes_from=pylock_path_or_url,
+            )
+        else:
+            return install_req_from_line(
+                package_directory_requirement_url(pylock_path_or_url, package_dist),
+                comes_from=pylock_path_or_url,
+            )
+    else:
+        # wheel or sdist
+        allow_binary = "binary" in format_control.get_allowed_formats(package.name)
+        if isinstance(package_dist, pylock.PackageWheel) and not allow_binary:
+            if not package.sdist:
+                raise InstallationError(
+                    f"binaries are not permitted for package {package.name!r} and "
+                    f"there is no source distribution for it in {pylock_path_or_url!r}"
+                )
+            package_dist = package.sdist
+        version = package.version
+        if isinstance(package_dist, pylock.PackageWheel):
+            if not version:
+                _, version, _, _ = parse_wheel_filename(package_dist.filename)
+            requirement_url = package_wheel_requirement_url(
+                pylock_path_or_url, package_dist
+            )
+        elif isinstance(package_dist, pylock.PackageSdist):
+            if not version:
+                _, version = parse_sdist_filename(package_dist.filename)
+            requirement_url = package_sdist_requirement_url(
+                pylock_path_or_url, package_dist
+            )
+        ireq = InstallRequirement(
+            req=Requirement(f"{package.name}=={version}"),
+            comes_from=pylock_path_or_url,
+            link=Link(requirement_url),
+            hash_options=_pylock_hashes_to_hash_options(package_dist.hashes),
+        )
+        ireq.original_link = None  # not a direct URL
+        return ireq

src/pip/_internal/utils/pylock.py

@@ -1,5 +1,11 @@
-from collections.abc import Iterable
+from __future__ import annotations
+
+import os
+import re
+from collections.abc import Iterable, Iterator
 from pathlib import Path
+from typing import TYPE_CHECKING
+from urllib.parse import urljoin, urlsplit
 
 from pip._vendor.packaging.pylock import (
     Package,
@@ -15,7 +21,11 @@
 from pip._internal.models.direct_url import ArchiveInfo, DirInfo, VcsInfo
 from pip._internal.models.link import Link
 from pip._internal.req.req_install import InstallRequirement
-from pip._internal.utils.urls import url_to_path
+from pip._internal.utils.compat import tomllib
+from pip._internal.utils.urls import path_to_url, url_to_path
+
+if TYPE_CHECKING:
+    from pip._internal.network.session import PipSession
 
 
 def _pylock_package_from_install_requirement(
@@ -114,3 +124,122 @@ def pylock_from_install_requirements(
             key=lambda p: p.name,
         ),
     )
+
+
+_SCHEME_RE = re.compile("^(http|https|file)://", re.IGNORECASE)
+
+
+def _is_url(s: str) -> bool:
+    return bool(_SCHEME_RE.match(s))
+
+
+def _package_dist_url(
+    pylock_path_or_url: str,
+    path: str | None,
+    url: str | None,
+) -> str:
+    """Compute an url from a Pylock package path and url.
+
+    Give priority to path over url. If path is relative,
+    compute an url using the pylock file location as base.
+    """
+    if path is not None:
+        if not os.path.isabs(path):
+            if _is_url(pylock_path_or_url):
+                return urljoin(pylock_path_or_url, path)
+            else:
+                return path_to_url(
+                    os.path.join(os.path.dirname(pylock_path_or_url), path)
+                )
+        else:
+            return path_to_url(path)
+    else:
+        assert url is not None  # guaranteed by packaging.pylock validation
+        return url
+
+
+def package_vcs_requirement_url(
+    pylock_path_or_url: str, package_vcs: PackageVcs
+) -> str:
+    url = (
+        package_vcs.type
+        + "+"
+        + _package_dist_url(pylock_path_or_url, package_vcs.path, package_vcs.url)
+        + "@"
+        + package_vcs.commit_id
+    )
+    if package_vcs.subdirectory:
+        assert "#" not in url  # TODO: prorper exception
+        url += "#" + package_vcs.subdirectory
+    return url
+
+
+def package_archive_requirement_url(
+    pylock_path_or_url: str, package_archive: PackageArchive
+) -> str:
+    url = _package_dist_url(
+        pylock_path_or_url, package_archive.path, package_archive.url
+    )
+    if package_archive.subdirectory:
+        assert "#" not in url  # TODO: prorper exception
+        url += "#" + package_archive.subdirectory
+    return url
+
+
+def package_directory_requirement_url(
+    pylock_path_or_url: str, package_directory: PackageDirectory
+) -> str:
+    url = _package_dist_url(pylock_path_or_url, package_directory.path, None)
+    assert url.startswith("file://")  # TODO: proper exception
+    if not url.endswith("/"):
+        url += "/"
+    if package_directory.subdirectory:
+        url += package_directory.subdirectory
+        if not url.endswith("/"):
+            url += "/"
+    return url
+
+
+def package_sdist_requirement_url(
+    pylock_path_or_url: str, package_sdist: PackageSdist
+) -> str:
+    return _package_dist_url(pylock_path_or_url, package_sdist.path, package_sdist.url)
+
+
+def package_wheel_requirement_url(
+    pylock_path_or_url: str, package_wheel: PackageWheel
+) -> str:
+    return _package_dist_url(pylock_path_or_url, package_wheel.path, package_wheel.url)
+
+
+def _get_pylock_path_or_url_content(path_or_url: str, session: PipSession) -> str:
+    # TODO: refactor - this is similar to req_file.get_file_content
+    scheme = urlsplit(path_or_url).scheme
+    # Pip has special support for file:// URLs (LocalFSAdapter).
+    if scheme in ["http", "https", "file"]:
+        # Delay importing heavy network modules until absolutely necessary.
+        from pip._internal.network.utils import raise_for_status
+
+        resp = session.get(path_or_url)
+        raise_for_status(resp)
+        return resp.text
+
+    # Assume this is a bare path.
+    return Path(path_or_url).read_text(encoding="utf-8")
+
+
+def select_from_pylock_path_or_url(
+    pylock_path_or_url: str,
+    session: PipSession,
+) -> Iterator[
+    tuple[
+        Package,
+        PackageVcs | PackageDirectory | PackageArchive | PackageWheel | PackageSdist,
+    ]
+]:
+    lock = Pylock.from_dict(
+        tomllib.loads(
+            _get_pylock_path_or_url_content(pylock_path_or_url, session),
+        )
+    )
+    yield from lock.select()

tests/data/lockfiles/pylock.directory.toml

@@ -0,0 +1,6 @@
+lock-version = "1.0"
+created-by = "pip"
+packages = [
+    { name = "simplewheel", directory = { path = "../src/simplewheel-2.0", editable = true } },
+    { name = "singlemodule", directory = { path = "../src/singlemodule" } },
+]

tests/data/lockfiles/pylock.toml

@@ -0,0 +1,33 @@
+lock-version = "1.0"
+created-by = "pip"
+
+[[packages]]
+name = "simple"
+version = "2.0"
+
+[packages.sdist]
+name = "simple-2.0.tar.gz"
+path = "../packages/simple-2.0.tar.gz"
+
+[packages.sdist.hashes]
+sha256 = "3a084929238d13bcd3bb928af04f3bac7ca2357d419e29f01459dc848e2d69a4"
+
+[[packages]]
+name = "simple2"
+
+[packages.archive]
+path = "../packages/simple2-3.0.tar.gz"
+
+[packages.archive.hashes]
+sha256 = "3ad45e1e9aa48b4462af0123f6a7e44a9115db1ef945d4d92c123dfe21815a06"
+
+[[packages]]
+name = "simplewheel"
+version = "2.0"
+
+[[packages.wheels]]
+name = "simplewheel-2.0-1-py2.py3-none-any.whl"
+path = "../packages/simplewheel-2.0-1-py2.py3-none-any.whl"
+
+[packages.wheels.hashes]
+sha256 = "71e1ca6b16ae3382a698c284013f66504f2581099b2ce4801f60e9536236ceee"