Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions docs/pypi/lock.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,7 @@ both `bzlmod` and `WORKSPACE`.

`pylock.toml` (PEP 751) files, e.g. exported with
`uv export --format pylock.toml`, can be consumed with
{attr}`pip.parse.pylock` under `bzlmod`; the `vcs`, `archive` and `directory`
sources are tracked in {gh-issue}`2787`.
{attr}`pip.parse.pylock` under `bzlmod` ({gh-issue}`2787`).
:::

## requirements.txt
Expand Down
4 changes: 2 additions & 2 deletions docs/pypi/rules-pycross-comparison.md
Original file line number Diff line number Diff line change
Expand Up @@ -202,8 +202,8 @@ module. Do not depend on the same distribution from both in one binary.
|---|---|---|
| `uv.lock` | {attr}`pip.parse.uv_lock` | have |
| uv workspace members, `projects`, `default-groups`, `extra_project_files` | the uv workspace members are used as the roots; {obj}`pip.configure` reads one `pyproject.toml` | partial |
| `pylock.toml` | {attr}`pip.parse.pylock`; `vcs`, `archive` and `directory` sources are planned | partial |
| `pdm.lock`, `poetry.lock` | export them to `pylock.toml` (`pdm export -f pylock`, `poetry-plugin-export` 1.10+) for {attr}`pip.parse.pylock` | partial: no `vcs`, `archive` or `directory` sources |
| `pylock.toml` | {attr}`pip.parse.pylock` | have |
| `pdm.lock`, `poetry.lock` | export them to `pylock.toml` (`pdm export -f pylock`, `poetry-plugin-export` 1.10+) for {attr}`pip.parse.pylock` | have |
| dependency groups, `transitive` | {obj}`pip.configure`, requires {gh-pr}`4395` | have |
| `;testonly` | none, see {gh-pr}`4428` | planned |
| extras targets | requires {gh-pr}`4490` and {gh-pr}`4523` | have |
Expand Down
3 changes: 3 additions & 0 deletions news/pip-parse-pylock-sources.added.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
(pypi) The {attr}`pip.parse.pylock` attribute now supports the `vcs` and
`archive` sources of the `pylock.toml` packages, which are built by `pip`, and
ignores the `directory` sources, the local projects (bzlmod only).
3 changes: 1 addition & 2 deletions news/pip-parse-pylock.added.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
(pypi) The {attr}`pip.parse.pylock` attribute reads a `pylock.toml` (PEP 751)
lock file, e.g. exported by `uv`, `pdm` or `pip lock`, instead of the
requirements files (bzlmod only). The packages from `vcs`, `archive` and
`directory` sources are not supported yet
requirements files (bzlmod only)
([#2787](https://github.com/bazel-contrib/rules_python/issues/2787)).
9 changes: 6 additions & 3 deletions python/private/pypi/extension.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -916,9 +916,12 @@ the platforms where its `marker` is true and it fails if its
`requires-python` does not match such a platform. Each platform needs to match
the top-level `requires-python`; the platforms that do not match the top-level
`environments` are ignored with a warning. The files without a `url` are
ignored, use {attr}`local_wheels` for them. The packages from `vcs`, `archive`
and `directory` sources are not supported yet. It cannot be used together with
the {attr}`uv_lock` or the requirements files.
ignored, use {attr}`local_wheels` for them. The `vcs` sources and the `archive`
sources that are not wheels (based on the URL file name) are downloaded and
built by `pip` from their URL when the repositories are fetched, so e.g. `git`
needs to be on the host and the Bazel downloader is not used for them. The
`directory` sources (the local projects) are ignored. It cannot be used
together with the {attr}`uv_lock` or the requirements files.

:::{versionadded} VERSION_NEXT_FEATURE
:::""",
Expand Down
67 changes: 60 additions & 7 deletions python/private/pypi/pylock.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,10 @@ def pylock_entries(pylock, *, all_platforms, platforms, local_wheels = [], logge
All of the extras, the default groups and the dependency groups that the
lock file lists are selected. A package is used on the platforms where its
`marker` is true; the dependency graph is not used, as the standard
requires.
requires. The `vcs` sources and the `archive` sources that are not wheels
are built by pip from their URL, the `archive` wheels are used like the
other wheels and the `directory` sources are the local projects, which are
ignored.

Args:
pylock: {type}`dict` The decoded pylock.toml contents.
Expand Down Expand Up @@ -64,10 +67,31 @@ def pylock_entries(pylock, *, all_platforms, platforms, local_wheels = [], logge
for pkg in pylock.get("packages", []):
name = pkg["name"]
version = pkg.get("version", "")
for key in ["vcs", "archive", "directory"]:
if key in pkg:
return _fail("The pylock.toml package '{}' has a '{}' source, which is not supported".format(name, key))
if not version:
if "directory" in pkg:
# A local project, e.g. the project that the lock file is for.
logger.debug(lambda: "Ignoring the pylock.toml package '{}' from the directory '{}'".format(
name,
pkg["directory"].get("path"),
))
continue

direct_requirement = ""
archive = None
if "vcs" in pkg:
direct_requirement = _vcs_requirement(name, pkg["vcs"], _fail = _fail)
if not direct_requirement:
return None
elif "archive" in pkg:
archive = pkg["archive"]
if "url" not in archive:
return _fail("The pylock.toml package '{}' has an 'archive' without a 'url', which is not supported".format(name))
if not _dist(archive).filename.endswith(".whl"):
# Let pip build the archive from its URL with the hash, so
# that it verifies the download and does not use a wheel
# from the index instead.
direct_requirement = _archive_requirement(archive)
archive = None
if not version and not direct_requirement:
return _fail("The pylock.toml package '{}' has no 'version'".format(name))

target_platforms = [
Expand Down Expand Up @@ -96,16 +120,20 @@ def pylock_entries(pylock, *, all_platforms, platforms, local_wheels = [], logge
used_local_wheels[whl.file] = None

sdist = pkg.get("sdist")
if [d for d in pkg.get("wheels", []) + ([sdist] if sdist else []) if "url" not in d]:
wheels = pkg.get("wheels", [])
if archive:
wheels = wheels + [archive]
if [d for d in wheels + ([sdist] if sdist else []) if "url" not in d]:
logger.warn(lambda: "The pylock.toml package '{}' has files without a 'url', which are ignored, use the 'local_wheels' instead".format(name))
entries.append(lock_entry(
name = name,
version = version,
target_platforms = target_platforms,
index_url = "{}/{}".format(pkg["index"].rstrip("/"), normalize_name(name).replace("_", "-")) if pkg.get("index") else "",
wheels = [_dist(wheel) for wheel in pkg.get("wheels", []) if "url" in wheel],
wheels = [_dist(wheel) for wheel in wheels if "url" in wheel],
local_wheels = local,
sdist = _dist(sdist) if sdist and "url" in sdist else None,
direct_requirement = direct_requirement,
))

for whls in local_by_name.values():
Expand All @@ -118,6 +146,31 @@ def pylock_entries(pylock, *, all_platforms, platforms, local_wheels = [], logge
platforms = sorted(envs),
)

def _vcs_requirement(name, vcs, *, _fail):
"""Turn a `vcs` source into a pip direct reference."""
if vcs.get("type") not in ["git", "hg", "svn", "bzr"] or not vcs.get("url") or not vcs.get("commit-id"):
return _fail("The pylock.toml package '{}' needs a 'vcs' source of the 'git', 'hg', 'svn' or 'bzr' type with a 'url' (not a 'path') and a 'commit-id'".format(name))

url = vcs["url"]
if not url.startswith(vcs["type"] + "+"):
url = "{}+{}".format(vcs["type"], url)
requirement = "{}@{}".format(url, vcs["commit-id"])
if vcs.get("subdirectory"):
requirement += "#subdirectory={}".format(vcs["subdirectory"])
return requirement

def _archive_requirement(archive):
"""Turn an `archive` source into a pip direct reference with its hash."""
fragments = [
"{}={}".format(algo, hex_digest)
for algo, hex_digest in sorted(archive.get("hashes", {}).items())
][:1]
if archive.get("subdirectory"):
fragments.append("subdirectory={}".format(archive["subdirectory"]))
if not fragments:
return archive["url"]
return "{}#{}".format(archive["url"], "&".join(fragments))

def _requires_python(specifiers, env):
"""Evaluate a `requires-python` specifier set, e.g. `>=3.9,<4`."""
return evaluate(
Expand Down
63 changes: 62 additions & 1 deletion tests/pypi/pylock/pylock_tests.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,8 @@ def _test_errors(env):
for contents, want in [
('lock-version = "2.0"', "Only the pylock.toml 'lock-version' 1.x is supported, got: '2.0'"),
('lock-version = "1.0"\nrequires-python = ">=3.13"', "The pylock.toml 'requires-python = \">=3.13\"' does not match the Python version 3.12.4 of the 'linux_x86_64' platform"),
('lock-version = "1.0"\n[[packages]]\nname = "foo"\nvcs = { type = "git", url = "https://example.org/foo.git", commit-id = "abc" }', "The pylock.toml package 'foo' has a 'vcs' source, which is not supported"),
('lock-version = "1.0"\n[[packages]]\nname = "foo"\nvcs = { type = "git", path = "../foo", commit-id = "abc" }', "The pylock.toml package 'foo' needs a 'vcs' source of the 'git', 'hg', 'svn' or 'bzr' type with a 'url' (not a 'path') and a 'commit-id'"),
('lock-version = "1.0"\n[[packages]]\nname = "foo"\nversion = "1.0"\narchive = { path = "foo.tar.gz" }', "The pylock.toml package 'foo' has an 'archive' without a 'url', which is not supported"),
('lock-version = "1.0"\n[[packages]]\nname = "foo"', "The pylock.toml package 'foo' has no 'version'"),
('lock-version = "1.0"\n[[packages]]\nname = "foo"\nversion = "1.0"\nrequires-python = "<3.12"', "The pylock.toml package 'foo' has 'requires-python = \"<3.12\"', which does not match the Python version 3.12.4 of the 'linux_x86_64' platform"),
]:
Expand Down Expand Up @@ -149,6 +150,66 @@ def _test_errors(env):

_tests.append(_test_errors)

_PYLOCK_SOURCES = """\
lock-version = "1.0"

[[packages]]
name = "my-project"
directory = { path = ".", editable = true }

[[packages]]
name = "from-git"
version = "0.1"
vcs = { type = "git", url = "https://github.com/org/repo.git", requested-revision = "main", commit-id = "0123abcd", subdirectory = "python" }

[[packages]]
name = "from-tarball"
version = "1.0"
archive = { url = "https://github.com/org/tarball/archive/v1.0.tar.gz", hashes = { sha256 = "deadbeef" } }

[[packages]]
name = "from-subdir"
version = "2.0"
archive = { url = "https://example.org/mono.zip", subdirectory = "pkg", hashes = { sha256 = "deadbaaf" } }

[[packages]]
name = "from-wheel"
version = "3.0"
archive = { url = "https://example.org/from_wheel-3.0-py3-none-any.whl", hashes = { sha256 = "deadb00f" } }
"""

def _test_sources(env):
failures = []
lock = pylock_entries(
toml.decode(_PYLOCK_SOURCES),
all_platforms = sorted(_PLATFORMS),
platforms = _PLATFORMS,
logger = _logger(failures),
_fail = failures.append,
)
env.expect.that_collection(failures).contains_exactly([])

# The local project is ignored
env.expect.that_collection([e.name for e in lock.entries]).contains_exactly([
"from-git",
"from-tarball",
"from-subdir",
"from-wheel",
])

got = lock_requirements(lock.entries, all_platforms = lock.platforms, platforms = _PLATFORMS, logger = _logger(failures))
env.expect.that_collection([
(r.name, [(s.requirement_line, s.filename, s.url, s.digest) for s in r.srcs])
for r in got
]).contains_exactly([
("from_git", [("from-git @ git+https://github.com/org/repo.git@0123abcd#subdirectory=python", "", "", "")]),
("from_subdir", [("from-subdir @ https://example.org/mono.zip#sha256=deadbaaf&subdirectory=pkg", "", "", "")]),
("from_tarball", [("from-tarball @ https://github.com/org/tarball/archive/v1.0.tar.gz#sha256=deadbeef", "", "", "")]),
("from_wheel", [("from-wheel==3.0", "from_wheel-3.0-py3-none-any.whl", "https://example.org/from_wheel-3.0-py3-none-any.whl", "sha256:deadb00f")]),
])

_tests.append(_test_sources)

def pylock_test_suite(name):
"""Create the test suite.

Expand Down
Loading