diff --git a/docs/pypi/lock.md b/docs/pypi/lock.md index 34c10db60c..70a477d2c2 100644 --- a/docs/pypi/lock.md +++ b/docs/pypi/lock.md @@ -12,8 +12,7 @@ both `bzlmod` and `WORKSPACE`. `pylock.toml` (PEP 751) files, e.g. exported with `uv export --format pylock.toml`, can be consumed with -{attr}`pip.parse.pylock` under `bzlmod`; the `vcs`, `archive` and `directory` -sources are tracked in {gh-issue}`2787`. +{attr}`pip.parse.pylock` under `bzlmod` ({gh-issue}`2787`). ::: ## requirements.txt diff --git a/docs/pypi/rules-pycross-comparison.md b/docs/pypi/rules-pycross-comparison.md index 15f719f569..a222f1e13f 100644 --- a/docs/pypi/rules-pycross-comparison.md +++ b/docs/pypi/rules-pycross-comparison.md @@ -202,8 +202,8 @@ module. Do not depend on the same distribution from both in one binary. |---|---|---| | `uv.lock` | {attr}`pip.parse.uv_lock` | have | | uv workspace members, `projects`, `default-groups`, `extra_project_files` | the uv workspace members are used as the roots; {obj}`pip.configure` reads one `pyproject.toml` | partial | -| `pylock.toml` | {attr}`pip.parse.pylock`; `vcs`, `archive` and `directory` sources are planned | partial | -| `pdm.lock`, `poetry.lock` | export them to `pylock.toml` (`pdm export -f pylock`, `poetry-plugin-export` 1.10+) for {attr}`pip.parse.pylock` | partial: no `vcs`, `archive` or `directory` sources | +| `pylock.toml` | {attr}`pip.parse.pylock` | have | +| `pdm.lock`, `poetry.lock` | export them to `pylock.toml` (`pdm export -f pylock`, `poetry-plugin-export` 1.10+) for {attr}`pip.parse.pylock` | have | | dependency groups, `transitive` | {obj}`pip.configure`, requires {gh-pr}`4395` | have | | `;testonly` | none, see {gh-pr}`4428` | planned | | extras targets | requires {gh-pr}`4490` and {gh-pr}`4523` | have | diff --git a/news/pip-parse-pylock-sources.added.md b/news/pip-parse-pylock-sources.added.md new file mode 100644 index 0000000000..dccf2a97fc --- /dev/null +++ b/news/pip-parse-pylock-sources.added.md @@ -0,0 +1,3 @@ +(pypi) The {attr}`pip.parse.pylock` attribute now supports the `vcs` and +`archive` sources of the `pylock.toml` packages, which are built by `pip`, and +ignores the `directory` sources, the local projects (bzlmod only). diff --git a/news/pip-parse-pylock.added.md b/news/pip-parse-pylock.added.md index 0f3ea439a2..681b7ecde3 100644 --- a/news/pip-parse-pylock.added.md +++ b/news/pip-parse-pylock.added.md @@ -1,5 +1,4 @@ (pypi) The {attr}`pip.parse.pylock` attribute reads a `pylock.toml` (PEP 751) lock file, e.g. exported by `uv`, `pdm` or `pip lock`, instead of the -requirements files (bzlmod only). The packages from `vcs`, `archive` and -`directory` sources are not supported yet +requirements files (bzlmod only) ([#2787](https://github.com/bazel-contrib/rules_python/issues/2787)). diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 3f3772a268..fae42f1f1a 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -916,9 +916,12 @@ the platforms where its `marker` is true and it fails if its `requires-python` does not match such a platform. Each platform needs to match the top-level `requires-python`; the platforms that do not match the top-level `environments` are ignored with a warning. The files without a `url` are -ignored, use {attr}`local_wheels` for them. The packages from `vcs`, `archive` -and `directory` sources are not supported yet. It cannot be used together with -the {attr}`uv_lock` or the requirements files. +ignored, use {attr}`local_wheels` for them. The `vcs` sources and the `archive` +sources that are not wheels (based on the URL file name) are downloaded and +built by `pip` from their URL when the repositories are fetched, so e.g. `git` +needs to be on the host and the Bazel downloader is not used for them. The +`directory` sources (the local projects) are ignored. It cannot be used +together with the {attr}`uv_lock` or the requirements files. :::{versionadded} VERSION_NEXT_FEATURE :::""", diff --git a/python/private/pypi/pylock.bzl b/python/private/pypi/pylock.bzl index a6452ce319..21b1837ddc 100644 --- a/python/private/pypi/pylock.bzl +++ b/python/private/pypi/pylock.bzl @@ -13,7 +13,10 @@ def pylock_entries(pylock, *, all_platforms, platforms, local_wheels = [], logge All of the extras, the default groups and the dependency groups that the lock file lists are selected. A package is used on the platforms where its `marker` is true; the dependency graph is not used, as the standard - requires. + requires. The `vcs` sources and the `archive` sources that are not wheels + are built by pip from their URL, the `archive` wheels are used like the + other wheels and the `directory` sources are the local projects, which are + ignored. Args: pylock: {type}`dict` The decoded pylock.toml contents. @@ -64,10 +67,31 @@ def pylock_entries(pylock, *, all_platforms, platforms, local_wheels = [], logge for pkg in pylock.get("packages", []): name = pkg["name"] version = pkg.get("version", "") - for key in ["vcs", "archive", "directory"]: - if key in pkg: - return _fail("The pylock.toml package '{}' has a '{}' source, which is not supported".format(name, key)) - if not version: + if "directory" in pkg: + # A local project, e.g. the project that the lock file is for. + logger.debug(lambda: "Ignoring the pylock.toml package '{}' from the directory '{}'".format( + name, + pkg["directory"].get("path"), + )) + continue + + direct_requirement = "" + archive = None + if "vcs" in pkg: + direct_requirement = _vcs_requirement(name, pkg["vcs"], _fail = _fail) + if not direct_requirement: + return None + elif "archive" in pkg: + archive = pkg["archive"] + if "url" not in archive: + return _fail("The pylock.toml package '{}' has an 'archive' without a 'url', which is not supported".format(name)) + if not _dist(archive).filename.endswith(".whl"): + # Let pip build the archive from its URL with the hash, so + # that it verifies the download and does not use a wheel + # from the index instead. + direct_requirement = _archive_requirement(archive) + archive = None + if not version and not direct_requirement: return _fail("The pylock.toml package '{}' has no 'version'".format(name)) target_platforms = [ @@ -96,16 +120,20 @@ def pylock_entries(pylock, *, all_platforms, platforms, local_wheels = [], logge used_local_wheels[whl.file] = None sdist = pkg.get("sdist") - if [d for d in pkg.get("wheels", []) + ([sdist] if sdist else []) if "url" not in d]: + wheels = pkg.get("wheels", []) + if archive: + wheels = wheels + [archive] + if [d for d in wheels + ([sdist] if sdist else []) if "url" not in d]: logger.warn(lambda: "The pylock.toml package '{}' has files without a 'url', which are ignored, use the 'local_wheels' instead".format(name)) entries.append(lock_entry( name = name, version = version, target_platforms = target_platforms, index_url = "{}/{}".format(pkg["index"].rstrip("/"), normalize_name(name).replace("_", "-")) if pkg.get("index") else "", - wheels = [_dist(wheel) for wheel in pkg.get("wheels", []) if "url" in wheel], + wheels = [_dist(wheel) for wheel in wheels if "url" in wheel], local_wheels = local, sdist = _dist(sdist) if sdist and "url" in sdist else None, + direct_requirement = direct_requirement, )) for whls in local_by_name.values(): @@ -118,6 +146,31 @@ def pylock_entries(pylock, *, all_platforms, platforms, local_wheels = [], logge platforms = sorted(envs), ) +def _vcs_requirement(name, vcs, *, _fail): + """Turn a `vcs` source into a pip direct reference.""" + if vcs.get("type") not in ["git", "hg", "svn", "bzr"] or not vcs.get("url") or not vcs.get("commit-id"): + return _fail("The pylock.toml package '{}' needs a 'vcs' source of the 'git', 'hg', 'svn' or 'bzr' type with a 'url' (not a 'path') and a 'commit-id'".format(name)) + + url = vcs["url"] + if not url.startswith(vcs["type"] + "+"): + url = "{}+{}".format(vcs["type"], url) + requirement = "{}@{}".format(url, vcs["commit-id"]) + if vcs.get("subdirectory"): + requirement += "#subdirectory={}".format(vcs["subdirectory"]) + return requirement + +def _archive_requirement(archive): + """Turn an `archive` source into a pip direct reference with its hash.""" + fragments = [ + "{}={}".format(algo, hex_digest) + for algo, hex_digest in sorted(archive.get("hashes", {}).items()) + ][:1] + if archive.get("subdirectory"): + fragments.append("subdirectory={}".format(archive["subdirectory"])) + if not fragments: + return archive["url"] + return "{}#{}".format(archive["url"], "&".join(fragments)) + def _requires_python(specifiers, env): """Evaluate a `requires-python` specifier set, e.g. `>=3.9,<4`.""" return evaluate( diff --git a/tests/pypi/pylock/pylock_tests.bzl b/tests/pypi/pylock/pylock_tests.bzl index 61ebe013ed..ab80d62585 100644 --- a/tests/pypi/pylock/pylock_tests.bzl +++ b/tests/pypi/pylock/pylock_tests.bzl @@ -114,7 +114,8 @@ def _test_errors(env): for contents, want in [ ('lock-version = "2.0"', "Only the pylock.toml 'lock-version' 1.x is supported, got: '2.0'"), ('lock-version = "1.0"\nrequires-python = ">=3.13"', "The pylock.toml 'requires-python = \">=3.13\"' does not match the Python version 3.12.4 of the 'linux_x86_64' platform"), - ('lock-version = "1.0"\n[[packages]]\nname = "foo"\nvcs = { type = "git", url = "https://example.org/foo.git", commit-id = "abc" }', "The pylock.toml package 'foo' has a 'vcs' source, which is not supported"), + ('lock-version = "1.0"\n[[packages]]\nname = "foo"\nvcs = { type = "git", path = "../foo", commit-id = "abc" }', "The pylock.toml package 'foo' needs a 'vcs' source of the 'git', 'hg', 'svn' or 'bzr' type with a 'url' (not a 'path') and a 'commit-id'"), + ('lock-version = "1.0"\n[[packages]]\nname = "foo"\nversion = "1.0"\narchive = { path = "foo.tar.gz" }', "The pylock.toml package 'foo' has an 'archive' without a 'url', which is not supported"), ('lock-version = "1.0"\n[[packages]]\nname = "foo"', "The pylock.toml package 'foo' has no 'version'"), ('lock-version = "1.0"\n[[packages]]\nname = "foo"\nversion = "1.0"\nrequires-python = "<3.12"', "The pylock.toml package 'foo' has 'requires-python = \"<3.12\"', which does not match the Python version 3.12.4 of the 'linux_x86_64' platform"), ]: @@ -149,6 +150,66 @@ def _test_errors(env): _tests.append(_test_errors) +_PYLOCK_SOURCES = """\ +lock-version = "1.0" + +[[packages]] +name = "my-project" +directory = { path = ".", editable = true } + +[[packages]] +name = "from-git" +version = "0.1" +vcs = { type = "git", url = "https://github.com/org/repo.git", requested-revision = "main", commit-id = "0123abcd", subdirectory = "python" } + +[[packages]] +name = "from-tarball" +version = "1.0" +archive = { url = "https://github.com/org/tarball/archive/v1.0.tar.gz", hashes = { sha256 = "deadbeef" } } + +[[packages]] +name = "from-subdir" +version = "2.0" +archive = { url = "https://example.org/mono.zip", subdirectory = "pkg", hashes = { sha256 = "deadbaaf" } } + +[[packages]] +name = "from-wheel" +version = "3.0" +archive = { url = "https://example.org/from_wheel-3.0-py3-none-any.whl", hashes = { sha256 = "deadb00f" } } +""" + +def _test_sources(env): + failures = [] + lock = pylock_entries( + toml.decode(_PYLOCK_SOURCES), + all_platforms = sorted(_PLATFORMS), + platforms = _PLATFORMS, + logger = _logger(failures), + _fail = failures.append, + ) + env.expect.that_collection(failures).contains_exactly([]) + + # The local project is ignored + env.expect.that_collection([e.name for e in lock.entries]).contains_exactly([ + "from-git", + "from-tarball", + "from-subdir", + "from-wheel", + ]) + + got = lock_requirements(lock.entries, all_platforms = lock.platforms, platforms = _PLATFORMS, logger = _logger(failures)) + env.expect.that_collection([ + (r.name, [(s.requirement_line, s.filename, s.url, s.digest) for s in r.srcs]) + for r in got + ]).contains_exactly([ + ("from_git", [("from-git @ git+https://github.com/org/repo.git@0123abcd#subdirectory=python", "", "", "")]), + ("from_subdir", [("from-subdir @ https://example.org/mono.zip#sha256=deadbaaf&subdirectory=pkg", "", "", "")]), + ("from_tarball", [("from-tarball @ https://github.com/org/tarball/archive/v1.0.tar.gz#sha256=deadbeef", "", "", "")]), + ("from_wheel", [("from-wheel==3.0", "from_wheel-3.0-py3-none-any.whl", "https://example.org/from_wheel-3.0-py3-none-any.whl", "sha256:deadb00f")]), + ]) + +_tests.append(_test_sources) + def pylock_test_suite(name): """Create the test suite.