From 40ec2c9010e9bc5ac478e94985250a0cbc131be3 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 22 Nov 2026 23:22:55 +0900 Subject: [PATCH] feat(pypi): resolve the extras of the dependencies in the whl repositories A dependency on an extra of another package from the wheel METADATA, e.g. coverage[toml], used the [toml] target of the hub, which only exists if the lock file requests the extra, and failed the build otherwise. The extras that a package provides are only known from its METADATA, so the whl repository of the dependency resolves them: the dependent package uses an extra_dep target that points to the _extras target of the dependency, which is forwarded by the hub. The missing_extra_error rule is no longer needed. Work towards #2786 --- docs/pypi/use.md | 21 +- news/2786.changed.md | 25 +- python/private/internal_dev_deps.bzl | 3 - python/private/pypi/BUILD.bazel | 3 +- python/private/pypi/hub_repository.bzl | 5 +- python/private/pypi/missing_package.bzl | 52 +- python/private/pypi/pip_repository.bzl | 6 +- python/private/pypi/pkg_aliases.bzl | 12 +- python/private/pypi/render_pkg_aliases.bzl | 2 +- .../private/pypi/whl_library_deps_targets.bzl | 79 ++- tests/pypi/pkg_aliases/pkg_aliases_test.bzl | 35 ++ tests/pypi/whl_library/BUILD.bazel | 25 +- .../consumer/consumer-1.0.dist-info/METADATA | 1 + tests/pypi/whl_library/testdata/packages.bzl | 4 +- .../whl_library/whl_library_query_test.py | 30 + .../whl_library_deps_targets_tests.bzl | 537 +++++++++++++++--- 16 files changed, 647 insertions(+), 193 deletions(-) create mode 100644 tests/pypi/whl_library/whl_library_query_test.py diff --git a/docs/pypi/use.md b/docs/pypi/use.md index 3a8af5983c..44eb06247f 100644 --- a/docs/pypi/use.md +++ b/docs/pypi/use.md @@ -119,12 +119,17 @@ platforms, for example `foo[bar]` on Linux and `foo[baz]` on macOS, then `@pypi//foo:[bar]` on macOS is the same as `@pypi//foo:[]`. The dependencies of the packages use the extras written in the wheel -`METADATA`, so if `foo` depends on `bar[socks]`, then `@pypi//foo` depends on -`@pypi//bar:[socks]`. If the extra is not in the lock file, then building -`@pypi//foo` fails with a message asking you to add `bar[socks]` to the -requirements or the lock file. For example, when compiling the requirements -with `uv pip compile`, pass `--no-strip-extras` to keep the extras in the -output. +`METADATA`, so if `foo` depends on `bar[socks]`, then `@pypi//foo` includes +`bar` and the dependencies of its `socks` extra. The extras of the +dependencies are resolved by the repository of `bar` from its own `METADATA`, +so they do not need to be in the lock file, e.g. when the requirements are +compiled with the extras stripped. This does not change `@pypi//bar`, which +only includes the extras requested in the lock file. The dependencies of the +extra that are not in the lock file are left out like any other dependency +that is not in the lock file. If `bar` does not provide the `socks` extra, +then only `bar` is used and a warning is printed when `@pypi//foo` is +analyzed, e.g. `WARNING: bar 1.0 does not provide the extra 'socks'`, which is +how `pip` handles the unknown extras. If the lock file requests an extra that the package does not provide, e.g. `foo[nope]`, then `@pypi//foo:[nope]` is the same as `@pypi//foo:[]`, because @@ -142,9 +147,7 @@ When the `requirements.bzl` file is vendored, there is no hub repo and the labels returned by the `requirement` helper point to the repositories of the wheels. Their `pkg` targets include the extras from the lock file and they have the `[]` target and the `[extra]` targets for the extras used in the lock -file and for all of the extras that the package provides. The dependencies -between the packages use the `pkg` targets and ignore the extras from the wheel -`METADATA` like before, so the extras need to be in the lock file. +file and for all of the extras that the package provides. The `whl` targets, e.g. `@pypi//foo:whl`, do not use the extras and include only the wheels of the package and its dependencies without extras. diff --git a/news/2786.changed.md b/news/2786.changed.md index 29a0897f0b..cc97dd9100 100644 --- a/news/2786.changed.md +++ b/news/2786.changed.md @@ -1,17 +1,16 @@ (pypi) The dependencies between the PyPI packages now use the extras from the wheel `METADATA`, e.g. if `foo` requires `bar[socks]`, then `@pypi//foo` -depends on `@pypi//bar:[socks]` instead of `@pypi//bar`. Previously such extras -were ignored. If the lock file of a hub does not request the extra, the build -fails with a message asking to add it, e.g. by compiling the requirements with -`--no-strip-extras`. This does not apply when there is no hub, e.g. with a -vendored `requirements.bzl`, where the dependencies use the `pkg` targets, -which include the extras from the lock file, as before. The `whl` targets no longer include the wheels of the -dependencies of the extras. A member of `experimental_requirement_cycles` -without extras, e.g. `foo`, is now only the package without extras, so the -cycles that go through `foo[bar]` need `foo[bar]` as a member. The same -distribution that is requested with different extras on different platforms -now uses a single whl repository. The `@pypi//foo` targets still include the -extras requested in the lock file. An extra that the package does not provide -is still ignored, but a warning is now printed like `pip` does, e.g. +includes `bar` and the dependencies of its `socks` extra. Previously such +extras were ignored. The extra does not need to be requested in the lock file +of the hub, because it is resolved by the whl repository of `bar`, and the +dependencies of the extra that are not in the lock file are left out. The +`whl` targets no longer include the wheels of the dependencies of the extras. +A member of `experimental_requirement_cycles` without extras, e.g. `foo`, is +now only the package without extras, so the cycles that go through `foo[bar]` +need `foo[bar]` as a member. The same distribution that is requested with +different extras on different platforms now uses a single whl repository. The +`@pypi//foo` targets still include the extras requested in the lock file. An +extra that the package does not provide is still ignored, but a warning is now +printed like `pip` does, e.g. `WARNING: foo 1.0 does not provide the extra 'nope'` ([#2786](https://github.com/bazel-contrib/rules_python/issues/2786)). diff --git a/python/private/internal_dev_deps.bzl b/python/private/internal_dev_deps.bzl index 7ca740aa6a..8aba3130f5 100644 --- a/python/private/internal_dev_deps.bzl +++ b/python/private/internal_dev_deps.bzl @@ -121,9 +121,6 @@ def _internal_dev_deps_impl(mctx): name = "whl_library_extras_pkg", root = "//tests/pypi/whl_library/testdata/pkg:BUILD.bazel", output = "pkg-1.0-any-none-any.whl", - # Without a hub the dependencies use the `pkg` target, which includes - # the extras of the requirement. - requirement = "pkg[optional]", # The following is necessary to enable pipstar and make tests faster config_load = "@rules_python//tests/pypi/whl_library/testdata:packages.bzl", dep_template = "@whl_library_extras_{name}//:{target}", diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 45e529b5a6..e292717746 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -386,6 +386,7 @@ bzl_library( name = "pkg_aliases", srcs = ["pkg_aliases.bzl"], deps = [ + ":extra_dep", ":labels", ":whl_library_deps_targets", "//python:py_library", @@ -643,8 +644,8 @@ bzl_library( srcs = ["whl_library_deps_targets.bzl"], deps = [ ":env_marker_setting", + ":extra_dep", ":labels", - ":missing_package", ":pep508_deps", "//python:py_library", "//python/private:normalize_name", diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index 27fa5fdece..ec43b0c3fa 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -57,8 +57,9 @@ def _impl(rctx): "config.bzl", rctx.attr._config_template, substitutions = { - # The available extras of each package for the spoke repositories - # to check if the extras of their dependencies are available. + # The requested extras of each package for the spoke repositories + # to create the targets of the extras that the package does not + # provide. "%%PACKAGES%%": render.dict(available_extras(whl_map), value_repr = render.list), }, ) diff --git a/python/private/pypi/missing_package.bzl b/python/private/pypi/missing_package.bzl index e5647d5786..46823e813b 100644 --- a/python/private/pypi/missing_package.bzl +++ b/python/private/pypi/missing_package.bzl @@ -1,16 +1,25 @@ -"""Rules for generating an execution-phase action failure when a PyPI package or its extra is missing.""" +"""Rule for generating an execution-phase action failure when a PyPI package is missing.""" load("//python/private:py_info.bzl", "PyInfo") load("//python/private:reexports.bzl", "BuiltinPyInfo") -def _error(ctx, message): +def _missing_package_error_impl(ctx): out = ctx.actions.declare_file(ctx.label.name + ".error") + if ctx.attr.hub_name: + hub_clause = ' when building under PyPI hub "{hub}". Try adding it to the requirements of this hub (e.g. requirements_lock or requirements_by_platform in pip.parse)'.format( + hub = ctx.attr.hub_name, + ) + else: + hub_clause = ' because no default PyPI hub was configured. Try designating a default hub via pip.default(default_hub = "...") or select a hub using --@rules_python//python/config_settings:venv' + # Register an action that fails when Bazel attempts to stage/build this file ctx.actions.run_shell( outputs = [out], - command = "echo \"$1\" >&2 && exit 1", - arguments = ["ERROR: " + message], + command = "echo 'ERROR: PyPI package \"{pkg}\" is not available{hub_clause}.' >&2 && exit 1".format( + pkg = ctx.attr.package_name, + hub_clause = hub_clause, + ), ) maybe_builtin = [BuiltinPyInfo(transitive_sources = depset([out]))] if BuiltinPyInfo != None else [] @@ -25,19 +34,6 @@ def _error(ctx, message): ), ] + maybe_builtin -def _missing_package_error_impl(ctx): - if ctx.attr.hub_name: - hub_clause = ' when building under PyPI hub "{hub}". Try adding it to the requirements of this hub (e.g. requirements_lock or requirements_by_platform in pip.parse)'.format( - hub = ctx.attr.hub_name, - ) - else: - hub_clause = ' because no default PyPI hub was configured. Try designating a default hub via pip.default(default_hub = "...") or select a hub using --@rules_python//python/config_settings:venv' - - return _error(ctx, 'PyPI package "{pkg}" is not available{hub_clause}.'.format( - pkg = ctx.attr.package_name, - hub_clause = hub_clause, - )) - missing_package_error = rule( implementation = _missing_package_error_impl, attrs = { @@ -45,25 +41,3 @@ missing_package_error = rule( "package_name": attr.string(mandatory = True), }, ) - -def _missing_extra_error_impl(ctx): - return _error(ctx, ( - 'The PyPI package "{pkg}" requires "{dep}[{extra}]", but the extra "{extra}" of ' + - '"{dep}" is not available in the hub. Add "{dep}[{extra}]" to the requirements or ' + - "the lock file of the hub, e.g. by compiling the requirements with the extras kept " + - "(`--no-strip-extras`) or by adding the extra to the project dependencies." - ).format( - pkg = ctx.attr.required_by, - dep = ctx.attr.package_name, - extra = ctx.attr.extra, - )) - -missing_extra_error = rule( - implementation = _missing_extra_error_impl, - doc = "A target that fails the build with a helpful message when an extra of a PyPI package is missing.", - attrs = { - "extra": attr.string(mandatory = True), - "package_name": attr.string(mandatory = True), - "required_by": attr.string(mandatory = True), - }, -) diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index a692296683..5fe8b6c1c7 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -286,9 +286,9 @@ def _pip_repository_impl(rctx): "config.bzl", rctx.attr._config_template, substitutions = { - # The available extras of each package for the spoke - # repositories to check if the extras of their dependencies - # are available. + # The requested extras of each package for the spoke + # repositories to create the targets of the extras that the + # package does not provide. "%%PACKAGES%%": render.dict({ pkg: sorted(selected_extras.get(pkg, [])) for pkg in bzl_packages diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index d64bc7a15f..a1fdfe72a9 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -35,6 +35,7 @@ load("@bazel_skylib//lib:selects.bzl", "selects") load("//python:py_library.bzl", "py_library") load("//python/private:common_labels.bzl", "labels") load("//python/private:text_util.bzl", "render") +load(":extra_dep.bzl", "EXTRAS_SELECT_TARGET", "extra_dep", "extra_select") load( ":labels.bzl", "DATA_LABEL", @@ -124,6 +125,8 @@ def pkg_aliases( _native = kwargs.pop("native", native) rules = kwargs.pop("rules", struct( + extra_dep = extra_dep, + extra_select = extra_select, py_library = py_library, whl_library_deps_targets = whl_library_deps_targets, )) @@ -157,6 +160,10 @@ def pkg_aliases( # targets do not use the extras, so only the package without extras # can be a member of the group. target_names = { + # The whl repositories resolve the extras requested by the + # packages that depend on them, so the hub only needs to forward + # to the whl repository of the target platform. + EXTRAS_SELECT_TARGET: EXTRAS_SELECT_TARGET, WHEEL_FILE_PUBLIC_LABEL: WHEEL_FILE_IMPL_LABEL if "" in group_extras else WHEEL_FILE_PUBLIC_LABEL, } @@ -214,7 +221,10 @@ def pkg_aliases( fail("The `actual` arg must be a dictionary or a string") kwargs = {} - if target_name.startswith("_"): + if target_name == EXTRAS_SELECT_TARGET: + # Used by the whl repositories that depend on the extras. + pass + elif target_name.startswith("_"): kwargs["visibility"] = ["//_groups:__subpackages__"] elif target_name in (WHEEL_FILE, PY_SRCS_LABEL): kwargs["visibility"] = ["//visibility:private"] diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 9ba55f3a75..bc0d98cdfe 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -313,7 +313,7 @@ def _non_versioned_platform(p, *, strict = False): fail("Expected to always have a platform in the form '{{abi}}_{{os}}_{{arch}}', got: {}".format(p)) def available_extras(aliases): - """Return the extras that are available in the hub for each package. + """Return the extras that are requested in the hub for each package. Args: aliases: {type}`dict[str, dict[struct, str]]` the packages to the diff --git a/python/private/pypi/whl_library_deps_targets.bzl b/python/private/pypi/whl_library_deps_targets.bzl index 68621e5f00..85d5a80797 100644 --- a/python/private/pypi/whl_library_deps_targets.bzl +++ b/python/private/pypi/whl_library_deps_targets.bzl @@ -3,6 +3,7 @@ load("//python:py_library.bzl", "py_library") load("//python/private:normalize_name.bzl", "normalize_name") load(":env_marker_setting.bzl", "env_marker_setting") +load(":extra_dep.bzl", "EXTRAS_SELECT_TARGET", "extra_dep", "extra_select", "extra_select_targets") load( ":labels.bzl", "DATA_LABEL", @@ -15,7 +16,6 @@ load( "extra_label", "group_member", ) -load(":missing_package.bzl", "missing_extra_error") load(":pep508_deps.bzl", "deps") def whl_library_deps_targets( @@ -37,7 +37,8 @@ def whl_library_deps_targets( rules = struct( py_library = py_library, env_marker_setting = env_marker_setting, - missing_extra_error = missing_extra_error, + extra_dep = extra_dep, + extra_select = extra_select, )): """Create all of the whl_library targets. @@ -54,6 +55,9 @@ def whl_library_deps_targets( provides_extra: {type}`list[str]` The list of extras that the package provides. For each extra the `[]` target is created that includes the package and the dependencies of that extra. + The `_extras` target selects one of them for the + {obj}`extra_dep` targets of the packages that depend on an extra + of this package. extras: {type}`list[str]` The list of requested extras. The `pkg` target includes them, which is how the extras are used when there is no hub repository to select them, e.g. when using a vendored @@ -64,11 +68,9 @@ def whl_library_deps_targets( `pip` does. include: {type}`list[str] | dict[str, list[str] | None]` The list of packages to include. If it is a dict, then the values are the - extras of each package that are available in the hub and the - dependencies on the extras that are not available will fail the - build with a helpful message. The `[]` targets are also - created for the extras of this package that are available in - the hub, see `extras` for the ones that the package does not + extras of each package that are requested in the hub. The + `[]` targets are also created for the requested extras of + this package, see `extras` for the ones that the package does not provide. group_name: {type}`str | None` name of the dependency group (if any). dep_template: {type}`str | None` The dep_template to use. @@ -156,13 +158,14 @@ def whl_library_deps_targets( if normalize_name(e) not in provided_extras }) extras_targets = sorted(provided_extras.keys() + unknown_extras) + distribution = " ".join([metadata_name] + [ + t.partition("=")[2] + for t in tags + if t.startswith("pypi_version=") + ]) for extra in unknown_extras: getattr(rules, "warn", _warn)("WARNING: {} does not provide the extra '{}'".format( - " ".join([metadata_name] + [ - t.partition("=")[2] - for t in tags - if t.startswith("pypi_version=") - ]), + distribution, extra, )) whl_file_label = _name("", WHEEL_FILE_PUBLIC_LABEL) @@ -210,6 +213,28 @@ def whl_library_deps_targets( visibility = visibility if group_label or not requires_dist else impl_vis, ) + def _extra_target(extra): + if group_name and not group_label and is_member(extra): + # The group library of the members is in the hub. + return dep_template.format(name = self_name, target = extra_label(extra)) + + return ":" + extra_label(extra) + + # The packages that depend on an extra of this package use this target + # with the requested extra, because the hub does not know which extras + # this package provides. An extra without a target is the same as the + # package without extras and a warning is printed when it is requested. + if hasattr(rules, "extra_select"): + extra_select_targets( + name = EXTRAS_SELECT_TARGET, + targets = {extra: _extra_target(extra) for extra in extras_targets}, + default = _extra_target(""), + distribution = distribution, + native = native, + rules = rules, + visibility = visibility, + ) + if not requires_dist: # If there are extras, then they will be visible in requires_dist. return @@ -218,21 +243,18 @@ def whl_library_deps_targets( # dep and the expression. settings = {} - # The dependencies on extras that are not available in the hub. - missing_extras = {} + # The dependencies on the extras of other packages. + extra_deps = {} def _pkg_label(dep): dep_name, _, extra = dep.partition("[") if not extra: return dep_template.format(name = dep_name, target = PY_LIBRARY_PUBLIC_LABEL) - extra = extra.rstrip("]") - available = include.get(dep_name) if type(include) == type({}) else None - if available != None and extra not in available: - missing_extras[dep] = (dep_name, extra) - return ":_missing_" + dep - - return dep_template.format(name = dep_name, target = extra_label(extra)) + # The extra does not need to be requested in the hub, so it is + # resolved by the package that provides it. + extra_deps[dep] = (dep_name, extra.rstrip("]")) + return ":_" + dep whl_label = lambda dep: dep_template.format(name = dep, target = WHEEL_FILE_PUBLIC_LABEL) @@ -266,11 +288,7 @@ def whl_library_deps_targets( include = include, excludes = members if is_member(extra) else [], extras = want_extras, - # Without the hub, e.g. with the vendored `requirements.bzl`, the - # `pkg` targets of the dependencies already include the extras - # from their requirement lines, so the dependencies use them - # like before. - keep_extras = "//:" not in dep_template, + keep_extras = True, ) rules.py_library( name = name, @@ -299,12 +317,11 @@ def whl_library_deps_targets( visibility = ["//visibility:private"], ) - for dep, (dep_name, extra) in sorted(missing_extras.items()): - rules.missing_extra_error( - name = "_missing_" + dep, - package_name = dep_name, + for dep, (dep_name, extra) in sorted(extra_deps.items()): + rules.extra_dep( + name = "_" + dep, + deps = [dep_template.format(name = dep_name, target = EXTRAS_SELECT_TARGET)], extra = extra, - required_by = metadata_name, visibility = ["//visibility:private"], ) diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index b1ff1bf698..4e26210c6a 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -43,6 +43,7 @@ def _test_legacy_aliases(env): "foo": ":pkg", "pkg": ":[]", "[]": "@repo//:[]", + "_extras": "@repo//:_extras", "srcs": "@repo//:srcs", "whl_file": "@repo//:whl_file", "whl": "@repo//:whl", @@ -56,6 +57,36 @@ def _test_legacy_aliases(env): _tests.append(_test_legacy_aliases) +def _test_extras_select_aliases(env): + got = {} + pkg_aliases( + name = "coverage", + actual = { + whl_config_setting(version = "3.10"): "coverage_310", + whl_config_setting(version = "3.12"): "coverage_312", + }, + native = struct( + alias = lambda *, name, actual, visibility = None, tags = None: got.update({name: (actual, visibility)}), + ), + select = lambda value, no_match_error = None: value, + ) + + # The hub does not know the extras that the package provides, so the + # packages that depend on an extra use the `_extras` target of the whl + # repository for the target platform. + env.expect.that_dict(got).contains_at_least({ + "_extras": ( + { + "//_config:is_cp310": "@coverage_310//:_extras", + "//_config:is_cp312": "@coverage_312//:_extras", + "//conditions:default": "_no_matching_repository", + }, + None, + ), + }) + +_tests.append(_test_extras_select_aliases) + def _test_config_setting_aliases(env): # Use this function as it is used in pip_repository got = {} @@ -221,6 +252,10 @@ def _test_group_aliases(env): "name": "foo", "actual": ":pkg", }, + { + "name": "_extras", + "actual": "@repo//:_extras", + }, { "name": "_pkg", "actual": "@repo//:[]", diff --git a/tests/pypi/whl_library/BUILD.bazel b/tests/pypi/whl_library/BUILD.bazel index 51e894693d..ad01e3bd54 100644 --- a/tests/pypi/whl_library/BUILD.bazel +++ b/tests/pypi/whl_library/BUILD.bazel @@ -40,7 +40,30 @@ py_test( srcs = ["whl_library_transitive_extras_test.py"], target_compatible_with = SUPPORTS_BZLMOD, deps = [ - # The METADATA of the consumer requires `pkg[optional]`. + # The METADATA of the consumer requires `pkg[optional]`, which is not + # requested in the hub, and `pkg[unknown]`, which `pkg` does not + # provide. "@whl_library_extras_consumer//:pkg", ], ) + +# `bazel query` does not know the configuration, so it needs all of the targets +# that the extras of the dependencies can resolve to. It fails with `no such +# target` if any of them is missing, e.g. for the `pkg[unknown]` dependency of +# the consumer, because `pkg` does not provide the `unknown` extra. +genquery( + name = "whl_library_transitive_extras_query", + expression = "deps(@whl_library_extras_consumer//:pkg)", + scope = ["@whl_library_extras_consumer//:pkg"], + target_compatible_with = SUPPORTS_BZLMOD, +) + +py_test( + name = "whl_library_query_test", + srcs = ["whl_library_query_test.py"], + data = [":whl_library_transitive_extras_query"], + env = { + "QUERY_OUTPUT": "$(rootpath :whl_library_transitive_extras_query)", + }, + target_compatible_with = SUPPORTS_BZLMOD, +) diff --git a/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA b/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA index 9a723c958a..e2cbed2ee5 100644 --- a/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA +++ b/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA @@ -1,3 +1,4 @@ Name: consumer Version: 1.0 Requires-Dist: pkg[optional] +Requires-Dist: pkg[unknown] diff --git a/tests/pypi/whl_library/testdata/packages.bzl b/tests/pypi/whl_library/testdata/packages.bzl index 627c3a84c2..812dc40f2f 100644 --- a/tests/pypi/whl_library/testdata/packages.bzl +++ b/tests/pypi/whl_library/testdata/packages.bzl @@ -1,12 +1,12 @@ """A list of packages that this logical testdata hub repo contains.""" # The values are the extras that are requested in the hub, the `pkg` package -# does not provide the `nope` extra. +# does not provide the `nope` extra. The `optional` extra of `pkg` is not +# requested in the hub, but the `consumer` package depends on it. packages = { "consumer": [], "optional_dep": [], "pkg": [ "nope", - "optional", ], } diff --git a/tests/pypi/whl_library/whl_library_query_test.py b/tests/pypi/whl_library/whl_library_query_test.py new file mode 100644 index 0000000000..ed31f7c4de --- /dev/null +++ b/tests/pypi/whl_library/whl_library_query_test.py @@ -0,0 +1,30 @@ +import os +import unittest + + +class QueryTest(unittest.TestCase): + def test_deps_include_the_extras(self): + with open(os.environ["QUERY_OUTPUT"]) as f: + got = { + # The canonical names of the repositories depend on the + # extension that creates them. + line.strip().rpartition("whl_library_extras_")[2] + for line in f + if "whl_library_extras_" in line + } + + # The edges in the `METADATA` of the consumer, the first extra is + # provided by `pkg` and the second one is not. + self.assertIn("consumer//:_pkg[optional]", got) + self.assertIn("consumer//:_pkg[unknown]", got) + # Both of them resolve to the target that selects the extra. + self.assertIn("pkg//:_extras", got) + self.assertIn("pkg//:[optional]", got) + self.assertIn("pkg//:[]", got) + self.assertNotIn("pkg//:[unknown]", got) + # The dependency of the `optional` extra. + self.assertIn("optional_dep//:pkg", got) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl index 551ab7c767..4fa6ad72ec 100644 --- a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl +++ b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl @@ -175,24 +175,25 @@ def _test_extras(env): rules = struct( py_library = lambda **kwargs: py_library_calls.append(kwargs), env_marker_setting = lambda **kwargs: env_marker_setting_calls.append(kwargs), + extra_dep = lambda **kwargs: None, ), ) # The `pkg` target has no extras because none were requested and there is # a target for each extra that the package provides. The `whl` target does - # not use the extras. Without the extras available in the hub, the - # dependencies use the targets with the extras directly. + # not use the extras. The dependencies on the extras of the other packages + # use the `extra_dep` targets. env.expect.that_collection([ (c["name"], c["deps"]) for c in py_library_calls ]).contains_exactly([ - ("pkg", ["srcs", "@pypi//bar:[socks]"]), - ("[all]", ["srcs", "@pypi//bar:[socks]", "@pypi//grpc:pkg"]), - ("[async]", ["srcs", "@pypi//bar:[socks]", "@pypi//uvloop:pkg"] + select({ + ("pkg", ["srcs", ":_bar[socks]"]), + ("[all]", ["srcs", ":_bar[socks]", "@pypi//grpc:pkg"]), + ("[async]", ["srcs", ":_bar[socks]", "@pypi//uvloop:pkg"] + select({ ":is_include_aiohttp_true": ["@pypi//aiohttp:pkg"], "//conditions:default": [], })), - ("[grpc]", ["srcs", "@pypi//bar:[socks]", "@pypi//grpc:pkg"]), + ("[grpc]", ["srcs", ":_bar[socks]", "@pypi//grpc:pkg"]), ]) env.expect.that_collection([ (c["name"], c["data"]) @@ -538,132 +539,494 @@ def _test_extras_in_group_legacy(env): _tests.append(_test_extras_in_group_legacy) -def _test_deps_with_extras(env): - filegroup_calls = [] - py_library_calls = [] - env_marker_setting_calls = [] - missing_calls = [] +def _targets(**kwargs): + """Create the targets and return the calls to the rules. + + Args: + **kwargs: the args for the `whl_library_deps_targets`. + + Returns: + A struct with the `py_library` deps and the `whl` data by the target + name and the calls to the other rules. + """ + calls = struct( + config_setting = [], + env_marker_setting = [], + extra_dep = [], + extra_select = [], + filegroup = [], + py_library = [], + warn = [], + ) whl_library_deps_targets( - name = "foo-0-py3-none-any.whl", + name = "{}-1.0-py3-none-any.whl".format(kwargs["metadata_name"]), + dep_template = kwargs.pop("dep_template", "@pypi//{name}:{target}"), + repo = None, + aliases = {}, + native = struct( + alias = lambda **kwargs: None, + config_setting = lambda **kwargs: calls.config_setting.append(kwargs), + filegroup = lambda **kwargs: calls.filegroup.append(kwargs), + ), + rules = struct( + env_marker_setting = lambda **kwargs: calls.env_marker_setting.append(kwargs), + extra_dep = lambda **kwargs: calls.extra_dep.append(kwargs), + extra_select = lambda **kwargs: calls.extra_select.append(kwargs), + py_library = lambda **kwargs: calls.py_library.append(kwargs), + warn = lambda msg: calls.warn.append(msg), + ), + **kwargs + ) + + return struct( + deps = {c["name"]: c["deps"] for c in calls.py_library}, + whl_data = {c["name"]: c["data"] for c in calls.filegroup}, + env_markers = {c["name"]: c["expression"] for c in calls.env_marker_setting}, + extra_deps = calls.extra_dep, + extra_selects = calls.extra_select, + config_settings = [c["name"] for c in calls.config_setting], + warnings = calls.warn, + ) + +def _test_deps_with_extras(env): + got = _targets( metadata_name = "foo", - dep_template = "@pypi//{name}:{target}", requires_dist = [ "bar[socks]", "baz[a,b]; sys_platform == 'linux'", "baz; sys_platform == 'darwin'", - "qux[missing]", + "qux[not-requested]", ], - # The values are the extras that are available in the hub + # The values are the extras that are requested in the hub, which are + # not needed for the extras of the dependencies. include = { "bar": ["socks"], "baz": ["a", "b"], "qux": [], }, - repo = None, - aliases = {}, - native = struct( - filegroup = lambda **kwargs: filegroup_calls.append(kwargs), - alias = lambda **kwargs: None, - ), - rules = struct( - py_library = lambda **kwargs: py_library_calls.append(kwargs), - env_marker_setting = lambda **kwargs: env_marker_setting_calls.append(kwargs), - missing_extra_error = lambda **kwargs: missing_calls.append(kwargs), - ), ) - # The dependencies use the targets with the extras, the missing extras use - # a target that fails the build with a helpful message. - env.expect.that_collection([ - (c["name"], c["deps"]) - for c in py_library_calls - ]).contains_exactly([ - ("pkg", ["srcs", "@pypi//bar:[socks]", ":_missing_qux[missing]"] + select({ + # The extras of the dependencies are resolved by the packages that provide + # them, so they are used via the `extra_dep` targets. + env.expect.that_dict(got.deps).contains_exactly({ + "pkg": ["srcs", ":_bar[socks]", ":_qux[not_requested]"] + select({ ":is_include_baz_2_true": ["@pypi//baz:pkg"], "//conditions:default": [], }) + select({ - ":is_include_baz[a]_true": ["@pypi//baz:[a]"], + ":is_include_baz[a]_true": [":_baz[a]"], "//conditions:default": [], }) + select({ - ":is_include_baz[b]_true": ["@pypi//baz:[b]"], + ":is_include_baz[b]_true": [":_baz[b]"], "//conditions:default": [], - })), - ]) - env.expect.that_collection(missing_calls).contains_exactly([ + }), + }) + env.expect.that_collection(got.extra_deps).contains_exactly([ { - "name": "_missing_qux[missing]", - "package_name": "qux", - "extra": "missing", - "required_by": "foo", + "name": "_" + dep, + "deps": ["@pypi//{}:_extras".format(name)], + "extra": extra, "visibility": ["//visibility:private"], - }, + } + for dep, name, extra in [ + ("bar[socks]", "bar", "socks"), + ("baz[a]", "baz", "a"), + ("baz[b]", "baz", "b"), + ("qux[not_requested]", "qux", "not_requested"), + ] ]) # buildifier: @unsorted-dict-items # The whl targets do not use the extras. - env.expect.that_collection([ - (c["name"], c["data"]) - for c in filegroup_calls - ]).contains_exactly([ - ("whl", ["@pypi//bar:whl", "@pypi//qux:whl"] + select({ + env.expect.that_dict(got.whl_data).contains_exactly({ + "whl": ["@pypi//bar:whl", "@pypi//qux:whl"] + select({ ":is_include_baz_true": ["@pypi//baz:whl"], "//conditions:default": [], - })), - ]) + }), + }) + env.expect.that_dict(got.env_markers).contains_exactly({ + "include_baz": "(sys_platform == \"darwin\") or (sys_platform == \"linux\")", + "include_baz[a]": "sys_platform == \"linux\"", + "include_baz[b]": "sys_platform == \"linux\"", + "include_baz_2": "sys_platform == \"darwin\"", + }) + + # The package does not provide any extras. + env.expect.that_collection(got.config_settings).contains_exactly([]) + env.expect.that_collection(got.extra_selects).contains_exactly([ + { + "name": "_extras", + "deps": [":[]"], + "distribution": "foo", + "extras": [], + "visibility": ["//visibility:public"], + }, + ]) # buildifier: @unsorted-dict-items + +_tests.append(_test_deps_with_extras) + +_COVERAGE_REQUIRES_DIST = [ + "tomli; python_full_version <= \"3.11.0a6\" and extra == \"toml\"", +] + +def _test_extra_that_is_not_requested_in_the_hub(env): + # The lock file has `coverage` without the `toml` extra, e.g. because the + # extras are stripped when compiling the requirements. + include = {"coverage": [], "pytest_cov": [], "tomli": []} + + got = _targets( + metadata_name = "pytest-cov", + requires_dist = ["coverage[toml]>=7.5", "pytest>=7"], + include = include, + ) + env.expect.that_dict(got.deps).contains_exactly({ + "pkg": ["srcs", ":_coverage[toml]"], + }) + env.expect.that_collection(got.extra_deps).contains_exactly([ + { + "name": "_coverage[toml]", + "deps": ["@pypi//coverage:_extras"], + "extra": "toml", + "visibility": ["//visibility:private"], + }, + ]) # buildifier: @unsorted-dict-items + + got = _targets( + metadata_name = "coverage", + requires_dist = _COVERAGE_REQUIRES_DIST, + provides_extra = ["toml"], + include = include, + tags = ["pypi_name=coverage", "pypi_version=7.6.1"], + ) + + # Only the target of the extra has `tomli` and only for the Python + # versions that need it, which is evaluated for each target platform. + env.expect.that_dict(got.deps).contains_exactly({ + "[toml]": ["srcs"] + select({ + ":is_include_tomli_true": ["@pypi//tomli:pkg"], + "//conditions:default": [], + }), + "pkg": ["srcs"], + }) + env.expect.that_dict(got.env_markers).contains_exactly({ + "include_tomli": "python_full_version <= \"3.11.0a6\"", + }) + env.expect.that_collection(got.config_settings).contains_exactly(["_extras_toml"]) + env.expect.that_collection(got.extra_selects).contains_exactly([ + { + "name": "_extras", + "deps": select({ + ":_extras_toml": [":[toml]"], + "//conditions:default": [":[]"], + }), + "distribution": "coverage 7.6.1", + "extras": ["toml"], + "visibility": ["//visibility:public"], + }, + ]) # buildifier: @unsorted-dict-items + +_tests.append(_test_extra_that_is_not_requested_in_the_hub) + +def _test_deps_of_the_extra_that_are_not_in_the_hub(env): + # `tomli` is not locked, e.g. because the lock file is only for the Python + # versions that do not need it. Like the other dependencies that are not + # in the hub, it is dropped. + got = _targets( + metadata_name = "coverage", + requires_dist = _COVERAGE_REQUIRES_DIST, + provides_extra = ["toml"], + include = {"coverage": [], "pytest_cov": []}, + ) + env.expect.that_dict(got.deps).contains_exactly({ + "[toml]": ["srcs"], + "pkg": ["srcs"], + }) + env.expect.that_dict(got.env_markers).contains_exactly({}) + + # The same for the dependencies of the extra without any markers. + got = _targets( + metadata_name = "bar", + requires_dist = ["baz; extra == \"x\"", "qux"], + provides_extra = ["x"], + include = {"bar": [], "foo": [], "qux": []}, + ) + env.expect.that_dict(got.deps).contains_exactly({ + "[x]": ["srcs", "@pypi//qux:pkg"], + "pkg": ["srcs", "@pypi//qux:pkg"], + }) + + # Without any other dependencies the targets are aliases and the extra can + # still be selected. + got = _targets( + metadata_name = "bar", + provides_extra = ["x"], + include = {"bar": [], "foo": []}, + ) + env.expect.that_dict(got.deps).contains_exactly({}) + env.expect.that_collection([ + (c["name"], c["extras"]) + for c in got.extra_selects + ]).contains_exactly([("_extras", ["x"])]) + +_tests.append(_test_deps_of_the_extra_that_are_not_in_the_hub) + +def _test_dep_with_extra_that_is_not_provided(env): + include = {"bar": [], "foo": []} + got = _targets( + metadata_name = "foo", + requires_dist = ["bar[nope]"], + include = include, + ) + + # The package does not know the extras of its dependencies. + env.expect.that_dict(got.deps).contains_exactly({ + "pkg": ["srcs", ":_bar[nope]"], + }) + env.expect.that_collection([ + (c["deps"], c["extra"]) + for c in got.extra_deps + ]).contains_exactly([(["@pypi//bar:_extras"], "nope")]) + + # The `nope` extra is not in the `extras` of the `_extras` target, so it + # selects the package without extras and prints a warning. + got = _targets( + metadata_name = "bar", + requires_dist = ["baz; extra == \"x\""], + provides_extra = ["x"], + include = include, + ) env.expect.that_collection([ - (c["name"], c["expression"]) - for c in env_marker_setting_calls + (c["deps"], c["extras"]) + for c in got.extra_selects ]).contains_exactly([ - ("include_baz", "(sys_platform == \"darwin\") or (sys_platform == \"linux\")"), - ("include_baz_2", "sys_platform == \"darwin\""), - ("include_baz[a]", "sys_platform == \"linux\""), - ("include_baz[b]", "sys_platform == \"linux\""), + ( + select({ + ":_extras_x": [":[x]"], + "//conditions:default": [":[]"], + }), + ["x"], + ), ]) -_tests.append(_test_deps_with_extras) - -def _test_deps_with_extras_legacy(env): - py_library_calls = [] - missing_calls = [] +_tests.append(_test_dep_with_extra_that_is_not_provided) - whl_library_deps_targets( - name = "foo-0-py3-none-any.whl", +def _test_self_referencing_extras_are_selected(env): + got = _targets( metadata_name = "foo", - # Without the hub, e.g. with the vendored `requirements.bzl`. - dep_template = "@pip_{name}//:{target}", requires_dist = [ - "bar[socks]", - "baz[a]; sys_platform == 'linux'", - "qux[missing]", + "foo[a,b]; extra == \"all\"", + "dep-a; extra == \"a\"", + "dep-b; extra == \"b\"", ], - repo = None, - aliases = {}, - native = struct( - alias = lambda **kwargs: None, - ), - rules = struct( - py_library = lambda **kwargs: py_library_calls.append(kwargs), - env_marker_setting = lambda **kwargs: None, - missing_extra_error = lambda **kwargs: missing_calls.append(kwargs), - ), + provides_extra = ["a", "all", "b"], + include = {"dep_a": [], "dep_b": [], "foo": []}, ) - # The dependencies use the `pkg` targets, which include the extras from - # the requirement lines of the dependencies, so the extras that are not - # provided or not requested do not result in missing targets. + # A dependency on `foo[all]` gets the dependencies of both extras and a + # dependency on `foo[a]` only gets the ones of `a`. + env.expect.that_dict(got.deps).contains_exactly({ + "[a]": ["srcs", "@pypi//dep_a:pkg"], + "[all]": ["srcs", "@pypi//dep_a:pkg", "@pypi//dep_b:pkg"], + "[b]": ["srcs", "@pypi//dep_b:pkg"], + "pkg": ["srcs"], + }) + env.expect.that_collection([c["deps"] for c in got.extra_selects]).contains_exactly([ + select({ + ":_extras_a": [":[a]"], + ":_extras_all": [":[all]"], + ":_extras_b": [":[b]"], + "//conditions:default": [":[]"], + }), + ]) + env.expect.that_collection(got.extra_deps).contains_exactly([]) + +_tests.append(_test_self_referencing_extras_are_selected) + +def _test_cycle_through_extras(env): + # `a[x]` requires `b[y]` and `b[y]` requires `a[x]`. Each package only + # reads its own `METADATA`, so nothing is resolved recursively here. + a = dict( + metadata_name = "a", + requires_dist = ["b[y]; extra == \"x\""], + provides_extra = ["x"], + include = {"a": [], "b": []}, + ) + b = dict( + metadata_name = "b", + requires_dist = ["a[x]; extra == \"y\""], + provides_extra = ["y"], + include = {"a": [], "b": []}, + ) + + # Without a group this is a dependency cycle between the targets like any + # other cycle between the packages. + env.expect.that_dict(_targets(**a).deps).contains_exactly({ + "[x]": ["srcs", ":_b[y]"], + "pkg": ["srcs"], + }) + env.expect.that_dict(_targets(**b).deps).contains_exactly({ + "[y]": ["srcs", ":_a[x]"], + "pkg": ["srcs"], + }) + + # With the extras as the members of a group the cycle is broken and the + # `_extras` targets select the group library of the hub. + group = dict(group_name = "cycle", group_deps = ["a[x]", "b[y]"]) + got = _targets(**(a | group)) + env.expect.that_dict(got.deps).contains_exactly({ + "[x]": ["srcs"], + "pkg": ["srcs"], + }) + env.expect.that_collection(got.extra_deps).contains_exactly([]) + env.expect.that_collection([c["deps"] for c in got.extra_selects]).contains_exactly([ + select({ + ":_extras_x": ["@pypi//a:[x]"], + "//conditions:default": [":[]"], + }), + ]) + got = _targets(**(b | group)) + env.expect.that_dict(got.deps).contains_exactly({ + "[y]": ["srcs"], + "pkg": ["srcs"], + }) + env.expect.that_collection(got.extra_deps).contains_exactly([]) + +_tests.append(_test_cycle_through_extras) + +def _test_different_extras_of_the_same_dep(env): + include = {"c": [], "foo_bar": [], "other_dep": [], "p": [], "q": [], "r": [], "tomli": []} + + # Each package only depends on the extra that it requires. + got = _targets(metadata_name = "p", requires_dist = ["c[toml]"], include = include) + env.expect.that_dict(got.deps).contains_exactly({"pkg": ["srcs", ":_c[toml]"]}) env.expect.that_collection([ - (c["name"], c["deps"]) - for c in py_library_calls + (c["name"], c["deps"], c["extra"]) + for c in got.extra_deps + ]).contains_exactly([("_c[toml]", ["@pypi//c:_extras"], "toml")]) + + got = _targets(metadata_name = "q", requires_dist = ["c[other]"], include = include) + env.expect.that_dict(got.deps).contains_exactly({"pkg": ["srcs", ":_c[other]"]}) + env.expect.that_collection([ + (c["name"], c["deps"], c["extra"]) + for c in got.extra_deps + ]).contains_exactly([("_c[other]", ["@pypi//c:_extras"], "other")]) + + got = _targets( + metadata_name = "c", + requires_dist = ["tomli; extra == \"toml\"", "other-dep; extra == \"other\""], + provides_extra = ["toml", "other"], + include = include, + ) + env.expect.that_dict(got.deps).contains_exactly({ + "[other]": ["srcs", "@pypi//other_dep:pkg"], + "[toml]": ["srcs", "@pypi//tomli:pkg"], + "pkg": ["srcs"], + }) + + # The names of the packages and the extras are normalized on both sides. + got = _targets(metadata_name = "r", requires_dist = ["Foo_Bar[TOML.Parser]"], include = include) + env.expect.that_dict(got.deps).contains_exactly({"pkg": ["srcs", ":_foo_bar[toml_parser]"]}) + env.expect.that_collection([ + (c["name"], c["deps"], c["extra"]) + for c in got.extra_deps + ]).contains_exactly([("_foo_bar[toml_parser]", ["@pypi//foo_bar:_extras"], "toml_parser")]) + + got = _targets( + metadata_name = "foo-bar", + requires_dist = ["tomli; extra == \"toml-parser\""], + provides_extra = ["toml-parser"], + include = include, + ) + env.expect.that_dict(got.deps).contains_exactly({ + "[toml_parser]": ["srcs", "@pypi//tomli:pkg"], + "pkg": ["srcs"], + }) + env.expect.that_collection(got.config_settings).contains_exactly(["_extras_toml_parser"]) + env.expect.that_collection([ + (c["deps"], c["extras"]) + for c in got.extra_selects ]).contains_exactly([ - ("pkg", ["srcs", "@pip_bar//:pkg", "@pip_qux//:pkg"] + select({ - ":is_include_baz_true": ["@pip_baz//:pkg"], + ( + select({ + ":_extras_toml_parser": [":[toml_parser]"], + "//conditions:default": [":[]"], + }), + ["toml_parser"], + ), + ]) + +_tests.append(_test_different_extras_of_the_same_dep) + +def _test_deps_with_extras_legacy(env): + # Without a hub, e.g. with a vendored `requirements.bzl`, the extras of + # the dependencies are resolved by the whl repositories in the same way + # and the config only has the names of the packages. + include = {"bar": None, "foo": None, "qux": None} + got = _targets( + metadata_name = "foo", + requires_dist = ["bar[socks]", "bar[nope]; sys_platform == \"linux\"", "qux"], + include = include, + dep_template = "@pip_{name}//:{target}", + ) + env.expect.that_dict(got.deps).contains_exactly({ + "pkg": ["srcs", ":_bar[socks]", "@pip_qux//:pkg"] + select({ + ":is_include_bar[nope]_true": [":_bar[nope]"], "//conditions:default": [], - })), + }), + }) + env.expect.that_collection([ + (c["name"], c["deps"], c["extra"]) + for c in got.extra_deps + ]).contains_exactly([ + ("_bar[nope]", ["@pip_bar//:_extras"], "nope"), + ("_bar[socks]", ["@pip_bar//:_extras"], "socks"), ]) - env.expect.that_collection(missing_calls).contains_exactly([]) + + # The repository of the dependency always has the target to select the + # extra, also for the extras that it does not provide and when it has no + # dependencies. + for requires_dist in [[], ["qux; extra == \"socks\""]]: + got = _targets( + metadata_name = "bar", + requires_dist = requires_dist, + provides_extra = ["socks"], + include = include, + dep_template = "@pip_{name}//:{target}", + ) + env.expect.that_collection([ + (c["name"], c["deps"], c["extras"]) + for c in got.extra_selects + ]).contains_exactly([( + "_extras", + select({ + ":_extras_socks": [":[socks]"], + "//conditions:default": [":[]"], + }), + ["socks"], + )]) _tests.append(_test_deps_with_extras_legacy) +def _test_extras_select_in_group_legacy(env): + # Without a hub the members of the group are aliases to the group library + # in the same repository. + got = _targets( + metadata_name = "foo", + requires_dist = ["bar; extra == \"a\"", "qux; extra == \"b\""], + provides_extra = ["a", "b"], + group_name = "group", + group_deps = ["foo[a]", "bar"], + dep_template = "@pypi_{name}//:{target}", + ) + env.expect.that_collection([c["deps"] for c in got.extra_selects]).contains_exactly([ + select({ + ":_extras_a": [":[a]"], + ":_extras_b": [":[b]"], + "//conditions:default": [":[]"], + }), + ]) + +_tests.append(_test_extras_select_in_group_legacy) + def whl_library_deps_targets_test_suite(name): """create the test suite.