From 3ef53b2098194d6754702dffa6defc24b0fd5566 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:07:15 +0900 Subject: [PATCH 01/55] refactor(pypi): select packages from the pyproject.toml dependency groups Read the dependencies, the optional dependencies and the PEP 735 dependency groups from the pyproject.toml and add a helper to select the packages using dependency group specifiers. --- python/private/pypi/BUILD.bazel | 9 ++ python/private/pypi/dependency_groups.bzl | 142 ++++++++++++++++++ python/private/pyproject_utils.bzl | 14 +- tests/pypi/dependency_groups/BUILD.bazel | 3 + .../dependency_groups_tests.bzl | 86 +++++++++++ 5 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 python/private/pypi/dependency_groups.bzl create mode 100644 tests/pypi/dependency_groups/BUILD.bazel create mode 100644 tests/pypi/dependency_groups/dependency_groups_tests.bzl diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index f94d24594f..af005f706e 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -632,6 +632,15 @@ bzl_library( ], ) +bzl_library( + name = "dependency_groups", + srcs = ["dependency_groups.bzl"], + deps = [ + ":pep508_requirement", + "//python/private:normalize_name", + ], +) + bzl_library( name = "argparse", srcs = ["argparse.bzl"], diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl new file mode 100644 index 0000000000..2ddadee3dd --- /dev/null +++ b/python/private/pypi/dependency_groups.bzl @@ -0,0 +1,142 @@ +"""Select PyPI packages using pyproject.toml dependency groups.""" + +load("//python/private:normalize_name.bzl", "normalize_name") +load(":pep508_requirement.bzl", "requirement") + +def select_dependency_groups(pyproject, specs, _fail = fail): + """Select the packages named by the given dependency group specifiers. + + Supported specifiers are: + + * `"default"`: the `[project] dependencies`. + * `"optional:"`: a `[project.optional-dependencies]` entry. + * `"group:"`: a PEP 735 `[dependency-groups]` entry. + * `"optional:*"` and `"group:*"`: all entries of the table. + * `"*"`: all of the above. + * `""`: a dependency group, or an optional dependency group if + there is no dependency group with that name. + * `"transitive"`: only supported together with `"*"` and selects all + of the packages, including the transitive dependencies. + + Args: + pyproject: {type}`struct` returned by `read_pyproject`. + specs: {type}`list[str]` the specifiers. + _fail: {type}`callable` the failure function, mainly for testing. + + Returns: + {type}`struct` with the attributes: + * `packages`: {type}`list[str]` the sorted normalized package names. + * `all`: {type}`bool` whether all of the packages including the + transitive dependencies should be selected, i.e. `"*"` and + `"transitive"` were given. + """ + groups = {"default": pyproject.dependencies} + for name, reqs in pyproject.optional_dependencies.items(): + groups["optional:" + normalize_name(name)] = reqs + for name, reqs in _expand_dependency_groups(pyproject.dependency_groups, _fail).items(): + groups["group:" + name] = reqs + + if "transitive" in specs: + if "*" not in specs: + _fail("The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: {}".format(specs)) + return struct(packages = [], all = True) + + selected = {} + for spec in specs: + keys = _match(spec, groups) + if not keys: + _fail("Unknown dependency group '{}'. Available groups: {}".format( + spec, + ", ".join(sorted(groups)), + )) + for key in keys: + selected[key] = None + + # A self reference like `my-project[test]` selects the optional + # dependencies, but only one level deep. + project = normalize_name(pyproject.name or "") + packages = {} + for key in selected: + for req in groups[key]: + req = requirement(req) + names = [req.name_] + if req.name_ == project: + names = [ + requirement(r).name_ + for extra in req.extras + for r in groups.get("optional:" + extra, []) + ] + for name in names: + if name: + packages[name] = None + + return struct(packages = sorted(packages), all = False) + +def _match(spec, groups): + if spec == "*": + return groups.keys() + if spec in ("optional:*", "group:*"): + prefix = spec[:-1] + return [key for key in groups if key.startswith(prefix)] + + kind, _, name = spec.rpartition(":") + name = normalize_name(name) + if kind: + candidates = [kind + ":" + name] + elif name == "default": + candidates = ["default"] + else: + candidates = ["group:" + name, "optional:" + name] + + for key in candidates: + if key in groups: + return [key] + return [] + +def _expand_dependency_groups(dependency_groups, _fail): + """Expand the PEP 735 `include-group` entries. + + Args: + dependency_groups: {type}`dict[str, list]` the `[dependency-groups]` + table. + _fail: {type}`callable` the failure function. + + Returns: + {type}`dict[str, list[str]]` normalized group names mapped to their + requirement strings. + """ + groups = { + normalize_name(name): items + for name, items in dependency_groups.items() + } + + # Each round expands at least one group unless there is a cycle. + expanded = {} + for _ in range(len(groups)): + for name, items in groups.items(): + if name in expanded: + continue + + reqs = [] + for item in items: + if type(item) == type(""): + reqs.append(item) + continue + + include = normalize_name(item.get("include-group", "")) + if include not in groups: + _fail("Unknown dependency group '{}' included from '{}'".format(include, name)) + return {} + if include not in expanded: + reqs = None + break + reqs.extend(expanded[include]) + + if reqs != None: + expanded[name] = reqs + + if len(expanded) != len(groups): + _fail("Cyclic dependency group includes: {}".format( + ", ".join(sorted([name for name in groups if name not in expanded])), + )) + return expanded diff --git a/python/private/pyproject_utils.bzl b/python/private/pyproject_utils.bzl index e96ffd2023..a9494b8ac2 100644 --- a/python/private/pyproject_utils.bzl +++ b/python/private/pyproject_utils.bzl @@ -18,10 +18,22 @@ def read_pyproject(module_ctx, pyproject): {type}`struct` with the attributes: * `requires_python`: {type}`str | None` the raw `requires-python` value (e.g. `"==3.13.9"`), or `None` if it is not set. + * `name`: {type}`str | None` the `[project] name`. + * `dependencies`: {type}`list[str]` the `[project] dependencies`. + * `optional_dependencies`: {type}`dict[str, list[str]]` the + `[project.optional-dependencies]` table. + * `dependency_groups`: {type}`dict[str, list]` the PEP 735 + `[dependency-groups]` table. Items are either requirement strings + or `{include-group = ""}` tables. """ data = toml.decode(module_ctx.read(module_ctx.path(pyproject), watch = "yes")) + project = data.get("project", {}) return struct( - requires_python = data.get("project", {}).get("requires-python"), + requires_python = project.get("requires-python"), + name = project.get("name"), + dependencies = project.get("dependencies") or [], + optional_dependencies = project.get("optional-dependencies") or {}, + dependency_groups = data.get("dependency-groups") or {}, ) def version_from_requires_python(requires_python): diff --git a/tests/pypi/dependency_groups/BUILD.bazel b/tests/pypi/dependency_groups/BUILD.bazel new file mode 100644 index 0000000000..ba6877e124 --- /dev/null +++ b/tests/pypi/dependency_groups/BUILD.bazel @@ -0,0 +1,3 @@ +load(":dependency_groups_tests.bzl", "dependency_groups_test_suite") + +dependency_groups_test_suite(name = "dependency_groups_tests") diff --git a/tests/pypi/dependency_groups/dependency_groups_tests.bzl b/tests/pypi/dependency_groups/dependency_groups_tests.bzl new file mode 100644 index 0000000000..ee23aec1fb --- /dev/null +++ b/tests/pypi/dependency_groups/dependency_groups_tests.bzl @@ -0,0 +1,86 @@ +"""dependency_groups tests""" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:dependency_groups.bzl", "select_dependency_groups") # buildifier: disable=bzl-visibility + +_tests = [] + +_PYPROJECT = struct( + name = "my-project", + dependencies = ["main-pkg>=1.0"], + optional_dependencies = { + "Docs": ["docs-pkg"], + "all": ["My_Project[docs, grpc]"], + "grpc": ["grpc-pkg"], + }, + dependency_groups = { + "dev": ["dev-pkg", "shared-pkg", {"include-group": "Test"}], + "test": ["test_pkg", "shared-pkg"], + }, +) + +def _select(specs, pyproject = _PYPROJECT): + failures = [] + got = select_dependency_groups(pyproject, specs, _fail = failures.append) + return got, failures + +def _test_select(env): + for specs, want in [ + (["default"], ["main_pkg"]), + (["optional:docs"], ["docs_pkg"]), + (["group:test"], ["shared_pkg", "test_pkg"]), + (["dev"], ["dev_pkg", "shared_pkg", "test_pkg"]), + (["grpc"], ["grpc_pkg"]), + (["all"], ["docs_pkg", "grpc_pkg"]), + (["optional:*"], ["docs_pkg", "grpc_pkg"]), + (["group:*"], ["dev_pkg", "shared_pkg", "test_pkg"]), + (["default", "group:test"], ["main_pkg", "shared_pkg", "test_pkg"]), + (["*"], ["dev_pkg", "docs_pkg", "grpc_pkg", "main_pkg", "shared_pkg", "test_pkg"]), + ]: + got, failures = _select(specs) + env.expect.where(specs = specs).that_collection(failures).contains_exactly([]) + env.expect.where(specs = specs).that_collection(got.packages).contains_exactly(want).in_order() + env.expect.where(specs = specs).that_bool(got.all).equals(False) + +_tests.append(_test_select) + +def _test_select_all(env): + got, failures = _select(["*", "transitive"]) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_bool(got.all).equals(True) + env.expect.that_collection(got.packages).contains_exactly([]) + +_tests.append(_test_select_all) + +def _test_select_errors(env): + for specs, want in [ + (["missing"], "Unknown dependency group 'missing'. Available groups: default, group:dev, group:test, optional:all, optional:docs, optional:grpc"), + (["default", "transitive"], "The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: [\"default\", \"transitive\"]"), + ]: + _, failures = _select(specs) + env.expect.where(specs = specs).that_collection(failures).contains_exactly([want]) + +_tests.append(_test_select_errors) + +def _test_include_group_errors(env): + for groups, want in [ + ({"a": [{"include-group": "missing"}]}, "Unknown dependency group 'missing' included from 'a'"), + ({"a": [{"include-group": "b"}], "b": [{"include-group": "a"}]}, "Cyclic dependency group includes: a, b"), + ]: + _, failures = _select(["*"], pyproject = struct( + name = None, + dependencies = [], + optional_dependencies = {}, + dependency_groups = groups, + )) + env.expect.that_collection(failures).contains_exactly([want]) + +_tests.append(_test_include_group_errors) + +def dependency_groups_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) From 755737072827673c74afba95147995086a579543 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:07:32 +0900 Subject: [PATCH 02/55] refactor(pypi): support hiding the unexposed packages of a hub Add the `hide_unexposed_packages` attribute to the hub repository. When set, the aliases of the packages that are not exposed are only visible to the wheel repositories that depend on them. --- python/private/pypi/hub_repository.bzl | 13 +++- python/private/pypi/render_pkg_aliases.bzl | 45 +++++++++++++- .../render_pkg_aliases_test.bzl | 62 +++++++++++++++++++ 3 files changed, 115 insertions(+), 5 deletions(-) diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index f915aa1c77..d421f3cc09 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -26,12 +26,16 @@ exports_files(["requirements.bzl"]) """ def _impl(rctx): - bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() + if rctx.attr.hide_unexposed_packages: + bzl_packages = rctx.attr.packages + else: + bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() aliases = render_multiplatform_pkg_aliases( aliases = { key: _whl_config_settings_from_json(values) for key, values in rctx.attr.whl_map.items() }, + exposed_packages = bzl_packages if rctx.attr.hide_unexposed_packages else None, extra_hub_aliases = rctx.attr.extra_hub_aliases, requirement_cycles = rctx.attr.groups, platform_config_settings = rctx.attr.platform_config_settings, @@ -78,10 +82,15 @@ hub_repository = repository_rule( "groups": attr.string_list_dict( mandatory = False, ), + "hide_unexposed_packages": attr.bool( + default = False, + doc = "Make the aliases of the packages that are not in `packages` visible only to the wheel repositories.", + ), "packages": attr.string_list( mandatory = False, doc = """\ -The list of packages that will be exposed via all_*requirements macros. Defaults to whl_map keys. +The list of packages that will be exposed via all_*requirements macros. Defaults to whl_map keys +unless `hide_unexposed_packages` is set. """, ), "platform_config_settings": attr.string_list_dict( diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 0a1c328491..5582cc00c9 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -65,7 +65,7 @@ def _repr_actual(aliases): else: return render.dict(aliases, key_repr = _repr_config_setting) -def _render_common_aliases(*, name, aliases, **kwargs): +def _render_common_aliases(*, name, aliases, visibility, **kwargs): pkg_aliases = render.call( "pkg_aliases", name = repr(name), @@ -80,14 +80,21 @@ def _render_common_aliases(*, name, aliases, **kwargs): return """\ load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") {extra_loads} -package(default_visibility = ["//visibility:public"]) +package(default_visibility = {visibility}) {aliases}""".format( aliases = pkg_aliases, extra_loads = extra_loads, + visibility = render.list(visibility), ) -def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases = {}, **kwargs): +def render_pkg_aliases( + *, + aliases, + requirement_cycles = None, + extra_hub_aliases = {}, + exposed_packages = None, + **kwargs): """Create alias declarations for each PyPI package. The aliases should be appended to the pip_repository BUILD.bazel file. These aliases @@ -100,6 +107,9 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases requirement_cycles: any package groups to also add. extra_hub_aliases: The list of extra aliases for each whl to be added in addition to the default ones. + exposed_packages: The packages that should be public. When set, the + aliases of the other packages are only visible to the wheel + repositories. **kwargs: Extra kwargs to pass to the rules. Returns: @@ -124,12 +134,18 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases for whl_name in group_whls } + public = ["//visibility:public"] + if exposed_packages != None: + exposed_packages = {normalize_name(name): None for name in exposed_packages} + internal_visibility = _internal_visibility(aliases, has_groups = bool(requirement_cycles)) + files = { "{}/BUILD.bazel".format(normalize_name(name)): _render_common_aliases( name = normalize_name(name), aliases = pkg_aliases, extra_aliases = extra_hub_aliases.get(normalize_name(name), []), group_name = whl_group_mapping.get(normalize_name(name)), + visibility = public if exposed_packages == None or normalize_name(name) in exposed_packages else internal_visibility, **kwargs ).strip() for name, pkg_aliases in aliases.items() @@ -139,6 +155,29 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases files["_groups/BUILD.bazel"] = generate_group_library_build_bazel("", requirement_cycles) return files +def _internal_visibility(aliases, has_groups = False): + """The visibility of the packages that are not exposed. + + The wheel repositories depend on the aliases of the hub, so the aliases + have to be visible to them. + """ + repo_names = {} + for pkg_aliases in aliases.values(): + if type(pkg_aliases) == type(""): + repo_names[pkg_aliases] = None + continue + + for repo_name in pkg_aliases.values(): + repo_names[repo_name] = None + + visibility = [ + "@{}//:__pkg__".format(repo_name) + for repo_name in sorted(repo_names) + ] + if has_groups: + visibility.append("//_groups:__pkg__") + return visibility + def _major_minor(python_version): major, _, tail = python_version.partition(".") minor, _, _ = tail.partition(".") diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index 9114f59279..13bee38ff5 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -58,6 +58,68 @@ pkg_aliases( _tests.append(_test_legacy_aliases) +def _test_hidden_aliases(env): + actual = render_pkg_aliases( + aliases = { + "bar": "pypi_bar", + "foo": "pypi_foo", + }, + exposed_packages = ["foo"], + ) + + env.expect.that_str(actual["bar/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = [ + "@pypi_bar//:__pkg__", + "@pypi_foo//:__pkg__", +]) + +pkg_aliases( + name = "bar", + actual = "pypi_bar", +)""") + env.expect.that_str(actual["foo/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = ["//visibility:public"]) + +pkg_aliases( + name = "foo", + actual = "pypi_foo", +)""") + +_tests.append(_test_hidden_aliases) + +def _test_hidden_aliases_in_groups(env): + actual = render_pkg_aliases( + aliases = { + "bar": "pypi_bar", + "baz": "pypi_baz", + }, + requirement_cycles = { + "group": ["bar", "baz"], + }, + exposed_packages = ["baz"], + ) + + env.expect.that_str(actual["bar/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = [ + "@pypi_bar//:__pkg__", + "@pypi_baz//:__pkg__", + "//_groups:__pkg__", +]) + +pkg_aliases( + name = "bar", + actual = "pypi_bar", + group_name = "group", +)""") + +_tests.append(_test_hidden_aliases_in_groups) + def _test_bzlmod_aliases(env): # Use this function as it is used in pip_repository actual = render_multiplatform_pkg_aliases( From 23127e5b1bea7fd8a1be0f3dfbab80b4134b8399 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:08:09 +0900 Subject: [PATCH 03/55] feat(pypi): add pip.configure to select the exposed packages of a hub Add the `pip.configure` tag class that limits the packages that a hub exposes to the ones listed in the dependency groups of a `pyproject.toml` file. The other packages of the hub are still used by the wheels that need them, but their aliases are not public. --- docs/pypi/download.md | 50 ++++++ news/4395.added.md | 2 + python/private/pypi/BUILD.bazel | 10 ++ python/private/pypi/extension.bzl | 108 +++++++++++++ python/private/pypi/hub_exposure.bzl | 84 ++++++++++ tests/integration/unified_pypi/BUILD.bazel | 16 ++ tests/integration/unified_pypi/MODULE.bazel | 14 ++ .../unified_pypi/pyproject_filtered.toml | 4 + tests/integration/unified_pypi_test.py | 13 ++ tests/pypi/extension/extension_tests.bzl | 92 ++++++++++- tests/pypi/extension/pip_parse.bzl | 12 ++ tests/pypi/hub_exposure/BUILD.bazel | 3 + .../pypi/hub_exposure/hub_exposure_tests.bzl | 153 ++++++++++++++++++ 13 files changed, 559 insertions(+), 2 deletions(-) create mode 100644 news/4395.added.md create mode 100644 python/private/pypi/hub_exposure.bzl create mode 100644 tests/integration/unified_pypi/pyproject_filtered.toml create mode 100644 tests/pypi/hub_exposure/BUILD.bazel create mode 100644 tests/pypi/hub_exposure/hub_exposure_tests.bzl diff --git a/docs/pypi/download.md b/docs/pypi/download.md index cc8b7890aa..5f16606d24 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -134,6 +134,56 @@ rename, and `requirement("numpy")` routes through the active hub like fixed at loading time, before the active hub is known, so load them from a concrete hub instead. Prefer `@pypi//` labels in new code. +(pip-configure)= +### Configuring the Exposed Packages (pip.configure) + +By default a hub exposes all of the packages from its requirements files or +`uv.lock` file, including the transitive dependencies. A `pip.configure` tag +can limit the hub to the packages listed in a `pyproject.toml` file: the +`[project] dependencies`, `[project.optional-dependencies]` and PEP 735 +`[dependency-groups]` tables. + +```toml +[project] +dependencies = ["numpy"] + +[dependency-groups] +dev = ["pytest"] +``` + +```starlark +pip.parse( + hub_name = "pypi_hub", + pyproject_toml = "//:pyproject.toml", + requirements_lock = "//:requirements_lock.txt", +) + +pip.configure( + hub_name = "pypi_hub", + dependency_groups = ["default", "group:dev"], +) +``` + +Here `@pypi_hub//numpy` and `@pypi_hub//pytest` can be used, but the +dependencies of `numpy` are only used through `@pypi_hub//numpy`. The +`all_requirements` lists in the hub's `requirements.bzl` and the +[Unified @pypi Hub](unified-pypi-hub) only contain the exposed packages. + +The {attr}`pip.configure.dependency_groups` attribute defaults to `["*"]`, +which selects every group. Use `["*", "transitive"]` to expose all of the +packages of the hub, including the transitive dependencies. Selecting the +transitive dependencies of only some of the groups is not supported. + +Multiple tags for the same hub are combined. Only the root module and the +module that created the hub can configure it; if both do, the root module +wins. A self reference such as `my-project[test]` in the `pyproject.toml` +selects the optional dependencies of the project. The packages that are not +part of the hub are ignored, so a single `pyproject.toml` can list tools that +are missing from the lock file. + +:::{versionadded} VERSION_NEXT_FEATURE +::: + ### Declaring Abstract Dependencies (pip.dep) :::{versionadded} 2.2.0 diff --git a/news/4395.added.md b/news/4395.added.md new file mode 100644 index 0000000000..62d7f10880 --- /dev/null +++ b/news/4395.added.md @@ -0,0 +1,2 @@ +(pypi) Added {obj}`pip.configure` to limit the packages that a hub exposes to +the ones listed in the dependency groups of a `pyproject.toml` file. diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index af005f706e..caf482b67f 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -129,6 +129,7 @@ bzl_library( deps = [ ":default_platforms", ":hub_builder", + ":hub_exposure", ":hub_repository", ":parse_whl_name", ":pep508_env", @@ -641,6 +642,15 @@ bzl_library( ], ) +bzl_library( + name = "hub_exposure", + srcs = ["hub_exposure.bzl"], + deps = [ + ":dependency_groups", + "//python/private:pyproject_utils", + ], +) + bzl_library( name = "argparse", srcs = ["argparse.bzl"], diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index f7a1dadaf7..e64e856489 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -24,6 +24,7 @@ load("//python/private:pyproject_utils.bzl", "read_pyproject", "version_from_req load("//python/private:repo_utils.bzl", "repo_utils") load(":default_platforms.bzl", "default_platforms") load(":hub_builder.bzl", "hub_builder") +load(":hub_exposure.bzl", "hub_exposure") load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") load(":parse_whl_name.bzl", "parse_whl_name") load(":pep508_env.bzl", "env") @@ -89,6 +90,7 @@ def build_config( unified_hub_requirements_bzl = False defaults = { "platforms": default_platforms(), + "pyproject_toml": None, "python_version": None, } for mod in module_ctx.modules: @@ -105,6 +107,8 @@ def build_config( unified_hub_requirements_bzl = True pyproject_toml = tag.pyproject_toml if pyproject_toml: + if mod.is_root: + defaults["pyproject_toml"] = pyproject_toml pyproject = read_pyproject(module_ctx, pyproject_toml) if pyproject.requires_python: defaults["python_version"] = version_from_requires_python(pyproject.requires_python) @@ -151,6 +155,7 @@ def build_config( for name, values in defaults["platforms"].items() }, enable_pipstar_extract = enable_pipstar_extract, + pyproject_toml = defaults["pyproject_toml"], toml_decode = toml.decode, unified_hub_requirements_bzl = unified_hub_requirements_bzl, ) @@ -243,6 +248,7 @@ You cannot use both the additive_build_content and additive_build_content_file a # dict[str repo, HubBuilder] # See `hub_builder.bzl%hub_builder()` for `HubBuilder` pip_hub_map = {} + hub_pyproject = {} simpleapi_cache = pypi_cache(mctx = module_ctx) is_pypi_hub_reserved = module_ctx.getenv("RULES_PYTHON_PYPI_HUB_RESERVED", "0") == "1" @@ -319,6 +325,9 @@ You cannot use both the additive_build_content and additive_build_content_file a else: builder = pip_hub_map[hub_name] + if pip_attr.pyproject_toml and hub_name not in hub_pyproject: + hub_pyproject[hub_name] = pip_attr.pyproject_toml + builder.pip_parse( module_ctx, pip_attr = pip_attr, @@ -342,6 +351,7 @@ You cannot use both the additive_build_content and additive_build_content_file a exposed_packages = {} extra_aliases = {} whl_libraries = {} + hubs = {} for hub in pip_hub_map.values(): out = hub.build() @@ -355,8 +365,25 @@ You cannot use both the additive_build_content and additive_build_content_file a extra_aliases[hub.name] = out.extra_aliases hub_group_map[hub.name] = out.group_map hub_whl_map[hub.name] = out.whl_map + hubs[hub.name] = struct( + module_name = hub.module_name, + pyproject = hub_pyproject.get(hub.name), + packages = out.whl_map.keys(), + exposed_packages = out.exposed_packages, + ) + + configured_hubs = hub_exposure( + module_ctx, + hubs = hubs, + default_pyproject = config.pyproject_toml, + _fail = _fail, + ) + if configured_hubs == None: + return None + exposed_packages.update(configured_hubs) return struct( + configured_hubs = configured_hubs, config = config, declared_deps = declared_deps, default_hub = config.default_hub or renamed_default_hub, @@ -511,6 +538,8 @@ def _pip_impl(module_ctx): packages = mods.exposed_packages.get(hub_name, []), platform_config_settings = mods.platform_config_settings.get(hub_name, {}), groups = mods.hub_group_map.get(hub_name), + # The hubs configured with `pip.configure` hide the other packages. + hide_unexposed_packages = hub_name in mods.configured_hubs, ) _create_unified_hub_repo(mods) @@ -638,6 +667,10 @@ This is designed to work with dependency management tools like Renovate. :::{versionadded} 2.3.0 ::: + +:::{versionchanged} VERSION_NEXT_FEATURE +Also used by {obj}`pip.configure` for the hubs of the root module. +::: """, ), "unified_hub_requirements_bzl": attr.bool( @@ -841,6 +874,10 @@ The version must be specified as `==X.Y.Z` (exact version with full semver). :::{versionadded} 2.3.0 ::: + +:::{versionchanged} VERSION_NEXT_FEATURE +Also used by {obj}`pip.configure` to select the packages that the hub exposes. +::: """, ), "python_version": attr.string( @@ -1026,6 +1063,72 @@ execution-phase error when built. """, ) +_configure_tag = tag_class( + attrs = { + "dependency_groups": attr.string_list( + default = ["*"], + doc = """\ +The `pyproject.toml` dependency groups whose packages the hub exposes. + +Each item is one of: + +* `"default"`: the `[project] dependencies`. +* `"optional:"`: the `[project.optional-dependencies]` entry ``. +* `"group:"`: the PEP 735 `[dependency-groups]` entry ``. +* `"optional:*"` or `"group:*"`: all entries of the table. +* `"*"`: all of the above. +* `""`: the dependency group `` if it exists, otherwise the + optional dependency group ``. +* `"transitive"`: only supported together with `"*"`, exposes all of the + packages of the hub including the transitive dependencies, i.e. all of the + packages from the requirements files or the `uv.lock` file. + +See [Configuring the Exposed Packages](#pip-configure) for examples. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + "hub_name": attr.string( + mandatory = True, + doc = """\ +The name of the hub to configure. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + "pyproject_toml": attr.label( + mandatory = False, + doc = """\ +The `pyproject.toml` file that defines the dependency groups. Defaults to the +{attr}`pip.parse.pyproject_toml` of the hub, then to +{attr}`pip.default.pyproject_toml` if the hub is defined by the root module. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + }, + doc = """\ +Configure the packages exposed by a hub. + +The packages are selected from a `pyproject.toml` file using the +`[project] dependencies`, `[project.optional-dependencies]` and PEP 735 +`[dependency-groups]` tables. The other packages of the hub, e.g. the +transitive dependencies, are still used by the packages that need them, but +they are not public. + +Only the root module and the module that created the hub can configure it. If +both do, the root module wins. Multiple tags for the same hub are combined. + +See [Configuring the Exposed Packages](#pip-configure) for examples. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", +) + pypi = module_extension( environ = ["RULES_PYTHON_PYPI_HUB_RESERVED"], doc = """\ @@ -1049,12 +1152,17 @@ Environment Variables: - `RULES_PYTHON_PYPI_HUB_RESERVED`: Enable fallback renaming for reserved hub name collisions. See the {envvar}`RULES_PYTHON_PYPI_HUB_RESERVED` documentation for details. +pip.configure: +This tag class configures the packages exposed by a hub using the dependency +groups from a pyproject.toml file. + pip.whl_mods: This tag class is used to help create JSON files to describe modifications to the BUILD files for wheels. """, implementation = _pip_impl, tag_classes = { + "configure": _configure_tag, "default": tag_class( attrs = _default_attrs, doc = """\ diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl new file mode 100644 index 0000000000..6ca9315a81 --- /dev/null +++ b/python/private/pypi/hub_exposure.bzl @@ -0,0 +1,84 @@ +"""Compute the packages exposed by the hubs configured with `pip.configure`.""" + +load("//python/private:pyproject_utils.bzl", "read_pyproject") +load(":dependency_groups.bzl", "select_dependency_groups") + +def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): + """Compute the packages exposed by the hubs configured with `pip.configure`. + + Args: + module_ctx: {type}`module_ctx` the module context. + hubs: {type}`dict[str, struct]` the hubs by name. Each struct has: + * `module_name`: {type}`str` the module that created the hub. + * `pyproject`: {type}`Label | None` from `pip.parse`. + * `packages`: {type}`list[str]` the normalized names of all + packages in the hub. + * `exposed_packages`: {type}`list[str]` the packages that the hub + exposes when it is not configured. + default_pyproject: {type}`Label | None` from `pip.default`, only used + for the hubs of the root module. + _fail: {type}`callable` the failure function, mainly for testing. + + Returns: + {type}`dict[str, list[str]] | None` the configured hubs mapped to the + sorted packages that they expose. The hubs that are not configured + are not included. Returns `None` on failure. + """ + + # The root module wins over the owner of the hub when both configure it. + root_modules = [mod for mod in module_ctx.modules if mod.is_root] + root_name = root_modules[0].name if root_modules else None + root_configured = { + tag.hub_name: None + for mod in root_modules + for tag in mod.tags.configure + } + + configured = {} + for mod in module_ctx.modules: + for tag in mod.tags.configure: + hub = tag.hub_name + if hub not in hubs: + _fail("pip.configure() refers to unknown hub '{}'. Available hubs: {}".format( + hub, + ", ".join(sorted(hubs)), + )) + return None + + info = hubs[hub] + if not mod.is_root and (mod.name != info.module_name or hub in root_configured): + # Only the root module and the module that created the hub can + # configure it. + continue + + pyproject = ( + tag.pyproject_toml or + info.pyproject or + (default_pyproject if info.module_name == root_name else None) + ) + if not pyproject: + _fail("pip.configure() for hub '{}' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`".format(hub)) + return None + + selection = select_dependency_groups( + read_pyproject(module_ctx, pyproject), + tag.dependency_groups, + _fail = _fail, + ) + if selection.all: + packages = info.exposed_packages + else: + packages = selection.packages + + # Multiple tags for the same hub are combined. + exposed = configured.setdefault(hub, {}) + for name in packages: + # The pyproject.toml may list packages that are not in the hub, + # e.g. tools that are not locked. + if name in info.packages: + exposed[name] = None + + return { + hub: sorted(packages) + for hub, packages in configured.items() + } diff --git a/tests/integration/unified_pypi/BUILD.bazel b/tests/integration/unified_pypi/BUILD.bazel index 13d188d087..f3a9513d1f 100644 --- a/tests/integration/unified_pypi/BUILD.bazel +++ b/tests/integration/unified_pypi/BUILD.bazel @@ -95,3 +95,19 @@ platform( "@platforms//os:windows", ], ) + +# The pypi_filtered hub only makes its pyproject.toml packages public. +py_test( + name = "test_filtered_direct", + srcs = ["test_default.py"], + main = "test_default.py", + deps = ["@pypi_filtered//colorama"], +) + +py_binary( + name = "bin_filtered_six", + srcs = ["bin_six_a.py"], + main = "bin_six_a.py", + tags = ["manual"], + deps = ["@pypi_filtered//six"], +) diff --git a/tests/integration/unified_pypi/MODULE.bazel b/tests/integration/unified_pypi/MODULE.bazel index ee8f622cb4..70e01ccdac 100644 --- a/tests/integration/unified_pypi/MODULE.bazel +++ b/tests/integration/unified_pypi/MODULE.bazel @@ -53,6 +53,20 @@ pip.parse( ) use_repo(pip, "pypi_b") +# `pypi_filtered` only exposes the `[project] dependencies` of the +# `pyproject.toml`, the other packages are not public. +pip.parse( + hub_name = "pypi_filtered", + pyproject_toml = "//:pyproject_filtered.toml", + python_version = "3.11", + requirements_lock = "//:requirements_b.txt", +) +pip.configure( + dependency_groups = ["default"], + hub_name = "pypi_filtered", +) +use_repo(pip, "pypi_filtered") + pip.default(default_hub = "pypi_b") pip.default(unified_hub_requirements_bzl = True) pip.dep( diff --git a/tests/integration/unified_pypi/pyproject_filtered.toml b/tests/integration/unified_pypi/pyproject_filtered.toml new file mode 100644 index 0000000000..632f72fe22 --- /dev/null +++ b/tests/integration/unified_pypi/pyproject_filtered.toml @@ -0,0 +1,4 @@ +[project] +name = "unified-pypi" +version = "0.0.0" +dependencies = ["colorama"] diff --git a/tests/integration/unified_pypi_test.py b/tests/integration/unified_pypi_test.py index 9ce41fafb7..5ac2f7df8b 100644 --- a/tests/integration/unified_pypi_test.py +++ b/tests/integration/unified_pypi_test.py @@ -7,6 +7,19 @@ class UnifiedPypiTest(runner.TestCase): + def test_configure_hides_unexposed_packages(self): + self.run_bazel("test", "//:test_filtered_direct") + result = self.run_bazel("build", "//:bin_filtered_six", check=False) + self.assertNotEqual( + result.exit_code, + 0, + "Expected the alias of the unexposed package to be hidden", + ) + self.assert_result_matches( + result, + r"alias '[^']*//six:six'.*is not visible from", + ) + def test_default_fallback_hub(self): self.run_bazel("test", "//:test_default") diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index ee8e58dfe4..17f8d6741e 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -21,7 +21,7 @@ load("//python/private/pypi:extension.bzl", "build_config", "parse_modules") # load("//python/private/pypi:platform.bzl", _plat = "platform") # buildifier: disable=bzl-visibility load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # buildifier: disable=bzl-visibility load("//tests/support/mocks:mocks.bzl", "mocks") -load(":pip_parse.bzl", _parse = "pip_parse") +load(":pip_parse.bzl", _configure = "configure", _parse = "pip_parse") _tests = [] @@ -101,10 +101,11 @@ def _dep(*, name, extra_targets = []): extra_targets = extra_targets, ) -def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], dep = [], is_root = True): +def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], dep = [], configure = [], is_root = True): return struct( name = name, tags = struct( + configure = configure, parse = parse, override = override, whl_mods = whl_mods, @@ -118,6 +119,7 @@ def _parse_modules(env, **kwargs): return env.expect.that_struct( parse_modules(**kwargs), attrs = dict( + configured_hubs = subjects.dict, declared_deps = subjects.dict, default_hub = subjects.str, exposed_packages = subjects.dict, @@ -766,6 +768,92 @@ simple==0.0.1 \ _tests.append(_test_default_index_setting) +_CONFIGURE_REQUIREMENTS = """\ +simple==0.0.1 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +other-pkg==0.0.2 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +transitive-pkg==0.0.3 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +""" + +_CONFIGURE_PYPROJECT = """\ +[project] +dependencies = ["simple"] + +[dependency-groups] +dev = ["other-pkg"] +test = ["simple"] +""" + +def _configure_parse_modules(env, *modules): + return _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + os_name = "linux", + arch_name = "x86_64", + mock_files = { + "pyproject.toml": _CONFIGURE_PYPROJECT, + "requirements.txt": _CONFIGURE_REQUIREMENTS, + }, + *modules + ), + available_interpreters = { + "python_3_15_host": "unit_test_interpreter_target", + }, + minor_mapping = {"3.15": "3.15.19"}, + ) + +def _configure_parse(*, hub_name = "pypi_a", pyproject_toml = "pyproject.toml"): + return _parse( + hub_name = hub_name, + pyproject_toml = pyproject_toml, + python_version = "3.15", + simpleapi_skip = ["simple", "other-pkg", "transitive-pkg"], + requirements_lock = "requirements.txt", + ) + +def _test_configure(env): + for groups, want in [ + (["*"], ["other_pkg", "simple"]), + (["test"], ["simple"]), + (["dev"], ["other_pkg"]), + (["*", "transitive"], ["other_pkg", "simple", "transitive_pkg"]), + ]: + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [_configure_parse()], + configure = [_configure(hub_name = "pypi_a", dependency_groups = groups)], + ), + ) + + pypi.exposed_packages().contains_exactly({"pypi_a": want}) + pypi.configured_hubs().contains_exactly({"pypi_a": want}) + +_tests.append(_test_configure) + +def _test_not_configured_exposes_all(env): + # The pyproject.toml does not filter the hub without `pip.configure`. + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [_configure_parse()], + ), + ) + + pypi.exposed_packages().contains_exactly({ + "pypi_a": ["other_pkg", "simple", "transitive_pkg"], + }) + pypi.configured_hubs().contains_exactly({}) + +_tests.append(_test_not_configured_exposes_all) + def extension_test_suite(name): """Create the test suite. diff --git a/tests/pypi/extension/pip_parse.bzl b/tests/pypi/extension/pip_parse.bzl index 7b5bdfdfd6..505c249169 100644 --- a/tests/pypi/extension/pip_parse.bzl +++ b/tests/pypi/extension/pip_parse.bzl @@ -1,5 +1,17 @@ """A simple test helper""" +def configure( + *, + dependency_groups = ["*"], + hub_name, + pyproject_toml = None): + """A simple helper for testing to simulate the PyPI extension configure tag class""" + return struct( + dependency_groups = dependency_groups, + hub_name = hub_name, + pyproject_toml = pyproject_toml, + ) + def pip_parse( *, hub_name, diff --git a/tests/pypi/hub_exposure/BUILD.bazel b/tests/pypi/hub_exposure/BUILD.bazel new file mode 100644 index 0000000000..187361e1ce --- /dev/null +++ b/tests/pypi/hub_exposure/BUILD.bazel @@ -0,0 +1,3 @@ +load(":hub_exposure_tests.bzl", "hub_exposure_test_suite") + +hub_exposure_test_suite(name = "hub_exposure_tests") diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl new file mode 100644 index 0000000000..cafbb229b5 --- /dev/null +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -0,0 +1,153 @@ +"""hub_exposure tests""" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:hub_exposure.bzl", "hub_exposure") # buildifier: disable=bzl-visibility +load("//tests/support/mocks:mocks.bzl", "mocks") + +_tests = [] + +_PYPROJECT = """\ +[project] +dependencies = ["simple"] + +[dependency-groups] +dev = ["other-pkg"] +test = ["simple", "not-in-hub"] +""" + +def _hub(*, module_name = "root", pyproject = None): + return struct( + module_name = module_name, + pyproject = pyproject, + packages = ["other_pkg", "simple", "transitive_pkg"], + exposed_packages = ["other_pkg", "simple", "transitive_pkg"], + ) + +def _configure(hub_name = "hub", dependency_groups = ["*"], pyproject_toml = None): + return mocks.tag( + hub_name = hub_name, + dependency_groups = dependency_groups, + pyproject_toml = pyproject_toml, + ) + +def _module(name = "root", is_root = True, configure = []): + return mocks.module(name = name, is_root = is_root, configure = configure) + +def _exposure(*modules, hubs = None, default_pyproject = None): + failures = [] + got = hub_exposure( + mocks.mctx( + modules = list(modules), + mock_files = { + "other.toml": "[project]\ndependencies = [\"other-pkg\"]\n", + "pyproject.toml": _PYPROJECT, + }, + ), + hubs = hubs or {"hub": _hub(pyproject = "pyproject.toml")}, + default_pyproject = default_pyproject, + _fail = failures.append, + ) + return got, failures + +def _test_not_configured(env): + got, failures = _exposure(_module()) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({}) + +_tests.append(_test_not_configured) + +def _test_configure(env): + for groups, want in [ + (["*"], ["other_pkg", "simple"]), + (["default"], ["simple"]), + (["dev"], ["other_pkg"]), + # `not-in-hub` is ignored + (["group:test"], ["simple"]), + (["*", "transitive"], ["other_pkg", "simple", "transitive_pkg"]), + ]: + got, failures = _exposure(_module(configure = [_configure(dependency_groups = groups)])) + env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) + env.expect.where(groups = groups).that_dict(got).contains_exactly({"hub": want}) + +_tests.append(_test_configure) + +def _test_configure_combines_tags(env): + got, failures = _exposure(_module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(dependency_groups = ["dev"]), + ])) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg", "simple"]}) + +_tests.append(_test_configure_combines_tags) + +def _test_configure_pyproject_precedence(env): + # The tag wins over the `pip.parse` value. + got, _ = _exposure(_module(configure = [ + _configure(pyproject_toml = "other.toml"), + ])) + env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) + + # The `pip.default` value is used for the hubs of the root module. + got, failures = _exposure( + _module(configure = [_configure(dependency_groups = ["default"])]), + hubs = {"hub": _hub()}, + default_pyproject = "pyproject.toml", + ) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({"hub": ["simple"]}) + +_tests.append(_test_configure_pyproject_precedence) + +def _test_configure_module_precedence(env): + hubs = {"hub": _hub(module_name = "dep", pyproject = "pyproject.toml")} + + # The owner of the hub can configure it. + got, _ = _exposure( + _module(), + _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + hubs = hubs, + ) + env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) + + # The root module wins over the owner. + got, _ = _exposure( + _module(configure = [_configure(dependency_groups = ["default"])]), + _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + hubs = hubs, + ) + env.expect.that_dict(got).contains_exactly({"hub": ["simple"]}) + + # Other modules are ignored. + got, _ = _exposure( + _module(), + _module(name = "other", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + hubs = hubs, + ) + env.expect.that_dict(got).contains_exactly({}) + +_tests.append(_test_configure_module_precedence) + +def _test_configure_errors(env): + _, failures = _exposure(_module(configure = [_configure(hub_name = "missing")])) + env.expect.that_collection(failures).contains_exactly([ + "pip.configure() refers to unknown hub 'missing'. Available hubs: hub", + ]) + + _, failures = _exposure( + _module(configure = [_configure()]), + hubs = {"hub": _hub()}, + ) + env.expect.that_collection(failures).contains_exactly([ + "pip.configure() for hub 'hub' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`", + ]) + +_tests.append(_test_configure_errors) + +def hub_exposure_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) From 6bb7f8379a68c92c78b12acb4a0fad3b8c99a36a Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:11:22 +0900 Subject: [PATCH 04/55] fix(pypi): only add the exposed packages of a configured hub to @pypi Otherwise the unified hub would point to the hidden aliases of the hub and fail with a visibility error instead of the missing package error. --- python/private/pypi/extension.bzl | 4 +++- tests/integration/unified_pypi/BUILD.bazel | 12 ++++++++++++ tests/integration/unified_pypi_test.py | 13 +++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index e64e856489..ce5e95492d 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -416,7 +416,9 @@ def _create_unified_hub_repo(mods): extra_aliases = {} for hub_name in hubs: - for pkg_name in mods.hub_whl_map[hub_name]: + # The hubs configured with `pip.configure` only expose some packages. + hub_packages = mods.configured_hubs.get(hub_name, mods.hub_whl_map[hub_name]) + for pkg_name in hub_packages: norm_pkg = normalize_name(pkg_name) if norm_pkg not in packages: packages[norm_pkg] = [] diff --git a/tests/integration/unified_pypi/BUILD.bazel b/tests/integration/unified_pypi/BUILD.bazel index f3a9513d1f..78f9a857ee 100644 --- a/tests/integration/unified_pypi/BUILD.bazel +++ b/tests/integration/unified_pypi/BUILD.bazel @@ -104,6 +104,18 @@ py_test( deps = ["@pypi_filtered//colorama"], ) +# six is not exposed by pypi_filtered, so it is missing in the unified hub. +py_binary( + name = "bin_filtered_six_unified", + srcs = ["bin_six_a.py"], + config_settings = { + "@rules_python//python/config_settings:venv": "pypi_filtered", + }, + main = "bin_six_a.py", + tags = ["manual"], + deps = ["@pypi//six"], +) + py_binary( name = "bin_filtered_six", srcs = ["bin_six_a.py"], diff --git a/tests/integration/unified_pypi_test.py b/tests/integration/unified_pypi_test.py index 5ac2f7df8b..f59a19fb42 100644 --- a/tests/integration/unified_pypi_test.py +++ b/tests/integration/unified_pypi_test.py @@ -20,6 +20,19 @@ def test_configure_hides_unexposed_packages(self): r"alias '[^']*//six:six'.*is not visible from", ) + def test_configure_hides_unexposed_packages_in_unified_hub(self): + self.run_bazel("cquery", "//:bin_filtered_six_unified") + result = self.run_bazel("build", "//:bin_filtered_six_unified", check=False) + self.assertNotEqual( + result.exit_code, + 0, + "Expected build to fail during execution phase", + ) + self.assert_result_matches( + result, + 'ERROR: PyPI package "six" is not available when building under PyPI hub "pypi_filtered"\\. Try adding it to the requirements of this hub', + ) + def test_default_fallback_hub(self): self.run_bazel("test", "//:test_default") From b31a734fd273538849adba68184df7b801212fce Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:05:33 +0900 Subject: [PATCH 05/55] fix(pypi): treat exposing all packages as not configuring the hub pip.configure with ["*", "transitive"] used the packages that are available on all platforms, which made the platform-specific packages private. Treat it as if the hub was not configured instead, which keeps these packages public and out of all_requirements as before. --- docs/pypi/download.md | 5 ++-- python/private/pypi/extension.bzl | 1 - python/private/pypi/hub_exposure.bzl | 24 +++++++++---------- .../pypi/hub_exposure/hub_exposure_tests.bzl | 14 +++++++++-- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/docs/pypi/download.md b/docs/pypi/download.md index 5f16606d24..654e156921 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -171,8 +171,9 @@ dependencies of `numpy` are only used through `@pypi_hub//numpy`. The The {attr}`pip.configure.dependency_groups` attribute defaults to `["*"]`, which selects every group. Use `["*", "transitive"]` to expose all of the -packages of the hub, including the transitive dependencies. Selecting the -transitive dependencies of only some of the groups is not supported. +packages of the hub, including the transitive dependencies, which is the same +as not configuring the hub. Selecting the transitive dependencies of only some +of the groups is not supported. Multiple tags for the same hub are combined. Only the root module and the module that created the hub can configure it; if both do, the root module diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index ce5e95492d..10e0c456a2 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -369,7 +369,6 @@ You cannot use both the additive_build_content and additive_build_content_file a module_name = hub.module_name, pyproject = hub_pyproject.get(hub.name), packages = out.whl_map.keys(), - exposed_packages = out.exposed_packages, ) configured_hubs = hub_exposure( diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl index 6ca9315a81..fe9e586153 100644 --- a/python/private/pypi/hub_exposure.bzl +++ b/python/private/pypi/hub_exposure.bzl @@ -13,28 +13,27 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): * `pyproject`: {type}`Label | None` from `pip.parse`. * `packages`: {type}`list[str]` the normalized names of all packages in the hub. - * `exposed_packages`: {type}`list[str]` the packages that the hub - exposes when it is not configured. default_pyproject: {type}`Label | None` from `pip.default`, only used for the hubs of the root module. _fail: {type}`callable` the failure function, mainly for testing. Returns: {type}`dict[str, list[str]] | None` the configured hubs mapped to the - sorted packages that they expose. The hubs that are not configured - are not included. Returns `None` on failure. + sorted packages that they expose. The hubs that are not configured or + that expose all of their packages are not included, so that they + behave the same. Returns `None` on failure. """ # The root module wins over the owner of the hub when both configure it. root_modules = [mod for mod in module_ctx.modules if mod.is_root] root_name = root_modules[0].name if root_modules else None - root_configured = { - tag.hub_name: None - for mod in root_modules - for tag in mod.tags.configure - } + root_configured = {} + for mod in root_modules: + for tag in mod.tags.configure: + root_configured[tag.hub_name] = None configured = {} + exposes_all = {} for mod in module_ctx.modules: for tag in mod.tags.configure: hub = tag.hub_name @@ -66,13 +65,11 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): _fail = _fail, ) if selection.all: - packages = info.exposed_packages - else: - packages = selection.packages + exposes_all[hub] = None # Multiple tags for the same hub are combined. exposed = configured.setdefault(hub, {}) - for name in packages: + for name in selection.packages: # The pyproject.toml may list packages that are not in the hub, # e.g. tools that are not locked. if name in info.packages: @@ -81,4 +78,5 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): return { hub: sorted(packages) for hub, packages in configured.items() + if hub not in exposes_all } diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl index cafbb229b5..5d84a351b5 100644 --- a/tests/pypi/hub_exposure/hub_exposure_tests.bzl +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -20,7 +20,6 @@ def _hub(*, module_name = "root", pyproject = None): module_name = module_name, pyproject = pyproject, packages = ["other_pkg", "simple", "transitive_pkg"], - exposed_packages = ["other_pkg", "simple", "transitive_pkg"], ) def _configure(hub_name = "hub", dependency_groups = ["*"], pyproject_toml = None): @@ -63,7 +62,6 @@ def _test_configure(env): (["dev"], ["other_pkg"]), # `not-in-hub` is ignored (["group:test"], ["simple"]), - (["*", "transitive"], ["other_pkg", "simple", "transitive_pkg"]), ]: got, failures = _exposure(_module(configure = [_configure(dependency_groups = groups)])) env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) @@ -71,6 +69,18 @@ def _test_configure(env): _tests.append(_test_configure) +def _test_configure_all_packages(env): + # Exposing all of the packages is the same as not configuring the hub, so + # that e.g. the platform-specific packages stay public. + got, failures = _exposure(_module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(dependency_groups = ["*", "transitive"]), + ])) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({}) + +_tests.append(_test_configure_all_packages) + def _test_configure_combines_tags(env): got, failures = _exposure(_module(configure = [ _configure(dependency_groups = ["default"]), From 9d97360242b44f6a73472793d17a17287cabb1f3 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:05:56 +0900 Subject: [PATCH 06/55] refactor(pypi): use explicit loops and simpler conditions in pip.configure Replace the nested comprehensions and the long inline conditions with explicit loops and use types.is_string. --- python/private/pypi/BUILD.bazel | 1 + python/private/pypi/dependency_groups.bzl | 19 ++++++++----------- python/private/pypi/hub_repository.bzl | 4 +++- python/private/pypi/render_pkg_aliases.bzl | 19 +++++++++++-------- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index caf482b67f..fe28b17413 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -639,6 +639,7 @@ bzl_library( deps = [ ":pep508_requirement", "//python/private:normalize_name", + "@bazel_skylib//lib:types", ], ) diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl index 2ddadee3dd..4e40c7e856 100644 --- a/python/private/pypi/dependency_groups.bzl +++ b/python/private/pypi/dependency_groups.bzl @@ -1,5 +1,6 @@ """Select PyPI packages using pyproject.toml dependency groups.""" +load("@bazel_skylib//lib:types.bzl", "types") load("//python/private:normalize_name.bzl", "normalize_name") load(":pep508_requirement.bzl", "requirement") @@ -59,16 +60,12 @@ def select_dependency_groups(pyproject, specs, _fail = fail): for key in selected: for req in groups[key]: req = requirement(req) - names = [req.name_] - if req.name_ == project: - names = [ - requirement(r).name_ - for extra in req.extras - for r in groups.get("optional:" + extra, []) - ] - for name in names: - if name: - packages[name] = None + if req.name_ != project: + packages[req.name_] = None + continue + for extra in req.extras: + for r in groups.get("optional:" + extra, []): + packages[requirement(r).name_] = None return struct(packages = sorted(packages), all = False) @@ -119,7 +116,7 @@ def _expand_dependency_groups(dependency_groups, _fail): reqs = [] for item in items: - if type(item) == type(""): + if types.is_string(item): reqs.append(item) continue diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index d421f3cc09..ff5311c6ea 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -28,14 +28,16 @@ exports_files(["requirements.bzl"]) def _impl(rctx): if rctx.attr.hide_unexposed_packages: bzl_packages = rctx.attr.packages + exposed_packages = bzl_packages else: bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() + exposed_packages = None aliases = render_multiplatform_pkg_aliases( aliases = { key: _whl_config_settings_from_json(values) for key, values in rctx.attr.whl_map.items() }, - exposed_packages = bzl_packages if rctx.attr.hide_unexposed_packages else None, + exposed_packages = exposed_packages, extra_hub_aliases = rctx.attr.extra_hub_aliases, requirement_cycles = rctx.attr.groups, platform_config_settings = rctx.attr.platform_config_settings, diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 5582cc00c9..2c86155876 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -139,17 +139,20 @@ def render_pkg_aliases( exposed_packages = {normalize_name(name): None for name in exposed_packages} internal_visibility = _internal_visibility(aliases, has_groups = bool(requirement_cycles)) - files = { - "{}/BUILD.bazel".format(normalize_name(name)): _render_common_aliases( - name = normalize_name(name), + files = {} + for name, pkg_aliases in aliases.items(): + name = normalize_name(name) + visibility = public + if exposed_packages != None and name not in exposed_packages: + visibility = internal_visibility + files["{}/BUILD.bazel".format(name)] = _render_common_aliases( + name = name, aliases = pkg_aliases, - extra_aliases = extra_hub_aliases.get(normalize_name(name), []), - group_name = whl_group_mapping.get(normalize_name(name)), - visibility = public if exposed_packages == None or normalize_name(name) in exposed_packages else internal_visibility, + extra_aliases = extra_hub_aliases.get(name, []), + group_name = whl_group_mapping.get(name), + visibility = visibility, **kwargs ).strip() - for name, pkg_aliases in aliases.items() - } if requirement_cycles: files["_groups/BUILD.bazel"] = generate_group_library_build_bazel("", requirement_cycles) From 35c066052cb0bea531aba78784149420d8056aaa Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:06:00 +0900 Subject: [PATCH 07/55] test(pypi): test the pip.default pyproject_toml with pip.configure Keep a single wiring test for pip.configure in the extension tests, as the selection is tested in hub_exposure_tests, and test that the pip.default pyproject_toml is used. Also link the issue in the news entry. --- news/4395.added.md | 3 +- tests/pypi/extension/extension_tests.bzl | 49 +++++++++++++++--------- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/news/4395.added.md b/news/4395.added.md index 62d7f10880..3e633cc997 100644 --- a/news/4395.added.md +++ b/news/4395.added.md @@ -1,2 +1,3 @@ (pypi) Added {obj}`pip.configure` to limit the packages that a hub exposes to -the ones listed in the dependency groups of a `pyproject.toml` file. +the ones listed in the dependency groups of a `pyproject.toml` file +([#3413](https://github.com/bazel-contrib/rules_python/issues/3413)). diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 17f8d6741e..680853a0d0 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -807,36 +807,47 @@ def _configure_parse_modules(env, *modules): minor_mapping = {"3.15": "3.15.19"}, ) -def _configure_parse(*, hub_name = "pypi_a", pyproject_toml = "pyproject.toml"): +def _configure_parse(**kwargs): return _parse( - hub_name = hub_name, - pyproject_toml = pyproject_toml, + hub_name = "pypi_a", python_version = "3.15", simpleapi_skip = ["simple", "other-pkg", "transitive-pkg"], requirements_lock = "requirements.txt", + **kwargs ) def _test_configure(env): - for groups, want in [ - (["*"], ["other_pkg", "simple"]), - (["test"], ["simple"]), - (["dev"], ["other_pkg"]), - (["*", "transitive"], ["other_pkg", "simple", "transitive_pkg"]), - ]: - pypi = _configure_parse_modules( - env, - _mod( - name = "rules_python", - parse = [_configure_parse()], - configure = [_configure(hub_name = "pypi_a", dependency_groups = groups)], - ), - ) + # The selection itself is tested in `hub_exposure_tests`. + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [_configure_parse(pyproject_toml = "pyproject.toml")], + configure = [_configure(hub_name = "pypi_a", dependency_groups = ["test"])], + ), + ) - pypi.exposed_packages().contains_exactly({"pypi_a": want}) - pypi.configured_hubs().contains_exactly({"pypi_a": want}) + pypi.exposed_packages().contains_exactly({"pypi_a": ["simple"]}) + pypi.configured_hubs().contains_exactly({"pypi_a": ["simple"]}) _tests.append(_test_configure) +def _test_configure_default_pyproject(env): + # The `pip.default` pyproject.toml is used for the hubs of the root module. + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + default = _default_tags_default + [_default(pyproject_toml = "pyproject.toml")], + parse = [_configure_parse()], + configure = [_configure(hub_name = "pypi_a", dependency_groups = ["dev"])], + ), + ) + + pypi.configured_hubs().contains_exactly({"pypi_a": ["other_pkg"]}) + +_tests.append(_test_configure_default_pyproject) + def _test_not_configured_exposes_all(env): # The pyproject.toml does not filter the hub without `pip.configure`. pypi = _configure_parse_modules( From 2873e71749076be9aff3492a0437f7cfd271d889 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:50:18 +0900 Subject: [PATCH 08/55] feat(pypi): add strip_extras to remove the extras from requirement lines --- python/private/pypi/pep508_requirement.bzl | 19 +++++++++++++++++++ tests/pypi/pep508/requirement_tests.bzl | 15 ++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/python/private/pypi/pep508_requirement.bzl b/python/private/pypi/pep508_requirement.bzl index 7552642572..8aaaa29e37 100644 --- a/python/private/pypi/pep508_requirement.bzl +++ b/python/private/pypi/pep508_requirement.bzl @@ -56,3 +56,22 @@ def requirement(spec): extras = extras, version = version, ) + +def strip_extras(requirement_line): + """Remove the extras from the requirement line. + + Args: + requirement_line: {type}`str` the requirement line, e.g. + `foo[bar]==1.0 --hash=sha256:...`. + + Returns: + The requirement line without the extras, e.g. `foo==1.0 --hash=sha256:...`. + """ + for i, c in enumerate(requirement_line.elems()): + if c in "=<>!~;@( ": + break + if c == "[": + _, _, tail = requirement_line[i:].partition("]") + return requirement_line[:i] + tail + + return requirement_line diff --git a/tests/pypi/pep508/requirement_tests.bzl b/tests/pypi/pep508/requirement_tests.bzl index 2ce45922da..c027209aa1 100644 --- a/tests/pypi/pep508/requirement_tests.bzl +++ b/tests/pypi/pep508/requirement_tests.bzl @@ -14,7 +14,7 @@ """Tests for parsing the requirement specifier.""" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private/pypi:pep508_requirement.bzl", "requirement") # buildifier: disable=bzl-visibility +load("//python/private/pypi:pep508_requirement.bzl", "requirement", "strip_extras") # buildifier: disable=bzl-visibility _tests = [] @@ -42,6 +42,19 @@ def _test_requirement_line_parsing(env): _tests.append(_test_requirement_line_parsing) +def _test_strip_extras(env): + for line, want in { + "foo": "foo", + "foo @ https://example.org/foo[1].whl": "foo @ https://example.org/foo[1].whl", + "foo==1.0 ; python_version < '3.12' --hash=sha256:deadbeef": "foo==1.0 ; python_version < '3.12' --hash=sha256:deadbeef", + "foo[bar,baz] @ https://example.org/foo.whl": "foo @ https://example.org/foo.whl", + "foo[bar]": "foo", + "foo[bar]==1.0 --hash=sha256:deadbeef": "foo==1.0 --hash=sha256:deadbeef", + }.items(): + env.expect.where(line = line).that_str(strip_extras(line)).equals(want) + +_tests.append(_test_strip_extras) + def requirement_test_suite(name): # buildifier: disable=function-docstring test_suite( name = name, From 2f92d4e4c2cf2457f84fef0b3249cb4ea518a1d6 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:50:44 +0900 Subject: [PATCH 09/55] feat(pypi): add keep_extras and dependency excludes to pep508_deps --- python/private/pypi/pep508_deps.bzl | 31 ++++++++++------- tests/pypi/pep508/deps_tests.bzl | 54 +++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 12 deletions(-) diff --git a/python/private/pypi/pep508_deps.bzl b/python/private/pypi/pep508_deps.bzl index e36ceadbd5..da42fbe5e9 100644 --- a/python/private/pypi/pep508_deps.bzl +++ b/python/private/pypi/pep508_deps.bzl @@ -26,23 +26,32 @@ def deps( requires_dist, extras = [], excludes = [], - include = []): + include = [], + keep_extras = False): """Parse the RequiresDist from wheel METADATA Args: name: {type}`str` the name of the wheel. requires_dist: {type}`list[str]` the list of RequiresDist lines from the METADATA file. - excludes: {type}`list[str]` what packages should we exclude. + excludes: {type}`list[str]` what dependencies should we exclude, e.g. + `foo` or `foo[bar]`, in the same format as the returned values. include: {type}`list[str]` what packages should we exclude. If it is not specified, then we will include all deps from `requires_dist`. extras: {type}`list[str]` the requested extras to generate targets for. + keep_extras: {type}`bool` keep the extras of the dependencies, if + `True`, the dependencies with extras are returned as `foo[bar]`. Returns: A struct with attributes: * deps: {type}`list[str]` dependencies to include unconditionally. * deps_select: {type}`dict[str, list[str]]` dependencies to include on particular subset of target platforms. + + The dependencies are normalized names, e.g. `foo`, or normalized names + with a single extra, e.g. `foo[bar]`, if the dependency is requested + with extras and `keep_extras` is set. A dependency with multiple + extras, e.g. `foo[bar,baz]`, is returned as `foo[bar]` and `foo[baz]`. """ if not requires_dist: return struct( @@ -60,25 +69,25 @@ def deps( want_extras = _resolve_extras(name, reqs, extras) include = [normalize_name(n) for n in include] - # drop self edges - excludes = [name] + [normalize_name(x) for x in excludes] - - reqs_by_name = {} + reqs_by_dep = {} for req in reqs: - if req.name_ in excludes: + # drop self edges + if req.name_ == name: continue if include and req.name_ not in include: continue - reqs_by_name.setdefault(req.name, []).append(req) + for dep in ["{}[{}]".format(req.name_, e) for e in req.extras if keep_extras] or [req.name_]: + if dep not in excludes: + reqs_by_dep.setdefault(dep, []).append(req) - for name, reqs in reqs_by_name.items(): + for dep, reqs in reqs_by_dep.items(): _add_reqs( deps, deps_select, - normalize_name(name), + dep, reqs, extras = want_extras, ) @@ -92,8 +101,6 @@ def deps( ) def _add(deps, deps_select, dep, markers = None): - dep = normalize_name(dep) - if not markers: deps[dep] = True elif len(markers) == 1: diff --git a/tests/pypi/pep508/deps_tests.bzl b/tests/pypi/pep508/deps_tests.bzl index 0d171d8e95..6e291e5a6c 100644 --- a/tests/pypi/pep508/deps_tests.bzl +++ b/tests/pypi/pep508/deps_tests.bzl @@ -320,6 +320,60 @@ def test_extras_with_hyphens_are_normalized(env): _tests.append(test_extras_with_hyphens_are_normalized) +def test_deps_with_extras(env): + got = deps( + "foo", + requires_dist = [ + "bar[Socks]", + "baz[a,b]; sys_platform == 'linux'", + "qux", + "qux[c]; extra == 'feat'", + ], + extras = ["feat"], + keep_extras = True, + ) + + # The extras of the dependencies are kept so that the targets with the + # extras can be used. + env.expect.that_collection(got.deps).contains_exactly(["bar[socks]", "qux", "qux[c]"]) + env.expect.that_dict(got.deps_select).contains_exactly({ + "baz[a]": "sys_platform == \"linux\"", + "baz[b]": "sys_platform == \"linux\"", + }) + +_tests.append(test_deps_with_extras) + +def test_deps_with_extras_excludes_and_includes(env): + got = deps( + "foo", + requires_dist = ["bar", "bar[socks]", "baz[a]", "baz[b]", "qux[b]"], + # The excludes are dependencies and the includes are packages + excludes = ["bar", "baz[a]"], + include = ["bar", "baz"], + keep_extras = True, + ) + + env.expect.that_collection(got.deps).contains_exactly(["bar[socks]", "baz[b]"]) + +_tests.append(test_deps_with_extras_excludes_and_includes) + +def test_deps_without_extras(env): + got = deps( + "foo", + requires_dist = [ + "bar[socks]", + "baz[a]; sys_platform == 'linux'", + "baz[b]; sys_platform == 'darwin'", + ], + ) + + env.expect.that_collection(got.deps).contains_exactly(["bar"]) + env.expect.that_dict(got.deps_select).contains_exactly({ + "baz": "(sys_platform == \"darwin\") or (sys_platform == \"linux\")", + }) + +_tests.append(test_deps_without_extras) + def deps_test_suite(name): # buildifier: disable=function-docstring test_suite( name = name, From efc64f051062f5f44216fb0acc96e33599e0e185 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:51:20 +0900 Subject: [PATCH 10/55] feat(pypi): add extra_label and group_member label helpers --- python/private/pypi/BUILD.bazel | 11 ++++++----- python/private/pypi/labels.bzl | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index f94d24594f..bea420c835 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -632,6 +632,12 @@ bzl_library( ], ) +bzl_library( + name = "labels", + srcs = ["labels.bzl"], + deps = ["//python/private:normalize_name"], +) + bzl_library( name = "argparse", srcs = ["argparse.bzl"], @@ -657,11 +663,6 @@ bzl_library( srcs = ["hash.bzl"], ) -bzl_library( - name = "labels", - srcs = ["labels.bzl"], -) - bzl_library( name = "package_annotation", srcs = ["package_annotation.bzl"], diff --git a/python/private/pypi/labels.bzl b/python/private/pypi/labels.bzl index c77167b593..f6257106e9 100644 --- a/python/private/pypi/labels.bzl +++ b/python/private/pypi/labels.bzl @@ -14,6 +14,8 @@ """Constants used by parts of pip_repository for naming libraries and wheels.""" +load("//python/private:normalize_name.bzl", "normalize_name") + EXTRACTED_WHEEL_FILES = "extracted_whl_files" WHEEL_FILE_PUBLIC_LABEL = "whl" WHEEL_FILE_IMPL_LABEL = "_whl" @@ -24,3 +26,30 @@ PY_SRCS_LABEL = "srcs" DATA_LABEL = "data" DIST_INFO_LABEL = "dist_info" NODEPS_LABEL = "no_deps" + +def extra_label(extra): + """Return the label name for the target that includes an extra. + + Args: + extra: {type}`str` the normalized name of the extra, an empty string + means the base package without any extras. + + Returns: + The target name, e.g. `[extra]`. + """ + return "[{}]".format(extra) + +def group_member(spec): + """Normalize a member of a dependency group. + + Args: + spec: {type}`str` the package name, e.g. `Foo`, or the package name + with a single extra, e.g. `Foo[Bar]`. + + Returns: + The normalized member, e.g. `foo` or `foo[bar]`. + """ + name, _, extra = spec.partition("[") + name = normalize_name(name.strip()) + extra = normalize_name(extra.partition("]")[0].strip()) if extra else "" + return "{}[{}]".format(name, extra) if extra else name From 552d8f167c6845de0deb77f3fff5f65236fdd138 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:52:36 +0900 Subject: [PATCH 11/55] feat(pypi): create a target for every provided extra in the whl repositories --- python/private/internal_dev_deps.bzl | 2 +- python/private/pypi/BUILD.bazel | 2 - .../pypi/generate_whl_library_build_bazel.bzl | 2 +- python/private/pypi/patch_and_extract_whl.bzl | 3 +- python/private/pypi/pkg_aliases.bzl | 4 - python/private/pypi/whl_deps_repo.bzl | 3 +- .../private/pypi/whl_library_deps_targets.bzl | 188 ++++++++----- python/private/pypi/whl_library_targets.bzl | 6 +- tests/pypi/pkg_aliases/pkg_aliases_test.bzl | 1 - tests/pypi/whl_library/BUILD.bazel | 2 +- .../whl_library_deps_targets_tests.bzl | 266 +++++++++++++++++- 11 files changed, 382 insertions(+), 97 deletions(-) diff --git a/python/private/internal_dev_deps.bzl b/python/private/internal_dev_deps.bzl index 5bbee232f9..111b839adf 100644 --- a/python/private/internal_dev_deps.bzl +++ b/python/private/internal_dev_deps.bzl @@ -117,7 +117,7 @@ def _internal_dev_deps_impl(mctx): name = "whl_library_extras_direct_dep", root = "//tests/pypi/whl_library/testdata/pkg:BUILD.bazel", output = "pkg-1.0-any-none-any.whl", - requirement = "pkg[optional]", + requirement = "pkg", # The following is necessary to enable pipstar and make tests faster config_load = "@rules_python//tests/pypi/whl_library/testdata:packages.bzl", dep_template = "@whl_library_extras_{name}//:{target}", diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index bea420c835..1e33b366db 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -586,7 +586,6 @@ bzl_library( srcs = ["whl_deps_repo.bzl"], deps = [ ":generate_whl_library_build_bazel", - ":pep508_requirement", "//python/private:repo_utils", ], ) @@ -597,7 +596,6 @@ bzl_library( deps = [ ":generate_whl_library_build_bazel", ":patch_whl", - ":pep508_requirement", ":pypi_repo_utils", ":whl_extract", ":whl_metadata", diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index bec45bb08f..bb8faa71dc 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -22,9 +22,9 @@ _RENDER = { "data": render.list, "data_exclude": render.list, "entry_points": render.dict_dict, - "extras": render.list, "group_deps": render.list, "include": str, + "provides_extra": render.list, "repo": lambda maybe_label: repr(str(maybe_label)), "requires_dist": render.list, "srcs_exclude": render.list, diff --git a/python/private/pypi/patch_and_extract_whl.bzl b/python/private/pypi/patch_and_extract_whl.bzl index 37c90e3221..cfacb4adf4 100644 --- a/python/private/pypi/patch_and_extract_whl.bzl +++ b/python/private/pypi/patch_and_extract_whl.bzl @@ -4,7 +4,6 @@ load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:repo_utils.bzl", "repo_utils") load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") load(":patch_whl.bzl", "get_whl_patches", "patch_whl") -load(":pep508_requirement.bzl", "requirement") load(":pypi_repo_utils.bzl", "pypi_repo_utils") load(":whl_extract.bzl", "whl_extract") load(":whl_metadata.bzl", "parse_entry_points", "whl_metadata") @@ -135,7 +134,7 @@ def patch_and_extract_whl(rctx, *, whl_path, logger, sdist_filename = None, whl_ group_deps = rctx.attr.group_deps, group_name = rctx.attr.group_name, namespace_package_files = namespace_package_files, - extras = requirement(rctx.attr.requirement).extras, + provides_extra = metadata.provides_extra, entry_points = entry_points, purl = _to_purl( index = rctx.attr.index_url, diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index 227e9bf10f..308d9a21fa 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -83,7 +83,6 @@ def pkg_aliases( group_name = None, extra_aliases = None, requires_dist = [], - extras = [], include = [], group_deps = [], **kwargs): @@ -103,8 +102,6 @@ def pkg_aliases( extra_aliases: {type}`list[str]` The extra aliases to be created. requires_dist: {type}`list[str]` The list of dependencies. Comes from METADATA or a lock file. - extras: {type}`list[str]` The extras for which we should add extra - dependencies when parsing the requires_dist. Comes from METADATA or a lock file. include: {type}`list[str]` The subset of packages to include. Comes from `//:config.bzl#packages` **kwargs: extra kwargs to pass to {bzl:obj}`get_config_settings`. @@ -129,7 +126,6 @@ def pkg_aliases( aliases = {}, # not none metadata_name = name, requires_dist = requires_dist, - extras = extras, include = include, group_deps = group_deps, group_name = group_name, diff --git a/python/private/pypi/whl_deps_repo.bzl b/python/private/pypi/whl_deps_repo.bzl index 20891da397..75e5a3771d 100644 --- a/python/private/pypi/whl_deps_repo.bzl +++ b/python/private/pypi/whl_deps_repo.bzl @@ -2,7 +2,6 @@ load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") -load(":pep508_requirement.bzl", "requirement") # Reusable common attributes for generating BUILD.bazel from Requires-Dist in the wheel METADATA. whl_deps_attrs = { @@ -62,7 +61,7 @@ def _whl_deps_repo_impl(rctx): repo = rctx.attr.repo or ( str(rctx.attr.metadata_file) if rctx.attr.metadata_file else None ), - extras = requirement(rctx.attr.requirement).extras, + provides_extra = getattr(metadata, "provides_extra", []), ) rctx.file("BUILD.bazel", build_file_contents) diff --git a/python/private/pypi/whl_library_deps_targets.bzl b/python/private/pypi/whl_library_deps_targets.bzl index e549f52a47..42de802267 100644 --- a/python/private/pypi/whl_library_deps_targets.bzl +++ b/python/private/pypi/whl_library_deps_targets.bzl @@ -8,12 +8,12 @@ load( "DATA_LABEL", "DIST_INFO_LABEL", "EXTRACTED_WHEEL_FILES", - "PY_LIBRARY_IMPL_LABEL", "PY_LIBRARY_PUBLIC_LABEL", "PY_SRCS_LABEL", "WHEEL_FILE", - "WHEEL_FILE_IMPL_LABEL", "WHEEL_FILE_PUBLIC_LABEL", + "extra_label", + "group_member", ) load(":pep508_deps.bzl", "deps") @@ -24,7 +24,7 @@ def whl_library_deps_targets( aliases = None, metadata_name, requires_dist = [], - extras = [], + provides_extra = [], include = [], group_deps = [], group_name = None, @@ -48,10 +48,9 @@ def whl_library_deps_targets( group rules which wrap this library and its fellows together. requires_dist: {type}`list[str]` The list of `Requires-Dist` values from the whl `METADATA`. Optional because some packages don't have them. - extras: {type}`list[str]` The list of requested extras. This essentially - includes extra transitive dependencies in the final targets - depending on the wheel `METADATA`. Optional because some packages - don't request them. + provides_extra: {type}`list[str]` The list of extras that the package + provides. For each extra the `[]` target is created + that includes the package and the dependencies of that extra. include: {type}`list[str]` The list of packages to include. group_name: {type}`str | None` name of the dependency group (if any). dep_template: {type}`str | None` The dep_template to use. @@ -75,27 +74,33 @@ def whl_library_deps_targets( DATA_LABEL: repo_label(DATA_LABEL), } + # The members of the group are the packages, e.g. `foo`, or the extras of + # the packages, e.g. `foo[bar]`. Only the targets of the members are part + # of the group and they do not depend on the other members to avoid cycles. + self_name = normalize_name(metadata_name) + members = {group_member(m): None for m in group_deps} if group_name else {} + is_member = lambda extra: ( + "{}[{}]".format(self_name, extra) if extra else self_name + ) in members + # If this library is a member of a group, its public label aliases need to # point to the group implementation rule not the implementation rules. We # also need to mark the implementation rules as visible to the group # implementation. + group_label = None if group_name and "//:" in dep_template: # This is the legacy behaviour where the group library is outside the hub repo # # It is expected to disappear when we drop WORKSPACE or drop the vendoring of # pip_parse `requirements.bzl` in WORKSPACE. The alternative would be to add # another argument to the macro, but it is already full of arguments. - label_tmpl = dep_template.format( + group_label = dep_template.format( name = "_config", target = normalize_name(group_name) + "_{}", ).replace( "//:", "//_groups:", ) - aliases = aliases | { - PY_LIBRARY_PUBLIC_LABEL: label_tmpl.format(PY_LIBRARY_PUBLIC_LABEL), - WHEEL_FILE_PUBLIC_LABEL: label_tmpl.format(WHEEL_FILE_PUBLIC_LABEL), - } impl_vis = [dep_template.format( name = "_config", target = "__pkg__", @@ -103,25 +108,40 @@ def whl_library_deps_targets( "//:", "//_groups:", )] - - py_library_label = PY_LIBRARY_IMPL_LABEL - whl_file_label = WHEEL_FILE_IMPL_LABEL + elif group_name: + impl_vis = [dep_template.format(name = "", target = "__subpackages__")] else: - py_library_label = PY_LIBRARY_PUBLIC_LABEL - whl_file_label = WHEEL_FILE_PUBLIC_LABEL - if group_name: - impl_vis = [dep_template.format(name = "", target = "__subpackages__")] - else: - impl_vis = visibility + impl_vis = visibility + + def _name(extra, public): + # In the legacy mode the targets of the members are implementation + # targets and the public targets are aliases to the group. + prefix = "_" if group_label and is_member(extra) else "" + return prefix + (extra_label(extra) if extra else public) + + def _visibility(extra): + # In the legacy mode the targets that are not members are public. + return visibility if group_label and not is_member(extra) else impl_vis + + # Create targets for every extra that the package provides, so that the hub + # can select only the extras that are requested on each platform without + # passing the extras to this repository. + extras_targets = [""] + sorted({normalize_name(e): None for e in provides_extra}) + whl_file_label = _name("", WHEEL_FILE_PUBLIC_LABEL) + + if group_label: + for extra in extras_targets: + if is_member(extra): + aliases[extra_label(extra) if extra else PY_LIBRARY_PUBLIC_LABEL] = group_label.format(PY_LIBRARY_PUBLIC_LABEL) + if is_member(""): + aliases[WHEEL_FILE_PUBLIC_LABEL] = group_label.format(WHEEL_FILE_PUBLIC_LABEL) if not requires_dist: - # If the package is in a group but has no deps, we still need the public labels to - # point at the srcs targets so that the group implementation can use them. We don't - # need any of the extra targets, so just create the aliases. - aliases = aliases | { - py_library_label: repo_label(PY_SRCS_LABEL), - whl_file_label: repo_label(WHEEL_FILE), - } + # If the package is in a group but has no deps, we still need the labels to point at + # the srcs targets so that the group implementation can use them. + aliases[whl_file_label] = repo_label(WHEEL_FILE) + for extra in extras_targets: + aliases[_name(extra, PY_LIBRARY_PUBLIC_LABEL)] = repo_label(PY_SRCS_LABEL) for alias, actual in aliases.items(): native.alias( @@ -134,37 +154,53 @@ def whl_library_deps_targets( # If there are extras, then they will be visible in requires_dist. return - package_deps = _parse_requires_dist( - name = metadata_name, - requires_dist = requires_dist, - excludes = group_deps, - extras = extras, - include = include, - ) + # The settings for the deps that are included conditionally keyed by the + # dep and the expression. + settings = {} + + def _pkg_label(dep): + dep_name, _, extra = dep.partition("[") + if not extra: + return dep_template.format(name = dep_name, target = PY_LIBRARY_PUBLIC_LABEL) - _config_settings( - dependencies_with_markers = package_deps.deps_select, - rules = rules, - visibility = ["//visibility:private"], - ) + return dep_template.format(name = dep_name, target = extra_label(extra.rstrip("]"))) + + whl_label = lambda dep: dep_template.format(name = dep, target = WHEEL_FILE_PUBLIC_LABEL) if hasattr(native, "filegroup"): # We include the whl file as srcs so that `$(location :whl)` expands to the whl file. - # The transitive dependencies are available via the `data` attribute. + # The transitive dependencies are available via the `data` attribute. The extras + # are not used for the whl targets, so only the packages without extras can be + # members of the group for the whl targets. + whl_deps = deps( + name = self_name, + requires_dist = requires_dist, + include = include, + excludes = members if is_member("") else [], + ) native.filegroup( name = whl_file_label, srcs = [repo_label(WHEEL_FILE)], data = _deps( deps = [], - package_deps = package_deps, - tmpl = dep_template.format(name = "{}", target = WHEEL_FILE_PUBLIC_LABEL), + package_deps = whl_deps, + conditions = _conditions(settings, whl_deps), + label = whl_label, ), - visibility = impl_vis, + visibility = _visibility(""), ) - if hasattr(rules, "py_library"): + for extra in extras_targets if hasattr(rules, "py_library") else []: + package_deps = deps( + name = self_name, + requires_dist = requires_dist, + include = include, + excludes = members if is_member(extra) else [], + extras = [extra] if extra else [], + keep_extras = True, + ) rules.py_library( - name = py_library_label, + name = _name(extra, PY_LIBRARY_PUBLIC_LABEL), # We include as srcs to ensure that the (locations :pkg) works as expected. srcs = [repo_label(PY_SRCS_LABEL)], deps = _deps( @@ -173,52 +209,48 @@ def whl_library_deps_targets( # `pyi_srcs`, see `tests/base_rules/py_library`. deps = [repo_label(PY_SRCS_LABEL)], package_deps = package_deps, - tmpl = dep_template.format(name = "{}", target = PY_LIBRARY_PUBLIC_LABEL), + conditions = _conditions(settings, package_deps), + label = _pkg_label, ), # Disable precompilation on this wrapper target to prevent duplicate # pyc generation; the underlying PY_SRCS_LABEL target handles it. precompile = "disabled", tags = tags, - visibility = impl_vis, + visibility = _visibility(extra), ) -def _parse_requires_dist( - *, - name, - requires_dist, - excludes, - include, - extras): - return deps( - name = normalize_name(name), - requires_dist = requires_dist, - excludes = excludes, - include = include, - extras = extras, - ) - -def _config_settings(dependencies_with_markers, rules, **kwargs): - """Generate config settings for the targets. - - Args: - dependencies_with_markers: {type}`dict[str, str]` The markers to evaluate by - each dep. - rules: used for testing - **kwargs: Extra kwargs to pass to the rule. - """ - for dep, expression in dependencies_with_markers.items(): + for (_, expression), name in settings.items(): rules.env_marker_setting( - name = "include_{}".format(dep), + name = name, expression = expression, - **kwargs + visibility = ["//visibility:private"], ) -def _deps(deps, package_deps, tmpl): - deps = [] + deps + [tmpl.format(d) for d in sorted(package_deps.deps)] +def _conditions(settings, package_deps): + return { + dep: _setting_name(settings, dep, expression) + for dep, expression in package_deps.deps_select.items() + } + +def _setting_name(settings, dep, expression): + """Return a unique name for the setting evaluating the expression for the dep. + + The first expression for a dep is named `include_` and any different + expressions for the same dep get a numeric suffix. + """ + key = (dep, expression) + if key not in settings: + n = len([d for d, _ in settings if d == dep]) + settings[key] = "include_{}".format(dep) + ("_{}".format(n + 1) if n else "") + + return settings[key] + +def _deps(deps, package_deps, conditions, label): + deps = [] + deps + [label(d) for d in sorted(package_deps.deps)] for dep in package_deps.deps_select: deps = deps + select({ - ":is_include_{}_true".format(dep): [tmpl.format(dep)], + ":is_{}_true".format(conditions[dep]): [label(dep)], "//conditions:default": [], }) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index a1cc727460..50be70c532 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -26,7 +26,7 @@ def whl_library_targets( whl_file = None, metadata_name = "", requires_dist = [], - extras = [], + provides_extra = [], entry_points = {}, include = [], group_deps = [], @@ -55,7 +55,7 @@ def whl_library_targets( group rules which wrap this library and its fellows together. requires_dist: {type}`list[str]` The list of `Requires-Dist` values from the whl `METADATA`. - extras: {type}`list[str]` The list of requested extras. This essentially includes extra transitive dependencies in the final targets depending on the wheel `METADATA`. + provides_extra: {type}`list[str]` The list of extras that the package provides. entry_points: {type}`list[dict]` A list of parsed entry point definitions. include: {type}`list[str]` The list of packages to include. group_name: {type}`str | None` name of the dependency group (if any). @@ -97,7 +97,7 @@ def whl_library_targets( dep_template = dep_template, # only needed if requires_dist or group_name is present group_deps = group_deps, # only needed if group_name is present group_name = group_name, # must specify group_deps together - extras = extras, # only needed if requires_dist is present + provides_extra = provides_extra, include = include, # only needed if requires_dist is present repo = None, # set aliases in the same repo aliases = {}, diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index 2d54bd34db..4343a7db1a 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -308,7 +308,6 @@ def _test_deps_and_aliases(env): want_deps_targets = { "aliases": {}, "dep_template": "//{name}:{target}", - "extras": [], "group_deps": ["bar", "foo"], "group_name": "my_group", "include": [], diff --git a/tests/pypi/whl_library/BUILD.bazel b/tests/pypi/whl_library/BUILD.bazel index cade0d2b8e..427780c9f2 100644 --- a/tests/pypi/whl_library/BUILD.bazel +++ b/tests/pypi/whl_library/BUILD.bazel @@ -6,6 +6,6 @@ py_test( srcs = ["whl_library_extras_test.py"], target_compatible_with = SUPPORTS_BZLMOD, deps = [ - "@whl_library_extras_direct_dep//:pkg", + "@whl_library_extras_direct_dep//:[optional]", ], ) diff --git a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl index e8659ff642..788bf44f7f 100644 --- a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl +++ b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl @@ -33,7 +33,6 @@ def _test_whl_library_deps_targets(env): # Overrides for testing repo = None, aliases = None, - extras = [], native = struct( filegroup = lambda **kwargs: filegroup_calls.append(kwargs), alias = lambda **kwargs: None, @@ -105,7 +104,6 @@ def _test_whl_library_deps_targets_no_deps(env): group_name = "qux", repo = None, aliases = {}, - extras = [], native = struct( filegroup = lambda **kwargs: filegroup_calls.append(kwargs), alias = lambda **kwargs: alias_calls.append(kwargs), @@ -139,6 +137,270 @@ def _test_whl_library_deps_targets_no_deps(env): _tests.append(_test_whl_library_deps_targets_no_deps) +def _test_extras(env): + filegroup_calls = [] + py_library_calls = [] + env_marker_setting_calls = [] + + whl_library_deps_targets( + name = "foo-0-py3-none-any.whl", + metadata_name = "Foo", + dep_template = "@pypi//{name}:{target}", + requires_dist = [ + "bar", + "grpc; extra == 'grpc'", + "aiohttp; extra == 'async' and python_version < '3.12'", + "uvloop; extra == 'async'", + "foo[grpc]; extra == 'all'", + ], + provides_extra = ["async", "grpc", "all"], + include = ["bar", "grpc", "aiohttp", "uvloop"], + repo = None, + aliases = {}, + native = struct( + filegroup = lambda **kwargs: filegroup_calls.append(kwargs), + alias = lambda **kwargs: None, + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + env_marker_setting = lambda **kwargs: env_marker_setting_calls.append(kwargs), + ), + ) + + # The `pkg` target has no extras because none were requested and there is + # a target for each extra that the package provides. The `whl` target does + # not use the extras. + env.expect.that_collection([ + (c["name"], c["deps"]) + for c in py_library_calls + ]).contains_exactly([ + ("pkg", ["srcs", "@pypi//bar:pkg"]), + ("[all]", ["srcs", "@pypi//bar:pkg", "@pypi//grpc:pkg"]), + ("[async]", ["srcs", "@pypi//bar:pkg", "@pypi//uvloop:pkg"] + select({ + ":is_include_aiohttp_true": ["@pypi//aiohttp:pkg"], + "//conditions:default": [], + })), + ("[grpc]", ["srcs", "@pypi//bar:pkg", "@pypi//grpc:pkg"]), + ]) + env.expect.that_collection([ + (c["name"], c["data"]) + for c in filegroup_calls + ]).contains_exactly([ + ("whl", ["@pypi//bar:whl"]), + ]) + env.expect.that_collection(env_marker_setting_calls).contains_exactly([ + { + "name": "include_aiohttp", + "expression": "python_version < \"3.12\"", + "visibility": ["//visibility:private"], + }, + ]) # buildifier: @unsorted-dict-items + +_tests.append(_test_extras) + +def _test_extras_with_different_markers(env): + py_library_calls = [] + env_marker_setting_calls = [] + + whl_library_deps_targets( + name = "foo-0-py3-none-any.whl", + metadata_name = "foo", + dep_template = "@pypi//{name}:{target}", + requires_dist = [ + "bar; sys_platform == 'win32'", + "bar; extra == 'a' and sys_platform == 'linux'", + ], + provides_extra = ["a"], + repo = None, + aliases = {}, + native = struct( + filegroup = lambda **kwargs: None, + alias = lambda **kwargs: None, + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + env_marker_setting = lambda **kwargs: env_marker_setting_calls.append(kwargs), + ), + ) + + env.expect.that_collection([ + (c["name"], c["deps"]) + for c in py_library_calls + ]).contains_exactly([ + ("pkg", ["srcs"] + select({ + ":is_include_bar_true": ["@pypi//bar:pkg"], + "//conditions:default": [], + })), + ("[a]", ["srcs"] + select({ + ":is_include_bar_2_true": ["@pypi//bar:pkg"], + "//conditions:default": [], + })), + ]) + env.expect.that_collection([ + (c["name"], c["expression"]) + for c in env_marker_setting_calls + ]).contains_exactly([ + ("include_bar", "sys_platform == \"win32\""), + ("include_bar_2", "(sys_platform == \"linux\") or (sys_platform == \"win32\")"), + ]) + +_tests.append(_test_extras_with_different_markers) + +def _test_extras_no_deps(env): + alias_calls = [] + + whl_library_deps_targets( + name = "foo-0-py3-none-any.whl", + metadata_name = "foo", + dep_template = "@pypi//{name}:{target}", + requires_dist = [], + provides_extra = ["A"], + repo = None, + aliases = {}, + native = struct( + alias = lambda **kwargs: alias_calls.append(kwargs), + ), + rules = struct(), + ) + + env.expect.that_collection([ + (c["name"], c["actual"]) + for c in alias_calls + ]).contains_exactly([ + ("pkg", "srcs"), + ("whl", "whl_file"), + ("[a]", "srcs"), + ]) + +_tests.append(_test_extras_no_deps) + +def _test_extras_in_group(env): + py_library_calls = [] + filegroup_calls = [] + + whl_library_deps_targets( + name = "foo-0-py3-none-any.whl", + metadata_name = "foo", + dep_template = "@pypi//{name}:{target}", + requires_dist = ["baz", "bar; extra == 'a'", "qux; extra == 'a'"], + provides_extra = ["a"], + group_name = "group", + # Only the `a` extra is in a cycle with `qux`. + group_deps = ["Foo[A]", "qux"], + repo = None, + aliases = {}, + native = struct( + filegroup = lambda **kwargs: filegroup_calls.append(kwargs), + alias = lambda **kwargs: None, + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + env_marker_setting = lambda **kwargs: None, + ), + ) + + # Only the member targets exclude the other members and the targets are + # visible to the hub that creates the group library. + env.expect.that_collection([ + (c["name"], c["deps"], c["visibility"]) + for c in py_library_calls + ]).contains_exactly([ + ("pkg", ["srcs", "@pypi//baz:pkg"], ["@pypi//:__subpackages__"]), + ("[a]", ["srcs", "@pypi//bar:pkg", "@pypi//baz:pkg"], ["@pypi//:__subpackages__"]), + ]) + env.expect.that_collection([ + (c["name"], c["data"]) + for c in filegroup_calls + ]).contains_exactly([ + ("whl", ["@pypi//baz:whl"]), + ]) + +_tests.append(_test_extras_in_group) + +def _test_extras_in_group_legacy(env): + alias_calls = [] + py_library_calls = [] + filegroup_calls = [] + + whl_library_deps_targets( + name = "foo-0-py3-none-any.whl", + metadata_name = "foo", + dep_template = "@pip_{name}//:{target}", + requires_dist = ["qux", "bar; extra == 'a'"], + provides_extra = ["a", "b"], + group_name = "group", + group_deps = ["foo", "foo[a]", "qux"], + repo = None, + aliases = {}, + native = struct( + filegroup = lambda **kwargs: filegroup_calls.append(kwargs), + alias = lambda **kwargs: alias_calls.append(kwargs), + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + env_marker_setting = lambda **kwargs: None, + ), + ) + + # The member targets are implementation targets and the public targets + # point to the group library in the config repo. + env.expect.that_collection([ + (c["name"], c["actual"]) + for c in alias_calls + ]).contains_exactly([ + ("pkg", "@pip__config//_groups:group_pkg"), + ("[a]", "@pip__config//_groups:group_pkg"), + ("whl", "@pip__config//_groups:group_whl"), + ]) + env.expect.that_collection([ + (c["name"], c["deps"], c["visibility"]) + for c in py_library_calls + ]).contains_exactly([ + ("_pkg", ["srcs"], ["@pip__config//_groups:__pkg__"]), + ("_[a]", ["srcs", "@pip_bar//:pkg"], ["@pip__config//_groups:__pkg__"]), + ("[b]", ["srcs", "@pip_qux//:pkg"], ["//visibility:public"]), + ]) + env.expect.that_collection([ + (c["name"], c["data"], c["visibility"]) + for c in filegroup_calls + ]).contains_exactly([ + ("_whl", [], ["@pip__config//_groups:__pkg__"]), + ]) + +_tests.append(_test_extras_in_group_legacy) + +def _test_deps_with_extras_without_hub_info(env): + py_library_calls = [] + + # When the dependencies refer to the whl repositories directly, then there + # is no information about the available extras and the targets with the + # extras in the dependency repositories are used. + whl_library_deps_targets( + name = "foo-0-py3-none-any.whl", + metadata_name = "foo", + dep_template = "@pypi_{name}//:{target}", + requires_dist = ["bar[socks]"], + repo = None, + aliases = {}, + native = struct( + filegroup = lambda **kwargs: None, + alias = lambda **kwargs: None, + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + env_marker_setting = lambda **kwargs: None, + ), + ) + + env.expect.that_collection([ + (c["name"], c["deps"]) + for c in py_library_calls + ]).contains_exactly([ + ("pkg", ["srcs", "@pypi_bar//:[socks]"]), + ]) + +_tests.append(_test_deps_with_extras_without_hub_info) + def whl_library_deps_targets_test_suite(name): """create the test suite. From ace0065e66d321e512acbb6871d0abc69f720212 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:53:21 +0900 Subject: [PATCH 12/55] feat(pypi): record the requested extras in whl_config_setting --- python/private/pypi/hub_repository.bzl | 2 ++ python/private/pypi/render_pkg_aliases.bzl | 3 ++- python/private/pypi/whl_config_setting.bzl | 5 ++++- .../render_pkg_aliases_test.bzl | 22 +++++++++++++++++++ 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index f915aa1c77..27b324f465 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -142,6 +142,8 @@ def whl_config_settings_to_json(repo_mapping): def _whl_config_setting_dict(a): ret = {} + if a.extras: + ret["extras"] = a.extras if a.target_platforms: ret["target_platforms"] = a.target_platforms if a.version: diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 0a1c328491..271a12ddfb 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -46,10 +46,11 @@ def _repr_dict(*, value_repr = repr, **kwargs): return {k: value_repr(v) for k, v in kwargs.items() if v} def _repr_config_setting(alias): - if alias.target_platforms: + if alias.target_platforms or alias.extras: return render.call( "whl_config_setting", **_repr_dict( + extras = alias.extras, target_platforms = alias.target_platforms, version = alias.version, ) diff --git a/python/private/pypi/whl_config_setting.bzl b/python/private/pypi/whl_config_setting.bzl index 1d868b1b65..7c2c595770 100644 --- a/python/private/pypi/whl_config_setting.bzl +++ b/python/private/pypi/whl_config_setting.bzl @@ -14,7 +14,7 @@ "A small function to create an alias for a whl distribution" -def whl_config_setting(*, version = None, target_platforms = None): +def whl_config_setting(*, version = None, target_platforms = None, extras = None): """The bzl_packages value used by by the render_pkg_aliases function. This contains the minimum amount of information required to generate correct @@ -27,6 +27,8 @@ def whl_config_setting(*, version = None, target_platforms = None): is no match found during a select. target_platforms: {type}`list[str] | None` the list of target_platforms for this distribution. + extras: {type}`list[str] | None` the extras of the distribution that + are requested on the target platforms. Returns: a struct with the validated and parsed values. @@ -55,4 +57,5 @@ def whl_config_setting(*, version = None, target_platforms = None): # Make the struct hashable target_platforms = tuple(target_platforms) if target_platforms else None, version = version, + extras = tuple(sorted(extras)) if extras else None, ) diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index 9114f59279..807c4728fd 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -247,6 +247,28 @@ def _test_get_flag_versions_from_alias_target_platforms(env): _tests.append(_test_get_flag_versions_from_alias_target_platforms) +def _test_aliases_with_extras(env): + actual = render_pkg_aliases( + aliases = { + "foo": { + whl_config_setting(version = "3.2", extras = ["b", "a"]): "pypi_32_foo", + }, + }, + ) + + env.expect.that_str(actual["foo/BUILD.bazel"]).contains("""\ +pkg_aliases( + name = "foo", + actual = { + whl_config_setting( + extras = ("a", "b"), + version = "3.2", + ): "pypi_32_foo", + }, +)""") + +_tests.append(_test_aliases_with_extras) + def render_pkg_aliases_test_suite(name): """Create the test suite. From c0bd9cc3abb789636f5d8fb4334b37ddda008a45 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:54:03 +0900 Subject: [PATCH 13/55] feat(pypi): add the per-platform extra targets to the hub packages --- python/private/pypi/BUILD.bazel | 1 + python/private/pypi/pkg_aliases.bzl | 150 ++++++++++++-- tests/pypi/pkg_aliases/pkg_aliases_test.bzl | 216 +++++++++++++++++++- 3 files changed, 349 insertions(+), 18 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 1e33b366db..8cee5b5210 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -353,6 +353,7 @@ bzl_library( deps = [ ":labels", ":whl_library_deps_targets", + "//python:py_library", "//python/private:common_labels", "//python/private:text_util", "@bazel_skylib//lib:selects", diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index 308d9a21fa..56157ca9a6 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -32,6 +32,7 @@ setting maps to and their precedence, refer to documentation on that page. """ load("@bazel_skylib//lib:selects.bzl", "selects") +load("//python:py_library.bzl", "py_library") load("//python/private:common_labels.bzl", "labels") load("//python/private:text_util.bzl", "render") load( @@ -39,12 +40,12 @@ load( "DATA_LABEL", "DIST_INFO_LABEL", "EXTRACTED_WHEEL_FILES", - "PY_LIBRARY_IMPL_LABEL", "PY_LIBRARY_PUBLIC_LABEL", "PY_SRCS_LABEL", "WHEEL_FILE", "WHEEL_FILE_IMPL_LABEL", "WHEEL_FILE_PUBLIC_LABEL", + "extra_label", ) load(":whl_library_deps_targets.bzl", "whl_library_deps_targets") @@ -75,6 +76,7 @@ _LABEL_NONE = labels.NONE _LABEL_CURRENT_CONFIG = Label("//python/config_settings:current_config") _LABEL_CURRENT_CONFIG_NO_MATCH = Label("//python/config_settings:is_not_matching_current_config") _INCOMPATIBLE = "_no_matching_repository" +_DEFAULT = "//conditions:default" def pkg_aliases( *, @@ -83,6 +85,7 @@ def pkg_aliases( group_name = None, extra_aliases = None, requires_dist = [], + extras = [], include = [], group_deps = [], **kwargs): @@ -102,12 +105,20 @@ def pkg_aliases( extra_aliases: {type}`list[str]` The extra aliases to be created. requires_dist: {type}`list[str]` The list of dependencies. Comes from METADATA or a lock file. + extras: {type}`list[str]` The requested extras of the package when + `actual` is the name of a single repo. When `actual` is a dict, the + requested extras come from the {obj}`whl_config_setting` keys. include: {type}`list[str]` The subset of packages to include. Comes from `//:config.bzl#packages` **kwargs: extra kwargs to pass to {bzl:obj}`get_config_settings`. """ + + # The package without extras is the member of the group. + group_extras = [""] if group_name else [] + _native = kwargs.pop("native", native) rules = kwargs.pop("rules", struct( + py_library = py_library, whl_library_deps_targets = whl_library_deps_targets, )) select = kwargs.pop("select", selects.with_or) @@ -136,16 +147,11 @@ def pkg_aliases( ) target_names = {} else: - if group_name: - py_library_target = PY_LIBRARY_IMPL_LABEL - whl_target = WHEEL_FILE_IMPL_LABEL - else: - py_library_target = PY_LIBRARY_PUBLIC_LABEL - whl_target = WHEEL_FILE_PUBLIC_LABEL - + # The `pkg` targets are created by `_pkg_targets` below. The whl + # targets do not use the extras, so only the package without extras + # can be a member of the group. target_names = { - PY_LIBRARY_PUBLIC_LABEL: py_library_target, - WHEEL_FILE_PUBLIC_LABEL: whl_target, + WHEEL_FILE_PUBLIC_LABEL: WHEEL_FILE_IMPL_LABEL if "" in group_extras else WHEEL_FILE_PUBLIC_LABEL, } target_names = target_names | { @@ -159,6 +165,7 @@ def pkg_aliases( for x in extra_aliases or [] } + extras_by_setting = _extras_by_setting(actual, extras) actual = multiplatform_whl_aliases(aliases = actual, **kwargs) if type(actual) == type({}) and "//conditions:default" not in actual: _native.alias( @@ -212,16 +219,129 @@ def pkg_aliases( **kwargs ) - if group_name: - _native.alias( - name = PY_LIBRARY_PUBLIC_LABEL, - actual = "//_groups:{}_pkg".format(group_name), - ) + if "" in group_extras: _native.alias( name = WHEEL_FILE_PUBLIC_LABEL, actual = "//_groups:{}_whl".format(group_name), ) + if requires_dist: + if group_name: + _native.alias( + name = PY_LIBRARY_PUBLIC_LABEL, + actual = "//_groups:{}_pkg".format(group_name), + ) + else: + _pkg_targets( + actual = actual, + extras_by_setting = extras_by_setting, + group_name = group_name, + group_extras = group_extras, + native = _native, + rules = rules, + select = select, + ) + +def _extras_by_setting(aliases, extras): + """Return the requested extras for each config setting label.""" + if type(aliases) == type(""): + # A single repo is used for all of the platforms, e.g. in `WORKSPACE`. + return {_DEFAULT: tuple(sorted(extras))} + + return { + "//_config" + setting: alias.extras + for alias in aliases + if type(alias) == "struct" and alias.extras + for setting in get_config_settings( + target_platforms = alias.target_platforms, + python_version = alias.version, + ) + } + +def _pkg_targets(*, actual, extras_by_setting, group_name, group_extras, native, rules, select): + """Create the `pkg`, `[]` and `[]` targets. + + The whl repositories have an `[]` target for every extra that the + package provides and the hub exposes only the extras requested on each + platform. The `pkg` target includes the extras requested on each platform. + + Args: + actual: {type}`str | dict[str, str]` the repo name or the config + setting labels to repo names. + extras_by_setting: {type}`dict[str, tuple[str]]` the config setting + labels to the extras requested on that platform. + group_name: {type}`str | None` the group name that the pkg belongs to. + group_extras: {type}`list[str]` the members of the group in this + package, where an empty string is the package without extras. + native: the native struct. + rules: the rules struct. + select: the select function. + """ + if type(actual) == type(""): + actual = {_DEFAULT: actual} + + def _select(values): + return values[_DEFAULT] if values.keys() == [_DEFAULT] else select(values) + + def _target(repo, name): + return repo if repo == _INCOMPATIBLE else "@{}//:{}".format(repo, name) + + group_label = "//_groups:{}_{}".format(group_name, PY_LIBRARY_PUBLIC_LABEL) + + def _hub_label(extra): + return group_label if extra in group_extras else ":" + extra_label(extra) + + all_extras = sorted({ + extra: None + for extras in list(extras_by_setting.values()) + [group_extras] + for extra in extras + if extra + }) + for extra in [""] + all_extras: + # The platforms that do not request the extra get the package without + # extras. + targets = _select({ + setting: _target( + repo, + extra_label(extra) if extra in extras_by_setting.get(setting, ()) else PY_LIBRARY_PUBLIC_LABEL, + ) + for setting, repo in actual.items() + }) + + if extra not in group_extras: + native.alias(name = extra_label(extra), actual = targets) + continue + + # The group library uses the implementation targets of the members. + native.alias( + name = "_" + (extra_label(extra) if extra else PY_LIBRARY_PUBLIC_LABEL), + actual = targets, + visibility = ["//_groups:__subpackages__"], + ) + native.alias(name = extra_label(extra), actual = group_label) + + # The `[]` and `[]` targets have the no match error, so we do not + # need to handle the incompatible platforms here. + targets = { + setting: sorted({ + _hub_label(e): None + for e in extras_by_setting.get(setting, ()) or [""] + }) + for setting in actual + } + unique = {label: None for t in targets.values() for label in t} + if len(unique) == 1: + native.alias(name = PY_LIBRARY_PUBLIC_LABEL, actual = unique.keys()[0]) + elif all([len(t) == 1 for t in targets.values()]): + native.alias( + name = PY_LIBRARY_PUBLIC_LABEL, + actual = _select({setting: t[0] for setting, t in targets.items()}), + ) + else: + # Some platforms request more than one extra, so we need to combine + # the targets. + rules.py_library(name = PY_LIBRARY_PUBLIC_LABEL, deps = _select(targets)) + def multiplatform_whl_aliases( *, aliases = []): diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index 4343a7db1a..0cbbfb88d9 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -41,7 +41,8 @@ def _test_legacy_aliases(env): # buildifier: disable=unsorted-dict-items want = { "foo": ":pkg", - "pkg": "@repo//:pkg", + "pkg": ":[]", + "[]": "@repo//:pkg", "srcs": "@repo//:srcs", "whl_file": "@repo//:whl_file", "whl": "@repo//:whl", @@ -79,7 +80,8 @@ def _test_config_setting_aliases(env): # buildifier: disable=unsorted-dict-items want = { - "pkg": { + "pkg": ":[]", + "[]": { "//:my_config_setting": "@bar_baz_repo//:pkg", "//conditions:default": "_no_matching_repository", }, @@ -180,7 +182,8 @@ def _test_multiplatform_whl_aliases(env): # buildifier: disable=unsorted-dict-items want = { - "pkg": { + "pkg": ":[]", + "[]": { "//:my_config_setting": "@bzlmod_repo//:pkg", "//_config:is_cp39": "@version_repo//:pkg", "//_config:is_cp39_linux_x86_64": "@version_platform_repo//:pkg", @@ -240,6 +243,10 @@ def _test_group_aliases(env): "name": "extracted_whl_files", "actual": "@repo//:extracted_whl_files", }, + { + "name": "[]", + "actual": "//_groups:my_group_pkg", + }, { "name": "pkg", "actual": "//_groups:my_group_pkg", @@ -520,6 +527,209 @@ def _test_config_settings_exist_legacy(env): _tests.append(_test_config_settings_exist_legacy) +_LINUX = "//_config:is_cp312_linux_x86_64" +_OSX = "//_config:is_cp312_osx_aarch64" +_WINDOWS = "//_config:is_cp312_windows_x86_64" + +def _extras_actual(extras_by_platform): + return { + whl_config_setting( + version = "3.12", + target_platforms = ["cp312_" + platform], + extras = extras, + ): repo + for platform, (repo, extras) in extras_by_platform.items() + } + +def _test_extras_aliases(env): + got = {} + py_library_calls = [] + pkg_aliases( + name = "foo", + actual = _extras_actual({ + "linux_x86_64": ("repo", ["bar"]), + "osx_aarch64": ("repo", ["baz"]), + "windows_x86_64": ("other_repo", []), + }), + native = struct( + alias = lambda *, name, actual, visibility = None, tags = None: got.update({name: actual}), + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + ), + select = lambda value, no_match_error = None: value, + ) + + # buildifier: disable=unsorted-dict-items + want = { + "foo": ":pkg", + "pkg": { + _LINUX: ":[bar]", + _OSX: ":[baz]", + _WINDOWS: ":[]", + # The `[]` target has the no match error. + "//conditions:default": ":[]", + }, + "whl": { + _LINUX: "@repo//:whl", + _OSX: "@repo//:whl", + _WINDOWS: "@other_repo//:whl", + "//conditions:default": "_no_matching_repository", + }, + "[]": { + _LINUX: "@repo//:pkg", + _OSX: "@repo//:pkg", + _WINDOWS: "@other_repo//:pkg", + "//conditions:default": "_no_matching_repository", + }, + "[bar]": { + _LINUX: "@repo//:[bar]", + _OSX: "@repo//:pkg", + _WINDOWS: "@other_repo//:pkg", + "//conditions:default": "_no_matching_repository", + }, + "[baz]": { + _LINUX: "@repo//:pkg", + _OSX: "@repo//:[baz]", + _WINDOWS: "@other_repo//:pkg", + "//conditions:default": "_no_matching_repository", + }, + } + env.expect.that_dict(got).contains_at_least(want) + env.expect.that_collection(py_library_calls).contains_exactly([]) + +_tests.append(_test_extras_aliases) + +def _test_extras_aliases_union(env): + got = {} + py_library_calls = [] + pkg_aliases( + name = "foo", + actual = _extras_actual({ + "linux_x86_64": ("repo", ["a", "b"]), + "osx_aarch64": ("repo", ["c"]), + "windows_x86_64": ("repo", []), + }), + native = struct( + alias = lambda *, name, actual, visibility = None, tags = None: got.update({name: actual}), + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + ), + select = lambda value, no_match_error = None: value, + ) + + # buildifier: disable=unsorted-dict-items + env.expect.that_collection(py_library_calls).contains_exactly([{ + "name": "pkg", + "deps": { + _LINUX: [":[a]", ":[b]"], + _OSX: [":[c]"], + _WINDOWS: [":[]"], + "//conditions:default": [":[]"], + }, + }]) + env.expect.that_collection(got.keys()).not_contains("pkg") + + # The whl target does not use the extras. + # buildifier: disable=unsorted-dict-items + env.expect.that_dict(got).contains_at_least({ + "whl": { + _LINUX: "@repo//:whl", + _OSX: "@repo//:whl", + _WINDOWS: "@repo//:whl", + "//conditions:default": "_no_matching_repository", + }, + }) + +_tests.append(_test_extras_aliases_union) + +def _test_extras_aliases_group(env): + got = {} + got_visibility = {} + + def _alias(*, name, actual, visibility = None, **_): + got[name] = actual + if visibility: + got_visibility[name] = visibility + + pkg_aliases( + name = "foo", + actual = _extras_actual({ + "linux_x86_64": ("repo", ["bar"]), + "osx_aarch64": ("repo", []), + }), + group_name = "group", + native = struct(alias = _alias), + select = lambda value, no_match_error = None: value, + ) + + # The package without extras is the member of the group by default, the + # `[bar]` extra is not a member of the group. + # buildifier: disable=unsorted-dict-items + env.expect.that_dict(got).contains_at_least({ + "pkg": { + _LINUX: ":[bar]", + _OSX: "//_groups:group_pkg", + "//conditions:default": "//_groups:group_pkg", + }, + "[]": "//_groups:group_pkg", + "_pkg": { + _LINUX: "@repo//:pkg", + _OSX: "@repo//:pkg", + "//conditions:default": "_no_matching_repository", + }, + "[bar]": { + _LINUX: "@repo//:[bar]", + _OSX: "@repo//:pkg", + "//conditions:default": "_no_matching_repository", + }, + "whl": "//_groups:group_whl", + "_whl": { + _LINUX: "@repo//:whl", + _OSX: "@repo//:whl", + "//conditions:default": "_no_matching_repository", + }, + }) + env.expect.that_dict(got_visibility).contains_at_least({ + "_pkg": ["//_groups:__subpackages__"], + "_whl": ["//_groups:__subpackages__"], + }) + +_tests.append(_test_extras_aliases_group) + +def _test_extras_aliases_single_repo(env): + got = {} + py_library_calls = [] + + # This is how `WORKSPACE` uses a single repo for each package. + pkg_aliases( + name = "foo", + actual = "repo", + extras = ["b", "a"], + native = struct( + alias = lambda *, name, actual, visibility = None, tags = None: got.update({name: actual}), + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + ), + select = lambda value, no_match_error = None: value, + ) + + # buildifier: disable=unsorted-dict-items + env.expect.that_dict(got).contains_at_least({ + "[]": "@repo//:pkg", + "[a]": "@repo//:[a]", + "[b]": "@repo//:[b]", + "whl": "@repo//:whl", + }) + env.expect.that_collection(py_library_calls).contains_exactly([{ + "deps": [":[a]", ":[b]"], + "name": "pkg", + }]) + +_tests.append(_test_extras_aliases_single_repo) + def pkg_aliases_test_suite(name): """Create the test suite. From 998146caaf9d1b7e97cb1e80111378a90cccbfc1 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:54:32 +0900 Subject: [PATCH 14/55] feat(pypi): use a single whl repository for the different extras of a distribution --- python/private/pypi/BUILD.bazel | 1 + python/private/pypi/hub_builder.bzl | 109 ++++++++++++++++--- tests/pypi/hub_builder/hub_builder_tests.bzl | 108 ++++++++++++------ 3 files changed, 167 insertions(+), 51 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 8cee5b5210..87885b18cc 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -185,6 +185,7 @@ bzl_library( ":parse_requirements", ":pep508_env", ":pep508_evaluate", + ":pep508_requirement", ":python_tag", ":requirements_files_by_platform", ":whl_config_setting", diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index ef6a7247c1..b6809660fd 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -12,6 +12,7 @@ load(":hash.bzl", "hash") load(":parse_requirements.bzl", "parse_requirements") load(":pep508_env.bzl", "env") load(":pep508_evaluate.bzl", "evaluate") +load(":pep508_requirement.bzl", "requirement", "strip_extras") load(":python_tag.bzl", "python_tag") load(":requirements_files_by_platform.bzl", "requirements_files_by_platform") load(":whl_config_setting.bzl", "whl_config_setting") @@ -362,16 +363,16 @@ def _add_whl_library(self, *, python_version, whl, repo): self._whl_libraries[repo_name] = repo.args mapping = self._whl_map.setdefault(whl.name, {}) - if repo.config_setting in mapping and mapping[repo.config_setting] != repo_name: - fail( - "attempting to override an existing repo '{}' for config setting '{}' with a new repo '{}'".format( - mapping[repo.config_setting], - repo.config_setting, - repo_name, - ), - ) - else: - mapping[repo.config_setting] = repo_name + for config_setting in repo.config_settings: + if config_setting in mapping and mapping[config_setting] != repo_name: + fail( + "attempting to override an existing repo '{}' for config setting '{}' with a new repo '{}'".format( + mapping[config_setting], + config_setting, + repo_name, + ), + ) + mapping[config_setting] = repo_name ### end of setters, below we have various functions to implement the public methods @@ -606,7 +607,8 @@ def _create_whl_repos( whl = whl, whl_modifications = whl_modifications, ) - for src in whl.srcs: + srcs = _merge_extras(whl.srcs) + for src in srcs: repo = _whl_repo( src = src, index_url = whl.index_url, @@ -616,7 +618,11 @@ def _create_whl_repos( use_downloader = src.url and _use_downloader(self, python_version, whl.name), auth_patterns = self._config.auth_patterns or pip_attr.auth_patterns, python_version = _major_minor_version(python_version), - is_multiple_versions = whl.is_multiple_versions, + # Different extras for the same distribution are not different versions. + is_multiple_versions = whl.is_multiple_versions and len({ + s.requirement_line: None + for s in srcs + }) > 1, interpreter = interpreter, enable_pipstar_extract = enable_pipstar_extract, ) @@ -627,6 +633,77 @@ def _create_whl_repos( repo = repo, ) +def _merge_extras(srcs): + """Merge the sources that differ only in the requested extras. + + The whl repositories create targets for all of the extras that a package + provides, so the extras are not passed to them and the same distribution + requested with different extras on different platforms uses a single whl + repository. The extras of each platform are recorded so that the hub can + expose only the requested extras for each platform. + + Args: + srcs: {type}`list[struct]` the sources returned by `parse_requirements`. + + Returns: + The list of sources without the extras in the requirement line and + with an extra `extras_by_platform` attribute. + """ + merged = {} + for src in srcs: + line = strip_extras(src.requirement_line) + entry = merged.setdefault( + (src.filename, src.url, line, tuple(src.extra_pip_args)), + struct( + src = src, + line = line, + extras_by_platform = {}, + target_platforms = [], + ), + ) + extras = sorted(requirement(src.requirement_line).extras) + for p in src.target_platforms: + if p not in entry.target_platforms: + entry.target_platforms.append(p) + entry.extras_by_platform[p] = extras + + return [ + struct( + distribution = entry.src.distribution, + extra_pip_args = entry.src.extra_pip_args, + requirement_line = entry.line, + target_platforms = entry.target_platforms, + extras_by_platform = entry.extras_by_platform, + filename = entry.src.filename, + digest = entry.src.digest, + url = entry.src.url, + yanked = entry.src.yanked, + ) + for entry in merged.values() + ] + +def _config_settings(*, src, version, target_platforms): + """Return the config settings for each set of extras requested by the platforms.""" + plats_by_extras = {} + for p in src.target_platforms: + plats_by_extras.setdefault(tuple(src.extras_by_platform.get(p, [])), []).append(p) + + if len(plats_by_extras) == 1: + return [whl_config_setting( + version = version, + target_platforms = target_platforms or None, + extras = plats_by_extras.keys()[0], + )] + + return [ + whl_config_setting( + version = version, + target_platforms = plats, + extras = extras, + ) + for extras, plats in plats_by_extras.items() + ] + def _common_args(self, module_ctx, *, pip_attr): # Construct args separately so that the lock file can be smaller and does not include unused # attrs. @@ -725,9 +802,10 @@ def _whl_repo( *target_platforms ), args = args, - config_setting = whl_config_setting( + config_settings = _config_settings( + src = src, version = python_version, - target_platforms = target_platforms or None, + target_platforms = target_platforms, ), ) @@ -758,7 +836,8 @@ def _whl_repo( return struct( repo_name = whl_repo_name(src.filename, src.digest, *target_platforms), args = args, - config_setting = whl_config_setting( + config_settings = _config_settings( + src = src, version = python_version, target_platforms = src.target_platforms, ), diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index 073ccc0f58..216dca27f7 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -176,6 +176,51 @@ def _test_uv_lock_only(env): _tests.append(_test_uv_lock_only) +def _test_uv_lock_extras(env): + builder = hub_builder(env) + builder.pip_parse( + _mock_mctx( + os_name = "osx", + arch_name = "aarch64", + mock_files = { + "uv.lock": """{"package":[{"name":"simple","provides-extras":["async-rest","grpc"],"source":{"registry":"https://pypi.org/simple"},"version":"0.0.1","wheels":[{"hash":"sha256:deadbeef","url":"https://files.pythonhosted.org/packages/simple-0.0.1-py3-none-any.whl"}]}]}""", + }, + ), + _parse( + hub_name = "pypi", + python_version = "3.15", + requirements_lock = None, + uv_lock = "uv.lock", + target_platforms = ["osx_aarch64", "linux_x86_64"], + ), + ) + pypi = builder.build() + + pypi.whl_map().contains_exactly({ + "simple": { + "pypi_315_simple_py3_none_any_deadbeef": [ + whl_config_setting( + version = "3.15", + target_platforms = ["cp315_linux_x86_64", "cp315_osx_aarch64"], + extras = ["async_rest", "grpc"], + ), + ], + }, + }) + pypi.whl_libraries().contains_at_least({ + "pypi_315_simple_py3_none_any_deadbeef": { + "config_load": "@pypi//:config.bzl", + "dep_template": "@pypi//{name}:{target}", + "filename": "simple-0.0.1-py3-none-any.whl", + "index_url": "https://pypi.org/simple/simple", + "integrity": "sha256-3q2+7w==", + "requirement": "simple==0.0.1", + "urls": ["https://files.pythonhosted.org/packages/simple-0.0.1-py3-none-any.whl"], + }, + }) + +_tests.append(_test_uv_lock_extras) + def _test_simple_multiple_requirements(env): sub_tests = { ("osx", "aarch64"): "simple==0.0.2 --hash=sha256:deadb00f", @@ -224,10 +269,10 @@ _tests.append(_test_simple_multiple_requirements) def _test_simple_extras_vs_no_extras(env): sub_tests = { - ("osx", "aarch64"): "simple[foo]==0.0.1 --hash=sha256:deadbeef", - ("windows", "aarch64"): "simple==0.0.1 --hash=sha256:deadbeef", + ("osx", "aarch64"): ("simple==0.0.1 --hash=sha256:deadbeef", ["foo"]), + ("windows", "aarch64"): ("simple==0.0.1 --hash=sha256:deadbeef", None), } - for (host_os, host_arch), want_requirement in sub_tests.items(): + for (host_os, host_arch), (want_requirement, want_extras) in sub_tests.items(): builder = hub_builder(env) builder.pip_parse( mocks.mctx( @@ -252,7 +297,7 @@ def _test_simple_extras_vs_no_extras(env): pypi.whl_map().contains_exactly({ "simple": { "pypi_315_simple": [ - whl_config_setting(version = "3.15"), + whl_config_setting(version = "3.15", extras = want_extras), ], }, }) @@ -314,37 +359,30 @@ def _test_simple_extras_vs_no_extras_simpleapi(env): pypi.exposed_packages().contains_exactly(["simple"]) pypi.group_map().contains_exactly({}) + + # The same wheel is used on both platforms, so a single repository has the + # extras for all of the platforms and the hub selects the extras per platform. pypi.whl_map().contains_exactly({ "simple": { - "pypi_315_simple_py3_none_any_deadbeef_osx_aarch64": [ + "pypi_315_simple_py3_none_any_deadbeef": [ whl_config_setting( target_platforms = [ - "cp315_osx_aarch64", + "cp315_windows_aarch64", ], version = "3.15", ), - ], - "pypi_315_simple_py3_none_any_deadbeef_windows_aarch64": [ whl_config_setting( target_platforms = [ - "cp315_windows_aarch64", + "cp315_osx_aarch64", ], version = "3.15", + extras = ["foo"], ), ], }, }) pypi.whl_libraries().contains_exactly({ - "pypi_315_simple_py3_none_any_deadbeef_osx_aarch64": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", - "filename": "simple-0.0.1-py3-none-any.whl", - "index_url": "https://example.com/simple/", - "integrity": "sha256-3q2+7w==", - "requirement": "simple[foo]==0.0.1", - "urls": ["/simple-0.0.1-py3-none-any.whl"], - }, - "pypi_315_simple_py3_none_any_deadbeef_windows_aarch64": { + "pypi_315_simple_py3_none_any_deadbeef": { "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "filename": "simple-0.0.1-py3-none-any.whl", @@ -1416,10 +1454,10 @@ _tests.append(_test_simple_get_index) def _test_optimum_sys_platform_extra(env): sub_tests = { - ("osx", "aarch64"): "optimum[onnxruntime]==1.17.1", - ("linux", "aarch64"): "optimum[onnxruntime-gpu]==1.17.1", + ("osx", "aarch64"): ("optimum==1.17.1", ["onnxruntime"]), + ("linux", "aarch64"): ("optimum==1.17.1", ["onnxruntime_gpu"]), } - for (host_os, host_arch), want_requirement in sub_tests.items(): + for (host_os, host_arch), (want_requirement, want_extras) in sub_tests.items(): builder = hub_builder( env, ) @@ -1447,7 +1485,7 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' pypi.whl_map().contains_exactly({ "optimum": { "pypi_315_optimum": [ - whl_config_setting(version = "3.15"), + whl_config_setting(version = "3.15", extras = want_extras), ], }, }) @@ -1509,38 +1547,36 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' pypi.exposed_packages().contains_exactly(["optimum"]) pypi.group_map().contains_exactly({}) + + # The same distribution is requested with different extras on different + # platforms, so a single repository has all of the extras and the hub + # selects the extras per platform. pypi.whl_map().contains_exactly({ "optimum": { - "pypi_315_optimum_mylinuxx86_64": [ + "pypi_315_optimum": [ whl_config_setting( version = "3.15", target_platforms = [ "cp315_mylinuxx86_64", ], + extras = ["onnxruntime_gpu"], ), - ], - "pypi_315_optimum_myosxaarch64": [ whl_config_setting( version = "3.15", target_platforms = [ "cp315_myosxaarch64", ], + extras = ["onnxruntime"], ), ], }, }) pypi.whl_libraries().contains_exactly({ - "pypi_315_optimum_mylinuxx86_64": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", - "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "optimum[onnxruntime-gpu]==1.17.1", - }, - "pypi_315_optimum_myosxaarch64": { + "pypi_315_optimum": { "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "optimum[onnxruntime]==1.17.1", + "requirement": "optimum==1.17.1", }, }) pypi.extra_aliases().contains_exactly({}) @@ -1598,7 +1634,7 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' pypi.whl_map().contains_exactly({ "optimum": { "pypi_315_optimum": [ - whl_config_setting(version = "3.15"), + whl_config_setting(version = "3.15", extras = ["onnxruntime_gpu"]), ], }, }) @@ -1607,7 +1643,7 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "optimum[onnxruntime-gpu]==1.17.1", + "requirement": "optimum==1.17.1", }, }) pypi.extra_aliases().contains_exactly({}) From dfaa556ea997b8c1cbfe1593dd5c18ba22d4bd80 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:54:49 +0900 Subject: [PATCH 15/55] feat(pypi): support extras in the requirement macro and add the requirements macro --- python/private/pypi/labels.bzl | 55 +++++++++++++++++++ .../private/pypi/requirements.bzl.tmpl.bzlmod | 24 +++++++- tests/pypi/labels/BUILD.bazel | 5 ++ tests/pypi/labels/labels_tests.bzl | 53 ++++++++++++++++++ 4 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 tests/pypi/labels/BUILD.bazel create mode 100644 tests/pypi/labels/labels_tests.bzl diff --git a/python/private/pypi/labels.bzl b/python/private/pypi/labels.bzl index f6257106e9..3414d586a7 100644 --- a/python/private/pypi/labels.bzl +++ b/python/private/pypi/labels.bzl @@ -39,6 +39,61 @@ def extra_label(extra): """ return "[{}]".format(extra) +def _labels(tmpl, requirement): + name, bracket, extras = requirement.partition("[") + name = normalize_name(name.strip()) + if not bracket: + return [tmpl.format(name, PY_LIBRARY_PUBLIC_LABEL)] + + extras, _, _ = extras.partition("]") + extras = sorted([ + normalize_name(extra.strip()) + for extra in extras.split(",") + if extra.strip() + ]) + return [tmpl.format(name, extra_label(extra)) for extra in extras or [""]] + +def requirement_label(tmpl, requirement, *, fail = fail): + """Return the label for a requirement in a hub repository. + + Args: + tmpl: {type}`str` the label template with two `{}` placeholders for + the package name and the target name. + requirement: {type}`str` the name of the package with at most one + extra, e.g. `foo`, `foo[]` or `foo[bar]`. + fail: the function to call on errors, used in tests. + + Returns: + {type}`str` the label of the `pkg` target if no extras are given, the + label of the `[]` target if empty extras are given, or the label of + the `[extra]` target if one extra is given. + """ + labels = _labels(tmpl, requirement) + if len(labels) > 1: + fail("Only a single extra is supported by 'requirement', use 'requirements(\"{}\")' instead to get a list of labels.".format(requirement)) + + return labels[0] + +def requirement_labels(tmpl, *requirements): + """Return the labels for requirements in a hub repository. + + Args: + tmpl: {type}`str` the label template with two `{}` placeholders for + the package name and the target name. + *requirements: {type}`str` the names of the packages with optional + extras, e.g. `foo`, `foo[]`, `foo[bar]` or `foo[bar,baz]`. + + Returns: + {type}`list[str]` the labels for each requirement, with an `[extra]` + label for each extra of a requirement. Each `[extra]` target includes + the package. + """ + return [ + label + for requirement in requirements + for label in _labels(tmpl, requirement) + ] + def group_member(spec): """Normalize a member of a dependency group. diff --git a/python/private/pypi/requirements.bzl.tmpl.bzlmod b/python/private/pypi/requirements.bzl.tmpl.bzlmod index ba227aeb2d..1b6a6c02dd 100644 --- a/python/private/pypi/requirements.bzl.tmpl.bzlmod +++ b/python/private/pypi/requirements.bzl.tmpl.bzlmod @@ -4,6 +4,7 @@ """ load("@rules_python//python:pip.bzl", "pip_utils") +load("@rules_python//python/private/pypi:labels.bzl", "requirement_label", "requirement_labels") all_requirements = %%ALL_REQUIREMENTS%% @@ -14,7 +15,28 @@ all_whl_requirements = all_whl_requirements_by_package.values() all_data_requirements = %%ALL_DATA_REQUIREMENTS%% def requirement(name): - return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "pkg") + """Return the label for the package. + + Args: + name: The name of the package with at most one extra, e.g. `foo`, + `foo[]` or `foo[bar]`. Use `requirements` for multiple extras. + + Returns: + The label of the package. + """ + return requirement_label("%%MACRO_TMPL%%", name) + +def requirements(*names): + """Return the labels for the packages. + + Args: + *names: The names of the packages with optional extras, e.g. `foo`, + `foo[]`, `foo[bar]` or `foo[bar,baz]`. + + Returns: + The list of labels, with a label for each extra of a package. + """ + return requirement_labels("%%MACRO_TMPL%%", *names) def whl_requirement(name): return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "whl") diff --git a/tests/pypi/labels/BUILD.bazel b/tests/pypi/labels/BUILD.bazel new file mode 100644 index 0000000000..ed55e66eae --- /dev/null +++ b/tests/pypi/labels/BUILD.bazel @@ -0,0 +1,5 @@ +load(":labels_tests.bzl", "labels_test_suite") + +labels_test_suite( + name = "labels_tests", +) diff --git a/tests/pypi/labels/labels_tests.bzl b/tests/pypi/labels/labels_tests.bzl new file mode 100644 index 0000000000..b26e6db471 --- /dev/null +++ b/tests/pypi/labels/labels_tests.bzl @@ -0,0 +1,53 @@ +"" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:labels.bzl", "requirement_label", "requirement_labels") # buildifier: disable=bzl-visibility + +_tests = [] + +_TMPL = "@@pypi//{}:{}" + +def _test_requirement_label(env): + for requirement, want in { + "Foo.Bar": "@@pypi//foo_bar:pkg", + "foo[ grpc ]": "@@pypi//foo:[grpc]", + "foo[Async-Rest]": "@@pypi//foo:[async_rest]", + "foo[]": "@@pypi//foo:[]", + "foo[grpc]": "@@pypi//foo:[grpc]", + }.items(): + env.expect.where(requirement = requirement).that_str(requirement_label(_TMPL, requirement)).equals(want) + +_tests.append(_test_requirement_label) + +def _test_requirement_label_multiple_extras_fails(env): + errors = [] + requirement_label( + _TMPL, + "google-api-core[grpc,async_rest]", + fail = lambda msg: errors.append(msg), + ) + env.expect.that_collection(errors).has_size(1) + env.expect.that_str(errors[0]).contains("requirements(\"google-api-core[grpc,async_rest]\")") + +_tests.append(_test_requirement_label_multiple_extras_fails) + +def _test_requirement_labels(env): + env.expect.that_collection( + requirement_labels(_TMPL, "google-api-core[grpc, async_rest]", "foobar[baz]", "foo", "bar[]"), + ).contains_exactly([ + "@@pypi//google_api_core:[async_rest]", + "@@pypi//google_api_core:[grpc]", + "@@pypi//foobar:[baz]", + "@@pypi//foo:pkg", + "@@pypi//bar:[]", + ]).in_order() + +_tests.append(_test_requirement_labels) + +def labels_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) From 6e7744866029e968d2440fdaf29b07072892c00f Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:55:41 +0900 Subject: [PATCH 16/55] refactor(pypi): extract the failing action of missing_package_error --- python/private/pypi/missing_package.bzl | 28 ++++++++++++++----------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/python/private/pypi/missing_package.bzl b/python/private/pypi/missing_package.bzl index 46823e813b..4b750caf45 100644 --- a/python/private/pypi/missing_package.bzl +++ b/python/private/pypi/missing_package.bzl @@ -3,23 +3,14 @@ load("//python/private:py_info.bzl", "PyInfo") load("//python/private:reexports.bzl", "BuiltinPyInfo") -def _missing_package_error_impl(ctx): +def _error(ctx, message): out = ctx.actions.declare_file(ctx.label.name + ".error") - if ctx.attr.hub_name: - hub_clause = ' when building under PyPI hub "{hub}". Try adding it to the requirements of this hub (e.g. requirements_lock or requirements_by_platform in pip.parse)'.format( - hub = ctx.attr.hub_name, - ) - else: - hub_clause = ' because no default PyPI hub was configured. Try designating a default hub via pip.default(default_hub = "...") or select a hub using --@rules_python//python/config_settings:venv' - # Register an action that fails when Bazel attempts to stage/build this file ctx.actions.run_shell( outputs = [out], - command = "echo 'ERROR: PyPI package \"{pkg}\" is not available{hub_clause}.' >&2 && exit 1".format( - pkg = ctx.attr.package_name, - hub_clause = hub_clause, - ), + command = "echo \"$1\" >&2 && exit 1", + arguments = ["ERROR: " + message], ) maybe_builtin = [BuiltinPyInfo(transitive_sources = depset([out]))] if BuiltinPyInfo != None else [] @@ -34,6 +25,19 @@ def _missing_package_error_impl(ctx): ), ] + maybe_builtin +def _missing_package_error_impl(ctx): + if ctx.attr.hub_name: + hub_clause = ' when building under PyPI hub "{hub}". Try adding it to the requirements of this hub (e.g. requirements_lock or requirements_by_platform in pip.parse)'.format( + hub = ctx.attr.hub_name, + ) + else: + hub_clause = ' because no default PyPI hub was configured. Try designating a default hub via pip.default(default_hub = "...") or select a hub using --@rules_python//python/config_settings:venv' + + return _error(ctx, 'PyPI package "{pkg}" is not available{hub_clause}.'.format( + pkg = ctx.attr.package_name, + hub_clause = hub_clause, + )) + missing_package_error = rule( implementation = _missing_package_error_impl, attrs = { From 3284430bbc15079e71218906de55dc7825ad4e16 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:55:51 +0900 Subject: [PATCH 17/55] feat(pypi): fail the build when a required extra of a dependency is missing --- python/private/pypi/BUILD.bazel | 1 + python/private/pypi/hub_repository.bzl | 15 ++-- python/private/pypi/missing_package.bzl | 24 +++++- python/private/pypi/render_pkg_aliases.bzl | 19 +++++ .../private/pypi/whl_library_deps_targets.bzl | 28 ++++++- .../render_pkg_aliases_test.bzl | 19 +++++ .../whl_library_deps_targets_tests.bzl | 84 +++++++++++++++++++ 7 files changed, 181 insertions(+), 9 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 87885b18cc..57dd8ad343 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -612,6 +612,7 @@ bzl_library( deps = [ ":env_marker_setting", ":labels", + ":missing_package", ":pep508_deps", "//python:py_library", "//python/private:normalize_name", diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index 27b324f465..7389c50964 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -15,7 +15,7 @@ "" load("//python/private:text_util.bzl", "render") -load(":render_pkg_aliases.bzl", "render_multiplatform_pkg_aliases") +load(":render_pkg_aliases.bzl", "available_extras", "render_multiplatform_pkg_aliases") load(":whl_config_setting.bzl", "whl_config_setting") _BUILD_FILE_CONTENTS = """\ @@ -27,11 +27,12 @@ exports_files(["requirements.bzl"]) def _impl(rctx): bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() + whl_map = { + key: _whl_config_settings_from_json(values) + for key, values in rctx.attr.whl_map.items() + } aliases = render_multiplatform_pkg_aliases( - aliases = { - key: _whl_config_settings_from_json(values) - for key, values in rctx.attr.whl_map.items() - }, + aliases = whl_map, extra_hub_aliases = rctx.attr.extra_hub_aliases, requirement_cycles = rctx.attr.groups, platform_config_settings = rctx.attr.platform_config_settings, @@ -50,7 +51,9 @@ def _impl(rctx): "config.bzl", rctx.attr._config_template, substitutions = { - "%%PACKAGES%%": render.dict(rctx.attr.whl_map, value_repr = lambda x: "None"), + # The available extras of each package for the spoke repositories + # to check if the extras of their dependencies are available. + "%%PACKAGES%%": render.dict(available_extras(whl_map), value_repr = render.list), }, ) rctx.template("requirements.bzl", rctx.attr._requirements_bzl_template, substitutions = { diff --git a/python/private/pypi/missing_package.bzl b/python/private/pypi/missing_package.bzl index 4b750caf45..e5647d5786 100644 --- a/python/private/pypi/missing_package.bzl +++ b/python/private/pypi/missing_package.bzl @@ -1,4 +1,4 @@ -"""Rule for generating an execution-phase action failure when a PyPI package is missing.""" +"""Rules for generating an execution-phase action failure when a PyPI package or its extra is missing.""" load("//python/private:py_info.bzl", "PyInfo") load("//python/private:reexports.bzl", "BuiltinPyInfo") @@ -45,3 +45,25 @@ missing_package_error = rule( "package_name": attr.string(mandatory = True), }, ) + +def _missing_extra_error_impl(ctx): + return _error(ctx, ( + 'The PyPI package "{pkg}" requires "{dep}[{extra}]", but the extra "{extra}" of ' + + '"{dep}" is not available in the hub. Add "{dep}[{extra}]" to the requirements or ' + + "the lock file of the hub, e.g. by compiling the requirements with the extras kept " + + "(`--no-strip-extras`) or by adding the extra to the project dependencies." + ).format( + pkg = ctx.attr.required_by, + dep = ctx.attr.package_name, + extra = ctx.attr.extra, + )) + +missing_extra_error = rule( + implementation = _missing_extra_error_impl, + doc = "A target that fails the build with a helpful message when an extra of a PyPI package is missing.", + attrs = { + "extra": attr.string(mandatory = True), + "package_name": attr.string(mandatory = True), + "required_by": attr.string(mandatory = True), + }, +) diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 271a12ddfb..44b80f0d28 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -243,3 +243,22 @@ def _non_versioned_platform(p, *, strict = False): return p else: fail("Expected to always have a platform in the form '{{abi}}_{{os}}_{{arch}}', got: {}".format(p)) + +def available_extras(aliases): + """Return the extras that are available in the hub for each package. + + Args: + aliases: {type}`dict[str, dict[struct, str]]` the packages to the + {obj}`whl_config_setting` instances and the repo names. + + Returns: + {type}`dict[str, list[str]]` the sorted extras for each package. + """ + return { + pkg: sorted({ + extra: None + for setting in settings + for extra in setting.extras or [] + }) + for pkg, settings in aliases.items() + } diff --git a/python/private/pypi/whl_library_deps_targets.bzl b/python/private/pypi/whl_library_deps_targets.bzl index 42de802267..5cce4e870f 100644 --- a/python/private/pypi/whl_library_deps_targets.bzl +++ b/python/private/pypi/whl_library_deps_targets.bzl @@ -15,6 +15,7 @@ load( "extra_label", "group_member", ) +load(":missing_package.bzl", "missing_extra_error") load(":pep508_deps.bzl", "deps") def whl_library_deps_targets( @@ -35,6 +36,7 @@ def whl_library_deps_targets( rules = struct( py_library = py_library, env_marker_setting = env_marker_setting, + missing_extra_error = missing_extra_error, )): """Create all of the whl_library targets. @@ -51,7 +53,11 @@ def whl_library_deps_targets( provides_extra: {type}`list[str]` The list of extras that the package provides. For each extra the `[]` target is created that includes the package and the dependencies of that extra. - include: {type}`list[str]` The list of packages to include. + include: {type}`list[str] | dict[str, list[str] | None]` The list + of packages to include. If it is a dict, then the values are the + extras of each package that are available in the hub and the + dependencies on the extras that are not available will fail the + build with a helpful message. group_name: {type}`str | None` name of the dependency group (if any). dep_template: {type}`str | None` The dep_template to use. tags: {type}`list[str]` The tags set on the targets. @@ -158,12 +164,21 @@ def whl_library_deps_targets( # dep and the expression. settings = {} + # The dependencies on extras that are not available in the hub. + missing_extras = {} + def _pkg_label(dep): dep_name, _, extra = dep.partition("[") if not extra: return dep_template.format(name = dep_name, target = PY_LIBRARY_PUBLIC_LABEL) - return dep_template.format(name = dep_name, target = extra_label(extra.rstrip("]"))) + extra = extra.rstrip("]") + available = include.get(dep_name) if type(include) == type({}) else None + if available != None and extra not in available: + missing_extras[dep] = (dep_name, extra) + return ":_missing_" + dep + + return dep_template.format(name = dep_name, target = extra_label(extra)) whl_label = lambda dep: dep_template.format(name = dep, target = WHEEL_FILE_PUBLIC_LABEL) @@ -226,6 +241,15 @@ def whl_library_deps_targets( visibility = ["//visibility:private"], ) + for dep, (dep_name, extra) in sorted(missing_extras.items()): + rules.missing_extra_error( + name = "_missing_" + dep, + package_name = dep_name, + extra = extra, + required_by = metadata_name, + visibility = ["//visibility:private"], + ) + def _conditions(settings, package_deps): return { dep: _setting_name(settings, dep, expression) diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index 807c4728fd..6786d27040 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -17,6 +17,7 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load( "//python/private/pypi:render_pkg_aliases.bzl", + "available_extras", "get_whl_flag_versions", "render_multiplatform_pkg_aliases", "render_pkg_aliases", @@ -269,6 +270,24 @@ pkg_aliases( _tests.append(_test_aliases_with_extras) +def _test_available_extras(env): + got = available_extras({ + "bar": { + whl_config_setting(version = "3.2"): "pypi_32_bar", + }, + "foo": { + whl_config_setting(version = "3.2", target_platforms = ["cp32_linux_x86_64"], extras = ["b"]): "pypi_32_foo", + whl_config_setting(version = "3.2", target_platforms = ["cp32_osx_aarch64"], extras = ["a", "b"]): "pypi_32_foo", + }, + }) + + env.expect.that_dict(got).contains_exactly({ + "bar": [], + "foo": ["a", "b"], + }) + +_tests.append(_test_available_extras) + def render_pkg_aliases_test_suite(name): """Create the test suite. diff --git a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl index 788bf44f7f..420efbdbcd 100644 --- a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl +++ b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl @@ -369,6 +369,90 @@ def _test_extras_in_group_legacy(env): _tests.append(_test_extras_in_group_legacy) +def _test_deps_with_extras(env): + filegroup_calls = [] + py_library_calls = [] + env_marker_setting_calls = [] + missing_calls = [] + + whl_library_deps_targets( + name = "foo-0-py3-none-any.whl", + metadata_name = "foo", + dep_template = "@pypi//{name}:{target}", + requires_dist = [ + "bar[socks]", + "baz[a,b]; sys_platform == 'linux'", + "baz; sys_platform == 'darwin'", + "qux[missing]", + ], + # The values are the extras that are available in the hub + include = { + "bar": ["socks"], + "baz": ["a", "b"], + "qux": [], + }, + repo = None, + aliases = {}, + native = struct( + filegroup = lambda **kwargs: filegroup_calls.append(kwargs), + alias = lambda **kwargs: None, + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + env_marker_setting = lambda **kwargs: env_marker_setting_calls.append(kwargs), + missing_extra_error = lambda **kwargs: missing_calls.append(kwargs), + ), + ) + + # The dependencies use the targets with the extras, the missing extras use + # a target that fails the build with a helpful message. + env.expect.that_collection([ + (c["name"], c["deps"]) + for c in py_library_calls + ]).contains_exactly([ + ("pkg", ["srcs", "@pypi//bar:[socks]", ":_missing_qux[missing]"] + select({ + ":is_include_baz_2_true": ["@pypi//baz:pkg"], + "//conditions:default": [], + }) + select({ + ":is_include_baz[a]_true": ["@pypi//baz:[a]"], + "//conditions:default": [], + }) + select({ + ":is_include_baz[b]_true": ["@pypi//baz:[b]"], + "//conditions:default": [], + })), + ]) + env.expect.that_collection(missing_calls).contains_exactly([ + { + "name": "_missing_qux[missing]", + "package_name": "qux", + "extra": "missing", + "required_by": "foo", + "visibility": ["//visibility:private"], + }, + ]) # buildifier: @unsorted-dict-items + + # The whl targets do not use the extras. + env.expect.that_collection([ + (c["name"], c["data"]) + for c in filegroup_calls + ]).contains_exactly([ + ("whl", ["@pypi//bar:whl", "@pypi//qux:whl"] + select({ + ":is_include_baz_true": ["@pypi//baz:whl"], + "//conditions:default": [], + })), + ]) + env.expect.that_collection([ + (c["name"], c["expression"]) + for c in env_marker_setting_calls + ]).contains_exactly([ + ("include_baz", "(sys_platform == \"darwin\") or (sys_platform == \"linux\")"), + ("include_baz_2", "sys_platform == \"darwin\""), + ("include_baz[a]", "sys_platform == \"linux\""), + ("include_baz[b]", "sys_platform == \"linux\""), + ]) + +_tests.append(_test_deps_with_extras) + def _test_deps_with_extras_without_hub_info(env): py_library_calls = [] From cca8d1bc28a8e5f492fcb4cdd7d00816a753024b Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:56:55 +0900 Subject: [PATCH 18/55] feat(pypi): support the extra targets in WORKSPACE --- python/private/pypi/BUILD.bazel | 1 + python/private/pypi/pip_repository.bzl | 19 +++++++++++---- python/private/pypi/render_pkg_aliases.bzl | 5 +++- .../pypi/requirements.bzl.tmpl.workspace | 23 ++++++++++++++++++- .../render_pkg_aliases_test.bzl | 17 ++++++++++++++ 5 files changed, 59 insertions(+), 6 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 57dd8ad343..b295f12a29 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -329,6 +329,7 @@ bzl_library( ":hash", ":parse_requirements", ":pep508_env", + ":pep508_requirement", ":pip_repository_attrs", ":pypi_repo_utils", ":render_pkg_aliases", diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index d02c62eae1..2b6721fc6f 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -24,6 +24,7 @@ load(":default_platforms.bzl", "default_platforms") load(":hash.bzl", "hash") load(":parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement") load(":pep508_env.bzl", "env") +load(":pep508_requirement.bzl", "strip_extras", parse_requirement = "requirement") load(":pip_repository_attrs.bzl", "ATTRS") load(":pypi_repo_utils.bzl", "pypi_repo_utils") load(":render_pkg_aliases.bzl", "render_pkg_aliases") @@ -150,6 +151,7 @@ def _pip_repository_impl(rctx): logger = logger, ) selected_requirements = {} + selected_extras = {} selected_srcs = {} options = None repository_platform = host_platform(rctx) @@ -161,7 +163,11 @@ def _pip_repository_impl(rctx): if not requirement: continue options = options or requirement.extra_pip_args - selected_requirements[whl.name] = requirement.requirement_line + + # The whl repositories have targets for all of the extras that a + # package provides, so the extras are exposed only via the hub. + selected_requirements[whl.name] = strip_extras(requirement.requirement_line) + selected_extras[whl.name] = parse_requirement(requirement.requirement_line).extras if not requirement.url: continue @@ -183,7 +189,7 @@ def _pip_repository_impl(rctx): # ensure that only the locked distributions can be used. digests = sorted({src.digest: None for src in whl.srcs}) if all(digests): - selected_requirements[whl.name] = " ".join([requirement.requirement_line] + [ + selected_requirements[whl.name] = " ".join([selected_requirements[whl.name]] + [ "--hash={}".format(d) for d in digests ]) @@ -224,6 +230,7 @@ def _pip_repository_impl(rctx): # NOTE: Maintain the order consistent with `buildifier` 'load("@rules_python//python:pip.bzl", "pip_utils")', 'load("@rules_python//python/pip_install:pip_repository.bzl", "whl_config_repo", "whl_library")', + 'load("@rules_python//python/private/pypi:labels.bzl", "requirement_label", "requirement_labels") # buildifier: disable=bzl-visibility', ] annotations = {} @@ -261,6 +268,7 @@ def _pip_repository_impl(rctx): for pkg in bzl_packages }, extra_hub_aliases = rctx.attr.extra_hub_aliases, + extras = selected_extras, requirement_cycles = requirement_cycles, ) for path, contents in aliases.items(): @@ -272,10 +280,13 @@ def _pip_repository_impl(rctx): "config.bzl", rctx.attr._config_template, substitutions = { + # The available extras of each package for the spoke + # repositories to check if the extras of their dependencies + # are available. "%%PACKAGES%%": render.dict({ - pkg: None + pkg: sorted(selected_extras.get(pkg, [])) for pkg in bzl_packages - }, value_repr = lambda x: "None"), + }, value_repr = render.list), }, ) config_repo_template = repr(rctx.attr.name) diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 44b80f0d28..9894f555c7 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -88,7 +88,7 @@ package(default_visibility = ["//visibility:public"]) extra_loads = extra_loads, ) -def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases = {}, **kwargs): +def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases = {}, extras = {}, **kwargs): """Create alias declarations for each PyPI package. The aliases should be appended to the pip_repository BUILD.bazel file. These aliases @@ -101,6 +101,8 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases requirement_cycles: any package groups to also add. extra_hub_aliases: The list of extra aliases for each whl to be added in addition to the default ones. + extras: {type}`dict[str, list[str]]` The requested extras for each + whl when the aliases are repo names. **kwargs: Extra kwargs to pass to the rules. Returns: @@ -130,6 +132,7 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases name = normalize_name(name), aliases = pkg_aliases, extra_aliases = extra_hub_aliases.get(normalize_name(name), []), + extras = extras.get(normalize_name(name), []), group_name = whl_group_mapping.get(normalize_name(name)), **kwargs ).strip() diff --git a/python/private/pypi/requirements.bzl.tmpl.workspace b/python/private/pypi/requirements.bzl.tmpl.workspace index faf774e7ed..65d9938cef 100644 --- a/python/private/pypi/requirements.bzl.tmpl.workspace +++ b/python/private/pypi/requirements.bzl.tmpl.workspace @@ -19,7 +19,28 @@ _config = %%CONFIG%% _annotations = %%ANNOTATIONS%% def requirement(name): - return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "pkg") + """Return the label for the package. + + Args: + name: The name of the package with at most one extra, e.g. `foo`, + `foo[]` or `foo[bar]`. Use `requirements` for multiple extras. + + Returns: + The label of the package. + """ + return requirement_label("%%MACRO_TMPL%%", name) + +def requirements(*names): + """Return the labels for the packages. + + Args: + *names: The names of the packages with optional extras, e.g. `foo`, + `foo[]`, `foo[bar]` or `foo[bar,baz]`. + + Returns: + The list of labels, with a label for each extra of a package. + """ + return requirement_labels("%%MACRO_TMPL%%", *names) def whl_requirement(name): return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "whl") diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index 6786d27040..40438134ef 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -288,6 +288,23 @@ def _test_available_extras(env): _tests.append(_test_available_extras) +def _test_aliases_with_extras_for_repo_names(env): + actual = render_pkg_aliases( + aliases = { + "foo": "pypi_foo", + }, + extras = {"foo": ["a"]}, + ) + + env.expect.that_str(actual["foo/BUILD.bazel"]).contains("""\ +pkg_aliases( + name = "foo", + actual = "pypi_foo", + extras = ["a"], +)""") + +_tests.append(_test_aliases_with_extras_for_repo_names) + def render_pkg_aliases_test_suite(name): """Create the test suite. From 181ddc9dbfa534c7989669b743d947572a29da9d Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:57:31 +0900 Subject: [PATCH 19/55] feat(pypi): support extras as members of the requirement cycle groups --- python/private/pypi/BUILD.bazel | 3 ++ .../generate_group_library_build_bazel.bzl | 47 ++++++++++--------- python/private/pypi/hub_builder.bzl | 13 +++-- python/private/pypi/pip_repository.bzl | 7 ++- python/private/pypi/pkg_aliases.bzl | 12 +++-- python/private/pypi/render_pkg_aliases.bzl | 22 ++++++--- .../pypi/requirements.bzl.tmpl.workspace | 4 +- ...nerate_group_library_build_bazel_tests.bzl | 27 +++++++++++ tests/pypi/hub_builder/hub_builder_tests.bzl | 40 ++++++++++++++++ tests/pypi/pkg_aliases/pkg_aliases_test.bzl | 29 ++++++++++++ .../render_pkg_aliases_test.bzl | 22 +++++++++ 11 files changed, 187 insertions(+), 39 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index b295f12a29..1678a5d94d 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -182,6 +182,7 @@ bzl_library( deps = [ ":attrs", ":hash", + ":labels", ":parse_requirements", ":pep508_env", ":pep508_evaluate", @@ -327,6 +328,7 @@ bzl_library( deps = [ ":default_platforms", ":hash", + ":labels", ":parse_requirements", ":pep508_env", ":pep508_requirement", @@ -389,6 +391,7 @@ bzl_library( srcs = ["render_pkg_aliases.bzl"], deps = [ ":generate_group_library_build_bazel", + ":labels", "//python/private:normalize_name", "//python/private:text_util", ], diff --git a/python/private/pypi/generate_group_library_build_bazel.bzl b/python/private/pypi/generate_group_library_build_bazel.bzl index 571cfd6b3f..114f96d41e 100644 --- a/python/private/pypi/generate_group_library_build_bazel.bzl +++ b/python/private/pypi/generate_group_library_build_bazel.bzl @@ -22,6 +22,8 @@ load( "PY_LIBRARY_PUBLIC_LABEL", "WHEEL_FILE_IMPL_LABEL", "WHEEL_FILE_PUBLIC_LABEL", + "extra_label", + "group_member", ) _PRELUDE = """\ @@ -60,35 +62,36 @@ def _generate_group_libraries(repo_prefix, group_name, group_members): repo_prefix: str; the pip_parse repo prefix. group_name: str; the name which the user provided for the dep group. group_members: list[str]; the names of the _packages_ (not repositories) - which make up the group. + which make up the group, optionally with a single extra, e.g. `foo[bar]`. """ - group_members = sorted(group_members) + # The members are the packages, e.g. `foo`, or the extras of the packages, + # e.g. `foo[bar]`. The whl targets do not use the extras, so only the + # packages are members for the whl targets. + members = sorted([group_member(d) for d in group_members]) + lib_targets = [] + whl_targets = [] + for member in members: + name, _, extra = member.partition("[") + if extra: + lib_targets.append((name, "_" + extra_label(extra.rstrip("]")))) + else: + lib_targets.append((name, PY_LIBRARY_IMPL_LABEL)) + whl_targets.append((name, WHEEL_FILE_IMPL_LABEL)) if repo_prefix: - lib_dependencies = [ - "@%s%s//:%s" % (repo_prefix, normalize_name(d), PY_LIBRARY_IMPL_LABEL) - for d in group_members - ] - whl_file_deps = [ - "@%s%s//:%s" % (repo_prefix, normalize_name(d), WHEEL_FILE_IMPL_LABEL) - for d in group_members - ] - visibility = [ - "@%s%s//:__pkg__" % (repo_prefix, normalize_name(d)) - for d in group_members - ] + label_tmpl = "@" + repo_prefix + "{}//:{}" + visibility = sorted({ + "@%s%s//:__pkg__" % (repo_prefix, name): None + for name, _ in lib_targets + }) else: - lib_dependencies = [ - "//%s:%s" % (normalize_name(d), PY_LIBRARY_IMPL_LABEL) - for d in group_members - ] - whl_file_deps = [ - "//%s:%s" % (normalize_name(d), WHEEL_FILE_IMPL_LABEL) - for d in group_members - ] + label_tmpl = "//{}:{}" visibility = ["//:__subpackages__"] + lib_dependencies = [label_tmpl.format(name, target) for name, target in lib_targets] + whl_file_deps = [label_tmpl.format(name, target) for name, target in whl_targets] + return _GROUP_TEMPLATE.format( name = normalize_name(group_name), whl_public_label = WHEEL_FILE_PUBLIC_LABEL, diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index b6809660fd..841f012242 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -9,6 +9,7 @@ load("//python/private:version.bzl", "version") load("//python/private:version_label.bzl", "version_label") load(":attrs.bzl", "use_isolated") load(":hash.bzl", "hash") +load(":labels.bzl", "group_member") load(":parse_requirements.bzl", "parse_requirements") load(":pep508_env.bzl", "env") load(":pep508_evaluate.bzl", "evaluate") @@ -258,18 +259,20 @@ def _add_group_map(self, group_map): # cycles for different abis/oses? For now we will need the users to # assume the same groups across all versions/platforms until we start # using an alternative cycle resolution strategy. + # The members are the packages, e.g. `foo`, or the extras of the packages, + # e.g. `foo[bar]`. group_map = { - name: [normalize_name(whl_name) for whl_name in whls] - for name, whls in group_map.items() + name: [group_member(member) for member in members] + for name, members in group_map.items() } self._group_map.clear() self._group_name_by_whl.clear() self._group_map.update(group_map) self._group_name_by_whl.update({ - whl_name: group_name - for group_name, group_whls in self._group_map.items() - for whl_name in group_whls + member.partition("[")[0]: group_name + for group_name, members in self._group_map.items() + for member in members }) def _add_extra_aliases(self, extra_hub_aliases): diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index 2b6721fc6f..67bbe1e945 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -22,6 +22,7 @@ load("//python/private:text_util.bzl", "render") load("//python/private:version.bzl", "version") load(":default_platforms.bzl", "default_platforms") load(":hash.bzl", "hash") +load(":labels.bzl", "group_member") load(":parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement") load(":pep508_env.bzl", "env") load(":pep508_requirement.bzl", "strip_extras", parse_requirement = "requirement") @@ -222,7 +223,11 @@ def _pip_repository_impl(rctx): # label referencing a package which may not be installed on the current # platform. requirement_cycles = { - normalize_name(name): sorted([normalize_name(d) for d in group if normalize_name(d) in bzl_packages]) + normalize_name(name): sorted([ + group_member(d) + for d in group + if group_member(d).partition("[")[0] in bzl_packages + ]) for name, group in requirement_cycles.items() } diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index 56157ca9a6..1920e2b971 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -83,6 +83,7 @@ def pkg_aliases( name, actual, group_name = None, + group_extras = None, extra_aliases = None, requires_dist = [], extras = [], @@ -100,6 +101,10 @@ def pkg_aliases( the aliases to point to mapping to repositories. The keys are passed to bazel skylib's `selects.with_or`, so they can be tuples as well. group_name: {type}`str` The group name that the pkg belongs to. + group_extras: {type}`list[str] | None` The members of the group in + this package, where an empty string is the package without extras + and the other values are the extras, e.g. `["", "bar"]`. Defaults + to the package without extras if `group_name` is set. group_deps: {type}`list[str]` The packages that are in the given group. Comes as an arg through the hub repository. extra_aliases: {type}`list[str]` The extra aliases to be created. @@ -112,9 +117,10 @@ def pkg_aliases( Comes from `//:config.bzl#packages` **kwargs: extra kwargs to pass to {bzl:obj}`get_config_settings`. """ - - # The package without extras is the member of the group. - group_extras = [""] if group_name else [] + if not group_name: + group_extras = [] + elif group_extras == None: + group_extras = [""] _native = kwargs.pop("native", native) rules = kwargs.pop("rules", struct( diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 9894f555c7..cd6c387fb1 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -22,6 +22,7 @@ load( ":generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel", ) # buildifier: disable=bzl-visibility +load(":labels.bzl", "group_member") NO_MATCH_ERROR_MESSAGE_TEMPLATE = """\ No matching wheel for current configuration's Python version. @@ -115,17 +116,23 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases fail("The aliases need to be provided as a dict, got: {}".format(type(aliases))) whl_group_mapping = {} + group_extras = {} if requirement_cycles: + # The members are the packages, e.g. `foo`, or the extras of the + # packages, e.g. `foo[bar]`. requirement_cycles = { - name: [normalize_name(whl_name) for whl_name in whls] - for name, whls in requirement_cycles.items() + name: [group_member(member) for member in members] + for name, members in requirement_cycles.items() } - whl_group_mapping = { - whl_name: group_name - for group_name, group_whls in requirement_cycles.items() - for whl_name in group_whls - } + for group_name, members in requirement_cycles.items(): + for member in members: + whl_name, _, extra = member.partition("[") + whl_group_mapping[whl_name] = group_name + group_extras.setdefault(whl_name, []).append(extra.rstrip("]")) + + # The package without extras is the default member. + group_extras = {k: v for k, v in group_extras.items() if v != [""]} files = { "{}/BUILD.bazel".format(normalize_name(name)): _render_common_aliases( @@ -134,6 +141,7 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases extra_aliases = extra_hub_aliases.get(normalize_name(name), []), extras = extras.get(normalize_name(name), []), group_name = whl_group_mapping.get(normalize_name(name)), + group_extras = group_extras.get(normalize_name(name)), **kwargs ).strip() for name, pkg_aliases in aliases.items() diff --git a/python/private/pypi/requirements.bzl.tmpl.workspace b/python/private/pypi/requirements.bzl.tmpl.workspace index 65d9938cef..61c191a3e7 100644 --- a/python/private/pypi/requirements.bzl.tmpl.workspace +++ b/python/private/pypi/requirements.bzl.tmpl.workspace @@ -68,8 +68,10 @@ def install_deps(**whl_library_kwargs): # Set up the requirement groups all_requirement_groups = %%ALL_REQUIREMENT_GROUPS%% + # The members are the packages, e.g. `foo`, or the extras of the packages, + # e.g. `foo[bar]`. requirement_group_mapping = { - requirement: group_name + requirement.partition("[")[0]: group_name for group_name, group_requirements in all_requirement_groups.items() for requirement in group_requirements } diff --git a/tests/pypi/generate_group_library_build_bazel/generate_group_library_build_bazel_tests.bzl b/tests/pypi/generate_group_library_build_bazel/generate_group_library_build_bazel_tests.bzl index a91f861a36..2fe22eb487 100644 --- a/tests/pypi/generate_group_library_build_bazel/generate_group_library_build_bazel_tests.bzl +++ b/tests/pypi/generate_group_library_build_bazel/generate_group_library_build_bazel_tests.bzl @@ -95,6 +95,33 @@ py_library( _tests.append(_test_in_hub) +def _test_in_hub_with_extras(env): + actual = generate_group_library_build_bazel( + repo_prefix = "", + groups = {"vbap": ["Pcodedmp", "oletools[Full]"]}, + ) + + # The extras are members only for the py_library targets + env.expect.that_str(actual).contains("""\ +filegroup( + name = "vbap_whl", + srcs = [], + data = ["//pcodedmp:_whl"], + visibility = ["//:__subpackages__"], +)""") + env.expect.that_str(actual).contains("""\ +py_library( + name = "vbap_pkg", + srcs = [], + deps = [ + "//oletools:_[full]", + "//pcodedmp:_pkg", + ], + visibility = ["//:__subpackages__"], +)""") + +_tests.append(_test_in_hub_with_extras) + def generate_group_library_build_bazel_test_suite(name): """Create the test suite. diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index 216dca27f7..88ef33e74a 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -1713,6 +1713,46 @@ Attempting to create a duplicate library pypi_315_foo for foo with different arg _tests.append(_test_err_duplicate_repos) +def _test_groups_with_extras(env): + builder = hub_builder(env) + builder.pip_parse( + _mock_mctx( + os_name = "osx", + arch_name = "aarch64", + mock_files = { + "requirements.txt": """\ +simple==0.0.1 --hash=sha256:deadbeef +other==0.0.1 --hash=sha256:deadbaaf +""", + }, + ), + _parse( + hub_name = "pypi", + python_version = "3.15", + requirements_lock = "requirements.txt", + experimental_requirement_cycles = { + "group": ["Simple[Foo]", "other"], + }, + ), + ) + pypi = builder.build() + + pypi.group_map().contains_exactly({ + "group": ["simple[foo]", "other"], + }) + pypi.whl_libraries().contains_at_least({ + "pypi_315_simple": { + "config_load": "@pypi//:config.bzl", + "dep_template": "@pypi//{name}:{target}", + "group_deps": ["simple[foo]", "other"], + "group_name": "group", + "python_interpreter_target": "unit_test_interpreter_target", + "requirement": "simple==0.0.1 --hash=sha256:deadbeef", + }, + }) + +_tests.append(_test_groups_with_extras) + def hub_builder_test_suite(name): """Create the test suite. diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index 0cbbfb88d9..8cdfc37a7a 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -698,6 +698,35 @@ def _test_extras_aliases_group(env): _tests.append(_test_extras_aliases_group) +def _test_extras_aliases_group_with_extras(env): + got = {} + + pkg_aliases( + name = "foo", + actual = "repo", + extras = ["bar"], + group_name = "group", + # Only the `bar` extra is the member of the group. + group_extras = ["bar"], + native = struct( + alias = lambda *, name, actual, visibility = None, tags = None: got.update({name: actual}), + ), + select = lambda value, no_match_error = None: value, + ) + + # buildifier: disable=unsorted-dict-items + env.expect.that_dict(got).contains_at_least({ + "pkg": "//_groups:group_pkg", + "[]": "@repo//:pkg", + "[bar]": "//_groups:group_pkg", + "_[bar]": "@repo//:[bar]", + "whl": "@repo//:whl", + }) + env.expect.that_collection(got.keys()).not_contains("_pkg") + env.expect.that_collection(got.keys()).not_contains("_whl") + +_tests.append(_test_extras_aliases_group_with_extras) + def _test_extras_aliases_single_repo(env): got = {} py_library_calls = [] diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index 40438134ef..07f00bb155 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -305,6 +305,28 @@ pkg_aliases( _tests.append(_test_aliases_with_extras_for_repo_names) +def _test_groups_with_extras(env): + actual = render_pkg_aliases( + aliases = { + "bar": {whl_config_setting(version = "3.1"): "pypi_31_bar"}, + "foo": {whl_config_setting(version = "3.1"): "pypi_31_foo"}, + }, + requirement_cycles = { + "group": ["Bar", "foo[Extra]"], + }, + ) + + env.expect.that_str(actual["foo/BUILD.bazel"]).contains("""\ + group_name = "group", + group_extras = ["extra"], +""") + env.expect.that_str(actual["bar/BUILD.bazel"]).contains("""\ + group_name = "group", +""") + env.expect.that_bool("group_extras" in actual["bar/BUILD.bazel"]).equals(False) + +_tests.append(_test_groups_with_extras) + def render_pkg_aliases_test_suite(name): """Create the test suite. From cf3b4b6d280c2c06b945d1645d2d4e47d0210a44 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:57:57 +0900 Subject: [PATCH 20/55] refactor(pypi): pass the requirement line only to the repositories using pip --- python/private/internal_dev_deps.bzl | 10 ---------- python/private/pypi/hub_builder.bzl | 5 ++++- python/private/pypi/pip_archive.bzl | 5 +++++ python/private/pypi/whl_deps_repo.bzl | 4 ---- python/private/pypi/whl_file_repo.bzl | 5 ++--- tests/pypi/extension/extension_tests.bzl | 1 - tests/pypi/hub_builder/hub_builder_tests.bzl | 13 ------------- 7 files changed, 11 insertions(+), 32 deletions(-) diff --git a/python/private/internal_dev_deps.bzl b/python/private/internal_dev_deps.bzl index 111b839adf..a708ea9a1b 100644 --- a/python/private/internal_dev_deps.bzl +++ b/python/private/internal_dev_deps.bzl @@ -39,7 +39,6 @@ def _internal_dev_deps_impl(mctx): whl_library( name = "somepkg_with_build_files", whl_file = "@whl_with_build_files//:somepkg-1.0-any-none-any.whl", - requirement = "somepkg", ) # Setup for //tests/implicit_namespace_packages @@ -51,7 +50,6 @@ def _internal_dev_deps_impl(mctx): whl_library( name = "implicit_namespace_ns_sub1", whl_file = "@implicit_namespace_ns_sub1_whl//:ns_sub1-1.0-any-none-any.whl", - requirement = "ns-sub1", enable_implicit_namespace_pkgs = False, ) @@ -63,7 +61,6 @@ def _internal_dev_deps_impl(mctx): whl_library( name = "implicit_namespace_ns_sub2", whl_file = "@implicit_namespace_ns_sub2_whl//:ns_sub2-1.0-any-none-any.whl", - requirement = "ns-sub2", enable_implicit_namespace_pkgs = False, ) @@ -75,7 +72,6 @@ def _internal_dev_deps_impl(mctx): whl_library( name = "pkgutil_nspkg1", whl_file = "@pkgutil_nspkg1_whl//:pkgutil_nspkg1-1.0-any-none-any.whl", - requirement = "pkgutil_nspkg1", enable_implicit_namespace_pkgs = False, ) @@ -87,7 +83,6 @@ def _internal_dev_deps_impl(mctx): whl_library( name = "pkgutil_nspkg2", whl_file = "@pkgutil_nspkg2_whl//:pkgutil_nspkg2-1.0-any-none-any.whl", - requirement = "pkgutil_nspkg2", enable_implicit_namespace_pkgs = False, ) @@ -99,7 +94,6 @@ def _internal_dev_deps_impl(mctx): whl_library( name = "whl_with_data1", whl_file = "@whl_with_data1_whl//:whl_with_data1-1.0-any-none-any.whl", - requirement = "whl-with-data1", ) whl_from_dir_repo( @@ -110,14 +104,12 @@ def _internal_dev_deps_impl(mctx): whl_library( name = "whl_with_data2", whl_file = "@whl_with_data2_whl//:whl_with_data2-1.0-any-none-any.whl", - requirement = "whl-with-data2", ) _whl_library_from_dir( name = "whl_library_extras_direct_dep", root = "//tests/pypi/whl_library/testdata/pkg:BUILD.bazel", output = "pkg-1.0-any-none-any.whl", - requirement = "pkg", # The following is necessary to enable pipstar and make tests faster config_load = "@rules_python//tests/pypi/whl_library/testdata:packages.bzl", dep_template = "@whl_library_extras_{name}//:{target}", @@ -126,7 +118,6 @@ def _internal_dev_deps_impl(mctx): name = "whl_library_extras_optional_dep", root = "//tests/pypi/whl_library/testdata/optional_dep:BUILD.bazel", output = "optional_dep-1.0-any-none-any.whl", - requirement = "optional_dep", # The following is necessary to enable pipstar and make tests faster config_load = "@rules_python//tests/pypi/whl_library/testdata:packages.bzl", ) @@ -140,7 +131,6 @@ def _internal_dev_deps_impl(mctx): whl_library( name = "patch_whl_pkg", whl_file = "@patch_whl_pkg_whl//:pkg-1.0-any-none-any.whl", - requirement = "pkg", whl_patches = { "//tests/pypi/patch_whl/testdata/patches:modify_pkg.patch": json.encode({ "patch_strip": 0, diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 841f012242..443212b1e3 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -775,8 +775,11 @@ def _whl_repo( interpreter, enable_pipstar_extract = False): args = dict(whl_library_args) - args["requirement"] = src.requirement_line is_whl = src.filename.endswith(".whl") + if not (src.url and is_whl): + # The requirement line is only needed for `pip` to download or build + # the wheel. + args["requirement"] = src.requirement_line if src.extra_pip_args and not is_whl: # pip is not used to download wheels and the python diff --git a/python/private/pypi/pip_archive.bzl b/python/private/pypi/pip_archive.bzl index 1764abe70d..9a641bd142 100644 --- a/python/private/pypi/pip_archive.bzl +++ b/python/private/pypi/pip_archive.bzl @@ -358,6 +358,11 @@ pip_archive_attrs = { "quiet", "timeout", ] +} | { + "requirement": attr.string( + mandatory = True, + doc = "Python requirement string describing the package to download or build with `pip`. The extras are not needed because the targets for all of the extras are created.", + ), } pip_archive = repository_rule( diff --git a/python/private/pypi/whl_deps_repo.bzl b/python/private/pypi/whl_deps_repo.bzl index 75e5a3771d..73a0051f11 100644 --- a/python/private/pypi/whl_deps_repo.bzl +++ b/python/private/pypi/whl_deps_repo.bzl @@ -25,10 +25,6 @@ For example if your whl depends on `numpy` and your Python package repo is named "group_name": attr.string( doc = "Name of the group, if any.", ), - "requirement": attr.string( - mandatory = True, - doc = "Python requirement string describing the package to make available, if 'urls' or 'whl_file' is given, then this only needs to include foo[any_extras] as a bare minimum.", - ), } def _whl_deps_repo_impl(rctx): diff --git a/python/private/pypi/whl_file_repo.bzl b/python/private/pypi/whl_file_repo.bzl index 40d79ac10b..0118402d11 100644 --- a/python/private/pypi/whl_file_repo.bzl +++ b/python/private/pypi/whl_file_repo.bzl @@ -36,8 +36,8 @@ def download_whl(rctx, *, logger): auth = get_auth(rctx, urls), ) if not rctx.attr.sha256 and not rctx.attr.integrity: - logger.warn("Please update the requirement line to include the hash:\n{} \\\n --hash=sha256:{}".format( - rctx.attr.requirement, + logger.warn("Please update the requirement line of '{}' to include the hash:\n --hash=sha256:{}".format( + filename, result.sha256, )) if not result.success: @@ -50,7 +50,6 @@ whl_file_repo_attrs = { "index_url": attr.string(doc = "The index URL used to resolve relative wheel URLs."), "integrity": attr.string(doc = "The expected SRI checksum, used when sha256 is unset."), "patches": attr.label_keyed_string_dict(doc = "Patch labels mapped to their strip levels."), - "requirement": attr.string(doc = "The requirement used in download diagnostics."), "sha256": attr.string(doc = "The expected SHA256 checksum."), "urls": attr.string_list(doc = "The wheel download URLs."), "whl_file": attr.label(doc = "A caller-owned wheel to use instead of downloading."), diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index b14e6feffb..02e45ee24a 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -808,7 +808,6 @@ simple==0.0.1 \ "filename": "simple-0.0.1-py3-none-any.whl", "index_url": test.want_index_url, "integrity": "sha256-3q2+7w==", - "requirement": "simple==0.0.1", "urls": ["file://simple-0.0.1-py3-none-any.whl"], }, }) diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index 88ef33e74a..aa441c21e6 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -214,7 +214,6 @@ def _test_uv_lock_extras(env): "filename": "simple-0.0.1-py3-none-any.whl", "index_url": "https://pypi.org/simple/simple", "integrity": "sha256-3q2+7w==", - "requirement": "simple==0.0.1", "urls": ["https://files.pythonhosted.org/packages/simple-0.0.1-py3-none-any.whl"], }, }) @@ -388,7 +387,6 @@ def _test_simple_extras_vs_no_extras_simpleapi(env): "filename": "simple-0.0.1-py3-none-any.whl", "index_url": "https://example.com/simple/", "integrity": "sha256-3q2+7w==", - "requirement": "simple==0.0.1", "urls": ["/simple-0.0.1-py3-none-any.whl"], }, }) @@ -460,7 +458,6 @@ def _test_simple_sha512_simpleapi(env): "filename": "simple-0.0.1-py3-none-any.whl", "index_url": "https://example.com/simple/", "integrity": "sha512-3q2+7w==", - "requirement": "simple==0.0.1", "urls": ["/simple-0.0.1-py3-none-any.whl"], }, }) @@ -851,7 +848,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", "index_url": "https://torch.index/torch/", "integrity": "sha256-iADe7wAmAR1QLAwlbMS2fQAjR/Y8OjjNjkXx9EXGE2Q=", - "requirement": "torch==2.4.1+cpu", "urls": ["/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-linux_x86_64.whl"], }, "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432_linux_aarch64": { @@ -860,7 +856,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", "index_url": "https://torch.index/torch/", "integrity": "sha256-NhCUMrEL1xY8mzDOiW88LMobhrl2X5VqFZTw/0MJHio=", - "requirement": "torch==2.4.1", "urls": ["/whl/cpu/torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"], }, "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c_windows_x86_64": { @@ -869,7 +864,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", "index_url": "https://torch.index/torch/", "integrity": "sha256-OlcOXFU0Fc293+Z5IHMns6OAayHGreoU+6d2hNFhnpc=", - "requirement": "torch==2.4.1+cpu", "urls": ["/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-win_amd64.whl"], }, "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5_osx_aarch64": { @@ -878,7 +872,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", "index_url": "https://torch.index/torch/", "integrity": "sha256-crSE1bbOwac1vz+locSIPQF0hpjF6c/b60/6t8eYfg0=", - "requirement": "torch==2.4.1", "urls": ["/whl/cpu/torch-2.4.1-cp312-none-macosx_11_0_arm64.whl"], }, }) @@ -1021,7 +1014,6 @@ simple==0.0.1 --hash=sha256:deadb00f "filename": "simple-0.0.1-py3-none-any.whl", "index_url": test.expect_index_url, "integrity": "sha256-3q2wDw==", - "requirement": "simple==0.0.1", "urls": [test.expect_url], } if getattr(test, "envsubst", []): @@ -1374,7 +1366,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "dep_template": "@pypi//{name}:{target}", "filename": "direct_without_sha-0.0.1-py3-none-any.whl", "integrity": "", - "requirement": "direct_without_sha==0.0.1", "urls": ["example-direct.org/direct_without_sha-0.0.1-py3-none-any.whl"], "whl_patches": {"my_patch": "1"}, }, @@ -1398,7 +1389,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "filename": "plat-pkg-0.0.4-py3-none-linux_x86_64.whl", "index_url": "https://pypi.org/simple", "integrity": "sha256-3q20Tw==", - "requirement": "plat_pkg==0.0.4", "urls": ["example2.org/index/plat_pkg/"], }, "pypi_315_simple_py3_none_any_deadb00f": { @@ -1407,7 +1397,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "filename": "simple-0.0.1-py3-none-any.whl", "index_url": "https://pypi.org/simple", "integrity": "sha256-3q2wDw==", - "requirement": "simple==0.0.1", "urls": ["example2.org"], }, "pypi_315_some_pkg_py3_none_any_deadbaaf": { @@ -1415,7 +1404,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "dep_template": "@pypi//{name}:{target}", "filename": "some_pkg-0.0.1-py3-none-any.whl", "integrity": "sha256-3q26rw==", - "requirement": "some_pkg==0.0.1", "urls": ["example-direct.org/some_pkg-0.0.1-py3-none-any.whl"], }, "pypi_315_some_py3_none_any_deadb33f": { @@ -1424,7 +1412,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "filename": "some-other-pkg-0.0.1-py3-none-any.whl", "index_url": "https://with_index_url", "integrity": "sha256-3q2zPw==", - "requirement": "some_other_pkg==0.0.1", "urls": ["example2.org/index/some_other_pkg/"], }, }) From be422b2bdd2eb30a92d974c6102b8eb6e2af3bee Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:58:08 +0900 Subject: [PATCH 21/55] test(pypi): check the extras of the dependencies from the METADATA --- MODULE.bazel | 1 + python/private/internal_dev_deps.bzl | 16 ++++++++++++++++ tests/pypi/whl_library/BUILD.bazel | 10 ++++++++++ .../whl_library/testdata/consumer/BUILD.bazel | 0 .../consumer/consumer-1.0.dist-info/METADATA | 3 +++ .../consumer/consumer-1.0.dist-info/RECORD | 0 .../consumer/consumer-1.0.dist-info/WHEEL | 1 + .../whl_library/testdata/consumer/consumer.py | 3 +++ tests/pypi/whl_library/testdata/packages.bzl | 1 + .../whl_library_transitive_extras_test.py | 12 ++++++++++++ 10 files changed, 47 insertions(+) create mode 100644 tests/pypi/whl_library/testdata/consumer/BUILD.bazel create mode 100644 tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA create mode 100644 tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/RECORD create mode 100644 tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/WHEEL create mode 100644 tests/pypi/whl_library/testdata/consumer/consumer.py create mode 100644 tests/pypi/whl_library/whl_library_transitive_extras_test.py diff --git a/MODULE.bazel b/MODULE.bazel index b7301c7291..d15a0ff231 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -335,6 +335,7 @@ use_repo( "pkgutil_nspkg2", "rules_python_runtime_env_tc_info", "somepkg_with_build_files", + "whl_library_extras_consumer", "whl_library_extras_direct_dep", "whl_with_build_files", "whl_with_data1", diff --git a/python/private/internal_dev_deps.bzl b/python/private/internal_dev_deps.bzl index a708ea9a1b..1c5d15e848 100644 --- a/python/private/internal_dev_deps.bzl +++ b/python/private/internal_dev_deps.bzl @@ -114,6 +114,22 @@ def _internal_dev_deps_impl(mctx): config_load = "@rules_python//tests/pypi/whl_library/testdata:packages.bzl", dep_template = "@whl_library_extras_{name}//:{target}", ) + _whl_library_from_dir( + name = "whl_library_extras_pkg", + root = "//tests/pypi/whl_library/testdata/pkg:BUILD.bazel", + output = "pkg-1.0-any-none-any.whl", + # The following is necessary to enable pipstar and make tests faster + config_load = "@rules_python//tests/pypi/whl_library/testdata:packages.bzl", + dep_template = "@whl_library_extras_{name}//:{target}", + ) + _whl_library_from_dir( + name = "whl_library_extras_consumer", + root = "//tests/pypi/whl_library/testdata/consumer:BUILD.bazel", + output = "consumer-1.0-any-none-any.whl", + # The following is necessary to enable pipstar and make tests faster + config_load = "@rules_python//tests/pypi/whl_library/testdata:packages.bzl", + dep_template = "@whl_library_extras_{name}//:{target}", + ) _whl_library_from_dir( name = "whl_library_extras_optional_dep", root = "//tests/pypi/whl_library/testdata/optional_dep:BUILD.bazel", diff --git a/tests/pypi/whl_library/BUILD.bazel b/tests/pypi/whl_library/BUILD.bazel index 427780c9f2..af2c4c24bf 100644 --- a/tests/pypi/whl_library/BUILD.bazel +++ b/tests/pypi/whl_library/BUILD.bazel @@ -9,3 +9,13 @@ py_test( "@whl_library_extras_direct_dep//:[optional]", ], ) + +py_test( + name = "whl_library_transitive_extras_test", + srcs = ["whl_library_transitive_extras_test.py"], + target_compatible_with = SUPPORTS_BZLMOD, + deps = [ + # The METADATA of the consumer requires `pkg[optional]`. + "@whl_library_extras_consumer//:pkg", + ], +) diff --git a/tests/pypi/whl_library/testdata/consumer/BUILD.bazel b/tests/pypi/whl_library/testdata/consumer/BUILD.bazel new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA b/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA new file mode 100644 index 0000000000..9a723c958a --- /dev/null +++ b/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA @@ -0,0 +1,3 @@ +Name: consumer +Version: 1.0 +Requires-Dist: pkg[optional] diff --git a/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/RECORD b/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/RECORD new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/WHEEL b/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/WHEEL new file mode 100644 index 0000000000..a64521a1cc --- /dev/null +++ b/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/WHEEL @@ -0,0 +1 @@ +Wheel-Version: 1.0 diff --git a/tests/pypi/whl_library/testdata/consumer/consumer.py b/tests/pypi/whl_library/testdata/consumer/consumer.py new file mode 100644 index 0000000000..20da50d156 --- /dev/null +++ b/tests/pypi/whl_library/testdata/consumer/consumer.py @@ -0,0 +1,3 @@ +import pkg + +WITH_EXTRAS = pkg.WITH_EXTRAS diff --git a/tests/pypi/whl_library/testdata/packages.bzl b/tests/pypi/whl_library/testdata/packages.bzl index e4a9b0af4c..fd12c500ca 100644 --- a/tests/pypi/whl_library/testdata/packages.bzl +++ b/tests/pypi/whl_library/testdata/packages.bzl @@ -1,6 +1,7 @@ """A list of packages that this logical testdata hub repo contains.""" packages = [ + "consumer", "optional_dep", "pkg", ] diff --git a/tests/pypi/whl_library/whl_library_transitive_extras_test.py b/tests/pypi/whl_library/whl_library_transitive_extras_test.py new file mode 100644 index 0000000000..9aa7a80f04 --- /dev/null +++ b/tests/pypi/whl_library/whl_library_transitive_extras_test.py @@ -0,0 +1,12 @@ +import unittest + + +class TransitiveExtrasTest(unittest.TestCase): + def test_extras_from_metadata_propagated(self): + import consumer + + self.assertEqual(consumer.WITH_EXTRAS, True) + + +if __name__ == "__main__": + unittest.main() From 646f2701e8520b2fe5591445e882e44d43106e75 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:58:14 +0900 Subject: [PATCH 22/55] docs(pypi): document the extra targets --- docs/pypi/circular-dependencies.md | 26 ++++++++++ docs/pypi/use.md | 80 ++++++++++++++++++++++++++++-- news/2786.added.md | 17 +++++++ 3 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 news/2786.added.md diff --git a/docs/pypi/circular-dependencies.md b/docs/pypi/circular-dependencies.md index 62613f489e..2d510316e7 100644 --- a/docs/pypi/circular-dependencies.md +++ b/docs/pypi/circular-dependencies.md @@ -65,6 +65,32 @@ be a part of the `airflow` cycle. For instance: ) ``` +Each extra of a package has its own target, so a cycle that goes through an +extra back to the package without extras, e.g. `foo[bar]` depending on `baz` +which depends on `foo`, is not a cycle in Bazel and does not need a group. If +the cycle includes the extra itself, e.g. `baz` depends on `foo[bar]`, then +only the extra needs to be a member of the group: + +```starlark + ... + experimental_requirement_cycles = { + "foo": [ + "foo[bar]", + "baz", + ] + } +) +``` + +Only the targets of the members, `@pypi//foo:[bar]` and `@pypi//baz`, are +then part of the group and the other targets of `foo`, e.g. `@pypi//foo:[]`, +can be used without the group. The `whl` targets do not use the extras, so +only the packages without extras are part of the group for the `whl` targets. + +:::{versionadded} VERSION_NEXT_FEATURE +Support for the extras as members of the groups. +::: + Alternatively, one could resolve the cycle by removing one leg of it. For example, while `apache-airflow-providers-sqlite` is "baked into" the Airflow diff --git a/docs/pypi/use.md b/docs/pypi/use.md index 9d0c54c4ab..2660749052 100644 --- a/docs/pypi/use.md +++ b/docs/pypi/use.md @@ -57,11 +57,85 @@ which can help you create a `py_binary` target for a particular console script e [whl_ep]: https://packaging.python.org/specifications/entry-points/ +{#extras} ## 'Extras' dependencies -Any "extras" specified in the requirements lock file will be automatically added -as transitive dependencies of the package. In the example above, you'd just put -`requirement("useful_dep")` or `@pypi//useful_dep`. +Any "extras" specified in the requirements lock file or the `uv.lock` file will +be automatically added as transitive dependencies of the package. In the +example above, you'd just put `requirement("useful_dep")` or +`@pypi//useful_dep`. + +If you need only some of the extras, the hub repo also has a target for the +package without any extras and a target for each extra used in the lock file. +For example, if the lock file has `google-api-core[async_rest,grpc]`, then the +hub has: + +``` +@pypi//google_api_core # The package with all of the extras +@pypi//google_api_core:[] # The package without any extras +@pypi//google_api_core:[async_rest] # The package with the async_rest extra +@pypi//google_api_core:[grpc] # The package with the grpc extra +``` + +Extra names are normalized in the same way as the package names, so +`async-rest` becomes `async_rest`. + +The `requirement` helper supports the same syntax: + +```starlark +load("@pypi//:requirements.bzl", "requirement") + +py_library( + name = "my_lib", + deps = [ + requirement("google-api-core[grpc]"), + ], +) +``` + +There are no targets for combinations of extras, because each `[extra]` target +already includes the package. The `requirement` helper accepts at most one +extra. To request more than one extra, use the `requirements` helper, which +accepts any number of packages and returns a list with a label for each +package and each extra: + +```starlark +load("@pypi//:requirements.bzl", "requirements") + +py_library( + name = "my_lib", + deps = [ + ":other_lib", + ] + requirements( + "google-api-core[async_rest,grpc]", + "numpy", + ), +) +``` + +If the lock file requests different extras for a package on different +platforms, for example `foo[bar]` on Linux and `foo[baz]` on macOS, then +`@pypi//foo` includes only the extras requested for the target platform and +`@pypi//foo:[bar]` on macOS is the same as `@pypi//foo:[]`. + +The dependencies of the packages use the extras written in the wheel +`METADATA`, so if `foo` depends on `bar[socks]`, then `@pypi//foo` depends on +`@pypi//bar:[socks]`. If the extra is not in the lock file, then building +`@pypi//foo` fails with a message asking you to add `bar[socks]` to the +requirements or the lock file. For example, when compiling the requirements +with `uv pip compile`, pass `--no-strip-extras` to keep the extras in the +output. + +The `whl` targets, e.g. `@pypi//foo:whl`, do not use the extras and include +only the wheels of the package and its dependencies without extras. + +The packages in a dependency group can also have only some of their extras in +the group, see the [circular dependencies](./circular-dependencies) docs. + +:::{versionadded} VERSION_NEXT_FEATURE +The `[]` and `[extra]` targets, the support for extras in the `requirement` +helper and the `requirements` helper were added. +::: ## Consuming Wheel Dists Directly diff --git a/news/2786.added.md b/news/2786.added.md new file mode 100644 index 0000000000..4c00959a7d --- /dev/null +++ b/news/2786.added.md @@ -0,0 +1,17 @@ +(pypi) The hub repositories now have a `[]` target for each package without +any extras and an `[extra]` target for each extra used in the lock file, e.g. +`@pypi//google_api_core:[grpc]`, in both bzlmod and `WORKSPACE`. The +`requirement` macro accepts a single extra, e.g. +`requirement("google-api-core[grpc]")`, and the new `requirements` macro +returns a list of labels for any number of packages and extras, e.g. +`requirements("google-api-core[async_rest,grpc]", "numpy")`. The whl +repositories now create the targets for every extra that the package provides +and the requested extras are no longer passed to them, so changing the +requested extras does not refetch them. The dependencies use the extras from +the wheel `METADATA` and the build fails with a helpful message if an extra is +missing in the lock file. The `whl` targets no longer include the dependencies +of the extras. The members of `experimental_requirement_cycles` can now be the +extras of the packages, e.g. `foo[bar]`. See +[the docs](https://rules-python.readthedocs.io/en/latest/pypi/use.html#extras) +([#2786](https://github.com/bazel-contrib/rules_python/issues/2786), +[#4168](https://github.com/bazel-contrib/rules_python/issues/4168)). From dafa8381190740bb40f47bd4fa0e24b196842741 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:58:14 +0900 Subject: [PATCH 23/55] chore(examples): update the vendored requirements.bzl --- examples/pip_parse_vendored/requirements.bzl | 28 ++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/examples/pip_parse_vendored/requirements.bzl b/examples/pip_parse_vendored/requirements.bzl index 82af35372d..0a308ac19b 100644 --- a/examples/pip_parse_vendored/requirements.bzl +++ b/examples/pip_parse_vendored/requirements.bzl @@ -5,6 +5,7 @@ load("@rules_python//python:pip.bzl", "pip_utils") load("@rules_python//python/pip_install:pip_repository.bzl", "whl_config_repo", "whl_library") +load("@rules_python//python/private/pypi:labels.bzl", "requirement_label", "requirement_labels") # buildifier: disable=bzl-visibility all_requirements = [ "@my_project_pip_deps_vendored_certifi//:pkg", @@ -79,7 +80,28 @@ _config = { _annotations = {} def requirement(name): - return "@my_project_pip_deps_vendored_{}//:{}".format(pip_utils.normalize_name(name), "pkg") + """Return the label for the package. + + Args: + name: The name of the package with at most one extra, e.g. `foo`, + `foo[]` or `foo[bar]`. Use `requirements` for multiple extras. + + Returns: + The label of the package. + """ + return requirement_label("@my_project_pip_deps_vendored_{}//:{}", name) + +def requirements(*names): + """Return the labels for the packages. + + Args: + *names: The names of the packages with optional extras, e.g. `foo`, + `foo[]`, `foo[bar]` or `foo[bar,baz]`. + + Returns: + The list of labels, with a label for each extra of a package. + """ + return requirement_labels("@my_project_pip_deps_vendored_{}//:{}", *names) def whl_requirement(name): return "@my_project_pip_deps_vendored_{}//:{}".format(pip_utils.normalize_name(name), "whl") @@ -107,8 +129,10 @@ def install_deps(**whl_library_kwargs): # Set up the requirement groups all_requirement_groups = {} + # The members are the packages, e.g. `foo`, or the extras of the packages, + # e.g. `foo[bar]`. requirement_group_mapping = { - requirement: group_name + requirement.partition("[")[0]: group_name for group_name, group_requirements in all_requirement_groups.items() for requirement in group_requirements } From 6431f8bfe5ac402bd81943b0f9a0439bb8c0f75c Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:01:38 +0900 Subject: [PATCH 24/55] fix(pypi): parse the whl METADATA with CRLF line endings --- python/private/pypi/whl_metadata.bzl | 6 +++--- .../pypi/whl_metadata/whl_metadata_tests.bzl | 21 +++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/python/private/pypi/whl_metadata.bzl b/python/private/pypi/whl_metadata.bzl index 2981a5d92f..203caafc58 100644 --- a/python/private/pypi/whl_metadata.bzl +++ b/python/private/pypi/whl_metadata.bzl @@ -56,7 +56,7 @@ def parse_whl_metadata(contents): "requires_dist": [], "version": "", } - for line in contents.strip().split("\n"): + for line in contents.strip().splitlines(): if not line: # Stop parsing on first empty line, which marks the end of the # headers containing the metadata. @@ -70,10 +70,10 @@ def parse_whl_metadata(contents): parsed["version"] = value.strip() elif line.startswith(_REQUIRES_DIST): _, _, value = line.partition(_REQUIRES_DIST) - parsed["requires_dist"].append(value.strip(" ")) + parsed["requires_dist"].append(value.strip()) elif line.startswith(_PROVIDES_EXTRA): _, _, value = line.partition(_PROVIDES_EXTRA) - parsed["provides_extra"].append(value.strip(" ")) + parsed["provides_extra"].append(value.strip()) return struct( name = parsed["name"], diff --git a/tests/pypi/whl_metadata/whl_metadata_tests.bzl b/tests/pypi/whl_metadata/whl_metadata_tests.bzl index 8131b0f452..8aca204146 100644 --- a/tests/pypi/whl_metadata/whl_metadata_tests.bzl +++ b/tests/pypi/whl_metadata/whl_metadata_tests.bzl @@ -246,6 +246,27 @@ FOOBAR = guimod:main _tests.append(_test_parse_entry_points_deduplicate) +def _test_parse_metadata_crlf(env): + got = _parse_whl_metadata( + env, + contents = "\r\n".join([ + "Name: foo", + "Version: 0.0.1", + "Requires-Dist: bar; extra == \"all\"", + "Provides-Extra: all", + "", + "Requires-Dist: this will be ignored", + ]), + ) + got.name().equals("foo") + got.version().equals("0.0.1") + got.requires_dist().contains_exactly([ + "bar; extra == \"all\"", + ]) + got.provides_extra().contains_exactly(["all"]) + +_tests.append(_test_parse_metadata_crlf) + def whl_metadata_test_suite(name): # buildifier: disable=function-docstring test_suite( name = name, From c2e07df71d78c17df418fdc577b51329b4f5ad1f Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:49:34 +0900 Subject: [PATCH 25/55] test(pypi): drop the requirement from the whl repositories in the integration test --- tests/integration/whl_library/MODULE.bazel | 2 -- tests/integration/whl_library/wheels.bzl | 3 --- 2 files changed, 5 deletions(-) diff --git a/tests/integration/whl_library/MODULE.bazel b/tests/integration/whl_library/MODULE.bazel index 9059879187..1cafbb2024 100644 --- a/tests/integration/whl_library/MODULE.bazel +++ b/tests/integration/whl_library/MODULE.bazel @@ -44,7 +44,6 @@ whl_archive( dep_template = DEP_TEMPLATE, filename = "requests-2.34.2-py3-none-any.whl", # https://pypi.org/project/requests/#requests-2.34.2-py3-none-any.whl - requirement = "requests", sha256 = "2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", urls = [ "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", @@ -57,7 +56,6 @@ whl_deps_repo( name = "whl_deps_repo", dep_template = DEP_TEMPLATE, metadata_file = "@whl_archive//:metadata.json", - requirement = "requests", ) wheels = use_extension("//:wheels.bzl", "wheels") diff --git a/tests/integration/whl_library/wheels.bzl b/tests/integration/whl_library/wheels.bzl index b82f0c0479..c690c7f027 100644 --- a/tests/integration/whl_library/wheels.bzl +++ b/tests/integration/whl_library/wheels.bzl @@ -16,7 +16,6 @@ def _wheels_impl(mrctx): common = { "envsubst": ["WHEEL_TEST_ADDRESS"], "filename": _FILENAME, - "requirement": "storage-pkg", "sha256": mrctx.read(Label("//:wheel.sha256")).strip(), "urls": ["http://${WHEEL_TEST_ADDRESS}/" + _FILENAME], } @@ -25,12 +24,10 @@ def _wheels_impl(mrctx): whl_library(name = "nonmatching", whl_patches = _patches("other.whl"), **common) whl_library( name = "provided", - requirement = "storage-pkg", whl_file = Label("//:" + _FILENAME), ) whl_library( name = "provided_patched", - requirement = "storage-pkg", whl_file = Label("//:" + _FILENAME), whl_patches = _patches(_FILENAME), ) From 9f9fc9f707128d0ed835758ba2da520fc14287af Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:28:54 +0900 Subject: [PATCH 26/55] refactor(pypi): keep the original hub_builder group map and repo mapping code --- python/private/pypi/hub_builder.bzl | 47 ++++++++++++---------- python/private/pypi/render_pkg_aliases.bzl | 4 +- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 443212b1e3..6ccda0a184 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -262,17 +262,17 @@ def _add_group_map(self, group_map): # The members are the packages, e.g. `foo`, or the extras of the packages, # e.g. `foo[bar]`. group_map = { - name: [group_member(member) for member in members] - for name, members in group_map.items() + name: [group_member(whl_name) for whl_name in whls] + for name, whls in group_map.items() } self._group_map.clear() self._group_name_by_whl.clear() self._group_map.update(group_map) self._group_name_by_whl.update({ - member.partition("[")[0]: group_name - for group_name, members in self._group_map.items() - for member in members + whl_name.partition("[")[0]: group_name + for group_name, group_whls in self._group_map.items() + for whl_name in group_whls }) def _add_extra_aliases(self, extra_hub_aliases): @@ -366,16 +366,16 @@ def _add_whl_library(self, *, python_version, whl, repo): self._whl_libraries[repo_name] = repo.args mapping = self._whl_map.setdefault(whl.name, {}) - for config_setting in repo.config_settings: - if config_setting in mapping and mapping[config_setting] != repo_name: - fail( - "attempting to override an existing repo '{}' for config setting '{}' with a new repo '{}'".format( - mapping[config_setting], - config_setting, - repo_name, - ), - ) - mapping[config_setting] = repo_name + if repo.config_setting in mapping and mapping[repo.config_setting] != repo_name: + fail( + "attempting to override an existing repo '{}' for config setting '{}' with a new repo '{}'".format( + mapping[repo.config_setting], + repo.config_setting, + repo_name, + ), + ) + else: + mapping[repo.config_setting] = repo_name ### end of setters, below we have various functions to implement the public methods @@ -629,12 +629,17 @@ def _create_whl_repos( interpreter = interpreter, enable_pipstar_extract = enable_pipstar_extract, ) - _add_whl_library( - self, - python_version = python_version, - whl = whl, - repo = repo, - ) + for config_setting in repo.config_settings if repo else []: + _add_whl_library( + self, + python_version = python_version, + whl = whl, + repo = struct( + repo_name = repo.repo_name, + args = repo.args, + config_setting = config_setting, + ), + ) def _merge_extras(srcs): """Merge the sources that differ only in the requested extras. diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index cd6c387fb1..1753f557ea 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -121,8 +121,8 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases # The members are the packages, e.g. `foo`, or the extras of the # packages, e.g. `foo[bar]`. requirement_cycles = { - name: [group_member(member) for member in members] - for name, members in requirement_cycles.items() + name: [group_member(whl_name) for whl_name in whls] + for name, whls in requirement_cycles.items() } for group_name, members in requirement_cycles.items(): From a24fac65e723af55a9e81b6e35a4f2b7f136f8d9 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:29:02 +0900 Subject: [PATCH 27/55] refactor(pypi): minimize the changes to the group library generation --- .../generate_group_library_build_bazel.bzl | 54 +++++++++++-------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/python/private/pypi/generate_group_library_build_bazel.bzl b/python/private/pypi/generate_group_library_build_bazel.bzl index 114f96d41e..92cf8f9870 100644 --- a/python/private/pypi/generate_group_library_build_bazel.bzl +++ b/python/private/pypi/generate_group_library_build_bazel.bzl @@ -65,33 +65,36 @@ def _generate_group_libraries(repo_prefix, group_name, group_members): which make up the group, optionally with a single extra, e.g. `foo[bar]`. """ - # The members are the packages, e.g. `foo`, or the extras of the packages, - # e.g. `foo[bar]`. The whl targets do not use the extras, so only the - # packages are members for the whl targets. - members = sorted([group_member(d) for d in group_members]) - lib_targets = [] - whl_targets = [] - for member in members: - name, _, extra = member.partition("[") - if extra: - lib_targets.append((name, "_" + extra_label(extra.rstrip("]")))) - else: - lib_targets.append((name, PY_LIBRARY_IMPL_LABEL)) - whl_targets.append((name, WHEEL_FILE_IMPL_LABEL)) + # The whl targets do not use the extras, so only the packages without + # extras are members for the whl targets. + group_members = sorted([group_member(d) for d in group_members]) + whl_members = [d for d in group_members if "[" not in d] + packages = sorted({_package(d): None for d in group_members}) if repo_prefix: - label_tmpl = "@" + repo_prefix + "{}//:{}" - visibility = sorted({ - "@%s%s//:__pkg__" % (repo_prefix, name): None - for name, _ in lib_targets - }) + lib_dependencies = [ + "@%s%s//:%s" % (repo_prefix, _package(d), _lib_impl_label(d)) + for d in group_members + ] + whl_file_deps = [ + "@%s%s//:%s" % (repo_prefix, normalize_name(d), WHEEL_FILE_IMPL_LABEL) + for d in whl_members + ] + visibility = [ + "@%s%s//:__pkg__" % (repo_prefix, normalize_name(d)) + for d in packages + ] else: - label_tmpl = "//{}:{}" + lib_dependencies = [ + "//%s:%s" % (_package(d), _lib_impl_label(d)) + for d in group_members + ] + whl_file_deps = [ + "//%s:%s" % (normalize_name(d), WHEEL_FILE_IMPL_LABEL) + for d in whl_members + ] visibility = ["//:__subpackages__"] - lib_dependencies = [label_tmpl.format(name, target) for name, target in lib_targets] - whl_file_deps = [label_tmpl.format(name, target) for name, target in whl_targets] - return _GROUP_TEMPLATE.format( name = normalize_name(group_name), whl_public_label = WHEEL_FILE_PUBLIC_LABEL, @@ -101,6 +104,13 @@ def _generate_group_libraries(repo_prefix, group_name, group_members): visibility = render.indent(render.list(visibility)).lstrip(), ) +def _package(member): + return member.partition("[")[0] + +def _lib_impl_label(member): + _, _, extra = member.partition("[") + return "_" + extra_label(extra.rstrip("]")) if extra else PY_LIBRARY_IMPL_LABEL + def generate_group_library_build_bazel( repo_prefix, groups): From 8f57a4feabb04b6b9345a88f9d1ce01bdcc10656 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:29:04 +0900 Subject: [PATCH 28/55] test(pypi): remove the overlapping extras tests --- tests/pypi/pep508/deps_tests.bzl | 17 --- tests/pypi/pkg_aliases/pkg_aliases_test.bzl | 51 +------ .../whl_library_deps_targets_tests.bzl | 127 ++---------------- 3 files changed, 14 insertions(+), 181 deletions(-) diff --git a/tests/pypi/pep508/deps_tests.bzl b/tests/pypi/pep508/deps_tests.bzl index 6e291e5a6c..b7a58a45a0 100644 --- a/tests/pypi/pep508/deps_tests.bzl +++ b/tests/pypi/pep508/deps_tests.bzl @@ -357,23 +357,6 @@ def test_deps_with_extras_excludes_and_includes(env): _tests.append(test_deps_with_extras_excludes_and_includes) -def test_deps_without_extras(env): - got = deps( - "foo", - requires_dist = [ - "bar[socks]", - "baz[a]; sys_platform == 'linux'", - "baz[b]; sys_platform == 'darwin'", - ], - ) - - env.expect.that_collection(got.deps).contains_exactly(["bar"]) - env.expect.that_dict(got.deps_select).contains_exactly({ - "baz": "(sys_platform == \"darwin\") or (sys_platform == \"linux\")", - }) - -_tests.append(test_deps_without_extras) - def deps_test_suite(name): # buildifier: disable=function-docstring test_suite( name = name, diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index 8cdfc37a7a..8cd4773612 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -631,17 +631,6 @@ def _test_extras_aliases_union(env): }]) env.expect.that_collection(got.keys()).not_contains("pkg") - # The whl target does not use the extras. - # buildifier: disable=unsorted-dict-items - env.expect.that_dict(got).contains_at_least({ - "whl": { - _LINUX: "@repo//:whl", - _OSX: "@repo//:whl", - _WINDOWS: "@repo//:whl", - "//conditions:default": "_no_matching_repository", - }, - }) - _tests.append(_test_extras_aliases_union) def _test_extras_aliases_group(env): @@ -684,16 +673,9 @@ def _test_extras_aliases_group(env): _OSX: "@repo//:pkg", "//conditions:default": "_no_matching_repository", }, - "whl": "//_groups:group_whl", - "_whl": { - _LINUX: "@repo//:whl", - _OSX: "@repo//:whl", - "//conditions:default": "_no_matching_repository", - }, }) env.expect.that_dict(got_visibility).contains_at_least({ "_pkg": ["//_groups:__subpackages__"], - "_whl": ["//_groups:__subpackages__"], }) _tests.append(_test_extras_aliases_group) @@ -701,6 +683,7 @@ _tests.append(_test_extras_aliases_group) def _test_extras_aliases_group_with_extras(env): got = {} + # This is how `WORKSPACE` uses a single repo for each package. pkg_aliases( name = "foo", actual = "repo", @@ -727,38 +710,6 @@ def _test_extras_aliases_group_with_extras(env): _tests.append(_test_extras_aliases_group_with_extras) -def _test_extras_aliases_single_repo(env): - got = {} - py_library_calls = [] - - # This is how `WORKSPACE` uses a single repo for each package. - pkg_aliases( - name = "foo", - actual = "repo", - extras = ["b", "a"], - native = struct( - alias = lambda *, name, actual, visibility = None, tags = None: got.update({name: actual}), - ), - rules = struct( - py_library = lambda **kwargs: py_library_calls.append(kwargs), - ), - select = lambda value, no_match_error = None: value, - ) - - # buildifier: disable=unsorted-dict-items - env.expect.that_dict(got).contains_at_least({ - "[]": "@repo//:pkg", - "[a]": "@repo//:[a]", - "[b]": "@repo//:[b]", - "whl": "@repo//:whl", - }) - env.expect.that_collection(py_library_calls).contains_exactly([{ - "deps": [":[a]", ":[b]"], - "name": "pkg", - }]) - -_tests.append(_test_extras_aliases_single_repo) - def pkg_aliases_test_suite(name): """Create the test suite. diff --git a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl index 420efbdbcd..1c520c148a 100644 --- a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl +++ b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl @@ -101,6 +101,7 @@ def _test_whl_library_deps_targets_no_deps(env): metadata_name = "Foo", dep_template = "@pypi//{name}:{target}", requires_dist = [], + provides_extra = ["A"], group_name = "qux", repo = None, aliases = {}, @@ -129,6 +130,11 @@ def _test_whl_library_deps_targets_no_deps(env): "actual": "whl_file", "visibility": ["//visibility:public"], }, + { + "name": "[a]", + "actual": "srcs", + "visibility": ["//visibility:public"], + }, ]) # buildifier: @unsorted-dict-items env.expect.that_collection(filegroup_calls).contains_exactly([]) @@ -147,7 +153,7 @@ def _test_extras(env): metadata_name = "Foo", dep_template = "@pypi//{name}:{target}", requires_dist = [ - "bar", + "bar[socks]", "grpc; extra == 'grpc'", "aiohttp; extra == 'async' and python_version < '3.12'", "uvloop; extra == 'async'", @@ -169,18 +175,19 @@ def _test_extras(env): # The `pkg` target has no extras because none were requested and there is # a target for each extra that the package provides. The `whl` target does - # not use the extras. + # not use the extras. Without the extras available in the hub, the + # dependencies use the targets with the extras directly. env.expect.that_collection([ (c["name"], c["deps"]) for c in py_library_calls ]).contains_exactly([ - ("pkg", ["srcs", "@pypi//bar:pkg"]), - ("[all]", ["srcs", "@pypi//bar:pkg", "@pypi//grpc:pkg"]), - ("[async]", ["srcs", "@pypi//bar:pkg", "@pypi//uvloop:pkg"] + select({ + ("pkg", ["srcs", "@pypi//bar:[socks]"]), + ("[all]", ["srcs", "@pypi//bar:[socks]", "@pypi//grpc:pkg"]), + ("[async]", ["srcs", "@pypi//bar:[socks]", "@pypi//uvloop:pkg"] + select({ ":is_include_aiohttp_true": ["@pypi//aiohttp:pkg"], "//conditions:default": [], })), - ("[grpc]", ["srcs", "@pypi//bar:pkg", "@pypi//grpc:pkg"]), + ("[grpc]", ["srcs", "@pypi//bar:[socks]", "@pypi//grpc:pkg"]), ]) env.expect.that_collection([ (c["name"], c["data"]) @@ -198,82 +205,6 @@ def _test_extras(env): _tests.append(_test_extras) -def _test_extras_with_different_markers(env): - py_library_calls = [] - env_marker_setting_calls = [] - - whl_library_deps_targets( - name = "foo-0-py3-none-any.whl", - metadata_name = "foo", - dep_template = "@pypi//{name}:{target}", - requires_dist = [ - "bar; sys_platform == 'win32'", - "bar; extra == 'a' and sys_platform == 'linux'", - ], - provides_extra = ["a"], - repo = None, - aliases = {}, - native = struct( - filegroup = lambda **kwargs: None, - alias = lambda **kwargs: None, - ), - rules = struct( - py_library = lambda **kwargs: py_library_calls.append(kwargs), - env_marker_setting = lambda **kwargs: env_marker_setting_calls.append(kwargs), - ), - ) - - env.expect.that_collection([ - (c["name"], c["deps"]) - for c in py_library_calls - ]).contains_exactly([ - ("pkg", ["srcs"] + select({ - ":is_include_bar_true": ["@pypi//bar:pkg"], - "//conditions:default": [], - })), - ("[a]", ["srcs"] + select({ - ":is_include_bar_2_true": ["@pypi//bar:pkg"], - "//conditions:default": [], - })), - ]) - env.expect.that_collection([ - (c["name"], c["expression"]) - for c in env_marker_setting_calls - ]).contains_exactly([ - ("include_bar", "sys_platform == \"win32\""), - ("include_bar_2", "(sys_platform == \"linux\") or (sys_platform == \"win32\")"), - ]) - -_tests.append(_test_extras_with_different_markers) - -def _test_extras_no_deps(env): - alias_calls = [] - - whl_library_deps_targets( - name = "foo-0-py3-none-any.whl", - metadata_name = "foo", - dep_template = "@pypi//{name}:{target}", - requires_dist = [], - provides_extra = ["A"], - repo = None, - aliases = {}, - native = struct( - alias = lambda **kwargs: alias_calls.append(kwargs), - ), - rules = struct(), - ) - - env.expect.that_collection([ - (c["name"], c["actual"]) - for c in alias_calls - ]).contains_exactly([ - ("pkg", "srcs"), - ("whl", "whl_file"), - ("[a]", "srcs"), - ]) - -_tests.append(_test_extras_no_deps) - def _test_extras_in_group(env): py_library_calls = [] filegroup_calls = [] @@ -453,38 +384,6 @@ def _test_deps_with_extras(env): _tests.append(_test_deps_with_extras) -def _test_deps_with_extras_without_hub_info(env): - py_library_calls = [] - - # When the dependencies refer to the whl repositories directly, then there - # is no information about the available extras and the targets with the - # extras in the dependency repositories are used. - whl_library_deps_targets( - name = "foo-0-py3-none-any.whl", - metadata_name = "foo", - dep_template = "@pypi_{name}//:{target}", - requires_dist = ["bar[socks]"], - repo = None, - aliases = {}, - native = struct( - filegroup = lambda **kwargs: None, - alias = lambda **kwargs: None, - ), - rules = struct( - py_library = lambda **kwargs: py_library_calls.append(kwargs), - env_marker_setting = lambda **kwargs: None, - ), - ) - - env.expect.that_collection([ - (c["name"], c["deps"]) - for c in py_library_calls - ]).contains_exactly([ - ("pkg", ["srcs", "@pypi_bar//:[socks]"]), - ]) - -_tests.append(_test_deps_with_extras_without_hub_info) - def whl_library_deps_targets_test_suite(name): """create the test suite. From df890f2fbcc4412958a04a9d638cc54f6654832c Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Wed, 18 Nov 2026 01:15:54 +0900 Subject: [PATCH 29/55] feat(pypi): collect the extras of the selected dependency groups select_dependency_groups now also returns the extras that the selected requirements request, e.g. requests[socks], so that pip.configure can expose them through the hub. --- python/private/pypi/dependency_groups.bzl | 32 ++++++++++++++---- .../dependency_groups_tests.bzl | 33 +++++++++++++++++++ 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl index 4e40c7e856..60c48eac83 100644 --- a/python/private/pypi/dependency_groups.bzl +++ b/python/private/pypi/dependency_groups.bzl @@ -30,6 +30,10 @@ def select_dependency_groups(pyproject, specs, _fail = fail): * `all`: {type}`bool` whether all of the packages including the transitive dependencies should be selected, i.e. `"*"` and `"transitive"` were given. + * `extras`: {type}`dict[str, list[str]]` the normalized package + names mapped to the sorted normalized extras that the selected + requirements request, e.g. `requests[socks]`. When `all` is set, + the extras of all of the groups are returned. """ groups = {"default": pyproject.dependencies} for name, reqs in pyproject.optional_dependencies.items(): @@ -37,10 +41,12 @@ def select_dependency_groups(pyproject, specs, _fail = fail): for name, reqs in _expand_dependency_groups(pyproject.dependency_groups, _fail).items(): groups["group:" + name] = reqs - if "transitive" in specs: + select_all = "transitive" in specs + if select_all: if "*" not in specs: _fail("The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: {}".format(specs)) - return struct(packages = [], all = True) + return struct(packages = [], all = True, extras = {}) + specs = ["*"] selected = {} for spec in specs: @@ -56,18 +62,30 @@ def select_dependency_groups(pyproject, specs, _fail = fail): # A self reference like `my-project[test]` selects the optional # dependencies, but only one level deep. project = normalize_name(pyproject.name or "") - packages = {} + reqs = [] for key in selected: for req in groups[key]: req = requirement(req) if req.name_ != project: - packages[req.name_] = None + reqs.append(req) continue for extra in req.extras: - for r in groups.get("optional:" + extra, []): - packages[requirement(r).name_] = None + reqs.extend([requirement(r) for r in groups.get("optional:" + extra, [])]) + + packages = {} + extras = {} + for req in reqs: + packages[req.name_] = None + if req.extras: + pkg_extras = extras.setdefault(req.name_, {}) + for extra in req.extras: + pkg_extras[extra] = None - return struct(packages = sorted(packages), all = False) + return struct( + packages = [] if select_all else sorted(packages), + all = select_all, + extras = {name: sorted(pkg_extras) for name, pkg_extras in extras.items()}, + ) def _match(spec, groups): if spec == "*": diff --git a/tests/pypi/dependency_groups/dependency_groups_tests.bzl b/tests/pypi/dependency_groups/dependency_groups_tests.bzl index ee23aec1fb..60c5a4efbb 100644 --- a/tests/pypi/dependency_groups/dependency_groups_tests.bzl +++ b/tests/pypi/dependency_groups/dependency_groups_tests.bzl @@ -52,6 +52,39 @@ def _test_select_all(env): _tests.append(_test_select_all) +def _test_select_extras(env): + pyproject = struct( + name = "my-project", + dependencies = ["main-pkg[Foo_Bar, baz]>=1.0", "plain-pkg"], + optional_dependencies = { + "all": ["my-project[docs]"], + "docs": ["docs-pkg[extra]", "main-pkg[docs]"], + }, + dependency_groups = { + "dev": ["dev-pkg[cli]", {"include-group": "test"}], + "test": ["test-pkg[cov]; sys_platform == 'linux'"], + }, + ) + for specs, want in [ + (["default"], {"main_pkg": ["baz", "foo_bar"]}), + (["dev"], {"dev_pkg": ["cli"], "test_pkg": ["cov"]}), + # The extras of the self reference are the optional dependencies of + # the project, not the extras of a package. + (["all"], {"docs_pkg": ["extra"], "main_pkg": ["docs"]}), + (["default", "docs"], {"docs_pkg": ["extra"], "main_pkg": ["baz", "docs", "foo_bar"]}), + (["*", "transitive"], { + "dev_pkg": ["cli"], + "docs_pkg": ["extra"], + "main_pkg": ["baz", "docs", "foo_bar"], + "test_pkg": ["cov"], + }), + ]: + got, failures = _select(specs, pyproject = pyproject) + env.expect.where(specs = specs).that_collection(failures).contains_exactly([]) + env.expect.where(specs = specs).that_dict(got.extras).contains_exactly(want) + +_tests.append(_test_select_extras) + def _test_select_errors(env): for specs, want in [ (["missing"], "Unknown dependency group 'missing'. Available groups: default, group:dev, group:test, optional:all, optional:docs, optional:grpc"), From c7b28a6acd19fe80c72e9a1e2f4816d56e8f3507 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Wed, 18 Nov 2026 01:16:14 +0900 Subject: [PATCH 30/55] feat(pypi): expose the pyproject.toml extras with pip.configure The extras requested in the selected pyproject.toml dependency groups are added to the hub, so @pypi//requests:[socks] exists and @pypi//requests includes the extra even if the lock file does not request it. hub_exposure now returns the extras together with the exposed packages and add_extras adds them to the config settings of the hub whl_map. --- python/private/pypi/BUILD.bazel | 1 + python/private/pypi/extension.bzl | 19 +++- python/private/pypi/hub_exposure.bzl | 74 ++++++++++++-- tests/pypi/extension/extension_tests.bzl | 42 ++++++++ .../pypi/hub_exposure/hub_exposure_tests.bzl | 98 +++++++++++++++++-- 5 files changed, 212 insertions(+), 22 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 05fc644c30..62f8c99125 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -650,6 +650,7 @@ bzl_library( srcs = ["hub_exposure.bzl"], deps = [ ":dependency_groups", + ":whl_config_setting", "//python/private:pyproject_utils", ], ) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index dcc7aa1a94..8ddc3c5847 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -24,7 +24,7 @@ load("//python/private:pyproject_utils.bzl", "read_pyproject", "version_from_req load("//python/private:repo_utils.bzl", "repo_utils") load(":default_platforms.bzl", "default_platforms") load(":hub_builder.bzl", "hub_builder") -load(":hub_exposure.bzl", "hub_exposure") +load(":hub_exposure.bzl", "add_extras", "hub_exposure") load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") load(":parse_whl_name.bzl", "parse_whl_name") load(":pep508_env.bzl", "env") @@ -374,16 +374,22 @@ You cannot use both the additive_build_content and additive_build_content_file a packages = out.whl_map.keys(), ) - configured_hubs = hub_exposure( + exposure = hub_exposure( module_ctx, hubs = hubs, default_pyproject = config.pyproject_toml, _fail = _fail, ) - if configured_hubs == None: + if exposure == None: return None + configured_hubs = exposure.packages exposed_packages.update(configured_hubs) + # The extras requested in the `pyproject.toml`, e.g. `requests[socks]`, + # are exposed by the hub even if the lock file does not have them. + for hub_name, extras in exposure.extras.items(): + hub_whl_map[hub_name] = add_extras(hub_whl_map[hub_name], extras) + return struct( configured_hubs = configured_hubs, config = config, @@ -1132,6 +1138,10 @@ Each item is one of: packages of the hub including the transitive dependencies, i.e. all of the packages from the requirements files or the `uv.lock` file. +The extras of the selected requirements, e.g. `requests[socks]`, are also +exposed by the hub, see [Configuring the Exposed Packages](#pip-configure). +With `"transitive"` the extras of all of the groups are exposed. + See [Configuring the Exposed Packages](#pip-configure) for examples. :::{versionadded} VERSION_NEXT_FEATURE @@ -1166,7 +1176,8 @@ The packages are selected from a `pyproject.toml` file using the `[project] dependencies`, `[project.optional-dependencies]` and PEP 735 `[dependency-groups]` tables. The other packages of the hub, e.g. the transitive dependencies, are still used by the packages that need them, but -they are not public. +they are not public. The extras requested by the selected requirements, e.g. +`requests[socks]`, are exposed even if the lock file does not request them. Only the root module and the module that created the hub can configure it. If both do, the root module wins. Multiple tags for the same hub are combined. diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl index fe9e586153..e2e441ccde 100644 --- a/python/private/pypi/hub_exposure.bzl +++ b/python/private/pypi/hub_exposure.bzl @@ -1,7 +1,8 @@ -"""Compute the packages exposed by the hubs configured with `pip.configure`.""" +"""Compute the packages and extras exposed by the hubs configured with `pip.configure`.""" load("//python/private:pyproject_utils.bzl", "read_pyproject") load(":dependency_groups.bzl", "select_dependency_groups") +load(":whl_config_setting.bzl", "whl_config_setting") def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): """Compute the packages exposed by the hubs configured with `pip.configure`. @@ -18,10 +19,15 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): _fail: {type}`callable` the failure function, mainly for testing. Returns: - {type}`dict[str, list[str]] | None` the configured hubs mapped to the - sorted packages that they expose. The hubs that are not configured or - that expose all of their packages are not included, so that they - behave the same. Returns `None` on failure. + {type}`struct | None` with the attributes below or `None` on failure: + * `packages`: {type}`dict[str, list[str]]` the configured hubs + mapped to the sorted packages that they expose. The hubs that are + not configured or that expose all of their packages are not + included, so that they behave the same. + * `extras`: {type}`dict[str, dict[str, list[str]]]` the configured + hubs mapped to the packages and the sorted extras that the + selected dependency groups request for them, e.g. + `{"hub": {"requests": ["socks"]}}`. """ # The root module wins over the owner of the hub when both configure it. @@ -34,6 +40,7 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): configured = {} exposes_all = {} + extras = {} for mod in module_ctx.modules: for tag in mod.tags.configure: hub = tag.hub_name @@ -75,8 +82,59 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): if name in info.packages: exposed[name] = None + hub_extras = extras.setdefault(hub, {}) + for name, pkg_extras in selection.extras.items(): + if name in info.packages: + for extra in pkg_extras: + hub_extras.setdefault(name, {})[extra] = None + + return struct( + packages = { + hub: sorted(packages) + for hub, packages in configured.items() + if hub not in exposes_all + }, + extras = { + hub: {name: sorted(pkg_extras) for name, pkg_extras in hub_extras.items()} + for hub, hub_extras in extras.items() + if hub_extras + }, + ) + +def add_extras(whl_map, extras): + """Add the extras requested by `pip.configure` to the hub `whl_map`. + + The extras are added to every config setting of the package, so the hub + exposes the `[]` targets and the package target includes them on + all of the platforms. + + Args: + whl_map: {type}`dict[str, dict[str, list[struct]]]` the packages + mapped to the repo names and their {obj}`whl_config_setting` + values. + extras: {type}`dict[str, list[str]]` the packages mapped to the + normalized extras to add. + + Returns: + {type}`dict[str, dict[str, list[struct]]]` the new `whl_map`. + """ + if not extras: + return whl_map + return { - hub: sorted(packages) - for hub, packages in configured.items() - if hub not in exposes_all + name: { + repo: [ + whl_config_setting( + version = setting.version, + target_platforms = setting.target_platforms, + extras = { + extra: None + for extra in list(setting.extras or []) + extras[name] + }.keys(), + ) + for setting in settings + ] + for repo, settings in repos.items() + } if name in extras else repos + for name, repos in whl_map.items() } diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index b888ea847e..ab547f7fa2 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -900,6 +900,48 @@ def _test_configure_default_pyproject(env): _tests.append(_test_configure_default_pyproject) +def _test_configure_extras(env): + # The extras requested in the pyproject.toml are added to the hub even if + # the lock file does not request them. + pypi = _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + _mod( + name = "rules_python", + parse = [_configure_parse(pyproject_toml = "pyproject.toml")], + configure = [_configure(hub_name = "pypi_a", dependency_groups = ["default"])], + ), + os_name = "linux", + arch_name = "x86_64", + mock_files = { + "pyproject.toml": """\ +[project] +dependencies = ["simple[foo]", "other-pkg"] +""", + "requirements.txt": _CONFIGURE_REQUIREMENTS, + }, + ), + available_interpreters = { + "python_3_15_host": "unit_test_interpreter_target", + }, + minor_mapping = {"3.15": "3.15.19"}, + ) + + pypi.configured_hubs().contains_exactly({"pypi_a": ["other_pkg", "simple"]}) + pypi.hub_whl_map().contains_exactly({"pypi_a": { + "other_pkg": { + "pypi_a_315_other_pkg": [whl_config_setting(version = "3.15")], + }, + "simple": { + "pypi_a_315_simple": [whl_config_setting(version = "3.15", extras = ["foo"])], + }, + "transitive_pkg": { + "pypi_a_315_transitive_pkg": [whl_config_setting(version = "3.15")], + }, + }}) + +_tests.append(_test_configure_extras) + def _test_not_configured_exposes_all(env): # The pyproject.toml does not filter the hub without `pip.configure`. pypi = _configure_parse_modules( diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl index 5d84a351b5..eeb57ccdbb 100644 --- a/tests/pypi/hub_exposure/hub_exposure_tests.bzl +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -1,7 +1,8 @@ """hub_exposure tests""" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private/pypi:hub_exposure.bzl", "hub_exposure") # buildifier: disable=bzl-visibility +load("//python/private/pypi:hub_exposure.bzl", "add_extras", "hub_exposure") # buildifier: disable=bzl-visibility +load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # buildifier: disable=bzl-visibility load("//tests/support/mocks:mocks.bzl", "mocks") _tests = [] @@ -15,6 +16,14 @@ dev = ["other-pkg"] test = ["simple", "not-in-hub"] """ +_EXTRAS_PYPROJECT = """\ +[project] +dependencies = ["simple[foo]", "simple[Bar]; sys_platform == 'linux'"] + +[dependency-groups] +dev = ["other-pkg[baz]", "simple[foo]", "not-in-hub[qux]"] +""" + def _hub(*, module_name = "root", pyproject = None): return struct( module_name = module_name, @@ -38,6 +47,7 @@ def _exposure(*modules, hubs = None, default_pyproject = None): mocks.mctx( modules = list(modules), mock_files = { + "extras.toml": _EXTRAS_PYPROJECT, "other.toml": "[project]\ndependencies = [\"other-pkg\"]\n", "pyproject.toml": _PYPROJECT, }, @@ -51,7 +61,7 @@ def _exposure(*modules, hubs = None, default_pyproject = None): def _test_not_configured(env): got, failures = _exposure(_module()) env.expect.that_collection(failures).contains_exactly([]) - env.expect.that_dict(got).contains_exactly({}) + env.expect.that_dict(got.packages).contains_exactly({}) _tests.append(_test_not_configured) @@ -65,7 +75,7 @@ def _test_configure(env): ]: got, failures = _exposure(_module(configure = [_configure(dependency_groups = groups)])) env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) - env.expect.where(groups = groups).that_dict(got).contains_exactly({"hub": want}) + env.expect.where(groups = groups).that_dict(got.packages).contains_exactly({"hub": want}) _tests.append(_test_configure) @@ -77,7 +87,7 @@ def _test_configure_all_packages(env): _configure(dependency_groups = ["*", "transitive"]), ])) env.expect.that_collection(failures).contains_exactly([]) - env.expect.that_dict(got).contains_exactly({}) + env.expect.that_dict(got.packages).contains_exactly({}) _tests.append(_test_configure_all_packages) @@ -87,7 +97,7 @@ def _test_configure_combines_tags(env): _configure(dependency_groups = ["dev"]), ])) env.expect.that_collection(failures).contains_exactly([]) - env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg", "simple"]}) + env.expect.that_dict(got.packages).contains_exactly({"hub": ["other_pkg", "simple"]}) _tests.append(_test_configure_combines_tags) @@ -96,7 +106,7 @@ def _test_configure_pyproject_precedence(env): got, _ = _exposure(_module(configure = [ _configure(pyproject_toml = "other.toml"), ])) - env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) + env.expect.that_dict(got.packages).contains_exactly({"hub": ["other_pkg"]}) # The `pip.default` value is used for the hubs of the root module. got, failures = _exposure( @@ -105,7 +115,7 @@ def _test_configure_pyproject_precedence(env): default_pyproject = "pyproject.toml", ) env.expect.that_collection(failures).contains_exactly([]) - env.expect.that_dict(got).contains_exactly({"hub": ["simple"]}) + env.expect.that_dict(got.packages).contains_exactly({"hub": ["simple"]}) _tests.append(_test_configure_pyproject_precedence) @@ -118,7 +128,7 @@ def _test_configure_module_precedence(env): _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), hubs = hubs, ) - env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) + env.expect.that_dict(got.packages).contains_exactly({"hub": ["other_pkg"]}) # The root module wins over the owner. got, _ = _exposure( @@ -126,7 +136,7 @@ def _test_configure_module_precedence(env): _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), hubs = hubs, ) - env.expect.that_dict(got).contains_exactly({"hub": ["simple"]}) + env.expect.that_dict(got.packages).contains_exactly({"hub": ["simple"]}) # Other modules are ignored. got, _ = _exposure( @@ -134,10 +144,78 @@ def _test_configure_module_precedence(env): _module(name = "other", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), hubs = hubs, ) - env.expect.that_dict(got).contains_exactly({}) + env.expect.that_dict(got.packages).contains_exactly({}) _tests.append(_test_configure_module_precedence) +def _test_configure_extras(env): + hubs = {"hub": _hub(pyproject = "extras.toml")} + for groups, want in [ + (["default"], {"hub": {"simple": ["bar", "foo"]}}), + # The packages that are not in the hub are ignored. + (["dev"], {"hub": {"other_pkg": ["baz"], "simple": ["foo"]}}), + # The extras are also exposed when all of the packages are exposed. + (["*", "transitive"], {"hub": {"other_pkg": ["baz"], "simple": ["bar", "foo"]}}), + ]: + got, failures = _exposure( + _module(configure = [_configure(dependency_groups = groups)]), + hubs = hubs, + ) + env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) + env.expect.where(groups = groups).that_dict(got.extras).contains_exactly(want) + + # The extras of multiple tags are combined. + got, _ = _exposure( + _module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(dependency_groups = ["dev"]), + ]), + hubs = hubs, + ) + env.expect.that_dict(got.extras).contains_exactly({ + "hub": {"other_pkg": ["baz"], "simple": ["bar", "foo"]}, + }) + + # No extras are requested. + got, _ = _exposure(_module(configure = [_configure()])) + env.expect.that_dict(got.extras).contains_exactly({}) + +_tests.append(_test_configure_extras) + +def _test_add_extras(env): + whl_map = { + "other_pkg": {"repo_other": [whl_config_setting(version = "3.15")]}, + "simple": { + "repo_linux": [whl_config_setting( + version = "3.15", + target_platforms = ["cp315_linux_x86_64"], + extras = ["bar"], + )], + "repo_osx": [whl_config_setting( + version = "3.15", + target_platforms = ["cp315_osx_aarch64"], + )], + }, + } + got = add_extras(whl_map, {"simple": ["bar", "foo"]}) + env.expect.that_dict(got).contains_exactly({ + "other_pkg": {"repo_other": [whl_config_setting(version = "3.15")]}, + "simple": { + "repo_linux": [whl_config_setting( + version = "3.15", + target_platforms = ["cp315_linux_x86_64"], + extras = ["bar", "foo"], + )], + "repo_osx": [whl_config_setting( + version = "3.15", + target_platforms = ["cp315_osx_aarch64"], + extras = ["bar", "foo"], + )], + }, + }) + +_tests.append(_test_add_extras) + def _test_configure_errors(env): _, failures = _exposure(_module(configure = [_configure(hub_name = "missing")])) env.expect.that_collection(failures).contains_exactly([ From 45c1c82c7c0f33fc716f82175a787b3ed0159172 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Wed, 18 Nov 2026 01:16:18 +0900 Subject: [PATCH 31/55] docs(pypi): document the pip.configure extras --- docs/pypi/download.md | 15 +++++++++++++++ news/pip-configure-extras.added.md | 5 +++++ 2 files changed, 20 insertions(+) create mode 100644 news/pip-configure-extras.added.md diff --git a/docs/pypi/download.md b/docs/pypi/download.md index 654e156921..a29b1387f7 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -182,6 +182,21 @@ selects the optional dependencies of the project. The packages that are not part of the hub are ignored, so a single `pyproject.toml` can list tools that are missing from the lock file. +The extras requested in the selected groups are exposed by the hub even if the +lock file does not request them, for example because the lock file was +generated with `--strip-extras`. With the following `pyproject.toml`: + +```toml +[project] +dependencies = ["requests[socks]"] +``` + +the hub has the `@pypi_hub//requests:[socks]` target and +`@pypi_hub//requests` includes the `socks` extra on all of the platforms. The +environment markers of the requirements are ignored when selecting the extras. +The dependencies of the extras, such as `pysocks`, still have to be in the lock +file. See [extras](./use.md#extras) for the targets of the extras. + :::{versionadded} VERSION_NEXT_FEATURE ::: diff --git a/news/pip-configure-extras.added.md b/news/pip-configure-extras.added.md new file mode 100644 index 0000000000..a722e3f9eb --- /dev/null +++ b/news/pip-configure-extras.added.md @@ -0,0 +1,5 @@ +(pypi) {obj}`pip.configure` now exposes the extras requested in the selected +`pyproject.toml` dependency groups, e.g. `requests[socks]`, as the +`@pypi//requests:[socks]` target and includes them in `@pypi//requests`, even +if the lock file does not request them +([#2786](https://github.com/bazel-contrib/rules_python/issues/2786)). From c54d57d6963883c6cf629370479c4346adf07077 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Wed, 18 Nov 2026 01:21:01 +0900 Subject: [PATCH 32/55] feat(pypi): add rules to depend on an extra of a package by its name The extra_dep target depends on the _extras target of a package with the requested extra set in a private build setting and the extra_select target selects the target of that extra and resets the setting, so that the targets of the extras are analyzed in the same configuration. The extras that the package does not provide fall back to the package itself, like pip. --- python/private/pypi/BUILD.bazel | 18 +++ python/private/pypi/extra_dep.bzl | 170 +++++++++++++++++++++++ tests/pypi/extra_dep/BUILD.bazel | 3 + tests/pypi/extra_dep/extra_dep_tests.bzl | 167 ++++++++++++++++++++++ 4 files changed, 358 insertions(+) create mode 100644 python/private/pypi/extra_dep.bzl create mode 100644 tests/pypi/extra_dep/BUILD.bazel create mode 100644 tests/pypi/extra_dep/extra_dep_tests.bzl diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 70aaa5a041..1d0903879a 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -15,6 +15,7 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("//python:py_binary.bzl", "py_binary") load("//python/private:distribution_filegroup.bzl", "distribution_filegroup") +load(":extra_dep.bzl", "requested_extra") package(default_visibility = ["//:__subpackages__"]) @@ -25,6 +26,13 @@ exports_files( visibility = ["//visibility:public"], ) +# The extra requested by the `extra_dep` targets in the whl repositories. +requested_extra( + name = "requested_extra", + build_setting_default = "", + visibility = ["//visibility:public"], +) + alias( name = "venv_entry_point_template", actual = select({ @@ -100,6 +108,16 @@ bzl_library( deps = ["//python/private:bazel_tools"], ) +bzl_library( + name = "extra_dep", + srcs = ["extra_dep.bzl"], + deps = [ + "//python/private:py_cc_link_params_info", + "//python/private:py_info", + "//python/private:reexports", + ], +) + bzl_library( name = "env_marker_setting", srcs = ["env_marker_setting.bzl"], diff --git a/python/private/pypi/extra_dep.bzl b/python/private/pypi/extra_dep.bzl new file mode 100644 index 0000000000..cee3324183 --- /dev/null +++ b/python/private/pypi/extra_dep.bzl @@ -0,0 +1,170 @@ +"""Rules to depend on an extra of a PyPI package without knowing its extras. + +The dependencies of an extra, e.g. `bar[socks]`, are only known by reading the +`METADATA` of `bar`, which is only available in the whl repository of `bar`. +Instead of creating the targets for the extras in the hub, which would require +the hub to know the extras that each package provides, the dependency on an +extra is wired as follows: + +1. The package that depends on `bar[socks]` creates an {obj}`extra_dep` + target, which depends on the `_extras` target of `bar` with the + {obj}`REQUESTED_EXTRA` setting set to `socks`. +2. The whl repository of `bar` creates an {obj}`extra_select` target named + `_extras`, which selects the `[socks]` target based on the value of the + setting and resets the setting to its default value, so that the targets of + the extras are analyzed in the same configuration as the rest of the build. + +The hub only needs to forward the `_extras` target of each package to the +right whl repository for the target platform. +""" + +load("//python/private:py_cc_link_params_info.bzl", "PyCcLinkParamsInfo") +load("//python/private:py_info.bzl", "PyInfo") +load("//python/private:reexports.bzl", "BuiltinPyInfo") + +# The name of the target that selects the requested extra of a package. +EXTRAS_SELECT_TARGET = "_extras" + +# The setting that carries the requested extra from the {obj}`extra_dep` to the +# {obj}`extra_select` target. +REQUESTED_EXTRA = str(Label("//python/private/pypi:requested_extra")) + +def _requested_extra_impl(_ctx): + return [] + +requested_extra = rule( + implementation = _requested_extra_impl, + build_setting = config.string(flag = False), + doc = """\ +The normalized name of the extra requested by an {obj}`extra_dep` target. + +This is only set on the edge between {obj}`extra_dep` and the +{obj}`extra_select` target and it is reset by the {obj}`extra_select` target. +""", +) + +def _set_requested_extra_impl(_settings, attr): + return {REQUESTED_EXTRA: attr.extra} + +_set_requested_extra = transition( + implementation = _set_requested_extra_impl, + inputs = [], + outputs = [REQUESTED_EXTRA], +) + +def _reset_requested_extra_impl(_settings, _attr): + return {REQUESTED_EXTRA: ""} + +_reset_requested_extra = transition( + implementation = _reset_requested_extra_impl, + inputs = [], + outputs = [REQUESTED_EXTRA], +) + +def _forward(ctx, target, attr_name): + if type(target) == type([]): + target = target[0] + + default_info = target[DefaultInfo] + providers = [ + DefaultInfo( + files = default_info.files, + default_runfiles = default_info.default_runfiles, + data_runfiles = default_info.data_runfiles, + ), + coverage_common.instrumented_files_info( + ctx, + dependency_attributes = [attr_name], + ), + ] + for provider in [PyInfo, BuiltinPyInfo, PyCcLinkParamsInfo]: + if provider != None and provider in target: + providers.append(target[provider]) + + return providers + +def _extra_dep_impl(ctx): + return _forward(ctx, ctx.attr.pkg, "pkg") + +extra_dep = rule( + implementation = _extra_dep_impl, + doc = """\ +Depend on an extra of a PyPI package. + +The `pkg` is the {obj}`extra_select` target of the package, which resolves the +`extra` to the target that includes the package and the dependencies of the +extra. + +""", + attrs = { + "extra": attr.string( + mandatory = True, + doc = "The normalized name of the extra.", + ), + "pkg": attr.label( + mandatory = True, + cfg = _set_requested_extra, + doc = "The {obj}`extra_select` target of the package.", + ), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + }, +) + +def _extra_select_impl(ctx): + return _forward(ctx, ctx.attr.actual, "actual") + +extra_select = rule( + implementation = _extra_select_impl, + doc = """\ +Forward the target of the extra that is requested by an {obj}`extra_dep`. + +The `actual` is expected to be a `select` on the {obj}`REQUESTED_EXTRA` +setting. The setting is reset on the edge to `actual` so that the targets of +the extras are analyzed in the same configuration regardless of how they are +reached. + +""", + attrs = { + "actual": attr.label( + mandatory = True, + cfg = _reset_requested_extra, + doc = "The target for the requested extra.", + ), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + }, +) + +def extra_select_targets(*, name, targets, default, native = native, rules = None, visibility = None): + """Create the {obj}`extra_select` target and the config settings for the extras. + + Args: + name: {type}`str` the name of the {obj}`extra_select` target. + targets: {type}`dict[str, str]` the normalized extras to the labels of + the targets that include the extra. + default: {type}`str` the label used when the requested extra is not + provided by the package, which follows `pip` that only warns about + unknown extras. + native: the native struct for overriding in tests. + rules: a struct with the `extra_select` rule for overriding in tests. + visibility: {type}`list[str]` the visibility of the target. + """ + rules = rules or struct(extra_select = extra_select) + actual = {} + for extra, label in sorted(targets.items()): + setting = "{}_{}".format(name, extra) + native.config_setting( + name = setting, + flag_values = {REQUESTED_EXTRA: extra}, + visibility = ["//visibility:private"], + ) + actual[":" + setting] = label + + rules.extra_select( + name = name, + actual = select(actual | {"//conditions:default": default}) if actual else default, + visibility = visibility, + ) diff --git a/tests/pypi/extra_dep/BUILD.bazel b/tests/pypi/extra_dep/BUILD.bazel new file mode 100644 index 0000000000..1236a6ad28 --- /dev/null +++ b/tests/pypi/extra_dep/BUILD.bazel @@ -0,0 +1,3 @@ +load(":extra_dep_tests.bzl", "extra_dep_test_suite") + +extra_dep_test_suite(name = "extra_dep_tests") diff --git a/tests/pypi/extra_dep/extra_dep_tests.bzl b/tests/pypi/extra_dep/extra_dep_tests.bzl new file mode 100644 index 0000000000..13d28fa7a8 --- /dev/null +++ b/tests/pypi/extra_dep/extra_dep_tests.bzl @@ -0,0 +1,167 @@ +"""Tests for the extra_dep and extra_select rules.""" + +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("@rules_testing//lib:util.bzl", rt_util = "util") +load("//python:py_info.bzl", "PyInfo") +load("//python:py_library.bzl", "py_library") +load("//python/private/pypi:extra_dep.bzl", "extra_dep", "extra_select_targets") # buildifier: disable=bzl-visibility + +_tests = [] +_basic_tests = [] + +def _py_library(name): + """A py_library with a generated source to check the output configuration.""" + rt_util.helper_target( + native.genrule, + name = name + "_gen", + outs = [name + ".py"], + cmd = "touch $@", + ) + rt_util.helper_target( + py_library, + name = name, + srcs = [name + ".py"], + ) + +def _setup(name): + """Create the targets of a package `foo` that provides the `a` extra.""" + _py_library(name + "_foo") + _py_library(name + "_dep_a") + rt_util.helper_target( + py_library, + name = name + "_foo[a]", + deps = [name + "_foo", name + "_dep_a"], + ) + extra_select_targets( + name = name + "_extras", + targets = {"a": name + "_foo[a]"}, + default = name + "_foo", + visibility = ["//visibility:private"], + ) + +def _sources(target): + return [f.basename for f in target[PyInfo].transitive_sources.to_list()] + +def _test_extra(name): + _setup(name) + rt_util.helper_target( + extra_dep, + name = name + "_subject", + pkg = name + "_extras", + extra = "a", + ) + analysis_test( + name = name, + impl = _test_extra_impl, + target = name + "_subject", + ) + +def _test_extra_impl(env, target): + env.expect.that_collection(_sources(target)).contains_exactly([ + "test_extra_dep_a.py", + "test_extra_foo.py", + ]) + + # The setting is reset, so the targets of the extra are analyzed in the + # same configuration as the target that depends on the extra. + env.expect.that_collection({ + f.root.path: None + for f in target[PyInfo].transitive_sources.to_list() + }.keys()).contains_exactly([env.ctx.bin_dir.path]) + +_tests.append(_test_extra) + +def _test_unknown_extra(name): + _setup(name) + rt_util.helper_target( + extra_dep, + name = name + "_subject", + pkg = name + "_extras", + extra = "unknown", + ) + analysis_test( + name = name, + impl = _test_unknown_extra_impl, + target = name + "_subject", + ) + +def _test_unknown_extra_impl(env, target): + # Like `pip`, the extras that the package does not provide are ignored. + env.expect.that_collection(_sources(target)).contains_exactly([ + "test_unknown_extra_foo.py", + ]) + +_tests.append(_test_unknown_extra) + +def _test_extra_select_targets(env): + config_settings = [] + extra_selects = [] + + extra_select_targets( + name = "_extras", + targets = {"a": ":[a]", "b": ":[b]"}, + default = ":pkg", + native = struct(config_setting = lambda **kwargs: config_settings.append(kwargs)), + rules = struct(extra_select = lambda **kwargs: extra_selects.append(kwargs)), + ) + + env.expect.that_collection(config_settings).contains_exactly([ + { + "name": "_extras_a", + "flag_values": {str(Label("//python/private/pypi:requested_extra")): "a"}, + "visibility": ["//visibility:private"], + }, + { + "name": "_extras_b", + "flag_values": {str(Label("//python/private/pypi:requested_extra")): "b"}, + "visibility": ["//visibility:private"], + }, + ]) # buildifier: @unsorted-dict-items + env.expect.that_collection(extra_selects).contains_exactly([ + { + "name": "_extras", + "actual": select({ + ":_extras_a": ":[a]", + ":_extras_b": ":[b]", + "//conditions:default": ":pkg", + }), + "visibility": None, + }, + ]) # buildifier: @unsorted-dict-items + +_basic_tests.append(_test_extra_select_targets) + +def _test_extra_select_targets_no_extras(env): + extra_selects = [] + + extra_select_targets( + name = "_extras", + targets = {}, + default = ":pkg", + native = struct(), + rules = struct(extra_select = lambda **kwargs: extra_selects.append(kwargs)), + visibility = ["//visibility:public"], + ) + + env.expect.that_collection(extra_selects).contains_exactly([ + { + "name": "_extras", + "actual": ":pkg", + "visibility": ["//visibility:public"], + }, + ]) # buildifier: @unsorted-dict-items + +_basic_tests.append(_test_extra_select_targets_no_extras) + +def extra_dep_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite( + name = name, + tests = _tests, + basic_tests = _basic_tests, + ) From b514331f6d618ec2aa13e052a0c7696e4744dcab Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Wed, 18 Nov 2026 01:21:01 +0900 Subject: [PATCH 33/55] docs(pypi): name the news entry after the PR --- news/{pip-configure-extras.added.md => 4522.added.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename news/{pip-configure-extras.added.md => 4522.added.md} (100%) diff --git a/news/pip-configure-extras.added.md b/news/4522.added.md similarity index 100% rename from news/pip-configure-extras.added.md rename to news/4522.added.md From a094169d235e3d7d6d68fd5d1fe1ba921a15767f Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Wed, 18 Nov 2026 01:28:12 +0900 Subject: [PATCH 34/55] feat(pypi): wire the extras of the dependencies without the hub The dependencies on the extras of other packages from the wheel METADATA, e.g. 'Requires-Dist: bar[socks]', used the '[socks]' target of the hub, which exists only if the extra is requested in the lock file, so the build failed if the lock file did not have the extra. Each whl repository now has an '_extras' target that selects the target of the extra requested via an extra_dep target, and the hub forwards it to the whl repository of the target platform. The extras that the package does not provide fall back to the package, like pip does. --- python/private/pypi/BUILD.bazel | 3 +- python/private/pypi/pkg_aliases.bzl | 12 +- .../private/pypi/whl_library_deps_targets.bzl | 60 ++++++--- tests/pypi/pkg_aliases/pkg_aliases_test.bzl | 14 ++ .../whl_library_deps_targets_tests.bzl | 126 ++++++++++++++---- 5 files changed, 166 insertions(+), 49 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 1d0903879a..bd9e5a23bf 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -372,6 +372,7 @@ bzl_library( name = "pkg_aliases", srcs = ["pkg_aliases.bzl"], deps = [ + ":extra_dep", ":labels", ":whl_library_deps_targets", "//python:py_library", @@ -630,8 +631,8 @@ bzl_library( srcs = ["whl_library_deps_targets.bzl"], deps = [ ":env_marker_setting", + ":extra_dep", ":labels", - ":missing_package", ":pep508_deps", "//python:py_library", "//python/private:normalize_name", diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index 1920e2b971..40d545302a 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -35,6 +35,7 @@ load("@bazel_skylib//lib:selects.bzl", "selects") load("//python:py_library.bzl", "py_library") load("//python/private:common_labels.bzl", "labels") load("//python/private:text_util.bzl", "render") +load(":extra_dep.bzl", "EXTRAS_SELECT_TARGET", "extra_dep", "extra_select") load( ":labels.bzl", "DATA_LABEL", @@ -124,6 +125,8 @@ def pkg_aliases( _native = kwargs.pop("native", native) rules = kwargs.pop("rules", struct( + extra_dep = extra_dep, + extra_select = extra_select, py_library = py_library, whl_library_deps_targets = whl_library_deps_targets, )) @@ -157,6 +160,10 @@ def pkg_aliases( # targets do not use the extras, so only the package without extras # can be a member of the group. target_names = { + # The whl repositories resolve the extras requested by the + # packages that depend on them, so the hub only needs to forward + # to the whl repository of the target platform. + EXTRAS_SELECT_TARGET: EXTRAS_SELECT_TARGET, WHEEL_FILE_PUBLIC_LABEL: WHEEL_FILE_IMPL_LABEL if "" in group_extras else WHEEL_FILE_PUBLIC_LABEL, } @@ -214,7 +221,10 @@ def pkg_aliases( fail("The `actual` arg must be a dictionary or a string") kwargs = {} - if target_name.startswith("_"): + if target_name == EXTRAS_SELECT_TARGET: + # Used by the whl repositories that depend on the extras. + pass + elif target_name.startswith("_"): kwargs["visibility"] = ["//_groups:__subpackages__"] elif target_name in (WHEEL_FILE, PY_SRCS_LABEL): kwargs["visibility"] = ["//visibility:private"] diff --git a/python/private/pypi/whl_library_deps_targets.bzl b/python/private/pypi/whl_library_deps_targets.bzl index 5cce4e870f..ed8e8384c0 100644 --- a/python/private/pypi/whl_library_deps_targets.bzl +++ b/python/private/pypi/whl_library_deps_targets.bzl @@ -3,6 +3,7 @@ load("//python:py_library.bzl", "py_library") load("//python/private:normalize_name.bzl", "normalize_name") load(":env_marker_setting.bzl", "env_marker_setting") +load(":extra_dep.bzl", "EXTRAS_SELECT_TARGET", "extra_dep", "extra_select", "extra_select_targets") load( ":labels.bzl", "DATA_LABEL", @@ -15,7 +16,6 @@ load( "extra_label", "group_member", ) -load(":missing_package.bzl", "missing_extra_error") load(":pep508_deps.bzl", "deps") def whl_library_deps_targets( @@ -36,7 +36,8 @@ def whl_library_deps_targets( rules = struct( py_library = py_library, env_marker_setting = env_marker_setting, - missing_extra_error = missing_extra_error, + extra_dep = extra_dep, + extra_select = extra_select, )): """Create all of the whl_library targets. @@ -53,11 +54,10 @@ def whl_library_deps_targets( provides_extra: {type}`list[str]` The list of extras that the package provides. For each extra the `[]` target is created that includes the package and the dependencies of that extra. - include: {type}`list[str] | dict[str, list[str] | None]` The list - of packages to include. If it is a dict, then the values are the - extras of each package that are available in the hub and the - dependencies on the extras that are not available will fail the - build with a helpful message. + The `_extras` target selects one of them for the + {obj}`extra_dep` targets of the packages that depend on an extra + of this package. + include: {type}`list[str] | dict[str, None]` The packages to include. group_name: {type}`str | None` name of the dependency group (if any). dep_template: {type}`str | None` The dep_template to use. tags: {type}`list[str]` The tags set on the targets. @@ -156,6 +156,30 @@ def whl_library_deps_targets( visibility = visibility, ) + def _extra_target(extra): + target = extra_label(extra) if extra else PY_LIBRARY_PUBLIC_LABEL + if group_name and not group_label and is_member(extra): + # The group library of the members is in the hub. + return dep_template.format(name = self_name, target = extra_label(extra)) + + return ":" + target + + # The packages that depend on an extra of this package use this target + # with the requested extra, so that the hub does not need to know which + # extras this package provides. + extra_select_targets( + name = EXTRAS_SELECT_TARGET, + targets = { + extra: _extra_target(extra) + for extra in extras_targets + if extra + }, + default = _extra_target(""), + native = native, + rules = rules, + visibility = visibility, + ) + if not requires_dist: # If there are extras, then they will be visible in requires_dist. return @@ -164,21 +188,16 @@ def whl_library_deps_targets( # dep and the expression. settings = {} - # The dependencies on extras that are not available in the hub. - missing_extras = {} + # The dependencies on the extras of other packages. + extra_deps = {} def _pkg_label(dep): dep_name, _, extra = dep.partition("[") if not extra: return dep_template.format(name = dep_name, target = PY_LIBRARY_PUBLIC_LABEL) - extra = extra.rstrip("]") - available = include.get(dep_name) if type(include) == type({}) else None - if available != None and extra not in available: - missing_extras[dep] = (dep_name, extra) - return ":_missing_" + dep - - return dep_template.format(name = dep_name, target = extra_label(extra)) + extra_deps[dep] = (dep_name, extra.rstrip("]")) + return ":_" + dep whl_label = lambda dep: dep_template.format(name = dep, target = WHEEL_FILE_PUBLIC_LABEL) @@ -241,12 +260,11 @@ def whl_library_deps_targets( visibility = ["//visibility:private"], ) - for dep, (dep_name, extra) in sorted(missing_extras.items()): - rules.missing_extra_error( - name = "_missing_" + dep, - package_name = dep_name, + for dep, (dep_name, extra) in sorted(extra_deps.items()): + rules.extra_dep( + name = "_" + dep, + pkg = dep_template.format(name = dep_name, target = EXTRAS_SELECT_TARGET), extra = extra, - required_by = metadata_name, visibility = ["//visibility:private"], ) diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index 8cd4773612..c3ee6c4f10 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -43,6 +43,7 @@ def _test_legacy_aliases(env): "foo": ":pkg", "pkg": ":[]", "[]": "@repo//:pkg", + "_extras": "@repo//:_extras", "srcs": "@repo//:srcs", "whl_file": "@repo//:whl_file", "whl": "@repo//:whl", @@ -221,6 +222,10 @@ def _test_group_aliases(env): "name": "foo", "actual": ":pkg", }, + { + "name": "_extras", + "actual": "@repo//:_extras", + }, { "name": "_pkg", "actual": "@repo//:pkg", @@ -352,6 +357,7 @@ def _test_deps_and_aliases_integration(env): # Use this function as it is used in pip_repository actual_aliases = [] actual_env_marker_settings = [] + actual_extra_select = [] actual_py_library = {} # buildifier: disable=unsorted-dict-items @@ -424,10 +430,18 @@ def _test_deps_and_aliases_integration(env): rules = struct( whl_library_deps_targets = whl_library_deps_targets, env_marker_setting = lambda **kwargs: actual_env_marker_settings.append(kwargs), + extra_select = lambda **kwargs: actual_extra_select.append(kwargs), py_library = lambda **kwargs: actual_py_library.update(kwargs), ), ) env.expect.that_collection(actual_aliases).contains_exactly(want_aliases) + env.expect.that_collection(actual_extra_select).contains_exactly([ + { + "name": "_extras", + "actual": ":pkg", + "visibility": ["//visibility:public"], + }, + ]) # buildifier: @unsorted-dict-items env.expect.that_collection(actual_env_marker_settings).contains_exactly(want_settings) env.expect.that_dict(actual_py_library).contains_at_least(want_library) diff --git a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl index 1c520c148a..462ea82b35 100644 --- a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl +++ b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl @@ -42,6 +42,7 @@ def _test_whl_library_deps_targets(env): rules = struct( py_library = lambda **kwargs: py_library_calls.append(kwargs), env_marker_setting = lambda **kwargs: env_marker_setting_calls.append(kwargs), + extra_select = lambda **kwargs: None, create_inits = lambda *args, **kwargs: ["_create_inits_target"], venv_rewrite_shebang = lambda **kwargs: None, ), @@ -95,6 +96,7 @@ def _test_whl_library_deps_targets_no_deps(env): filegroup_calls = [] py_library_calls = [] env_marker_setting_calls = [] + extra_select_calls = [] whl_library_deps_targets( name = "foo-0-py3-none-any.whl", @@ -114,9 +116,22 @@ def _test_whl_library_deps_targets_no_deps(env): rules = struct( py_library = lambda **kwargs: py_library_calls.append(kwargs), env_marker_setting = lambda **kwargs: env_marker_setting_calls.append(kwargs), + extra_select = lambda **kwargs: extra_select_calls.append(kwargs), ), ) + # The packages depending on the extras of this package can still use them. + env.expect.that_collection(extra_select_calls).contains_exactly([ + { + "name": "_extras", + "actual": select({ + ":_extras_a": ":[a]", + "//conditions:default": ":pkg", + }), + "visibility": ["//visibility:public"], + }, + ]) # buildifier: @unsorted-dict-items + # If the package is in a group but has no deps, then the public labels should be aliases # to the srcs targets and no other targets should be created. env.expect.that_collection(alias_calls).contains_exactly([ @@ -147,6 +162,8 @@ def _test_extras(env): filegroup_calls = [] py_library_calls = [] env_marker_setting_calls = [] + extra_dep_calls = [] + extra_select_calls = [] whl_library_deps_targets( name = "foo-0-py3-none-any.whl", @@ -166,29 +183,53 @@ def _test_extras(env): native = struct( filegroup = lambda **kwargs: filegroup_calls.append(kwargs), alias = lambda **kwargs: None, + config_setting = lambda **kwargs: None, ), rules = struct( py_library = lambda **kwargs: py_library_calls.append(kwargs), env_marker_setting = lambda **kwargs: env_marker_setting_calls.append(kwargs), + extra_dep = lambda **kwargs: extra_dep_calls.append(kwargs), + extra_select = lambda **kwargs: extra_select_calls.append(kwargs), ), ) # The `pkg` target has no extras because none were requested and there is # a target for each extra that the package provides. The `whl` target does - # not use the extras. Without the extras available in the hub, the - # dependencies use the targets with the extras directly. + # not use the extras. The dependencies on the extras of other packages use + # the `_extras` target of the package, so that the hub does not need to + # know about the extras. env.expect.that_collection([ (c["name"], c["deps"]) for c in py_library_calls ]).contains_exactly([ - ("pkg", ["srcs", "@pypi//bar:[socks]"]), - ("[all]", ["srcs", "@pypi//bar:[socks]", "@pypi//grpc:pkg"]), - ("[async]", ["srcs", "@pypi//bar:[socks]", "@pypi//uvloop:pkg"] + select({ + ("pkg", ["srcs", ":_bar[socks]"]), + ("[all]", ["srcs", ":_bar[socks]", "@pypi//grpc:pkg"]), + ("[async]", ["srcs", ":_bar[socks]", "@pypi//uvloop:pkg"] + select({ ":is_include_aiohttp_true": ["@pypi//aiohttp:pkg"], "//conditions:default": [], })), - ("[grpc]", ["srcs", "@pypi//bar:[socks]", "@pypi//grpc:pkg"]), + ("[grpc]", ["srcs", ":_bar[socks]", "@pypi//grpc:pkg"]), ]) + env.expect.that_collection(extra_dep_calls).contains_exactly([ + { + "name": "_bar[socks]", + "pkg": "@pypi//bar:_extras", + "extra": "socks", + "visibility": ["//visibility:private"], + }, + ]) # buildifier: @unsorted-dict-items + env.expect.that_collection(extra_select_calls).contains_exactly([ + { + "name": "_extras", + "actual": select({ + ":_extras_all": ":[all]", + ":_extras_async": ":[async]", + ":_extras_grpc": ":[grpc]", + "//conditions:default": ":pkg", + }), + "visibility": ["//visibility:public"], + }, + ]) # buildifier: @unsorted-dict-items env.expect.that_collection([ (c["name"], c["data"]) for c in filegroup_calls @@ -208,6 +249,7 @@ _tests.append(_test_extras) def _test_extras_in_group(env): py_library_calls = [] filegroup_calls = [] + extra_select_calls = [] whl_library_deps_targets( name = "foo-0-py3-none-any.whl", @@ -223,13 +265,26 @@ def _test_extras_in_group(env): native = struct( filegroup = lambda **kwargs: filegroup_calls.append(kwargs), alias = lambda **kwargs: None, + config_setting = lambda **kwargs: None, ), rules = struct( py_library = lambda **kwargs: py_library_calls.append(kwargs), env_marker_setting = lambda **kwargs: None, + extra_select = lambda **kwargs: extra_select_calls.append(kwargs), ), ) + # The members of the group are selected via the group library in the hub. + env.expect.that_collection([ + c["actual"] + for c in extra_select_calls + ]).contains_exactly([ + select({ + ":_extras_a": "@pypi//foo:[a]", + "//conditions:default": ":pkg", + }), + ]) + # Only the member targets exclude the other members and the targets are # visible to the hub that creates the group library. env.expect.that_collection([ @@ -252,6 +307,7 @@ def _test_extras_in_group_legacy(env): alias_calls = [] py_library_calls = [] filegroup_calls = [] + extra_select_calls = [] whl_library_deps_targets( name = "foo-0-py3-none-any.whl", @@ -266,13 +322,27 @@ def _test_extras_in_group_legacy(env): native = struct( filegroup = lambda **kwargs: filegroup_calls.append(kwargs), alias = lambda **kwargs: alias_calls.append(kwargs), + config_setting = lambda **kwargs: None, ), rules = struct( py_library = lambda **kwargs: py_library_calls.append(kwargs), env_marker_setting = lambda **kwargs: None, + extra_select = lambda **kwargs: extra_select_calls.append(kwargs), ), ) + # The public targets of the members point to the group library. + env.expect.that_collection([ + c["actual"] + for c in extra_select_calls + ]).contains_exactly([ + select({ + ":_extras_a": ":[a]", + ":_extras_b": ":[b]", + "//conditions:default": ":pkg", + }), + ]) + # The member targets are implementation targets and the public targets # point to the group library in the config repo. env.expect.that_collection([ @@ -304,7 +374,7 @@ def _test_deps_with_extras(env): filegroup_calls = [] py_library_calls = [] env_marker_setting_calls = [] - missing_calls = [] + extra_dep_calls = [] whl_library_deps_targets( name = "foo-0-py3-none-any.whl", @@ -316,51 +386,55 @@ def _test_deps_with_extras(env): "baz; sys_platform == 'darwin'", "qux[missing]", ], - # The values are the extras that are available in the hub + # The hub does not need to know which extras are available. include = { - "bar": ["socks"], - "baz": ["a", "b"], - "qux": [], + "bar": None, + "baz": None, + "qux": None, }, repo = None, aliases = {}, native = struct( filegroup = lambda **kwargs: filegroup_calls.append(kwargs), alias = lambda **kwargs: None, + config_setting = lambda **kwargs: None, ), rules = struct( py_library = lambda **kwargs: py_library_calls.append(kwargs), env_marker_setting = lambda **kwargs: env_marker_setting_calls.append(kwargs), - missing_extra_error = lambda **kwargs: missing_calls.append(kwargs), + extra_dep = lambda **kwargs: extra_dep_calls.append(kwargs), + extra_select = lambda **kwargs: None, ), ) - # The dependencies use the targets with the extras, the missing extras use - # a target that fails the build with a helpful message. + # The dependencies on the extras use the `_extras` target of the packages + # with the requested extra, which is resolved by the package that provides + # it. If the package does not provide the extra, then only the package is + # used. env.expect.that_collection([ (c["name"], c["deps"]) for c in py_library_calls ]).contains_exactly([ - ("pkg", ["srcs", "@pypi//bar:[socks]", ":_missing_qux[missing]"] + select({ + ("pkg", ["srcs", ":_bar[socks]", ":_qux[missing]"] + select({ ":is_include_baz_2_true": ["@pypi//baz:pkg"], "//conditions:default": [], }) + select({ - ":is_include_baz[a]_true": ["@pypi//baz:[a]"], + ":is_include_baz[a]_true": [":_baz[a]"], "//conditions:default": [], }) + select({ - ":is_include_baz[b]_true": ["@pypi//baz:[b]"], + ":is_include_baz[b]_true": [":_baz[b]"], "//conditions:default": [], })), ]) - env.expect.that_collection(missing_calls).contains_exactly([ - { - "name": "_missing_qux[missing]", - "package_name": "qux", - "extra": "missing", - "required_by": "foo", - "visibility": ["//visibility:private"], - }, - ]) # buildifier: @unsorted-dict-items + env.expect.that_collection([ + (c["name"], c["pkg"], c["extra"]) + for c in extra_dep_calls + ]).contains_exactly([ + ("_bar[socks]", "@pypi//bar:_extras", "socks"), + ("_baz[a]", "@pypi//baz:_extras", "a"), + ("_baz[b]", "@pypi//baz:_extras", "b"), + ("_qux[missing]", "@pypi//qux:_extras", "missing"), + ]) # The whl targets do not use the extras. env.expect.that_collection([ From a0553952d7619ea2cc8e8d92c5866730d87dfd0a Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Wed, 18 Nov 2026 01:29:48 +0900 Subject: [PATCH 35/55] refactor(pypi): stop passing the available extras to the whl repositories The whl repositories resolve the extras of their dependencies without the hub, so the config.bzl of the hub lists only the packages again and the missing_extra_error rule is removed. This means that the BUILD files of the whl repositories do not change when the extras in the lock file change. --- python/private/pypi/hub_repository.bzl | 15 +++++------- python/private/pypi/missing_package.bzl | 24 +------------------ python/private/pypi/pip_repository.bzl | 7 ++---- python/private/pypi/render_pkg_aliases.bzl | 19 --------------- .../render_pkg_aliases_test.bzl | 19 --------------- 5 files changed, 9 insertions(+), 75 deletions(-) diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index 7389c50964..27b324f465 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -15,7 +15,7 @@ "" load("//python/private:text_util.bzl", "render") -load(":render_pkg_aliases.bzl", "available_extras", "render_multiplatform_pkg_aliases") +load(":render_pkg_aliases.bzl", "render_multiplatform_pkg_aliases") load(":whl_config_setting.bzl", "whl_config_setting") _BUILD_FILE_CONTENTS = """\ @@ -27,12 +27,11 @@ exports_files(["requirements.bzl"]) def _impl(rctx): bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() - whl_map = { - key: _whl_config_settings_from_json(values) - for key, values in rctx.attr.whl_map.items() - } aliases = render_multiplatform_pkg_aliases( - aliases = whl_map, + aliases = { + key: _whl_config_settings_from_json(values) + for key, values in rctx.attr.whl_map.items() + }, extra_hub_aliases = rctx.attr.extra_hub_aliases, requirement_cycles = rctx.attr.groups, platform_config_settings = rctx.attr.platform_config_settings, @@ -51,9 +50,7 @@ def _impl(rctx): "config.bzl", rctx.attr._config_template, substitutions = { - # The available extras of each package for the spoke repositories - # to check if the extras of their dependencies are available. - "%%PACKAGES%%": render.dict(available_extras(whl_map), value_repr = render.list), + "%%PACKAGES%%": render.dict(rctx.attr.whl_map, value_repr = lambda x: "None"), }, ) rctx.template("requirements.bzl", rctx.attr._requirements_bzl_template, substitutions = { diff --git a/python/private/pypi/missing_package.bzl b/python/private/pypi/missing_package.bzl index e5647d5786..4b750caf45 100644 --- a/python/private/pypi/missing_package.bzl +++ b/python/private/pypi/missing_package.bzl @@ -1,4 +1,4 @@ -"""Rules for generating an execution-phase action failure when a PyPI package or its extra is missing.""" +"""Rule for generating an execution-phase action failure when a PyPI package is missing.""" load("//python/private:py_info.bzl", "PyInfo") load("//python/private:reexports.bzl", "BuiltinPyInfo") @@ -45,25 +45,3 @@ missing_package_error = rule( "package_name": attr.string(mandatory = True), }, ) - -def _missing_extra_error_impl(ctx): - return _error(ctx, ( - 'The PyPI package "{pkg}" requires "{dep}[{extra}]", but the extra "{extra}" of ' + - '"{dep}" is not available in the hub. Add "{dep}[{extra}]" to the requirements or ' + - "the lock file of the hub, e.g. by compiling the requirements with the extras kept " + - "(`--no-strip-extras`) or by adding the extra to the project dependencies." - ).format( - pkg = ctx.attr.required_by, - dep = ctx.attr.package_name, - extra = ctx.attr.extra, - )) - -missing_extra_error = rule( - implementation = _missing_extra_error_impl, - doc = "A target that fails the build with a helpful message when an extra of a PyPI package is missing.", - attrs = { - "extra": attr.string(mandatory = True), - "package_name": attr.string(mandatory = True), - "required_by": attr.string(mandatory = True), - }, -) diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index 67bbe1e945..190d763ddd 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -285,13 +285,10 @@ def _pip_repository_impl(rctx): "config.bzl", rctx.attr._config_template, substitutions = { - # The available extras of each package for the spoke - # repositories to check if the extras of their dependencies - # are available. "%%PACKAGES%%": render.dict({ - pkg: sorted(selected_extras.get(pkg, [])) + pkg: None for pkg in bzl_packages - }, value_repr = render.list), + }, value_repr = lambda x: "None"), }, ) config_repo_template = repr(rctx.attr.name) diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 1753f557ea..3c7a11266b 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -254,22 +254,3 @@ def _non_versioned_platform(p, *, strict = False): return p else: fail("Expected to always have a platform in the form '{{abi}}_{{os}}_{{arch}}', got: {}".format(p)) - -def available_extras(aliases): - """Return the extras that are available in the hub for each package. - - Args: - aliases: {type}`dict[str, dict[struct, str]]` the packages to the - {obj}`whl_config_setting` instances and the repo names. - - Returns: - {type}`dict[str, list[str]]` the sorted extras for each package. - """ - return { - pkg: sorted({ - extra: None - for setting in settings - for extra in setting.extras or [] - }) - for pkg, settings in aliases.items() - } diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index 07f00bb155..5aae828d78 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -17,7 +17,6 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load( "//python/private/pypi:render_pkg_aliases.bzl", - "available_extras", "get_whl_flag_versions", "render_multiplatform_pkg_aliases", "render_pkg_aliases", @@ -270,24 +269,6 @@ pkg_aliases( _tests.append(_test_aliases_with_extras) -def _test_available_extras(env): - got = available_extras({ - "bar": { - whl_config_setting(version = "3.2"): "pypi_32_bar", - }, - "foo": { - whl_config_setting(version = "3.2", target_platforms = ["cp32_linux_x86_64"], extras = ["b"]): "pypi_32_foo", - whl_config_setting(version = "3.2", target_platforms = ["cp32_osx_aarch64"], extras = ["a", "b"]): "pypi_32_foo", - }, - }) - - env.expect.that_dict(got).contains_exactly({ - "bar": [], - "foo": ["a", "b"], - }) - -_tests.append(_test_available_extras) - def _test_aliases_with_extras_for_repo_names(env): actual = render_pkg_aliases( aliases = { From 45555f36a903b196a2db2141f11edd0d94d52c8c Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Wed, 18 Nov 2026 01:29:49 +0900 Subject: [PATCH 36/55] docs(pypi): document that the extras of the dependencies need not be locked Also check that the extras that a package does not provide are ignored in the whl_library integration test. --- docs/pypi/use.md | 13 +++++++------ news/2786.added.md | 9 +++++---- .../consumer/consumer-1.0.dist-info/METADATA | 1 + 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/pypi/use.md b/docs/pypi/use.md index 2660749052..932f5a00c9 100644 --- a/docs/pypi/use.md +++ b/docs/pypi/use.md @@ -119,12 +119,13 @@ platforms, for example `foo[bar]` on Linux and `foo[baz]` on macOS, then `@pypi//foo:[bar]` on macOS is the same as `@pypi//foo:[]`. The dependencies of the packages use the extras written in the wheel -`METADATA`, so if `foo` depends on `bar[socks]`, then `@pypi//foo` depends on -`@pypi//bar:[socks]`. If the extra is not in the lock file, then building -`@pypi//foo` fails with a message asking you to add `bar[socks]` to the -requirements or the lock file. For example, when compiling the requirements -with `uv pip compile`, pass `--no-strip-extras` to keep the extras in the -output. +`METADATA`, so if `foo` depends on `bar[socks]`, then `@pypi//foo` includes +`bar` and the dependencies of its `socks` extra. The extras of the +dependencies are resolved by the repository of `bar` from its own `METADATA`, +so they do not need to be in the lock file, e.g. when the requirements are +compiled with the extras stripped, but the packages that the extra depends on +need to be in the lock file. If `bar` does not provide the `socks` extra, then +only `bar` is used, like `pip` does. The `whl` targets, e.g. `@pypi//foo:whl`, do not use the extras and include only the wheels of the package and its dependencies without extras. diff --git a/news/2786.added.md b/news/2786.added.md index 4c00959a7d..33d9107600 100644 --- a/news/2786.added.md +++ b/news/2786.added.md @@ -8,10 +8,11 @@ returns a list of labels for any number of packages and extras, e.g. repositories now create the targets for every extra that the package provides and the requested extras are no longer passed to them, so changing the requested extras does not refetch them. The dependencies use the extras from -the wheel `METADATA` and the build fails with a helpful message if an extra is -missing in the lock file. The `whl` targets no longer include the dependencies -of the extras. The members of `experimental_requirement_cycles` can now be the -extras of the packages, e.g. `foo[bar]`. See +the wheel `METADATA`, which are resolved by the whl repositories, so the +extras of the dependencies do not need to be in the lock file. The `whl` +targets no longer include the dependencies of the extras. The members of +`experimental_requirement_cycles` can now be the extras of the packages, e.g. +`foo[bar]`. See [the docs](https://rules-python.readthedocs.io/en/latest/pypi/use.html#extras) ([#2786](https://github.com/bazel-contrib/rules_python/issues/2786), [#4168](https://github.com/bazel-contrib/rules_python/issues/4168)). diff --git a/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA b/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA index 9a723c958a..e2cbed2ee5 100644 --- a/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA +++ b/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA @@ -1,3 +1,4 @@ Name: consumer Version: 1.0 Requires-Dist: pkg[optional] +Requires-Dist: pkg[unknown] From 1ffebb84dee0f508ea8b237bf7eea252fd4e18b0 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 18 Nov 2026 19:45:58 -0800 Subject: [PATCH 37/55] refactor(pypi): polish pip.configure docs, docstrings, and formatting Align the new `pip.configure` documentation, docstrings, and Starlark files with repository style and documentation conventions. Add `requires-python` to the `docs/pypi/download.md` example, move the `pip.configure` section to top-level `##` heading order, clarify the `pyproject_toml` attribute docstrings and `{ref}` links, document helper arguments/returns and returned `struct` fields, flatten `hub_exposure` guards, move module-level test constants before functions, and wrap Starlark lines and `doc` strings to 80 columns. --- docs/pypi/download.md | 87 ++++++++++--------- python/private/pypi/dependency_groups.bzl | 39 +++++++-- python/private/pypi/extension.bzl | 52 +++++++---- python/private/pypi/hub_exposure.bzl | 16 ++-- python/private/pypi/hub_repository.bzl | 11 ++- python/private/pypi/render_pkg_aliases.bzl | 19 +++- python/private/pyproject_utils.bzl | 5 ++ .../dependency_groups_tests.bzl | 49 +++++++++-- tests/pypi/extension/extension_tests.bzl | 65 ++++++++------ tests/pypi/extension/pip_parse.bzl | 2 +- .../pypi/hub_exposure/hub_exposure_tests.bzl | 48 +++++++--- 11 files changed, 266 insertions(+), 127 deletions(-) diff --git a/docs/pypi/download.md b/docs/pypi/download.md index 654e156921..ef069da5b5 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -134,8 +134,50 @@ rename, and `requirement("numpy")` routes through the active hub like fixed at loading time, before the active hub is known, so load them from a concrete hub instead. Prefer `@pypi//` labels in new code. +### Declaring Abstract Dependencies (pip.dep) + +:::{versionadded} 2.2.0 +Declaring abstract PyPI dependencies via `pip.dep` tags. +::: + +Sometimes a shared library target or a ruleset needs to depend on a PyPI +package (e.g., `@pypi//numpy`), but does not want to force a specific package +version or a concrete `requirements.txt` lock file on its consumers. + +Instead of calling `pip.parse()`, the module can declare its dependency using +the `pip.dep` tag: + +```starlark +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") + +# Declare an abstract dependency on 'numpy' and specify extra targets that +# are expected to be available in the package. +pip.dep( + name = "numpy", + extra_targets = ["extra-alias"], +) +``` + +This ensures that the target structure `@pypi//numpy` (and +`@pypi//numpy:extra-alias`) exists in the unified `@pypi` hub repository, so the +declaring module can compile and analyze successfully without needing any local +requirements file. + +The actual concrete implementation and version of the package must be provided +by a downstream module calling `pip.parse`. + +If a downstream module attempts to build a target that depends on an abstract +dependency, but has not provided a concrete implementation for it via any +`pip.parse` call, the build will fail at execution time. + + +As with any repository rule or extension, if you would like to ensure that `pip_parse` is +re-executed to pick up a non-hermetic change to your environment (e.g., updating your system +`python` interpreter), you can force it to re-execute by running `bazel sync --only [pip_parse +name]`. + (pip-configure)= -### Configuring the Exposed Packages (pip.configure) +## Configuring the Exposed Packages (pip.configure) By default a hub exposes all of the packages from its requirements files or `uv.lock` file, including the transitive dependencies. A `pip.configure` tag @@ -145,6 +187,7 @@ can limit the hub to the packages listed in a `pyproject.toml` file: the ```toml [project] +requires-python = "==3.13.0" dependencies = ["numpy"] [dependency-groups] @@ -185,48 +228,6 @@ are missing from the lock file. :::{versionadded} VERSION_NEXT_FEATURE ::: -### Declaring Abstract Dependencies (pip.dep) - -:::{versionadded} 2.2.0 -Declaring abstract PyPI dependencies via `pip.dep` tags. -::: - -Sometimes a shared library target or a ruleset needs to depend on a PyPI -package (e.g., `@pypi//numpy`), but does not want to force a specific package -version or a concrete `requirements.txt` lock file on its consumers. - -Instead of calling `pip.parse()`, the module can declare its dependency using -the `pip.dep` tag: - -```starlark -pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") - -# Declare an abstract dependency on 'numpy' and specify extra targets that -# are expected to be available in the package. -pip.dep( - name = "numpy", - extra_targets = ["extra-alias"], -) -``` - -This ensures that the target structure `@pypi//numpy` (and -`@pypi//numpy:extra-alias`) exists in the unified `@pypi` hub repository, so the -declaring module can compile and analyze successfully without needing any local -requirements file. - -The actual concrete implementation and version of the package must be provided -by a downstream module calling `pip.parse`. - -If a downstream module attempts to build a target that depends on an abstract -dependency, but has not provided a concrete implementation for it via any -`pip.parse` call, the build will fail at execution time. - - -As with any repository rule or extension, if you would like to ensure that `pip_parse` is -re-executed to pick up a non-hermetic change to your environment (e.g., updating your system -`python` interpreter), you can force it to re-execute by running `bazel sync --only [pip_parse -name]`. - (per-os-arch-requirements)= ## Requirements for a specific OS/Architecture diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl index 4e40c7e856..e40e67a4c3 100644 --- a/python/private/pypi/dependency_groups.bzl +++ b/python/private/pypi/dependency_groups.bzl @@ -34,13 +34,21 @@ def select_dependency_groups(pyproject, specs, _fail = fail): groups = {"default": pyproject.dependencies} for name, reqs in pyproject.optional_dependencies.items(): groups["optional:" + normalize_name(name)] = reqs - for name, reqs in _expand_dependency_groups(pyproject.dependency_groups, _fail).items(): + expanded = _expand_dependency_groups(pyproject.dependency_groups, _fail) + for name, reqs in expanded.items(): groups["group:" + name] = reqs if "transitive" in specs: if "*" not in specs: - _fail("The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: {}".format(specs)) - return struct(packages = [], all = True) + _fail("The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: {}".format( + specs, + )) + return struct( + # list[str]: sorted normalized package names selected by `specs`. + packages = [], + # bool: True when all hub packages are selected. + all = True, + ) selected = {} for spec in specs: @@ -67,9 +75,24 @@ def select_dependency_groups(pyproject, specs, _fail = fail): for r in groups.get("optional:" + extra, []): packages[requirement(r).name_] = None - return struct(packages = sorted(packages), all = False) + return struct( + # list[str]: sorted normalized package names selected by `specs`. + packages = sorted(packages), + # bool: True when all packages including transitive ones are selected. + all = False, + ) def _match(spec, groups): + """Match a dependency group specifier against normalized group keys. + + Args: + spec: {type}`str` the dependency group specifier. + groups: {type}`dict[str, list[str]]` available groups keyed by + prefixed group name. + + Returns: + {type}`list[str]` matching keys in `groups`. + """ if spec == "*": return groups.keys() if spec in ("optional:*", "group:*"): @@ -122,7 +145,10 @@ def _expand_dependency_groups(dependency_groups, _fail): include = normalize_name(item.get("include-group", "")) if include not in groups: - _fail("Unknown dependency group '{}' included from '{}'".format(include, name)) + _fail("Unknown dependency group '{}' included from '{}'".format( + include, + name, + )) return {} if include not in expanded: reqs = None @@ -133,7 +159,8 @@ def _expand_dependency_groups(dependency_groups, _fail): expanded[name] = reqs if len(expanded) != len(groups): + unexpanded = [name for name in groups if name not in expanded] _fail("Cyclic dependency group includes: {}".format( - ", ".join(sorted([name for name in groups if name not in expanded])), + ", ".join(sorted(unexpanded)), )) return expanded diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index dcc7aa1a94..78119835af 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -157,6 +157,7 @@ def build_config( for name, values in defaults["platforms"].items() }, enable_pipstar_extract = enable_pipstar_extract, + # Label | None: default `pyproject.toml` from `pip.default`. pyproject_toml = defaults["pyproject_toml"], toml_decode = toml.decode, unified_hub_requirements_bzl = unified_hub_requirements_bzl, @@ -369,8 +370,11 @@ You cannot use both the additive_build_content and additive_build_content_file a hub_group_map[hub.name] = out.group_map hub_whl_map[hub.name] = out.whl_map hubs[hub.name] = struct( + # str: module that created the hub. module_name = hub.module_name, + # Label | None: `pyproject_toml` set on `pip.parse`. pyproject = hub_pyproject.get(hub.name), + # list[str]: normalized names of all packages in the hub. packages = out.whl_map.keys(), ) @@ -385,6 +389,7 @@ You cannot use both the additive_build_content and additive_build_content_file a exposed_packages.update(configured_hubs) return struct( + # dict[str, list[str]]: configured hubs and exposed packages. configured_hubs = configured_hubs, config = config, declared_deps = declared_deps, @@ -419,7 +424,10 @@ def _create_unified_hub_repo(mods): for hub_name in hubs: # The hubs configured with `pip.configure` only expose some packages. - hub_packages = mods.configured_hubs.get(hub_name, mods.hub_whl_map[hub_name]) + hub_packages = mods.configured_hubs.get( + hub_name, + mods.hub_whl_map[hub_name], + ) for pkg_name in hub_packages: norm_pkg = normalize_name(pkg_name) if norm_pkg not in packages: @@ -673,15 +681,18 @@ If you are defining custom platforms in your project and don't want things to cl ), "pyproject_toml": attr.label( mandatory = False, - doc = """\ -Label pointing to pyproject.toml file to read the default Python version from. -When specified, reads the `requires-python` field from pyproject.toml and uses -it as the default python_version for all `pip.parse()` calls that don't -explicitly specify one. + doc = """ +Label pointing to a `pyproject.toml` file used to read the default Python +version and, for hubs of the root module, to select exposed packages via +{obj}`pip.configure`. + +When `[project] requires-python` is present, it is used as the default +`python_version` for `pip.parse()` calls that do not specify one. :::{note} -The version must be specified as `==X.Y.Z` (exact version with full semver). -This is designed to work with dependency management tools like Renovate. +If `requires-python` is set, the version must be specified as `==X.Y.Z` (exact +version with full semver). This is designed to work with dependency management +tools like Renovate. ::: :::{versionadded} 2.3.0 @@ -898,13 +909,16 @@ find in case extra indexes are specified. ), "pyproject_toml": attr.label( mandatory = False, - doc = """\ -Label pointing to a pyproject.toml file to read the Python version from. -When specified, the `requires-python` field is used as the `python_version` -for this `pip.parse()` call, unless `python_version` is set explicitly. + doc = """ +Label pointing to a `pyproject.toml` file used to read the Python version and +to select exposed packages via {obj}`pip.configure`. + +When `python_version` is not set explicitly, the `[project] requires-python` +field is used as the `python_version` for this `pip.parse()` call. :::{note} -The version must be specified as `==X.Y.Z` (exact version with full semver). +When used to derive `python_version`, `requires-python` must be specified as +`==X.Y.Z` (exact version with full semver). ::: :::{versionadded} 2.3.0 @@ -1116,7 +1130,7 @@ _configure_tag = tag_class( attrs = { "dependency_groups": attr.string_list( default = ["*"], - doc = """\ + doc = """ The `pyproject.toml` dependency groups whose packages the hub exposes. Each item is one of: @@ -1132,7 +1146,7 @@ Each item is one of: packages of the hub including the transitive dependencies, i.e. all of the packages from the requirements files or the `uv.lock` file. -See [Configuring the Exposed Packages](#pip-configure) for examples. +See {ref}`Configuring the Exposed Packages ` for examples. :::{versionadded} VERSION_NEXT_FEATURE ::: @@ -1140,7 +1154,7 @@ See [Configuring the Exposed Packages](#pip-configure) for examples. ), "hub_name": attr.string( mandatory = True, - doc = """\ + doc = """ The name of the hub to configure. :::{versionadded} VERSION_NEXT_FEATURE @@ -1149,7 +1163,7 @@ The name of the hub to configure. ), "pyproject_toml": attr.label( mandatory = False, - doc = """\ + doc = """ The `pyproject.toml` file that defines the dependency groups. Defaults to the {attr}`pip.parse.pyproject_toml` of the hub, then to {attr}`pip.default.pyproject_toml` if the hub is defined by the root module. @@ -1159,7 +1173,7 @@ The `pyproject.toml` file that defines the dependency groups. Defaults to the """, ), }, - doc = """\ + doc = """ Configure the packages exposed by a hub. The packages are selected from a `pyproject.toml` file using the @@ -1171,7 +1185,7 @@ they are not public. Only the root module and the module that created the hub can configure it. If both do, the root module wins. Multiple tags for the same hub are combined. -See [Configuring the Exposed Packages](#pip-configure) for examples. +See {ref}`Configuring the Exposed Packages ` for examples. :::{versionadded} VERSION_NEXT_FEATURE ::: diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl index fe9e586153..ff9f24694d 100644 --- a/python/private/pypi/hub_exposure.bzl +++ b/python/private/pypi/hub_exposure.bzl @@ -45,18 +45,20 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): return None info = hubs[hub] - if not mod.is_root and (mod.name != info.module_name or hub in root_configured): + if not mod.is_root and mod.name != info.module_name: # Only the root module and the module that created the hub can # configure it. continue + if not mod.is_root and hub in root_configured: + continue - pyproject = ( - tag.pyproject_toml or - info.pyproject or - (default_pyproject if info.module_name == root_name else None) - ) + pyproject = tag.pyproject_toml or info.pyproject + if not pyproject and info.module_name == root_name: + pyproject = default_pyproject if not pyproject: - _fail("pip.configure() for hub '{}' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`".format(hub)) + _fail("pip.configure() for hub '{}' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`".format( + hub, + )) return None selection = select_dependency_groups( diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index ff5311c6ea..99f659ba31 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -86,13 +86,16 @@ hub_repository = repository_rule( ), "hide_unexposed_packages": attr.bool( default = False, - doc = "Make the aliases of the packages that are not in `packages` visible only to the wheel repositories.", + doc = """ +Make the aliases of the packages that are not in `packages` visible only to +the wheel repositories. +""", ), "packages": attr.string_list( mandatory = False, - doc = """\ -The list of packages that will be exposed via all_*requirements macros. Defaults to whl_map keys -unless `hide_unexposed_packages` is set. + doc = """ +The list of packages that will be exposed via all_*requirements macros. +Defaults to whl_map keys unless `hide_unexposed_packages` is set. """, ), "platform_config_settings": attr.string_list_dict( diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 2c86155876..ba0ad2cc75 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -136,8 +136,14 @@ def render_pkg_aliases( public = ["//visibility:public"] if exposed_packages != None: - exposed_packages = {normalize_name(name): None for name in exposed_packages} - internal_visibility = _internal_visibility(aliases, has_groups = bool(requirement_cycles)) + exposed_packages = { + normalize_name(name): None + for name in exposed_packages + } + internal_visibility = _internal_visibility( + aliases, + has_groups = bool(requirement_cycles), + ) files = {} for name, pkg_aliases in aliases.items(): @@ -159,10 +165,17 @@ def render_pkg_aliases( return files def _internal_visibility(aliases, has_groups = False): - """The visibility of the packages that are not exposed. + """Return the visibility list for packages that are not exposed. The wheel repositories depend on the aliases of the hub, so the aliases have to be visible to them. + + Args: + aliases: {type}`dict[str, str | dict]` package-to-repo alias mapping. + has_groups: {type}`bool` whether requirement cycle groups exist. + + Returns: + {type}`list[str]` visibility package specifications. """ repo_names = {} for pkg_aliases in aliases.values(): diff --git a/python/private/pyproject_utils.bzl b/python/private/pyproject_utils.bzl index a9494b8ac2..ceb5c803aa 100644 --- a/python/private/pyproject_utils.bzl +++ b/python/private/pyproject_utils.bzl @@ -29,10 +29,15 @@ def read_pyproject(module_ctx, pyproject): data = toml.decode(module_ctx.read(module_ctx.path(pyproject), watch = "yes")) project = data.get("project", {}) return struct( + # str | None: raw `[project] requires-python` value. requires_python = project.get("requires-python"), + # str | None: `[project] name`. name = project.get("name"), + # list[str]: `[project] dependencies` requirement strings. dependencies = project.get("dependencies") or [], + # dict[str, list[str]]: `[project.optional-dependencies]` table. optional_dependencies = project.get("optional-dependencies") or {}, + # dict[str, list]: PEP 735 `[dependency-groups]` table. dependency_groups = data.get("dependency-groups") or {}, ) diff --git a/tests/pypi/dependency_groups/dependency_groups_tests.bzl b/tests/pypi/dependency_groups/dependency_groups_tests.bzl index ee23aec1fb..d494cae9d7 100644 --- a/tests/pypi/dependency_groups/dependency_groups_tests.bzl +++ b/tests/pypi/dependency_groups/dependency_groups_tests.bzl @@ -1,7 +1,10 @@ """dependency_groups tests""" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private/pypi:dependency_groups.bzl", "select_dependency_groups") # buildifier: disable=bzl-visibility +load( + "//python/private/pypi:dependency_groups.bzl", + "select_dependency_groups", +) # buildifier: disable=bzl-visibility _tests = [] @@ -35,11 +38,25 @@ def _test_select(env): (["optional:*"], ["docs_pkg", "grpc_pkg"]), (["group:*"], ["dev_pkg", "shared_pkg", "test_pkg"]), (["default", "group:test"], ["main_pkg", "shared_pkg", "test_pkg"]), - (["*"], ["dev_pkg", "docs_pkg", "grpc_pkg", "main_pkg", "shared_pkg", "test_pkg"]), + ( + ["*"], + [ + "dev_pkg", + "docs_pkg", + "grpc_pkg", + "main_pkg", + "shared_pkg", + "test_pkg", + ], + ), ]: got, failures = _select(specs) - env.expect.where(specs = specs).that_collection(failures).contains_exactly([]) - env.expect.where(specs = specs).that_collection(got.packages).contains_exactly(want).in_order() + env.expect.where(specs = specs).that_collection( + failures, + ).contains_exactly([]) + env.expect.where(specs = specs).that_collection( + got.packages, + ).contains_exactly(want).in_order() env.expect.where(specs = specs).that_bool(got.all).equals(False) _tests.append(_test_select) @@ -54,18 +71,32 @@ _tests.append(_test_select_all) def _test_select_errors(env): for specs, want in [ - (["missing"], "Unknown dependency group 'missing'. Available groups: default, group:dev, group:test, optional:all, optional:docs, optional:grpc"), - (["default", "transitive"], "The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: [\"default\", \"transitive\"]"), + ( + ["missing"], + "Unknown dependency group 'missing'. Available groups: default, group:dev, group:test, optional:all, optional:docs, optional:grpc", + ), + ( + ["default", "transitive"], + "The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: [\"default\", \"transitive\"]", + ), ]: _, failures = _select(specs) - env.expect.where(specs = specs).that_collection(failures).contains_exactly([want]) + env.expect.where(specs = specs).that_collection( + failures, + ).contains_exactly([want]) _tests.append(_test_select_errors) def _test_include_group_errors(env): for groups, want in [ - ({"a": [{"include-group": "missing"}]}, "Unknown dependency group 'missing' included from 'a'"), - ({"a": [{"include-group": "b"}], "b": [{"include-group": "a"}]}, "Cyclic dependency group includes: a, b"), + ( + {"a": [{"include-group": "missing"}]}, + "Unknown dependency group 'missing' included from 'a'", + ), + ( + {"a": [{"include-group": "b"}], "b": [{"include-group": "a"}]}, + "Cyclic dependency group includes: a, b", + ), ]: _, failures = _select(["*"], pyproject = struct( name = None, diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index bc80ed679c..b3a6e7837c 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -25,6 +25,27 @@ load(":pip_parse.bzl", _configure = "configure", _parse = "pip_parse") _tests = [] +_CONFIGURE_REQUIREMENTS = """\ +simple==0.0.1 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +other-pkg==0.0.2 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +transitive-pkg==0.0.3 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +""" + +_CONFIGURE_PYPROJECT = """\ +[project] +dependencies = ["simple"] + +[dependency-groups] +dev = ["other-pkg"] +test = ["simple"] +""" + def _pypi_mock_mctx(*modules, os_name = "unittest", arch_name = "exotic", environ = {}, read = None, mock_files = {}): _ = read # @unused return mocks.mctx( @@ -107,7 +128,16 @@ def _dep(*, name, extra_targets = []): extra_targets = extra_targets, ) -def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], dep = [], configure = [], is_root = True): +def _mod( + *, + name, + default = _default_tags_default, + parse = [], + override = [], + whl_mods = [], + dep = [], + configure = [], + is_root = True): return struct( name = name, tags = struct( @@ -821,27 +851,6 @@ simple==0.0.1 \ _tests.append(_test_default_index_setting) -_CONFIGURE_REQUIREMENTS = """\ -simple==0.0.1 \ - --hash=sha256:deadbeef \ - --hash=sha256:deadbaaf -other-pkg==0.0.2 \ - --hash=sha256:deadbeef \ - --hash=sha256:deadbaaf -transitive-pkg==0.0.3 \ - --hash=sha256:deadbeef \ - --hash=sha256:deadbaaf -""" - -_CONFIGURE_PYPROJECT = """\ -[project] -dependencies = ["simple"] - -[dependency-groups] -dev = ["other-pkg"] -test = ["simple"] -""" - def _configure_parse_modules(env, *modules): return _parse_modules( env, @@ -876,7 +885,9 @@ def _test_configure(env): _mod( name = "rules_python", parse = [_configure_parse(pyproject_toml = "pyproject.toml")], - configure = [_configure(hub_name = "pypi_a", dependency_groups = ["test"])], + configure = [ + _configure(hub_name = "pypi_a", dependency_groups = ["test"]), + ], ), ) @@ -891,9 +902,13 @@ def _test_configure_default_pyproject(env): env, _mod( name = "rules_python", - default = _default_tags_default + [_default(pyproject_toml = "pyproject.toml")], + default = _default_tags_default + [ + _default(pyproject_toml = "pyproject.toml"), + ], parse = [_configure_parse()], - configure = [_configure(hub_name = "pypi_a", dependency_groups = ["dev"])], + configure = [ + _configure(hub_name = "pypi_a", dependency_groups = ["dev"]), + ], ), ) diff --git a/tests/pypi/extension/pip_parse.bzl b/tests/pypi/extension/pip_parse.bzl index 3670282a92..db3008d431 100644 --- a/tests/pypi/extension/pip_parse.bzl +++ b/tests/pypi/extension/pip_parse.bzl @@ -5,7 +5,7 @@ def configure( dependency_groups = ["*"], hub_name, pyproject_toml = None): - """A simple helper for testing to simulate the PyPI extension configure tag class""" + """Simulate the PyPI extension configure tag class for testing.""" return struct( dependency_groups = dependency_groups, hub_name = hub_name, diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl index 5d84a351b5..a51deb9c56 100644 --- a/tests/pypi/hub_exposure/hub_exposure_tests.bzl +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -1,7 +1,10 @@ """hub_exposure tests""" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private/pypi:hub_exposure.bzl", "hub_exposure") # buildifier: disable=bzl-visibility +load( + "//python/private/pypi:hub_exposure.bzl", + "hub_exposure", +) # buildifier: disable=bzl-visibility load("//tests/support/mocks:mocks.bzl", "mocks") _tests = [] @@ -22,7 +25,10 @@ def _hub(*, module_name = "root", pyproject = None): packages = ["other_pkg", "simple", "transitive_pkg"], ) -def _configure(hub_name = "hub", dependency_groups = ["*"], pyproject_toml = None): +def _configure( + hub_name = "hub", + dependency_groups = ["*"], + pyproject_toml = None): return mocks.tag( hub_name = hub_name, dependency_groups = dependency_groups, @@ -63,9 +69,15 @@ def _test_configure(env): # `not-in-hub` is ignored (["group:test"], ["simple"]), ]: - got, failures = _exposure(_module(configure = [_configure(dependency_groups = groups)])) - env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) - env.expect.where(groups = groups).that_dict(got).contains_exactly({"hub": want}) + got, failures = _exposure( + _module(configure = [_configure(dependency_groups = groups)]), + ) + env.expect.where(groups = groups).that_collection( + failures, + ).contains_exactly([]) + env.expect.where(groups = groups).that_dict(got).contains_exactly( + {"hub": want}, + ) _tests.append(_test_configure) @@ -87,7 +99,9 @@ def _test_configure_combines_tags(env): _configure(dependency_groups = ["dev"]), ])) env.expect.that_collection(failures).contains_exactly([]) - env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg", "simple"]}) + env.expect.that_dict(got).contains_exactly( + {"hub": ["other_pkg", "simple"]}, + ) _tests.append(_test_configure_combines_tags) @@ -115,7 +129,11 @@ def _test_configure_module_precedence(env): # The owner of the hub can configure it. got, _ = _exposure( _module(), - _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + _module( + name = "dep", + is_root = False, + configure = [_configure(dependency_groups = ["dev"])], + ), hubs = hubs, ) env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) @@ -123,7 +141,11 @@ def _test_configure_module_precedence(env): # The root module wins over the owner. got, _ = _exposure( _module(configure = [_configure(dependency_groups = ["default"])]), - _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + _module( + name = "dep", + is_root = False, + configure = [_configure(dependency_groups = ["dev"])], + ), hubs = hubs, ) env.expect.that_dict(got).contains_exactly({"hub": ["simple"]}) @@ -131,7 +153,11 @@ def _test_configure_module_precedence(env): # Other modules are ignored. got, _ = _exposure( _module(), - _module(name = "other", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + _module( + name = "other", + is_root = False, + configure = [_configure(dependency_groups = ["dev"])], + ), hubs = hubs, ) env.expect.that_dict(got).contains_exactly({}) @@ -139,7 +165,9 @@ def _test_configure_module_precedence(env): _tests.append(_test_configure_module_precedence) def _test_configure_errors(env): - _, failures = _exposure(_module(configure = [_configure(hub_name = "missing")])) + _, failures = _exposure( + _module(configure = [_configure(hub_name = "missing")]), + ) env.expect.that_collection(failures).contains_exactly([ "pip.configure() refers to unknown hub 'missing'. Available hubs: hub", ]) From a40d1d5e00aee45af7f6724304cdf369a7c3aebc Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 18 Nov 2026 19:49:15 -0800 Subject: [PATCH 38/55] refactor(pypi): restore pyproject_toml attribute docstrings Restore the `pip.default.pyproject_toml` and `pip.parse.pyproject_toml` attribute docstrings to their original text per review decision. --- python/private/pypi/extension.bzl | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 78119835af..7814246e62 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -681,18 +681,15 @@ If you are defining custom platforms in your project and don't want things to cl ), "pyproject_toml": attr.label( mandatory = False, - doc = """ -Label pointing to a `pyproject.toml` file used to read the default Python -version and, for hubs of the root module, to select exposed packages via -{obj}`pip.configure`. - -When `[project] requires-python` is present, it is used as the default -`python_version` for `pip.parse()` calls that do not specify one. + doc = """\ +Label pointing to pyproject.toml file to read the default Python version from. +When specified, reads the `requires-python` field from pyproject.toml and uses +it as the default python_version for all `pip.parse()` calls that don't +explicitly specify one. :::{note} -If `requires-python` is set, the version must be specified as `==X.Y.Z` (exact -version with full semver). This is designed to work with dependency management -tools like Renovate. +The version must be specified as `==X.Y.Z` (exact version with full semver). +This is designed to work with dependency management tools like Renovate. ::: :::{versionadded} 2.3.0 @@ -909,16 +906,13 @@ find in case extra indexes are specified. ), "pyproject_toml": attr.label( mandatory = False, - doc = """ -Label pointing to a `pyproject.toml` file used to read the Python version and -to select exposed packages via {obj}`pip.configure`. - -When `python_version` is not set explicitly, the `[project] requires-python` -field is used as the `python_version` for this `pip.parse()` call. + doc = """\ +Label pointing to a pyproject.toml file to read the Python version from. +When specified, the `requires-python` field is used as the `python_version` +for this `pip.parse()` call, unless `python_version` is set explicitly. :::{note} -When used to derive `python_version`, `requires-python` must be specified as -`==X.Y.Z` (exact version with full semver). +The version must be specified as `==X.Y.Z` (exact version with full semver). ::: :::{versionadded} 2.3.0 From 1291919372fc4283939655250438ee646f9e6ba3 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 18 Nov 2026 20:20:05 -0800 Subject: [PATCH 39/55] refactor(pypi): short-circuit _fail paths with None in group selection When a non-aborting `_fail` callback is injected in tests, `select_dependency_groups` and `_expand_dependency_groups` previously continued executing after `_fail(...)`, causing `hub_exposure` to fail on `None` attribute access or return partial results instead of `None`. Return `None` immediately after each `_fail(...)` call in `dependency_groups.bzl`, guard `selection == None` in `hub_exposure.bzl`, and assert `None` return values in error tests. --- python/private/pypi/dependency_groups.bzl | 14 ++++++++++---- python/private/pypi/hub_exposure.bzl | 2 ++ .../dependency_groups/dependency_groups_tests.bzl | 6 ++++-- tests/pypi/hub_exposure/hub_exposure_tests.bzl | 14 ++++++++++++-- 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl index e40e67a4c3..511041417d 100644 --- a/python/private/pypi/dependency_groups.bzl +++ b/python/private/pypi/dependency_groups.bzl @@ -25,16 +25,19 @@ def select_dependency_groups(pyproject, specs, _fail = fail): _fail: {type}`callable` the failure function, mainly for testing. Returns: - {type}`struct` with the attributes: + {type}`struct | None` with the attributes: * `packages`: {type}`list[str]` the sorted normalized package names. * `all`: {type}`bool` whether all of the packages including the transitive dependencies should be selected, i.e. `"*"` and `"transitive"` were given. + Returns `None` on failure. """ groups = {"default": pyproject.dependencies} for name, reqs in pyproject.optional_dependencies.items(): groups["optional:" + normalize_name(name)] = reqs expanded = _expand_dependency_groups(pyproject.dependency_groups, _fail) + if expanded == None: + return None for name, reqs in expanded.items(): groups["group:" + name] = reqs @@ -43,6 +46,7 @@ def select_dependency_groups(pyproject, specs, _fail = fail): _fail("The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: {}".format( specs, )) + return None return struct( # list[str]: sorted normalized package names selected by `specs`. packages = [], @@ -58,6 +62,7 @@ def select_dependency_groups(pyproject, specs, _fail = fail): spec, ", ".join(sorted(groups)), )) + return None for key in keys: selected[key] = None @@ -122,8 +127,8 @@ def _expand_dependency_groups(dependency_groups, _fail): _fail: {type}`callable` the failure function. Returns: - {type}`dict[str, list[str]]` normalized group names mapped to their - requirement strings. + {type}`dict[str, list[str]] | None` normalized group names mapped to + their requirement strings, or `None` on failure. """ groups = { normalize_name(name): items @@ -149,7 +154,7 @@ def _expand_dependency_groups(dependency_groups, _fail): include, name, )) - return {} + return None if include not in expanded: reqs = None break @@ -163,4 +168,5 @@ def _expand_dependency_groups(dependency_groups, _fail): _fail("Cyclic dependency group includes: {}".format( ", ".join(sorted(unexpanded)), )) + return None return expanded diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl index ff9f24694d..ed103fd2d6 100644 --- a/python/private/pypi/hub_exposure.bzl +++ b/python/private/pypi/hub_exposure.bzl @@ -66,6 +66,8 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): tag.dependency_groups, _fail = _fail, ) + if selection == None: + return None if selection.all: exposes_all[hub] = None diff --git a/tests/pypi/dependency_groups/dependency_groups_tests.bzl b/tests/pypi/dependency_groups/dependency_groups_tests.bzl index d494cae9d7..208b30639f 100644 --- a/tests/pypi/dependency_groups/dependency_groups_tests.bzl +++ b/tests/pypi/dependency_groups/dependency_groups_tests.bzl @@ -80,7 +80,8 @@ def _test_select_errors(env): "The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: [\"default\", \"transitive\"]", ), ]: - _, failures = _select(specs) + got, failures = _select(specs) + env.expect.where(specs = specs).that_str(got).equals(None) env.expect.where(specs = specs).that_collection( failures, ).contains_exactly([want]) @@ -98,12 +99,13 @@ def _test_include_group_errors(env): "Cyclic dependency group includes: a, b", ), ]: - _, failures = _select(["*"], pyproject = struct( + got, failures = _select(["a"], pyproject = struct( name = None, dependencies = [], optional_dependencies = {}, dependency_groups = groups, )) + env.expect.that_str(got).equals(None) env.expect.that_collection(failures).contains_exactly([want]) _tests.append(_test_include_group_errors) diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl index a51deb9c56..2713b9355b 100644 --- a/tests/pypi/hub_exposure/hub_exposure_tests.bzl +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -165,21 +165,31 @@ def _test_configure_module_precedence(env): _tests.append(_test_configure_module_precedence) def _test_configure_errors(env): - _, failures = _exposure( + got, failures = _exposure( _module(configure = [_configure(hub_name = "missing")]), ) + env.expect.that_str(got).equals(None) env.expect.that_collection(failures).contains_exactly([ "pip.configure() refers to unknown hub 'missing'. Available hubs: hub", ]) - _, failures = _exposure( + got, failures = _exposure( _module(configure = [_configure()]), hubs = {"hub": _hub()}, ) + env.expect.that_str(got).equals(None) env.expect.that_collection(failures).contains_exactly([ "pip.configure() for hub 'hub' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`", ]) + got, failures = _exposure( + _module(configure = [_configure(dependency_groups = ["missing"])]), + ) + env.expect.that_str(got).equals(None) + env.expect.that_collection(failures).contains_exactly([ + "Unknown dependency group 'missing'. Available groups: default, group:dev, group:test", + ]) + _tests.append(_test_configure_errors) def hub_exposure_test_suite(name): From 28b48d5ba2771d3c019e03bcffbd3a0993364162 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Wed, 18 Nov 2026 20:30:56 -0800 Subject: [PATCH 40/55] refactor(pypi): expand nested self-extras and refine type comments Single-level self-reference expansion dropped nested extras from composite extras such as `my-project[all]` and added the project name itself to `packages`, and `dependency_groups` type comments omitted list element types. Expand self-referenced optional groups iteratively with a bounded queue in `dependency_groups.bzl` (with unit tests covering nested, bare, and cyclic self-references), and specify parameterized element types in `pyproject_utils.bzl`, `dependency_groups.bzl`, and `render_pkg_aliases.bzl`. --- python/private/pypi/dependency_groups.bzl | 19 +++++++----- python/private/pypi/render_pkg_aliases.bzl | 3 +- python/private/pyproject_utils.bzl | 12 ++++--- .../dependency_groups_tests.bzl | 31 +++++++++++++++++++ 4 files changed, 52 insertions(+), 13 deletions(-) diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl index 511041417d..c3b435a956 100644 --- a/python/private/pypi/dependency_groups.bzl +++ b/python/private/pypi/dependency_groups.bzl @@ -67,18 +67,23 @@ def select_dependency_groups(pyproject, specs, _fail = fail): selected[key] = None # A self reference like `my-project[test]` selects the optional - # dependencies, but only one level deep. + # dependencies, including nested composite extras. project = normalize_name(pyproject.name or "") packages = {} - for key in selected: - for req in groups[key]: + queue = list(selected) + for _ in range(len(groups)): + if not queue: + break + for req in groups[queue.pop(0)]: req = requirement(req) if req.name_ != project: packages[req.name_] = None continue for extra in req.extras: - for r in groups.get("optional:" + extra, []): - packages[requirement(r).name_] = None + opt_key = "optional:" + extra + if opt_key in groups and opt_key not in selected: + selected[opt_key] = None + queue.append(opt_key) return struct( # list[str]: sorted normalized package names selected by `specs`. @@ -122,8 +127,8 @@ def _expand_dependency_groups(dependency_groups, _fail): """Expand the PEP 735 `include-group` entries. Args: - dependency_groups: {type}`dict[str, list]` the `[dependency-groups]` - table. + dependency_groups: {type}`dict[str, list[str | dict[str, str]]]` the + `[dependency-groups]` table. _fail: {type}`callable` the failure function. Returns: diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index ba0ad2cc75..f03c33355e 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -171,7 +171,8 @@ def _internal_visibility(aliases, has_groups = False): have to be visible to them. Args: - aliases: {type}`dict[str, str | dict]` package-to-repo alias mapping. + aliases: {type}`dict[str, str | dict[struct, str]]` package-to-repo + alias mapping. has_groups: {type}`bool` whether requirement cycle groups exist. Returns: diff --git a/python/private/pyproject_utils.bzl b/python/private/pyproject_utils.bzl index ceb5c803aa..c65157df83 100644 --- a/python/private/pyproject_utils.bzl +++ b/python/private/pyproject_utils.bzl @@ -22,11 +22,12 @@ def read_pyproject(module_ctx, pyproject): * `dependencies`: {type}`list[str]` the `[project] dependencies`. * `optional_dependencies`: {type}`dict[str, list[str]]` the `[project.optional-dependencies]` table. - * `dependency_groups`: {type}`dict[str, list]` the PEP 735 - `[dependency-groups]` table. Items are either requirement strings - or `{include-group = ""}` tables. + * `dependency_groups`: {type}`dict[str, list[str | dict[str, str]]]` + the PEP 735 `[dependency-groups]` table. Items are either + requirement strings or `{include-group = ""}` tables. """ - data = toml.decode(module_ctx.read(module_ctx.path(pyproject), watch = "yes")) + raw = module_ctx.read(module_ctx.path(pyproject), watch = "yes") + data = toml.decode(raw) project = data.get("project", {}) return struct( # str | None: raw `[project] requires-python` value. @@ -37,7 +38,8 @@ def read_pyproject(module_ctx, pyproject): dependencies = project.get("dependencies") or [], # dict[str, list[str]]: `[project.optional-dependencies]` table. optional_dependencies = project.get("optional-dependencies") or {}, - # dict[str, list]: PEP 735 `[dependency-groups]` table. + # dict[str, list[str | dict[str, str]]]: PEP 735 `[dependency-groups]` + # table of requirement strings and `{"include-group": str}` dicts. dependency_groups = data.get("dependency-groups") or {}, ) diff --git a/tests/pypi/dependency_groups/dependency_groups_tests.bzl b/tests/pypi/dependency_groups/dependency_groups_tests.bzl index 208b30639f..ab56109c7c 100644 --- a/tests/pypi/dependency_groups/dependency_groups_tests.bzl +++ b/tests/pypi/dependency_groups/dependency_groups_tests.bzl @@ -61,6 +61,37 @@ def _test_select(env): _tests.append(_test_select) +def _test_self_reference_extras(env): + pyproject = struct( + name = "my-project", + dependencies = [], + optional_dependencies = { + "a": ["a-pkg", "my-project[b]"], + "all": ["My_Project[docs, grpc]"], + "b": ["b-pkg", "my-project[a]"], + "docs": ["docs-pkg"], + "everything": ["my-project[all]", "my-project"], + "grpc": ["grpc-pkg"], + }, + dependency_groups = { + "ci": ["ci-pkg", "my-project[all]"], + }, + ) + for specs, want in [ + (["everything"], ["docs_pkg", "grpc_pkg"]), + (["ci"], ["ci_pkg", "docs_pkg", "grpc_pkg"]), + (["a"], ["a_pkg", "b_pkg"]), + ]: + got, failures = _select(specs, pyproject = pyproject) + env.expect.where(specs = specs).that_collection( + failures, + ).contains_exactly([]) + env.expect.where(specs = specs).that_collection( + got.packages, + ).contains_exactly(want).in_order() + +_tests.append(_test_self_reference_extras) + def _test_select_all(env): got, failures = _select(["*", "transitive"]) env.expect.that_collection(failures).contains_exactly([]) From 0f35bd1b060054334ec1b1c0ff0c9eb000b21f92 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:32:52 +0900 Subject: [PATCH 41/55] feat(pypi): make pip.configure a builder that does not require a pyproject.toml The dependency_groups default is now empty, so a pip.configure tag only needs a pyproject.toml when it selects dependency groups. A pyproject_toml set on a tag is used by the later tags of the same module and hub. --- docs/pypi/download.md | 36 ++++++++--- python/private/pypi/extension.bzl | 23 +++++-- python/private/pypi/hub_exposure.bzl | 17 +++++- tests/pypi/extension/pip_parse.bzl | 2 +- .../pypi/hub_exposure/hub_exposure_tests.bzl | 61 +++++++++++++++++-- 5 files changed, 116 insertions(+), 23 deletions(-) diff --git a/docs/pypi/download.md b/docs/pypi/download.md index ef069da5b5..931b64cf57 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -212,15 +212,33 @@ dependencies of `numpy` are only used through `@pypi_hub//numpy`. The `all_requirements` lists in the hub's `requirements.bzl` and the [Unified @pypi Hub](unified-pypi-hub) only contain the exposed packages. -The {attr}`pip.configure.dependency_groups` attribute defaults to `["*"]`, -which selects every group. Use `["*", "transitive"]` to expose all of the -packages of the hub, including the transitive dependencies, which is the same -as not configuring the hub. Selecting the transitive dependencies of only some -of the groups is not supported. - -Multiple tags for the same hub are combined. Only the root module and the -module that created the hub can configure it; if both do, the root module -wins. A self reference such as `my-project[test]` in the `pyproject.toml` +Use `["*"]` to select every group, or `["*", "transitive"]` to expose all of +the packages of the hub, including the transitive dependencies, which is the +same as not configuring the hub. Selecting the transitive dependencies of only +some of the groups is not supported. A tag without +{attr}`pip.configure.dependency_groups` does not limit the hub and does not +need a `pyproject.toml`. + +The `pip.configure` tags work like a builder: they are applied in order and +the tags for the same hub are combined. A `pyproject_toml` set on a tag is used +by the later tags of the same module and hub, so it only needs to be set once: + +```starlark +pip.configure( + hub_name = "pypi_hub", + pyproject_toml = "//tools:pyproject.toml", + dependency_groups = ["default"], +) + +# Uses //tools:pyproject.toml +pip.configure( + hub_name = "pypi_hub", + dependency_groups = ["group:dev"], +) +``` + +Only the root module and the module that created the hub can configure it; if +both do, the root module wins. A self reference such as `my-project[test]` in the `pyproject.toml` selects the optional dependencies of the project. The packages that are not part of the hub are ignored, so a single `pyproject.toml` can list tools that are missing from the lock file. diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 7814246e62..38ec1d0e09 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -1123,9 +1123,11 @@ execution-phase error when built. _configure_tag = tag_class( attrs = { "dependency_groups": attr.string_list( - default = ["*"], + default = [], doc = """ -The `pyproject.toml` dependency groups whose packages the hub exposes. +The `pyproject.toml` dependency groups whose packages the hub exposes. The +default, `[]`, does not select any packages, so the tag does not limit the hub +and it does not need a `pyproject.toml`. Each item is one of: @@ -1158,9 +1160,12 @@ The name of the hub to configure. "pyproject_toml": attr.label( mandatory = False, doc = """ -The `pyproject.toml` file that defines the dependency groups. Defaults to the -{attr}`pip.parse.pyproject_toml` of the hub, then to -{attr}`pip.default.pyproject_toml` if the hub is defined by the root module. +The `pyproject.toml` file that defines the dependency groups. It is also used +by the later `pip.configure` tags of the same module and hub that don't set it. +Defaults to the last `pyproject_toml` set by an earlier `pip.configure` tag of +the same module and hub, then to the {attr}`pip.parse.pyproject_toml` of the +hub, then to {attr}`pip.default.pyproject_toml` if the hub is defined by the +root module. It is only needed when {attr}`dependency_groups` is set. :::{versionadded} VERSION_NEXT_FEATURE ::: @@ -1176,8 +1181,14 @@ The packages are selected from a `pyproject.toml` file using the transitive dependencies, are still used by the packages that need them, but they are not public. +The tags work like a builder: they are applied in order and multiple tags for +the same hub are combined, so each tag can configure a different part of the +hub. A `pyproject_toml` set on a tag is used by the later tags of the same +module and hub. A tag that does not select any {attr}`dependency_groups` does +not limit the hub. + Only the root module and the module that created the hub can configure it. If -both do, the root module wins. Multiple tags for the same hub are combined. +both do, the root module wins. See {ref}`Configuring the Exposed Packages ` for examples. diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl index ed103fd2d6..7393c28fbe 100644 --- a/python/private/pypi/hub_exposure.bzl +++ b/python/private/pypi/hub_exposure.bzl @@ -30,11 +30,15 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): root_configured = {} for mod in root_modules: for tag in mod.tags.configure: - root_configured[tag.hub_name] = None + if tag.dependency_groups: + root_configured[tag.hub_name] = None configured = {} exposes_all = {} for mod in module_ctx.modules: + # The tags of a module are applied in order like a builder: a tag + # without a `pyproject_toml` uses the last one set for the hub. + last_pyproject = {} for tag in mod.tags.configure: hub = tag.hub_name if hub not in hubs: @@ -52,12 +56,19 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): if not mod.is_root and hub in root_configured: continue - pyproject = tag.pyproject_toml or info.pyproject + if tag.pyproject_toml: + last_pyproject[hub] = tag.pyproject_toml + if not tag.dependency_groups: + # The tag does not select any packages. + continue + + pyproject = last_pyproject.get(hub) or info.pyproject if not pyproject and info.module_name == root_name: pyproject = default_pyproject if not pyproject: - _fail("pip.configure() for hub '{}' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`".format( + _fail("pip.configure() for hub '{}' selects the dependency groups {}, which need a pyproject.toml, set `pyproject_toml` on this or an earlier `pip.configure` or on `pip.parse`".format( hub, + tag.dependency_groups, )) return None diff --git a/tests/pypi/extension/pip_parse.bzl b/tests/pypi/extension/pip_parse.bzl index db3008d431..d048f99bd0 100644 --- a/tests/pypi/extension/pip_parse.bzl +++ b/tests/pypi/extension/pip_parse.bzl @@ -2,7 +2,7 @@ def configure( *, - dependency_groups = ["*"], + dependency_groups = [], hub_name, pyproject_toml = None): """Simulate the PyPI extension configure tag class for testing.""" diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl index 2713b9355b..1f2b13622c 100644 --- a/tests/pypi/hub_exposure/hub_exposure_tests.bzl +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -27,7 +27,7 @@ def _hub(*, module_name = "root", pyproject = None): def _configure( hub_name = "hub", - dependency_groups = ["*"], + dependency_groups = [], pyproject_toml = None): return mocks.tag( hub_name = hub_name, @@ -108,7 +108,7 @@ _tests.append(_test_configure_combines_tags) def _test_configure_pyproject_precedence(env): # The tag wins over the `pip.parse` value. got, _ = _exposure(_module(configure = [ - _configure(pyproject_toml = "other.toml"), + _configure(pyproject_toml = "other.toml", dependency_groups = ["*"]), ])) env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) @@ -123,6 +123,59 @@ def _test_configure_pyproject_precedence(env): _tests.append(_test_configure_pyproject_precedence) +def _test_configure_without_pyproject(env): + # A tag without the dependency groups does not need a pyproject.toml and + # does not limit the hub. + got, failures = _exposure( + _module(configure = [_configure()]), + hubs = {"hub": _hub()}, + ) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({}) + +_tests.append(_test_configure_without_pyproject) + +def _test_configure_builder(env): + # The `pyproject_toml` of a tag is used by the later tags of the hub. + got, failures = _exposure( + _module(configure = [ + _configure(pyproject_toml = "other.toml"), + _configure(dependency_groups = ["default"]), + ]), + hubs = {"hub": _hub(pyproject = "pyproject.toml")}, + ) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) + + # The tags before it use the `pip.parse` value. + got, failures = _exposure( + _module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(pyproject_toml = "other.toml", dependency_groups = ["default"]), + ]), + hubs = {"hub": _hub(pyproject = "pyproject.toml")}, + ) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg", "simple"]}) + + # It is not used by the other modules. + hubs = {"hub": _hub(module_name = "dep")} + got, failures = _exposure( + _module(configure = [_configure(hub_name = "hub", pyproject_toml = "other.toml")]), + _module( + name = "dep", + is_root = False, + configure = [_configure(hub_name = "hub", dependency_groups = ["default"])], + ), + hubs = hubs, + ) + env.expect.that_str(got).equals(None) + env.expect.that_collection(failures).contains_exactly([ + "pip.configure() for hub 'hub' selects the dependency groups [\"default\"], which need a pyproject.toml, set `pyproject_toml` on this or an earlier `pip.configure` or on `pip.parse`", + ]) + +_tests.append(_test_configure_builder) + def _test_configure_module_precedence(env): hubs = {"hub": _hub(module_name = "dep", pyproject = "pyproject.toml")} @@ -174,12 +227,12 @@ def _test_configure_errors(env): ]) got, failures = _exposure( - _module(configure = [_configure()]), + _module(configure = [_configure(dependency_groups = ["*"])]), hubs = {"hub": _hub()}, ) env.expect.that_str(got).equals(None) env.expect.that_collection(failures).contains_exactly([ - "pip.configure() for hub 'hub' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`", + "pip.configure() for hub 'hub' selects the dependency groups [\"*\"], which need a pyproject.toml, set `pyproject_toml` on this or an earlier `pip.configure` or on `pip.parse`", ]) got, failures = _exposure( From 7c58d7b20a47efb5a0f33d9db412d439ad97191d Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:36:34 +0900 Subject: [PATCH 42/55] docs(pypi): rewrap the pip.configure docs --- docs/pypi/download.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/pypi/download.md b/docs/pypi/download.md index 931b64cf57..ad8b40e70d 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -238,10 +238,10 @@ pip.configure( ``` Only the root module and the module that created the hub can configure it; if -both do, the root module wins. A self reference such as `my-project[test]` in the `pyproject.toml` -selects the optional dependencies of the project. The packages that are not -part of the hub are ignored, so a single `pyproject.toml` can list tools that -are missing from the lock file. +both do, the root module wins. A self reference such as `my-project[test]` in +the `pyproject.toml` selects the optional dependencies of the project. The +packages that are not part of the hub are ignored, so a single `pyproject.toml` +can list tools that are missing from the lock file. :::{versionadded} VERSION_NEXT_FEATURE ::: From ad8c5cb2db26df3d681123932a2cb080bb13f2b4 Mon Sep 17 00:00:00 2001 From: Austin Schuh Date: Sun, 22 Nov 2026 11:11:24 +0000 Subject: [PATCH 43/55] test(windows): skip the symlink case of the cached venv test without symlinks (#4571) `test_replaces_dangling_package_link_before_creating_child_link` needs to create symlinks, so it fails on Windows without the privilege to create them, which is the case #4509 is for. Skip it there. The junction case still runs. Tested on Windows 11 26H2 with Developer Mode off: the symlink case is skipped and the other two cases pass. On `main`, the symlink case fails. The tests also pass on Linux. --- .../cached_venv_dangling_symlink_test.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/bootstrap_impls/cached_venv_dangling_symlink_test.py b/tests/bootstrap_impls/cached_venv_dangling_symlink_test.py index 363736d0a0..35f9c54bad 100644 --- a/tests/bootstrap_impls/cached_venv_dangling_symlink_test.py +++ b/tests/bootstrap_impls/cached_venv_dangling_symlink_test.py @@ -41,7 +41,17 @@ def _load_bootstrap() -> dict: return bootstrap +def _can_create_symlinks() -> bool: + with tempfile.TemporaryDirectory() as temp: + try: + os.symlink(temp, os.path.join(temp, "link"), target_is_directory=True) + except OSError: + return False + return True + + class CachedVenvDanglingSymlinkTest(unittest.TestCase): + @unittest.skipUnless(_can_create_symlinks(), "can't create symlinks") def test_replaces_dangling_package_link_before_creating_child_link(self) -> None: self._test_replaces_dangling_package_link(symlinks_supported=True) From 4ae37bdb5262492c51e9381b79e1e08287879f1a Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 22 Nov 2026 21:54:17 +0900 Subject: [PATCH 44/55] fix(pypi): forward all providers and propagate aspects in extra_dep The `extra_dep` and `extra_select` rules used the `pkg` and `actual` attributes for their dependency, so the aspects that propagate along `deps` did not reach the package, and they only forwarded a subset of the providers, e.g. `OutputGroupInfo` was dropped. Use the `deps` attribute and forward all of the providers that the targets of the whl and hub repositories have. The `extra_select` rule also prints a warning when an extra without a target is requested, which is how `pip` handles the unknown extras. --- python/private/pypi/BUILD.bazel | 1 + python/private/pypi/extra_dep.bzl | 127 ++++++--- tests/pypi/extra_dep/extra_dep_tests.bzl | 349 ++++++++++++++++++++--- tests/pypi/extra_dep/instrumented.py | 1 + 4 files changed, 398 insertions(+), 80 deletions(-) create mode 100644 tests/pypi/extra_dep/instrumented.py diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 1d83ef9ad5..45e529b5a6 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -125,6 +125,7 @@ bzl_library( "//python/private:py_cc_link_params_info", "//python/private:py_info", "//python/private:reexports", + "@bazel_skylib//rules:common_settings", ], ) diff --git a/python/private/pypi/extra_dep.bzl b/python/private/pypi/extra_dep.bzl index cee3324183..4837d6e5b1 100644 --- a/python/private/pypi/extra_dep.bzl +++ b/python/private/pypi/extra_dep.bzl @@ -2,9 +2,9 @@ The dependencies of an extra, e.g. `bar[socks]`, are only known by reading the `METADATA` of `bar`, which is only available in the whl repository of `bar`. -Instead of creating the targets for the extras in the hub, which would require -the hub to know the extras that each package provides, the dependency on an -extra is wired as follows: +The hub cannot create the `[socks]` target for every extra that `bar` +provides without fetching the wheels, so the dependency on an extra is wired +as follows: 1. The package that depends on `bar[socks]` creates an {obj}`extra_dep` target, which depends on the `_extras` target of `bar` with the @@ -16,8 +16,13 @@ extra is wired as follows: The hub only needs to forward the `_extras` target of each package to the right whl repository for the target platform. + +Both rules forward the providers of their only dependency, which is in the +`deps` attribute so that the aspects propagating along `deps` reach the +package. """ +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") load("//python/private:py_cc_link_params_info.bzl", "PyCcLinkParamsInfo") load("//python/private:py_info.bzl", "PyInfo") load("//python/private:reexports.bzl", "BuiltinPyInfo") @@ -29,8 +34,22 @@ EXTRAS_SELECT_TARGET = "_extras" # {obj}`extra_select` target. REQUESTED_EXTRA = str(Label("//python/private/pypi:requested_extra")) -def _requested_extra_impl(_ctx): - return [] +# All of the providers of the targets in the whl and hub repositories, i.e. +# `py_library` and `missing_package_error`. `DefaultInfo` is always present. +_FORWARDED_PROVIDERS = [ + p + for p in [ + PyInfo, + BuiltinPyInfo, + PyCcLinkParamsInfo, + InstrumentedFilesInfo, + OutputGroupInfo, + ] + if p != None +] + +def _requested_extra_impl(ctx): + return [BuildSettingInfo(value = ctx.build_setting_value)] requested_extra = rule( implementation = _requested_extra_impl, @@ -61,51 +80,39 @@ _reset_requested_extra = transition( outputs = [REQUESTED_EXTRA], ) -def _forward(ctx, target, attr_name): - if type(target) == type([]): - target = target[0] +def _forward(ctx): + if len(ctx.attr.deps) != 1: + fail("Expected exactly one target in 'deps', got: {}".format(ctx.attr.deps)) - default_info = target[DefaultInfo] - providers = [ - DefaultInfo( - files = default_info.files, - default_runfiles = default_info.default_runfiles, - data_runfiles = default_info.data_runfiles, - ), - coverage_common.instrumented_files_info( - ctx, - dependency_attributes = [attr_name], - ), + target = ctx.attr.deps[0] + return [target[DefaultInfo]] + [ + target[provider] + for provider in _FORWARDED_PROVIDERS + if provider in target ] - for provider in [PyInfo, BuiltinPyInfo, PyCcLinkParamsInfo]: - if provider != None and provider in target: - providers.append(target[provider]) - - return providers def _extra_dep_impl(ctx): - return _forward(ctx, ctx.attr.pkg, "pkg") + return _forward(ctx) extra_dep = rule( implementation = _extra_dep_impl, doc = """\ Depend on an extra of a PyPI package. -The `pkg` is the {obj}`extra_select` target of the package, which resolves the -`extra` to the target that includes the package and the dependencies of the -extra. - +The only target in `deps` is the {obj}`extra_select` target of the package, +which resolves the `extra` to the target that includes the package and the +dependencies of the extra. All of its providers are forwarded. """, attrs = { - "extra": attr.string( - mandatory = True, - doc = "The normalized name of the extra.", - ), - "pkg": attr.label( + "deps": attr.label_list( mandatory = True, cfg = _set_requested_extra, doc = "The {obj}`extra_select` target of the package.", ), + "extra": attr.string( + mandatory = True, + doc = "The normalized name of the extra.", + ), "_allowlist_function_transition": attr.label( default = "@bazel_tools//tools/allowlists/function_transition_allowlist", ), @@ -113,32 +120,58 @@ extra. ) def _extra_select_impl(ctx): - return _forward(ctx, ctx.attr.actual, "actual") + extra = ctx.attr._requested_extra[BuildSettingInfo].value + if extra and extra not in ctx.attr.extras: + # This is how `pip` handles the unknown extras. + # buildifier: disable=print + print("WARNING: {} does not provide the extra '{}'".format( + ctx.attr.distribution, + extra, + )) + + return _forward(ctx) extra_select = rule( implementation = _extra_select_impl, doc = """\ Forward the target of the extra that is requested by an {obj}`extra_dep`. -The `actual` is expected to be a `select` on the {obj}`REQUESTED_EXTRA` -setting. The setting is reset on the edge to `actual` so that the targets of -the extras are analyzed in the same configuration regardless of how they are -reached. - +The `deps` is expected to be a `select` on the {obj}`REQUESTED_EXTRA` +setting that results in a single target. The setting is reset on the edge to +`deps` so that the targets of the extras are analyzed in the same +configuration regardless of how they are reached. All of the providers of +the target are forwarded. """, attrs = { - "actual": attr.label( + "deps": attr.label_list( mandatory = True, cfg = _reset_requested_extra, doc = "The target for the requested extra.", ), + "distribution": attr.string( + doc = "The name and the version of the package for the warning about the unknown extras.", + ), + "extras": attr.string_list( + doc = "The normalized extras that have a target, a warning is printed if any other extra is requested.", + ), "_allowlist_function_transition": attr.label( default = "@bazel_tools//tools/allowlists/function_transition_allowlist", ), + "_requested_extra": attr.label( + default = REQUESTED_EXTRA, + ), }, ) -def extra_select_targets(*, name, targets, default, native = native, rules = None, visibility = None): +def extra_select_targets( + *, + name, + targets, + default, + distribution = "", + native = native, + rules = None, + visibility = None): """Create the {obj}`extra_select` target and the config settings for the extras. Args: @@ -148,12 +181,14 @@ def extra_select_targets(*, name, targets, default, native = native, rules = Non default: {type}`str` the label used when the requested extra is not provided by the package, which follows `pip` that only warns about unknown extras. + distribution: {type}`str` the name and the version of the package + used in the warning about the unknown extras. native: the native struct for overriding in tests. rules: a struct with the `extra_select` rule for overriding in tests. visibility: {type}`list[str]` the visibility of the target. """ rules = rules or struct(extra_select = extra_select) - actual = {} + deps = {} for extra, label in sorted(targets.items()): setting = "{}_{}".format(name, extra) native.config_setting( @@ -161,10 +196,12 @@ def extra_select_targets(*, name, targets, default, native = native, rules = Non flag_values = {REQUESTED_EXTRA: extra}, visibility = ["//visibility:private"], ) - actual[":" + setting] = label + deps[":" + setting] = [label] rules.extra_select( name = name, - actual = select(actual | {"//conditions:default": default}) if actual else default, + deps = select(deps | {"//conditions:default": [default]}) if deps else [default], + distribution = distribution, + extras = sorted(targets), visibility = visibility, ) diff --git a/tests/pypi/extra_dep/extra_dep_tests.bzl b/tests/pypi/extra_dep/extra_dep_tests.bzl index 13d28fa7a8..13adcadd3e 100644 --- a/tests/pypi/extra_dep/extra_dep_tests.bzl +++ b/tests/pypi/extra_dep/extra_dep_tests.bzl @@ -5,12 +5,15 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("@rules_testing//lib:util.bzl", rt_util = "util") load("//python:py_info.bzl", "PyInfo") load("//python:py_library.bzl", "py_library") +load("//python/private:common_labels.bzl", "labels") # buildifier: disable=bzl-visibility +load("//python/private:py_cc_link_params_info.bzl", "PyCcLinkParamsInfo") # buildifier: disable=bzl-visibility +load("//python/private/pypi:env_marker_setting.bzl", "env_marker_setting") # buildifier: disable=bzl-visibility load("//python/private/pypi:extra_dep.bzl", "extra_dep", "extra_select_targets") # buildifier: disable=bzl-visibility _tests = [] _basic_tests = [] -def _py_library(name): +def _py_library(name, **kwargs): """A py_library with a generated source to check the output configuration.""" rt_util.helper_target( native.genrule, @@ -22,35 +25,48 @@ def _py_library(name): py_library, name = name, srcs = [name + ".py"], + **kwargs ) -def _setup(name): - """Create the targets of a package `foo` that provides the `a` extra.""" - _py_library(name + "_foo") - _py_library(name + "_dep_a") - rt_util.helper_target( - py_library, - name = name + "_foo[a]", - deps = [name + "_foo", name + "_dep_a"], - ) +def _pkg(name, extras = {}): + """Create the targets of a package like the whl repositories do. + + Args: + name: the name of the package, the `_extras` target is created + for the packages that depend on its extras. + extras: the extras to the dependencies of the extra. + """ + _py_library(name) + for extra, deps in extras.items(): + rt_util.helper_target( + py_library, + name = "{}[{}]".format(name, extra), + deps = [name] + deps, + ) + extra_select_targets( name = name + "_extras", - targets = {"a": name + "_foo[a]"}, - default = name + "_foo", + targets = {extra: "{}[{}]".format(name, extra) for extra in extras}, + default = name, + distribution = name + " 1.0", visibility = ["//visibility:private"], ) -def _sources(target): - return [f.basename for f in target[PyInfo].transitive_sources.to_list()] - -def _test_extra(name): - _setup(name) +def _extra_dep(name, pkg, extra): rt_util.helper_target( extra_dep, - name = name + "_subject", - pkg = name + "_extras", - extra = "a", + name = name, + deps = [pkg + "_extras"], + extra = extra, ) + +def _sources(target): + return sorted([f.basename for f in target[PyInfo].transitive_sources.to_list()]) + +def _test_extra(name): + _py_library(name + "_dep_a") + _pkg(name + "_foo", {"a": [name + "_dep_a"]}) + _extra_dep(name + "_subject", name + "_foo", "a") analysis_test( name = name, impl = _test_extra_impl, @@ -73,13 +89,9 @@ def _test_extra_impl(env, target): _tests.append(_test_extra) def _test_unknown_extra(name): - _setup(name) - rt_util.helper_target( - extra_dep, - name = name + "_subject", - pkg = name + "_extras", - extra = "unknown", - ) + _py_library(name + "_dep_a") + _pkg(name + "_foo", {"a": [name + "_dep_a"]}) + _extra_dep(name + "_subject", name + "_foo", "unknown") analysis_test( name = name, impl = _test_unknown_extra_impl, @@ -87,13 +99,275 @@ def _test_unknown_extra(name): ) def _test_unknown_extra_impl(env, target): - # Like `pip`, the extras that the package does not provide are ignored. + # Like `pip`, the extras that the package does not provide are ignored and + # a warning is printed. env.expect.that_collection(_sources(target)).contains_exactly([ "test_unknown_extra_foo.py", ]) _tests.append(_test_unknown_extra) +def _test_no_extras(name): + _pkg(name + "_foo") + _extra_dep(name + "_subject", name + "_foo", "a") + analysis_test( + name = name, + impl = _test_no_extras_impl, + target = name + "_subject", + ) + +def _test_no_extras_impl(env, target): + env.expect.that_collection(_sources(target)).contains_exactly([ + "test_no_extras_foo.py", + ]) + +_tests.append(_test_no_extras) + +def _test_different_extras_of_the_same_package(name): + # Both `p` and `q` depend on `c`, but each of them gets only the + # dependencies of the extra that it requests. + _py_library(name + "_tomli") + _py_library(name + "_other_dep") + _pkg(name + "_c", { + "other": [name + "_other_dep"], + "toml": [name + "_tomli"], + }) + _extra_dep(name + "_p_c[toml]", name + "_c", "toml") + _extra_dep(name + "_q_c[other]", name + "_c", "other") + _py_library(name + "_p", deps = [name + "_p_c[toml]"]) + _py_library(name + "_q", deps = [name + "_q_c[other]"]) + analysis_test( + name = name, + impl = _test_different_extras_of_the_same_package_impl, + targets = { + "c": name + "_c", + "p": name + "_p", + "q": name + "_q", + }, + ) + +def _test_different_extras_of_the_same_package_impl(env, targets): + prefix = "test_different_extras_of_the_same_package_" + env.expect.that_collection(_sources(targets.p)).contains_exactly([ + prefix + "c.py", + prefix + "p.py", + prefix + "tomli.py", + ]) + env.expect.that_collection(_sources(targets.q)).contains_exactly([ + prefix + "c.py", + prefix + "other_dep.py", + prefix + "q.py", + ]) + + # The package without extras does not get the dependencies of the extras. + env.expect.that_collection(_sources(targets.c)).contains_exactly([ + prefix + "c.py", + ]) + +_tests.append(_test_different_extras_of_the_same_package) + +def _test_env_markers_of_the_extra(name): + # This is `coverage[toml]`, which requires + # `tomli; python_full_version <= "3.11.0a6" and extra == "toml"`, and + # `pytest-cov` that requires `coverage[toml]`. + _py_library(name + "_tomli") + rt_util.helper_target( + env_marker_setting, + name = name + "_include_tomli", + expression = "python_full_version <= \"3.11.0a6\"", + ) + _pkg(name + "_coverage", { + "toml": select({ + ":is_{}_include_tomli_true".format(name): [name + "_tomli"], + "//conditions:default": [], + }), + }) + _extra_dep(name + "_dep_coverage[toml]", name + "_coverage", "toml") + _py_library(name + "_pytest_cov", deps = [name + "_dep_coverage[toml]"]) + + for version, want in { + "3.10": ["coverage.py", "pytest_cov.py", "tomli.py"], + "3.12": ["coverage.py", "pytest_cov.py"], + }.items(): + analysis_test( + name = "{}_{}".format(name, version.replace(".", "_")), + impl = _test_env_markers_of_the_extra_impl, + target = name + "_pytest_cov", + config_settings = { + labels.PYTHON_VERSION: version, + }, + attr_values = { + "want": [name + "_" + w for w in want], + }, + attrs = { + "want": attr.string_list(), + }, + ) + + native.test_suite( + name = name, + tests = [ + name + "_3_10", + name + "_3_12", + ], + ) + +def _test_env_markers_of_the_extra_impl(env, target): + # The setting of the python version is kept when the setting of the extra + # is set and reset. + env.expect.that_collection(_sources(target)).contains_exactly(env.ctx.attr.want) + +_tests.append(_test_env_markers_of_the_extra) + +_ProvidersInfo = provider( + doc = "The providers of a target to compare them in the tests", + fields = ["target"], +) + +def _providers_impl(ctx): + return [_ProvidersInfo(target = ctx.attr.target)] + +_providers = rule( + implementation = _providers_impl, + attrs = {"target": attr.label()}, +) + +def _test_providers_are_forwarded(name): + _py_library(name + "_generated", data = [name + "_data.txt"]) + rt_util.helper_target( + py_library, + name = name + "_dep_a", + srcs = ["instrumented.py"], + deps = [name + "_generated"], + ) + rt_util.helper_target( + native.genrule, + name = name + "_data", + outs = [name + "_data.txt"], + cmd = "touch $@", + ) + _pkg(name + "_foo", {"a": [name + "_dep_a"]}) + _extra_dep(name + "_subject", name + "_foo", "a") + rt_util.helper_target( + _providers, + name = name + "_want", + target = name + "_foo[a]", + ) + analysis_test( + name = name, + impl = _test_providers_are_forwarded_impl, + targets = { + "subject": name + "_subject", + "want": name + "_want", + }, + config_settings = { + "//command_line_option:collect_code_coverage": True, + "//command_line_option:instrumentation_filter": "//tests/pypi/extra_dep[:/]", + }, + ) + +def _test_providers_are_forwarded_impl(env, targets): + got = targets.subject + want = targets.want[_ProvidersInfo].target + + for provider in [PyInfo, PyCcLinkParamsInfo, InstrumentedFilesInfo, OutputGroupInfo]: + env.expect.where(provider = str(provider)).that_bool(provider in got).equals(True) + + env.expect.that_bool(got[PyInfo] == want[PyInfo]).equals(True) + env.expect.that_bool(got[PyCcLinkParamsInfo] == want[PyCcLinkParamsInfo]).equals(True) + + # The output groups of the target. + env.expect.that_collection(dir(got[OutputGroupInfo])).contains_exactly(dir(want[OutputGroupInfo])) + for group in dir(want[OutputGroupInfo]): + env.expect.where(group = group).that_depset_of_files( + getattr(got[OutputGroupInfo], group), + ).contains_exactly([ + f.short_path + for f in getattr(want[OutputGroupInfo], group).to_list() + ]) + + # The files and the runfiles of the target. + env.expect.that_depset_of_files(got[DefaultInfo].files).contains_exactly([ + f.short_path + for f in want[DefaultInfo].files.to_list() + ]) + for runfiles in ["default_runfiles", "data_runfiles"]: + got_files = sorted([f.short_path for f in getattr(got[DefaultInfo], runfiles).files.to_list()]) + want_files = sorted([f.short_path for f in getattr(want[DefaultInfo], runfiles).files.to_list()]) + env.expect.where(runfiles = runfiles).that_collection(got_files).contains_exactly(want_files) + env.expect.where(runfiles = runfiles).that_collection(got_files).contains( + "tests/pypi/extra_dep/test_providers_are_forwarded_data.txt", + ) + + # The coverage is collected for the sources of the extra. + got_instrumented = sorted([f.short_path for f in got[InstrumentedFilesInfo].instrumented_files.to_list()]) + env.expect.that_collection(got_instrumented).contains_exactly([ + f.short_path + for f in want[InstrumentedFilesInfo].instrumented_files.to_list() + ]) + env.expect.that_collection(got_instrumented).contains( + "tests/pypi/extra_dep/instrumented.py", + ) + +_tests.append(_test_providers_are_forwarded) + +_VisitedInfo = provider( + doc = "The targets visited by an aspect that propagates along the `deps`", + fields = ["labels"], +) + +def _deps_aspect_impl(target, ctx): + return [_VisitedInfo(labels = depset( + [target.label.name], + transitive = [ + dep[_VisitedInfo].labels + for dep in getattr(ctx.rule.attr, "deps", []) + if _VisitedInfo in dep + ], + ))] + +_deps_aspect = aspect( + implementation = _deps_aspect_impl, + attr_aspects = ["deps"], +) + +def _with_deps_aspect_impl(ctx): + return [ctx.attr.target[_VisitedInfo]] + +_with_deps_aspect = rule( + implementation = _with_deps_aspect_impl, + attrs = {"target": attr.label(aspects = [_deps_aspect])}, +) + +def _test_aspects_propagate_along_deps(name): + _py_library(name + "_dep_a") + _pkg(name + "_foo", {"a": [name + "_dep_a"]}) + _extra_dep(name + "_dep_foo[a]", name + "_foo", "a") + _py_library(name + "_consumer", deps = [name + "_dep_foo[a]"]) + rt_util.helper_target( + _with_deps_aspect, + name = name + "_subject", + target = name + "_consumer", + ) + analysis_test( + name = name, + impl = _test_aspects_propagate_along_deps_impl, + target = name + "_subject", + ) + +def _test_aspects_propagate_along_deps_impl(env, target): + prefix = "test_aspects_propagate_along_deps_" + env.expect.that_collection(target[_VisitedInfo].labels.to_list()).contains_exactly([ + prefix + "consumer", + prefix + "dep_foo[a]", + prefix + "foo_extras", + prefix + "foo[a]", + prefix + "foo", + prefix + "dep_a", + ]) + +_tests.append(_test_aspects_propagate_along_deps) + def _test_extra_select_targets(env): config_settings = [] extra_selects = [] @@ -101,7 +375,8 @@ def _test_extra_select_targets(env): extra_select_targets( name = "_extras", targets = {"a": ":[a]", "b": ":[b]"}, - default = ":pkg", + default = ":[]", + distribution = "foo 1.0", native = struct(config_setting = lambda **kwargs: config_settings.append(kwargs)), rules = struct(extra_select = lambda **kwargs: extra_selects.append(kwargs)), ) @@ -121,11 +396,13 @@ def _test_extra_select_targets(env): env.expect.that_collection(extra_selects).contains_exactly([ { "name": "_extras", - "actual": select({ - ":_extras_a": ":[a]", - ":_extras_b": ":[b]", - "//conditions:default": ":pkg", + "deps": select({ + ":_extras_a": [":[a]"], + ":_extras_b": [":[b]"], + "//conditions:default": [":[]"], }), + "distribution": "foo 1.0", + "extras": ["a", "b"], "visibility": None, }, ]) # buildifier: @unsorted-dict-items @@ -138,7 +415,7 @@ def _test_extra_select_targets_no_extras(env): extra_select_targets( name = "_extras", targets = {}, - default = ":pkg", + default = ":[]", native = struct(), rules = struct(extra_select = lambda **kwargs: extra_selects.append(kwargs)), visibility = ["//visibility:public"], @@ -147,7 +424,9 @@ def _test_extra_select_targets_no_extras(env): env.expect.that_collection(extra_selects).contains_exactly([ { "name": "_extras", - "actual": ":pkg", + "deps": [":[]"], + "distribution": "", + "extras": [], "visibility": ["//visibility:public"], }, ]) # buildifier: @unsorted-dict-items diff --git a/tests/pypi/extra_dep/instrumented.py b/tests/pypi/extra_dep/instrumented.py new file mode 100644 index 0000000000..8667111328 --- /dev/null +++ b/tests/pypi/extra_dep/instrumented.py @@ -0,0 +1 @@ +"""A source file for the tests, only the source files are instrumented.""" From 4fb97d1da4515660dcf6f7631c0ae19d6563059d Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 22 Nov 2026 21:54:51 +0900 Subject: [PATCH 45/55] feat(pypi): resolve the extras of the dependencies in the whl repos A dependency on an extra of another package, e.g. `pytest-cov` requiring `coverage[toml]`, used the `[toml]` target of the hub, which only exists if the lock file requests the extra, and failed the build otherwise. The lock files often do not have the extras, e.g. `pip-compile` strips them by default, even though the packages of the extras are locked. The hub cannot create the targets for all of the extras, because the extras that a package provides are only known from its `METADATA`. So the whl repository of the dependent package creates an `extra_dep` target for each such dependency and the whl repository of the dependency selects its `[]` target in the `_extras` target, which the hub forwards to the whl repository of the target platform like the other targets. The dependencies of the extra that are not in the hub are left out like all of the other dependencies that are not in the hub, and an extra that the dependency does not provide is the same as the dependency without extras, a warning is printed for it like `pip` does. --- python/private/pypi/BUILD.bazel | 3 +- python/private/pypi/hub_repository.bzl | 5 +- python/private/pypi/pip_repository.bzl | 6 +- python/private/pypi/pkg_aliases.bzl | 12 +- python/private/pypi/render_pkg_aliases.bzl | 2 +- .../private/pypi/whl_library_deps_targets.bzl | 73 ++- tests/pypi/pkg_aliases/pkg_aliases_test.bzl | 35 ++ tests/pypi/whl_library/BUILD.bazel | 4 +- .../consumer/consumer-1.0.dist-info/METADATA | 1 + tests/pypi/whl_library/testdata/packages.bzl | 4 +- .../whl_library_deps_targets_tests.bzl | 464 +++++++++++++++--- 11 files changed, 516 insertions(+), 93 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 45e529b5a6..e292717746 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -386,6 +386,7 @@ bzl_library( name = "pkg_aliases", srcs = ["pkg_aliases.bzl"], deps = [ + ":extra_dep", ":labels", ":whl_library_deps_targets", "//python:py_library", @@ -643,8 +644,8 @@ bzl_library( srcs = ["whl_library_deps_targets.bzl"], deps = [ ":env_marker_setting", + ":extra_dep", ":labels", - ":missing_package", ":pep508_deps", "//python:py_library", "//python/private:normalize_name", diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index 27fa5fdece..ec43b0c3fa 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -57,8 +57,9 @@ def _impl(rctx): "config.bzl", rctx.attr._config_template, substitutions = { - # The available extras of each package for the spoke repositories - # to check if the extras of their dependencies are available. + # The requested extras of each package for the spoke repositories + # to create the targets of the extras that the package does not + # provide. "%%PACKAGES%%": render.dict(available_extras(whl_map), value_repr = render.list), }, ) diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index a692296683..5fe8b6c1c7 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -286,9 +286,9 @@ def _pip_repository_impl(rctx): "config.bzl", rctx.attr._config_template, substitutions = { - # The available extras of each package for the spoke - # repositories to check if the extras of their dependencies - # are available. + # The requested extras of each package for the spoke + # repositories to create the targets of the extras that the + # package does not provide. "%%PACKAGES%%": render.dict({ pkg: sorted(selected_extras.get(pkg, [])) for pkg in bzl_packages diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index d64bc7a15f..a1fdfe72a9 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -35,6 +35,7 @@ load("@bazel_skylib//lib:selects.bzl", "selects") load("//python:py_library.bzl", "py_library") load("//python/private:common_labels.bzl", "labels") load("//python/private:text_util.bzl", "render") +load(":extra_dep.bzl", "EXTRAS_SELECT_TARGET", "extra_dep", "extra_select") load( ":labels.bzl", "DATA_LABEL", @@ -124,6 +125,8 @@ def pkg_aliases( _native = kwargs.pop("native", native) rules = kwargs.pop("rules", struct( + extra_dep = extra_dep, + extra_select = extra_select, py_library = py_library, whl_library_deps_targets = whl_library_deps_targets, )) @@ -157,6 +160,10 @@ def pkg_aliases( # targets do not use the extras, so only the package without extras # can be a member of the group. target_names = { + # The whl repositories resolve the extras requested by the + # packages that depend on them, so the hub only needs to forward + # to the whl repository of the target platform. + EXTRAS_SELECT_TARGET: EXTRAS_SELECT_TARGET, WHEEL_FILE_PUBLIC_LABEL: WHEEL_FILE_IMPL_LABEL if "" in group_extras else WHEEL_FILE_PUBLIC_LABEL, } @@ -214,7 +221,10 @@ def pkg_aliases( fail("The `actual` arg must be a dictionary or a string") kwargs = {} - if target_name.startswith("_"): + if target_name == EXTRAS_SELECT_TARGET: + # Used by the whl repositories that depend on the extras. + pass + elif target_name.startswith("_"): kwargs["visibility"] = ["//_groups:__subpackages__"] elif target_name in (WHEEL_FILE, PY_SRCS_LABEL): kwargs["visibility"] = ["//visibility:private"] diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 9ba55f3a75..bc0d98cdfe 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -313,7 +313,7 @@ def _non_versioned_platform(p, *, strict = False): fail("Expected to always have a platform in the form '{{abi}}_{{os}}_{{arch}}', got: {}".format(p)) def available_extras(aliases): - """Return the extras that are available in the hub for each package. + """Return the extras that are requested in the hub for each package. Args: aliases: {type}`dict[str, dict[struct, str]]` the packages to the diff --git a/python/private/pypi/whl_library_deps_targets.bzl b/python/private/pypi/whl_library_deps_targets.bzl index 8b633e6c99..85d5a80797 100644 --- a/python/private/pypi/whl_library_deps_targets.bzl +++ b/python/private/pypi/whl_library_deps_targets.bzl @@ -3,6 +3,7 @@ load("//python:py_library.bzl", "py_library") load("//python/private:normalize_name.bzl", "normalize_name") load(":env_marker_setting.bzl", "env_marker_setting") +load(":extra_dep.bzl", "EXTRAS_SELECT_TARGET", "extra_dep", "extra_select", "extra_select_targets") load( ":labels.bzl", "DATA_LABEL", @@ -15,7 +16,6 @@ load( "extra_label", "group_member", ) -load(":missing_package.bzl", "missing_extra_error") load(":pep508_deps.bzl", "deps") def whl_library_deps_targets( @@ -37,7 +37,8 @@ def whl_library_deps_targets( rules = struct( py_library = py_library, env_marker_setting = env_marker_setting, - missing_extra_error = missing_extra_error, + extra_dep = extra_dep, + extra_select = extra_select, )): """Create all of the whl_library targets. @@ -54,6 +55,9 @@ def whl_library_deps_targets( provides_extra: {type}`list[str]` The list of extras that the package provides. For each extra the `[]` target is created that includes the package and the dependencies of that extra. + The `_extras` target selects one of them for the + {obj}`extra_dep` targets of the packages that depend on an extra + of this package. extras: {type}`list[str]` The list of requested extras. The `pkg` target includes them, which is how the extras are used when there is no hub repository to select them, e.g. when using a vendored @@ -64,11 +68,9 @@ def whl_library_deps_targets( `pip` does. include: {type}`list[str] | dict[str, list[str] | None]` The list of packages to include. If it is a dict, then the values are the - extras of each package that are available in the hub and the - dependencies on the extras that are not available will fail the - build with a helpful message. The `[]` targets are also - created for the extras of this package that are available in - the hub, see `extras` for the ones that the package does not + extras of each package that are requested in the hub. The + `[]` targets are also created for the requested extras of + this package, see `extras` for the ones that the package does not provide. group_name: {type}`str | None` name of the dependency group (if any). dep_template: {type}`str | None` The dep_template to use. @@ -156,13 +158,14 @@ def whl_library_deps_targets( if normalize_name(e) not in provided_extras }) extras_targets = sorted(provided_extras.keys() + unknown_extras) + distribution = " ".join([metadata_name] + [ + t.partition("=")[2] + for t in tags + if t.startswith("pypi_version=") + ]) for extra in unknown_extras: getattr(rules, "warn", _warn)("WARNING: {} does not provide the extra '{}'".format( - " ".join([metadata_name] + [ - t.partition("=")[2] - for t in tags - if t.startswith("pypi_version=") - ]), + distribution, extra, )) whl_file_label = _name("", WHEEL_FILE_PUBLIC_LABEL) @@ -210,6 +213,28 @@ def whl_library_deps_targets( visibility = visibility if group_label or not requires_dist else impl_vis, ) + def _extra_target(extra): + if group_name and not group_label and is_member(extra): + # The group library of the members is in the hub. + return dep_template.format(name = self_name, target = extra_label(extra)) + + return ":" + extra_label(extra) + + # The packages that depend on an extra of this package use this target + # with the requested extra, because the hub does not know which extras + # this package provides. An extra without a target is the same as the + # package without extras and a warning is printed when it is requested. + if hasattr(rules, "extra_select"): + extra_select_targets( + name = EXTRAS_SELECT_TARGET, + targets = {extra: _extra_target(extra) for extra in extras_targets}, + default = _extra_target(""), + distribution = distribution, + native = native, + rules = rules, + visibility = visibility, + ) + if not requires_dist: # If there are extras, then they will be visible in requires_dist. return @@ -218,21 +243,18 @@ def whl_library_deps_targets( # dep and the expression. settings = {} - # The dependencies on extras that are not available in the hub. - missing_extras = {} + # The dependencies on the extras of other packages. + extra_deps = {} def _pkg_label(dep): dep_name, _, extra = dep.partition("[") if not extra: return dep_template.format(name = dep_name, target = PY_LIBRARY_PUBLIC_LABEL) - extra = extra.rstrip("]") - available = include.get(dep_name) if type(include) == type({}) else None - if available != None and extra not in available: - missing_extras[dep] = (dep_name, extra) - return ":_missing_" + dep - - return dep_template.format(name = dep_name, target = extra_label(extra)) + # The extra does not need to be requested in the hub, so it is + # resolved by the package that provides it. + extra_deps[dep] = (dep_name, extra.rstrip("]")) + return ":_" + dep whl_label = lambda dep: dep_template.format(name = dep, target = WHEEL_FILE_PUBLIC_LABEL) @@ -295,12 +317,11 @@ def whl_library_deps_targets( visibility = ["//visibility:private"], ) - for dep, (dep_name, extra) in sorted(missing_extras.items()): - rules.missing_extra_error( - name = "_missing_" + dep, - package_name = dep_name, + for dep, (dep_name, extra) in sorted(extra_deps.items()): + rules.extra_dep( + name = "_" + dep, + deps = [dep_template.format(name = dep_name, target = EXTRAS_SELECT_TARGET)], extra = extra, - required_by = metadata_name, visibility = ["//visibility:private"], ) diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index b1ff1bf698..4e26210c6a 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -43,6 +43,7 @@ def _test_legacy_aliases(env): "foo": ":pkg", "pkg": ":[]", "[]": "@repo//:[]", + "_extras": "@repo//:_extras", "srcs": "@repo//:srcs", "whl_file": "@repo//:whl_file", "whl": "@repo//:whl", @@ -56,6 +57,36 @@ def _test_legacy_aliases(env): _tests.append(_test_legacy_aliases) +def _test_extras_select_aliases(env): + got = {} + pkg_aliases( + name = "coverage", + actual = { + whl_config_setting(version = "3.10"): "coverage_310", + whl_config_setting(version = "3.12"): "coverage_312", + }, + native = struct( + alias = lambda *, name, actual, visibility = None, tags = None: got.update({name: (actual, visibility)}), + ), + select = lambda value, no_match_error = None: value, + ) + + # The hub does not know the extras that the package provides, so the + # packages that depend on an extra use the `_extras` target of the whl + # repository for the target platform. + env.expect.that_dict(got).contains_at_least({ + "_extras": ( + { + "//_config:is_cp310": "@coverage_310//:_extras", + "//_config:is_cp312": "@coverage_312//:_extras", + "//conditions:default": "_no_matching_repository", + }, + None, + ), + }) + +_tests.append(_test_extras_select_aliases) + def _test_config_setting_aliases(env): # Use this function as it is used in pip_repository got = {} @@ -221,6 +252,10 @@ def _test_group_aliases(env): "name": "foo", "actual": ":pkg", }, + { + "name": "_extras", + "actual": "@repo//:_extras", + }, { "name": "_pkg", "actual": "@repo//:[]", diff --git a/tests/pypi/whl_library/BUILD.bazel b/tests/pypi/whl_library/BUILD.bazel index 51e894693d..e7b012ebb3 100644 --- a/tests/pypi/whl_library/BUILD.bazel +++ b/tests/pypi/whl_library/BUILD.bazel @@ -40,7 +40,9 @@ py_test( srcs = ["whl_library_transitive_extras_test.py"], target_compatible_with = SUPPORTS_BZLMOD, deps = [ - # The METADATA of the consumer requires `pkg[optional]`. + # The METADATA of the consumer requires `pkg[optional]`, which is not + # requested in the hub, and `pkg[unknown]`, which `pkg` does not + # provide. "@whl_library_extras_consumer//:pkg", ], ) diff --git a/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA b/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA index 9a723c958a..e2cbed2ee5 100644 --- a/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA +++ b/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA @@ -1,3 +1,4 @@ Name: consumer Version: 1.0 Requires-Dist: pkg[optional] +Requires-Dist: pkg[unknown] diff --git a/tests/pypi/whl_library/testdata/packages.bzl b/tests/pypi/whl_library/testdata/packages.bzl index 627c3a84c2..812dc40f2f 100644 --- a/tests/pypi/whl_library/testdata/packages.bzl +++ b/tests/pypi/whl_library/testdata/packages.bzl @@ -1,12 +1,12 @@ """A list of packages that this logical testdata hub repo contains.""" # The values are the extras that are requested in the hub, the `pkg` package -# does not provide the `nope` extra. +# does not provide the `nope` extra. The `optional` extra of `pkg` is not +# requested in the hub, but the `consumer` package depends on it. packages = { "consumer": [], "optional_dep": [], "pkg": [ "nope", - "optional", ], } diff --git a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl index 12b1f9fa7c..7347ff0743 100644 --- a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl +++ b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl @@ -175,24 +175,25 @@ def _test_extras(env): rules = struct( py_library = lambda **kwargs: py_library_calls.append(kwargs), env_marker_setting = lambda **kwargs: env_marker_setting_calls.append(kwargs), + extra_dep = lambda **kwargs: None, ), ) # The `pkg` target has no extras because none were requested and there is # a target for each extra that the package provides. The `whl` target does - # not use the extras. Without the extras available in the hub, the - # dependencies use the targets with the extras directly. + # not use the extras. The dependencies on the extras of the other packages + # use the `extra_dep` targets. env.expect.that_collection([ (c["name"], c["deps"]) for c in py_library_calls ]).contains_exactly([ - ("pkg", ["srcs", "@pypi//bar:[socks]"]), - ("[all]", ["srcs", "@pypi//bar:[socks]", "@pypi//grpc:pkg"]), - ("[async]", ["srcs", "@pypi//bar:[socks]", "@pypi//uvloop:pkg"] + select({ + ("pkg", ["srcs", ":_bar[socks]"]), + ("[all]", ["srcs", ":_bar[socks]", "@pypi//grpc:pkg"]), + ("[async]", ["srcs", ":_bar[socks]", "@pypi//uvloop:pkg"] + select({ ":is_include_aiohttp_true": ["@pypi//aiohttp:pkg"], "//conditions:default": [], })), - ("[grpc]", ["srcs", "@pypi//bar:[socks]", "@pypi//grpc:pkg"]), + ("[grpc]", ["srcs", ":_bar[socks]", "@pypi//grpc:pkg"]), ]) env.expect.that_collection([ (c["name"], c["data"]) @@ -538,89 +539,440 @@ def _test_extras_in_group_legacy(env): _tests.append(_test_extras_in_group_legacy) -def _test_deps_with_extras(env): - filegroup_calls = [] - py_library_calls = [] - env_marker_setting_calls = [] - missing_calls = [] +def _targets(**kwargs): + """Create the targets and return the calls to the rules. + + Args: + **kwargs: the args for the `whl_library_deps_targets`. + + Returns: + A struct with the `py_library` deps and the `whl` data by the target + name and the calls to the other rules. + """ + calls = struct( + config_setting = [], + env_marker_setting = [], + extra_dep = [], + extra_select = [], + filegroup = [], + py_library = [], + ) whl_library_deps_targets( - name = "foo-0-py3-none-any.whl", + name = "{}-1.0-py3-none-any.whl".format(kwargs["metadata_name"]), + dep_template = kwargs.pop("dep_template", "@pypi//{name}:{target}"), + repo = None, + aliases = {}, + native = struct( + alias = lambda **kwargs: None, + config_setting = lambda **kwargs: calls.config_setting.append(kwargs), + filegroup = lambda **kwargs: calls.filegroup.append(kwargs), + ), + rules = struct( + env_marker_setting = lambda **kwargs: calls.env_marker_setting.append(kwargs), + extra_dep = lambda **kwargs: calls.extra_dep.append(kwargs), + extra_select = lambda **kwargs: calls.extra_select.append(kwargs), + py_library = lambda **kwargs: calls.py_library.append(kwargs), + ), + **kwargs + ) + + return struct( + deps = {c["name"]: c["deps"] for c in calls.py_library}, + whl_data = {c["name"]: c["data"] for c in calls.filegroup}, + env_markers = {c["name"]: c["expression"] for c in calls.env_marker_setting}, + extra_deps = calls.extra_dep, + extra_selects = calls.extra_select, + config_settings = [c["name"] for c in calls.config_setting], + ) + +def _test_deps_with_extras(env): + got = _targets( metadata_name = "foo", - dep_template = "@pypi//{name}:{target}", requires_dist = [ "bar[socks]", "baz[a,b]; sys_platform == 'linux'", "baz; sys_platform == 'darwin'", - "qux[missing]", + "qux[not-requested]", ], - # The values are the extras that are available in the hub + # The values are the extras that are requested in the hub, which are + # not needed for the extras of the dependencies. include = { "bar": ["socks"], "baz": ["a", "b"], "qux": [], }, - repo = None, - aliases = {}, - native = struct( - filegroup = lambda **kwargs: filegroup_calls.append(kwargs), - alias = lambda **kwargs: None, - ), - rules = struct( - py_library = lambda **kwargs: py_library_calls.append(kwargs), - env_marker_setting = lambda **kwargs: env_marker_setting_calls.append(kwargs), - missing_extra_error = lambda **kwargs: missing_calls.append(kwargs), - ), ) - # The dependencies use the targets with the extras, the missing extras use - # a target that fails the build with a helpful message. - env.expect.that_collection([ - (c["name"], c["deps"]) - for c in py_library_calls - ]).contains_exactly([ - ("pkg", ["srcs", "@pypi//bar:[socks]", ":_missing_qux[missing]"] + select({ + # The extras of the dependencies are resolved by the packages that provide + # them, so they are used via the `extra_dep` targets. + env.expect.that_dict(got.deps).contains_exactly({ + "pkg": ["srcs", ":_bar[socks]", ":_qux[not_requested]"] + select({ ":is_include_baz_2_true": ["@pypi//baz:pkg"], "//conditions:default": [], }) + select({ - ":is_include_baz[a]_true": ["@pypi//baz:[a]"], + ":is_include_baz[a]_true": [":_baz[a]"], "//conditions:default": [], }) + select({ - ":is_include_baz[b]_true": ["@pypi//baz:[b]"], + ":is_include_baz[b]_true": [":_baz[b]"], "//conditions:default": [], - })), - ]) - env.expect.that_collection(missing_calls).contains_exactly([ + }), + }) + env.expect.that_collection(got.extra_deps).contains_exactly([ { - "name": "_missing_qux[missing]", - "package_name": "qux", - "extra": "missing", - "required_by": "foo", + "name": "_" + dep, + "deps": ["@pypi//{}:_extras".format(name)], + "extra": extra, "visibility": ["//visibility:private"], - }, + } + for dep, name, extra in [ + ("bar[socks]", "bar", "socks"), + ("baz[a]", "baz", "a"), + ("baz[b]", "baz", "b"), + ("qux[not_requested]", "qux", "not_requested"), + ] ]) # buildifier: @unsorted-dict-items # The whl targets do not use the extras. - env.expect.that_collection([ - (c["name"], c["data"]) - for c in filegroup_calls - ]).contains_exactly([ - ("whl", ["@pypi//bar:whl", "@pypi//qux:whl"] + select({ + env.expect.that_dict(got.whl_data).contains_exactly({ + "whl": ["@pypi//bar:whl", "@pypi//qux:whl"] + select({ ":is_include_baz_true": ["@pypi//baz:whl"], "//conditions:default": [], - })), + }), + }) + env.expect.that_dict(got.env_markers).contains_exactly({ + "include_baz": "(sys_platform == \"darwin\") or (sys_platform == \"linux\")", + "include_baz[a]": "sys_platform == \"linux\"", + "include_baz[b]": "sys_platform == \"linux\"", + "include_baz_2": "sys_platform == \"darwin\"", + }) + + # The package does not provide any extras. + env.expect.that_collection(got.config_settings).contains_exactly([]) + env.expect.that_collection(got.extra_selects).contains_exactly([ + { + "name": "_extras", + "deps": [":[]"], + "distribution": "foo", + "extras": [], + "visibility": ["//visibility:public"], + }, + ]) # buildifier: @unsorted-dict-items + +_tests.append(_test_deps_with_extras) + +_COVERAGE_REQUIRES_DIST = [ + "tomli; python_full_version <= \"3.11.0a6\" and extra == \"toml\"", +] + +def _test_extra_that_is_not_requested_in_the_hub(env): + # The lock file has `coverage` without the `toml` extra, e.g. because the + # extras are stripped when compiling the requirements. + include = {"coverage": [], "pytest_cov": [], "tomli": []} + + got = _targets( + metadata_name = "pytest-cov", + requires_dist = ["coverage[toml]>=7.5", "pytest>=7"], + include = include, + ) + env.expect.that_dict(got.deps).contains_exactly({ + "pkg": ["srcs", ":_coverage[toml]"], + }) + env.expect.that_collection(got.extra_deps).contains_exactly([ + { + "name": "_coverage[toml]", + "deps": ["@pypi//coverage:_extras"], + "extra": "toml", + "visibility": ["//visibility:private"], + }, + ]) # buildifier: @unsorted-dict-items + + got = _targets( + metadata_name = "coverage", + requires_dist = _COVERAGE_REQUIRES_DIST, + provides_extra = ["toml"], + include = include, + tags = ["pypi_name=coverage", "pypi_version=7.6.1"], + ) + + # Only the target of the extra has `tomli` and only for the Python + # versions that need it, which is evaluated for each target platform. + env.expect.that_dict(got.deps).contains_exactly({ + "[toml]": ["srcs"] + select({ + ":is_include_tomli_true": ["@pypi//tomli:pkg"], + "//conditions:default": [], + }), + "pkg": ["srcs"], + }) + env.expect.that_dict(got.env_markers).contains_exactly({ + "include_tomli": "python_full_version <= \"3.11.0a6\"", + }) + env.expect.that_collection(got.config_settings).contains_exactly(["_extras_toml"]) + env.expect.that_collection(got.extra_selects).contains_exactly([ + { + "name": "_extras", + "deps": select({ + ":_extras_toml": [":[toml]"], + "//conditions:default": [":[]"], + }), + "distribution": "coverage 7.6.1", + "extras": ["toml"], + "visibility": ["//visibility:public"], + }, + ]) # buildifier: @unsorted-dict-items + +_tests.append(_test_extra_that_is_not_requested_in_the_hub) + +def _test_deps_of_the_extra_that_are_not_in_the_hub(env): + # `tomli` is not locked, e.g. because the lock file is only for the Python + # versions that do not need it. Like the other dependencies that are not + # in the hub, it is dropped. + got = _targets( + metadata_name = "coverage", + requires_dist = _COVERAGE_REQUIRES_DIST, + provides_extra = ["toml"], + include = {"coverage": [], "pytest_cov": []}, + ) + env.expect.that_dict(got.deps).contains_exactly({ + "[toml]": ["srcs"], + "pkg": ["srcs"], + }) + env.expect.that_dict(got.env_markers).contains_exactly({}) + + # The same for the dependencies of the extra without any markers. + got = _targets( + metadata_name = "bar", + requires_dist = ["baz; extra == \"x\"", "qux"], + provides_extra = ["x"], + include = {"bar": [], "foo": [], "qux": []}, + ) + env.expect.that_dict(got.deps).contains_exactly({ + "[x]": ["srcs", "@pypi//qux:pkg"], + "pkg": ["srcs", "@pypi//qux:pkg"], + }) + + # Without any other dependencies the targets are aliases and the extra can + # still be selected. + got = _targets( + metadata_name = "bar", + provides_extra = ["x"], + include = {"bar": [], "foo": []}, + ) + env.expect.that_dict(got.deps).contains_exactly({}) + env.expect.that_collection([ + (c["name"], c["extras"]) + for c in got.extra_selects + ]).contains_exactly([("_extras", ["x"])]) + +_tests.append(_test_deps_of_the_extra_that_are_not_in_the_hub) + +def _test_dep_with_extra_that_is_not_provided(env): + include = {"bar": [], "foo": []} + got = _targets( + metadata_name = "foo", + requires_dist = ["bar[nope]"], + include = include, + ) + + # The package does not know the extras of its dependencies. + env.expect.that_dict(got.deps).contains_exactly({ + "pkg": ["srcs", ":_bar[nope]"], + }) + env.expect.that_collection([ + (c["deps"], c["extra"]) + for c in got.extra_deps + ]).contains_exactly([(["@pypi//bar:_extras"], "nope")]) + + # The `nope` extra is not in the `extras` of the `_extras` target, so it + # selects the package without extras and prints a warning. + got = _targets( + metadata_name = "bar", + requires_dist = ["baz; extra == \"x\""], + provides_extra = ["x"], + include = include, + ) + env.expect.that_collection([ + (c["deps"], c["extras"]) + for c in got.extra_selects + ]).contains_exactly([ + ( + select({ + ":_extras_x": [":[x]"], + "//conditions:default": [":[]"], + }), + ["x"], + ), + ]) + +_tests.append(_test_dep_with_extra_that_is_not_provided) + +def _test_self_referencing_extras_are_selected(env): + got = _targets( + metadata_name = "foo", + requires_dist = [ + "foo[a,b]; extra == \"all\"", + "dep-a; extra == \"a\"", + "dep-b; extra == \"b\"", + ], + provides_extra = ["a", "all", "b"], + include = {"dep_a": [], "dep_b": [], "foo": []}, + ) + + # A dependency on `foo[all]` gets the dependencies of both extras and a + # dependency on `foo[a]` only gets the ones of `a`. + env.expect.that_dict(got.deps).contains_exactly({ + "[a]": ["srcs", "@pypi//dep_a:pkg"], + "[all]": ["srcs", "@pypi//dep_a:pkg", "@pypi//dep_b:pkg"], + "[b]": ["srcs", "@pypi//dep_b:pkg"], + "pkg": ["srcs"], + }) + env.expect.that_collection([c["deps"] for c in got.extra_selects]).contains_exactly([ + select({ + ":_extras_a": [":[a]"], + ":_extras_all": [":[all]"], + ":_extras_b": [":[b]"], + "//conditions:default": [":[]"], + }), ]) + env.expect.that_collection(got.extra_deps).contains_exactly([]) + +_tests.append(_test_self_referencing_extras_are_selected) + +def _test_cycle_through_extras(env): + # `a[x]` requires `b[y]` and `b[y]` requires `a[x]`. Each package only + # reads its own `METADATA`, so nothing is resolved recursively here. + a = dict( + metadata_name = "a", + requires_dist = ["b[y]; extra == \"x\""], + provides_extra = ["x"], + include = {"a": [], "b": []}, + ) + b = dict( + metadata_name = "b", + requires_dist = ["a[x]; extra == \"y\""], + provides_extra = ["y"], + include = {"a": [], "b": []}, + ) + + # Without a group this is a dependency cycle between the targets like any + # other cycle between the packages. + env.expect.that_dict(_targets(**a).deps).contains_exactly({ + "[x]": ["srcs", ":_b[y]"], + "pkg": ["srcs"], + }) + env.expect.that_dict(_targets(**b).deps).contains_exactly({ + "[y]": ["srcs", ":_a[x]"], + "pkg": ["srcs"], + }) + + # With the extras as the members of a group the cycle is broken and the + # `_extras` targets select the group library of the hub. + group = dict(group_name = "cycle", group_deps = ["a[x]", "b[y]"]) + got = _targets(**(a | group)) + env.expect.that_dict(got.deps).contains_exactly({ + "[x]": ["srcs"], + "pkg": ["srcs"], + }) + env.expect.that_collection(got.extra_deps).contains_exactly([]) + env.expect.that_collection([c["deps"] for c in got.extra_selects]).contains_exactly([ + select({ + ":_extras_x": ["@pypi//a:[x]"], + "//conditions:default": [":[]"], + }), + ]) + got = _targets(**(b | group)) + env.expect.that_dict(got.deps).contains_exactly({ + "[y]": ["srcs"], + "pkg": ["srcs"], + }) + env.expect.that_collection(got.extra_deps).contains_exactly([]) + +_tests.append(_test_cycle_through_extras) + +def _test_different_extras_of_the_same_dep(env): + include = {"c": [], "foo_bar": [], "other_dep": [], "p": [], "q": [], "r": [], "tomli": []} + + # Each package only depends on the extra that it requires. + got = _targets(metadata_name = "p", requires_dist = ["c[toml]"], include = include) + env.expect.that_dict(got.deps).contains_exactly({"pkg": ["srcs", ":_c[toml]"]}) + env.expect.that_collection([ + (c["name"], c["deps"], c["extra"]) + for c in got.extra_deps + ]).contains_exactly([("_c[toml]", ["@pypi//c:_extras"], "toml")]) + + got = _targets(metadata_name = "q", requires_dist = ["c[other]"], include = include) + env.expect.that_dict(got.deps).contains_exactly({"pkg": ["srcs", ":_c[other]"]}) env.expect.that_collection([ - (c["name"], c["expression"]) - for c in env_marker_setting_calls + (c["name"], c["deps"], c["extra"]) + for c in got.extra_deps + ]).contains_exactly([("_c[other]", ["@pypi//c:_extras"], "other")]) + + got = _targets( + metadata_name = "c", + requires_dist = ["tomli; extra == \"toml\"", "other-dep; extra == \"other\""], + provides_extra = ["toml", "other"], + include = include, + ) + env.expect.that_dict(got.deps).contains_exactly({ + "[other]": ["srcs", "@pypi//other_dep:pkg"], + "[toml]": ["srcs", "@pypi//tomli:pkg"], + "pkg": ["srcs"], + }) + + # The names of the packages and the extras are normalized on both sides. + got = _targets(metadata_name = "r", requires_dist = ["Foo_Bar[TOML.Parser]"], include = include) + env.expect.that_dict(got.deps).contains_exactly({"pkg": ["srcs", ":_foo_bar[toml_parser]"]}) + env.expect.that_collection([ + (c["name"], c["deps"], c["extra"]) + for c in got.extra_deps + ]).contains_exactly([("_foo_bar[toml_parser]", ["@pypi//foo_bar:_extras"], "toml_parser")]) + + got = _targets( + metadata_name = "foo-bar", + requires_dist = ["tomli; extra == \"toml-parser\""], + provides_extra = ["toml-parser"], + include = include, + ) + env.expect.that_dict(got.deps).contains_exactly({ + "[toml_parser]": ["srcs", "@pypi//tomli:pkg"], + "pkg": ["srcs"], + }) + env.expect.that_collection(got.config_settings).contains_exactly(["_extras_toml_parser"]) + env.expect.that_collection([ + (c["deps"], c["extras"]) + for c in got.extra_selects ]).contains_exactly([ - ("include_baz", "(sys_platform == \"darwin\") or (sys_platform == \"linux\")"), - ("include_baz_2", "sys_platform == \"darwin\""), - ("include_baz[a]", "sys_platform == \"linux\""), - ("include_baz[b]", "sys_platform == \"linux\""), + ( + select({ + ":_extras_toml_parser": [":[toml_parser]"], + "//conditions:default": [":[]"], + }), + ["toml_parser"], + ), ]) -_tests.append(_test_deps_with_extras) +_tests.append(_test_different_extras_of_the_same_dep) + +def _test_extras_select_in_group_legacy(env): + # Without a hub the members of the group are aliases to the group library + # in the same repository. + got = _targets( + metadata_name = "foo", + requires_dist = ["bar; extra == \"a\"", "qux; extra == \"b\""], + provides_extra = ["a", "b"], + group_name = "group", + group_deps = ["foo[a]", "bar"], + dep_template = "@pypi_{name}//:{target}", + ) + env.expect.that_collection([c["deps"] for c in got.extra_selects]).contains_exactly([ + select({ + ":_extras_a": [":[a]"], + ":_extras_b": [":[b]"], + "//conditions:default": [":[]"], + }), + ]) + +_tests.append(_test_extras_select_in_group_legacy) def whl_library_deps_targets_test_suite(name): """create the test suite. From 4c4672d8d2be30794690496b56bde44388d06fc0 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 22 Nov 2026 21:54:58 +0900 Subject: [PATCH 46/55] refactor(pypi): remove the unused missing_extra_error rule The extras of the dependencies no longer need to be requested in the hub, so nothing creates the targets that fail the build for them. --- python/private/pypi/missing_package.bzl | 52 +++++++------------------ 1 file changed, 13 insertions(+), 39 deletions(-) diff --git a/python/private/pypi/missing_package.bzl b/python/private/pypi/missing_package.bzl index e5647d5786..46823e813b 100644 --- a/python/private/pypi/missing_package.bzl +++ b/python/private/pypi/missing_package.bzl @@ -1,16 +1,25 @@ -"""Rules for generating an execution-phase action failure when a PyPI package or its extra is missing.""" +"""Rule for generating an execution-phase action failure when a PyPI package is missing.""" load("//python/private:py_info.bzl", "PyInfo") load("//python/private:reexports.bzl", "BuiltinPyInfo") -def _error(ctx, message): +def _missing_package_error_impl(ctx): out = ctx.actions.declare_file(ctx.label.name + ".error") + if ctx.attr.hub_name: + hub_clause = ' when building under PyPI hub "{hub}". Try adding it to the requirements of this hub (e.g. requirements_lock or requirements_by_platform in pip.parse)'.format( + hub = ctx.attr.hub_name, + ) + else: + hub_clause = ' because no default PyPI hub was configured. Try designating a default hub via pip.default(default_hub = "...") or select a hub using --@rules_python//python/config_settings:venv' + # Register an action that fails when Bazel attempts to stage/build this file ctx.actions.run_shell( outputs = [out], - command = "echo \"$1\" >&2 && exit 1", - arguments = ["ERROR: " + message], + command = "echo 'ERROR: PyPI package \"{pkg}\" is not available{hub_clause}.' >&2 && exit 1".format( + pkg = ctx.attr.package_name, + hub_clause = hub_clause, + ), ) maybe_builtin = [BuiltinPyInfo(transitive_sources = depset([out]))] if BuiltinPyInfo != None else [] @@ -25,19 +34,6 @@ def _error(ctx, message): ), ] + maybe_builtin -def _missing_package_error_impl(ctx): - if ctx.attr.hub_name: - hub_clause = ' when building under PyPI hub "{hub}". Try adding it to the requirements of this hub (e.g. requirements_lock or requirements_by_platform in pip.parse)'.format( - hub = ctx.attr.hub_name, - ) - else: - hub_clause = ' because no default PyPI hub was configured. Try designating a default hub via pip.default(default_hub = "...") or select a hub using --@rules_python//python/config_settings:venv' - - return _error(ctx, 'PyPI package "{pkg}" is not available{hub_clause}.'.format( - pkg = ctx.attr.package_name, - hub_clause = hub_clause, - )) - missing_package_error = rule( implementation = _missing_package_error_impl, attrs = { @@ -45,25 +41,3 @@ missing_package_error = rule( "package_name": attr.string(mandatory = True), }, ) - -def _missing_extra_error_impl(ctx): - return _error(ctx, ( - 'The PyPI package "{pkg}" requires "{dep}[{extra}]", but the extra "{extra}" of ' + - '"{dep}" is not available in the hub. Add "{dep}[{extra}]" to the requirements or ' + - "the lock file of the hub, e.g. by compiling the requirements with the extras kept " + - "(`--no-strip-extras`) or by adding the extra to the project dependencies." - ).format( - pkg = ctx.attr.required_by, - dep = ctx.attr.package_name, - extra = ctx.attr.extra, - )) - -missing_extra_error = rule( - implementation = _missing_extra_error_impl, - doc = "A target that fails the build with a helpful message when an extra of a PyPI package is missing.", - attrs = { - "extra": attr.string(mandatory = True), - "package_name": attr.string(mandatory = True), - "required_by": attr.string(mandatory = True), - }, -) From d611f6ecca4bb0fce7a453dddb67f8c55370215c Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 22 Nov 2026 21:54:59 +0900 Subject: [PATCH 47/55] docs(pypi): document that the extras of the dependencies need not be locked --- docs/pypi/use.md | 17 +++++++++++------ news/2786.changed.md | 23 ++++++++++++----------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/docs/pypi/use.md b/docs/pypi/use.md index 2162c20268..2fef87cea3 100644 --- a/docs/pypi/use.md +++ b/docs/pypi/use.md @@ -119,12 +119,17 @@ platforms, for example `foo[bar]` on Linux and `foo[baz]` on macOS, then `@pypi//foo:[bar]` on macOS is the same as `@pypi//foo:[]`. The dependencies of the packages use the extras written in the wheel -`METADATA`, so if `foo` depends on `bar[socks]`, then `@pypi//foo` depends on -`@pypi//bar:[socks]`. If the extra is not in the lock file, then building -`@pypi//foo` fails with a message asking you to add `bar[socks]` to the -requirements or the lock file. For example, when compiling the requirements -with `uv pip compile`, pass `--no-strip-extras` to keep the extras in the -output. +`METADATA`, so if `foo` depends on `bar[socks]`, then `@pypi//foo` includes +`bar` and the dependencies of its `socks` extra. The extras of the +dependencies are resolved by the repository of `bar` from its own `METADATA`, +so they do not need to be in the lock file, e.g. when the requirements are +compiled with the extras stripped. This does not change `@pypi//bar`, which +only includes the extras requested in the lock file. The dependencies of the +extra that are not in the lock file are left out like any other dependency +that is not in the lock file. If `bar` does not provide the `socks` extra, +then only `bar` is used and a warning is printed when `@pypi//foo` is +analyzed, e.g. `WARNING: bar 1.0 does not provide the extra 'socks'`, which is +how `pip` handles the unknown extras. If the lock file requests an extra that the package does not provide, e.g. `foo[nope]`, then `@pypi//foo:[nope]` is the same as `@pypi//foo:[]`, because diff --git a/news/2786.changed.md b/news/2786.changed.md index 256ff9f01f..cc97dd9100 100644 --- a/news/2786.changed.md +++ b/news/2786.changed.md @@ -1,15 +1,16 @@ (pypi) The dependencies between the PyPI packages now use the extras from the wheel `METADATA`, e.g. if `foo` requires `bar[socks]`, then `@pypi//foo` -depends on `@pypi//bar:[socks]` instead of `@pypi//bar`. Previously such extras -were ignored. If the lock file of a hub does not request the extra, the build -fails with a message asking to add it, e.g. by compiling the requirements with -`--no-strip-extras`. The `whl` targets no longer include the wheels of the -dependencies of the extras. A member of `experimental_requirement_cycles` -without extras, e.g. `foo`, is now only the package without extras, so the -cycles that go through `foo[bar]` need `foo[bar]` as a member. The same -distribution that is requested with different extras on different platforms -now uses a single whl repository. The `@pypi//foo` targets still include the -extras requested in the lock file. An extra that the package does not provide -is still ignored, but a warning is now printed like `pip` does, e.g. +includes `bar` and the dependencies of its `socks` extra. Previously such +extras were ignored. The extra does not need to be requested in the lock file +of the hub, because it is resolved by the whl repository of `bar`, and the +dependencies of the extra that are not in the lock file are left out. The +`whl` targets no longer include the wheels of the dependencies of the extras. +A member of `experimental_requirement_cycles` without extras, e.g. `foo`, is +now only the package without extras, so the cycles that go through `foo[bar]` +need `foo[bar]` as a member. The same distribution that is requested with +different extras on different platforms now uses a single whl repository. The +`@pypi//foo` targets still include the extras requested in the lock file. An +extra that the package does not provide is still ignored, but a warning is now +printed like `pip` does, e.g. `WARNING: foo 1.0 does not provide the extra 'nope'` ([#2786](https://github.com/bazel-contrib/rules_python/issues/2786)). From f1a428a534f3551390ae353df8618a25eb913dae Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 22 Nov 2026 22:01:43 +0900 Subject: [PATCH 48/55] feat(pypi): collect the extras of the selected dependency groups `select_dependency_groups` now also returns the extras that the selected requirements request, e.g. `requests[socks]`, so that `pip.configure` can add them to the hub. The extras are keyed by the environment markers of the requirements, including the markers of the self references that select the group of the requirement, so that the extras can be added only for the target platforms that request them. --- python/private/pypi/dependency_groups.bzl | 95 ++++++++++++++--- .../dependency_groups_tests.bzl | 100 ++++++++++++++++++ 2 files changed, 180 insertions(+), 15 deletions(-) diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl index 13b164c2b4..3355fdc0b5 100644 --- a/python/private/pypi/dependency_groups.bzl +++ b/python/private/pypi/dependency_groups.bzl @@ -4,6 +4,11 @@ load("@bazel_skylib//lib:types.bzl", "types") load("//python/private:normalize_name.bzl", "normalize_name") load(":pep508_requirement.bzl", "requirement") +# The limit for processing the groups selected by the self references, e.g. +# `my-project[test]`. A group is processed once for each different set of +# markers that it is selected with. +_MAX_SELF_REFERENCES = 10000 + def select_dependency_groups(pyproject, specs, _fail = fail): """Select the packages named by the given dependency group specifiers. @@ -30,6 +35,13 @@ def select_dependency_groups(pyproject, specs, _fail = fail): * `all`: {type}`bool` whether all of the packages including the transitive dependencies should be selected, i.e. `"*"` and `"transitive"` were given. + * `extras`: {type}`dict[str, dict[tuple[str], list[str]]]` the + normalized package names mapped to the extras that the selected + requirements request, e.g. `requests[socks]`. The extras are + keyed by the environment markers that all need to be true for + the extras to be requested, e.g. `("sys_platform == 'win32'",)` + or an empty tuple if there are no markers. When `all` is set, + the extras of all of the groups are returned. Returns `None` on failure. """ groups = {"default": pyproject.dependencies} @@ -41,18 +53,17 @@ def select_dependency_groups(pyproject, specs, _fail = fail): for name, reqs in expanded.items(): groups["group:" + name] = reqs - if "transitive" in specs: + select_all = "transitive" in specs + if select_all: if "*" not in specs: _fail("The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: {}".format( specs, )) return None - return struct( - # list[str]: sorted normalized package names selected by `specs`. - packages = [], - # bool: True when all hub packages are selected. - all = True, - ) + + # All of the packages are selected, but the extras are only known + # from the groups. + specs = ["*"] selected = {} for spec in specs: @@ -70,30 +81,84 @@ def select_dependency_groups(pyproject, specs, _fail = fail): # A self reference like `my-project[test]` selects the optional # dependencies, including nested composite extras. + # + # The groups are queued together with the markers of the self references + # that selected them, because the requirements of such groups are only + # requested when the markers are true. A group is processed again if it is + # selected with markers that are not known to be stricter than the ones + # that it was processed with. project = normalize_name(pyproject.name or "") packages = {} - queue = list(selected) - for _ in range(len(groups)): + extras = {} + processed = {} + queue = [(key, ()) for key in selected] + for _ in range(_MAX_SELF_REFERENCES): if not queue: break - for req in groups[queue.pop(0)]: + key, markers = queue.pop(0) + if _is_processed(processed.setdefault(key, []), markers): + continue + processed[key].append(markers) + for req in groups[key]: req = requirement(req) + req_markers = _add_marker(markers, req.marker) if req.name_ != project: packages[req.name_] = None + if req.extras: + pkg_extras = extras.setdefault(req.name_, {}).setdefault(req_markers, {}) + for extra in req.extras: + pkg_extras[extra] = None continue for extra in req.extras: opt_key = "optional:" + extra - if opt_key in groups and opt_key not in selected: - selected[opt_key] = None - queue.append(opt_key) + if opt_key in groups: + queue.append((opt_key, req_markers)) + + if queue: + _fail("Too many self references with different markers in the dependency groups of '{}'".format( + pyproject.name, + )) + return None return struct( # list[str]: sorted normalized package names selected by `specs`. - packages = sorted(packages), + packages = [] if select_all else sorted(packages), # bool: True when all packages including transitive ones are selected. - all = False, + all = select_all, + # dict[str, dict[tuple[str], list[str]]]: the extras of the packages + # keyed by the markers that need to be true. + extras = { + name: { + markers: sorted(pkg_extras) + for markers, pkg_extras in sorted(by_markers.items()) + } + for name, by_markers in sorted(extras.items()) + }, ) +def _add_marker(markers, marker): + if not marker or marker in markers: + return markers + return tuple(sorted(markers + (marker,))) + +def _is_processed(processed, markers): + """Tell if a group was already processed with the same or looser markers. + + Args: + processed: {type}`list[tuple[str]]` the markers that the group was + processed with. + markers: {type}`tuple[str]` the markers to check. + + Returns: + {type}`bool` `True` if all of the markers of a processed entry are in + `markers`, i.e. the requirements of the group are already requested + whenever `markers` are true. + """ + for known in processed: + if len([m for m in known if m in markers]) == len(known): + return True + return False + def _match(spec, groups): """Match a dependency group specifier against normalized group keys. diff --git a/tests/pypi/dependency_groups/dependency_groups_tests.bzl b/tests/pypi/dependency_groups/dependency_groups_tests.bzl index a53905bb04..a3817cd835 100644 --- a/tests/pypi/dependency_groups/dependency_groups_tests.bzl +++ b/tests/pypi/dependency_groups/dependency_groups_tests.bzl @@ -100,6 +100,106 @@ def _test_select_all(env): _tests.append(_test_select_all) +def _test_select_extras(env): + pyproject = struct( + name = "my-project", + dependencies = ["main-pkg[Foo_Bar, baz]>=1.0", "plain-pkg"], + optional_dependencies = { + "all": ["my-project[docs]"], + "docs": ["docs-pkg[extra]", "main-pkg[docs]"], + }, + dependency_groups = { + "dev": ["dev-pkg[cli]", {"include-group": "test"}], + "test": [ + "test-pkg[cov]; sys_platform == 'linux'", + "test-pkg[win]; sys_platform == 'win32'", + "test-pkg[cov]", + ], + }, + ) + for specs, want in [ + (["default"], {"main_pkg": {(): ["baz", "foo_bar"]}}), + # The markers of the requirements are kept, so that the extras can be + # added only for the platforms that request them. + (["dev"], { + "dev_pkg": {(): ["cli"]}, + "test_pkg": { + (): ["cov"], + ("sys_platform == 'linux'",): ["cov"], + ("sys_platform == 'win32'",): ["win"], + }, + }), + # The extras of the self reference are the optional dependencies of + # the project, not the extras of a package. + (["all"], {"docs_pkg": {(): ["extra"]}, "main_pkg": {(): ["docs"]}}), + (["default", "docs"], { + "docs_pkg": {(): ["extra"]}, + "main_pkg": {(): ["baz", "docs", "foo_bar"]}, + }), + ]: + got, failures = _select(specs, pyproject = pyproject) + env.expect.where(specs = specs).that_collection(failures).contains_exactly([]) + env.expect.where(specs = specs).that_dict(got.extras).contains_exactly(want) + + # The extras of all of the groups are returned when everything is selected. + got, failures = _select(["*", "transitive"], pyproject = pyproject) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_bool(got.all).equals(True) + env.expect.that_collection(got.packages).contains_exactly([]) + env.expect.that_dict(got.extras).keys().contains_exactly([ + "dev_pkg", + "docs_pkg", + "main_pkg", + "test_pkg", + ]) + +_tests.append(_test_select_extras) + +def _test_select_extras_self_reference_markers(env): + pyproject = struct( + name = "my-project", + dependencies = [ + "my-project[win]; sys_platform == 'win32'", + "my-project[cycle]; python_version < '3.12'", + ], + optional_dependencies = { + "cycle": ["my-project[win]", "my-project[cycle]", "cycle-pkg[c]"], + "win": [ + "win-pkg[a]", + "win-pkg[b]; python_version < '3.12'", + "my-project[cycle]", + ], + }, + dependency_groups = {}, + ) + + # The requirements of a group that is selected by a self reference with a + # marker are only requested if the marker is true. + got, failures = _select(["default"], pyproject = pyproject) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_collection(got.packages).contains_exactly(["cycle_pkg", "win_pkg"]) + env.expect.that_dict(got.extras).contains_exactly({ + "cycle_pkg": { + ("python_version < '3.12'",): ["c"], + ("sys_platform == 'win32'",): ["c"], + }, + "win_pkg": { + ("python_version < '3.12'",): ["a", "b"], + ("python_version < '3.12'", "sys_platform == 'win32'"): ["b"], + ("sys_platform == 'win32'",): ["a"], + }, + }) + + # Selecting the group directly requests it on all of the platforms. + got, failures = _select(["default", "win"], pyproject = pyproject) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got.extras["win_pkg"]).contains_at_least({ + (): ["a"], + ("python_version < '3.12'",): ["b"], + }) + +_tests.append(_test_select_extras_self_reference_markers) + def _test_select_empty_wildcards(env): pyproject = struct( name = "my-project", From b2c276d23139628959636862e64d5ae36f030399 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 22 Nov 2026 22:01:54 +0900 Subject: [PATCH 49/55] feat(pypi): add the pyproject.toml extras to the hubs with pip.configure The extras requested in the selected `pyproject.toml` dependency groups are added to the hub, so `@pypi//requests:[socks]` exists and `@pypi//requests` includes the extra even if the lock file does not request it. `hub_exposure` now returns the extras together with the exposed packages and `add_extras` adds them to the config settings of the hub `whl_map`. The environment markers of the requirements are evaluated for each target platform using the marker environments that the hub builder now returns, like it is done for the requirements files. A requirement like `foo[bar]; sys_platform == "win32"` only adds the extra for Windows and the config setting of the whl repository is split if its platforms request different extras. The whl repositories get the requested extras from the hub, so an extra that the package does not provide has a target that is the same as the package without extras and a warning is printed. --- python/private/pypi/BUILD.bazel | 2 + python/private/pypi/extension.bzl | 27 +- python/private/pypi/hub_builder.bzl | 12 + python/private/pypi/hub_exposure.bzl | 128 +++++++++- tests/pypi/extension/extension_tests.bzl | 64 +++++ .../pypi/hub_exposure/hub_exposure_tests.bzl | 238 +++++++++++++++++- 6 files changed, 458 insertions(+), 13 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 0d660f0d4d..e5fa9de49c 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -667,6 +667,8 @@ bzl_library( srcs = ["hub_exposure.bzl"], deps = [ ":dependency_groups", + ":pep508_evaluate", + ":whl_config_setting", "//python/private:pyproject_utils", ], ) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 1d921f539d..081f8ca94d 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -24,7 +24,7 @@ load("//python/private:pyproject_utils.bzl", "read_pyproject", "version_from_req load("//python/private:repo_utils.bzl", "repo_utils") load(":default_platforms.bzl", "default_platforms") load(":hub_builder.bzl", "hub_builder") -load(":hub_exposure.bzl", "hub_exposure") +load(":hub_exposure.bzl", "add_extras", "hub_exposure") load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") load(":labels.bzl", "extra_label") load(":parse_whl_name.bzl", "parse_whl_name") @@ -357,6 +357,7 @@ You cannot use both the additive_build_content and additive_build_content_file a extra_aliases = {} whl_libraries = {} hubs = {} + hub_platforms = {} for hub in pip_hub_map.values(): out = hub.build() @@ -370,6 +371,7 @@ You cannot use both the additive_build_content and additive_build_content_file a extra_aliases[hub.name] = out.extra_aliases hub_group_map[hub.name] = out.group_map hub_whl_map[hub.name] = out.whl_map + hub_platforms[hub.name] = out.platforms hubs[hub.name] = struct( # str: module that created the hub. module_name = hub.module_name, @@ -379,14 +381,24 @@ You cannot use both the additive_build_content and additive_build_content_file a packages = out.whl_map.keys(), ) - configured_hubs = hub_exposure( + exposure = hub_exposure( module_ctx, hubs = hubs, default_pyproject = config.pyproject_toml, _fail = _fail, ) - if configured_hubs == None: + if exposure == None: return None + configured_hubs = exposure.packages + + # The extras requested in the `pyproject.toml`, e.g. `requests[socks]`, + # are added to the hub even if the lock file does not have them. + for hub_name, extras in exposure.extras.items(): + hub_whl_map[hub_name] = add_extras( + hub_whl_map[hub_name], + extras, + platforms = hub_platforms[hub_name], + ) # The `all_*requirements` only list the selected packages that are # available on all of the platforms, while all of the selected packages @@ -1176,6 +1188,11 @@ Each item is one of: packages of the hub including the transitive dependencies, i.e. all of the packages from the requirements files or the `uv.lock` file. +The extras of the selected requirements, e.g. `requests[socks]`, are also +added to the hub for the target platforms that match the environment markers +of the requirements. With `"transitive"` the extras of all of the groups are +added. + See {ref}`Configuring the Exposed Packages ` for examples. :::{versionadded} VERSION_NEXT_FEATURE @@ -1213,7 +1230,9 @@ The packages are selected from a `pyproject.toml` file using the `[project] dependencies`, `[project.optional-dependencies]` and PEP 735 `[dependency-groups]` tables. The other packages of the hub, e.g. the transitive dependencies, are still used by the packages that need them, but -they are not public. +they are not public. The extras requested by the selected requirements, e.g. +`requests[socks]`, are added to the hub even if the lock file does not request +them. The tags work like a builder: they are applied in order and multiple tags for the same hub are combined, so each tag can configure a different part of the diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 05def3daf4..83f41eb9fc 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -116,6 +116,7 @@ def _build(self): extra_aliases = {}, exposed_packages = [], whl_libraries = {}, + platforms = {}, ) if self._logger.failed(): return ret @@ -145,6 +146,17 @@ def _build(self): # Mapping of whl_library repo names and their kwargs. # dict[str repo_name, dict[str, object] kwargs] whl_libraries = self._whl_libraries, + # The marker environments of the target platforms for each Python + # version, the keys are the same as in the `whl_config_setting` + # values of the `whl_map`. + # dict[str python_version, dict[str target_platform, dict[str, str]]] + platforms = { + _major_minor_version(python_version): { + whl_config_setting(target_platforms = [name]).target_platforms[0]: platform.env + for name, platform in platforms.items() + } + for python_version, platforms in self._platforms.items() + }, ) def _pip_parse(self, module_ctx, pip_attr, default_envsubst = [], python_version = None): diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl index 7393c28fbe..08abf25438 100644 --- a/python/private/pypi/hub_exposure.bzl +++ b/python/private/pypi/hub_exposure.bzl @@ -1,7 +1,9 @@ -"""Compute the packages exposed by the hubs configured with `pip.configure`.""" +"""Compute the packages and extras exposed by the hubs configured with `pip.configure`.""" load("//python/private:pyproject_utils.bzl", "read_pyproject") load(":dependency_groups.bzl", "select_dependency_groups") +load(":pep508_evaluate.bzl", "evaluate") +load(":whl_config_setting.bzl", "whl_config_setting") def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): """Compute the packages exposed by the hubs configured with `pip.configure`. @@ -18,10 +20,16 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): _fail: {type}`callable` the failure function, mainly for testing. Returns: - {type}`dict[str, list[str]] | None` the configured hubs mapped to the - sorted packages that they expose. The hubs that are not configured or - that expose all of their packages are not included, so that they - behave the same. Returns `None` on failure. + {type}`struct | None` with the attributes below or `None` on failure: + * `packages`: {type}`dict[str, list[str]]` the configured hubs + mapped to the sorted packages that they expose. The hubs that are + not configured or that expose all of their packages are not + included, so that they behave the same. + * `extras`: {type}`dict[str, dict[str, dict[tuple[str], list[str]]]]` + the configured hubs mapped to the packages and the extras that + the selected dependency groups request for them, see + {obj}`select_dependency_groups` for the format. It is used with + {obj}`add_extras`. """ # The root module wins over the owner of the hub when both configure it. @@ -35,6 +43,7 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): configured = {} exposes_all = {} + extras = {} for mod in module_ctx.modules: # The tags of a module are applied in order like a builder: a tag # without a `pyproject_toml` uses the last one set for the hub. @@ -90,8 +99,111 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): if name in info.packages: exposed[name] = None + hub_extras = extras.setdefault(hub, {}) + for name, by_markers in selection.extras.items(): + if name not in info.packages: + continue + for markers, pkg_extras in by_markers.items(): + merged = hub_extras.setdefault(name, {}).setdefault(markers, {}) + for extra in pkg_extras: + merged[extra] = None + + return struct( + # dict[str, list[str]]: configured hubs and exposed packages. + packages = { + hub: sorted(packages) + for hub, packages in configured.items() + if hub not in exposes_all + }, + # dict[str, dict[str, dict[tuple[str], list[str]]]]: configured hubs + # and the extras of their packages by the markers. + extras = { + hub: { + name: { + markers: sorted(pkg_extras) + for markers, pkg_extras in sorted(by_markers.items()) + } + for name, by_markers in sorted(hub_extras.items()) + } + for hub, hub_extras in extras.items() + if hub_extras + }, + ) + +def add_extras(whl_map, extras, *, platforms): + """Add the extras requested by `pip.configure` to the hub `whl_map`. + + The extras are added to the config settings of the package, so that the + hub has the `[]` targets and the package target includes them. The + environment markers of the requirements are evaluated for each target + platform, like it is done for the requirements files, so an extra is only + added for the platforms that request it. If the platforms of a config + setting request different extras, the config setting is split. + + Args: + whl_map: {type}`dict[str, dict[str, list[struct]]]` the packages + mapped to the repo names and their {obj}`whl_config_setting` + values. + extras: {type}`dict[str, dict[tuple[str], list[str]]]` the packages + mapped to the normalized extras to add keyed by the markers that + all need to be true on a target platform. + platforms: {type}`dict[str, dict[str, dict[str, str]]]` the Python + versions of the hub mapped to their target platforms and the + marker environments of the platforms. The versions and the + platforms are the same as in the {obj}`whl_config_setting` + values. + + Returns: + {type}`dict[str, dict[str, list[struct]]]` the new `whl_map`. + """ + if not extras: + return whl_map + return { - hub: sorted(packages) - for hub, packages in configured.items() - if hub not in exposes_all + name: { + repo: [ + new_setting + for setting in settings + for new_setting in _add_extras(setting, extras[name], platforms) + ] + for repo, settings in repos.items() + } if name in extras else repos + for name, repos in whl_map.items() } + +def _add_extras(setting, extras, platforms): + envs = platforms.get(setting.version, {}) + + # A config setting without the platforms is used for all of them. + target_platforms = setting.target_platforms or sorted(envs) + if not target_platforms: + # The platforms are not known, so only the extras without markers can + # be added. + target_platforms = [None] + + plats_by_extras = {} + for platform in target_platforms: + env = envs.get(platform) + plat_extras = {extra: None for extra in setting.extras or []} + for markers, pkg_extras in extras.items(): + if markers and not env: + continue + if len([m for m in markers if evaluate(m, env = env)]) == len(markers): + plat_extras.update({extra: None for extra in pkg_extras}) + plats_by_extras.setdefault(tuple(sorted(plat_extras)), []).append(platform) + + if len(plats_by_extras) == 1: + return [whl_config_setting( + version = setting.version, + target_platforms = setting.target_platforms, + extras = plats_by_extras.keys()[0], + )] + + return [ + whl_config_setting( + version = setting.version, + target_platforms = plats, + extras = plat_extras, + ) + for plat_extras, plats in plats_by_extras.items() + ] diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 19bb16dee3..6e3deedc04 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -1000,6 +1000,70 @@ def _test_configure_default_pyproject(env): _tests.append(_test_configure_default_pyproject) +def _test_configure_extras(env): + # The extras requested in the pyproject.toml are added to the hub even if + # the lock file does not request them, but only on the platforms that + # match the markers. + pypi = _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + _mod( + name = "rules_python", + parse = [_configure_parse( + pyproject_toml = "pyproject.toml", + target_platforms = ["linux_x86_64", "windows_aarch64"], + )], + configure = [_configure(hub_name = "pypi_a", dependency_groups = ["default"])], + ), + os_name = "linux", + arch_name = "x86_64", + mock_files = { + "pyproject.toml": """\ +[project] +dependencies = [ + "simple[foo]", + "simple[win]; sys_platform == 'win32'", + "other-pkg[not-provided]", + "other-pkg[old]; python_version < '3.10'", +] +""", + "requirements.txt": _CONFIGURE_REQUIREMENTS, + }, + ), + available_interpreters = { + "python_3_15_host": "unit_test_interpreter_target", + }, + minor_mapping = {"3.15": "3.15.19"}, + ) + + pypi.configured_hubs().contains_exactly({"pypi_a": ["other_pkg", "simple"]}) + pypi.hub_whl_map().contains_exactly({"pypi_a": { + "other_pkg": { + "pypi_a_315_other_pkg": [ + whl_config_setting(version = "3.15", extras = ["not_provided"]), + ], + }, + "simple": { + "pypi_a_315_simple": [ + whl_config_setting( + version = "3.15", + target_platforms = ["cp315_linux_x86_64"], + extras = ["foo"], + ), + whl_config_setting( + version = "3.15", + target_platforms = ["cp315_windows_aarch64"], + extras = ["foo", "win"], + ), + ], + }, + "transitive_pkg": { + "pypi_a_315_transitive_pkg": [whl_config_setting(version = "3.15")], + }, + }}) + +_tests.append(_test_configure_extras) + def _test_not_configured_exposes_all(env): # The pyproject.toml does not filter the hub without `pip.configure`. pypi = _configure_parse_modules( diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl index 1f2b13622c..f5f971b923 100644 --- a/tests/pypi/hub_exposure/hub_exposure_tests.bzl +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -3,8 +3,12 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load( "//python/private/pypi:hub_exposure.bzl", + "add_extras", "hub_exposure", ) # buildifier: disable=bzl-visibility +load("//python/private/pypi:pep508_env.bzl", pep508_env = "env") # buildifier: disable=bzl-visibility +load("//python/private/pypi:render_pkg_aliases.bzl", "available_extras") # buildifier: disable=bzl-visibility +load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # buildifier: disable=bzl-visibility load("//tests/support/mocks:mocks.bzl", "mocks") _tests = [] @@ -18,6 +22,14 @@ dev = ["other-pkg"] test = ["simple", "not-in-hub"] """ +_EXTRAS_PYPROJECT = """\ +[project] +dependencies = ["simple[foo]", "simple[Bar]; sys_platform == 'linux'"] + +[dependency-groups] +dev = ["other-pkg[baz]", "simple[foo]", "not-in-hub[qux]"] +""" + def _hub(*, module_name = "root", pyproject = None): return struct( module_name = module_name, @@ -38,12 +50,18 @@ def _configure( def _module(name = "root", is_root = True, configure = []): return mocks.module(name = name, is_root = is_root, configure = configure) -def _exposure(*modules, hubs = None, default_pyproject = None): +def _exposure(*modules, **kwargs): + """Return the exposed packages of the hubs.""" + got, failures = _hub_exposure(*modules, **kwargs) + return got.packages if got != None else None, failures + +def _hub_exposure(*modules, hubs = None, default_pyproject = None): failures = [] got = hub_exposure( mocks.mctx( modules = list(modules), mock_files = { + "extras.toml": _EXTRAS_PYPROJECT, "other.toml": "[project]\ndependencies = [\"other-pkg\"]\n", "pyproject.toml": _PYPROJECT, }, @@ -217,6 +235,224 @@ def _test_configure_module_precedence(env): _tests.append(_test_configure_module_precedence) +def _test_configure_extras(env): + hubs = {"hub": _hub(pyproject = "extras.toml")} + linux = ("sys_platform == 'linux'",) + for groups, want in [ + (["default"], {"hub": {"simple": {(): ["foo"], linux: ["bar"]}}}), + # The packages that are not in the hub are ignored. + (["dev"], {"hub": {"other_pkg": {(): ["baz"]}, "simple": {(): ["foo"]}}}), + # The extras are also returned when all of the packages are exposed. + (["*", "transitive"], {"hub": { + "other_pkg": {(): ["baz"]}, + "simple": {(): ["foo"], linux: ["bar"]}, + }}), + ]: + got, failures = _hub_exposure( + _module(configure = [_configure(dependency_groups = groups)]), + hubs = hubs, + ) + env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) + env.expect.where(groups = groups).that_dict(got.extras).contains_exactly(want) + + # The extras of multiple tags are combined. + got, _ = _hub_exposure( + _module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(dependency_groups = ["dev"]), + ]), + hubs = hubs, + ) + env.expect.that_dict(got.extras).contains_exactly({ + "hub": { + "other_pkg": {(): ["baz"]}, + "simple": {(): ["foo"], linux: ["bar"]}, + }, + }) + + # No extras are requested. + got, _ = _hub_exposure(_module(configure = [_configure(dependency_groups = ["*"])])) + env.expect.that_dict(got.extras).contains_exactly({}) + got, _ = _hub_exposure(_module(configure = [_configure()])) + env.expect.that_dict(got.extras).contains_exactly({}) + +_tests.append(_test_configure_extras) + +def _platforms(*platforms, version = "3.15"): + """The marker environments of the hub platforms like `hub_builder` returns them.""" + return { + version: { + "cp{}_{}_{}".format(version.replace(".", ""), os, arch): pep508_env( + os = os, + arch = arch, + python_version = version + ".0", + ) + for os, arch in platforms + }, + } + +_LINUX = "cp315_linux_x86_64" +_OSX = "cp315_osx_aarch64" +_WINDOWS = "cp315_windows_x86_64" + +def _test_add_extras(env): + whl_map = { + "other_pkg": {"repo_other": [whl_config_setting(version = "3.15")]}, + "simple": { + "repo_linux": [whl_config_setting( + version = "3.15", + target_platforms = [_LINUX], + extras = ["bar"], + )], + "repo_osx": [whl_config_setting( + version = "3.15", + target_platforms = [_OSX], + )], + }, + } + got = add_extras( + whl_map, + {"simple": {(): ["bar", "foo"]}}, + platforms = _platforms(("linux", "x86_64"), ("osx", "aarch64")), + ) + + # The extras without markers are added on all of the platforms. + env.expect.that_dict(got).contains_exactly({ + "other_pkg": {"repo_other": [whl_config_setting(version = "3.15")]}, + "simple": { + "repo_linux": [whl_config_setting( + version = "3.15", + target_platforms = [_LINUX], + extras = ["bar", "foo"], + )], + "repo_osx": [whl_config_setting( + version = "3.15", + target_platforms = [_OSX], + extras = ["bar", "foo"], + )], + }, + }) + + # Nothing to add. + env.expect.that_dict(add_extras(whl_map, {}, platforms = {})).contains_exactly(whl_map) + +_tests.append(_test_add_extras) + +def _test_add_extras_markers(env): + platforms = _platforms(("linux", "x86_64"), ("osx", "aarch64"), ("windows", "x86_64")) + whl_map = { + # A repo for all of the platforms, e.g. a pure Python wheel. + "any": {"repo_any": [whl_config_setting(version = "3.15")]}, + "simple": { + "repo_unix": [whl_config_setting( + version = "3.15", + target_platforms = [_LINUX, _OSX], + extras = ["locked"], + )], + "repo_windows": [whl_config_setting( + version = "3.15", + target_platforms = [_WINDOWS], + )], + }, + } + extras = { + "any": { + ("sys_platform == 'win32'",): ["win"], + # All of the markers need to be true. + ("python_version < '3.12'", "sys_platform == 'linux'"): ["old"], + }, + "simple": { + (): ["foo"], + ("sys_platform == 'win32'",): ["win"], + ("sys_platform == 'linux'",): ["lin"], + }, + } + + got = add_extras(whl_map, extras, platforms = platforms) + + # `foo[win]; sys_platform == "win32"` only adds the extra on Windows, so + # the other platforms do not get the dependencies of the extra, which may + # not be available for them. + env.expect.that_dict(got).contains_exactly({ + "any": { + "repo_any": [ + whl_config_setting( + version = "3.15", + target_platforms = [_LINUX, _OSX], + ), + whl_config_setting( + version = "3.15", + target_platforms = [_WINDOWS], + extras = ["win"], + ), + ], + }, + "simple": { + "repo_unix": [ + whl_config_setting( + version = "3.15", + target_platforms = [_LINUX], + extras = ["foo", "lin", "locked"], + ), + whl_config_setting( + version = "3.15", + target_platforms = [_OSX], + extras = ["foo", "locked"], + ), + ], + "repo_windows": [whl_config_setting( + version = "3.15", + target_platforms = [_WINDOWS], + extras = ["foo", "win"], + )], + }, + }) + + # The hub tells the whl repositories which extras are requested, so that + # they create the targets for the extras that the package does not + # provide, i.e. the hub never points to a target that does not exist. + env.expect.that_dict(available_extras({ + name: { + setting: repo + for repo, settings in repos.items() + for setting in settings + } + for name, repos in got.items() + })).contains_exactly({ + "any": ["win"], + "simple": ["foo", "lin", "locked", "win"], + }) + + # A marker that is the same on all of the platforms does not split the + # config setting. + got = add_extras( + whl_map, + {"any": {("python_version >= '3.12'",): ["new"], ("python_version < '3.12'",): ["old"]}}, + platforms = platforms, + ) + env.expect.that_dict(got["any"]).contains_exactly({ + "repo_any": [whl_config_setting(version = "3.15", extras = ["new"])], + }) + + # Only the extras without markers are added if the platforms of the Python + # version are not known. + got = add_extras(whl_map, extras, platforms = {}) + env.expect.that_dict(got["any"]).contains_exactly(whl_map["any"]) + env.expect.that_dict(got["simple"]).contains_exactly({ + "repo_unix": [whl_config_setting( + version = "3.15", + target_platforms = [_LINUX, _OSX], + extras = ["foo", "locked"], + )], + "repo_windows": [whl_config_setting( + version = "3.15", + target_platforms = [_WINDOWS], + extras = ["foo"], + )], + }) + +_tests.append(_test_add_extras_markers) + def _test_configure_errors(env): got, failures = _exposure( _module(configure = [_configure(hub_name = "missing")]), From 2017d05a6c27b33d0f13da179c8b1c04080e3ce4 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 22 Nov 2026 22:02:00 +0900 Subject: [PATCH 50/55] docs(pypi): document the pip.configure extras --- docs/pypi/download.md | 23 +++++++++++++++++++++++ news/4522.added.md | 10 ++++++---- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/docs/pypi/download.md b/docs/pypi/download.md index ad8b40e70d..39f725d6b9 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -243,6 +243,29 @@ the `pyproject.toml` selects the optional dependencies of the project. The packages that are not part of the hub are ignored, so a single `pyproject.toml` can list tools that are missing from the lock file. +The extras requested in the selected groups are added to the hub even if the +lock file does not request them, for example because the lock file was +generated with `--strip-extras`. With the following `pyproject.toml`: + +```toml +[project] +dependencies = [ + "requests[socks]", + "keyring[completion]; sys_platform == 'win32'", +] +``` + +the hub has the `@pypi_hub//requests:[socks]` target and +`@pypi_hub//requests` includes the `socks` extra. The environment markers of +the requirements are evaluated for each target platform of the hub, so +`@pypi_hub//keyring` only includes the `completion` extra on Windows and +`@pypi_hub//keyring:[completion]` is the same as `@pypi_hub//keyring:[]` on +the other platforms. With `["*", "transitive"]` the extras of all of the +groups are added. The dependencies of the extras, such as `pysocks`, still +have to be in the lock file, and an extra that the package does not provide is +ignored with a warning. See [extras](./use.md#extras) for the targets of the +extras. + :::{versionadded} VERSION_NEXT_FEATURE ::: diff --git a/news/4522.added.md b/news/4522.added.md index a722e3f9eb..98b513aee3 100644 --- a/news/4522.added.md +++ b/news/4522.added.md @@ -1,5 +1,7 @@ -(pypi) {obj}`pip.configure` now exposes the extras requested in the selected -`pyproject.toml` dependency groups, e.g. `requests[socks]`, as the -`@pypi//requests:[socks]` target and includes them in `@pypi//requests`, even -if the lock file does not request them +(pypi) {obj}`pip.configure` now adds the extras requested in the selected +`pyproject.toml` dependency groups, e.g. `requests[socks]`, to the hub, so +that the `@pypi//requests:[socks]` target exists and `@pypi//requests` +includes the extra even if the lock file does not request it. The environment +markers of the requirements are evaluated for each target platform, so an +extra is only added for the platforms that request it ([#2786](https://github.com/bazel-contrib/rules_python/issues/2786)). From 0fd20ceb19448258730390ecee9269a5a82efd9a Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 22 Nov 2026 23:07:21 +0900 Subject: [PATCH 51/55] feat(pypi): return the dependencies that are left out but needed on the target platforms The dependencies that are not in the `include` list are dropped by `pep508_deps.deps` without telling the caller. Accept the marker environments of the target platforms and return the dropped dependencies with markers that can be true on any of them, so that the caller can warn about them. --- python/private/pypi/pep508_deps.bzl | 43 +++++++++++++++++++- tests/pypi/pep508/deps_tests.bzl | 61 +++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 1 deletion(-) diff --git a/python/private/pypi/pep508_deps.bzl b/python/private/pypi/pep508_deps.bzl index 8aa74194fa..90002b1dc1 100644 --- a/python/private/pypi/pep508_deps.bzl +++ b/python/private/pypi/pep508_deps.bzl @@ -28,7 +28,8 @@ def deps( extras = [], excludes = [], include = [], - keep_extras = False): + keep_extras = False, + envs = []): """Parse the RequiresDist from wheel METADATA Args: @@ -42,12 +43,19 @@ def deps( extras: {type}`list[str]` the requested extras to generate targets for. keep_extras: {type}`bool` keep the extras of the dependencies, if `True`, the dependencies with extras are returned as `foo[bar]`. + envs: {type}`list[dict[str, str]]` the marker environments of the + target platforms. If specified, the dependencies that are left + out because they are not in `include` are returned as `missing` + if they are needed on any of the target platforms. Returns: A struct with attributes: * deps: {type}`list[str]` dependencies to include unconditionally. * deps_select: {type}`dict[str, list[str]]` dependencies to include on particular subset of target platforms. + * missing: {type}`dict[str, list[str]]` the names of the + dependencies that are not in `include`, but are needed on at + least one of the `envs`, and the requirements that need them. The dependencies are normalized names, e.g. `foo`, or normalized names with a single extra, e.g. `foo[bar]`, if the dependency is requested @@ -58,6 +66,7 @@ def deps( return struct( deps = [], deps_select = {}, + missing = {}, ) reqs = sorted( @@ -72,6 +81,7 @@ def deps( excludes = {group_member(x): None for x in excludes} reqs_by_dep = {} + missing = {} for req in reqs: # drop self edges @@ -79,6 +89,8 @@ def deps( continue if include and req.name_ not in include: + if envs and _is_needed(req, extras = want_extras, envs = envs): + missing.setdefault(req.name, []).append(_format(req)) continue for dep in ["{}[{}]".format(req.name_, e) for e in req.extras if keep_extras] or [req.name_]: @@ -100,8 +112,37 @@ def deps( d: markers for d, markers in sorted(deps_select.items()) }, + missing = missing, ) +def _format(req): + return "{}{}{}".format( + req.name, + "[{}]".format(",".join(req.extras)) if req.extras else "", + "; {}".format(req.marker) if req.marker else "", + ) + +def _is_needed(req, *, extras, envs): + """Tell if the requirement is needed on any of the target platforms.""" + if not req.marker: + return True + + for extra in extras: + # Most of the requirements are for the other extras, so do not go + # through the target platforms for them. + env = create_env() | {"extra": extra} + m = evaluate(req.marker, env = env, strict = False) + if m == False: + continue + elif m == True: + return True + + for target_env in envs: + if evaluate(req.marker, env = env | target_env, strict = False) != False: + return True + + return False + def _add(deps, deps_select, dep, markers = None): if not markers: deps[dep] = True diff --git a/tests/pypi/pep508/deps_tests.bzl b/tests/pypi/pep508/deps_tests.bzl index be288fe308..adb2d02e49 100644 --- a/tests/pypi/pep508/deps_tests.bzl +++ b/tests/pypi/pep508/deps_tests.bzl @@ -376,6 +376,67 @@ def test_deps_with_extras_excludes_and_includes(env): _tests.append(test_deps_with_extras_excludes_and_includes) +def test_missing_deps(env): + """The deps that are not included are returned if the target platforms need them. + + Args: + env: the test env. + """ + requires_dist = [ + "bar", + "not-locked", + "tomli>=2; python_version < \"3.11\"", + "tomli-w[all]; python_version < \"3.11\" and extra == \"write\"", + "pywin32; sys_platform == \"win32\"", + "colorama; sys_platform == \"win32\" and extra == \"color\"", + "foo[write]; extra == \"all\"", + ] + linux = lambda v: {"python_version": v, "sys_platform": "linux"} + + got = deps("foo", requires_dist = requires_dist, include = ["bar"]) + env.expect.that_dict(got.missing).contains_exactly({}) + + got = deps("foo", requires_dist = requires_dist, envs = [linux("3.10")]) + env.expect.that_collection(got.deps).contains_at_least(["bar", "not_locked"]) + env.expect.that_dict(got.missing).contains_exactly({}) + + got = deps("foo", requires_dist = requires_dist, include = ["bar"], envs = [linux("3.12")]) + env.expect.that_dict(got.missing).contains_exactly({"not-locked": ["not-locked"]}) + + got = deps("foo", requires_dist = requires_dist, include = ["bar"], envs = [linux("3.12"), linux("3.10")]) + env.expect.that_dict(got.missing).contains_exactly({ + "not-locked": ["not-locked"], + "tomli": ["tomli; python_version < \"3.11\""], + }) + + # The extras that are included via the self references are checked. + got = deps( + "foo", + requires_dist = requires_dist, + include = ["bar"], + extras = ["all", "color"], + envs = [linux("3.12"), linux("3.10")], + ) + env.expect.that_dict(got.missing).contains_exactly({ + "not-locked": ["not-locked"], + "tomli": ["tomli; python_version < \"3.11\""], + "tomli-w": ["tomli-w[all]; python_version < \"3.11\" and extra == \"write\""], + }) + + got = deps( + "foo", + requires_dist = requires_dist, + include = ["bar", "not_locked"], + extras = ["color"], + envs = [{"python_version": "3.12", "sys_platform": "win32"}], + ) + env.expect.that_dict(got.missing).contains_exactly({ + "colorama": ["colorama; sys_platform == \"win32\" and extra == \"color\""], + "pywin32": ["pywin32; sys_platform == \"win32\""], + }) + +_tests.append(test_missing_deps) + def deps_test_suite(name): # buildifier: disable=function-docstring test_suite( name = name, From 6d99d5b0b0623f7a66aec822932cc0cbe03d4020 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 22 Nov 2026 23:07:23 +0900 Subject: [PATCH 52/55] feat(pypi): export the marker environments of the target platforms from the hub The whl repositories load `config.bzl` of the hub to know which packages are in the hub. Also export the marker environments of the target platforms that have requirements, so that the whl repositories can tell if a dependency that is not in the hub is needed on any of them. The list is empty in the WORKSPACE mode, where the requirements are only for the host platform. --- python/private/pypi/config.bzl.tmpl | 5 +++++ python/private/pypi/extension.bzl | 6 +++++ python/private/pypi/hub_builder.bzl | 23 ++++++++++++++++++++ python/private/pypi/hub_repository.bzl | 12 ++++++++++ python/private/pypi/whl_config_repo.bzl | 3 +++ tests/pypi/hub_builder/hub_builder_tests.bzl | 22 +++++++++++++++++++ 6 files changed, 71 insertions(+) diff --git a/python/private/pypi/config.bzl.tmpl b/python/private/pypi/config.bzl.tmpl index 1037153cac..d10006b174 100644 --- a/python/private/pypi/config.bzl.tmpl +++ b/python/private/pypi/config.bzl.tmpl @@ -5,3 +5,8 @@ with your usecase. This may change in between rules_python versions without any """ packages = %%PACKAGES%% + +# The marker environments of the target platforms that have requirements. They +# are used to warn about the dependencies that are not in `packages`, but are +# needed on any of the target platforms. +target_platform_envs = %%TARGET_PLATFORM_ENVS%% diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 1d921f539d..263ffe090d 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -352,6 +352,7 @@ You cannot use both the additive_build_content and additive_build_content_file a # dict[hub, dict[whl, dict[version, str pip]]] # Where hub, whl, and pip are the repo names hub_whl_map = {} + hub_target_platform_envs = {} hub_group_map = {} exposed_packages = {} extra_aliases = {} @@ -370,6 +371,7 @@ You cannot use both the additive_build_content and additive_build_content_file a extra_aliases[hub.name] = out.extra_aliases hub_group_map[hub.name] = out.group_map hub_whl_map[hub.name] = out.whl_map + hub_target_platform_envs[hub.name] = out.target_platform_envs hubs[hub.name] = struct( # str: module that created the hub. module_name = hub.module_name, @@ -410,6 +412,9 @@ You cannot use both the additive_build_content and additive_build_content_file a facts = simpleapi_cache.get_facts(), hub_group_map = hub_group_map, hub_whl_map = hub_whl_map, + # dict[str, list[dict[str, str]]]: the marker environments of the + # target platforms that have requirements in each hub. + hub_target_platform_envs = hub_target_platform_envs, unified_hub_requirements_bzl = config.unified_hub_requirements_bzl, whl_libraries = whl_libraries, whl_mods = whl_mods, @@ -583,6 +588,7 @@ def _pip_impl(module_ctx): packages = mods.exposed_packages.get(hub_name, []), platform_config_settings = mods.platform_config_settings.get(hub_name, {}), groups = mods.hub_group_map.get(hub_name), + target_platform_envs = json.encode(mods.hub_target_platform_envs.get(hub_name, [])), # The hubs configured with `pip.configure` hide the other packages. hide_unexposed_packages = hub_name in mods.configured_hubs, public_packages = mods.configured_hubs.get(hub_name, []), diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 05def3daf4..c67e42a267 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -78,6 +78,10 @@ def hub_builder( # setting originated from. # dict[str whl_name, dict[str config_setting, str repo_name]] _whl_map = {}, # modified by _add_whl_library + # The marker environments of the target platforms that have + # requirements. Dict acting as a set of the values. + # dict[str json, dict[str, str] env] + _target_platform_envs = {}, # modified by _create_whl_repos # Internal @@ -116,6 +120,7 @@ def _build(self): extra_aliases = {}, exposed_packages = [], whl_libraries = {}, + target_platform_envs = [], ) if self._logger.failed(): return ret @@ -145,6 +150,10 @@ def _build(self): # Mapping of whl_library repo names and their kwargs. # dict[str repo_name, dict[str, object] kwargs] whl_libraries = self._whl_libraries, + # The marker environments of the target platforms that have + # requirements. + # list[dict[str, str]] + target_platform_envs = self._target_platform_envs.values(), ) def _pip_parse(self, module_ctx, pip_attr, default_envsubst = [], python_version = None): @@ -609,6 +618,20 @@ def _create_whl_repos( ) for whl in requirements_by_platform: + for src in whl.srcs: + for p in src.target_platforms: + if p not in platforms: + continue + + # The functions for normalizing the values are added back + # when evaluating the markers. + env = { + k: v + for k, v in platforms[p].env.items() + if not k.startswith("_") + } + self._target_platform_envs[json.encode(env)] = env + whl_library_args = common_args | _whl_library_args( self, whl = whl, diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index ec43b0c3fa..67d8f0af1a 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -61,6 +61,10 @@ def _impl(rctx): # to create the targets of the extras that the package does not # provide. "%%PACKAGES%%": render.dict(available_extras(whl_map), value_repr = render.list), + "%%TARGET_PLATFORM_ENVS%%": render.list( + json.decode(rctx.attr.target_platform_envs or "[]"), + value_repr = render.dict, + ), }, ) rctx.template("requirements.bzl", rctx.attr._requirements_bzl_template, substitutions = { @@ -118,6 +122,14 @@ the platforms. mandatory = True, doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", ), + "target_platform_envs": attr.string( + doc = """\ +The JSON encoded list of the marker environments of the target platforms that +have requirements in the hub. The spoke repositories use it to warn about the +dependencies that are not in the hub, but are needed on any of the target +platforms. +""", + ), "whl_map": attr.string_dict( mandatory = True, doc = """\ diff --git a/python/private/pypi/whl_config_repo.bzl b/python/private/pypi/whl_config_repo.bzl index b7cea5a8b7..e347b430cd 100644 --- a/python/private/pypi/whl_config_repo.bzl +++ b/python/private/pypi/whl_config_repo.bzl @@ -15,6 +15,9 @@ def _whl_config_repo_impl(rctx): rctx.attr._config_template, substitutions = { "%%PACKAGES%%": render.dict(rctx.attr.whl_map or {}, value_repr = lambda x: "None"), + # The requirements are only for the host platform, the + # environment of which is not known when the wheels are extracted. + "%%TARGET_PLATFORM_ENVS%%": "[]", }, ) diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index f9e2b0a905..a08cb17de1 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -113,6 +113,7 @@ def hub_builder( whl_map = subjects.dict, whl_libraries = subjects.dict, extra_aliases = subjects.dict, + target_platform_envs = subjects.collection, ), ), pip_parse = builder.pip_parse, @@ -1288,6 +1289,27 @@ simple==0.0.3 \ }) pypi.extra_aliases().contains_exactly({}) + # Only the target platforms that have requirements. + pypi.target_platform_envs().contains_exactly([ + { + "implementation_name": "cpython", + "implementation_version": "3.15.19", + "os_name": "posix", + "platform_machine": platform_machine, + "platform_python_implementation": "CPython", + "platform_release": "", + "platform_system": platform_system, + "platform_version": "0", + "python_full_version": "3.15.19", + "python_version": "3.15", + "sys_platform": sys_platform, + } + for platform_machine, platform_system, sys_platform in [ + ("x86_64", "Linux", "linux"), + ("aarch64", "Darwin", "darwin"), + ] + ]) + _tests.append(_test_download_only_multiple) def _test_simple_get_index(env): From 99435c21a86ee9b003b37ad232e40497dae6256d Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 22 Nov 2026 23:08:54 +0900 Subject: [PATCH 53/55] feat(pypi): warn about the dependencies that are not in the hub but are needed The dependencies from the wheel `METADATA` that are not in the hub are left out silently, which is only correct if they are not needed on any of the target platforms, e.g. `pywin32; sys_platform == "win32"` in a hub for Linux. If the lock file is compiled for fewer platforms or Python versions than the hub is used with, then the missing dependency is only noticed as an `ImportError` at runtime. Print a warning if the marker of such a dependency can be true on any of the target platforms of the hub. The warnings about the dependencies of the package and of the extras requested in the lock file are printed when the targets of the whl repository are created. The warnings about the dependencies of the other extras are printed by the `extra_select` target when another package depends on the extra, otherwise every package with extras that are not used would print them. --- docs/pypi/use.md | 17 ++- news/2786.changed.md | 5 +- python/private/pypi/extra_dep.bzl | 16 +++ .../pypi/generate_whl_library_build_bazel.bzl | 4 +- .../private/pypi/whl_library_deps_targets.bzl | 75 +++++++++-- python/private/pypi/whl_library_targets.bzl | 5 + tests/pypi/extra_dep/extra_dep_tests.bzl | 2 + ...generate_whl_library_build_bazel_tests.bzl | 12 +- tests/pypi/whl_library/testdata/packages.bzl | 16 +++ .../testdata/pkg/pkg-1.0.dist-info/METADATA | 6 + .../whl_library_deps_targets_tests.bzl | 119 ++++++++++++++++++ 11 files changed, 258 insertions(+), 19 deletions(-) diff --git a/docs/pypi/use.md b/docs/pypi/use.md index 44eb06247f..1117ed529d 100644 --- a/docs/pypi/use.md +++ b/docs/pypi/use.md @@ -126,11 +126,26 @@ so they do not need to be in the lock file, e.g. when the requirements are compiled with the extras stripped. This does not change `@pypi//bar`, which only includes the extras requested in the lock file. The dependencies of the extra that are not in the lock file are left out like any other dependency -that is not in the lock file. If `bar` does not provide the `socks` extra, +that is not in the lock file, see below for the warning that is printed if +they are needed. If `bar` does not provide the `socks` extra, then only `bar` is used and a warning is printed when `@pypi//foo` is analyzed, e.g. `WARNING: bar 1.0 does not provide the extra 'socks'`, which is how `pip` handles the unknown extras. +The dependencies from the wheel `METADATA` that are not in the lock file are +left out, which is correct if they are not needed on any of the target +platforms of the hub, e.g. `pywin32; sys_platform == "win32"` in a hub that is +only for Linux. If the marker of such a dependency can be true on any of the +target platforms that have requirements, then a warning that names the missing +package is printed, because importing it may fail at runtime. For example, if +the lock file is compiled for Python 3.12 only and the hub is also used with +Python 3.10, where `coverage[toml]` needs `tomli`, then the warning asks to add +`tomli` to the lock file. The fix is to lock for all of the target platforms +and Python versions of the hub. The warnings about the dependencies of an +extra are only printed if the extra is used, i.e. it is requested in the lock +file or another package depends on it. The warnings are not printed when using +`pip_parse` in `WORKSPACE`. + If the lock file requests an extra that the package does not provide, e.g. `foo[nope]`, then `@pypi//foo:[nope]` is the same as `@pypi//foo:[]`, because an unknown extra does not have any dependencies. A warning is printed when the diff --git a/news/2786.changed.md b/news/2786.changed.md index cc97dd9100..c78f1673a7 100644 --- a/news/2786.changed.md +++ b/news/2786.changed.md @@ -12,5 +12,8 @@ different extras on different platforms now uses a single whl repository. The `@pypi//foo` targets still include the extras requested in the lock file. An extra that the package does not provide is still ignored, but a warning is now printed like `pip` does, e.g. -`WARNING: foo 1.0 does not provide the extra 'nope'` +`WARNING: foo 1.0 does not provide the extra 'nope'`. A warning is also +printed about the dependencies from the wheel `METADATA` that are left out +because they are not in the lock file, if they are needed on any of the +target platforms of the hub ([#2786](https://github.com/bazel-contrib/rules_python/issues/2786)). diff --git a/python/private/pypi/extra_dep.bzl b/python/private/pypi/extra_dep.bzl index 4837d6e5b1..cbeef12fd6 100644 --- a/python/private/pypi/extra_dep.bzl +++ b/python/private/pypi/extra_dep.bzl @@ -129,6 +129,10 @@ def _extra_select_impl(ctx): extra, )) + for warning in ctx.attr.warnings.get(extra, []): + # buildifier: disable=print + print(warning) + return _forward(ctx) extra_select = rule( @@ -154,6 +158,14 @@ the target are forwarded. "extras": attr.string_list( doc = "The normalized extras that have a target, a warning is printed if any other extra is requested.", ), + "warnings": attr.string_list_dict( + doc = """\ +The warnings to print when the extra in the key is requested, e.g. about the +dependencies of the extra that are not in the hub. They are printed here and +not when the targets are created, because only the extras that are used +matter. +""", + ), "_allowlist_function_transition": attr.label( default = "@bazel_tools//tools/allowlists/function_transition_allowlist", ), @@ -169,6 +181,7 @@ def extra_select_targets( targets, default, distribution = "", + warnings = {}, native = native, rules = None, visibility = None): @@ -183,6 +196,8 @@ def extra_select_targets( unknown extras. distribution: {type}`str` the name and the version of the package used in the warning about the unknown extras. + warnings: {type}`dict[str, list[str]]` the warnings to print when + the extra in the key is requested. native: the native struct for overriding in tests. rules: a struct with the `extra_select` rule for overriding in tests. visibility: {type}`list[str]` the visibility of the target. @@ -203,5 +218,6 @@ def extra_select_targets( deps = select(deps | {"//conditions:default": [default]}) if deps else [default], distribution = distribution, extras = sorted(targets), + warnings = warnings, visibility = visibility, ) diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index 3b3f64cc81..2e4a862d65 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -30,6 +30,7 @@ _RENDER = { "requires_dist": render.list, "srcs_exclude": render.list, "tags": render.list, + "target_platform_envs": str, } # NOTE @aignas 2024-10-25: We have to keep this so that files in @@ -98,8 +99,9 @@ def generate_whl_library_build_bazel( if config_load: # The config is also needed without any deps, because it has the # extras that are requested from this package. - loads.append("""load("{}", "{}")""".format(config_load, "packages")) + loads.append("""load("{}", "packages", "target_platform_envs")""".format(config_load)) kwargs["include"] = "packages" + kwargs["target_platform_envs"] = "target_platform_envs" if requires_dist: kwargs["requires_dist"] = requires_dist diff --git a/python/private/pypi/whl_library_deps_targets.bzl b/python/private/pypi/whl_library_deps_targets.bzl index 85d5a80797..6fa47f28a3 100644 --- a/python/private/pypi/whl_library_deps_targets.bzl +++ b/python/private/pypi/whl_library_deps_targets.bzl @@ -28,6 +28,7 @@ def whl_library_deps_targets( provides_extra = [], extras = [], include = [], + target_platform_envs = [], group_deps = [], group_name = None, dep_template, @@ -72,6 +73,12 @@ def whl_library_deps_targets( `[]` targets are also created for the requested extras of this package, see `extras` for the ones that the package does not provide. + target_platform_envs: {type}`list[dict[str, str]]` The marker + environments of the target platforms of the hub. They are used + to warn about the dependencies that are left out because they + are not in `include`, but are needed on any of the target + platforms. No warnings are printed about such dependencies if it + is empty. group_name: {type}`str | None` name of the dependency group (if any). dep_template: {type}`str | None` The dep_template to use. tags: {type}`list[str]` The tags set on the targets. @@ -220,6 +227,44 @@ def whl_library_deps_targets( return ":" + extra_label(extra) + # The dependencies of each target. The dependencies that are not in the + # hub are left out, which is only correct if they are not needed on any of + # the target platforms, so warn about the rest of them. The warnings about + # the dependencies of the extras that are not requested in the hub are + # only printed when another package depends on the extra, otherwise every + # package with extras that are not used would print them. + package_deps = {} + missing = {} + missing_by_extra = {} + for target, extra, want_extras in pkg_targets if requires_dist and hasattr(rules, "py_library") else []: + package_deps[target] = deps( + name = self_name, + requires_dist = requires_dist, + include = include, + excludes = members if is_member(extra) else [], + extras = want_extras, + keep_extras = True, + envs = target_platform_envs, + ) + if not extra or extra in hub_extras: + for dep, reqs in package_deps[target].missing.items(): + missing.setdefault(dep, {}).update({r: None for r in reqs}) + else: + missing_by_extra[extra] = package_deps[target].missing + + for dep, reqs in sorted(missing.items()): + getattr(rules, "warn", _warn)(_missing_dep_warning(distribution, dep, reqs)) + + extra_warnings = {} + for extra, extra_missing in sorted(missing_by_extra.items()): + for dep, reqs in sorted(extra_missing.items()): + # The rest are already printed. + reqs = [r for r in reqs if r not in missing.get(dep, {})] + if reqs: + extra_warnings.setdefault(extra, []).append( + _missing_dep_warning(distribution, dep, reqs), + ) + # The packages that depend on an extra of this package use this target # with the requested extra, because the hub does not know which extras # this package provides. An extra without a target is the same as the @@ -230,6 +275,7 @@ def whl_library_deps_targets( targets = {extra: _extra_target(extra) for extra in extras_targets}, default = _extra_target(""), distribution = distribution, + warnings = extra_warnings, native = native, rules = rules, visibility = visibility, @@ -281,17 +327,9 @@ def whl_library_deps_targets( visibility = _visibility(""), ) - for name, extra, want_extras in pkg_targets if hasattr(rules, "py_library") else []: - package_deps = deps( - name = self_name, - requires_dist = requires_dist, - include = include, - excludes = members if is_member(extra) else [], - extras = want_extras, - keep_extras = True, - ) + for target, extra, _ in pkg_targets if hasattr(rules, "py_library") else []: rules.py_library( - name = name, + name = target, # We include as srcs to ensure that the (locations :pkg) works as expected. srcs = [repo_label(PY_SRCS_LABEL)], deps = _deps( @@ -299,8 +337,8 @@ def whl_library_deps_targets( # propagated. Just passing the target as `srcs` is not enough to propagate # `pyi_srcs`, see `tests/base_rules/py_library`. deps = [repo_label(PY_SRCS_LABEL)], - package_deps = package_deps, - conditions = _conditions(settings, package_deps), + package_deps = package_deps[target], + conditions = _conditions(settings, package_deps[target]), label = _pkg_label, ), # Disable precompilation on this wrapper target to prevent duplicate @@ -325,6 +363,19 @@ def whl_library_deps_targets( visibility = ["//visibility:private"], ) +def _missing_dep_warning(distribution, dep, reqs): + return ( + "WARNING: {distribution} requires '{reqs}', but '{dep}' is not in " + + "the requirements of the hub, so it is left out and it may be " + + "missing at runtime on some of the target platforms. Add '{dep}' " + + "to the lock file, e.g. by locking for all of the target platforms " + + "and Python versions of the hub." + ).format( + distribution = distribution, + reqs = "', '".join(sorted(reqs)), + dep = dep, + ) + def _warn(msg): # buildifier: disable=print print(msg) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 7abc0c4ec5..a08d5ec5f8 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -30,6 +30,7 @@ def whl_library_targets( extras = [], entry_points = {}, include = [], + target_platform_envs = [], group_deps = [], group_name = None, dep_template = None, @@ -61,6 +62,9 @@ def whl_library_targets( {obj}`whl_library_deps_targets`. entry_points: {type}`list[dict]` A list of parsed entry point definitions. include: {type}`list[str]` The list of packages to include. + target_platform_envs: {type}`list[dict[str, str]]` The marker + environments of the target platforms of the hub, see + {obj}`whl_library_deps_targets`. group_name: {type}`str | None` name of the dependency group (if any). dep_template: {type}`str | None` The dep_template to use. data_exclude: {type}`list[str]` The globs for data attribute exclusion. @@ -103,6 +107,7 @@ def whl_library_targets( provides_extra = provides_extra, extras = extras, include = include, # only needed if requires_dist is present + target_platform_envs = target_platform_envs, repo = None, # set aliases in the same repo aliases = {}, **kwargs diff --git a/tests/pypi/extra_dep/extra_dep_tests.bzl b/tests/pypi/extra_dep/extra_dep_tests.bzl index 13adcadd3e..f58f7a94f9 100644 --- a/tests/pypi/extra_dep/extra_dep_tests.bzl +++ b/tests/pypi/extra_dep/extra_dep_tests.bzl @@ -403,6 +403,7 @@ def _test_extra_select_targets(env): }), "distribution": "foo 1.0", "extras": ["a", "b"], + "warnings": {}, "visibility": None, }, ]) # buildifier: @unsorted-dict-items @@ -427,6 +428,7 @@ def _test_extra_select_targets_no_extras(env): "deps": [":[]"], "distribution": "", "extras": [], + "warnings": {}, "visibility": ["//visibility:public"], }, ]) # buildifier: @unsorted-dict-items diff --git a/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl index 21af3416a9..7a23f83c25 100644 --- a/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl +++ b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl @@ -22,7 +22,7 @@ _tests = [] def _test_all_workspace(env): want = """\ load("@package_metadata//rules:package_metadata.bzl", "package_metadata") -load("@pypi//:config.bzl", "packages") +load("@pypi//:config.bzl", "packages", "target_platform_envs") load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets") package(default_visibility = ["//visibility:public"]) @@ -59,6 +59,7 @@ whl_library_targets( "pypi_name=foo", "pypi_version=0", ], + target_platform_envs = target_platform_envs, ) package_metadata( @@ -96,7 +97,7 @@ _tests.append(_test_all_workspace) def _test_all(env): want = """\ load("@package_metadata//rules:package_metadata.bzl", "package_metadata") -load("@pypi//:config.bzl", "packages") +load("@pypi//:config.bzl", "packages", "target_platform_envs") load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets") package(default_visibility = ["//visibility:public"]) @@ -134,6 +135,7 @@ whl_library_targets( "pypi_name=foo", "pypi_version=0", ], + target_platform_envs = target_platform_envs, ) package_metadata( @@ -172,7 +174,7 @@ _tests.append(_test_all) def _test_all_with_loads(env): want = """\ load("@package_metadata//rules:package_metadata.bzl", "package_metadata") -load("@pypi//:config.bzl", "packages") +load("@pypi//:config.bzl", "packages", "target_platform_envs") load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets") package(default_visibility = ["//visibility:public"]) @@ -209,6 +211,7 @@ whl_library_targets( "pypi_name=foo", "pypi_version=0", ], + target_platform_envs = target_platform_envs, ) package_metadata( @@ -248,7 +251,7 @@ def _test_no_deps_with_config(env): # are requested from the package in the hub. want = """\ load("@package_metadata//rules:package_metadata.bzl", "package_metadata") -load("@pypi//:config.bzl", "packages") +load("@pypi//:config.bzl", "packages", "target_platform_envs") load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets") package(default_visibility = ["//visibility:public"]) @@ -262,6 +265,7 @@ whl_library_targets( "pypi_name=foo", "pypi_version=0", ], + target_platform_envs = target_platform_envs, ) """ actual = generate_whl_library_build_bazel( diff --git a/tests/pypi/whl_library/testdata/packages.bzl b/tests/pypi/whl_library/testdata/packages.bzl index 812dc40f2f..6584dfeda3 100644 --- a/tests/pypi/whl_library/testdata/packages.bzl +++ b/tests/pypi/whl_library/testdata/packages.bzl @@ -10,3 +10,19 @@ packages = { "nope", ], } + +# The marker environments of the target platforms of the hub. The `pkg` +# package has dependencies that are not in `packages` and a warning is only +# printed for the ones that are needed on these platforms. +target_platform_envs = [ + { + "implementation_name": "cpython", + "os_name": "posix", + "platform_machine": "x86_64", + "platform_system": "Linux", + "python_full_version": python_full_version, + "python_version": python_full_version.rpartition(".")[0], + "sys_platform": "linux", + } + for python_full_version in ["3.10.20", "3.12.13"] +] diff --git a/tests/pypi/whl_library/testdata/pkg/pkg-1.0.dist-info/METADATA b/tests/pypi/whl_library/testdata/pkg/pkg-1.0.dist-info/METADATA index 712b44edbf..f9ce00ff1f 100644 --- a/tests/pypi/whl_library/testdata/pkg/pkg-1.0.dist-info/METADATA +++ b/tests/pypi/whl_library/testdata/pkg/pkg-1.0.dist-info/METADATA @@ -2,3 +2,9 @@ Name: pkg Version: 1.0 Requires-Dist: optional_dep; extra == "optional" Provides-Extra: optional +Requires-Dist: not_locked; python_version < "3.11" +Requires-Dist: not_locked_optional; sys_platform == "linux" and extra == "optional" +Requires-Dist: not_locked_windows; sys_platform == "win32" +Requires-Dist: not_locked_windows_optional; sys_platform == "win32" and extra == "optional" +Provides-Extra: unused +Requires-Dist: not_locked_unused; extra == "unused" diff --git a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl index 7347ff0743..9fb1d2d30e 100644 --- a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl +++ b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl @@ -556,6 +556,7 @@ def _targets(**kwargs): extra_select = [], filegroup = [], py_library = [], + warn = [], ) whl_library_deps_targets( @@ -573,6 +574,7 @@ def _targets(**kwargs): extra_dep = lambda **kwargs: calls.extra_dep.append(kwargs), extra_select = lambda **kwargs: calls.extra_select.append(kwargs), py_library = lambda **kwargs: calls.py_library.append(kwargs), + warn = lambda msg: calls.warn.append(msg), ), **kwargs ) @@ -584,6 +586,7 @@ def _targets(**kwargs): extra_deps = calls.extra_dep, extra_selects = calls.extra_select, config_settings = [c["name"] for c in calls.config_setting], + warnings = calls.warn, ) def _test_deps_with_extras(env): @@ -655,6 +658,7 @@ def _test_deps_with_extras(env): "deps": [":[]"], "distribution": "foo", "extras": [], + "warnings": {}, "visibility": ["//visibility:public"], }, ]) # buildifier: @unsorted-dict-items @@ -717,6 +721,7 @@ def _test_extra_that_is_not_requested_in_the_hub(env): }), "distribution": "coverage 7.6.1", "extras": ["toml"], + "warnings": {}, "visibility": ["//visibility:public"], }, ]) # buildifier: @unsorted-dict-items @@ -766,6 +771,120 @@ def _test_deps_of_the_extra_that_are_not_in_the_hub(env): _tests.append(_test_deps_of_the_extra_that_are_not_in_the_hub) +def _envs(*python_versions, sys_platform = "linux"): + return [ + { + "os_name": "nt" if sys_platform == "win32" else "posix", + "python_full_version": v, + "python_version": v.rpartition(".")[0], + "sys_platform": sys_platform, + } + for v in python_versions + ] + +_MISSING_DEP_WARNING = ( + "WARNING: {} requires '{}', but '{}' is not in the requirements of the " + + "hub, so it is left out and it may be missing at runtime on some of the " + + "target platforms. Add '{}' to the lock file, e.g. by locking for all " + + "of the target platforms and Python versions of the hub." +) + +def _test_warn_about_missing_deps(env): + # The lock file is only for Python 3.12, but the hub is also used with + # Python 3.10, where `exceptiongroup` is needed. + kwargs = dict( + metadata_name = "foo", + requires_dist = [ + "bar", + "exceptiongroup>=1.0; python_version < \"3.11\"", + "pywin32; sys_platform == \"win32\"", + "not-locked", + ], + include = {"bar": [], "foo": []}, + tags = ["pypi_name=foo", "pypi_version=1.0"], + ) + got = _targets(target_platform_envs = _envs("3.10.20", "3.12.13"), **kwargs) + env.expect.that_dict(got.deps).contains_exactly({ + "pkg": ["srcs", "@pypi//bar:pkg"], + }) + env.expect.that_collection(got.warnings).contains_exactly([ + _MISSING_DEP_WARNING.format( + "foo 1.0", + "exceptiongroup; python_version < \"3.11\"", + "exceptiongroup", + "exceptiongroup", + ), + _MISSING_DEP_WARNING.format("foo 1.0", "not-locked", "not-locked", "not-locked"), + ]) + env.expect.that_collection([c["warnings"] for c in got.extra_selects]).contains_exactly([{}]) + + # Only the dependencies without markers are needed on Python 3.12. + got = _targets(target_platform_envs = _envs("3.12.13"), **kwargs) + env.expect.that_collection(got.warnings).contains_exactly([ + _MISSING_DEP_WARNING.format("foo 1.0", "not-locked", "not-locked", "not-locked"), + ]) + + # The target platforms are not known, e.g. in the WORKSPACE mode. + got = _targets(**kwargs) + env.expect.that_collection(got.warnings).contains_exactly([]) + +_tests.append(_test_warn_about_missing_deps) + +def _test_warn_about_missing_deps_of_extras(env): + tomli = "tomli; python_full_version <= \"3.11.0a6\" and extra == \"toml\"" + warning = _MISSING_DEP_WARNING.format("coverage 7.6.1", tomli, "tomli", "tomli") + kwargs = dict( + metadata_name = "coverage", + requires_dist = _COVERAGE_REQUIRES_DIST + [ + "not-locked", + "also-not-locked; extra == \"unused\"", + ], + provides_extra = ["toml", "unused"], + tags = ["pypi_name=coverage", "pypi_version=7.6.1"], + target_platform_envs = _envs("3.10.20", "3.12.13"), + ) + not_locked = _MISSING_DEP_WARNING.format("coverage 7.6.1", "not-locked", "not-locked", "not-locked") + + # The extra is not requested in the hub, so the warnings about its + # dependencies are only printed when another package depends on the + # extra. The dependencies without the extra are not repeated there. + got = _targets(include = {"coverage": [], "pytest_cov": []}, **kwargs) + env.expect.that_collection(got.warnings).contains_exactly([not_locked]) + env.expect.that_collection([c["warnings"] for c in got.extra_selects]).contains_exactly([{ + "toml": [warning], + "unused": [_MISSING_DEP_WARNING.format( + "coverage 7.6.1", + "also-not-locked; extra == \"unused\"", + "also-not-locked", + "also-not-locked", + )], + }]) + + # The extra is requested in the hub, so it is used. + got = _targets(include = {"coverage": ["toml"], "pytest_cov": []}, **kwargs) + env.expect.that_collection(got.warnings).contains_exactly([not_locked, warning]) + env.expect.that_collection([c["warnings"].keys() for c in got.extra_selects]).contains_exactly([["unused"]]) + + # The extra is requested without the hub and the `pkg` target includes it. + got = _targets( + include = {"coverage": None, "pytest_cov": None}, + extras = ["toml"], + dep_template = "@pip_{name}//:{target}", + **kwargs + ) + env.expect.that_collection(got.warnings).contains_exactly([not_locked, warning]) + env.expect.that_collection([c["warnings"].keys() for c in got.extra_selects]).contains_exactly([["unused"]]) + + # The dependency of the extra is not needed on any of the target platforms. + got = _targets( + include = {"coverage": ["toml"], "pytest_cov": []}, + **(kwargs | {"target_platform_envs": _envs("3.12.13")}) + ) + env.expect.that_collection(got.warnings).contains_exactly([not_locked]) + env.expect.that_collection([c["warnings"].keys() for c in got.extra_selects]).contains_exactly([["unused"]]) + +_tests.append(_test_warn_about_missing_deps_of_extras) + def _test_dep_with_extra_that_is_not_provided(env): include = {"bar": [], "foo": []} got = _targets( From b95ab164a1d3fa410171f840013c2b5e84d00dc9 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 22 Nov 2026 23:08:56 +0900 Subject: [PATCH 54/55] test(pypi): query the dependencies with extras through the whl repositories `bazel query` needs all of the targets that the extras of the dependencies can resolve to, so use `genquery` on the consumer that depends on an extra that is provided and on an extra that is not provided by its dependency. --- tests/pypi/whl_library/BUILD.bazel | 21 +++++++++++++ .../whl_library/whl_library_query_test.py | 30 +++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 tests/pypi/whl_library/whl_library_query_test.py diff --git a/tests/pypi/whl_library/BUILD.bazel b/tests/pypi/whl_library/BUILD.bazel index e7b012ebb3..ad01e3bd54 100644 --- a/tests/pypi/whl_library/BUILD.bazel +++ b/tests/pypi/whl_library/BUILD.bazel @@ -46,3 +46,24 @@ py_test( "@whl_library_extras_consumer//:pkg", ], ) + +# `bazel query` does not know the configuration, so it needs all of the targets +# that the extras of the dependencies can resolve to. It fails with `no such +# target` if any of them is missing, e.g. for the `pkg[unknown]` dependency of +# the consumer, because `pkg` does not provide the `unknown` extra. +genquery( + name = "whl_library_transitive_extras_query", + expression = "deps(@whl_library_extras_consumer//:pkg)", + scope = ["@whl_library_extras_consumer//:pkg"], + target_compatible_with = SUPPORTS_BZLMOD, +) + +py_test( + name = "whl_library_query_test", + srcs = ["whl_library_query_test.py"], + data = [":whl_library_transitive_extras_query"], + env = { + "QUERY_OUTPUT": "$(rootpath :whl_library_transitive_extras_query)", + }, + target_compatible_with = SUPPORTS_BZLMOD, +) diff --git a/tests/pypi/whl_library/whl_library_query_test.py b/tests/pypi/whl_library/whl_library_query_test.py new file mode 100644 index 0000000000..ed31f7c4de --- /dev/null +++ b/tests/pypi/whl_library/whl_library_query_test.py @@ -0,0 +1,30 @@ +import os +import unittest + + +class QueryTest(unittest.TestCase): + def test_deps_include_the_extras(self): + with open(os.environ["QUERY_OUTPUT"]) as f: + got = { + # The canonical names of the repositories depend on the + # extension that creates them. + line.strip().rpartition("whl_library_extras_")[2] + for line in f + if "whl_library_extras_" in line + } + + # The edges in the `METADATA` of the consumer, the first extra is + # provided by `pkg` and the second one is not. + self.assertIn("consumer//:_pkg[optional]", got) + self.assertIn("consumer//:_pkg[unknown]", got) + # Both of them resolve to the target that selects the extra. + self.assertIn("pkg//:_extras", got) + self.assertIn("pkg//:[optional]", got) + self.assertIn("pkg//:[]", got) + self.assertNotIn("pkg//:[unknown]", got) + # The dependency of the `optional` extra. + self.assertIn("optional_dep//:pkg", got) + + +if __name__ == "__main__": + unittest.main() From d7601dbb0244f987424e57456c67a623b5b6a5a3 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 22 Nov 2026 23:10:22 +0900 Subject: [PATCH 55/55] test(pypi): cover the extras of the dependencies without a hub The whl repositories created by `pip_parse` in WORKSPACE or a vendored `requirements.bzl` use the `@_{name}//:{target}` template for the dependencies, so the `extra_dep` targets depend on the `_extras` target of the whl repository directly. --- .../whl_library_deps_targets_tests.bzl | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl index 9fb1d2d30e..c8e3006bd1 100644 --- a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl +++ b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl @@ -1072,6 +1072,56 @@ def _test_different_extras_of_the_same_dep(env): _tests.append(_test_different_extras_of_the_same_dep) +def _test_deps_with_extras_legacy(env): + # Without a hub, e.g. with a vendored `requirements.bzl`, the extras of + # the dependencies are resolved by the whl repositories in the same way + # and the config only has the names of the packages. + include = {"bar": None, "foo": None, "qux": None} + got = _targets( + metadata_name = "foo", + requires_dist = ["bar[socks]", "bar[nope]; sys_platform == \"linux\"", "qux"], + include = include, + dep_template = "@pip_{name}//:{target}", + ) + env.expect.that_dict(got.deps).contains_exactly({ + "pkg": ["srcs", ":_bar[socks]", "@pip_qux//:pkg"] + select({ + ":is_include_bar[nope]_true": [":_bar[nope]"], + "//conditions:default": [], + }), + }) + env.expect.that_collection([ + (c["name"], c["deps"], c["extra"]) + for c in got.extra_deps + ]).contains_exactly([ + ("_bar[nope]", ["@pip_bar//:_extras"], "nope"), + ("_bar[socks]", ["@pip_bar//:_extras"], "socks"), + ]) + + # The repository of the dependency always has the target to select the + # extra, also for the extras that it does not provide and when it has no + # dependencies. + for requires_dist in [[], ["qux; extra == \"socks\""]]: + got = _targets( + metadata_name = "bar", + requires_dist = requires_dist, + provides_extra = ["socks"], + include = include, + dep_template = "@pip_{name}//:{target}", + ) + env.expect.that_collection([ + (c["name"], c["deps"], c["extras"]) + for c in got.extra_selects + ]).contains_exactly([( + "_extras", + select({ + ":_extras_socks": [":[socks]"], + "//conditions:default": [":[]"], + }), + ["socks"], + )]) + +_tests.append(_test_deps_with_extras_legacy) + def _test_extras_select_in_group_legacy(env): # Without a hub the members of the group are aliases to the group library # in the same repository.