From d50eea3fcc6af6bfe9619f8312fecd8eb476b71f Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 22 Nov 2026 23:24:53 +0900 Subject: [PATCH] feat(pypi): export the marker environments of the target platforms from the hub The whl repositories need the marker environments of the target platforms of the hub to tell which of the dependencies that are not in the hub may be needed. The hub now exports the environments of the target platforms that have requirements as target_platform_envs in its config.bzl. Nothing loads them yet. Work towards #2786 --- python/private/pypi/config.bzl.tmpl | 5 +++++ python/private/pypi/extension.bzl | 6 +++++ python/private/pypi/hub_builder.bzl | 23 ++++++++++++++++++++ python/private/pypi/hub_repository.bzl | 12 ++++++++++ python/private/pypi/pip_repository.bzl | 2 ++ python/private/pypi/whl_config_repo.bzl | 3 +++ tests/pypi/hub_builder/hub_builder_tests.bzl | 22 +++++++++++++++++++ 7 files changed, 73 insertions(+) diff --git a/python/private/pypi/config.bzl.tmpl b/python/private/pypi/config.bzl.tmpl index 1037153cac..d10006b174 100644 --- a/python/private/pypi/config.bzl.tmpl +++ b/python/private/pypi/config.bzl.tmpl @@ -5,3 +5,8 @@ with your usecase. This may change in between rules_python versions without any """ packages = %%PACKAGES%% + +# The marker environments of the target platforms that have requirements. They +# are used to warn about the dependencies that are not in `packages`, but are +# needed on any of the target platforms. +target_platform_envs = %%TARGET_PLATFORM_ENVS%% diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 1d921f539d..263ffe090d 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -352,6 +352,7 @@ You cannot use both the additive_build_content and additive_build_content_file a # dict[hub, dict[whl, dict[version, str pip]]] # Where hub, whl, and pip are the repo names hub_whl_map = {} + hub_target_platform_envs = {} hub_group_map = {} exposed_packages = {} extra_aliases = {} @@ -370,6 +371,7 @@ You cannot use both the additive_build_content and additive_build_content_file a extra_aliases[hub.name] = out.extra_aliases hub_group_map[hub.name] = out.group_map hub_whl_map[hub.name] = out.whl_map + hub_target_platform_envs[hub.name] = out.target_platform_envs hubs[hub.name] = struct( # str: module that created the hub. module_name = hub.module_name, @@ -410,6 +412,9 @@ You cannot use both the additive_build_content and additive_build_content_file a facts = simpleapi_cache.get_facts(), hub_group_map = hub_group_map, hub_whl_map = hub_whl_map, + # dict[str, list[dict[str, str]]]: the marker environments of the + # target platforms that have requirements in each hub. + hub_target_platform_envs = hub_target_platform_envs, unified_hub_requirements_bzl = config.unified_hub_requirements_bzl, whl_libraries = whl_libraries, whl_mods = whl_mods, @@ -583,6 +588,7 @@ def _pip_impl(module_ctx): packages = mods.exposed_packages.get(hub_name, []), platform_config_settings = mods.platform_config_settings.get(hub_name, {}), groups = mods.hub_group_map.get(hub_name), + target_platform_envs = json.encode(mods.hub_target_platform_envs.get(hub_name, [])), # The hubs configured with `pip.configure` hide the other packages. hide_unexposed_packages = hub_name in mods.configured_hubs, public_packages = mods.configured_hubs.get(hub_name, []), diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 05def3daf4..c67e42a267 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -78,6 +78,10 @@ def hub_builder( # setting originated from. # dict[str whl_name, dict[str config_setting, str repo_name]] _whl_map = {}, # modified by _add_whl_library + # The marker environments of the target platforms that have + # requirements. Dict acting as a set of the values. + # dict[str json, dict[str, str] env] + _target_platform_envs = {}, # modified by _create_whl_repos # Internal @@ -116,6 +120,7 @@ def _build(self): extra_aliases = {}, exposed_packages = [], whl_libraries = {}, + target_platform_envs = [], ) if self._logger.failed(): return ret @@ -145,6 +150,10 @@ def _build(self): # Mapping of whl_library repo names and their kwargs. # dict[str repo_name, dict[str, object] kwargs] whl_libraries = self._whl_libraries, + # The marker environments of the target platforms that have + # requirements. + # list[dict[str, str]] + target_platform_envs = self._target_platform_envs.values(), ) def _pip_parse(self, module_ctx, pip_attr, default_envsubst = [], python_version = None): @@ -609,6 +618,20 @@ def _create_whl_repos( ) for whl in requirements_by_platform: + for src in whl.srcs: + for p in src.target_platforms: + if p not in platforms: + continue + + # The functions for normalizing the values are added back + # when evaluating the markers. + env = { + k: v + for k, v in platforms[p].env.items() + if not k.startswith("_") + } + self._target_platform_envs[json.encode(env)] = env + whl_library_args = common_args | _whl_library_args( self, whl = whl, diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index ec43b0c3fa..67d8f0af1a 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -61,6 +61,10 @@ def _impl(rctx): # to create the targets of the extras that the package does not # provide. "%%PACKAGES%%": render.dict(available_extras(whl_map), value_repr = render.list), + "%%TARGET_PLATFORM_ENVS%%": render.list( + json.decode(rctx.attr.target_platform_envs or "[]"), + value_repr = render.dict, + ), }, ) rctx.template("requirements.bzl", rctx.attr._requirements_bzl_template, substitutions = { @@ -118,6 +122,14 @@ the platforms. mandatory = True, doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", ), + "target_platform_envs": attr.string( + doc = """\ +The JSON encoded list of the marker environments of the target platforms that +have requirements in the hub. The spoke repositories use it to warn about the +dependencies that are not in the hub, but are needed on any of the target +platforms. +""", + ), "whl_map": attr.string_dict( mandatory = True, doc = """\ diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index 5fe8b6c1c7..837cdd4014 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -293,6 +293,8 @@ def _pip_repository_impl(rctx): pkg: sorted(selected_extras.get(pkg, [])) for pkg in bzl_packages }, value_repr = render.list), + # The target platforms are not known in WORKSPACE. + "%%TARGET_PLATFORM_ENVS%%": "[]", }, ) config_repo_template = repr(rctx.attr.name) diff --git a/python/private/pypi/whl_config_repo.bzl b/python/private/pypi/whl_config_repo.bzl index b7cea5a8b7..e347b430cd 100644 --- a/python/private/pypi/whl_config_repo.bzl +++ b/python/private/pypi/whl_config_repo.bzl @@ -15,6 +15,9 @@ def _whl_config_repo_impl(rctx): rctx.attr._config_template, substitutions = { "%%PACKAGES%%": render.dict(rctx.attr.whl_map or {}, value_repr = lambda x: "None"), + # The requirements are only for the host platform, the + # environment of which is not known when the wheels are extracted. + "%%TARGET_PLATFORM_ENVS%%": "[]", }, ) diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index f9e2b0a905..a08cb17de1 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -113,6 +113,7 @@ def hub_builder( whl_map = subjects.dict, whl_libraries = subjects.dict, extra_aliases = subjects.dict, + target_platform_envs = subjects.collection, ), ), pip_parse = builder.pip_parse, @@ -1288,6 +1289,27 @@ simple==0.0.3 \ }) pypi.extra_aliases().contains_exactly({}) + # Only the target platforms that have requirements. + pypi.target_platform_envs().contains_exactly([ + { + "implementation_name": "cpython", + "implementation_version": "3.15.19", + "os_name": "posix", + "platform_machine": platform_machine, + "platform_python_implementation": "CPython", + "platform_release": "", + "platform_system": platform_system, + "platform_version": "0", + "python_full_version": "3.15.19", + "python_version": "3.15", + "sys_platform": sys_platform, + } + for platform_machine, platform_system, sys_platform in [ + ("x86_64", "Linux", "linux"), + ("aarch64", "Darwin", "darwin"), + ] + ]) + _tests.append(_test_download_only_multiple) def _test_simple_get_index(env):