diff --git a/python/private/pypi/config.bzl.tmpl b/python/private/pypi/config.bzl.tmpl index 1037153cac..d10006b174 100644 --- a/python/private/pypi/config.bzl.tmpl +++ b/python/private/pypi/config.bzl.tmpl @@ -5,3 +5,8 @@ with your usecase. This may change in between rules_python versions without any """ packages = %%PACKAGES%% + +# The marker environments of the target platforms that have requirements. They +# are used to warn about the dependencies that are not in `packages`, but are +# needed on any of the target platforms. +target_platform_envs = %%TARGET_PLATFORM_ENVS%% diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 1d921f539d..263ffe090d 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -352,6 +352,7 @@ You cannot use both the additive_build_content and additive_build_content_file a # dict[hub, dict[whl, dict[version, str pip]]] # Where hub, whl, and pip are the repo names hub_whl_map = {} + hub_target_platform_envs = {} hub_group_map = {} exposed_packages = {} extra_aliases = {} @@ -370,6 +371,7 @@ You cannot use both the additive_build_content and additive_build_content_file a extra_aliases[hub.name] = out.extra_aliases hub_group_map[hub.name] = out.group_map hub_whl_map[hub.name] = out.whl_map + hub_target_platform_envs[hub.name] = out.target_platform_envs hubs[hub.name] = struct( # str: module that created the hub. module_name = hub.module_name, @@ -410,6 +412,9 @@ You cannot use both the additive_build_content and additive_build_content_file a facts = simpleapi_cache.get_facts(), hub_group_map = hub_group_map, hub_whl_map = hub_whl_map, + # dict[str, list[dict[str, str]]]: the marker environments of the + # target platforms that have requirements in each hub. + hub_target_platform_envs = hub_target_platform_envs, unified_hub_requirements_bzl = config.unified_hub_requirements_bzl, whl_libraries = whl_libraries, whl_mods = whl_mods, @@ -583,6 +588,7 @@ def _pip_impl(module_ctx): packages = mods.exposed_packages.get(hub_name, []), platform_config_settings = mods.platform_config_settings.get(hub_name, {}), groups = mods.hub_group_map.get(hub_name), + target_platform_envs = json.encode(mods.hub_target_platform_envs.get(hub_name, [])), # The hubs configured with `pip.configure` hide the other packages. hide_unexposed_packages = hub_name in mods.configured_hubs, public_packages = mods.configured_hubs.get(hub_name, []), diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 05def3daf4..c67e42a267 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -78,6 +78,10 @@ def hub_builder( # setting originated from. # dict[str whl_name, dict[str config_setting, str repo_name]] _whl_map = {}, # modified by _add_whl_library + # The marker environments of the target platforms that have + # requirements. Dict acting as a set of the values. + # dict[str json, dict[str, str] env] + _target_platform_envs = {}, # modified by _create_whl_repos # Internal @@ -116,6 +120,7 @@ def _build(self): extra_aliases = {}, exposed_packages = [], whl_libraries = {}, + target_platform_envs = [], ) if self._logger.failed(): return ret @@ -145,6 +150,10 @@ def _build(self): # Mapping of whl_library repo names and their kwargs. # dict[str repo_name, dict[str, object] kwargs] whl_libraries = self._whl_libraries, + # The marker environments of the target platforms that have + # requirements. + # list[dict[str, str]] + target_platform_envs = self._target_platform_envs.values(), ) def _pip_parse(self, module_ctx, pip_attr, default_envsubst = [], python_version = None): @@ -609,6 +618,20 @@ def _create_whl_repos( ) for whl in requirements_by_platform: + for src in whl.srcs: + for p in src.target_platforms: + if p not in platforms: + continue + + # The functions for normalizing the values are added back + # when evaluating the markers. + env = { + k: v + for k, v in platforms[p].env.items() + if not k.startswith("_") + } + self._target_platform_envs[json.encode(env)] = env + whl_library_args = common_args | _whl_library_args( self, whl = whl, diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index ec43b0c3fa..67d8f0af1a 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -61,6 +61,10 @@ def _impl(rctx): # to create the targets of the extras that the package does not # provide. "%%PACKAGES%%": render.dict(available_extras(whl_map), value_repr = render.list), + "%%TARGET_PLATFORM_ENVS%%": render.list( + json.decode(rctx.attr.target_platform_envs or "[]"), + value_repr = render.dict, + ), }, ) rctx.template("requirements.bzl", rctx.attr._requirements_bzl_template, substitutions = { @@ -118,6 +122,14 @@ the platforms. mandatory = True, doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", ), + "target_platform_envs": attr.string( + doc = """\ +The JSON encoded list of the marker environments of the target platforms that +have requirements in the hub. The spoke repositories use it to warn about the +dependencies that are not in the hub, but are needed on any of the target +platforms. +""", + ), "whl_map": attr.string_dict( mandatory = True, doc = """\ diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index 5fe8b6c1c7..837cdd4014 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -293,6 +293,8 @@ def _pip_repository_impl(rctx): pkg: sorted(selected_extras.get(pkg, [])) for pkg in bzl_packages }, value_repr = render.list), + # The target platforms are not known in WORKSPACE. + "%%TARGET_PLATFORM_ENVS%%": "[]", }, ) config_repo_template = repr(rctx.attr.name) diff --git a/python/private/pypi/whl_config_repo.bzl b/python/private/pypi/whl_config_repo.bzl index b7cea5a8b7..e347b430cd 100644 --- a/python/private/pypi/whl_config_repo.bzl +++ b/python/private/pypi/whl_config_repo.bzl @@ -15,6 +15,9 @@ def _whl_config_repo_impl(rctx): rctx.attr._config_template, substitutions = { "%%PACKAGES%%": render.dict(rctx.attr.whl_map or {}, value_repr = lambda x: "None"), + # The requirements are only for the host platform, the + # environment of which is not known when the wheels are extracted. + "%%TARGET_PLATFORM_ENVS%%": "[]", }, ) diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index f9e2b0a905..a08cb17de1 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -113,6 +113,7 @@ def hub_builder( whl_map = subjects.dict, whl_libraries = subjects.dict, extra_aliases = subjects.dict, + target_platform_envs = subjects.collection, ), ), pip_parse = builder.pip_parse, @@ -1288,6 +1289,27 @@ simple==0.0.3 \ }) pypi.extra_aliases().contains_exactly({}) + # Only the target platforms that have requirements. + pypi.target_platform_envs().contains_exactly([ + { + "implementation_name": "cpython", + "implementation_version": "3.15.19", + "os_name": "posix", + "platform_machine": platform_machine, + "platform_python_implementation": "CPython", + "platform_release": "", + "platform_system": platform_system, + "platform_version": "0", + "python_full_version": "3.15.19", + "python_version": "3.15", + "sys_platform": sys_platform, + } + for platform_machine, platform_system, sys_platform in [ + ("x86_64", "Linux", "linux"), + ("aarch64", "Darwin", "darwin"), + ] + ]) + _tests.append(_test_download_only_multiple) def _test_simple_get_index(env):