diff --git a/docs/pypi/download.md b/docs/pypi/download.md index ad8b40e70d..39f725d6b9 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -243,6 +243,29 @@ the `pyproject.toml` selects the optional dependencies of the project. The packages that are not part of the hub are ignored, so a single `pyproject.toml` can list tools that are missing from the lock file. +The extras requested in the selected groups are added to the hub even if the +lock file does not request them, for example because the lock file was +generated with `--strip-extras`. With the following `pyproject.toml`: + +```toml +[project] +dependencies = [ + "requests[socks]", + "keyring[completion]; sys_platform == 'win32'", +] +``` + +the hub has the `@pypi_hub//requests:[socks]` target and +`@pypi_hub//requests` includes the `socks` extra. The environment markers of +the requirements are evaluated for each target platform of the hub, so +`@pypi_hub//keyring` only includes the `completion` extra on Windows and +`@pypi_hub//keyring:[completion]` is the same as `@pypi_hub//keyring:[]` on +the other platforms. With `["*", "transitive"]` the extras of all of the +groups are added. The dependencies of the extras, such as `pysocks`, still +have to be in the lock file, and an extra that the package does not provide is +ignored with a warning. See [extras](./use.md#extras) for the targets of the +extras. + :::{versionadded} VERSION_NEXT_FEATURE ::: diff --git a/news/4522.added.md b/news/4522.added.md new file mode 100644 index 0000000000..98b513aee3 --- /dev/null +++ b/news/4522.added.md @@ -0,0 +1,7 @@ +(pypi) {obj}`pip.configure` now adds the extras requested in the selected +`pyproject.toml` dependency groups, e.g. `requests[socks]`, to the hub, so +that the `@pypi//requests:[socks]` target exists and `@pypi//requests` +includes the extra even if the lock file does not request it. The environment +markers of the requirements are evaluated for each target platform, so an +extra is only added for the platforms that request it +([#2786](https://github.com/bazel-contrib/rules_python/issues/2786)). diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index e292717746..7d510471c3 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -687,6 +687,8 @@ bzl_library( srcs = ["hub_exposure.bzl"], deps = [ ":dependency_groups", + ":pep508_evaluate", + ":whl_config_setting", "//python/private:pyproject_utils", ], ) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 263ffe090d..3bfc0d85fd 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -24,7 +24,7 @@ load("//python/private:pyproject_utils.bzl", "read_pyproject", "version_from_req load("//python/private:repo_utils.bzl", "repo_utils") load(":default_platforms.bzl", "default_platforms") load(":hub_builder.bzl", "hub_builder") -load(":hub_exposure.bzl", "hub_exposure") +load(":hub_exposure.bzl", "add_extras", "hub_exposure") load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") load(":labels.bzl", "extra_label") load(":parse_whl_name.bzl", "parse_whl_name") @@ -358,6 +358,7 @@ You cannot use both the additive_build_content and additive_build_content_file a extra_aliases = {} whl_libraries = {} hubs = {} + hub_platforms = {} for hub in pip_hub_map.values(): out = hub.build() @@ -372,6 +373,7 @@ You cannot use both the additive_build_content and additive_build_content_file a hub_group_map[hub.name] = out.group_map hub_whl_map[hub.name] = out.whl_map hub_target_platform_envs[hub.name] = out.target_platform_envs + hub_platforms[hub.name] = out.platforms hubs[hub.name] = struct( # str: module that created the hub. module_name = hub.module_name, @@ -381,14 +383,24 @@ You cannot use both the additive_build_content and additive_build_content_file a packages = out.whl_map.keys(), ) - configured_hubs = hub_exposure( + exposure = hub_exposure( module_ctx, hubs = hubs, default_pyproject = config.pyproject_toml, _fail = _fail, ) - if configured_hubs == None: + if exposure == None: return None + configured_hubs = exposure.packages + + # The extras requested in the `pyproject.toml`, e.g. `requests[socks]`, + # are added to the hub even if the lock file does not have them. + for hub_name, extras in exposure.extras.items(): + hub_whl_map[hub_name] = add_extras( + hub_whl_map[hub_name], + extras, + platforms = hub_platforms[hub_name], + ) # The `all_*requirements` only list the selected packages that are # available on all of the platforms, while all of the selected packages @@ -1182,6 +1194,11 @@ Each item is one of: packages of the hub including the transitive dependencies, i.e. all of the packages from the requirements files or the `uv.lock` file. +The extras of the selected requirements, e.g. `requests[socks]`, are also +added to the hub for the target platforms that match the environment markers +of the requirements. With `"transitive"` the extras of all of the groups are +added. + See {ref}`Configuring the Exposed Packages ` for examples. :::{versionadded} VERSION_NEXT_FEATURE @@ -1219,7 +1236,9 @@ The packages are selected from a `pyproject.toml` file using the `[project] dependencies`, `[project.optional-dependencies]` and PEP 735 `[dependency-groups]` tables. The other packages of the hub, e.g. the transitive dependencies, are still used by the packages that need them, but -they are not public. +they are not public. The extras requested by the selected requirements, e.g. +`requests[socks]`, are added to the hub even if the lock file does not request +them. The tags work like a builder: they are applied in order and multiple tags for the same hub are combined, so each tag can configure a different part of the diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index c67e42a267..c85701344a 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -121,6 +121,7 @@ def _build(self): exposed_packages = [], whl_libraries = {}, target_platform_envs = [], + platforms = {}, ) if self._logger.failed(): return ret @@ -154,6 +155,17 @@ def _build(self): # requirements. # list[dict[str, str]] target_platform_envs = self._target_platform_envs.values(), + # The marker environments of the target platforms for each Python + # version, the keys are the same as in the `whl_config_setting` + # values of the `whl_map`. + # dict[str python_version, dict[str target_platform, dict[str, str]]] + platforms = { + _major_minor_version(python_version): { + whl_config_setting(target_platforms = [name]).target_platforms[0]: platform.env + for name, platform in platforms.items() + } + for python_version, platforms in self._platforms.items() + }, ) def _pip_parse(self, module_ctx, pip_attr, default_envsubst = [], python_version = None): diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl index 7393c28fbe..08abf25438 100644 --- a/python/private/pypi/hub_exposure.bzl +++ b/python/private/pypi/hub_exposure.bzl @@ -1,7 +1,9 @@ -"""Compute the packages exposed by the hubs configured with `pip.configure`.""" +"""Compute the packages and extras exposed by the hubs configured with `pip.configure`.""" load("//python/private:pyproject_utils.bzl", "read_pyproject") load(":dependency_groups.bzl", "select_dependency_groups") +load(":pep508_evaluate.bzl", "evaluate") +load(":whl_config_setting.bzl", "whl_config_setting") def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): """Compute the packages exposed by the hubs configured with `pip.configure`. @@ -18,10 +20,16 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): _fail: {type}`callable` the failure function, mainly for testing. Returns: - {type}`dict[str, list[str]] | None` the configured hubs mapped to the - sorted packages that they expose. The hubs that are not configured or - that expose all of their packages are not included, so that they - behave the same. Returns `None` on failure. + {type}`struct | None` with the attributes below or `None` on failure: + * `packages`: {type}`dict[str, list[str]]` the configured hubs + mapped to the sorted packages that they expose. The hubs that are + not configured or that expose all of their packages are not + included, so that they behave the same. + * `extras`: {type}`dict[str, dict[str, dict[tuple[str], list[str]]]]` + the configured hubs mapped to the packages and the extras that + the selected dependency groups request for them, see + {obj}`select_dependency_groups` for the format. It is used with + {obj}`add_extras`. """ # The root module wins over the owner of the hub when both configure it. @@ -35,6 +43,7 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): configured = {} exposes_all = {} + extras = {} for mod in module_ctx.modules: # The tags of a module are applied in order like a builder: a tag # without a `pyproject_toml` uses the last one set for the hub. @@ -90,8 +99,111 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): if name in info.packages: exposed[name] = None + hub_extras = extras.setdefault(hub, {}) + for name, by_markers in selection.extras.items(): + if name not in info.packages: + continue + for markers, pkg_extras in by_markers.items(): + merged = hub_extras.setdefault(name, {}).setdefault(markers, {}) + for extra in pkg_extras: + merged[extra] = None + + return struct( + # dict[str, list[str]]: configured hubs and exposed packages. + packages = { + hub: sorted(packages) + for hub, packages in configured.items() + if hub not in exposes_all + }, + # dict[str, dict[str, dict[tuple[str], list[str]]]]: configured hubs + # and the extras of their packages by the markers. + extras = { + hub: { + name: { + markers: sorted(pkg_extras) + for markers, pkg_extras in sorted(by_markers.items()) + } + for name, by_markers in sorted(hub_extras.items()) + } + for hub, hub_extras in extras.items() + if hub_extras + }, + ) + +def add_extras(whl_map, extras, *, platforms): + """Add the extras requested by `pip.configure` to the hub `whl_map`. + + The extras are added to the config settings of the package, so that the + hub has the `[]` targets and the package target includes them. The + environment markers of the requirements are evaluated for each target + platform, like it is done for the requirements files, so an extra is only + added for the platforms that request it. If the platforms of a config + setting request different extras, the config setting is split. + + Args: + whl_map: {type}`dict[str, dict[str, list[struct]]]` the packages + mapped to the repo names and their {obj}`whl_config_setting` + values. + extras: {type}`dict[str, dict[tuple[str], list[str]]]` the packages + mapped to the normalized extras to add keyed by the markers that + all need to be true on a target platform. + platforms: {type}`dict[str, dict[str, dict[str, str]]]` the Python + versions of the hub mapped to their target platforms and the + marker environments of the platforms. The versions and the + platforms are the same as in the {obj}`whl_config_setting` + values. + + Returns: + {type}`dict[str, dict[str, list[struct]]]` the new `whl_map`. + """ + if not extras: + return whl_map + return { - hub: sorted(packages) - for hub, packages in configured.items() - if hub not in exposes_all + name: { + repo: [ + new_setting + for setting in settings + for new_setting in _add_extras(setting, extras[name], platforms) + ] + for repo, settings in repos.items() + } if name in extras else repos + for name, repos in whl_map.items() } + +def _add_extras(setting, extras, platforms): + envs = platforms.get(setting.version, {}) + + # A config setting without the platforms is used for all of them. + target_platforms = setting.target_platforms or sorted(envs) + if not target_platforms: + # The platforms are not known, so only the extras without markers can + # be added. + target_platforms = [None] + + plats_by_extras = {} + for platform in target_platforms: + env = envs.get(platform) + plat_extras = {extra: None for extra in setting.extras or []} + for markers, pkg_extras in extras.items(): + if markers and not env: + continue + if len([m for m in markers if evaluate(m, env = env)]) == len(markers): + plat_extras.update({extra: None for extra in pkg_extras}) + plats_by_extras.setdefault(tuple(sorted(plat_extras)), []).append(platform) + + if len(plats_by_extras) == 1: + return [whl_config_setting( + version = setting.version, + target_platforms = setting.target_platforms, + extras = plats_by_extras.keys()[0], + )] + + return [ + whl_config_setting( + version = setting.version, + target_platforms = plats, + extras = plat_extras, + ) + for plat_extras, plats in plats_by_extras.items() + ] diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 19bb16dee3..6e3deedc04 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -1000,6 +1000,70 @@ def _test_configure_default_pyproject(env): _tests.append(_test_configure_default_pyproject) +def _test_configure_extras(env): + # The extras requested in the pyproject.toml are added to the hub even if + # the lock file does not request them, but only on the platforms that + # match the markers. + pypi = _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + _mod( + name = "rules_python", + parse = [_configure_parse( + pyproject_toml = "pyproject.toml", + target_platforms = ["linux_x86_64", "windows_aarch64"], + )], + configure = [_configure(hub_name = "pypi_a", dependency_groups = ["default"])], + ), + os_name = "linux", + arch_name = "x86_64", + mock_files = { + "pyproject.toml": """\ +[project] +dependencies = [ + "simple[foo]", + "simple[win]; sys_platform == 'win32'", + "other-pkg[not-provided]", + "other-pkg[old]; python_version < '3.10'", +] +""", + "requirements.txt": _CONFIGURE_REQUIREMENTS, + }, + ), + available_interpreters = { + "python_3_15_host": "unit_test_interpreter_target", + }, + minor_mapping = {"3.15": "3.15.19"}, + ) + + pypi.configured_hubs().contains_exactly({"pypi_a": ["other_pkg", "simple"]}) + pypi.hub_whl_map().contains_exactly({"pypi_a": { + "other_pkg": { + "pypi_a_315_other_pkg": [ + whl_config_setting(version = "3.15", extras = ["not_provided"]), + ], + }, + "simple": { + "pypi_a_315_simple": [ + whl_config_setting( + version = "3.15", + target_platforms = ["cp315_linux_x86_64"], + extras = ["foo"], + ), + whl_config_setting( + version = "3.15", + target_platforms = ["cp315_windows_aarch64"], + extras = ["foo", "win"], + ), + ], + }, + "transitive_pkg": { + "pypi_a_315_transitive_pkg": [whl_config_setting(version = "3.15")], + }, + }}) + +_tests.append(_test_configure_extras) + def _test_not_configured_exposes_all(env): # The pyproject.toml does not filter the hub without `pip.configure`. pypi = _configure_parse_modules( diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl index 1f2b13622c..f5f971b923 100644 --- a/tests/pypi/hub_exposure/hub_exposure_tests.bzl +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -3,8 +3,12 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load( "//python/private/pypi:hub_exposure.bzl", + "add_extras", "hub_exposure", ) # buildifier: disable=bzl-visibility +load("//python/private/pypi:pep508_env.bzl", pep508_env = "env") # buildifier: disable=bzl-visibility +load("//python/private/pypi:render_pkg_aliases.bzl", "available_extras") # buildifier: disable=bzl-visibility +load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # buildifier: disable=bzl-visibility load("//tests/support/mocks:mocks.bzl", "mocks") _tests = [] @@ -18,6 +22,14 @@ dev = ["other-pkg"] test = ["simple", "not-in-hub"] """ +_EXTRAS_PYPROJECT = """\ +[project] +dependencies = ["simple[foo]", "simple[Bar]; sys_platform == 'linux'"] + +[dependency-groups] +dev = ["other-pkg[baz]", "simple[foo]", "not-in-hub[qux]"] +""" + def _hub(*, module_name = "root", pyproject = None): return struct( module_name = module_name, @@ -38,12 +50,18 @@ def _configure( def _module(name = "root", is_root = True, configure = []): return mocks.module(name = name, is_root = is_root, configure = configure) -def _exposure(*modules, hubs = None, default_pyproject = None): +def _exposure(*modules, **kwargs): + """Return the exposed packages of the hubs.""" + got, failures = _hub_exposure(*modules, **kwargs) + return got.packages if got != None else None, failures + +def _hub_exposure(*modules, hubs = None, default_pyproject = None): failures = [] got = hub_exposure( mocks.mctx( modules = list(modules), mock_files = { + "extras.toml": _EXTRAS_PYPROJECT, "other.toml": "[project]\ndependencies = [\"other-pkg\"]\n", "pyproject.toml": _PYPROJECT, }, @@ -217,6 +235,224 @@ def _test_configure_module_precedence(env): _tests.append(_test_configure_module_precedence) +def _test_configure_extras(env): + hubs = {"hub": _hub(pyproject = "extras.toml")} + linux = ("sys_platform == 'linux'",) + for groups, want in [ + (["default"], {"hub": {"simple": {(): ["foo"], linux: ["bar"]}}}), + # The packages that are not in the hub are ignored. + (["dev"], {"hub": {"other_pkg": {(): ["baz"]}, "simple": {(): ["foo"]}}}), + # The extras are also returned when all of the packages are exposed. + (["*", "transitive"], {"hub": { + "other_pkg": {(): ["baz"]}, + "simple": {(): ["foo"], linux: ["bar"]}, + }}), + ]: + got, failures = _hub_exposure( + _module(configure = [_configure(dependency_groups = groups)]), + hubs = hubs, + ) + env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) + env.expect.where(groups = groups).that_dict(got.extras).contains_exactly(want) + + # The extras of multiple tags are combined. + got, _ = _hub_exposure( + _module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(dependency_groups = ["dev"]), + ]), + hubs = hubs, + ) + env.expect.that_dict(got.extras).contains_exactly({ + "hub": { + "other_pkg": {(): ["baz"]}, + "simple": {(): ["foo"], linux: ["bar"]}, + }, + }) + + # No extras are requested. + got, _ = _hub_exposure(_module(configure = [_configure(dependency_groups = ["*"])])) + env.expect.that_dict(got.extras).contains_exactly({}) + got, _ = _hub_exposure(_module(configure = [_configure()])) + env.expect.that_dict(got.extras).contains_exactly({}) + +_tests.append(_test_configure_extras) + +def _platforms(*platforms, version = "3.15"): + """The marker environments of the hub platforms like `hub_builder` returns them.""" + return { + version: { + "cp{}_{}_{}".format(version.replace(".", ""), os, arch): pep508_env( + os = os, + arch = arch, + python_version = version + ".0", + ) + for os, arch in platforms + }, + } + +_LINUX = "cp315_linux_x86_64" +_OSX = "cp315_osx_aarch64" +_WINDOWS = "cp315_windows_x86_64" + +def _test_add_extras(env): + whl_map = { + "other_pkg": {"repo_other": [whl_config_setting(version = "3.15")]}, + "simple": { + "repo_linux": [whl_config_setting( + version = "3.15", + target_platforms = [_LINUX], + extras = ["bar"], + )], + "repo_osx": [whl_config_setting( + version = "3.15", + target_platforms = [_OSX], + )], + }, + } + got = add_extras( + whl_map, + {"simple": {(): ["bar", "foo"]}}, + platforms = _platforms(("linux", "x86_64"), ("osx", "aarch64")), + ) + + # The extras without markers are added on all of the platforms. + env.expect.that_dict(got).contains_exactly({ + "other_pkg": {"repo_other": [whl_config_setting(version = "3.15")]}, + "simple": { + "repo_linux": [whl_config_setting( + version = "3.15", + target_platforms = [_LINUX], + extras = ["bar", "foo"], + )], + "repo_osx": [whl_config_setting( + version = "3.15", + target_platforms = [_OSX], + extras = ["bar", "foo"], + )], + }, + }) + + # Nothing to add. + env.expect.that_dict(add_extras(whl_map, {}, platforms = {})).contains_exactly(whl_map) + +_tests.append(_test_add_extras) + +def _test_add_extras_markers(env): + platforms = _platforms(("linux", "x86_64"), ("osx", "aarch64"), ("windows", "x86_64")) + whl_map = { + # A repo for all of the platforms, e.g. a pure Python wheel. + "any": {"repo_any": [whl_config_setting(version = "3.15")]}, + "simple": { + "repo_unix": [whl_config_setting( + version = "3.15", + target_platforms = [_LINUX, _OSX], + extras = ["locked"], + )], + "repo_windows": [whl_config_setting( + version = "3.15", + target_platforms = [_WINDOWS], + )], + }, + } + extras = { + "any": { + ("sys_platform == 'win32'",): ["win"], + # All of the markers need to be true. + ("python_version < '3.12'", "sys_platform == 'linux'"): ["old"], + }, + "simple": { + (): ["foo"], + ("sys_platform == 'win32'",): ["win"], + ("sys_platform == 'linux'",): ["lin"], + }, + } + + got = add_extras(whl_map, extras, platforms = platforms) + + # `foo[win]; sys_platform == "win32"` only adds the extra on Windows, so + # the other platforms do not get the dependencies of the extra, which may + # not be available for them. + env.expect.that_dict(got).contains_exactly({ + "any": { + "repo_any": [ + whl_config_setting( + version = "3.15", + target_platforms = [_LINUX, _OSX], + ), + whl_config_setting( + version = "3.15", + target_platforms = [_WINDOWS], + extras = ["win"], + ), + ], + }, + "simple": { + "repo_unix": [ + whl_config_setting( + version = "3.15", + target_platforms = [_LINUX], + extras = ["foo", "lin", "locked"], + ), + whl_config_setting( + version = "3.15", + target_platforms = [_OSX], + extras = ["foo", "locked"], + ), + ], + "repo_windows": [whl_config_setting( + version = "3.15", + target_platforms = [_WINDOWS], + extras = ["foo", "win"], + )], + }, + }) + + # The hub tells the whl repositories which extras are requested, so that + # they create the targets for the extras that the package does not + # provide, i.e. the hub never points to a target that does not exist. + env.expect.that_dict(available_extras({ + name: { + setting: repo + for repo, settings in repos.items() + for setting in settings + } + for name, repos in got.items() + })).contains_exactly({ + "any": ["win"], + "simple": ["foo", "lin", "locked", "win"], + }) + + # A marker that is the same on all of the platforms does not split the + # config setting. + got = add_extras( + whl_map, + {"any": {("python_version >= '3.12'",): ["new"], ("python_version < '3.12'",): ["old"]}}, + platforms = platforms, + ) + env.expect.that_dict(got["any"]).contains_exactly({ + "repo_any": [whl_config_setting(version = "3.15", extras = ["new"])], + }) + + # Only the extras without markers are added if the platforms of the Python + # version are not known. + got = add_extras(whl_map, extras, platforms = {}) + env.expect.that_dict(got["any"]).contains_exactly(whl_map["any"]) + env.expect.that_dict(got["simple"]).contains_exactly({ + "repo_unix": [whl_config_setting( + version = "3.15", + target_platforms = [_LINUX, _OSX], + extras = ["foo", "locked"], + )], + "repo_windows": [whl_config_setting( + version = "3.15", + target_platforms = [_WINDOWS], + extras = ["foo"], + )], + }) + +_tests.append(_test_add_extras_markers) + def _test_configure_errors(env): got, failures = _exposure( _module(configure = [_configure(hub_name = "missing")]),