Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions docs/pypi/download.md
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,29 @@ the `pyproject.toml` selects the optional dependencies of the project. The
packages that are not part of the hub are ignored, so a single `pyproject.toml`
can list tools that are missing from the lock file.

The extras requested in the selected groups are added to the hub even if the
lock file does not request them, for example because the lock file was
generated with `--strip-extras`. With the following `pyproject.toml`:

```toml
[project]
dependencies = [
"requests[socks]",
"keyring[completion]; sys_platform == 'win32'",
]
```

the hub has the `@pypi_hub//requests:[socks]` target and
`@pypi_hub//requests` includes the `socks` extra. The environment markers of
the requirements are evaluated for each target platform of the hub, so
`@pypi_hub//keyring` only includes the `completion` extra on Windows and
`@pypi_hub//keyring:[completion]` is the same as `@pypi_hub//keyring:[]` on
the other platforms. With `["*", "transitive"]` the extras of all of the
groups are added. The dependencies of the extras, such as `pysocks`, still
have to be in the lock file, and an extra that the package does not provide is
ignored with a warning. See [extras](./use.md#extras) for the targets of the
extras.

:::{versionadded} VERSION_NEXT_FEATURE
:::

Expand Down
7 changes: 7 additions & 0 deletions news/4522.added.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
(pypi) {obj}`pip.configure` now adds the extras requested in the selected
`pyproject.toml` dependency groups, e.g. `requests[socks]`, to the hub, so
that the `@pypi//requests:[socks]` target exists and `@pypi//requests`
includes the extra even if the lock file does not request it. The environment
markers of the requirements are evaluated for each target platform, so an
extra is only added for the platforms that request it
([#2786](https://github.com/bazel-contrib/rules_python/issues/2786)).
2 changes: 2 additions & 0 deletions python/private/pypi/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -687,6 +687,8 @@ bzl_library(
srcs = ["hub_exposure.bzl"],
deps = [
":dependency_groups",
":pep508_evaluate",
":whl_config_setting",
"//python/private:pyproject_utils",
],
)
Expand Down
27 changes: 23 additions & 4 deletions python/private/pypi/extension.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ load("//python/private:pyproject_utils.bzl", "read_pyproject", "version_from_req
load("//python/private:repo_utils.bzl", "repo_utils")
load(":default_platforms.bzl", "default_platforms")
load(":hub_builder.bzl", "hub_builder")
load(":hub_exposure.bzl", "hub_exposure")
load(":hub_exposure.bzl", "add_extras", "hub_exposure")
load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json")
load(":labels.bzl", "extra_label")
load(":parse_whl_name.bzl", "parse_whl_name")
Expand Down Expand Up @@ -358,6 +358,7 @@ You cannot use both the additive_build_content and additive_build_content_file a
extra_aliases = {}
whl_libraries = {}
hubs = {}
hub_platforms = {}
for hub in pip_hub_map.values():
out = hub.build()

Expand All @@ -372,6 +373,7 @@ You cannot use both the additive_build_content and additive_build_content_file a
hub_group_map[hub.name] = out.group_map
hub_whl_map[hub.name] = out.whl_map
hub_target_platform_envs[hub.name] = out.target_platform_envs
hub_platforms[hub.name] = out.platforms
hubs[hub.name] = struct(
# str: module that created the hub.
module_name = hub.module_name,
Expand All @@ -381,14 +383,24 @@ You cannot use both the additive_build_content and additive_build_content_file a
packages = out.whl_map.keys(),
)

configured_hubs = hub_exposure(
exposure = hub_exposure(
module_ctx,
hubs = hubs,
default_pyproject = config.pyproject_toml,
_fail = _fail,
)
if configured_hubs == None:
if exposure == None:
return None
configured_hubs = exposure.packages

# The extras requested in the `pyproject.toml`, e.g. `requests[socks]`,
# are added to the hub even if the lock file does not have them.
for hub_name, extras in exposure.extras.items():
hub_whl_map[hub_name] = add_extras(
hub_whl_map[hub_name],
extras,
platforms = hub_platforms[hub_name],
)

# The `all_*requirements` only list the selected packages that are
# available on all of the platforms, while all of the selected packages
Expand Down Expand Up @@ -1182,6 +1194,11 @@ Each item is one of:
packages of the hub including the transitive dependencies, i.e. all of the
packages from the requirements files or the `uv.lock` file.

The extras of the selected requirements, e.g. `requests[socks]`, are also
added to the hub for the target platforms that match the environment markers
of the requirements. With `"transitive"` the extras of all of the groups are
added.

See {ref}`Configuring the Exposed Packages <pip-configure>` for examples.

:::{versionadded} VERSION_NEXT_FEATURE
Expand Down Expand Up @@ -1219,7 +1236,9 @@ The packages are selected from a `pyproject.toml` file using the
`[project] dependencies`, `[project.optional-dependencies]` and PEP 735
`[dependency-groups]` tables. The other packages of the hub, e.g. the
transitive dependencies, are still used by the packages that need them, but
they are not public.
they are not public. The extras requested by the selected requirements, e.g.
`requests[socks]`, are added to the hub even if the lock file does not request
them.

The tags work like a builder: they are applied in order and multiple tags for
the same hub are combined, so each tag can configure a different part of the
Expand Down
12 changes: 12 additions & 0 deletions python/private/pypi/hub_builder.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ def _build(self):
exposed_packages = [],
whl_libraries = {},
target_platform_envs = [],
platforms = {},
)
if self._logger.failed():
return ret
Expand Down Expand Up @@ -154,6 +155,17 @@ def _build(self):
# requirements.
# list[dict[str, str]]
target_platform_envs = self._target_platform_envs.values(),
# The marker environments of the target platforms for each Python
# version, the keys are the same as in the `whl_config_setting`
# values of the `whl_map`.
# dict[str python_version, dict[str target_platform, dict[str, str]]]
platforms = {
_major_minor_version(python_version): {
whl_config_setting(target_platforms = [name]).target_platforms[0]: platform.env
for name, platform in platforms.items()
}
for python_version, platforms in self._platforms.items()
},
)

def _pip_parse(self, module_ctx, pip_attr, default_envsubst = [], python_version = None):
Expand Down
128 changes: 120 additions & 8 deletions python/private/pypi/hub_exposure.bzl
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
"""Compute the packages exposed by the hubs configured with `pip.configure`."""
"""Compute the packages and extras exposed by the hubs configured with `pip.configure`."""

load("//python/private:pyproject_utils.bzl", "read_pyproject")
load(":dependency_groups.bzl", "select_dependency_groups")
load(":pep508_evaluate.bzl", "evaluate")
load(":whl_config_setting.bzl", "whl_config_setting")

def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail):
"""Compute the packages exposed by the hubs configured with `pip.configure`.
Expand All @@ -18,10 +20,16 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail):
_fail: {type}`callable` the failure function, mainly for testing.

Returns:
{type}`dict[str, list[str]] | None` the configured hubs mapped to the
sorted packages that they expose. The hubs that are not configured or
that expose all of their packages are not included, so that they
behave the same. Returns `None` on failure.
{type}`struct | None` with the attributes below or `None` on failure:
* `packages`: {type}`dict[str, list[str]]` the configured hubs
mapped to the sorted packages that they expose. The hubs that are
not configured or that expose all of their packages are not
included, so that they behave the same.
* `extras`: {type}`dict[str, dict[str, dict[tuple[str], list[str]]]]`
the configured hubs mapped to the packages and the extras that
the selected dependency groups request for them, see
{obj}`select_dependency_groups` for the format. It is used with
{obj}`add_extras`.
"""

# The root module wins over the owner of the hub when both configure it.
Expand All @@ -35,6 +43,7 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail):

configured = {}
exposes_all = {}
extras = {}
for mod in module_ctx.modules:
# The tags of a module are applied in order like a builder: a tag
# without a `pyproject_toml` uses the last one set for the hub.
Expand Down Expand Up @@ -90,8 +99,111 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail):
if name in info.packages:
exposed[name] = None

hub_extras = extras.setdefault(hub, {})
for name, by_markers in selection.extras.items():
if name not in info.packages:
continue
for markers, pkg_extras in by_markers.items():
merged = hub_extras.setdefault(name, {}).setdefault(markers, {})
for extra in pkg_extras:
merged[extra] = None

return struct(
# dict[str, list[str]]: configured hubs and exposed packages.
packages = {
hub: sorted(packages)
for hub, packages in configured.items()
if hub not in exposes_all
},
# dict[str, dict[str, dict[tuple[str], list[str]]]]: configured hubs
# and the extras of their packages by the markers.
extras = {
hub: {
name: {
markers: sorted(pkg_extras)
for markers, pkg_extras in sorted(by_markers.items())
}
for name, by_markers in sorted(hub_extras.items())
}
for hub, hub_extras in extras.items()
if hub_extras
},
)

def add_extras(whl_map, extras, *, platforms):
"""Add the extras requested by `pip.configure` to the hub `whl_map`.

The extras are added to the config settings of the package, so that the
hub has the `[<extra>]` targets and the package target includes them. The
environment markers of the requirements are evaluated for each target
platform, like it is done for the requirements files, so an extra is only
added for the platforms that request it. If the platforms of a config
setting request different extras, the config setting is split.

Args:
whl_map: {type}`dict[str, dict[str, list[struct]]]` the packages
mapped to the repo names and their {obj}`whl_config_setting`
values.
extras: {type}`dict[str, dict[tuple[str], list[str]]]` the packages
mapped to the normalized extras to add keyed by the markers that
all need to be true on a target platform.
platforms: {type}`dict[str, dict[str, dict[str, str]]]` the Python
versions of the hub mapped to their target platforms and the
marker environments of the platforms. The versions and the
platforms are the same as in the {obj}`whl_config_setting`
values.

Returns:
{type}`dict[str, dict[str, list[struct]]]` the new `whl_map`.
"""
if not extras:
return whl_map

return {
hub: sorted(packages)
for hub, packages in configured.items()
if hub not in exposes_all
name: {
repo: [
new_setting
for setting in settings
for new_setting in _add_extras(setting, extras[name], platforms)
]
for repo, settings in repos.items()
} if name in extras else repos
for name, repos in whl_map.items()
}

def _add_extras(setting, extras, platforms):
envs = platforms.get(setting.version, {})

# A config setting without the platforms is used for all of them.
target_platforms = setting.target_platforms or sorted(envs)
if not target_platforms:
# The platforms are not known, so only the extras without markers can
# be added.
target_platforms = [None]

plats_by_extras = {}
for platform in target_platforms:
env = envs.get(platform)
plat_extras = {extra: None for extra in setting.extras or []}
for markers, pkg_extras in extras.items():
if markers and not env:
continue
if len([m for m in markers if evaluate(m, env = env)]) == len(markers):
plat_extras.update({extra: None for extra in pkg_extras})
plats_by_extras.setdefault(tuple(sorted(plat_extras)), []).append(platform)

if len(plats_by_extras) == 1:
return [whl_config_setting(
version = setting.version,
target_platforms = setting.target_platforms,
extras = plats_by_extras.keys()[0],
)]

return [
whl_config_setting(
version = setting.version,
target_platforms = plats,
extras = plat_extras,
)
for plat_extras, plats in plats_by_extras.items()
]
64 changes: 64 additions & 0 deletions tests/pypi/extension/extension_tests.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -1000,6 +1000,70 @@ def _test_configure_default_pyproject(env):

_tests.append(_test_configure_default_pyproject)

def _test_configure_extras(env):
# The extras requested in the pyproject.toml are added to the hub even if
# the lock file does not request them, but only on the platforms that
# match the markers.
pypi = _parse_modules(
env,
module_ctx = _pypi_mock_mctx(
_mod(
name = "rules_python",
parse = [_configure_parse(
pyproject_toml = "pyproject.toml",
target_platforms = ["linux_x86_64", "windows_aarch64"],
)],
configure = [_configure(hub_name = "pypi_a", dependency_groups = ["default"])],
),
os_name = "linux",
arch_name = "x86_64",
mock_files = {
"pyproject.toml": """\
[project]
dependencies = [
"simple[foo]",
"simple[win]; sys_platform == 'win32'",
"other-pkg[not-provided]",
"other-pkg[old]; python_version < '3.10'",
]
""",
"requirements.txt": _CONFIGURE_REQUIREMENTS,
},
),
available_interpreters = {
"python_3_15_host": "unit_test_interpreter_target",
},
minor_mapping = {"3.15": "3.15.19"},
)

pypi.configured_hubs().contains_exactly({"pypi_a": ["other_pkg", "simple"]})
pypi.hub_whl_map().contains_exactly({"pypi_a": {
"other_pkg": {
"pypi_a_315_other_pkg": [
whl_config_setting(version = "3.15", extras = ["not_provided"]),
],
},
"simple": {
"pypi_a_315_simple": [
whl_config_setting(
version = "3.15",
target_platforms = ["cp315_linux_x86_64"],
extras = ["foo"],
),
whl_config_setting(
version = "3.15",
target_platforms = ["cp315_windows_aarch64"],
extras = ["foo", "win"],
),
],
},
"transitive_pkg": {
"pypi_a_315_transitive_pkg": [whl_config_setting(version = "3.15")],
},
}})

_tests.append(_test_configure_extras)

def _test_not_configured_exposes_all(env):
# The pyproject.toml does not filter the hub without `pip.configure`.
pypi = _configure_parse_modules(
Expand Down
Loading
Loading