From 59fd85cf8f7e0d2887f8eb7618762a729b609d3e Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 22 Nov 2026 23:25:41 +0900 Subject: [PATCH] feat(pypi): collect the extras of the selected dependency groups The requirements in the pyproject.toml dependency groups can request extras, e.g. requests[socks]. select_dependency_groups now returns them keyed by the environment markers that need to be true for the extra to be requested, including the markers of the self references that select a group. Nothing uses them yet. Work towards #2786 --- python/private/pypi/dependency_groups.bzl | 95 ++++++++++++++--- .../dependency_groups_tests.bzl | 100 ++++++++++++++++++ 2 files changed, 180 insertions(+), 15 deletions(-) diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl index 13b164c2b4..3355fdc0b5 100644 --- a/python/private/pypi/dependency_groups.bzl +++ b/python/private/pypi/dependency_groups.bzl @@ -4,6 +4,11 @@ load("@bazel_skylib//lib:types.bzl", "types") load("//python/private:normalize_name.bzl", "normalize_name") load(":pep508_requirement.bzl", "requirement") +# The limit for processing the groups selected by the self references, e.g. +# `my-project[test]`. A group is processed once for each different set of +# markers that it is selected with. +_MAX_SELF_REFERENCES = 10000 + def select_dependency_groups(pyproject, specs, _fail = fail): """Select the packages named by the given dependency group specifiers. @@ -30,6 +35,13 @@ def select_dependency_groups(pyproject, specs, _fail = fail): * `all`: {type}`bool` whether all of the packages including the transitive dependencies should be selected, i.e. `"*"` and `"transitive"` were given. + * `extras`: {type}`dict[str, dict[tuple[str], list[str]]]` the + normalized package names mapped to the extras that the selected + requirements request, e.g. `requests[socks]`. The extras are + keyed by the environment markers that all need to be true for + the extras to be requested, e.g. `("sys_platform == 'win32'",)` + or an empty tuple if there are no markers. When `all` is set, + the extras of all of the groups are returned. Returns `None` on failure. """ groups = {"default": pyproject.dependencies} @@ -41,18 +53,17 @@ def select_dependency_groups(pyproject, specs, _fail = fail): for name, reqs in expanded.items(): groups["group:" + name] = reqs - if "transitive" in specs: + select_all = "transitive" in specs + if select_all: if "*" not in specs: _fail("The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: {}".format( specs, )) return None - return struct( - # list[str]: sorted normalized package names selected by `specs`. - packages = [], - # bool: True when all hub packages are selected. - all = True, - ) + + # All of the packages are selected, but the extras are only known + # from the groups. + specs = ["*"] selected = {} for spec in specs: @@ -70,30 +81,84 @@ def select_dependency_groups(pyproject, specs, _fail = fail): # A self reference like `my-project[test]` selects the optional # dependencies, including nested composite extras. + # + # The groups are queued together with the markers of the self references + # that selected them, because the requirements of such groups are only + # requested when the markers are true. A group is processed again if it is + # selected with markers that are not known to be stricter than the ones + # that it was processed with. project = normalize_name(pyproject.name or "") packages = {} - queue = list(selected) - for _ in range(len(groups)): + extras = {} + processed = {} + queue = [(key, ()) for key in selected] + for _ in range(_MAX_SELF_REFERENCES): if not queue: break - for req in groups[queue.pop(0)]: + key, markers = queue.pop(0) + if _is_processed(processed.setdefault(key, []), markers): + continue + processed[key].append(markers) + for req in groups[key]: req = requirement(req) + req_markers = _add_marker(markers, req.marker) if req.name_ != project: packages[req.name_] = None + if req.extras: + pkg_extras = extras.setdefault(req.name_, {}).setdefault(req_markers, {}) + for extra in req.extras: + pkg_extras[extra] = None continue for extra in req.extras: opt_key = "optional:" + extra - if opt_key in groups and opt_key not in selected: - selected[opt_key] = None - queue.append(opt_key) + if opt_key in groups: + queue.append((opt_key, req_markers)) + + if queue: + _fail("Too many self references with different markers in the dependency groups of '{}'".format( + pyproject.name, + )) + return None return struct( # list[str]: sorted normalized package names selected by `specs`. - packages = sorted(packages), + packages = [] if select_all else sorted(packages), # bool: True when all packages including transitive ones are selected. - all = False, + all = select_all, + # dict[str, dict[tuple[str], list[str]]]: the extras of the packages + # keyed by the markers that need to be true. + extras = { + name: { + markers: sorted(pkg_extras) + for markers, pkg_extras in sorted(by_markers.items()) + } + for name, by_markers in sorted(extras.items()) + }, ) +def _add_marker(markers, marker): + if not marker or marker in markers: + return markers + return tuple(sorted(markers + (marker,))) + +def _is_processed(processed, markers): + """Tell if a group was already processed with the same or looser markers. + + Args: + processed: {type}`list[tuple[str]]` the markers that the group was + processed with. + markers: {type}`tuple[str]` the markers to check. + + Returns: + {type}`bool` `True` if all of the markers of a processed entry are in + `markers`, i.e. the requirements of the group are already requested + whenever `markers` are true. + """ + for known in processed: + if len([m for m in known if m in markers]) == len(known): + return True + return False + def _match(spec, groups): """Match a dependency group specifier against normalized group keys. diff --git a/tests/pypi/dependency_groups/dependency_groups_tests.bzl b/tests/pypi/dependency_groups/dependency_groups_tests.bzl index a53905bb04..a3817cd835 100644 --- a/tests/pypi/dependency_groups/dependency_groups_tests.bzl +++ b/tests/pypi/dependency_groups/dependency_groups_tests.bzl @@ -100,6 +100,106 @@ def _test_select_all(env): _tests.append(_test_select_all) +def _test_select_extras(env): + pyproject = struct( + name = "my-project", + dependencies = ["main-pkg[Foo_Bar, baz]>=1.0", "plain-pkg"], + optional_dependencies = { + "all": ["my-project[docs]"], + "docs": ["docs-pkg[extra]", "main-pkg[docs]"], + }, + dependency_groups = { + "dev": ["dev-pkg[cli]", {"include-group": "test"}], + "test": [ + "test-pkg[cov]; sys_platform == 'linux'", + "test-pkg[win]; sys_platform == 'win32'", + "test-pkg[cov]", + ], + }, + ) + for specs, want in [ + (["default"], {"main_pkg": {(): ["baz", "foo_bar"]}}), + # The markers of the requirements are kept, so that the extras can be + # added only for the platforms that request them. + (["dev"], { + "dev_pkg": {(): ["cli"]}, + "test_pkg": { + (): ["cov"], + ("sys_platform == 'linux'",): ["cov"], + ("sys_platform == 'win32'",): ["win"], + }, + }), + # The extras of the self reference are the optional dependencies of + # the project, not the extras of a package. + (["all"], {"docs_pkg": {(): ["extra"]}, "main_pkg": {(): ["docs"]}}), + (["default", "docs"], { + "docs_pkg": {(): ["extra"]}, + "main_pkg": {(): ["baz", "docs", "foo_bar"]}, + }), + ]: + got, failures = _select(specs, pyproject = pyproject) + env.expect.where(specs = specs).that_collection(failures).contains_exactly([]) + env.expect.where(specs = specs).that_dict(got.extras).contains_exactly(want) + + # The extras of all of the groups are returned when everything is selected. + got, failures = _select(["*", "transitive"], pyproject = pyproject) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_bool(got.all).equals(True) + env.expect.that_collection(got.packages).contains_exactly([]) + env.expect.that_dict(got.extras).keys().contains_exactly([ + "dev_pkg", + "docs_pkg", + "main_pkg", + "test_pkg", + ]) + +_tests.append(_test_select_extras) + +def _test_select_extras_self_reference_markers(env): + pyproject = struct( + name = "my-project", + dependencies = [ + "my-project[win]; sys_platform == 'win32'", + "my-project[cycle]; python_version < '3.12'", + ], + optional_dependencies = { + "cycle": ["my-project[win]", "my-project[cycle]", "cycle-pkg[c]"], + "win": [ + "win-pkg[a]", + "win-pkg[b]; python_version < '3.12'", + "my-project[cycle]", + ], + }, + dependency_groups = {}, + ) + + # The requirements of a group that is selected by a self reference with a + # marker are only requested if the marker is true. + got, failures = _select(["default"], pyproject = pyproject) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_collection(got.packages).contains_exactly(["cycle_pkg", "win_pkg"]) + env.expect.that_dict(got.extras).contains_exactly({ + "cycle_pkg": { + ("python_version < '3.12'",): ["c"], + ("sys_platform == 'win32'",): ["c"], + }, + "win_pkg": { + ("python_version < '3.12'",): ["a", "b"], + ("python_version < '3.12'", "sys_platform == 'win32'"): ["b"], + ("sys_platform == 'win32'",): ["a"], + }, + }) + + # Selecting the group directly requests it on all of the platforms. + got, failures = _select(["default", "win"], pyproject = pyproject) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got.extras["win_pkg"]).contains_at_least({ + (): ["a"], + ("python_version < '3.12'",): ["b"], + }) + +_tests.append(_test_select_extras_self_reference_markers) + def _test_select_empty_wildcards(env): pyproject = struct( name = "my-project",