Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 80 additions & 15 deletions python/private/pypi/dependency_groups.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@ load("@bazel_skylib//lib:types.bzl", "types")
load("//python/private:normalize_name.bzl", "normalize_name")
load(":pep508_requirement.bzl", "requirement")

# The limit for processing the groups selected by the self references, e.g.
# `my-project[test]`. A group is processed once for each different set of
# markers that it is selected with.
_MAX_SELF_REFERENCES = 10000

def select_dependency_groups(pyproject, specs, _fail = fail):
"""Select the packages named by the given dependency group specifiers.

Expand All @@ -30,6 +35,13 @@ def select_dependency_groups(pyproject, specs, _fail = fail):
* `all`: {type}`bool` whether all of the packages including the
transitive dependencies should be selected, i.e. `"*"` and
`"transitive"` were given.
* `extras`: {type}`dict[str, dict[tuple[str], list[str]]]` the
normalized package names mapped to the extras that the selected
requirements request, e.g. `requests[socks]`. The extras are
keyed by the environment markers that all need to be true for
the extras to be requested, e.g. `("sys_platform == 'win32'",)`
or an empty tuple if there are no markers. When `all` is set,
the extras of all of the groups are returned.
Returns `None` on failure.
"""
groups = {"default": pyproject.dependencies}
Expand All @@ -41,18 +53,17 @@ def select_dependency_groups(pyproject, specs, _fail = fail):
for name, reqs in expanded.items():
groups["group:" + name] = reqs

if "transitive" in specs:
select_all = "transitive" in specs
if select_all:
if "*" not in specs:
_fail("The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: {}".format(
specs,
))
return None
return struct(
# list[str]: sorted normalized package names selected by `specs`.
packages = [],
# bool: True when all hub packages are selected.
all = True,
)

# All of the packages are selected, but the extras are only known
# from the groups.
specs = ["*"]

selected = {}
for spec in specs:
Expand All @@ -70,30 +81,84 @@ def select_dependency_groups(pyproject, specs, _fail = fail):

# A self reference like `my-project[test]` selects the optional
# dependencies, including nested composite extras.
#
# The groups are queued together with the markers of the self references
# that selected them, because the requirements of such groups are only
# requested when the markers are true. A group is processed again if it is
# selected with markers that are not known to be stricter than the ones
# that it was processed with.
project = normalize_name(pyproject.name or "")
packages = {}
queue = list(selected)
for _ in range(len(groups)):
extras = {}
processed = {}
queue = [(key, ()) for key in selected]
for _ in range(_MAX_SELF_REFERENCES):
if not queue:
break
for req in groups[queue.pop(0)]:
key, markers = queue.pop(0)
if _is_processed(processed.setdefault(key, []), markers):
continue
processed[key].append(markers)
for req in groups[key]:
req = requirement(req)
req_markers = _add_marker(markers, req.marker)
if req.name_ != project:
packages[req.name_] = None
if req.extras:
pkg_extras = extras.setdefault(req.name_, {}).setdefault(req_markers, {})
for extra in req.extras:
pkg_extras[extra] = None
continue
for extra in req.extras:
opt_key = "optional:" + extra
if opt_key in groups and opt_key not in selected:
selected[opt_key] = None
queue.append(opt_key)
if opt_key in groups:
queue.append((opt_key, req_markers))

if queue:
_fail("Too many self references with different markers in the dependency groups of '{}'".format(
pyproject.name,
))
return None

return struct(
# list[str]: sorted normalized package names selected by `specs`.
packages = sorted(packages),
packages = [] if select_all else sorted(packages),
# bool: True when all packages including transitive ones are selected.
all = False,
all = select_all,
# dict[str, dict[tuple[str], list[str]]]: the extras of the packages
# keyed by the markers that need to be true.
extras = {
name: {
markers: sorted(pkg_extras)
for markers, pkg_extras in sorted(by_markers.items())
}
for name, by_markers in sorted(extras.items())
},
)

def _add_marker(markers, marker):
if not marker or marker in markers:
return markers
return tuple(sorted(markers + (marker,)))

def _is_processed(processed, markers):
"""Tell if a group was already processed with the same or looser markers.

Args:
processed: {type}`list[tuple[str]]` the markers that the group was
processed with.
markers: {type}`tuple[str]` the markers to check.

Returns:
{type}`bool` `True` if all of the markers of a processed entry are in
`markers`, i.e. the requirements of the group are already requested
whenever `markers` are true.
"""
for known in processed:
if len([m for m in known if m in markers]) == len(known):
return True
return False

def _match(spec, groups):
"""Match a dependency group specifier against normalized group keys.

Expand Down
100 changes: 100 additions & 0 deletions tests/pypi/dependency_groups/dependency_groups_tests.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,106 @@ def _test_select_all(env):

_tests.append(_test_select_all)

def _test_select_extras(env):
pyproject = struct(
name = "my-project",
dependencies = ["main-pkg[Foo_Bar, baz]>=1.0", "plain-pkg"],
optional_dependencies = {
"all": ["my-project[docs]"],
"docs": ["docs-pkg[extra]", "main-pkg[docs]"],
},
dependency_groups = {
"dev": ["dev-pkg[cli]", {"include-group": "test"}],
"test": [
"test-pkg[cov]; sys_platform == 'linux'",
"test-pkg[win]; sys_platform == 'win32'",
"test-pkg[cov]",
],
},
)
for specs, want in [
(["default"], {"main_pkg": {(): ["baz", "foo_bar"]}}),
# The markers of the requirements are kept, so that the extras can be
# added only for the platforms that request them.
(["dev"], {
"dev_pkg": {(): ["cli"]},
"test_pkg": {
(): ["cov"],
("sys_platform == 'linux'",): ["cov"],
("sys_platform == 'win32'",): ["win"],
},
}),
# The extras of the self reference are the optional dependencies of
# the project, not the extras of a package.
(["all"], {"docs_pkg": {(): ["extra"]}, "main_pkg": {(): ["docs"]}}),
(["default", "docs"], {
"docs_pkg": {(): ["extra"]},
"main_pkg": {(): ["baz", "docs", "foo_bar"]},
}),
]:
got, failures = _select(specs, pyproject = pyproject)
env.expect.where(specs = specs).that_collection(failures).contains_exactly([])
env.expect.where(specs = specs).that_dict(got.extras).contains_exactly(want)

# The extras of all of the groups are returned when everything is selected.
got, failures = _select(["*", "transitive"], pyproject = pyproject)
env.expect.that_collection(failures).contains_exactly([])
env.expect.that_bool(got.all).equals(True)
env.expect.that_collection(got.packages).contains_exactly([])
env.expect.that_dict(got.extras).keys().contains_exactly([
"dev_pkg",
"docs_pkg",
"main_pkg",
"test_pkg",
])

_tests.append(_test_select_extras)

def _test_select_extras_self_reference_markers(env):
pyproject = struct(
name = "my-project",
dependencies = [
"my-project[win]; sys_platform == 'win32'",
"my-project[cycle]; python_version < '3.12'",
],
optional_dependencies = {
"cycle": ["my-project[win]", "my-project[cycle]", "cycle-pkg[c]"],
"win": [
"win-pkg[a]",
"win-pkg[b]; python_version < '3.12'",
"my-project[cycle]",
],
},
dependency_groups = {},
)

# The requirements of a group that is selected by a self reference with a
# marker are only requested if the marker is true.
got, failures = _select(["default"], pyproject = pyproject)
env.expect.that_collection(failures).contains_exactly([])
env.expect.that_collection(got.packages).contains_exactly(["cycle_pkg", "win_pkg"])
env.expect.that_dict(got.extras).contains_exactly({
"cycle_pkg": {
("python_version < '3.12'",): ["c"],
("sys_platform == 'win32'",): ["c"],
},
"win_pkg": {
("python_version < '3.12'",): ["a", "b"],
("python_version < '3.12'", "sys_platform == 'win32'"): ["b"],
("sys_platform == 'win32'",): ["a"],
},
})

# Selecting the group directly requests it on all of the platforms.
got, failures = _select(["default", "win"], pyproject = pyproject)
env.expect.that_collection(failures).contains_exactly([])
env.expect.that_dict(got.extras["win_pkg"]).contains_at_least({
(): ["a"],
("python_version < '3.12'",): ["b"],
})

_tests.append(_test_select_extras_self_reference_markers)

def _test_select_empty_wildcards(env):
pyproject = struct(
name = "my-project",
Expand Down
Loading