Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion docs/pypi/use.md
Original file line number Diff line number Diff line change
Expand Up @@ -126,11 +126,26 @@ so they do not need to be in the lock file, e.g. when the requirements are
compiled with the extras stripped. This does not change `@pypi//bar`, which
only includes the extras requested in the lock file. The dependencies of the
extra that are not in the lock file are left out like any other dependency
that is not in the lock file. If `bar` does not provide the `socks` extra,
that is not in the lock file, see below for the warning that is printed if
they are needed. If `bar` does not provide the `socks` extra,
then only `bar` is used and a warning is printed when `@pypi//foo` is
analyzed, e.g. `WARNING: bar 1.0 does not provide the extra 'socks'`, which is
how `pip` handles the unknown extras.

The dependencies from the wheel `METADATA` that are not in the lock file are
left out, which is correct if they are not needed on any of the target
platforms of the hub, e.g. `pywin32; sys_platform == "win32"` in a hub that is
only for Linux. If the marker of such a dependency can be true on any of the
target platforms that have requirements, then a warning that names the missing
package is printed, because importing it may fail at runtime. For example, if
the lock file is compiled for Python 3.12 only and the hub is also used with
Python 3.10, where `coverage[toml]` needs `tomli`, then the warning asks to add
`tomli` to the lock file. The fix is to lock for all of the target platforms
and Python versions of the hub. The warnings about the dependencies of an
extra are only printed if the extra is used, i.e. it is requested in the lock
file or another package depends on it. The warnings are not printed when using
`pip_parse` in `WORKSPACE`.

If the lock file requests an extra that the package does not provide, e.g.
`foo[nope]`, then `@pypi//foo:[nope]` is the same as `@pypi//foo:[]`, because
an unknown extra does not have any dependencies. A warning is printed when the
Expand Down
5 changes: 4 additions & 1 deletion news/2786.changed.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,5 +12,8 @@ different extras on different platforms now uses a single whl repository. The
`@pypi//foo` targets still include the extras requested in the lock file. An
extra that the package does not provide is still ignored, but a warning is now
printed like `pip` does, e.g.
`WARNING: foo 1.0 does not provide the extra 'nope'`
`WARNING: foo 1.0 does not provide the extra 'nope'`. A warning is also
printed about the dependencies from the wheel `METADATA` that are left out
because they are not in the lock file, if they are needed on any of the
target platforms of the hub
([#2786](https://github.com/bazel-contrib/rules_python/issues/2786)).
16 changes: 16 additions & 0 deletions python/private/pypi/extra_dep.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,10 @@ def _extra_select_impl(ctx):
extra,
))

for warning in ctx.attr.warnings.get(extra, []):
# buildifier: disable=print
print(warning)

return _forward(ctx)

extra_select = rule(
Expand All @@ -154,6 +158,14 @@ the target are forwarded.
"extras": attr.string_list(
doc = "The normalized extras that have a target, a warning is printed if any other extra is requested.",
),
"warnings": attr.string_list_dict(
doc = """\
The warnings to print when the extra in the key is requested, e.g. about the
dependencies of the extra that are not in the hub. They are printed here and
not when the targets are created, because only the extras that are used
matter.
""",
),
"_allowlist_function_transition": attr.label(
default = "@bazel_tools//tools/allowlists/function_transition_allowlist",
),
Expand All @@ -169,6 +181,7 @@ def extra_select_targets(
targets,
default,
distribution = "",
warnings = {},
native = native,
rules = None,
visibility = None):
Expand All @@ -183,6 +196,8 @@ def extra_select_targets(
unknown extras.
distribution: {type}`str` the name and the version of the package
used in the warning about the unknown extras.
warnings: {type}`dict[str, list[str]]` the warnings to print when
the extra in the key is requested.
native: the native struct for overriding in tests.
rules: a struct with the `extra_select` rule for overriding in tests.
visibility: {type}`list[str]` the visibility of the target.
Expand All @@ -203,5 +218,6 @@ def extra_select_targets(
deps = select(deps | {"//conditions:default": [default]}) if deps else [default],
distribution = distribution,
extras = sorted(targets),
warnings = warnings,
visibility = visibility,
)
4 changes: 3 additions & 1 deletion python/private/pypi/generate_whl_library_build_bazel.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ _RENDER = {
"requires_dist": render.list,
"srcs_exclude": render.list,
"tags": render.list,
"target_platform_envs": str,
}

# NOTE @aignas 2024-10-25: We have to keep this so that files in
Expand Down Expand Up @@ -98,8 +99,9 @@ def generate_whl_library_build_bazel(
if config_load:
# The config is also needed without any deps, because it has the
# extras that are requested from this package.
loads.append("""load("{}", "{}")""".format(config_load, "packages"))
loads.append("""load("{}", "packages", "target_platform_envs")""".format(config_load))
kwargs["include"] = "packages"
kwargs["target_platform_envs"] = "target_platform_envs"
if requires_dist:
kwargs["requires_dist"] = requires_dist

Expand Down
75 changes: 63 additions & 12 deletions python/private/pypi/whl_library_deps_targets.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ def whl_library_deps_targets(
provides_extra = [],
extras = [],
include = [],
target_platform_envs = [],
group_deps = [],
group_name = None,
dep_template,
Expand Down Expand Up @@ -72,6 +73,12 @@ def whl_library_deps_targets(
`[<extra>]` targets are also created for the requested extras of
this package, see `extras` for the ones that the package does not
provide.
target_platform_envs: {type}`list[dict[str, str]]` The marker
environments of the target platforms of the hub. They are used
to warn about the dependencies that are left out because they
are not in `include`, but are needed on any of the target
platforms. No warnings are printed about such dependencies if it
is empty.
group_name: {type}`str | None` name of the dependency group (if any).
dep_template: {type}`str | None` The dep_template to use.
tags: {type}`list[str]` The tags set on the targets.
Expand Down Expand Up @@ -220,6 +227,44 @@ def whl_library_deps_targets(

return ":" + extra_label(extra)

# The dependencies of each target. The dependencies that are not in the
# hub are left out, which is only correct if they are not needed on any of
# the target platforms, so warn about the rest of them. The warnings about
# the dependencies of the extras that are not requested in the hub are
# only printed when another package depends on the extra, otherwise every
# package with extras that are not used would print them.
package_deps = {}
missing = {}
missing_by_extra = {}
for target, extra, want_extras in pkg_targets if requires_dist and hasattr(rules, "py_library") else []:
package_deps[target] = deps(
name = self_name,
requires_dist = requires_dist,
include = include,
excludes = members if is_member(extra) else [],
extras = want_extras,
keep_extras = True,
envs = target_platform_envs,
)
if not extra or extra in hub_extras:
for dep, reqs in package_deps[target].missing.items():
missing.setdefault(dep, {}).update({r: None for r in reqs})
else:
missing_by_extra[extra] = package_deps[target].missing

for dep, reqs in sorted(missing.items()):
getattr(rules, "warn", _warn)(_missing_dep_warning(distribution, dep, reqs))

extra_warnings = {}
for extra, extra_missing in sorted(missing_by_extra.items()):
for dep, reqs in sorted(extra_missing.items()):
# The rest are already printed.
reqs = [r for r in reqs if r not in missing.get(dep, {})]
if reqs:
extra_warnings.setdefault(extra, []).append(
_missing_dep_warning(distribution, dep, reqs),
)

# The packages that depend on an extra of this package use this target
# with the requested extra, because the hub does not know which extras
# this package provides. An extra without a target is the same as the
Expand All @@ -230,6 +275,7 @@ def whl_library_deps_targets(
targets = {extra: _extra_target(extra) for extra in extras_targets},
default = _extra_target(""),
distribution = distribution,
warnings = extra_warnings,
native = native,
rules = rules,
visibility = visibility,
Expand Down Expand Up @@ -281,26 +327,18 @@ def whl_library_deps_targets(
visibility = _visibility(""),
)

for name, extra, want_extras in pkg_targets if hasattr(rules, "py_library") else []:
package_deps = deps(
name = self_name,
requires_dist = requires_dist,
include = include,
excludes = members if is_member(extra) else [],
extras = want_extras,
keep_extras = True,
)
for target, extra, _ in pkg_targets if hasattr(rules, "py_library") else []:
rules.py_library(
name = name,
name = target,
# We include as srcs to ensure that the (locations :pkg) works as expected.
srcs = [repo_label(PY_SRCS_LABEL)],
deps = _deps(
# We include as deps, so that `PyInfo` and friends (e.g. `pyi_srcs`) get
# propagated. Just passing the target as `srcs` is not enough to propagate
# `pyi_srcs`, see `tests/base_rules/py_library`.
deps = [repo_label(PY_SRCS_LABEL)],
package_deps = package_deps,
conditions = _conditions(settings, package_deps),
package_deps = package_deps[target],
conditions = _conditions(settings, package_deps[target]),
label = _pkg_label,
),
# Disable precompilation on this wrapper target to prevent duplicate
Expand All @@ -325,6 +363,19 @@ def whl_library_deps_targets(
visibility = ["//visibility:private"],
)

def _missing_dep_warning(distribution, dep, reqs):
return (
"WARNING: {distribution} requires '{reqs}', but '{dep}' is not in " +
"the requirements of the hub, so it is left out and it may be " +
"missing at runtime on some of the target platforms. Add '{dep}' " +
"to the lock file, e.g. by locking for all of the target platforms " +
"and Python versions of the hub."
).format(
distribution = distribution,
reqs = "', '".join(sorted(reqs)),
dep = dep,
)

def _warn(msg):
# buildifier: disable=print
print(msg)
Expand Down
5 changes: 5 additions & 0 deletions python/private/pypi/whl_library_targets.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ def whl_library_targets(
extras = [],
entry_points = {},
include = [],
target_platform_envs = [],
group_deps = [],
group_name = None,
dep_template = None,
Expand Down Expand Up @@ -61,6 +62,9 @@ def whl_library_targets(
{obj}`whl_library_deps_targets`.
entry_points: {type}`list[dict]` A list of parsed entry point definitions.
include: {type}`list[str]` The list of packages to include.
target_platform_envs: {type}`list[dict[str, str]]` The marker
environments of the target platforms of the hub, see
{obj}`whl_library_deps_targets`.
group_name: {type}`str | None` name of the dependency group (if any).
dep_template: {type}`str | None` The dep_template to use.
data_exclude: {type}`list[str]` The globs for data attribute exclusion.
Expand Down Expand Up @@ -103,6 +107,7 @@ def whl_library_targets(
provides_extra = provides_extra,
extras = extras,
include = include, # only needed if requires_dist is present
target_platform_envs = target_platform_envs,
repo = None, # set aliases in the same repo
aliases = {},
**kwargs
Expand Down
2 changes: 2 additions & 0 deletions tests/pypi/extra_dep/extra_dep_tests.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,7 @@ def _test_extra_select_targets(env):
}),
"distribution": "foo 1.0",
"extras": ["a", "b"],
"warnings": {},
"visibility": None,
},
]) # buildifier: @unsorted-dict-items
Expand All @@ -427,6 +428,7 @@ def _test_extra_select_targets_no_extras(env):
"deps": [":[]"],
"distribution": "",
"extras": [],
"warnings": {},
"visibility": ["//visibility:public"],
},
]) # buildifier: @unsorted-dict-items
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ _tests = []
def _test_all_workspace(env):
want = """\
load("@package_metadata//rules:package_metadata.bzl", "package_metadata")
load("@pypi//:config.bzl", "packages")
load("@pypi//:config.bzl", "packages", "target_platform_envs")
load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets")

package(default_visibility = ["//visibility:public"])
Expand Down Expand Up @@ -59,6 +59,7 @@ whl_library_targets(
"pypi_name=foo",
"pypi_version=0",
],
target_platform_envs = target_platform_envs,
)

package_metadata(
Expand Down Expand Up @@ -96,7 +97,7 @@ _tests.append(_test_all_workspace)
def _test_all(env):
want = """\
load("@package_metadata//rules:package_metadata.bzl", "package_metadata")
load("@pypi//:config.bzl", "packages")
load("@pypi//:config.bzl", "packages", "target_platform_envs")
load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets")

package(default_visibility = ["//visibility:public"])
Expand Down Expand Up @@ -134,6 +135,7 @@ whl_library_targets(
"pypi_name=foo",
"pypi_version=0",
],
target_platform_envs = target_platform_envs,
)

package_metadata(
Expand Down Expand Up @@ -172,7 +174,7 @@ _tests.append(_test_all)
def _test_all_with_loads(env):
want = """\
load("@package_metadata//rules:package_metadata.bzl", "package_metadata")
load("@pypi//:config.bzl", "packages")
load("@pypi//:config.bzl", "packages", "target_platform_envs")
load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets")

package(default_visibility = ["//visibility:public"])
Expand Down Expand Up @@ -209,6 +211,7 @@ whl_library_targets(
"pypi_name=foo",
"pypi_version=0",
],
target_platform_envs = target_platform_envs,
)

package_metadata(
Expand Down Expand Up @@ -248,7 +251,7 @@ def _test_no_deps_with_config(env):
# are requested from the package in the hub.
want = """\
load("@package_metadata//rules:package_metadata.bzl", "package_metadata")
load("@pypi//:config.bzl", "packages")
load("@pypi//:config.bzl", "packages", "target_platform_envs")
load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets")

package(default_visibility = ["//visibility:public"])
Expand All @@ -262,6 +265,7 @@ whl_library_targets(
"pypi_name=foo",
"pypi_version=0",
],
target_platform_envs = target_platform_envs,
)
"""
actual = generate_whl_library_build_bazel(
Expand Down
16 changes: 16 additions & 0 deletions tests/pypi/whl_library/testdata/packages.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,19 @@ packages = {
"nope",
],
}

# The marker environments of the target platforms of the hub. The `pkg`
# package has dependencies that are not in `packages` and a warning is only
# printed for the ones that are needed on these platforms.
target_platform_envs = [
{
"implementation_name": "cpython",
"os_name": "posix",
"platform_machine": "x86_64",
"platform_system": "Linux",
"python_full_version": python_full_version,
"python_version": python_full_version.rpartition(".")[0],
"sys_platform": "linux",
}
for python_full_version in ["3.10.20", "3.12.13"]
]
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,9 @@ Name: pkg
Version: 1.0
Requires-Dist: optional_dep; extra == "optional"
Provides-Extra: optional
Requires-Dist: not_locked; python_version < "3.11"
Requires-Dist: not_locked_optional; sys_platform == "linux" and extra == "optional"
Requires-Dist: not_locked_windows; sys_platform == "win32"
Requires-Dist: not_locked_windows_optional; sys_platform == "win32" and extra == "optional"
Provides-Extra: unused
Requires-Dist: not_locked_unused; extra == "unused"
Loading
Loading