From eec36bb35b2f0876cfb6e47aa2c5a810eabf0cd8 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 22 Nov 2026 23:25:19 +0900 Subject: [PATCH] feat(pypi): warn about the dependencies that are not in the hub The dependencies from the wheel METADATA that are not in the hub are left out, which is only correct if they are not needed on any of the target platforms, e.g. when the lock file is compiled for Python 3.12 and the hub is also used with Python 3.10. The whl repositories now load the marker environments of the target platforms from the hub and print a warning that names the package, the missing dependency and its marker. The warnings about the dependencies of an extra are only printed if the extra is used, i.e. it is requested in the hub or another package depends on it through its extra_select target. Work towards #2786 --- docs/pypi/use.md | 17 ++- news/2786.changed.md | 5 +- python/private/pypi/extra_dep.bzl | 16 +++ .../pypi/generate_whl_library_build_bazel.bzl | 4 +- .../private/pypi/whl_library_deps_targets.bzl | 75 +++++++++-- python/private/pypi/whl_library_targets.bzl | 5 + tests/pypi/extra_dep/extra_dep_tests.bzl | 2 + ...generate_whl_library_build_bazel_tests.bzl | 12 +- tests/pypi/whl_library/testdata/packages.bzl | 16 +++ .../testdata/pkg/pkg-1.0.dist-info/METADATA | 6 + .../whl_library_deps_targets_tests.bzl | 116 ++++++++++++++++++ 11 files changed, 255 insertions(+), 19 deletions(-) diff --git a/docs/pypi/use.md b/docs/pypi/use.md index 44eb06247f..1117ed529d 100644 --- a/docs/pypi/use.md +++ b/docs/pypi/use.md @@ -126,11 +126,26 @@ so they do not need to be in the lock file, e.g. when the requirements are compiled with the extras stripped. This does not change `@pypi//bar`, which only includes the extras requested in the lock file. The dependencies of the extra that are not in the lock file are left out like any other dependency -that is not in the lock file. If `bar` does not provide the `socks` extra, +that is not in the lock file, see below for the warning that is printed if +they are needed. If `bar` does not provide the `socks` extra, then only `bar` is used and a warning is printed when `@pypi//foo` is analyzed, e.g. `WARNING: bar 1.0 does not provide the extra 'socks'`, which is how `pip` handles the unknown extras. +The dependencies from the wheel `METADATA` that are not in the lock file are +left out, which is correct if they are not needed on any of the target +platforms of the hub, e.g. `pywin32; sys_platform == "win32"` in a hub that is +only for Linux. If the marker of such a dependency can be true on any of the +target platforms that have requirements, then a warning that names the missing +package is printed, because importing it may fail at runtime. For example, if +the lock file is compiled for Python 3.12 only and the hub is also used with +Python 3.10, where `coverage[toml]` needs `tomli`, then the warning asks to add +`tomli` to the lock file. The fix is to lock for all of the target platforms +and Python versions of the hub. The warnings about the dependencies of an +extra are only printed if the extra is used, i.e. it is requested in the lock +file or another package depends on it. The warnings are not printed when using +`pip_parse` in `WORKSPACE`. + If the lock file requests an extra that the package does not provide, e.g. `foo[nope]`, then `@pypi//foo:[nope]` is the same as `@pypi//foo:[]`, because an unknown extra does not have any dependencies. A warning is printed when the diff --git a/news/2786.changed.md b/news/2786.changed.md index cc97dd9100..c78f1673a7 100644 --- a/news/2786.changed.md +++ b/news/2786.changed.md @@ -12,5 +12,8 @@ different extras on different platforms now uses a single whl repository. The `@pypi//foo` targets still include the extras requested in the lock file. An extra that the package does not provide is still ignored, but a warning is now printed like `pip` does, e.g. -`WARNING: foo 1.0 does not provide the extra 'nope'` +`WARNING: foo 1.0 does not provide the extra 'nope'`. A warning is also +printed about the dependencies from the wheel `METADATA` that are left out +because they are not in the lock file, if they are needed on any of the +target platforms of the hub ([#2786](https://github.com/bazel-contrib/rules_python/issues/2786)). diff --git a/python/private/pypi/extra_dep.bzl b/python/private/pypi/extra_dep.bzl index 4837d6e5b1..cbeef12fd6 100644 --- a/python/private/pypi/extra_dep.bzl +++ b/python/private/pypi/extra_dep.bzl @@ -129,6 +129,10 @@ def _extra_select_impl(ctx): extra, )) + for warning in ctx.attr.warnings.get(extra, []): + # buildifier: disable=print + print(warning) + return _forward(ctx) extra_select = rule( @@ -154,6 +158,14 @@ the target are forwarded. "extras": attr.string_list( doc = "The normalized extras that have a target, a warning is printed if any other extra is requested.", ), + "warnings": attr.string_list_dict( + doc = """\ +The warnings to print when the extra in the key is requested, e.g. about the +dependencies of the extra that are not in the hub. They are printed here and +not when the targets are created, because only the extras that are used +matter. +""", + ), "_allowlist_function_transition": attr.label( default = "@bazel_tools//tools/allowlists/function_transition_allowlist", ), @@ -169,6 +181,7 @@ def extra_select_targets( targets, default, distribution = "", + warnings = {}, native = native, rules = None, visibility = None): @@ -183,6 +196,8 @@ def extra_select_targets( unknown extras. distribution: {type}`str` the name and the version of the package used in the warning about the unknown extras. + warnings: {type}`dict[str, list[str]]` the warnings to print when + the extra in the key is requested. native: the native struct for overriding in tests. rules: a struct with the `extra_select` rule for overriding in tests. visibility: {type}`list[str]` the visibility of the target. @@ -203,5 +218,6 @@ def extra_select_targets( deps = select(deps | {"//conditions:default": [default]}) if deps else [default], distribution = distribution, extras = sorted(targets), + warnings = warnings, visibility = visibility, ) diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index 3b3f64cc81..2e4a862d65 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -30,6 +30,7 @@ _RENDER = { "requires_dist": render.list, "srcs_exclude": render.list, "tags": render.list, + "target_platform_envs": str, } # NOTE @aignas 2024-10-25: We have to keep this so that files in @@ -98,8 +99,9 @@ def generate_whl_library_build_bazel( if config_load: # The config is also needed without any deps, because it has the # extras that are requested from this package. - loads.append("""load("{}", "{}")""".format(config_load, "packages")) + loads.append("""load("{}", "packages", "target_platform_envs")""".format(config_load)) kwargs["include"] = "packages" + kwargs["target_platform_envs"] = "target_platform_envs" if requires_dist: kwargs["requires_dist"] = requires_dist diff --git a/python/private/pypi/whl_library_deps_targets.bzl b/python/private/pypi/whl_library_deps_targets.bzl index 85d5a80797..6fa47f28a3 100644 --- a/python/private/pypi/whl_library_deps_targets.bzl +++ b/python/private/pypi/whl_library_deps_targets.bzl @@ -28,6 +28,7 @@ def whl_library_deps_targets( provides_extra = [], extras = [], include = [], + target_platform_envs = [], group_deps = [], group_name = None, dep_template, @@ -72,6 +73,12 @@ def whl_library_deps_targets( `[]` targets are also created for the requested extras of this package, see `extras` for the ones that the package does not provide. + target_platform_envs: {type}`list[dict[str, str]]` The marker + environments of the target platforms of the hub. They are used + to warn about the dependencies that are left out because they + are not in `include`, but are needed on any of the target + platforms. No warnings are printed about such dependencies if it + is empty. group_name: {type}`str | None` name of the dependency group (if any). dep_template: {type}`str | None` The dep_template to use. tags: {type}`list[str]` The tags set on the targets. @@ -220,6 +227,44 @@ def whl_library_deps_targets( return ":" + extra_label(extra) + # The dependencies of each target. The dependencies that are not in the + # hub are left out, which is only correct if they are not needed on any of + # the target platforms, so warn about the rest of them. The warnings about + # the dependencies of the extras that are not requested in the hub are + # only printed when another package depends on the extra, otherwise every + # package with extras that are not used would print them. + package_deps = {} + missing = {} + missing_by_extra = {} + for target, extra, want_extras in pkg_targets if requires_dist and hasattr(rules, "py_library") else []: + package_deps[target] = deps( + name = self_name, + requires_dist = requires_dist, + include = include, + excludes = members if is_member(extra) else [], + extras = want_extras, + keep_extras = True, + envs = target_platform_envs, + ) + if not extra or extra in hub_extras: + for dep, reqs in package_deps[target].missing.items(): + missing.setdefault(dep, {}).update({r: None for r in reqs}) + else: + missing_by_extra[extra] = package_deps[target].missing + + for dep, reqs in sorted(missing.items()): + getattr(rules, "warn", _warn)(_missing_dep_warning(distribution, dep, reqs)) + + extra_warnings = {} + for extra, extra_missing in sorted(missing_by_extra.items()): + for dep, reqs in sorted(extra_missing.items()): + # The rest are already printed. + reqs = [r for r in reqs if r not in missing.get(dep, {})] + if reqs: + extra_warnings.setdefault(extra, []).append( + _missing_dep_warning(distribution, dep, reqs), + ) + # The packages that depend on an extra of this package use this target # with the requested extra, because the hub does not know which extras # this package provides. An extra without a target is the same as the @@ -230,6 +275,7 @@ def whl_library_deps_targets( targets = {extra: _extra_target(extra) for extra in extras_targets}, default = _extra_target(""), distribution = distribution, + warnings = extra_warnings, native = native, rules = rules, visibility = visibility, @@ -281,17 +327,9 @@ def whl_library_deps_targets( visibility = _visibility(""), ) - for name, extra, want_extras in pkg_targets if hasattr(rules, "py_library") else []: - package_deps = deps( - name = self_name, - requires_dist = requires_dist, - include = include, - excludes = members if is_member(extra) else [], - extras = want_extras, - keep_extras = True, - ) + for target, extra, _ in pkg_targets if hasattr(rules, "py_library") else []: rules.py_library( - name = name, + name = target, # We include as srcs to ensure that the (locations :pkg) works as expected. srcs = [repo_label(PY_SRCS_LABEL)], deps = _deps( @@ -299,8 +337,8 @@ def whl_library_deps_targets( # propagated. Just passing the target as `srcs` is not enough to propagate # `pyi_srcs`, see `tests/base_rules/py_library`. deps = [repo_label(PY_SRCS_LABEL)], - package_deps = package_deps, - conditions = _conditions(settings, package_deps), + package_deps = package_deps[target], + conditions = _conditions(settings, package_deps[target]), label = _pkg_label, ), # Disable precompilation on this wrapper target to prevent duplicate @@ -325,6 +363,19 @@ def whl_library_deps_targets( visibility = ["//visibility:private"], ) +def _missing_dep_warning(distribution, dep, reqs): + return ( + "WARNING: {distribution} requires '{reqs}', but '{dep}' is not in " + + "the requirements of the hub, so it is left out and it may be " + + "missing at runtime on some of the target platforms. Add '{dep}' " + + "to the lock file, e.g. by locking for all of the target platforms " + + "and Python versions of the hub." + ).format( + distribution = distribution, + reqs = "', '".join(sorted(reqs)), + dep = dep, + ) + def _warn(msg): # buildifier: disable=print print(msg) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 7abc0c4ec5..a08d5ec5f8 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -30,6 +30,7 @@ def whl_library_targets( extras = [], entry_points = {}, include = [], + target_platform_envs = [], group_deps = [], group_name = None, dep_template = None, @@ -61,6 +62,9 @@ def whl_library_targets( {obj}`whl_library_deps_targets`. entry_points: {type}`list[dict]` A list of parsed entry point definitions. include: {type}`list[str]` The list of packages to include. + target_platform_envs: {type}`list[dict[str, str]]` The marker + environments of the target platforms of the hub, see + {obj}`whl_library_deps_targets`. group_name: {type}`str | None` name of the dependency group (if any). dep_template: {type}`str | None` The dep_template to use. data_exclude: {type}`list[str]` The globs for data attribute exclusion. @@ -103,6 +107,7 @@ def whl_library_targets( provides_extra = provides_extra, extras = extras, include = include, # only needed if requires_dist is present + target_platform_envs = target_platform_envs, repo = None, # set aliases in the same repo aliases = {}, **kwargs diff --git a/tests/pypi/extra_dep/extra_dep_tests.bzl b/tests/pypi/extra_dep/extra_dep_tests.bzl index 13adcadd3e..f58f7a94f9 100644 --- a/tests/pypi/extra_dep/extra_dep_tests.bzl +++ b/tests/pypi/extra_dep/extra_dep_tests.bzl @@ -403,6 +403,7 @@ def _test_extra_select_targets(env): }), "distribution": "foo 1.0", "extras": ["a", "b"], + "warnings": {}, "visibility": None, }, ]) # buildifier: @unsorted-dict-items @@ -427,6 +428,7 @@ def _test_extra_select_targets_no_extras(env): "deps": [":[]"], "distribution": "", "extras": [], + "warnings": {}, "visibility": ["//visibility:public"], }, ]) # buildifier: @unsorted-dict-items diff --git a/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl index 21af3416a9..7a23f83c25 100644 --- a/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl +++ b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl @@ -22,7 +22,7 @@ _tests = [] def _test_all_workspace(env): want = """\ load("@package_metadata//rules:package_metadata.bzl", "package_metadata") -load("@pypi//:config.bzl", "packages") +load("@pypi//:config.bzl", "packages", "target_platform_envs") load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets") package(default_visibility = ["//visibility:public"]) @@ -59,6 +59,7 @@ whl_library_targets( "pypi_name=foo", "pypi_version=0", ], + target_platform_envs = target_platform_envs, ) package_metadata( @@ -96,7 +97,7 @@ _tests.append(_test_all_workspace) def _test_all(env): want = """\ load("@package_metadata//rules:package_metadata.bzl", "package_metadata") -load("@pypi//:config.bzl", "packages") +load("@pypi//:config.bzl", "packages", "target_platform_envs") load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets") package(default_visibility = ["//visibility:public"]) @@ -134,6 +135,7 @@ whl_library_targets( "pypi_name=foo", "pypi_version=0", ], + target_platform_envs = target_platform_envs, ) package_metadata( @@ -172,7 +174,7 @@ _tests.append(_test_all) def _test_all_with_loads(env): want = """\ load("@package_metadata//rules:package_metadata.bzl", "package_metadata") -load("@pypi//:config.bzl", "packages") +load("@pypi//:config.bzl", "packages", "target_platform_envs") load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets") package(default_visibility = ["//visibility:public"]) @@ -209,6 +211,7 @@ whl_library_targets( "pypi_name=foo", "pypi_version=0", ], + target_platform_envs = target_platform_envs, ) package_metadata( @@ -248,7 +251,7 @@ def _test_no_deps_with_config(env): # are requested from the package in the hub. want = """\ load("@package_metadata//rules:package_metadata.bzl", "package_metadata") -load("@pypi//:config.bzl", "packages") +load("@pypi//:config.bzl", "packages", "target_platform_envs") load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets") package(default_visibility = ["//visibility:public"]) @@ -262,6 +265,7 @@ whl_library_targets( "pypi_name=foo", "pypi_version=0", ], + target_platform_envs = target_platform_envs, ) """ actual = generate_whl_library_build_bazel( diff --git a/tests/pypi/whl_library/testdata/packages.bzl b/tests/pypi/whl_library/testdata/packages.bzl index 812dc40f2f..6584dfeda3 100644 --- a/tests/pypi/whl_library/testdata/packages.bzl +++ b/tests/pypi/whl_library/testdata/packages.bzl @@ -10,3 +10,19 @@ packages = { "nope", ], } + +# The marker environments of the target platforms of the hub. The `pkg` +# package has dependencies that are not in `packages` and a warning is only +# printed for the ones that are needed on these platforms. +target_platform_envs = [ + { + "implementation_name": "cpython", + "os_name": "posix", + "platform_machine": "x86_64", + "platform_system": "Linux", + "python_full_version": python_full_version, + "python_version": python_full_version.rpartition(".")[0], + "sys_platform": "linux", + } + for python_full_version in ["3.10.20", "3.12.13"] +] diff --git a/tests/pypi/whl_library/testdata/pkg/pkg-1.0.dist-info/METADATA b/tests/pypi/whl_library/testdata/pkg/pkg-1.0.dist-info/METADATA index 712b44edbf..f9ce00ff1f 100644 --- a/tests/pypi/whl_library/testdata/pkg/pkg-1.0.dist-info/METADATA +++ b/tests/pypi/whl_library/testdata/pkg/pkg-1.0.dist-info/METADATA @@ -2,3 +2,9 @@ Name: pkg Version: 1.0 Requires-Dist: optional_dep; extra == "optional" Provides-Extra: optional +Requires-Dist: not_locked; python_version < "3.11" +Requires-Dist: not_locked_optional; sys_platform == "linux" and extra == "optional" +Requires-Dist: not_locked_windows; sys_platform == "win32" +Requires-Dist: not_locked_windows_optional; sys_platform == "win32" and extra == "optional" +Provides-Extra: unused +Requires-Dist: not_locked_unused; extra == "unused" diff --git a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl index 4fa6ad72ec..c8e3006bd1 100644 --- a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl +++ b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl @@ -658,6 +658,7 @@ def _test_deps_with_extras(env): "deps": [":[]"], "distribution": "foo", "extras": [], + "warnings": {}, "visibility": ["//visibility:public"], }, ]) # buildifier: @unsorted-dict-items @@ -720,6 +721,7 @@ def _test_extra_that_is_not_requested_in_the_hub(env): }), "distribution": "coverage 7.6.1", "extras": ["toml"], + "warnings": {}, "visibility": ["//visibility:public"], }, ]) # buildifier: @unsorted-dict-items @@ -769,6 +771,120 @@ def _test_deps_of_the_extra_that_are_not_in_the_hub(env): _tests.append(_test_deps_of_the_extra_that_are_not_in_the_hub) +def _envs(*python_versions, sys_platform = "linux"): + return [ + { + "os_name": "nt" if sys_platform == "win32" else "posix", + "python_full_version": v, + "python_version": v.rpartition(".")[0], + "sys_platform": sys_platform, + } + for v in python_versions + ] + +_MISSING_DEP_WARNING = ( + "WARNING: {} requires '{}', but '{}' is not in the requirements of the " + + "hub, so it is left out and it may be missing at runtime on some of the " + + "target platforms. Add '{}' to the lock file, e.g. by locking for all " + + "of the target platforms and Python versions of the hub." +) + +def _test_warn_about_missing_deps(env): + # The lock file is only for Python 3.12, but the hub is also used with + # Python 3.10, where `exceptiongroup` is needed. + kwargs = dict( + metadata_name = "foo", + requires_dist = [ + "bar", + "exceptiongroup>=1.0; python_version < \"3.11\"", + "pywin32; sys_platform == \"win32\"", + "not-locked", + ], + include = {"bar": [], "foo": []}, + tags = ["pypi_name=foo", "pypi_version=1.0"], + ) + got = _targets(target_platform_envs = _envs("3.10.20", "3.12.13"), **kwargs) + env.expect.that_dict(got.deps).contains_exactly({ + "pkg": ["srcs", "@pypi//bar:pkg"], + }) + env.expect.that_collection(got.warnings).contains_exactly([ + _MISSING_DEP_WARNING.format( + "foo 1.0", + "exceptiongroup; python_version < \"3.11\"", + "exceptiongroup", + "exceptiongroup", + ), + _MISSING_DEP_WARNING.format("foo 1.0", "not-locked", "not-locked", "not-locked"), + ]) + env.expect.that_collection([c["warnings"] for c in got.extra_selects]).contains_exactly([{}]) + + # Only the dependencies without markers are needed on Python 3.12. + got = _targets(target_platform_envs = _envs("3.12.13"), **kwargs) + env.expect.that_collection(got.warnings).contains_exactly([ + _MISSING_DEP_WARNING.format("foo 1.0", "not-locked", "not-locked", "not-locked"), + ]) + + # The target platforms are not known, e.g. in the WORKSPACE mode. + got = _targets(**kwargs) + env.expect.that_collection(got.warnings).contains_exactly([]) + +_tests.append(_test_warn_about_missing_deps) + +def _test_warn_about_missing_deps_of_extras(env): + tomli = "tomli; python_full_version <= \"3.11.0a6\" and extra == \"toml\"" + warning = _MISSING_DEP_WARNING.format("coverage 7.6.1", tomli, "tomli", "tomli") + kwargs = dict( + metadata_name = "coverage", + requires_dist = _COVERAGE_REQUIRES_DIST + [ + "not-locked", + "also-not-locked; extra == \"unused\"", + ], + provides_extra = ["toml", "unused"], + tags = ["pypi_name=coverage", "pypi_version=7.6.1"], + target_platform_envs = _envs("3.10.20", "3.12.13"), + ) + not_locked = _MISSING_DEP_WARNING.format("coverage 7.6.1", "not-locked", "not-locked", "not-locked") + + # The extra is not requested in the hub, so the warnings about its + # dependencies are only printed when another package depends on the + # extra. The dependencies without the extra are not repeated there. + got = _targets(include = {"coverage": [], "pytest_cov": []}, **kwargs) + env.expect.that_collection(got.warnings).contains_exactly([not_locked]) + env.expect.that_collection([c["warnings"] for c in got.extra_selects]).contains_exactly([{ + "toml": [warning], + "unused": [_MISSING_DEP_WARNING.format( + "coverage 7.6.1", + "also-not-locked; extra == \"unused\"", + "also-not-locked", + "also-not-locked", + )], + }]) + + # The extra is requested in the hub, so it is used. + got = _targets(include = {"coverage": ["toml"], "pytest_cov": []}, **kwargs) + env.expect.that_collection(got.warnings).contains_exactly([not_locked, warning]) + env.expect.that_collection([c["warnings"].keys() for c in got.extra_selects]).contains_exactly([["unused"]]) + + # The extra is requested without the hub and the `pkg` target includes it. + got = _targets( + include = {"coverage": None, "pytest_cov": None}, + extras = ["toml"], + dep_template = "@pip_{name}//:{target}", + **kwargs + ) + env.expect.that_collection(got.warnings).contains_exactly([not_locked, warning]) + env.expect.that_collection([c["warnings"].keys() for c in got.extra_selects]).contains_exactly([["unused"]]) + + # The dependency of the extra is not needed on any of the target platforms. + got = _targets( + include = {"coverage": ["toml"], "pytest_cov": []}, + **(kwargs | {"target_platform_envs": _envs("3.12.13")}) + ) + env.expect.that_collection(got.warnings).contains_exactly([not_locked]) + env.expect.that_collection([c["warnings"].keys() for c in got.extra_selects]).contains_exactly([["unused"]]) + +_tests.append(_test_warn_about_missing_deps_of_extras) + def _test_dep_with_extra_that_is_not_provided(env): include = {"bar": [], "foo": []} got = _targets(