diff --git a/docs/pypi/use.md b/docs/pypi/use.md index 44eb06247f..1117ed529d 100644 --- a/docs/pypi/use.md +++ b/docs/pypi/use.md @@ -126,11 +126,26 @@ so they do not need to be in the lock file, e.g. when the requirements are compiled with the extras stripped. This does not change `@pypi//bar`, which only includes the extras requested in the lock file. The dependencies of the extra that are not in the lock file are left out like any other dependency -that is not in the lock file. If `bar` does not provide the `socks` extra, +that is not in the lock file, see below for the warning that is printed if +they are needed. If `bar` does not provide the `socks` extra, then only `bar` is used and a warning is printed when `@pypi//foo` is analyzed, e.g. `WARNING: bar 1.0 does not provide the extra 'socks'`, which is how `pip` handles the unknown extras. +The dependencies from the wheel `METADATA` that are not in the lock file are +left out, which is correct if they are not needed on any of the target +platforms of the hub, e.g. `pywin32; sys_platform == "win32"` in a hub that is +only for Linux. If the marker of such a dependency can be true on any of the +target platforms that have requirements, then a warning that names the missing +package is printed, because importing it may fail at runtime. For example, if +the lock file is compiled for Python 3.12 only and the hub is also used with +Python 3.10, where `coverage[toml]` needs `tomli`, then the warning asks to add +`tomli` to the lock file. The fix is to lock for all of the target platforms +and Python versions of the hub. The warnings about the dependencies of an +extra are only printed if the extra is used, i.e. it is requested in the lock +file or another package depends on it. The warnings are not printed when using +`pip_parse` in `WORKSPACE`. + If the lock file requests an extra that the package does not provide, e.g. `foo[nope]`, then `@pypi//foo:[nope]` is the same as `@pypi//foo:[]`, because an unknown extra does not have any dependencies. A warning is printed when the diff --git a/news/2786.changed.md b/news/2786.changed.md index cc97dd9100..c78f1673a7 100644 --- a/news/2786.changed.md +++ b/news/2786.changed.md @@ -12,5 +12,8 @@ different extras on different platforms now uses a single whl repository. The `@pypi//foo` targets still include the extras requested in the lock file. An extra that the package does not provide is still ignored, but a warning is now printed like `pip` does, e.g. -`WARNING: foo 1.0 does not provide the extra 'nope'` +`WARNING: foo 1.0 does not provide the extra 'nope'`. A warning is also +printed about the dependencies from the wheel `METADATA` that are left out +because they are not in the lock file, if they are needed on any of the +target platforms of the hub ([#2786](https://github.com/bazel-contrib/rules_python/issues/2786)). diff --git a/python/private/pypi/extra_dep.bzl b/python/private/pypi/extra_dep.bzl index 4837d6e5b1..cbeef12fd6 100644 --- a/python/private/pypi/extra_dep.bzl +++ b/python/private/pypi/extra_dep.bzl @@ -129,6 +129,10 @@ def _extra_select_impl(ctx): extra, )) + for warning in ctx.attr.warnings.get(extra, []): + # buildifier: disable=print + print(warning) + return _forward(ctx) extra_select = rule( @@ -154,6 +158,14 @@ the target are forwarded. "extras": attr.string_list( doc = "The normalized extras that have a target, a warning is printed if any other extra is requested.", ), + "warnings": attr.string_list_dict( + doc = """\ +The warnings to print when the extra in the key is requested, e.g. about the +dependencies of the extra that are not in the hub. They are printed here and +not when the targets are created, because only the extras that are used +matter. +""", + ), "_allowlist_function_transition": attr.label( default = "@bazel_tools//tools/allowlists/function_transition_allowlist", ), @@ -169,6 +181,7 @@ def extra_select_targets( targets, default, distribution = "", + warnings = {}, native = native, rules = None, visibility = None): @@ -183,6 +196,8 @@ def extra_select_targets( unknown extras. distribution: {type}`str` the name and the version of the package used in the warning about the unknown extras. + warnings: {type}`dict[str, list[str]]` the warnings to print when + the extra in the key is requested. native: the native struct for overriding in tests. rules: a struct with the `extra_select` rule for overriding in tests. visibility: {type}`list[str]` the visibility of the target. @@ -203,5 +218,6 @@ def extra_select_targets( deps = select(deps | {"//conditions:default": [default]}) if deps else [default], distribution = distribution, extras = sorted(targets), + warnings = warnings, visibility = visibility, ) diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index 3b3f64cc81..2e4a862d65 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -30,6 +30,7 @@ _RENDER = { "requires_dist": render.list, "srcs_exclude": render.list, "tags": render.list, + "target_platform_envs": str, } # NOTE @aignas 2024-10-25: We have to keep this so that files in @@ -98,8 +99,9 @@ def generate_whl_library_build_bazel( if config_load: # The config is also needed without any deps, because it has the # extras that are requested from this package. - loads.append("""load("{}", "{}")""".format(config_load, "packages")) + loads.append("""load("{}", "packages", "target_platform_envs")""".format(config_load)) kwargs["include"] = "packages" + kwargs["target_platform_envs"] = "target_platform_envs" if requires_dist: kwargs["requires_dist"] = requires_dist diff --git a/python/private/pypi/whl_library_deps_targets.bzl b/python/private/pypi/whl_library_deps_targets.bzl index 85d5a80797..6fa47f28a3 100644 --- a/python/private/pypi/whl_library_deps_targets.bzl +++ b/python/private/pypi/whl_library_deps_targets.bzl @@ -28,6 +28,7 @@ def whl_library_deps_targets( provides_extra = [], extras = [], include = [], + target_platform_envs = [], group_deps = [], group_name = None, dep_template, @@ -72,6 +73,12 @@ def whl_library_deps_targets( `[]` targets are also created for the requested extras of this package, see `extras` for the ones that the package does not provide. + target_platform_envs: {type}`list[dict[str, str]]` The marker + environments of the target platforms of the hub. They are used + to warn about the dependencies that are left out because they + are not in `include`, but are needed on any of the target + platforms. No warnings are printed about such dependencies if it + is empty. group_name: {type}`str | None` name of the dependency group (if any). dep_template: {type}`str | None` The dep_template to use. tags: {type}`list[str]` The tags set on the targets. @@ -220,6 +227,44 @@ def whl_library_deps_targets( return ":" + extra_label(extra) + # The dependencies of each target. The dependencies that are not in the + # hub are left out, which is only correct if they are not needed on any of + # the target platforms, so warn about the rest of them. The warnings about + # the dependencies of the extras that are not requested in the hub are + # only printed when another package depends on the extra, otherwise every + # package with extras that are not used would print them. + package_deps = {} + missing = {} + missing_by_extra = {} + for target, extra, want_extras in pkg_targets if requires_dist and hasattr(rules, "py_library") else []: + package_deps[target] = deps( + name = self_name, + requires_dist = requires_dist, + include = include, + excludes = members if is_member(extra) else [], + extras = want_extras, + keep_extras = True, + envs = target_platform_envs, + ) + if not extra or extra in hub_extras: + for dep, reqs in package_deps[target].missing.items(): + missing.setdefault(dep, {}).update({r: None for r in reqs}) + else: + missing_by_extra[extra] = package_deps[target].missing + + for dep, reqs in sorted(missing.items()): + getattr(rules, "warn", _warn)(_missing_dep_warning(distribution, dep, reqs)) + + extra_warnings = {} + for extra, extra_missing in sorted(missing_by_extra.items()): + for dep, reqs in sorted(extra_missing.items()): + # The rest are already printed. + reqs = [r for r in reqs if r not in missing.get(dep, {})] + if reqs: + extra_warnings.setdefault(extra, []).append( + _missing_dep_warning(distribution, dep, reqs), + ) + # The packages that depend on an extra of this package use this target # with the requested extra, because the hub does not know which extras # this package provides. An extra without a target is the same as the @@ -230,6 +275,7 @@ def whl_library_deps_targets( targets = {extra: _extra_target(extra) for extra in extras_targets}, default = _extra_target(""), distribution = distribution, + warnings = extra_warnings, native = native, rules = rules, visibility = visibility, @@ -281,17 +327,9 @@ def whl_library_deps_targets( visibility = _visibility(""), ) - for name, extra, want_extras in pkg_targets if hasattr(rules, "py_library") else []: - package_deps = deps( - name = self_name, - requires_dist = requires_dist, - include = include, - excludes = members if is_member(extra) else [], - extras = want_extras, - keep_extras = True, - ) + for target, extra, _ in pkg_targets if hasattr(rules, "py_library") else []: rules.py_library( - name = name, + name = target, # We include as srcs to ensure that the (locations :pkg) works as expected. srcs = [repo_label(PY_SRCS_LABEL)], deps = _deps( @@ -299,8 +337,8 @@ def whl_library_deps_targets( # propagated. Just passing the target as `srcs` is not enough to propagate # `pyi_srcs`, see `tests/base_rules/py_library`. deps = [repo_label(PY_SRCS_LABEL)], - package_deps = package_deps, - conditions = _conditions(settings, package_deps), + package_deps = package_deps[target], + conditions = _conditions(settings, package_deps[target]), label = _pkg_label, ), # Disable precompilation on this wrapper target to prevent duplicate @@ -325,6 +363,19 @@ def whl_library_deps_targets( visibility = ["//visibility:private"], ) +def _missing_dep_warning(distribution, dep, reqs): + return ( + "WARNING: {distribution} requires '{reqs}', but '{dep}' is not in " + + "the requirements of the hub, so it is left out and it may be " + + "missing at runtime on some of the target platforms. Add '{dep}' " + + "to the lock file, e.g. by locking for all of the target platforms " + + "and Python versions of the hub." + ).format( + distribution = distribution, + reqs = "', '".join(sorted(reqs)), + dep = dep, + ) + def _warn(msg): # buildifier: disable=print print(msg) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 7abc0c4ec5..a08d5ec5f8 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -30,6 +30,7 @@ def whl_library_targets( extras = [], entry_points = {}, include = [], + target_platform_envs = [], group_deps = [], group_name = None, dep_template = None, @@ -61,6 +62,9 @@ def whl_library_targets( {obj}`whl_library_deps_targets`. entry_points: {type}`list[dict]` A list of parsed entry point definitions. include: {type}`list[str]` The list of packages to include. + target_platform_envs: {type}`list[dict[str, str]]` The marker + environments of the target platforms of the hub, see + {obj}`whl_library_deps_targets`. group_name: {type}`str | None` name of the dependency group (if any). dep_template: {type}`str | None` The dep_template to use. data_exclude: {type}`list[str]` The globs for data attribute exclusion. @@ -103,6 +107,7 @@ def whl_library_targets( provides_extra = provides_extra, extras = extras, include = include, # only needed if requires_dist is present + target_platform_envs = target_platform_envs, repo = None, # set aliases in the same repo aliases = {}, **kwargs diff --git a/tests/pypi/extra_dep/extra_dep_tests.bzl b/tests/pypi/extra_dep/extra_dep_tests.bzl index 13adcadd3e..f58f7a94f9 100644 --- a/tests/pypi/extra_dep/extra_dep_tests.bzl +++ b/tests/pypi/extra_dep/extra_dep_tests.bzl @@ -403,6 +403,7 @@ def _test_extra_select_targets(env): }), "distribution": "foo 1.0", "extras": ["a", "b"], + "warnings": {}, "visibility": None, }, ]) # buildifier: @unsorted-dict-items @@ -427,6 +428,7 @@ def _test_extra_select_targets_no_extras(env): "deps": [":[]"], "distribution": "", "extras": [], + "warnings": {}, "visibility": ["//visibility:public"], }, ]) # buildifier: @unsorted-dict-items diff --git a/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl index 21af3416a9..7a23f83c25 100644 --- a/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl +++ b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl @@ -22,7 +22,7 @@ _tests = [] def _test_all_workspace(env): want = """\ load("@package_metadata//rules:package_metadata.bzl", "package_metadata") -load("@pypi//:config.bzl", "packages") +load("@pypi//:config.bzl", "packages", "target_platform_envs") load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets") package(default_visibility = ["//visibility:public"]) @@ -59,6 +59,7 @@ whl_library_targets( "pypi_name=foo", "pypi_version=0", ], + target_platform_envs = target_platform_envs, ) package_metadata( @@ -96,7 +97,7 @@ _tests.append(_test_all_workspace) def _test_all(env): want = """\ load("@package_metadata//rules:package_metadata.bzl", "package_metadata") -load("@pypi//:config.bzl", "packages") +load("@pypi//:config.bzl", "packages", "target_platform_envs") load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets") package(default_visibility = ["//visibility:public"]) @@ -134,6 +135,7 @@ whl_library_targets( "pypi_name=foo", "pypi_version=0", ], + target_platform_envs = target_platform_envs, ) package_metadata( @@ -172,7 +174,7 @@ _tests.append(_test_all) def _test_all_with_loads(env): want = """\ load("@package_metadata//rules:package_metadata.bzl", "package_metadata") -load("@pypi//:config.bzl", "packages") +load("@pypi//:config.bzl", "packages", "target_platform_envs") load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets") package(default_visibility = ["//visibility:public"]) @@ -209,6 +211,7 @@ whl_library_targets( "pypi_name=foo", "pypi_version=0", ], + target_platform_envs = target_platform_envs, ) package_metadata( @@ -248,7 +251,7 @@ def _test_no_deps_with_config(env): # are requested from the package in the hub. want = """\ load("@package_metadata//rules:package_metadata.bzl", "package_metadata") -load("@pypi//:config.bzl", "packages") +load("@pypi//:config.bzl", "packages", "target_platform_envs") load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets") package(default_visibility = ["//visibility:public"]) @@ -262,6 +265,7 @@ whl_library_targets( "pypi_name=foo", "pypi_version=0", ], + target_platform_envs = target_platform_envs, ) """ actual = generate_whl_library_build_bazel( diff --git a/tests/pypi/whl_library/testdata/packages.bzl b/tests/pypi/whl_library/testdata/packages.bzl index 812dc40f2f..6584dfeda3 100644 --- a/tests/pypi/whl_library/testdata/packages.bzl +++ b/tests/pypi/whl_library/testdata/packages.bzl @@ -10,3 +10,19 @@ packages = { "nope", ], } + +# The marker environments of the target platforms of the hub. The `pkg` +# package has dependencies that are not in `packages` and a warning is only +# printed for the ones that are needed on these platforms. +target_platform_envs = [ + { + "implementation_name": "cpython", + "os_name": "posix", + "platform_machine": "x86_64", + "platform_system": "Linux", + "python_full_version": python_full_version, + "python_version": python_full_version.rpartition(".")[0], + "sys_platform": "linux", + } + for python_full_version in ["3.10.20", "3.12.13"] +] diff --git a/tests/pypi/whl_library/testdata/pkg/pkg-1.0.dist-info/METADATA b/tests/pypi/whl_library/testdata/pkg/pkg-1.0.dist-info/METADATA index 712b44edbf..f9ce00ff1f 100644 --- a/tests/pypi/whl_library/testdata/pkg/pkg-1.0.dist-info/METADATA +++ b/tests/pypi/whl_library/testdata/pkg/pkg-1.0.dist-info/METADATA @@ -2,3 +2,9 @@ Name: pkg Version: 1.0 Requires-Dist: optional_dep; extra == "optional" Provides-Extra: optional +Requires-Dist: not_locked; python_version < "3.11" +Requires-Dist: not_locked_optional; sys_platform == "linux" and extra == "optional" +Requires-Dist: not_locked_windows; sys_platform == "win32" +Requires-Dist: not_locked_windows_optional; sys_platform == "win32" and extra == "optional" +Provides-Extra: unused +Requires-Dist: not_locked_unused; extra == "unused" diff --git a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl index 4fa6ad72ec..c8e3006bd1 100644 --- a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl +++ b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl @@ -658,6 +658,7 @@ def _test_deps_with_extras(env): "deps": [":[]"], "distribution": "foo", "extras": [], + "warnings": {}, "visibility": ["//visibility:public"], }, ]) # buildifier: @unsorted-dict-items @@ -720,6 +721,7 @@ def _test_extra_that_is_not_requested_in_the_hub(env): }), "distribution": "coverage 7.6.1", "extras": ["toml"], + "warnings": {}, "visibility": ["//visibility:public"], }, ]) # buildifier: @unsorted-dict-items @@ -769,6 +771,120 @@ def _test_deps_of_the_extra_that_are_not_in_the_hub(env): _tests.append(_test_deps_of_the_extra_that_are_not_in_the_hub) +def _envs(*python_versions, sys_platform = "linux"): + return [ + { + "os_name": "nt" if sys_platform == "win32" else "posix", + "python_full_version": v, + "python_version": v.rpartition(".")[0], + "sys_platform": sys_platform, + } + for v in python_versions + ] + +_MISSING_DEP_WARNING = ( + "WARNING: {} requires '{}', but '{}' is not in the requirements of the " + + "hub, so it is left out and it may be missing at runtime on some of the " + + "target platforms. Add '{}' to the lock file, e.g. by locking for all " + + "of the target platforms and Python versions of the hub." +) + +def _test_warn_about_missing_deps(env): + # The lock file is only for Python 3.12, but the hub is also used with + # Python 3.10, where `exceptiongroup` is needed. + kwargs = dict( + metadata_name = "foo", + requires_dist = [ + "bar", + "exceptiongroup>=1.0; python_version < \"3.11\"", + "pywin32; sys_platform == \"win32\"", + "not-locked", + ], + include = {"bar": [], "foo": []}, + tags = ["pypi_name=foo", "pypi_version=1.0"], + ) + got = _targets(target_platform_envs = _envs("3.10.20", "3.12.13"), **kwargs) + env.expect.that_dict(got.deps).contains_exactly({ + "pkg": ["srcs", "@pypi//bar:pkg"], + }) + env.expect.that_collection(got.warnings).contains_exactly([ + _MISSING_DEP_WARNING.format( + "foo 1.0", + "exceptiongroup; python_version < \"3.11\"", + "exceptiongroup", + "exceptiongroup", + ), + _MISSING_DEP_WARNING.format("foo 1.0", "not-locked", "not-locked", "not-locked"), + ]) + env.expect.that_collection([c["warnings"] for c in got.extra_selects]).contains_exactly([{}]) + + # Only the dependencies without markers are needed on Python 3.12. + got = _targets(target_platform_envs = _envs("3.12.13"), **kwargs) + env.expect.that_collection(got.warnings).contains_exactly([ + _MISSING_DEP_WARNING.format("foo 1.0", "not-locked", "not-locked", "not-locked"), + ]) + + # The target platforms are not known, e.g. in the WORKSPACE mode. + got = _targets(**kwargs) + env.expect.that_collection(got.warnings).contains_exactly([]) + +_tests.append(_test_warn_about_missing_deps) + +def _test_warn_about_missing_deps_of_extras(env): + tomli = "tomli; python_full_version <= \"3.11.0a6\" and extra == \"toml\"" + warning = _MISSING_DEP_WARNING.format("coverage 7.6.1", tomli, "tomli", "tomli") + kwargs = dict( + metadata_name = "coverage", + requires_dist = _COVERAGE_REQUIRES_DIST + [ + "not-locked", + "also-not-locked; extra == \"unused\"", + ], + provides_extra = ["toml", "unused"], + tags = ["pypi_name=coverage", "pypi_version=7.6.1"], + target_platform_envs = _envs("3.10.20", "3.12.13"), + ) + not_locked = _MISSING_DEP_WARNING.format("coverage 7.6.1", "not-locked", "not-locked", "not-locked") + + # The extra is not requested in the hub, so the warnings about its + # dependencies are only printed when another package depends on the + # extra. The dependencies without the extra are not repeated there. + got = _targets(include = {"coverage": [], "pytest_cov": []}, **kwargs) + env.expect.that_collection(got.warnings).contains_exactly([not_locked]) + env.expect.that_collection([c["warnings"] for c in got.extra_selects]).contains_exactly([{ + "toml": [warning], + "unused": [_MISSING_DEP_WARNING.format( + "coverage 7.6.1", + "also-not-locked; extra == \"unused\"", + "also-not-locked", + "also-not-locked", + )], + }]) + + # The extra is requested in the hub, so it is used. + got = _targets(include = {"coverage": ["toml"], "pytest_cov": []}, **kwargs) + env.expect.that_collection(got.warnings).contains_exactly([not_locked, warning]) + env.expect.that_collection([c["warnings"].keys() for c in got.extra_selects]).contains_exactly([["unused"]]) + + # The extra is requested without the hub and the `pkg` target includes it. + got = _targets( + include = {"coverage": None, "pytest_cov": None}, + extras = ["toml"], + dep_template = "@pip_{name}//:{target}", + **kwargs + ) + env.expect.that_collection(got.warnings).contains_exactly([not_locked, warning]) + env.expect.that_collection([c["warnings"].keys() for c in got.extra_selects]).contains_exactly([["unused"]]) + + # The dependency of the extra is not needed on any of the target platforms. + got = _targets( + include = {"coverage": ["toml"], "pytest_cov": []}, + **(kwargs | {"target_platform_envs": _envs("3.12.13")}) + ) + env.expect.that_collection(got.warnings).contains_exactly([not_locked]) + env.expect.that_collection([c["warnings"].keys() for c in got.extra_selects]).contains_exactly([["unused"]]) + +_tests.append(_test_warn_about_missing_deps_of_extras) + def _test_dep_with_extra_that_is_not_provided(env): include = {"bar": [], "foo": []} got = _targets(