diff --git a/docs/pypi/download.md b/docs/pypi/download.md index ad8b40e70d..39f725d6b9 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -243,6 +243,29 @@ the `pyproject.toml` selects the optional dependencies of the project. The packages that are not part of the hub are ignored, so a single `pyproject.toml` can list tools that are missing from the lock file. +The extras requested in the selected groups are added to the hub even if the +lock file does not request them, for example because the lock file was +generated with `--strip-extras`. With the following `pyproject.toml`: + +```toml +[project] +dependencies = [ + "requests[socks]", + "keyring[completion]; sys_platform == 'win32'", +] +``` + +the hub has the `@pypi_hub//requests:[socks]` target and +`@pypi_hub//requests` includes the `socks` extra. The environment markers of +the requirements are evaluated for each target platform of the hub, so +`@pypi_hub//keyring` only includes the `completion` extra on Windows and +`@pypi_hub//keyring:[completion]` is the same as `@pypi_hub//keyring:[]` on +the other platforms. With `["*", "transitive"]` the extras of all of the +groups are added. The dependencies of the extras, such as `pysocks`, still +have to be in the lock file, and an extra that the package does not provide is +ignored with a warning. See [extras](./use.md#extras) for the targets of the +extras. + :::{versionadded} VERSION_NEXT_FEATURE ::: diff --git a/docs/pypi/use.md b/docs/pypi/use.md index 3a8af5983c..1117ed529d 100644 --- a/docs/pypi/use.md +++ b/docs/pypi/use.md @@ -119,12 +119,32 @@ platforms, for example `foo[bar]` on Linux and `foo[baz]` on macOS, then `@pypi//foo:[bar]` on macOS is the same as `@pypi//foo:[]`. The dependencies of the packages use the extras written in the wheel -`METADATA`, so if `foo` depends on `bar[socks]`, then `@pypi//foo` depends on -`@pypi//bar:[socks]`. If the extra is not in the lock file, then building -`@pypi//foo` fails with a message asking you to add `bar[socks]` to the -requirements or the lock file. For example, when compiling the requirements -with `uv pip compile`, pass `--no-strip-extras` to keep the extras in the -output. +`METADATA`, so if `foo` depends on `bar[socks]`, then `@pypi//foo` includes +`bar` and the dependencies of its `socks` extra. The extras of the +dependencies are resolved by the repository of `bar` from its own `METADATA`, +so they do not need to be in the lock file, e.g. when the requirements are +compiled with the extras stripped. This does not change `@pypi//bar`, which +only includes the extras requested in the lock file. The dependencies of the +extra that are not in the lock file are left out like any other dependency +that is not in the lock file, see below for the warning that is printed if +they are needed. If `bar` does not provide the `socks` extra, +then only `bar` is used and a warning is printed when `@pypi//foo` is +analyzed, e.g. `WARNING: bar 1.0 does not provide the extra 'socks'`, which is +how `pip` handles the unknown extras. + +The dependencies from the wheel `METADATA` that are not in the lock file are +left out, which is correct if they are not needed on any of the target +platforms of the hub, e.g. `pywin32; sys_platform == "win32"` in a hub that is +only for Linux. If the marker of such a dependency can be true on any of the +target platforms that have requirements, then a warning that names the missing +package is printed, because importing it may fail at runtime. For example, if +the lock file is compiled for Python 3.12 only and the hub is also used with +Python 3.10, where `coverage[toml]` needs `tomli`, then the warning asks to add +`tomli` to the lock file. The fix is to lock for all of the target platforms +and Python versions of the hub. The warnings about the dependencies of an +extra are only printed if the extra is used, i.e. it is requested in the lock +file or another package depends on it. The warnings are not printed when using +`pip_parse` in `WORKSPACE`. If the lock file requests an extra that the package does not provide, e.g. `foo[nope]`, then `@pypi//foo:[nope]` is the same as `@pypi//foo:[]`, because @@ -142,9 +162,7 @@ When the `requirements.bzl` file is vendored, there is no hub repo and the labels returned by the `requirement` helper point to the repositories of the wheels. Their `pkg` targets include the extras from the lock file and they have the `[]` target and the `[extra]` targets for the extras used in the lock -file and for all of the extras that the package provides. The dependencies -between the packages use the `pkg` targets and ignore the extras from the wheel -`METADATA` like before, so the extras need to be in the lock file. +file and for all of the extras that the package provides. The `whl` targets, e.g. `@pypi//foo:whl`, do not use the extras and include only the wheels of the package and its dependencies without extras. diff --git a/news/2786.changed.md b/news/2786.changed.md index 29a0897f0b..c78f1673a7 100644 --- a/news/2786.changed.md +++ b/news/2786.changed.md @@ -1,17 +1,19 @@ (pypi) The dependencies between the PyPI packages now use the extras from the wheel `METADATA`, e.g. if `foo` requires `bar[socks]`, then `@pypi//foo` -depends on `@pypi//bar:[socks]` instead of `@pypi//bar`. Previously such extras -were ignored. If the lock file of a hub does not request the extra, the build -fails with a message asking to add it, e.g. by compiling the requirements with -`--no-strip-extras`. This does not apply when there is no hub, e.g. with a -vendored `requirements.bzl`, where the dependencies use the `pkg` targets, -which include the extras from the lock file, as before. The `whl` targets no longer include the wheels of the -dependencies of the extras. A member of `experimental_requirement_cycles` -without extras, e.g. `foo`, is now only the package without extras, so the -cycles that go through `foo[bar]` need `foo[bar]` as a member. The same -distribution that is requested with different extras on different platforms -now uses a single whl repository. The `@pypi//foo` targets still include the -extras requested in the lock file. An extra that the package does not provide -is still ignored, but a warning is now printed like `pip` does, e.g. -`WARNING: foo 1.0 does not provide the extra 'nope'` +includes `bar` and the dependencies of its `socks` extra. Previously such +extras were ignored. The extra does not need to be requested in the lock file +of the hub, because it is resolved by the whl repository of `bar`, and the +dependencies of the extra that are not in the lock file are left out. The +`whl` targets no longer include the wheels of the dependencies of the extras. +A member of `experimental_requirement_cycles` without extras, e.g. `foo`, is +now only the package without extras, so the cycles that go through `foo[bar]` +need `foo[bar]` as a member. The same distribution that is requested with +different extras on different platforms now uses a single whl repository. The +`@pypi//foo` targets still include the extras requested in the lock file. An +extra that the package does not provide is still ignored, but a warning is now +printed like `pip` does, e.g. +`WARNING: foo 1.0 does not provide the extra 'nope'`. A warning is also +printed about the dependencies from the wheel `METADATA` that are left out +because they are not in the lock file, if they are needed on any of the +target platforms of the hub ([#2786](https://github.com/bazel-contrib/rules_python/issues/2786)). diff --git a/news/4522.added.md b/news/4522.added.md new file mode 100644 index 0000000000..98b513aee3 --- /dev/null +++ b/news/4522.added.md @@ -0,0 +1,7 @@ +(pypi) {obj}`pip.configure` now adds the extras requested in the selected +`pyproject.toml` dependency groups, e.g. `requests[socks]`, to the hub, so +that the `@pypi//requests:[socks]` target exists and `@pypi//requests` +includes the extra even if the lock file does not request it. The environment +markers of the requirements are evaluated for each target platform, so an +extra is only added for the platforms that request it +([#2786](https://github.com/bazel-contrib/rules_python/issues/2786)). diff --git a/python/private/internal_dev_deps.bzl b/python/private/internal_dev_deps.bzl index 7ca740aa6a..8aba3130f5 100644 --- a/python/private/internal_dev_deps.bzl +++ b/python/private/internal_dev_deps.bzl @@ -121,9 +121,6 @@ def _internal_dev_deps_impl(mctx): name = "whl_library_extras_pkg", root = "//tests/pypi/whl_library/testdata/pkg:BUILD.bazel", output = "pkg-1.0-any-none-any.whl", - # Without a hub the dependencies use the `pkg` target, which includes - # the extras of the requirement. - requirement = "pkg[optional]", # The following is necessary to enable pipstar and make tests faster config_load = "@rules_python//tests/pypi/whl_library/testdata:packages.bzl", dep_template = "@whl_library_extras_{name}//:{target}", diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 0d660f0d4d..7d510471c3 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -16,6 +16,7 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("//python:py_binary.bzl", "py_binary") load("//python:py_library.bzl", "py_library") load("//python/private:distribution_filegroup.bzl", "distribution_filegroup") +load(":extra_dep.bzl", "requested_extra") package(default_visibility = ["//:__subpackages__"]) @@ -26,6 +27,13 @@ exports_files( visibility = ["//visibility:public"], ) +# The extra requested by the `extra_dep` targets in the whl repositories. +requested_extra( + name = "requested_extra", + build_setting_default = "", + visibility = ["//visibility:public"], +) + py_library( name = "rewriters_lib", srcs = [ @@ -110,6 +118,17 @@ bzl_library( deps = ["//python/private:bazel_tools"], ) +bzl_library( + name = "extra_dep", + srcs = ["extra_dep.bzl"], + deps = [ + "//python/private:py_cc_link_params_info", + "//python/private:py_info", + "//python/private:reexports", + "@bazel_skylib//rules:common_settings", + ], +) + bzl_library( name = "env_marker_setting", srcs = ["env_marker_setting.bzl"], @@ -367,6 +386,7 @@ bzl_library( name = "pkg_aliases", srcs = ["pkg_aliases.bzl"], deps = [ + ":extra_dep", ":labels", ":whl_library_deps_targets", "//python:py_library", @@ -624,8 +644,8 @@ bzl_library( srcs = ["whl_library_deps_targets.bzl"], deps = [ ":env_marker_setting", + ":extra_dep", ":labels", - ":missing_package", ":pep508_deps", "//python:py_library", "//python/private:normalize_name", @@ -667,6 +687,8 @@ bzl_library( srcs = ["hub_exposure.bzl"], deps = [ ":dependency_groups", + ":pep508_evaluate", + ":whl_config_setting", "//python/private:pyproject_utils", ], ) diff --git a/python/private/pypi/config.bzl.tmpl b/python/private/pypi/config.bzl.tmpl index 1037153cac..d10006b174 100644 --- a/python/private/pypi/config.bzl.tmpl +++ b/python/private/pypi/config.bzl.tmpl @@ -5,3 +5,8 @@ with your usecase. This may change in between rules_python versions without any """ packages = %%PACKAGES%% + +# The marker environments of the target platforms that have requirements. They +# are used to warn about the dependencies that are not in `packages`, but are +# needed on any of the target platforms. +target_platform_envs = %%TARGET_PLATFORM_ENVS%% diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl index 13b164c2b4..3355fdc0b5 100644 --- a/python/private/pypi/dependency_groups.bzl +++ b/python/private/pypi/dependency_groups.bzl @@ -4,6 +4,11 @@ load("@bazel_skylib//lib:types.bzl", "types") load("//python/private:normalize_name.bzl", "normalize_name") load(":pep508_requirement.bzl", "requirement") +# The limit for processing the groups selected by the self references, e.g. +# `my-project[test]`. A group is processed once for each different set of +# markers that it is selected with. +_MAX_SELF_REFERENCES = 10000 + def select_dependency_groups(pyproject, specs, _fail = fail): """Select the packages named by the given dependency group specifiers. @@ -30,6 +35,13 @@ def select_dependency_groups(pyproject, specs, _fail = fail): * `all`: {type}`bool` whether all of the packages including the transitive dependencies should be selected, i.e. `"*"` and `"transitive"` were given. + * `extras`: {type}`dict[str, dict[tuple[str], list[str]]]` the + normalized package names mapped to the extras that the selected + requirements request, e.g. `requests[socks]`. The extras are + keyed by the environment markers that all need to be true for + the extras to be requested, e.g. `("sys_platform == 'win32'",)` + or an empty tuple if there are no markers. When `all` is set, + the extras of all of the groups are returned. Returns `None` on failure. """ groups = {"default": pyproject.dependencies} @@ -41,18 +53,17 @@ def select_dependency_groups(pyproject, specs, _fail = fail): for name, reqs in expanded.items(): groups["group:" + name] = reqs - if "transitive" in specs: + select_all = "transitive" in specs + if select_all: if "*" not in specs: _fail("The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: {}".format( specs, )) return None - return struct( - # list[str]: sorted normalized package names selected by `specs`. - packages = [], - # bool: True when all hub packages are selected. - all = True, - ) + + # All of the packages are selected, but the extras are only known + # from the groups. + specs = ["*"] selected = {} for spec in specs: @@ -70,30 +81,84 @@ def select_dependency_groups(pyproject, specs, _fail = fail): # A self reference like `my-project[test]` selects the optional # dependencies, including nested composite extras. + # + # The groups are queued together with the markers of the self references + # that selected them, because the requirements of such groups are only + # requested when the markers are true. A group is processed again if it is + # selected with markers that are not known to be stricter than the ones + # that it was processed with. project = normalize_name(pyproject.name or "") packages = {} - queue = list(selected) - for _ in range(len(groups)): + extras = {} + processed = {} + queue = [(key, ()) for key in selected] + for _ in range(_MAX_SELF_REFERENCES): if not queue: break - for req in groups[queue.pop(0)]: + key, markers = queue.pop(0) + if _is_processed(processed.setdefault(key, []), markers): + continue + processed[key].append(markers) + for req in groups[key]: req = requirement(req) + req_markers = _add_marker(markers, req.marker) if req.name_ != project: packages[req.name_] = None + if req.extras: + pkg_extras = extras.setdefault(req.name_, {}).setdefault(req_markers, {}) + for extra in req.extras: + pkg_extras[extra] = None continue for extra in req.extras: opt_key = "optional:" + extra - if opt_key in groups and opt_key not in selected: - selected[opt_key] = None - queue.append(opt_key) + if opt_key in groups: + queue.append((opt_key, req_markers)) + + if queue: + _fail("Too many self references with different markers in the dependency groups of '{}'".format( + pyproject.name, + )) + return None return struct( # list[str]: sorted normalized package names selected by `specs`. - packages = sorted(packages), + packages = [] if select_all else sorted(packages), # bool: True when all packages including transitive ones are selected. - all = False, + all = select_all, + # dict[str, dict[tuple[str], list[str]]]: the extras of the packages + # keyed by the markers that need to be true. + extras = { + name: { + markers: sorted(pkg_extras) + for markers, pkg_extras in sorted(by_markers.items()) + } + for name, by_markers in sorted(extras.items()) + }, ) +def _add_marker(markers, marker): + if not marker or marker in markers: + return markers + return tuple(sorted(markers + (marker,))) + +def _is_processed(processed, markers): + """Tell if a group was already processed with the same or looser markers. + + Args: + processed: {type}`list[tuple[str]]` the markers that the group was + processed with. + markers: {type}`tuple[str]` the markers to check. + + Returns: + {type}`bool` `True` if all of the markers of a processed entry are in + `markers`, i.e. the requirements of the group are already requested + whenever `markers` are true. + """ + for known in processed: + if len([m for m in known if m in markers]) == len(known): + return True + return False + def _match(spec, groups): """Match a dependency group specifier against normalized group keys. diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 1d921f539d..3bfc0d85fd 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -24,7 +24,7 @@ load("//python/private:pyproject_utils.bzl", "read_pyproject", "version_from_req load("//python/private:repo_utils.bzl", "repo_utils") load(":default_platforms.bzl", "default_platforms") load(":hub_builder.bzl", "hub_builder") -load(":hub_exposure.bzl", "hub_exposure") +load(":hub_exposure.bzl", "add_extras", "hub_exposure") load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") load(":labels.bzl", "extra_label") load(":parse_whl_name.bzl", "parse_whl_name") @@ -352,11 +352,13 @@ You cannot use both the additive_build_content and additive_build_content_file a # dict[hub, dict[whl, dict[version, str pip]]] # Where hub, whl, and pip are the repo names hub_whl_map = {} + hub_target_platform_envs = {} hub_group_map = {} exposed_packages = {} extra_aliases = {} whl_libraries = {} hubs = {} + hub_platforms = {} for hub in pip_hub_map.values(): out = hub.build() @@ -370,6 +372,8 @@ You cannot use both the additive_build_content and additive_build_content_file a extra_aliases[hub.name] = out.extra_aliases hub_group_map[hub.name] = out.group_map hub_whl_map[hub.name] = out.whl_map + hub_target_platform_envs[hub.name] = out.target_platform_envs + hub_platforms[hub.name] = out.platforms hubs[hub.name] = struct( # str: module that created the hub. module_name = hub.module_name, @@ -379,14 +383,24 @@ You cannot use both the additive_build_content and additive_build_content_file a packages = out.whl_map.keys(), ) - configured_hubs = hub_exposure( + exposure = hub_exposure( module_ctx, hubs = hubs, default_pyproject = config.pyproject_toml, _fail = _fail, ) - if configured_hubs == None: + if exposure == None: return None + configured_hubs = exposure.packages + + # The extras requested in the `pyproject.toml`, e.g. `requests[socks]`, + # are added to the hub even if the lock file does not have them. + for hub_name, extras in exposure.extras.items(): + hub_whl_map[hub_name] = add_extras( + hub_whl_map[hub_name], + extras, + platforms = hub_platforms[hub_name], + ) # The `all_*requirements` only list the selected packages that are # available on all of the platforms, while all of the selected packages @@ -410,6 +424,9 @@ You cannot use both the additive_build_content and additive_build_content_file a facts = simpleapi_cache.get_facts(), hub_group_map = hub_group_map, hub_whl_map = hub_whl_map, + # dict[str, list[dict[str, str]]]: the marker environments of the + # target platforms that have requirements in each hub. + hub_target_platform_envs = hub_target_platform_envs, unified_hub_requirements_bzl = config.unified_hub_requirements_bzl, whl_libraries = whl_libraries, whl_mods = whl_mods, @@ -583,6 +600,7 @@ def _pip_impl(module_ctx): packages = mods.exposed_packages.get(hub_name, []), platform_config_settings = mods.platform_config_settings.get(hub_name, {}), groups = mods.hub_group_map.get(hub_name), + target_platform_envs = json.encode(mods.hub_target_platform_envs.get(hub_name, [])), # The hubs configured with `pip.configure` hide the other packages. hide_unexposed_packages = hub_name in mods.configured_hubs, public_packages = mods.configured_hubs.get(hub_name, []), @@ -1176,6 +1194,11 @@ Each item is one of: packages of the hub including the transitive dependencies, i.e. all of the packages from the requirements files or the `uv.lock` file. +The extras of the selected requirements, e.g. `requests[socks]`, are also +added to the hub for the target platforms that match the environment markers +of the requirements. With `"transitive"` the extras of all of the groups are +added. + See {ref}`Configuring the Exposed Packages ` for examples. :::{versionadded} VERSION_NEXT_FEATURE @@ -1213,7 +1236,9 @@ The packages are selected from a `pyproject.toml` file using the `[project] dependencies`, `[project.optional-dependencies]` and PEP 735 `[dependency-groups]` tables. The other packages of the hub, e.g. the transitive dependencies, are still used by the packages that need them, but -they are not public. +they are not public. The extras requested by the selected requirements, e.g. +`requests[socks]`, are added to the hub even if the lock file does not request +them. The tags work like a builder: they are applied in order and multiple tags for the same hub are combined, so each tag can configure a different part of the diff --git a/python/private/pypi/extra_dep.bzl b/python/private/pypi/extra_dep.bzl new file mode 100644 index 0000000000..cbeef12fd6 --- /dev/null +++ b/python/private/pypi/extra_dep.bzl @@ -0,0 +1,223 @@ +"""Rules to depend on an extra of a PyPI package without knowing its extras. + +The dependencies of an extra, e.g. `bar[socks]`, are only known by reading the +`METADATA` of `bar`, which is only available in the whl repository of `bar`. +The hub cannot create the `[socks]` target for every extra that `bar` +provides without fetching the wheels, so the dependency on an extra is wired +as follows: + +1. The package that depends on `bar[socks]` creates an {obj}`extra_dep` + target, which depends on the `_extras` target of `bar` with the + {obj}`REQUESTED_EXTRA` setting set to `socks`. +2. The whl repository of `bar` creates an {obj}`extra_select` target named + `_extras`, which selects the `[socks]` target based on the value of the + setting and resets the setting to its default value, so that the targets of + the extras are analyzed in the same configuration as the rest of the build. + +The hub only needs to forward the `_extras` target of each package to the +right whl repository for the target platform. + +Both rules forward the providers of their only dependency, which is in the +`deps` attribute so that the aspects propagating along `deps` reach the +package. +""" + +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load("//python/private:py_cc_link_params_info.bzl", "PyCcLinkParamsInfo") +load("//python/private:py_info.bzl", "PyInfo") +load("//python/private:reexports.bzl", "BuiltinPyInfo") + +# The name of the target that selects the requested extra of a package. +EXTRAS_SELECT_TARGET = "_extras" + +# The setting that carries the requested extra from the {obj}`extra_dep` to the +# {obj}`extra_select` target. +REQUESTED_EXTRA = str(Label("//python/private/pypi:requested_extra")) + +# All of the providers of the targets in the whl and hub repositories, i.e. +# `py_library` and `missing_package_error`. `DefaultInfo` is always present. +_FORWARDED_PROVIDERS = [ + p + for p in [ + PyInfo, + BuiltinPyInfo, + PyCcLinkParamsInfo, + InstrumentedFilesInfo, + OutputGroupInfo, + ] + if p != None +] + +def _requested_extra_impl(ctx): + return [BuildSettingInfo(value = ctx.build_setting_value)] + +requested_extra = rule( + implementation = _requested_extra_impl, + build_setting = config.string(flag = False), + doc = """\ +The normalized name of the extra requested by an {obj}`extra_dep` target. + +This is only set on the edge between {obj}`extra_dep` and the +{obj}`extra_select` target and it is reset by the {obj}`extra_select` target. +""", +) + +def _set_requested_extra_impl(_settings, attr): + return {REQUESTED_EXTRA: attr.extra} + +_set_requested_extra = transition( + implementation = _set_requested_extra_impl, + inputs = [], + outputs = [REQUESTED_EXTRA], +) + +def _reset_requested_extra_impl(_settings, _attr): + return {REQUESTED_EXTRA: ""} + +_reset_requested_extra = transition( + implementation = _reset_requested_extra_impl, + inputs = [], + outputs = [REQUESTED_EXTRA], +) + +def _forward(ctx): + if len(ctx.attr.deps) != 1: + fail("Expected exactly one target in 'deps', got: {}".format(ctx.attr.deps)) + + target = ctx.attr.deps[0] + return [target[DefaultInfo]] + [ + target[provider] + for provider in _FORWARDED_PROVIDERS + if provider in target + ] + +def _extra_dep_impl(ctx): + return _forward(ctx) + +extra_dep = rule( + implementation = _extra_dep_impl, + doc = """\ +Depend on an extra of a PyPI package. + +The only target in `deps` is the {obj}`extra_select` target of the package, +which resolves the `extra` to the target that includes the package and the +dependencies of the extra. All of its providers are forwarded. +""", + attrs = { + "deps": attr.label_list( + mandatory = True, + cfg = _set_requested_extra, + doc = "The {obj}`extra_select` target of the package.", + ), + "extra": attr.string( + mandatory = True, + doc = "The normalized name of the extra.", + ), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + }, +) + +def _extra_select_impl(ctx): + extra = ctx.attr._requested_extra[BuildSettingInfo].value + if extra and extra not in ctx.attr.extras: + # This is how `pip` handles the unknown extras. + # buildifier: disable=print + print("WARNING: {} does not provide the extra '{}'".format( + ctx.attr.distribution, + extra, + )) + + for warning in ctx.attr.warnings.get(extra, []): + # buildifier: disable=print + print(warning) + + return _forward(ctx) + +extra_select = rule( + implementation = _extra_select_impl, + doc = """\ +Forward the target of the extra that is requested by an {obj}`extra_dep`. + +The `deps` is expected to be a `select` on the {obj}`REQUESTED_EXTRA` +setting that results in a single target. The setting is reset on the edge to +`deps` so that the targets of the extras are analyzed in the same +configuration regardless of how they are reached. All of the providers of +the target are forwarded. +""", + attrs = { + "deps": attr.label_list( + mandatory = True, + cfg = _reset_requested_extra, + doc = "The target for the requested extra.", + ), + "distribution": attr.string( + doc = "The name and the version of the package for the warning about the unknown extras.", + ), + "extras": attr.string_list( + doc = "The normalized extras that have a target, a warning is printed if any other extra is requested.", + ), + "warnings": attr.string_list_dict( + doc = """\ +The warnings to print when the extra in the key is requested, e.g. about the +dependencies of the extra that are not in the hub. They are printed here and +not when the targets are created, because only the extras that are used +matter. +""", + ), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + "_requested_extra": attr.label( + default = REQUESTED_EXTRA, + ), + }, +) + +def extra_select_targets( + *, + name, + targets, + default, + distribution = "", + warnings = {}, + native = native, + rules = None, + visibility = None): + """Create the {obj}`extra_select` target and the config settings for the extras. + + Args: + name: {type}`str` the name of the {obj}`extra_select` target. + targets: {type}`dict[str, str]` the normalized extras to the labels of + the targets that include the extra. + default: {type}`str` the label used when the requested extra is not + provided by the package, which follows `pip` that only warns about + unknown extras. + distribution: {type}`str` the name and the version of the package + used in the warning about the unknown extras. + warnings: {type}`dict[str, list[str]]` the warnings to print when + the extra in the key is requested. + native: the native struct for overriding in tests. + rules: a struct with the `extra_select` rule for overriding in tests. + visibility: {type}`list[str]` the visibility of the target. + """ + rules = rules or struct(extra_select = extra_select) + deps = {} + for extra, label in sorted(targets.items()): + setting = "{}_{}".format(name, extra) + native.config_setting( + name = setting, + flag_values = {REQUESTED_EXTRA: extra}, + visibility = ["//visibility:private"], + ) + deps[":" + setting] = [label] + + rules.extra_select( + name = name, + deps = select(deps | {"//conditions:default": [default]}) if deps else [default], + distribution = distribution, + extras = sorted(targets), + warnings = warnings, + visibility = visibility, + ) diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index 3b3f64cc81..2e4a862d65 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -30,6 +30,7 @@ _RENDER = { "requires_dist": render.list, "srcs_exclude": render.list, "tags": render.list, + "target_platform_envs": str, } # NOTE @aignas 2024-10-25: We have to keep this so that files in @@ -98,8 +99,9 @@ def generate_whl_library_build_bazel( if config_load: # The config is also needed without any deps, because it has the # extras that are requested from this package. - loads.append("""load("{}", "{}")""".format(config_load, "packages")) + loads.append("""load("{}", "packages", "target_platform_envs")""".format(config_load)) kwargs["include"] = "packages" + kwargs["target_platform_envs"] = "target_platform_envs" if requires_dist: kwargs["requires_dist"] = requires_dist diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 05def3daf4..c85701344a 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -78,6 +78,10 @@ def hub_builder( # setting originated from. # dict[str whl_name, dict[str config_setting, str repo_name]] _whl_map = {}, # modified by _add_whl_library + # The marker environments of the target platforms that have + # requirements. Dict acting as a set of the values. + # dict[str json, dict[str, str] env] + _target_platform_envs = {}, # modified by _create_whl_repos # Internal @@ -116,6 +120,8 @@ def _build(self): extra_aliases = {}, exposed_packages = [], whl_libraries = {}, + target_platform_envs = [], + platforms = {}, ) if self._logger.failed(): return ret @@ -145,6 +151,21 @@ def _build(self): # Mapping of whl_library repo names and their kwargs. # dict[str repo_name, dict[str, object] kwargs] whl_libraries = self._whl_libraries, + # The marker environments of the target platforms that have + # requirements. + # list[dict[str, str]] + target_platform_envs = self._target_platform_envs.values(), + # The marker environments of the target platforms for each Python + # version, the keys are the same as in the `whl_config_setting` + # values of the `whl_map`. + # dict[str python_version, dict[str target_platform, dict[str, str]]] + platforms = { + _major_minor_version(python_version): { + whl_config_setting(target_platforms = [name]).target_platforms[0]: platform.env + for name, platform in platforms.items() + } + for python_version, platforms in self._platforms.items() + }, ) def _pip_parse(self, module_ctx, pip_attr, default_envsubst = [], python_version = None): @@ -609,6 +630,20 @@ def _create_whl_repos( ) for whl in requirements_by_platform: + for src in whl.srcs: + for p in src.target_platforms: + if p not in platforms: + continue + + # The functions for normalizing the values are added back + # when evaluating the markers. + env = { + k: v + for k, v in platforms[p].env.items() + if not k.startswith("_") + } + self._target_platform_envs[json.encode(env)] = env + whl_library_args = common_args | _whl_library_args( self, whl = whl, diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl index 7393c28fbe..08abf25438 100644 --- a/python/private/pypi/hub_exposure.bzl +++ b/python/private/pypi/hub_exposure.bzl @@ -1,7 +1,9 @@ -"""Compute the packages exposed by the hubs configured with `pip.configure`.""" +"""Compute the packages and extras exposed by the hubs configured with `pip.configure`.""" load("//python/private:pyproject_utils.bzl", "read_pyproject") load(":dependency_groups.bzl", "select_dependency_groups") +load(":pep508_evaluate.bzl", "evaluate") +load(":whl_config_setting.bzl", "whl_config_setting") def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): """Compute the packages exposed by the hubs configured with `pip.configure`. @@ -18,10 +20,16 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): _fail: {type}`callable` the failure function, mainly for testing. Returns: - {type}`dict[str, list[str]] | None` the configured hubs mapped to the - sorted packages that they expose. The hubs that are not configured or - that expose all of their packages are not included, so that they - behave the same. Returns `None` on failure. + {type}`struct | None` with the attributes below or `None` on failure: + * `packages`: {type}`dict[str, list[str]]` the configured hubs + mapped to the sorted packages that they expose. The hubs that are + not configured or that expose all of their packages are not + included, so that they behave the same. + * `extras`: {type}`dict[str, dict[str, dict[tuple[str], list[str]]]]` + the configured hubs mapped to the packages and the extras that + the selected dependency groups request for them, see + {obj}`select_dependency_groups` for the format. It is used with + {obj}`add_extras`. """ # The root module wins over the owner of the hub when both configure it. @@ -35,6 +43,7 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): configured = {} exposes_all = {} + extras = {} for mod in module_ctx.modules: # The tags of a module are applied in order like a builder: a tag # without a `pyproject_toml` uses the last one set for the hub. @@ -90,8 +99,111 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): if name in info.packages: exposed[name] = None + hub_extras = extras.setdefault(hub, {}) + for name, by_markers in selection.extras.items(): + if name not in info.packages: + continue + for markers, pkg_extras in by_markers.items(): + merged = hub_extras.setdefault(name, {}).setdefault(markers, {}) + for extra in pkg_extras: + merged[extra] = None + + return struct( + # dict[str, list[str]]: configured hubs and exposed packages. + packages = { + hub: sorted(packages) + for hub, packages in configured.items() + if hub not in exposes_all + }, + # dict[str, dict[str, dict[tuple[str], list[str]]]]: configured hubs + # and the extras of their packages by the markers. + extras = { + hub: { + name: { + markers: sorted(pkg_extras) + for markers, pkg_extras in sorted(by_markers.items()) + } + for name, by_markers in sorted(hub_extras.items()) + } + for hub, hub_extras in extras.items() + if hub_extras + }, + ) + +def add_extras(whl_map, extras, *, platforms): + """Add the extras requested by `pip.configure` to the hub `whl_map`. + + The extras are added to the config settings of the package, so that the + hub has the `[]` targets and the package target includes them. The + environment markers of the requirements are evaluated for each target + platform, like it is done for the requirements files, so an extra is only + added for the platforms that request it. If the platforms of a config + setting request different extras, the config setting is split. + + Args: + whl_map: {type}`dict[str, dict[str, list[struct]]]` the packages + mapped to the repo names and their {obj}`whl_config_setting` + values. + extras: {type}`dict[str, dict[tuple[str], list[str]]]` the packages + mapped to the normalized extras to add keyed by the markers that + all need to be true on a target platform. + platforms: {type}`dict[str, dict[str, dict[str, str]]]` the Python + versions of the hub mapped to their target platforms and the + marker environments of the platforms. The versions and the + platforms are the same as in the {obj}`whl_config_setting` + values. + + Returns: + {type}`dict[str, dict[str, list[struct]]]` the new `whl_map`. + """ + if not extras: + return whl_map + return { - hub: sorted(packages) - for hub, packages in configured.items() - if hub not in exposes_all + name: { + repo: [ + new_setting + for setting in settings + for new_setting in _add_extras(setting, extras[name], platforms) + ] + for repo, settings in repos.items() + } if name in extras else repos + for name, repos in whl_map.items() } + +def _add_extras(setting, extras, platforms): + envs = platforms.get(setting.version, {}) + + # A config setting without the platforms is used for all of them. + target_platforms = setting.target_platforms or sorted(envs) + if not target_platforms: + # The platforms are not known, so only the extras without markers can + # be added. + target_platforms = [None] + + plats_by_extras = {} + for platform in target_platforms: + env = envs.get(platform) + plat_extras = {extra: None for extra in setting.extras or []} + for markers, pkg_extras in extras.items(): + if markers and not env: + continue + if len([m for m in markers if evaluate(m, env = env)]) == len(markers): + plat_extras.update({extra: None for extra in pkg_extras}) + plats_by_extras.setdefault(tuple(sorted(plat_extras)), []).append(platform) + + if len(plats_by_extras) == 1: + return [whl_config_setting( + version = setting.version, + target_platforms = setting.target_platforms, + extras = plats_by_extras.keys()[0], + )] + + return [ + whl_config_setting( + version = setting.version, + target_platforms = plats, + extras = plat_extras, + ) + for plat_extras, plats in plats_by_extras.items() + ] diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index 27fa5fdece..67d8f0af1a 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -57,9 +57,14 @@ def _impl(rctx): "config.bzl", rctx.attr._config_template, substitutions = { - # The available extras of each package for the spoke repositories - # to check if the extras of their dependencies are available. + # The requested extras of each package for the spoke repositories + # to create the targets of the extras that the package does not + # provide. "%%PACKAGES%%": render.dict(available_extras(whl_map), value_repr = render.list), + "%%TARGET_PLATFORM_ENVS%%": render.list( + json.decode(rctx.attr.target_platform_envs or "[]"), + value_repr = render.dict, + ), }, ) rctx.template("requirements.bzl", rctx.attr._requirements_bzl_template, substitutions = { @@ -117,6 +122,14 @@ the platforms. mandatory = True, doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", ), + "target_platform_envs": attr.string( + doc = """\ +The JSON encoded list of the marker environments of the target platforms that +have requirements in the hub. The spoke repositories use it to warn about the +dependencies that are not in the hub, but are needed on any of the target +platforms. +""", + ), "whl_map": attr.string_dict( mandatory = True, doc = """\ diff --git a/python/private/pypi/missing_package.bzl b/python/private/pypi/missing_package.bzl index e5647d5786..46823e813b 100644 --- a/python/private/pypi/missing_package.bzl +++ b/python/private/pypi/missing_package.bzl @@ -1,16 +1,25 @@ -"""Rules for generating an execution-phase action failure when a PyPI package or its extra is missing.""" +"""Rule for generating an execution-phase action failure when a PyPI package is missing.""" load("//python/private:py_info.bzl", "PyInfo") load("//python/private:reexports.bzl", "BuiltinPyInfo") -def _error(ctx, message): +def _missing_package_error_impl(ctx): out = ctx.actions.declare_file(ctx.label.name + ".error") + if ctx.attr.hub_name: + hub_clause = ' when building under PyPI hub "{hub}". Try adding it to the requirements of this hub (e.g. requirements_lock or requirements_by_platform in pip.parse)'.format( + hub = ctx.attr.hub_name, + ) + else: + hub_clause = ' because no default PyPI hub was configured. Try designating a default hub via pip.default(default_hub = "...") or select a hub using --@rules_python//python/config_settings:venv' + # Register an action that fails when Bazel attempts to stage/build this file ctx.actions.run_shell( outputs = [out], - command = "echo \"$1\" >&2 && exit 1", - arguments = ["ERROR: " + message], + command = "echo 'ERROR: PyPI package \"{pkg}\" is not available{hub_clause}.' >&2 && exit 1".format( + pkg = ctx.attr.package_name, + hub_clause = hub_clause, + ), ) maybe_builtin = [BuiltinPyInfo(transitive_sources = depset([out]))] if BuiltinPyInfo != None else [] @@ -25,19 +34,6 @@ def _error(ctx, message): ), ] + maybe_builtin -def _missing_package_error_impl(ctx): - if ctx.attr.hub_name: - hub_clause = ' when building under PyPI hub "{hub}". Try adding it to the requirements of this hub (e.g. requirements_lock or requirements_by_platform in pip.parse)'.format( - hub = ctx.attr.hub_name, - ) - else: - hub_clause = ' because no default PyPI hub was configured. Try designating a default hub via pip.default(default_hub = "...") or select a hub using --@rules_python//python/config_settings:venv' - - return _error(ctx, 'PyPI package "{pkg}" is not available{hub_clause}.'.format( - pkg = ctx.attr.package_name, - hub_clause = hub_clause, - )) - missing_package_error = rule( implementation = _missing_package_error_impl, attrs = { @@ -45,25 +41,3 @@ missing_package_error = rule( "package_name": attr.string(mandatory = True), }, ) - -def _missing_extra_error_impl(ctx): - return _error(ctx, ( - 'The PyPI package "{pkg}" requires "{dep}[{extra}]", but the extra "{extra}" of ' + - '"{dep}" is not available in the hub. Add "{dep}[{extra}]" to the requirements or ' + - "the lock file of the hub, e.g. by compiling the requirements with the extras kept " + - "(`--no-strip-extras`) or by adding the extra to the project dependencies." - ).format( - pkg = ctx.attr.required_by, - dep = ctx.attr.package_name, - extra = ctx.attr.extra, - )) - -missing_extra_error = rule( - implementation = _missing_extra_error_impl, - doc = "A target that fails the build with a helpful message when an extra of a PyPI package is missing.", - attrs = { - "extra": attr.string(mandatory = True), - "package_name": attr.string(mandatory = True), - "required_by": attr.string(mandatory = True), - }, -) diff --git a/python/private/pypi/pep508_deps.bzl b/python/private/pypi/pep508_deps.bzl index 8aa74194fa..90002b1dc1 100644 --- a/python/private/pypi/pep508_deps.bzl +++ b/python/private/pypi/pep508_deps.bzl @@ -28,7 +28,8 @@ def deps( extras = [], excludes = [], include = [], - keep_extras = False): + keep_extras = False, + envs = []): """Parse the RequiresDist from wheel METADATA Args: @@ -42,12 +43,19 @@ def deps( extras: {type}`list[str]` the requested extras to generate targets for. keep_extras: {type}`bool` keep the extras of the dependencies, if `True`, the dependencies with extras are returned as `foo[bar]`. + envs: {type}`list[dict[str, str]]` the marker environments of the + target platforms. If specified, the dependencies that are left + out because they are not in `include` are returned as `missing` + if they are needed on any of the target platforms. Returns: A struct with attributes: * deps: {type}`list[str]` dependencies to include unconditionally. * deps_select: {type}`dict[str, list[str]]` dependencies to include on particular subset of target platforms. + * missing: {type}`dict[str, list[str]]` the names of the + dependencies that are not in `include`, but are needed on at + least one of the `envs`, and the requirements that need them. The dependencies are normalized names, e.g. `foo`, or normalized names with a single extra, e.g. `foo[bar]`, if the dependency is requested @@ -58,6 +66,7 @@ def deps( return struct( deps = [], deps_select = {}, + missing = {}, ) reqs = sorted( @@ -72,6 +81,7 @@ def deps( excludes = {group_member(x): None for x in excludes} reqs_by_dep = {} + missing = {} for req in reqs: # drop self edges @@ -79,6 +89,8 @@ def deps( continue if include and req.name_ not in include: + if envs and _is_needed(req, extras = want_extras, envs = envs): + missing.setdefault(req.name, []).append(_format(req)) continue for dep in ["{}[{}]".format(req.name_, e) for e in req.extras if keep_extras] or [req.name_]: @@ -100,8 +112,37 @@ def deps( d: markers for d, markers in sorted(deps_select.items()) }, + missing = missing, ) +def _format(req): + return "{}{}{}".format( + req.name, + "[{}]".format(",".join(req.extras)) if req.extras else "", + "; {}".format(req.marker) if req.marker else "", + ) + +def _is_needed(req, *, extras, envs): + """Tell if the requirement is needed on any of the target platforms.""" + if not req.marker: + return True + + for extra in extras: + # Most of the requirements are for the other extras, so do not go + # through the target platforms for them. + env = create_env() | {"extra": extra} + m = evaluate(req.marker, env = env, strict = False) + if m == False: + continue + elif m == True: + return True + + for target_env in envs: + if evaluate(req.marker, env = env | target_env, strict = False) != False: + return True + + return False + def _add(deps, deps_select, dep, markers = None): if not markers: deps[dep] = True diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index a692296683..5fe8b6c1c7 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -286,9 +286,9 @@ def _pip_repository_impl(rctx): "config.bzl", rctx.attr._config_template, substitutions = { - # The available extras of each package for the spoke - # repositories to check if the extras of their dependencies - # are available. + # The requested extras of each package for the spoke + # repositories to create the targets of the extras that the + # package does not provide. "%%PACKAGES%%": render.dict({ pkg: sorted(selected_extras.get(pkg, [])) for pkg in bzl_packages diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index d64bc7a15f..a1fdfe72a9 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -35,6 +35,7 @@ load("@bazel_skylib//lib:selects.bzl", "selects") load("//python:py_library.bzl", "py_library") load("//python/private:common_labels.bzl", "labels") load("//python/private:text_util.bzl", "render") +load(":extra_dep.bzl", "EXTRAS_SELECT_TARGET", "extra_dep", "extra_select") load( ":labels.bzl", "DATA_LABEL", @@ -124,6 +125,8 @@ def pkg_aliases( _native = kwargs.pop("native", native) rules = kwargs.pop("rules", struct( + extra_dep = extra_dep, + extra_select = extra_select, py_library = py_library, whl_library_deps_targets = whl_library_deps_targets, )) @@ -157,6 +160,10 @@ def pkg_aliases( # targets do not use the extras, so only the package without extras # can be a member of the group. target_names = { + # The whl repositories resolve the extras requested by the + # packages that depend on them, so the hub only needs to forward + # to the whl repository of the target platform. + EXTRAS_SELECT_TARGET: EXTRAS_SELECT_TARGET, WHEEL_FILE_PUBLIC_LABEL: WHEEL_FILE_IMPL_LABEL if "" in group_extras else WHEEL_FILE_PUBLIC_LABEL, } @@ -214,7 +221,10 @@ def pkg_aliases( fail("The `actual` arg must be a dictionary or a string") kwargs = {} - if target_name.startswith("_"): + if target_name == EXTRAS_SELECT_TARGET: + # Used by the whl repositories that depend on the extras. + pass + elif target_name.startswith("_"): kwargs["visibility"] = ["//_groups:__subpackages__"] elif target_name in (WHEEL_FILE, PY_SRCS_LABEL): kwargs["visibility"] = ["//visibility:private"] diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 9ba55f3a75..bc0d98cdfe 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -313,7 +313,7 @@ def _non_versioned_platform(p, *, strict = False): fail("Expected to always have a platform in the form '{{abi}}_{{os}}_{{arch}}', got: {}".format(p)) def available_extras(aliases): - """Return the extras that are available in the hub for each package. + """Return the extras that are requested in the hub for each package. Args: aliases: {type}`dict[str, dict[struct, str]]` the packages to the diff --git a/python/private/pypi/whl_config_repo.bzl b/python/private/pypi/whl_config_repo.bzl index b7cea5a8b7..e347b430cd 100644 --- a/python/private/pypi/whl_config_repo.bzl +++ b/python/private/pypi/whl_config_repo.bzl @@ -15,6 +15,9 @@ def _whl_config_repo_impl(rctx): rctx.attr._config_template, substitutions = { "%%PACKAGES%%": render.dict(rctx.attr.whl_map or {}, value_repr = lambda x: "None"), + # The requirements are only for the host platform, the + # environment of which is not known when the wheels are extracted. + "%%TARGET_PLATFORM_ENVS%%": "[]", }, ) diff --git a/python/private/pypi/whl_library_deps_targets.bzl b/python/private/pypi/whl_library_deps_targets.bzl index 68621e5f00..6fa47f28a3 100644 --- a/python/private/pypi/whl_library_deps_targets.bzl +++ b/python/private/pypi/whl_library_deps_targets.bzl @@ -3,6 +3,7 @@ load("//python:py_library.bzl", "py_library") load("//python/private:normalize_name.bzl", "normalize_name") load(":env_marker_setting.bzl", "env_marker_setting") +load(":extra_dep.bzl", "EXTRAS_SELECT_TARGET", "extra_dep", "extra_select", "extra_select_targets") load( ":labels.bzl", "DATA_LABEL", @@ -15,7 +16,6 @@ load( "extra_label", "group_member", ) -load(":missing_package.bzl", "missing_extra_error") load(":pep508_deps.bzl", "deps") def whl_library_deps_targets( @@ -28,6 +28,7 @@ def whl_library_deps_targets( provides_extra = [], extras = [], include = [], + target_platform_envs = [], group_deps = [], group_name = None, dep_template, @@ -37,7 +38,8 @@ def whl_library_deps_targets( rules = struct( py_library = py_library, env_marker_setting = env_marker_setting, - missing_extra_error = missing_extra_error, + extra_dep = extra_dep, + extra_select = extra_select, )): """Create all of the whl_library targets. @@ -54,6 +56,9 @@ def whl_library_deps_targets( provides_extra: {type}`list[str]` The list of extras that the package provides. For each extra the `[]` target is created that includes the package and the dependencies of that extra. + The `_extras` target selects one of them for the + {obj}`extra_dep` targets of the packages that depend on an extra + of this package. extras: {type}`list[str]` The list of requested extras. The `pkg` target includes them, which is how the extras are used when there is no hub repository to select them, e.g. when using a vendored @@ -64,12 +69,16 @@ def whl_library_deps_targets( `pip` does. include: {type}`list[str] | dict[str, list[str] | None]` The list of packages to include. If it is a dict, then the values are the - extras of each package that are available in the hub and the - dependencies on the extras that are not available will fail the - build with a helpful message. The `[]` targets are also - created for the extras of this package that are available in - the hub, see `extras` for the ones that the package does not + extras of each package that are requested in the hub. The + `[]` targets are also created for the requested extras of + this package, see `extras` for the ones that the package does not provide. + target_platform_envs: {type}`list[dict[str, str]]` The marker + environments of the target platforms of the hub. They are used + to warn about the dependencies that are left out because they + are not in `include`, but are needed on any of the target + platforms. No warnings are printed about such dependencies if it + is empty. group_name: {type}`str | None` name of the dependency group (if any). dep_template: {type}`str | None` The dep_template to use. tags: {type}`list[str]` The tags set on the targets. @@ -156,13 +165,14 @@ def whl_library_deps_targets( if normalize_name(e) not in provided_extras }) extras_targets = sorted(provided_extras.keys() + unknown_extras) + distribution = " ".join([metadata_name] + [ + t.partition("=")[2] + for t in tags + if t.startswith("pypi_version=") + ]) for extra in unknown_extras: getattr(rules, "warn", _warn)("WARNING: {} does not provide the extra '{}'".format( - " ".join([metadata_name] + [ - t.partition("=")[2] - for t in tags - if t.startswith("pypi_version=") - ]), + distribution, extra, )) whl_file_label = _name("", WHEEL_FILE_PUBLIC_LABEL) @@ -210,6 +220,67 @@ def whl_library_deps_targets( visibility = visibility if group_label or not requires_dist else impl_vis, ) + def _extra_target(extra): + if group_name and not group_label and is_member(extra): + # The group library of the members is in the hub. + return dep_template.format(name = self_name, target = extra_label(extra)) + + return ":" + extra_label(extra) + + # The dependencies of each target. The dependencies that are not in the + # hub are left out, which is only correct if they are not needed on any of + # the target platforms, so warn about the rest of them. The warnings about + # the dependencies of the extras that are not requested in the hub are + # only printed when another package depends on the extra, otherwise every + # package with extras that are not used would print them. + package_deps = {} + missing = {} + missing_by_extra = {} + for target, extra, want_extras in pkg_targets if requires_dist and hasattr(rules, "py_library") else []: + package_deps[target] = deps( + name = self_name, + requires_dist = requires_dist, + include = include, + excludes = members if is_member(extra) else [], + extras = want_extras, + keep_extras = True, + envs = target_platform_envs, + ) + if not extra or extra in hub_extras: + for dep, reqs in package_deps[target].missing.items(): + missing.setdefault(dep, {}).update({r: None for r in reqs}) + else: + missing_by_extra[extra] = package_deps[target].missing + + for dep, reqs in sorted(missing.items()): + getattr(rules, "warn", _warn)(_missing_dep_warning(distribution, dep, reqs)) + + extra_warnings = {} + for extra, extra_missing in sorted(missing_by_extra.items()): + for dep, reqs in sorted(extra_missing.items()): + # The rest are already printed. + reqs = [r for r in reqs if r not in missing.get(dep, {})] + if reqs: + extra_warnings.setdefault(extra, []).append( + _missing_dep_warning(distribution, dep, reqs), + ) + + # The packages that depend on an extra of this package use this target + # with the requested extra, because the hub does not know which extras + # this package provides. An extra without a target is the same as the + # package without extras and a warning is printed when it is requested. + if hasattr(rules, "extra_select"): + extra_select_targets( + name = EXTRAS_SELECT_TARGET, + targets = {extra: _extra_target(extra) for extra in extras_targets}, + default = _extra_target(""), + distribution = distribution, + warnings = extra_warnings, + native = native, + rules = rules, + visibility = visibility, + ) + if not requires_dist: # If there are extras, then they will be visible in requires_dist. return @@ -218,21 +289,18 @@ def whl_library_deps_targets( # dep and the expression. settings = {} - # The dependencies on extras that are not available in the hub. - missing_extras = {} + # The dependencies on the extras of other packages. + extra_deps = {} def _pkg_label(dep): dep_name, _, extra = dep.partition("[") if not extra: return dep_template.format(name = dep_name, target = PY_LIBRARY_PUBLIC_LABEL) - extra = extra.rstrip("]") - available = include.get(dep_name) if type(include) == type({}) else None - if available != None and extra not in available: - missing_extras[dep] = (dep_name, extra) - return ":_missing_" + dep - - return dep_template.format(name = dep_name, target = extra_label(extra)) + # The extra does not need to be requested in the hub, so it is + # resolved by the package that provides it. + extra_deps[dep] = (dep_name, extra.rstrip("]")) + return ":_" + dep whl_label = lambda dep: dep_template.format(name = dep, target = WHEEL_FILE_PUBLIC_LABEL) @@ -259,21 +327,9 @@ def whl_library_deps_targets( visibility = _visibility(""), ) - for name, extra, want_extras in pkg_targets if hasattr(rules, "py_library") else []: - package_deps = deps( - name = self_name, - requires_dist = requires_dist, - include = include, - excludes = members if is_member(extra) else [], - extras = want_extras, - # Without the hub, e.g. with the vendored `requirements.bzl`, the - # `pkg` targets of the dependencies already include the extras - # from their requirement lines, so the dependencies use them - # like before. - keep_extras = "//:" not in dep_template, - ) + for target, extra, _ in pkg_targets if hasattr(rules, "py_library") else []: rules.py_library( - name = name, + name = target, # We include as srcs to ensure that the (locations :pkg) works as expected. srcs = [repo_label(PY_SRCS_LABEL)], deps = _deps( @@ -281,8 +337,8 @@ def whl_library_deps_targets( # propagated. Just passing the target as `srcs` is not enough to propagate # `pyi_srcs`, see `tests/base_rules/py_library`. deps = [repo_label(PY_SRCS_LABEL)], - package_deps = package_deps, - conditions = _conditions(settings, package_deps), + package_deps = package_deps[target], + conditions = _conditions(settings, package_deps[target]), label = _pkg_label, ), # Disable precompilation on this wrapper target to prevent duplicate @@ -299,15 +355,27 @@ def whl_library_deps_targets( visibility = ["//visibility:private"], ) - for dep, (dep_name, extra) in sorted(missing_extras.items()): - rules.missing_extra_error( - name = "_missing_" + dep, - package_name = dep_name, + for dep, (dep_name, extra) in sorted(extra_deps.items()): + rules.extra_dep( + name = "_" + dep, + deps = [dep_template.format(name = dep_name, target = EXTRAS_SELECT_TARGET)], extra = extra, - required_by = metadata_name, visibility = ["//visibility:private"], ) +def _missing_dep_warning(distribution, dep, reqs): + return ( + "WARNING: {distribution} requires '{reqs}', but '{dep}' is not in " + + "the requirements of the hub, so it is left out and it may be " + + "missing at runtime on some of the target platforms. Add '{dep}' " + + "to the lock file, e.g. by locking for all of the target platforms " + + "and Python versions of the hub." + ).format( + distribution = distribution, + reqs = "', '".join(sorted(reqs)), + dep = dep, + ) + def _warn(msg): # buildifier: disable=print print(msg) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 7abc0c4ec5..a08d5ec5f8 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -30,6 +30,7 @@ def whl_library_targets( extras = [], entry_points = {}, include = [], + target_platform_envs = [], group_deps = [], group_name = None, dep_template = None, @@ -61,6 +62,9 @@ def whl_library_targets( {obj}`whl_library_deps_targets`. entry_points: {type}`list[dict]` A list of parsed entry point definitions. include: {type}`list[str]` The list of packages to include. + target_platform_envs: {type}`list[dict[str, str]]` The marker + environments of the target platforms of the hub, see + {obj}`whl_library_deps_targets`. group_name: {type}`str | None` name of the dependency group (if any). dep_template: {type}`str | None` The dep_template to use. data_exclude: {type}`list[str]` The globs for data attribute exclusion. @@ -103,6 +107,7 @@ def whl_library_targets( provides_extra = provides_extra, extras = extras, include = include, # only needed if requires_dist is present + target_platform_envs = target_platform_envs, repo = None, # set aliases in the same repo aliases = {}, **kwargs diff --git a/tests/bootstrap_impls/cached_venv_dangling_symlink_test.py b/tests/bootstrap_impls/cached_venv_dangling_symlink_test.py index 363736d0a0..35f9c54bad 100644 --- a/tests/bootstrap_impls/cached_venv_dangling_symlink_test.py +++ b/tests/bootstrap_impls/cached_venv_dangling_symlink_test.py @@ -41,7 +41,17 @@ def _load_bootstrap() -> dict: return bootstrap +def _can_create_symlinks() -> bool: + with tempfile.TemporaryDirectory() as temp: + try: + os.symlink(temp, os.path.join(temp, "link"), target_is_directory=True) + except OSError: + return False + return True + + class CachedVenvDanglingSymlinkTest(unittest.TestCase): + @unittest.skipUnless(_can_create_symlinks(), "can't create symlinks") def test_replaces_dangling_package_link_before_creating_child_link(self) -> None: self._test_replaces_dangling_package_link(symlinks_supported=True) diff --git a/tests/pypi/dependency_groups/dependency_groups_tests.bzl b/tests/pypi/dependency_groups/dependency_groups_tests.bzl index a53905bb04..a3817cd835 100644 --- a/tests/pypi/dependency_groups/dependency_groups_tests.bzl +++ b/tests/pypi/dependency_groups/dependency_groups_tests.bzl @@ -100,6 +100,106 @@ def _test_select_all(env): _tests.append(_test_select_all) +def _test_select_extras(env): + pyproject = struct( + name = "my-project", + dependencies = ["main-pkg[Foo_Bar, baz]>=1.0", "plain-pkg"], + optional_dependencies = { + "all": ["my-project[docs]"], + "docs": ["docs-pkg[extra]", "main-pkg[docs]"], + }, + dependency_groups = { + "dev": ["dev-pkg[cli]", {"include-group": "test"}], + "test": [ + "test-pkg[cov]; sys_platform == 'linux'", + "test-pkg[win]; sys_platform == 'win32'", + "test-pkg[cov]", + ], + }, + ) + for specs, want in [ + (["default"], {"main_pkg": {(): ["baz", "foo_bar"]}}), + # The markers of the requirements are kept, so that the extras can be + # added only for the platforms that request them. + (["dev"], { + "dev_pkg": {(): ["cli"]}, + "test_pkg": { + (): ["cov"], + ("sys_platform == 'linux'",): ["cov"], + ("sys_platform == 'win32'",): ["win"], + }, + }), + # The extras of the self reference are the optional dependencies of + # the project, not the extras of a package. + (["all"], {"docs_pkg": {(): ["extra"]}, "main_pkg": {(): ["docs"]}}), + (["default", "docs"], { + "docs_pkg": {(): ["extra"]}, + "main_pkg": {(): ["baz", "docs", "foo_bar"]}, + }), + ]: + got, failures = _select(specs, pyproject = pyproject) + env.expect.where(specs = specs).that_collection(failures).contains_exactly([]) + env.expect.where(specs = specs).that_dict(got.extras).contains_exactly(want) + + # The extras of all of the groups are returned when everything is selected. + got, failures = _select(["*", "transitive"], pyproject = pyproject) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_bool(got.all).equals(True) + env.expect.that_collection(got.packages).contains_exactly([]) + env.expect.that_dict(got.extras).keys().contains_exactly([ + "dev_pkg", + "docs_pkg", + "main_pkg", + "test_pkg", + ]) + +_tests.append(_test_select_extras) + +def _test_select_extras_self_reference_markers(env): + pyproject = struct( + name = "my-project", + dependencies = [ + "my-project[win]; sys_platform == 'win32'", + "my-project[cycle]; python_version < '3.12'", + ], + optional_dependencies = { + "cycle": ["my-project[win]", "my-project[cycle]", "cycle-pkg[c]"], + "win": [ + "win-pkg[a]", + "win-pkg[b]; python_version < '3.12'", + "my-project[cycle]", + ], + }, + dependency_groups = {}, + ) + + # The requirements of a group that is selected by a self reference with a + # marker are only requested if the marker is true. + got, failures = _select(["default"], pyproject = pyproject) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_collection(got.packages).contains_exactly(["cycle_pkg", "win_pkg"]) + env.expect.that_dict(got.extras).contains_exactly({ + "cycle_pkg": { + ("python_version < '3.12'",): ["c"], + ("sys_platform == 'win32'",): ["c"], + }, + "win_pkg": { + ("python_version < '3.12'",): ["a", "b"], + ("python_version < '3.12'", "sys_platform == 'win32'"): ["b"], + ("sys_platform == 'win32'",): ["a"], + }, + }) + + # Selecting the group directly requests it on all of the platforms. + got, failures = _select(["default", "win"], pyproject = pyproject) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got.extras["win_pkg"]).contains_at_least({ + (): ["a"], + ("python_version < '3.12'",): ["b"], + }) + +_tests.append(_test_select_extras_self_reference_markers) + def _test_select_empty_wildcards(env): pyproject = struct( name = "my-project", diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 19bb16dee3..6e3deedc04 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -1000,6 +1000,70 @@ def _test_configure_default_pyproject(env): _tests.append(_test_configure_default_pyproject) +def _test_configure_extras(env): + # The extras requested in the pyproject.toml are added to the hub even if + # the lock file does not request them, but only on the platforms that + # match the markers. + pypi = _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + _mod( + name = "rules_python", + parse = [_configure_parse( + pyproject_toml = "pyproject.toml", + target_platforms = ["linux_x86_64", "windows_aarch64"], + )], + configure = [_configure(hub_name = "pypi_a", dependency_groups = ["default"])], + ), + os_name = "linux", + arch_name = "x86_64", + mock_files = { + "pyproject.toml": """\ +[project] +dependencies = [ + "simple[foo]", + "simple[win]; sys_platform == 'win32'", + "other-pkg[not-provided]", + "other-pkg[old]; python_version < '3.10'", +] +""", + "requirements.txt": _CONFIGURE_REQUIREMENTS, + }, + ), + available_interpreters = { + "python_3_15_host": "unit_test_interpreter_target", + }, + minor_mapping = {"3.15": "3.15.19"}, + ) + + pypi.configured_hubs().contains_exactly({"pypi_a": ["other_pkg", "simple"]}) + pypi.hub_whl_map().contains_exactly({"pypi_a": { + "other_pkg": { + "pypi_a_315_other_pkg": [ + whl_config_setting(version = "3.15", extras = ["not_provided"]), + ], + }, + "simple": { + "pypi_a_315_simple": [ + whl_config_setting( + version = "3.15", + target_platforms = ["cp315_linux_x86_64"], + extras = ["foo"], + ), + whl_config_setting( + version = "3.15", + target_platforms = ["cp315_windows_aarch64"], + extras = ["foo", "win"], + ), + ], + }, + "transitive_pkg": { + "pypi_a_315_transitive_pkg": [whl_config_setting(version = "3.15")], + }, + }}) + +_tests.append(_test_configure_extras) + def _test_not_configured_exposes_all(env): # The pyproject.toml does not filter the hub without `pip.configure`. pypi = _configure_parse_modules( diff --git a/tests/pypi/extra_dep/BUILD.bazel b/tests/pypi/extra_dep/BUILD.bazel new file mode 100644 index 0000000000..1236a6ad28 --- /dev/null +++ b/tests/pypi/extra_dep/BUILD.bazel @@ -0,0 +1,3 @@ +load(":extra_dep_tests.bzl", "extra_dep_test_suite") + +extra_dep_test_suite(name = "extra_dep_tests") diff --git a/tests/pypi/extra_dep/extra_dep_tests.bzl b/tests/pypi/extra_dep/extra_dep_tests.bzl new file mode 100644 index 0000000000..f58f7a94f9 --- /dev/null +++ b/tests/pypi/extra_dep/extra_dep_tests.bzl @@ -0,0 +1,448 @@ +"""Tests for the extra_dep and extra_select rules.""" + +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("@rules_testing//lib:util.bzl", rt_util = "util") +load("//python:py_info.bzl", "PyInfo") +load("//python:py_library.bzl", "py_library") +load("//python/private:common_labels.bzl", "labels") # buildifier: disable=bzl-visibility +load("//python/private:py_cc_link_params_info.bzl", "PyCcLinkParamsInfo") # buildifier: disable=bzl-visibility +load("//python/private/pypi:env_marker_setting.bzl", "env_marker_setting") # buildifier: disable=bzl-visibility +load("//python/private/pypi:extra_dep.bzl", "extra_dep", "extra_select_targets") # buildifier: disable=bzl-visibility + +_tests = [] +_basic_tests = [] + +def _py_library(name, **kwargs): + """A py_library with a generated source to check the output configuration.""" + rt_util.helper_target( + native.genrule, + name = name + "_gen", + outs = [name + ".py"], + cmd = "touch $@", + ) + rt_util.helper_target( + py_library, + name = name, + srcs = [name + ".py"], + **kwargs + ) + +def _pkg(name, extras = {}): + """Create the targets of a package like the whl repositories do. + + Args: + name: the name of the package, the `_extras` target is created + for the packages that depend on its extras. + extras: the extras to the dependencies of the extra. + """ + _py_library(name) + for extra, deps in extras.items(): + rt_util.helper_target( + py_library, + name = "{}[{}]".format(name, extra), + deps = [name] + deps, + ) + + extra_select_targets( + name = name + "_extras", + targets = {extra: "{}[{}]".format(name, extra) for extra in extras}, + default = name, + distribution = name + " 1.0", + visibility = ["//visibility:private"], + ) + +def _extra_dep(name, pkg, extra): + rt_util.helper_target( + extra_dep, + name = name, + deps = [pkg + "_extras"], + extra = extra, + ) + +def _sources(target): + return sorted([f.basename for f in target[PyInfo].transitive_sources.to_list()]) + +def _test_extra(name): + _py_library(name + "_dep_a") + _pkg(name + "_foo", {"a": [name + "_dep_a"]}) + _extra_dep(name + "_subject", name + "_foo", "a") + analysis_test( + name = name, + impl = _test_extra_impl, + target = name + "_subject", + ) + +def _test_extra_impl(env, target): + env.expect.that_collection(_sources(target)).contains_exactly([ + "test_extra_dep_a.py", + "test_extra_foo.py", + ]) + + # The setting is reset, so the targets of the extra are analyzed in the + # same configuration as the target that depends on the extra. + env.expect.that_collection({ + f.root.path: None + for f in target[PyInfo].transitive_sources.to_list() + }.keys()).contains_exactly([env.ctx.bin_dir.path]) + +_tests.append(_test_extra) + +def _test_unknown_extra(name): + _py_library(name + "_dep_a") + _pkg(name + "_foo", {"a": [name + "_dep_a"]}) + _extra_dep(name + "_subject", name + "_foo", "unknown") + analysis_test( + name = name, + impl = _test_unknown_extra_impl, + target = name + "_subject", + ) + +def _test_unknown_extra_impl(env, target): + # Like `pip`, the extras that the package does not provide are ignored and + # a warning is printed. + env.expect.that_collection(_sources(target)).contains_exactly([ + "test_unknown_extra_foo.py", + ]) + +_tests.append(_test_unknown_extra) + +def _test_no_extras(name): + _pkg(name + "_foo") + _extra_dep(name + "_subject", name + "_foo", "a") + analysis_test( + name = name, + impl = _test_no_extras_impl, + target = name + "_subject", + ) + +def _test_no_extras_impl(env, target): + env.expect.that_collection(_sources(target)).contains_exactly([ + "test_no_extras_foo.py", + ]) + +_tests.append(_test_no_extras) + +def _test_different_extras_of_the_same_package(name): + # Both `p` and `q` depend on `c`, but each of them gets only the + # dependencies of the extra that it requests. + _py_library(name + "_tomli") + _py_library(name + "_other_dep") + _pkg(name + "_c", { + "other": [name + "_other_dep"], + "toml": [name + "_tomli"], + }) + _extra_dep(name + "_p_c[toml]", name + "_c", "toml") + _extra_dep(name + "_q_c[other]", name + "_c", "other") + _py_library(name + "_p", deps = [name + "_p_c[toml]"]) + _py_library(name + "_q", deps = [name + "_q_c[other]"]) + analysis_test( + name = name, + impl = _test_different_extras_of_the_same_package_impl, + targets = { + "c": name + "_c", + "p": name + "_p", + "q": name + "_q", + }, + ) + +def _test_different_extras_of_the_same_package_impl(env, targets): + prefix = "test_different_extras_of_the_same_package_" + env.expect.that_collection(_sources(targets.p)).contains_exactly([ + prefix + "c.py", + prefix + "p.py", + prefix + "tomli.py", + ]) + env.expect.that_collection(_sources(targets.q)).contains_exactly([ + prefix + "c.py", + prefix + "other_dep.py", + prefix + "q.py", + ]) + + # The package without extras does not get the dependencies of the extras. + env.expect.that_collection(_sources(targets.c)).contains_exactly([ + prefix + "c.py", + ]) + +_tests.append(_test_different_extras_of_the_same_package) + +def _test_env_markers_of_the_extra(name): + # This is `coverage[toml]`, which requires + # `tomli; python_full_version <= "3.11.0a6" and extra == "toml"`, and + # `pytest-cov` that requires `coverage[toml]`. + _py_library(name + "_tomli") + rt_util.helper_target( + env_marker_setting, + name = name + "_include_tomli", + expression = "python_full_version <= \"3.11.0a6\"", + ) + _pkg(name + "_coverage", { + "toml": select({ + ":is_{}_include_tomli_true".format(name): [name + "_tomli"], + "//conditions:default": [], + }), + }) + _extra_dep(name + "_dep_coverage[toml]", name + "_coverage", "toml") + _py_library(name + "_pytest_cov", deps = [name + "_dep_coverage[toml]"]) + + for version, want in { + "3.10": ["coverage.py", "pytest_cov.py", "tomli.py"], + "3.12": ["coverage.py", "pytest_cov.py"], + }.items(): + analysis_test( + name = "{}_{}".format(name, version.replace(".", "_")), + impl = _test_env_markers_of_the_extra_impl, + target = name + "_pytest_cov", + config_settings = { + labels.PYTHON_VERSION: version, + }, + attr_values = { + "want": [name + "_" + w for w in want], + }, + attrs = { + "want": attr.string_list(), + }, + ) + + native.test_suite( + name = name, + tests = [ + name + "_3_10", + name + "_3_12", + ], + ) + +def _test_env_markers_of_the_extra_impl(env, target): + # The setting of the python version is kept when the setting of the extra + # is set and reset. + env.expect.that_collection(_sources(target)).contains_exactly(env.ctx.attr.want) + +_tests.append(_test_env_markers_of_the_extra) + +_ProvidersInfo = provider( + doc = "The providers of a target to compare them in the tests", + fields = ["target"], +) + +def _providers_impl(ctx): + return [_ProvidersInfo(target = ctx.attr.target)] + +_providers = rule( + implementation = _providers_impl, + attrs = {"target": attr.label()}, +) + +def _test_providers_are_forwarded(name): + _py_library(name + "_generated", data = [name + "_data.txt"]) + rt_util.helper_target( + py_library, + name = name + "_dep_a", + srcs = ["instrumented.py"], + deps = [name + "_generated"], + ) + rt_util.helper_target( + native.genrule, + name = name + "_data", + outs = [name + "_data.txt"], + cmd = "touch $@", + ) + _pkg(name + "_foo", {"a": [name + "_dep_a"]}) + _extra_dep(name + "_subject", name + "_foo", "a") + rt_util.helper_target( + _providers, + name = name + "_want", + target = name + "_foo[a]", + ) + analysis_test( + name = name, + impl = _test_providers_are_forwarded_impl, + targets = { + "subject": name + "_subject", + "want": name + "_want", + }, + config_settings = { + "//command_line_option:collect_code_coverage": True, + "//command_line_option:instrumentation_filter": "//tests/pypi/extra_dep[:/]", + }, + ) + +def _test_providers_are_forwarded_impl(env, targets): + got = targets.subject + want = targets.want[_ProvidersInfo].target + + for provider in [PyInfo, PyCcLinkParamsInfo, InstrumentedFilesInfo, OutputGroupInfo]: + env.expect.where(provider = str(provider)).that_bool(provider in got).equals(True) + + env.expect.that_bool(got[PyInfo] == want[PyInfo]).equals(True) + env.expect.that_bool(got[PyCcLinkParamsInfo] == want[PyCcLinkParamsInfo]).equals(True) + + # The output groups of the target. + env.expect.that_collection(dir(got[OutputGroupInfo])).contains_exactly(dir(want[OutputGroupInfo])) + for group in dir(want[OutputGroupInfo]): + env.expect.where(group = group).that_depset_of_files( + getattr(got[OutputGroupInfo], group), + ).contains_exactly([ + f.short_path + for f in getattr(want[OutputGroupInfo], group).to_list() + ]) + + # The files and the runfiles of the target. + env.expect.that_depset_of_files(got[DefaultInfo].files).contains_exactly([ + f.short_path + for f in want[DefaultInfo].files.to_list() + ]) + for runfiles in ["default_runfiles", "data_runfiles"]: + got_files = sorted([f.short_path for f in getattr(got[DefaultInfo], runfiles).files.to_list()]) + want_files = sorted([f.short_path for f in getattr(want[DefaultInfo], runfiles).files.to_list()]) + env.expect.where(runfiles = runfiles).that_collection(got_files).contains_exactly(want_files) + env.expect.where(runfiles = runfiles).that_collection(got_files).contains( + "tests/pypi/extra_dep/test_providers_are_forwarded_data.txt", + ) + + # The coverage is collected for the sources of the extra. + got_instrumented = sorted([f.short_path for f in got[InstrumentedFilesInfo].instrumented_files.to_list()]) + env.expect.that_collection(got_instrumented).contains_exactly([ + f.short_path + for f in want[InstrumentedFilesInfo].instrumented_files.to_list() + ]) + env.expect.that_collection(got_instrumented).contains( + "tests/pypi/extra_dep/instrumented.py", + ) + +_tests.append(_test_providers_are_forwarded) + +_VisitedInfo = provider( + doc = "The targets visited by an aspect that propagates along the `deps`", + fields = ["labels"], +) + +def _deps_aspect_impl(target, ctx): + return [_VisitedInfo(labels = depset( + [target.label.name], + transitive = [ + dep[_VisitedInfo].labels + for dep in getattr(ctx.rule.attr, "deps", []) + if _VisitedInfo in dep + ], + ))] + +_deps_aspect = aspect( + implementation = _deps_aspect_impl, + attr_aspects = ["deps"], +) + +def _with_deps_aspect_impl(ctx): + return [ctx.attr.target[_VisitedInfo]] + +_with_deps_aspect = rule( + implementation = _with_deps_aspect_impl, + attrs = {"target": attr.label(aspects = [_deps_aspect])}, +) + +def _test_aspects_propagate_along_deps(name): + _py_library(name + "_dep_a") + _pkg(name + "_foo", {"a": [name + "_dep_a"]}) + _extra_dep(name + "_dep_foo[a]", name + "_foo", "a") + _py_library(name + "_consumer", deps = [name + "_dep_foo[a]"]) + rt_util.helper_target( + _with_deps_aspect, + name = name + "_subject", + target = name + "_consumer", + ) + analysis_test( + name = name, + impl = _test_aspects_propagate_along_deps_impl, + target = name + "_subject", + ) + +def _test_aspects_propagate_along_deps_impl(env, target): + prefix = "test_aspects_propagate_along_deps_" + env.expect.that_collection(target[_VisitedInfo].labels.to_list()).contains_exactly([ + prefix + "consumer", + prefix + "dep_foo[a]", + prefix + "foo_extras", + prefix + "foo[a]", + prefix + "foo", + prefix + "dep_a", + ]) + +_tests.append(_test_aspects_propagate_along_deps) + +def _test_extra_select_targets(env): + config_settings = [] + extra_selects = [] + + extra_select_targets( + name = "_extras", + targets = {"a": ":[a]", "b": ":[b]"}, + default = ":[]", + distribution = "foo 1.0", + native = struct(config_setting = lambda **kwargs: config_settings.append(kwargs)), + rules = struct(extra_select = lambda **kwargs: extra_selects.append(kwargs)), + ) + + env.expect.that_collection(config_settings).contains_exactly([ + { + "name": "_extras_a", + "flag_values": {str(Label("//python/private/pypi:requested_extra")): "a"}, + "visibility": ["//visibility:private"], + }, + { + "name": "_extras_b", + "flag_values": {str(Label("//python/private/pypi:requested_extra")): "b"}, + "visibility": ["//visibility:private"], + }, + ]) # buildifier: @unsorted-dict-items + env.expect.that_collection(extra_selects).contains_exactly([ + { + "name": "_extras", + "deps": select({ + ":_extras_a": [":[a]"], + ":_extras_b": [":[b]"], + "//conditions:default": [":[]"], + }), + "distribution": "foo 1.0", + "extras": ["a", "b"], + "warnings": {}, + "visibility": None, + }, + ]) # buildifier: @unsorted-dict-items + +_basic_tests.append(_test_extra_select_targets) + +def _test_extra_select_targets_no_extras(env): + extra_selects = [] + + extra_select_targets( + name = "_extras", + targets = {}, + default = ":[]", + native = struct(), + rules = struct(extra_select = lambda **kwargs: extra_selects.append(kwargs)), + visibility = ["//visibility:public"], + ) + + env.expect.that_collection(extra_selects).contains_exactly([ + { + "name": "_extras", + "deps": [":[]"], + "distribution": "", + "extras": [], + "warnings": {}, + "visibility": ["//visibility:public"], + }, + ]) # buildifier: @unsorted-dict-items + +_basic_tests.append(_test_extra_select_targets_no_extras) + +def extra_dep_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite( + name = name, + tests = _tests, + basic_tests = _basic_tests, + ) diff --git a/tests/pypi/extra_dep/instrumented.py b/tests/pypi/extra_dep/instrumented.py new file mode 100644 index 0000000000..8667111328 --- /dev/null +++ b/tests/pypi/extra_dep/instrumented.py @@ -0,0 +1 @@ +"""A source file for the tests, only the source files are instrumented.""" diff --git a/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl index 21af3416a9..7a23f83c25 100644 --- a/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl +++ b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl @@ -22,7 +22,7 @@ _tests = [] def _test_all_workspace(env): want = """\ load("@package_metadata//rules:package_metadata.bzl", "package_metadata") -load("@pypi//:config.bzl", "packages") +load("@pypi//:config.bzl", "packages", "target_platform_envs") load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets") package(default_visibility = ["//visibility:public"]) @@ -59,6 +59,7 @@ whl_library_targets( "pypi_name=foo", "pypi_version=0", ], + target_platform_envs = target_platform_envs, ) package_metadata( @@ -96,7 +97,7 @@ _tests.append(_test_all_workspace) def _test_all(env): want = """\ load("@package_metadata//rules:package_metadata.bzl", "package_metadata") -load("@pypi//:config.bzl", "packages") +load("@pypi//:config.bzl", "packages", "target_platform_envs") load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets") package(default_visibility = ["//visibility:public"]) @@ -134,6 +135,7 @@ whl_library_targets( "pypi_name=foo", "pypi_version=0", ], + target_platform_envs = target_platform_envs, ) package_metadata( @@ -172,7 +174,7 @@ _tests.append(_test_all) def _test_all_with_loads(env): want = """\ load("@package_metadata//rules:package_metadata.bzl", "package_metadata") -load("@pypi//:config.bzl", "packages") +load("@pypi//:config.bzl", "packages", "target_platform_envs") load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets") package(default_visibility = ["//visibility:public"]) @@ -209,6 +211,7 @@ whl_library_targets( "pypi_name=foo", "pypi_version=0", ], + target_platform_envs = target_platform_envs, ) package_metadata( @@ -248,7 +251,7 @@ def _test_no_deps_with_config(env): # are requested from the package in the hub. want = """\ load("@package_metadata//rules:package_metadata.bzl", "package_metadata") -load("@pypi//:config.bzl", "packages") +load("@pypi//:config.bzl", "packages", "target_platform_envs") load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets") package(default_visibility = ["//visibility:public"]) @@ -262,6 +265,7 @@ whl_library_targets( "pypi_name=foo", "pypi_version=0", ], + target_platform_envs = target_platform_envs, ) """ actual = generate_whl_library_build_bazel( diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index f9e2b0a905..a08cb17de1 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -113,6 +113,7 @@ def hub_builder( whl_map = subjects.dict, whl_libraries = subjects.dict, extra_aliases = subjects.dict, + target_platform_envs = subjects.collection, ), ), pip_parse = builder.pip_parse, @@ -1288,6 +1289,27 @@ simple==0.0.3 \ }) pypi.extra_aliases().contains_exactly({}) + # Only the target platforms that have requirements. + pypi.target_platform_envs().contains_exactly([ + { + "implementation_name": "cpython", + "implementation_version": "3.15.19", + "os_name": "posix", + "platform_machine": platform_machine, + "platform_python_implementation": "CPython", + "platform_release": "", + "platform_system": platform_system, + "platform_version": "0", + "python_full_version": "3.15.19", + "python_version": "3.15", + "sys_platform": sys_platform, + } + for platform_machine, platform_system, sys_platform in [ + ("x86_64", "Linux", "linux"), + ("aarch64", "Darwin", "darwin"), + ] + ]) + _tests.append(_test_download_only_multiple) def _test_simple_get_index(env): diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl index 1f2b13622c..f5f971b923 100644 --- a/tests/pypi/hub_exposure/hub_exposure_tests.bzl +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -3,8 +3,12 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load( "//python/private/pypi:hub_exposure.bzl", + "add_extras", "hub_exposure", ) # buildifier: disable=bzl-visibility +load("//python/private/pypi:pep508_env.bzl", pep508_env = "env") # buildifier: disable=bzl-visibility +load("//python/private/pypi:render_pkg_aliases.bzl", "available_extras") # buildifier: disable=bzl-visibility +load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # buildifier: disable=bzl-visibility load("//tests/support/mocks:mocks.bzl", "mocks") _tests = [] @@ -18,6 +22,14 @@ dev = ["other-pkg"] test = ["simple", "not-in-hub"] """ +_EXTRAS_PYPROJECT = """\ +[project] +dependencies = ["simple[foo]", "simple[Bar]; sys_platform == 'linux'"] + +[dependency-groups] +dev = ["other-pkg[baz]", "simple[foo]", "not-in-hub[qux]"] +""" + def _hub(*, module_name = "root", pyproject = None): return struct( module_name = module_name, @@ -38,12 +50,18 @@ def _configure( def _module(name = "root", is_root = True, configure = []): return mocks.module(name = name, is_root = is_root, configure = configure) -def _exposure(*modules, hubs = None, default_pyproject = None): +def _exposure(*modules, **kwargs): + """Return the exposed packages of the hubs.""" + got, failures = _hub_exposure(*modules, **kwargs) + return got.packages if got != None else None, failures + +def _hub_exposure(*modules, hubs = None, default_pyproject = None): failures = [] got = hub_exposure( mocks.mctx( modules = list(modules), mock_files = { + "extras.toml": _EXTRAS_PYPROJECT, "other.toml": "[project]\ndependencies = [\"other-pkg\"]\n", "pyproject.toml": _PYPROJECT, }, @@ -217,6 +235,224 @@ def _test_configure_module_precedence(env): _tests.append(_test_configure_module_precedence) +def _test_configure_extras(env): + hubs = {"hub": _hub(pyproject = "extras.toml")} + linux = ("sys_platform == 'linux'",) + for groups, want in [ + (["default"], {"hub": {"simple": {(): ["foo"], linux: ["bar"]}}}), + # The packages that are not in the hub are ignored. + (["dev"], {"hub": {"other_pkg": {(): ["baz"]}, "simple": {(): ["foo"]}}}), + # The extras are also returned when all of the packages are exposed. + (["*", "transitive"], {"hub": { + "other_pkg": {(): ["baz"]}, + "simple": {(): ["foo"], linux: ["bar"]}, + }}), + ]: + got, failures = _hub_exposure( + _module(configure = [_configure(dependency_groups = groups)]), + hubs = hubs, + ) + env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) + env.expect.where(groups = groups).that_dict(got.extras).contains_exactly(want) + + # The extras of multiple tags are combined. + got, _ = _hub_exposure( + _module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(dependency_groups = ["dev"]), + ]), + hubs = hubs, + ) + env.expect.that_dict(got.extras).contains_exactly({ + "hub": { + "other_pkg": {(): ["baz"]}, + "simple": {(): ["foo"], linux: ["bar"]}, + }, + }) + + # No extras are requested. + got, _ = _hub_exposure(_module(configure = [_configure(dependency_groups = ["*"])])) + env.expect.that_dict(got.extras).contains_exactly({}) + got, _ = _hub_exposure(_module(configure = [_configure()])) + env.expect.that_dict(got.extras).contains_exactly({}) + +_tests.append(_test_configure_extras) + +def _platforms(*platforms, version = "3.15"): + """The marker environments of the hub platforms like `hub_builder` returns them.""" + return { + version: { + "cp{}_{}_{}".format(version.replace(".", ""), os, arch): pep508_env( + os = os, + arch = arch, + python_version = version + ".0", + ) + for os, arch in platforms + }, + } + +_LINUX = "cp315_linux_x86_64" +_OSX = "cp315_osx_aarch64" +_WINDOWS = "cp315_windows_x86_64" + +def _test_add_extras(env): + whl_map = { + "other_pkg": {"repo_other": [whl_config_setting(version = "3.15")]}, + "simple": { + "repo_linux": [whl_config_setting( + version = "3.15", + target_platforms = [_LINUX], + extras = ["bar"], + )], + "repo_osx": [whl_config_setting( + version = "3.15", + target_platforms = [_OSX], + )], + }, + } + got = add_extras( + whl_map, + {"simple": {(): ["bar", "foo"]}}, + platforms = _platforms(("linux", "x86_64"), ("osx", "aarch64")), + ) + + # The extras without markers are added on all of the platforms. + env.expect.that_dict(got).contains_exactly({ + "other_pkg": {"repo_other": [whl_config_setting(version = "3.15")]}, + "simple": { + "repo_linux": [whl_config_setting( + version = "3.15", + target_platforms = [_LINUX], + extras = ["bar", "foo"], + )], + "repo_osx": [whl_config_setting( + version = "3.15", + target_platforms = [_OSX], + extras = ["bar", "foo"], + )], + }, + }) + + # Nothing to add. + env.expect.that_dict(add_extras(whl_map, {}, platforms = {})).contains_exactly(whl_map) + +_tests.append(_test_add_extras) + +def _test_add_extras_markers(env): + platforms = _platforms(("linux", "x86_64"), ("osx", "aarch64"), ("windows", "x86_64")) + whl_map = { + # A repo for all of the platforms, e.g. a pure Python wheel. + "any": {"repo_any": [whl_config_setting(version = "3.15")]}, + "simple": { + "repo_unix": [whl_config_setting( + version = "3.15", + target_platforms = [_LINUX, _OSX], + extras = ["locked"], + )], + "repo_windows": [whl_config_setting( + version = "3.15", + target_platforms = [_WINDOWS], + )], + }, + } + extras = { + "any": { + ("sys_platform == 'win32'",): ["win"], + # All of the markers need to be true. + ("python_version < '3.12'", "sys_platform == 'linux'"): ["old"], + }, + "simple": { + (): ["foo"], + ("sys_platform == 'win32'",): ["win"], + ("sys_platform == 'linux'",): ["lin"], + }, + } + + got = add_extras(whl_map, extras, platforms = platforms) + + # `foo[win]; sys_platform == "win32"` only adds the extra on Windows, so + # the other platforms do not get the dependencies of the extra, which may + # not be available for them. + env.expect.that_dict(got).contains_exactly({ + "any": { + "repo_any": [ + whl_config_setting( + version = "3.15", + target_platforms = [_LINUX, _OSX], + ), + whl_config_setting( + version = "3.15", + target_platforms = [_WINDOWS], + extras = ["win"], + ), + ], + }, + "simple": { + "repo_unix": [ + whl_config_setting( + version = "3.15", + target_platforms = [_LINUX], + extras = ["foo", "lin", "locked"], + ), + whl_config_setting( + version = "3.15", + target_platforms = [_OSX], + extras = ["foo", "locked"], + ), + ], + "repo_windows": [whl_config_setting( + version = "3.15", + target_platforms = [_WINDOWS], + extras = ["foo", "win"], + )], + }, + }) + + # The hub tells the whl repositories which extras are requested, so that + # they create the targets for the extras that the package does not + # provide, i.e. the hub never points to a target that does not exist. + env.expect.that_dict(available_extras({ + name: { + setting: repo + for repo, settings in repos.items() + for setting in settings + } + for name, repos in got.items() + })).contains_exactly({ + "any": ["win"], + "simple": ["foo", "lin", "locked", "win"], + }) + + # A marker that is the same on all of the platforms does not split the + # config setting. + got = add_extras( + whl_map, + {"any": {("python_version >= '3.12'",): ["new"], ("python_version < '3.12'",): ["old"]}}, + platforms = platforms, + ) + env.expect.that_dict(got["any"]).contains_exactly({ + "repo_any": [whl_config_setting(version = "3.15", extras = ["new"])], + }) + + # Only the extras without markers are added if the platforms of the Python + # version are not known. + got = add_extras(whl_map, extras, platforms = {}) + env.expect.that_dict(got["any"]).contains_exactly(whl_map["any"]) + env.expect.that_dict(got["simple"]).contains_exactly({ + "repo_unix": [whl_config_setting( + version = "3.15", + target_platforms = [_LINUX, _OSX], + extras = ["foo", "locked"], + )], + "repo_windows": [whl_config_setting( + version = "3.15", + target_platforms = [_WINDOWS], + extras = ["foo"], + )], + }) + +_tests.append(_test_add_extras_markers) + def _test_configure_errors(env): got, failures = _exposure( _module(configure = [_configure(hub_name = "missing")]), diff --git a/tests/pypi/pep508/deps_tests.bzl b/tests/pypi/pep508/deps_tests.bzl index be288fe308..adb2d02e49 100644 --- a/tests/pypi/pep508/deps_tests.bzl +++ b/tests/pypi/pep508/deps_tests.bzl @@ -376,6 +376,67 @@ def test_deps_with_extras_excludes_and_includes(env): _tests.append(test_deps_with_extras_excludes_and_includes) +def test_missing_deps(env): + """The deps that are not included are returned if the target platforms need them. + + Args: + env: the test env. + """ + requires_dist = [ + "bar", + "not-locked", + "tomli>=2; python_version < \"3.11\"", + "tomli-w[all]; python_version < \"3.11\" and extra == \"write\"", + "pywin32; sys_platform == \"win32\"", + "colorama; sys_platform == \"win32\" and extra == \"color\"", + "foo[write]; extra == \"all\"", + ] + linux = lambda v: {"python_version": v, "sys_platform": "linux"} + + got = deps("foo", requires_dist = requires_dist, include = ["bar"]) + env.expect.that_dict(got.missing).contains_exactly({}) + + got = deps("foo", requires_dist = requires_dist, envs = [linux("3.10")]) + env.expect.that_collection(got.deps).contains_at_least(["bar", "not_locked"]) + env.expect.that_dict(got.missing).contains_exactly({}) + + got = deps("foo", requires_dist = requires_dist, include = ["bar"], envs = [linux("3.12")]) + env.expect.that_dict(got.missing).contains_exactly({"not-locked": ["not-locked"]}) + + got = deps("foo", requires_dist = requires_dist, include = ["bar"], envs = [linux("3.12"), linux("3.10")]) + env.expect.that_dict(got.missing).contains_exactly({ + "not-locked": ["not-locked"], + "tomli": ["tomli; python_version < \"3.11\""], + }) + + # The extras that are included via the self references are checked. + got = deps( + "foo", + requires_dist = requires_dist, + include = ["bar"], + extras = ["all", "color"], + envs = [linux("3.12"), linux("3.10")], + ) + env.expect.that_dict(got.missing).contains_exactly({ + "not-locked": ["not-locked"], + "tomli": ["tomli; python_version < \"3.11\""], + "tomli-w": ["tomli-w[all]; python_version < \"3.11\" and extra == \"write\""], + }) + + got = deps( + "foo", + requires_dist = requires_dist, + include = ["bar", "not_locked"], + extras = ["color"], + envs = [{"python_version": "3.12", "sys_platform": "win32"}], + ) + env.expect.that_dict(got.missing).contains_exactly({ + "colorama": ["colorama; sys_platform == \"win32\" and extra == \"color\""], + "pywin32": ["pywin32; sys_platform == \"win32\""], + }) + +_tests.append(test_missing_deps) + def deps_test_suite(name): # buildifier: disable=function-docstring test_suite( name = name, diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index b1ff1bf698..4e26210c6a 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -43,6 +43,7 @@ def _test_legacy_aliases(env): "foo": ":pkg", "pkg": ":[]", "[]": "@repo//:[]", + "_extras": "@repo//:_extras", "srcs": "@repo//:srcs", "whl_file": "@repo//:whl_file", "whl": "@repo//:whl", @@ -56,6 +57,36 @@ def _test_legacy_aliases(env): _tests.append(_test_legacy_aliases) +def _test_extras_select_aliases(env): + got = {} + pkg_aliases( + name = "coverage", + actual = { + whl_config_setting(version = "3.10"): "coverage_310", + whl_config_setting(version = "3.12"): "coverage_312", + }, + native = struct( + alias = lambda *, name, actual, visibility = None, tags = None: got.update({name: (actual, visibility)}), + ), + select = lambda value, no_match_error = None: value, + ) + + # The hub does not know the extras that the package provides, so the + # packages that depend on an extra use the `_extras` target of the whl + # repository for the target platform. + env.expect.that_dict(got).contains_at_least({ + "_extras": ( + { + "//_config:is_cp310": "@coverage_310//:_extras", + "//_config:is_cp312": "@coverage_312//:_extras", + "//conditions:default": "_no_matching_repository", + }, + None, + ), + }) + +_tests.append(_test_extras_select_aliases) + def _test_config_setting_aliases(env): # Use this function as it is used in pip_repository got = {} @@ -221,6 +252,10 @@ def _test_group_aliases(env): "name": "foo", "actual": ":pkg", }, + { + "name": "_extras", + "actual": "@repo//:_extras", + }, { "name": "_pkg", "actual": "@repo//:[]", diff --git a/tests/pypi/whl_library/BUILD.bazel b/tests/pypi/whl_library/BUILD.bazel index 51e894693d..ad01e3bd54 100644 --- a/tests/pypi/whl_library/BUILD.bazel +++ b/tests/pypi/whl_library/BUILD.bazel @@ -40,7 +40,30 @@ py_test( srcs = ["whl_library_transitive_extras_test.py"], target_compatible_with = SUPPORTS_BZLMOD, deps = [ - # The METADATA of the consumer requires `pkg[optional]`. + # The METADATA of the consumer requires `pkg[optional]`, which is not + # requested in the hub, and `pkg[unknown]`, which `pkg` does not + # provide. "@whl_library_extras_consumer//:pkg", ], ) + +# `bazel query` does not know the configuration, so it needs all of the targets +# that the extras of the dependencies can resolve to. It fails with `no such +# target` if any of them is missing, e.g. for the `pkg[unknown]` dependency of +# the consumer, because `pkg` does not provide the `unknown` extra. +genquery( + name = "whl_library_transitive_extras_query", + expression = "deps(@whl_library_extras_consumer//:pkg)", + scope = ["@whl_library_extras_consumer//:pkg"], + target_compatible_with = SUPPORTS_BZLMOD, +) + +py_test( + name = "whl_library_query_test", + srcs = ["whl_library_query_test.py"], + data = [":whl_library_transitive_extras_query"], + env = { + "QUERY_OUTPUT": "$(rootpath :whl_library_transitive_extras_query)", + }, + target_compatible_with = SUPPORTS_BZLMOD, +) diff --git a/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA b/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA index 9a723c958a..e2cbed2ee5 100644 --- a/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA +++ b/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA @@ -1,3 +1,4 @@ Name: consumer Version: 1.0 Requires-Dist: pkg[optional] +Requires-Dist: pkg[unknown] diff --git a/tests/pypi/whl_library/testdata/packages.bzl b/tests/pypi/whl_library/testdata/packages.bzl index 627c3a84c2..6584dfeda3 100644 --- a/tests/pypi/whl_library/testdata/packages.bzl +++ b/tests/pypi/whl_library/testdata/packages.bzl @@ -1,12 +1,28 @@ """A list of packages that this logical testdata hub repo contains.""" # The values are the extras that are requested in the hub, the `pkg` package -# does not provide the `nope` extra. +# does not provide the `nope` extra. The `optional` extra of `pkg` is not +# requested in the hub, but the `consumer` package depends on it. packages = { "consumer": [], "optional_dep": [], "pkg": [ "nope", - "optional", ], } + +# The marker environments of the target platforms of the hub. The `pkg` +# package has dependencies that are not in `packages` and a warning is only +# printed for the ones that are needed on these platforms. +target_platform_envs = [ + { + "implementation_name": "cpython", + "os_name": "posix", + "platform_machine": "x86_64", + "platform_system": "Linux", + "python_full_version": python_full_version, + "python_version": python_full_version.rpartition(".")[0], + "sys_platform": "linux", + } + for python_full_version in ["3.10.20", "3.12.13"] +] diff --git a/tests/pypi/whl_library/testdata/pkg/pkg-1.0.dist-info/METADATA b/tests/pypi/whl_library/testdata/pkg/pkg-1.0.dist-info/METADATA index 712b44edbf..f9ce00ff1f 100644 --- a/tests/pypi/whl_library/testdata/pkg/pkg-1.0.dist-info/METADATA +++ b/tests/pypi/whl_library/testdata/pkg/pkg-1.0.dist-info/METADATA @@ -2,3 +2,9 @@ Name: pkg Version: 1.0 Requires-Dist: optional_dep; extra == "optional" Provides-Extra: optional +Requires-Dist: not_locked; python_version < "3.11" +Requires-Dist: not_locked_optional; sys_platform == "linux" and extra == "optional" +Requires-Dist: not_locked_windows; sys_platform == "win32" +Requires-Dist: not_locked_windows_optional; sys_platform == "win32" and extra == "optional" +Provides-Extra: unused +Requires-Dist: not_locked_unused; extra == "unused" diff --git a/tests/pypi/whl_library/whl_library_query_test.py b/tests/pypi/whl_library/whl_library_query_test.py new file mode 100644 index 0000000000..ed31f7c4de --- /dev/null +++ b/tests/pypi/whl_library/whl_library_query_test.py @@ -0,0 +1,30 @@ +import os +import unittest + + +class QueryTest(unittest.TestCase): + def test_deps_include_the_extras(self): + with open(os.environ["QUERY_OUTPUT"]) as f: + got = { + # The canonical names of the repositories depend on the + # extension that creates them. + line.strip().rpartition("whl_library_extras_")[2] + for line in f + if "whl_library_extras_" in line + } + + # The edges in the `METADATA` of the consumer, the first extra is + # provided by `pkg` and the second one is not. + self.assertIn("consumer//:_pkg[optional]", got) + self.assertIn("consumer//:_pkg[unknown]", got) + # Both of them resolve to the target that selects the extra. + self.assertIn("pkg//:_extras", got) + self.assertIn("pkg//:[optional]", got) + self.assertIn("pkg//:[]", got) + self.assertNotIn("pkg//:[unknown]", got) + # The dependency of the `optional` extra. + self.assertIn("optional_dep//:pkg", got) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl index 551ab7c767..c8e3006bd1 100644 --- a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl +++ b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl @@ -175,24 +175,25 @@ def _test_extras(env): rules = struct( py_library = lambda **kwargs: py_library_calls.append(kwargs), env_marker_setting = lambda **kwargs: env_marker_setting_calls.append(kwargs), + extra_dep = lambda **kwargs: None, ), ) # The `pkg` target has no extras because none were requested and there is # a target for each extra that the package provides. The `whl` target does - # not use the extras. Without the extras available in the hub, the - # dependencies use the targets with the extras directly. + # not use the extras. The dependencies on the extras of the other packages + # use the `extra_dep` targets. env.expect.that_collection([ (c["name"], c["deps"]) for c in py_library_calls ]).contains_exactly([ - ("pkg", ["srcs", "@pypi//bar:[socks]"]), - ("[all]", ["srcs", "@pypi//bar:[socks]", "@pypi//grpc:pkg"]), - ("[async]", ["srcs", "@pypi//bar:[socks]", "@pypi//uvloop:pkg"] + select({ + ("pkg", ["srcs", ":_bar[socks]"]), + ("[all]", ["srcs", ":_bar[socks]", "@pypi//grpc:pkg"]), + ("[async]", ["srcs", ":_bar[socks]", "@pypi//uvloop:pkg"] + select({ ":is_include_aiohttp_true": ["@pypi//aiohttp:pkg"], "//conditions:default": [], })), - ("[grpc]", ["srcs", "@pypi//bar:[socks]", "@pypi//grpc:pkg"]), + ("[grpc]", ["srcs", ":_bar[socks]", "@pypi//grpc:pkg"]), ]) env.expect.that_collection([ (c["name"], c["data"]) @@ -538,132 +539,610 @@ def _test_extras_in_group_legacy(env): _tests.append(_test_extras_in_group_legacy) -def _test_deps_with_extras(env): - filegroup_calls = [] - py_library_calls = [] - env_marker_setting_calls = [] - missing_calls = [] +def _targets(**kwargs): + """Create the targets and return the calls to the rules. + + Args: + **kwargs: the args for the `whl_library_deps_targets`. + + Returns: + A struct with the `py_library` deps and the `whl` data by the target + name and the calls to the other rules. + """ + calls = struct( + config_setting = [], + env_marker_setting = [], + extra_dep = [], + extra_select = [], + filegroup = [], + py_library = [], + warn = [], + ) whl_library_deps_targets( - name = "foo-0-py3-none-any.whl", + name = "{}-1.0-py3-none-any.whl".format(kwargs["metadata_name"]), + dep_template = kwargs.pop("dep_template", "@pypi//{name}:{target}"), + repo = None, + aliases = {}, + native = struct( + alias = lambda **kwargs: None, + config_setting = lambda **kwargs: calls.config_setting.append(kwargs), + filegroup = lambda **kwargs: calls.filegroup.append(kwargs), + ), + rules = struct( + env_marker_setting = lambda **kwargs: calls.env_marker_setting.append(kwargs), + extra_dep = lambda **kwargs: calls.extra_dep.append(kwargs), + extra_select = lambda **kwargs: calls.extra_select.append(kwargs), + py_library = lambda **kwargs: calls.py_library.append(kwargs), + warn = lambda msg: calls.warn.append(msg), + ), + **kwargs + ) + + return struct( + deps = {c["name"]: c["deps"] for c in calls.py_library}, + whl_data = {c["name"]: c["data"] for c in calls.filegroup}, + env_markers = {c["name"]: c["expression"] for c in calls.env_marker_setting}, + extra_deps = calls.extra_dep, + extra_selects = calls.extra_select, + config_settings = [c["name"] for c in calls.config_setting], + warnings = calls.warn, + ) + +def _test_deps_with_extras(env): + got = _targets( metadata_name = "foo", - dep_template = "@pypi//{name}:{target}", requires_dist = [ "bar[socks]", "baz[a,b]; sys_platform == 'linux'", "baz; sys_platform == 'darwin'", - "qux[missing]", + "qux[not-requested]", ], - # The values are the extras that are available in the hub + # The values are the extras that are requested in the hub, which are + # not needed for the extras of the dependencies. include = { "bar": ["socks"], "baz": ["a", "b"], "qux": [], }, - repo = None, - aliases = {}, - native = struct( - filegroup = lambda **kwargs: filegroup_calls.append(kwargs), - alias = lambda **kwargs: None, - ), - rules = struct( - py_library = lambda **kwargs: py_library_calls.append(kwargs), - env_marker_setting = lambda **kwargs: env_marker_setting_calls.append(kwargs), - missing_extra_error = lambda **kwargs: missing_calls.append(kwargs), - ), ) - # The dependencies use the targets with the extras, the missing extras use - # a target that fails the build with a helpful message. - env.expect.that_collection([ - (c["name"], c["deps"]) - for c in py_library_calls - ]).contains_exactly([ - ("pkg", ["srcs", "@pypi//bar:[socks]", ":_missing_qux[missing]"] + select({ + # The extras of the dependencies are resolved by the packages that provide + # them, so they are used via the `extra_dep` targets. + env.expect.that_dict(got.deps).contains_exactly({ + "pkg": ["srcs", ":_bar[socks]", ":_qux[not_requested]"] + select({ ":is_include_baz_2_true": ["@pypi//baz:pkg"], "//conditions:default": [], }) + select({ - ":is_include_baz[a]_true": ["@pypi//baz:[a]"], + ":is_include_baz[a]_true": [":_baz[a]"], "//conditions:default": [], }) + select({ - ":is_include_baz[b]_true": ["@pypi//baz:[b]"], + ":is_include_baz[b]_true": [":_baz[b]"], "//conditions:default": [], - })), - ]) - env.expect.that_collection(missing_calls).contains_exactly([ + }), + }) + env.expect.that_collection(got.extra_deps).contains_exactly([ { - "name": "_missing_qux[missing]", - "package_name": "qux", - "extra": "missing", - "required_by": "foo", + "name": "_" + dep, + "deps": ["@pypi//{}:_extras".format(name)], + "extra": extra, "visibility": ["//visibility:private"], - }, + } + for dep, name, extra in [ + ("bar[socks]", "bar", "socks"), + ("baz[a]", "baz", "a"), + ("baz[b]", "baz", "b"), + ("qux[not_requested]", "qux", "not_requested"), + ] ]) # buildifier: @unsorted-dict-items # The whl targets do not use the extras. - env.expect.that_collection([ - (c["name"], c["data"]) - for c in filegroup_calls - ]).contains_exactly([ - ("whl", ["@pypi//bar:whl", "@pypi//qux:whl"] + select({ + env.expect.that_dict(got.whl_data).contains_exactly({ + "whl": ["@pypi//bar:whl", "@pypi//qux:whl"] + select({ ":is_include_baz_true": ["@pypi//baz:whl"], "//conditions:default": [], - })), - ]) - env.expect.that_collection([ - (c["name"], c["expression"]) - for c in env_marker_setting_calls - ]).contains_exactly([ - ("include_baz", "(sys_platform == \"darwin\") or (sys_platform == \"linux\")"), - ("include_baz_2", "sys_platform == \"darwin\""), - ("include_baz[a]", "sys_platform == \"linux\""), - ("include_baz[b]", "sys_platform == \"linux\""), - ]) + }), + }) + env.expect.that_dict(got.env_markers).contains_exactly({ + "include_baz": "(sys_platform == \"darwin\") or (sys_platform == \"linux\")", + "include_baz[a]": "sys_platform == \"linux\"", + "include_baz[b]": "sys_platform == \"linux\"", + "include_baz_2": "sys_platform == \"darwin\"", + }) + + # The package does not provide any extras. + env.expect.that_collection(got.config_settings).contains_exactly([]) + env.expect.that_collection(got.extra_selects).contains_exactly([ + { + "name": "_extras", + "deps": [":[]"], + "distribution": "foo", + "extras": [], + "warnings": {}, + "visibility": ["//visibility:public"], + }, + ]) # buildifier: @unsorted-dict-items _tests.append(_test_deps_with_extras) -def _test_deps_with_extras_legacy(env): - py_library_calls = [] - missing_calls = [] +_COVERAGE_REQUIRES_DIST = [ + "tomli; python_full_version <= \"3.11.0a6\" and extra == \"toml\"", +] - whl_library_deps_targets( - name = "foo-0-py3-none-any.whl", +def _test_extra_that_is_not_requested_in_the_hub(env): + # The lock file has `coverage` without the `toml` extra, e.g. because the + # extras are stripped when compiling the requirements. + include = {"coverage": [], "pytest_cov": [], "tomli": []} + + got = _targets( + metadata_name = "pytest-cov", + requires_dist = ["coverage[toml]>=7.5", "pytest>=7"], + include = include, + ) + env.expect.that_dict(got.deps).contains_exactly({ + "pkg": ["srcs", ":_coverage[toml]"], + }) + env.expect.that_collection(got.extra_deps).contains_exactly([ + { + "name": "_coverage[toml]", + "deps": ["@pypi//coverage:_extras"], + "extra": "toml", + "visibility": ["//visibility:private"], + }, + ]) # buildifier: @unsorted-dict-items + + got = _targets( + metadata_name = "coverage", + requires_dist = _COVERAGE_REQUIRES_DIST, + provides_extra = ["toml"], + include = include, + tags = ["pypi_name=coverage", "pypi_version=7.6.1"], + ) + + # Only the target of the extra has `tomli` and only for the Python + # versions that need it, which is evaluated for each target platform. + env.expect.that_dict(got.deps).contains_exactly({ + "[toml]": ["srcs"] + select({ + ":is_include_tomli_true": ["@pypi//tomli:pkg"], + "//conditions:default": [], + }), + "pkg": ["srcs"], + }) + env.expect.that_dict(got.env_markers).contains_exactly({ + "include_tomli": "python_full_version <= \"3.11.0a6\"", + }) + env.expect.that_collection(got.config_settings).contains_exactly(["_extras_toml"]) + env.expect.that_collection(got.extra_selects).contains_exactly([ + { + "name": "_extras", + "deps": select({ + ":_extras_toml": [":[toml]"], + "//conditions:default": [":[]"], + }), + "distribution": "coverage 7.6.1", + "extras": ["toml"], + "warnings": {}, + "visibility": ["//visibility:public"], + }, + ]) # buildifier: @unsorted-dict-items + +_tests.append(_test_extra_that_is_not_requested_in_the_hub) + +def _test_deps_of_the_extra_that_are_not_in_the_hub(env): + # `tomli` is not locked, e.g. because the lock file is only for the Python + # versions that do not need it. Like the other dependencies that are not + # in the hub, it is dropped. + got = _targets( + metadata_name = "coverage", + requires_dist = _COVERAGE_REQUIRES_DIST, + provides_extra = ["toml"], + include = {"coverage": [], "pytest_cov": []}, + ) + env.expect.that_dict(got.deps).contains_exactly({ + "[toml]": ["srcs"], + "pkg": ["srcs"], + }) + env.expect.that_dict(got.env_markers).contains_exactly({}) + + # The same for the dependencies of the extra without any markers. + got = _targets( + metadata_name = "bar", + requires_dist = ["baz; extra == \"x\"", "qux"], + provides_extra = ["x"], + include = {"bar": [], "foo": [], "qux": []}, + ) + env.expect.that_dict(got.deps).contains_exactly({ + "[x]": ["srcs", "@pypi//qux:pkg"], + "pkg": ["srcs", "@pypi//qux:pkg"], + }) + + # Without any other dependencies the targets are aliases and the extra can + # still be selected. + got = _targets( + metadata_name = "bar", + provides_extra = ["x"], + include = {"bar": [], "foo": []}, + ) + env.expect.that_dict(got.deps).contains_exactly({}) + env.expect.that_collection([ + (c["name"], c["extras"]) + for c in got.extra_selects + ]).contains_exactly([("_extras", ["x"])]) + +_tests.append(_test_deps_of_the_extra_that_are_not_in_the_hub) + +def _envs(*python_versions, sys_platform = "linux"): + return [ + { + "os_name": "nt" if sys_platform == "win32" else "posix", + "python_full_version": v, + "python_version": v.rpartition(".")[0], + "sys_platform": sys_platform, + } + for v in python_versions + ] + +_MISSING_DEP_WARNING = ( + "WARNING: {} requires '{}', but '{}' is not in the requirements of the " + + "hub, so it is left out and it may be missing at runtime on some of the " + + "target platforms. Add '{}' to the lock file, e.g. by locking for all " + + "of the target platforms and Python versions of the hub." +) + +def _test_warn_about_missing_deps(env): + # The lock file is only for Python 3.12, but the hub is also used with + # Python 3.10, where `exceptiongroup` is needed. + kwargs = dict( metadata_name = "foo", - # Without the hub, e.g. with the vendored `requirements.bzl`. - dep_template = "@pip_{name}//:{target}", requires_dist = [ - "bar[socks]", - "baz[a]; sys_platform == 'linux'", - "qux[missing]", + "bar", + "exceptiongroup>=1.0; python_version < \"3.11\"", + "pywin32; sys_platform == \"win32\"", + "not-locked", ], - repo = None, - aliases = {}, - native = struct( - alias = lambda **kwargs: None, + include = {"bar": [], "foo": []}, + tags = ["pypi_name=foo", "pypi_version=1.0"], + ) + got = _targets(target_platform_envs = _envs("3.10.20", "3.12.13"), **kwargs) + env.expect.that_dict(got.deps).contains_exactly({ + "pkg": ["srcs", "@pypi//bar:pkg"], + }) + env.expect.that_collection(got.warnings).contains_exactly([ + _MISSING_DEP_WARNING.format( + "foo 1.0", + "exceptiongroup; python_version < \"3.11\"", + "exceptiongroup", + "exceptiongroup", ), - rules = struct( - py_library = lambda **kwargs: py_library_calls.append(kwargs), - env_marker_setting = lambda **kwargs: None, - missing_extra_error = lambda **kwargs: missing_calls.append(kwargs), + _MISSING_DEP_WARNING.format("foo 1.0", "not-locked", "not-locked", "not-locked"), + ]) + env.expect.that_collection([c["warnings"] for c in got.extra_selects]).contains_exactly([{}]) + + # Only the dependencies without markers are needed on Python 3.12. + got = _targets(target_platform_envs = _envs("3.12.13"), **kwargs) + env.expect.that_collection(got.warnings).contains_exactly([ + _MISSING_DEP_WARNING.format("foo 1.0", "not-locked", "not-locked", "not-locked"), + ]) + + # The target platforms are not known, e.g. in the WORKSPACE mode. + got = _targets(**kwargs) + env.expect.that_collection(got.warnings).contains_exactly([]) + +_tests.append(_test_warn_about_missing_deps) + +def _test_warn_about_missing_deps_of_extras(env): + tomli = "tomli; python_full_version <= \"3.11.0a6\" and extra == \"toml\"" + warning = _MISSING_DEP_WARNING.format("coverage 7.6.1", tomli, "tomli", "tomli") + kwargs = dict( + metadata_name = "coverage", + requires_dist = _COVERAGE_REQUIRES_DIST + [ + "not-locked", + "also-not-locked; extra == \"unused\"", + ], + provides_extra = ["toml", "unused"], + tags = ["pypi_name=coverage", "pypi_version=7.6.1"], + target_platform_envs = _envs("3.10.20", "3.12.13"), + ) + not_locked = _MISSING_DEP_WARNING.format("coverage 7.6.1", "not-locked", "not-locked", "not-locked") + + # The extra is not requested in the hub, so the warnings about its + # dependencies are only printed when another package depends on the + # extra. The dependencies without the extra are not repeated there. + got = _targets(include = {"coverage": [], "pytest_cov": []}, **kwargs) + env.expect.that_collection(got.warnings).contains_exactly([not_locked]) + env.expect.that_collection([c["warnings"] for c in got.extra_selects]).contains_exactly([{ + "toml": [warning], + "unused": [_MISSING_DEP_WARNING.format( + "coverage 7.6.1", + "also-not-locked; extra == \"unused\"", + "also-not-locked", + "also-not-locked", + )], + }]) + + # The extra is requested in the hub, so it is used. + got = _targets(include = {"coverage": ["toml"], "pytest_cov": []}, **kwargs) + env.expect.that_collection(got.warnings).contains_exactly([not_locked, warning]) + env.expect.that_collection([c["warnings"].keys() for c in got.extra_selects]).contains_exactly([["unused"]]) + + # The extra is requested without the hub and the `pkg` target includes it. + got = _targets( + include = {"coverage": None, "pytest_cov": None}, + extras = ["toml"], + dep_template = "@pip_{name}//:{target}", + **kwargs + ) + env.expect.that_collection(got.warnings).contains_exactly([not_locked, warning]) + env.expect.that_collection([c["warnings"].keys() for c in got.extra_selects]).contains_exactly([["unused"]]) + + # The dependency of the extra is not needed on any of the target platforms. + got = _targets( + include = {"coverage": ["toml"], "pytest_cov": []}, + **(kwargs | {"target_platform_envs": _envs("3.12.13")}) + ) + env.expect.that_collection(got.warnings).contains_exactly([not_locked]) + env.expect.that_collection([c["warnings"].keys() for c in got.extra_selects]).contains_exactly([["unused"]]) + +_tests.append(_test_warn_about_missing_deps_of_extras) + +def _test_dep_with_extra_that_is_not_provided(env): + include = {"bar": [], "foo": []} + got = _targets( + metadata_name = "foo", + requires_dist = ["bar[nope]"], + include = include, + ) + + # The package does not know the extras of its dependencies. + env.expect.that_dict(got.deps).contains_exactly({ + "pkg": ["srcs", ":_bar[nope]"], + }) + env.expect.that_collection([ + (c["deps"], c["extra"]) + for c in got.extra_deps + ]).contains_exactly([(["@pypi//bar:_extras"], "nope")]) + + # The `nope` extra is not in the `extras` of the `_extras` target, so it + # selects the package without extras and prints a warning. + got = _targets( + metadata_name = "bar", + requires_dist = ["baz; extra == \"x\""], + provides_extra = ["x"], + include = include, + ) + env.expect.that_collection([ + (c["deps"], c["extras"]) + for c in got.extra_selects + ]).contains_exactly([ + ( + select({ + ":_extras_x": [":[x]"], + "//conditions:default": [":[]"], + }), + ["x"], ), + ]) + +_tests.append(_test_dep_with_extra_that_is_not_provided) + +def _test_self_referencing_extras_are_selected(env): + got = _targets( + metadata_name = "foo", + requires_dist = [ + "foo[a,b]; extra == \"all\"", + "dep-a; extra == \"a\"", + "dep-b; extra == \"b\"", + ], + provides_extra = ["a", "all", "b"], + include = {"dep_a": [], "dep_b": [], "foo": []}, ) - # The dependencies use the `pkg` targets, which include the extras from - # the requirement lines of the dependencies, so the extras that are not - # provided or not requested do not result in missing targets. + # A dependency on `foo[all]` gets the dependencies of both extras and a + # dependency on `foo[a]` only gets the ones of `a`. + env.expect.that_dict(got.deps).contains_exactly({ + "[a]": ["srcs", "@pypi//dep_a:pkg"], + "[all]": ["srcs", "@pypi//dep_a:pkg", "@pypi//dep_b:pkg"], + "[b]": ["srcs", "@pypi//dep_b:pkg"], + "pkg": ["srcs"], + }) + env.expect.that_collection([c["deps"] for c in got.extra_selects]).contains_exactly([ + select({ + ":_extras_a": [":[a]"], + ":_extras_all": [":[all]"], + ":_extras_b": [":[b]"], + "//conditions:default": [":[]"], + }), + ]) + env.expect.that_collection(got.extra_deps).contains_exactly([]) + +_tests.append(_test_self_referencing_extras_are_selected) + +def _test_cycle_through_extras(env): + # `a[x]` requires `b[y]` and `b[y]` requires `a[x]`. Each package only + # reads its own `METADATA`, so nothing is resolved recursively here. + a = dict( + metadata_name = "a", + requires_dist = ["b[y]; extra == \"x\""], + provides_extra = ["x"], + include = {"a": [], "b": []}, + ) + b = dict( + metadata_name = "b", + requires_dist = ["a[x]; extra == \"y\""], + provides_extra = ["y"], + include = {"a": [], "b": []}, + ) + + # Without a group this is a dependency cycle between the targets like any + # other cycle between the packages. + env.expect.that_dict(_targets(**a).deps).contains_exactly({ + "[x]": ["srcs", ":_b[y]"], + "pkg": ["srcs"], + }) + env.expect.that_dict(_targets(**b).deps).contains_exactly({ + "[y]": ["srcs", ":_a[x]"], + "pkg": ["srcs"], + }) + + # With the extras as the members of a group the cycle is broken and the + # `_extras` targets select the group library of the hub. + group = dict(group_name = "cycle", group_deps = ["a[x]", "b[y]"]) + got = _targets(**(a | group)) + env.expect.that_dict(got.deps).contains_exactly({ + "[x]": ["srcs"], + "pkg": ["srcs"], + }) + env.expect.that_collection(got.extra_deps).contains_exactly([]) + env.expect.that_collection([c["deps"] for c in got.extra_selects]).contains_exactly([ + select({ + ":_extras_x": ["@pypi//a:[x]"], + "//conditions:default": [":[]"], + }), + ]) + got = _targets(**(b | group)) + env.expect.that_dict(got.deps).contains_exactly({ + "[y]": ["srcs"], + "pkg": ["srcs"], + }) + env.expect.that_collection(got.extra_deps).contains_exactly([]) + +_tests.append(_test_cycle_through_extras) + +def _test_different_extras_of_the_same_dep(env): + include = {"c": [], "foo_bar": [], "other_dep": [], "p": [], "q": [], "r": [], "tomli": []} + + # Each package only depends on the extra that it requires. + got = _targets(metadata_name = "p", requires_dist = ["c[toml]"], include = include) + env.expect.that_dict(got.deps).contains_exactly({"pkg": ["srcs", ":_c[toml]"]}) env.expect.that_collection([ - (c["name"], c["deps"]) - for c in py_library_calls + (c["name"], c["deps"], c["extra"]) + for c in got.extra_deps + ]).contains_exactly([("_c[toml]", ["@pypi//c:_extras"], "toml")]) + + got = _targets(metadata_name = "q", requires_dist = ["c[other]"], include = include) + env.expect.that_dict(got.deps).contains_exactly({"pkg": ["srcs", ":_c[other]"]}) + env.expect.that_collection([ + (c["name"], c["deps"], c["extra"]) + for c in got.extra_deps + ]).contains_exactly([("_c[other]", ["@pypi//c:_extras"], "other")]) + + got = _targets( + metadata_name = "c", + requires_dist = ["tomli; extra == \"toml\"", "other-dep; extra == \"other\""], + provides_extra = ["toml", "other"], + include = include, + ) + env.expect.that_dict(got.deps).contains_exactly({ + "[other]": ["srcs", "@pypi//other_dep:pkg"], + "[toml]": ["srcs", "@pypi//tomli:pkg"], + "pkg": ["srcs"], + }) + + # The names of the packages and the extras are normalized on both sides. + got = _targets(metadata_name = "r", requires_dist = ["Foo_Bar[TOML.Parser]"], include = include) + env.expect.that_dict(got.deps).contains_exactly({"pkg": ["srcs", ":_foo_bar[toml_parser]"]}) + env.expect.that_collection([ + (c["name"], c["deps"], c["extra"]) + for c in got.extra_deps + ]).contains_exactly([("_foo_bar[toml_parser]", ["@pypi//foo_bar:_extras"], "toml_parser")]) + + got = _targets( + metadata_name = "foo-bar", + requires_dist = ["tomli; extra == \"toml-parser\""], + provides_extra = ["toml-parser"], + include = include, + ) + env.expect.that_dict(got.deps).contains_exactly({ + "[toml_parser]": ["srcs", "@pypi//tomli:pkg"], + "pkg": ["srcs"], + }) + env.expect.that_collection(got.config_settings).contains_exactly(["_extras_toml_parser"]) + env.expect.that_collection([ + (c["deps"], c["extras"]) + for c in got.extra_selects ]).contains_exactly([ - ("pkg", ["srcs", "@pip_bar//:pkg", "@pip_qux//:pkg"] + select({ - ":is_include_baz_true": ["@pip_baz//:pkg"], + ( + select({ + ":_extras_toml_parser": [":[toml_parser]"], + "//conditions:default": [":[]"], + }), + ["toml_parser"], + ), + ]) + +_tests.append(_test_different_extras_of_the_same_dep) + +def _test_deps_with_extras_legacy(env): + # Without a hub, e.g. with a vendored `requirements.bzl`, the extras of + # the dependencies are resolved by the whl repositories in the same way + # and the config only has the names of the packages. + include = {"bar": None, "foo": None, "qux": None} + got = _targets( + metadata_name = "foo", + requires_dist = ["bar[socks]", "bar[nope]; sys_platform == \"linux\"", "qux"], + include = include, + dep_template = "@pip_{name}//:{target}", + ) + env.expect.that_dict(got.deps).contains_exactly({ + "pkg": ["srcs", ":_bar[socks]", "@pip_qux//:pkg"] + select({ + ":is_include_bar[nope]_true": [":_bar[nope]"], "//conditions:default": [], - })), + }), + }) + env.expect.that_collection([ + (c["name"], c["deps"], c["extra"]) + for c in got.extra_deps + ]).contains_exactly([ + ("_bar[nope]", ["@pip_bar//:_extras"], "nope"), + ("_bar[socks]", ["@pip_bar//:_extras"], "socks"), ]) - env.expect.that_collection(missing_calls).contains_exactly([]) + + # The repository of the dependency always has the target to select the + # extra, also for the extras that it does not provide and when it has no + # dependencies. + for requires_dist in [[], ["qux; extra == \"socks\""]]: + got = _targets( + metadata_name = "bar", + requires_dist = requires_dist, + provides_extra = ["socks"], + include = include, + dep_template = "@pip_{name}//:{target}", + ) + env.expect.that_collection([ + (c["name"], c["deps"], c["extras"]) + for c in got.extra_selects + ]).contains_exactly([( + "_extras", + select({ + ":_extras_socks": [":[socks]"], + "//conditions:default": [":[]"], + }), + ["socks"], + )]) _tests.append(_test_deps_with_extras_legacy) +def _test_extras_select_in_group_legacy(env): + # Without a hub the members of the group are aliases to the group library + # in the same repository. + got = _targets( + metadata_name = "foo", + requires_dist = ["bar; extra == \"a\"", "qux; extra == \"b\""], + provides_extra = ["a", "b"], + group_name = "group", + group_deps = ["foo[a]", "bar"], + dep_template = "@pypi_{name}//:{target}", + ) + env.expect.that_collection([c["deps"] for c in got.extra_selects]).contains_exactly([ + select({ + ":_extras_a": [":[a]"], + ":_extras_b": [":[b]"], + "//conditions:default": [":[]"], + }), + ]) + +_tests.append(_test_extras_select_in_group_legacy) + def whl_library_deps_targets_test_suite(name): """create the test suite.