diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 0d660f0d4d..45e529b5a6 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -16,6 +16,7 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("//python:py_binary.bzl", "py_binary") load("//python:py_library.bzl", "py_library") load("//python/private:distribution_filegroup.bzl", "distribution_filegroup") +load(":extra_dep.bzl", "requested_extra") package(default_visibility = ["//:__subpackages__"]) @@ -26,6 +27,13 @@ exports_files( visibility = ["//visibility:public"], ) +# The extra requested by the `extra_dep` targets in the whl repositories. +requested_extra( + name = "requested_extra", + build_setting_default = "", + visibility = ["//visibility:public"], +) + py_library( name = "rewriters_lib", srcs = [ @@ -110,6 +118,17 @@ bzl_library( deps = ["//python/private:bazel_tools"], ) +bzl_library( + name = "extra_dep", + srcs = ["extra_dep.bzl"], + deps = [ + "//python/private:py_cc_link_params_info", + "//python/private:py_info", + "//python/private:reexports", + "@bazel_skylib//rules:common_settings", + ], +) + bzl_library( name = "env_marker_setting", srcs = ["env_marker_setting.bzl"], diff --git a/python/private/pypi/extra_dep.bzl b/python/private/pypi/extra_dep.bzl new file mode 100644 index 0000000000..4837d6e5b1 --- /dev/null +++ b/python/private/pypi/extra_dep.bzl @@ -0,0 +1,207 @@ +"""Rules to depend on an extra of a PyPI package without knowing its extras. + +The dependencies of an extra, e.g. `bar[socks]`, are only known by reading the +`METADATA` of `bar`, which is only available in the whl repository of `bar`. +The hub cannot create the `[socks]` target for every extra that `bar` +provides without fetching the wheels, so the dependency on an extra is wired +as follows: + +1. The package that depends on `bar[socks]` creates an {obj}`extra_dep` + target, which depends on the `_extras` target of `bar` with the + {obj}`REQUESTED_EXTRA` setting set to `socks`. +2. The whl repository of `bar` creates an {obj}`extra_select` target named + `_extras`, which selects the `[socks]` target based on the value of the + setting and resets the setting to its default value, so that the targets of + the extras are analyzed in the same configuration as the rest of the build. + +The hub only needs to forward the `_extras` target of each package to the +right whl repository for the target platform. + +Both rules forward the providers of their only dependency, which is in the +`deps` attribute so that the aspects propagating along `deps` reach the +package. +""" + +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load("//python/private:py_cc_link_params_info.bzl", "PyCcLinkParamsInfo") +load("//python/private:py_info.bzl", "PyInfo") +load("//python/private:reexports.bzl", "BuiltinPyInfo") + +# The name of the target that selects the requested extra of a package. +EXTRAS_SELECT_TARGET = "_extras" + +# The setting that carries the requested extra from the {obj}`extra_dep` to the +# {obj}`extra_select` target. +REQUESTED_EXTRA = str(Label("//python/private/pypi:requested_extra")) + +# All of the providers of the targets in the whl and hub repositories, i.e. +# `py_library` and `missing_package_error`. `DefaultInfo` is always present. +_FORWARDED_PROVIDERS = [ + p + for p in [ + PyInfo, + BuiltinPyInfo, + PyCcLinkParamsInfo, + InstrumentedFilesInfo, + OutputGroupInfo, + ] + if p != None +] + +def _requested_extra_impl(ctx): + return [BuildSettingInfo(value = ctx.build_setting_value)] + +requested_extra = rule( + implementation = _requested_extra_impl, + build_setting = config.string(flag = False), + doc = """\ +The normalized name of the extra requested by an {obj}`extra_dep` target. + +This is only set on the edge between {obj}`extra_dep` and the +{obj}`extra_select` target and it is reset by the {obj}`extra_select` target. +""", +) + +def _set_requested_extra_impl(_settings, attr): + return {REQUESTED_EXTRA: attr.extra} + +_set_requested_extra = transition( + implementation = _set_requested_extra_impl, + inputs = [], + outputs = [REQUESTED_EXTRA], +) + +def _reset_requested_extra_impl(_settings, _attr): + return {REQUESTED_EXTRA: ""} + +_reset_requested_extra = transition( + implementation = _reset_requested_extra_impl, + inputs = [], + outputs = [REQUESTED_EXTRA], +) + +def _forward(ctx): + if len(ctx.attr.deps) != 1: + fail("Expected exactly one target in 'deps', got: {}".format(ctx.attr.deps)) + + target = ctx.attr.deps[0] + return [target[DefaultInfo]] + [ + target[provider] + for provider in _FORWARDED_PROVIDERS + if provider in target + ] + +def _extra_dep_impl(ctx): + return _forward(ctx) + +extra_dep = rule( + implementation = _extra_dep_impl, + doc = """\ +Depend on an extra of a PyPI package. + +The only target in `deps` is the {obj}`extra_select` target of the package, +which resolves the `extra` to the target that includes the package and the +dependencies of the extra. All of its providers are forwarded. +""", + attrs = { + "deps": attr.label_list( + mandatory = True, + cfg = _set_requested_extra, + doc = "The {obj}`extra_select` target of the package.", + ), + "extra": attr.string( + mandatory = True, + doc = "The normalized name of the extra.", + ), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + }, +) + +def _extra_select_impl(ctx): + extra = ctx.attr._requested_extra[BuildSettingInfo].value + if extra and extra not in ctx.attr.extras: + # This is how `pip` handles the unknown extras. + # buildifier: disable=print + print("WARNING: {} does not provide the extra '{}'".format( + ctx.attr.distribution, + extra, + )) + + return _forward(ctx) + +extra_select = rule( + implementation = _extra_select_impl, + doc = """\ +Forward the target of the extra that is requested by an {obj}`extra_dep`. + +The `deps` is expected to be a `select` on the {obj}`REQUESTED_EXTRA` +setting that results in a single target. The setting is reset on the edge to +`deps` so that the targets of the extras are analyzed in the same +configuration regardless of how they are reached. All of the providers of +the target are forwarded. +""", + attrs = { + "deps": attr.label_list( + mandatory = True, + cfg = _reset_requested_extra, + doc = "The target for the requested extra.", + ), + "distribution": attr.string( + doc = "The name and the version of the package for the warning about the unknown extras.", + ), + "extras": attr.string_list( + doc = "The normalized extras that have a target, a warning is printed if any other extra is requested.", + ), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + "_requested_extra": attr.label( + default = REQUESTED_EXTRA, + ), + }, +) + +def extra_select_targets( + *, + name, + targets, + default, + distribution = "", + native = native, + rules = None, + visibility = None): + """Create the {obj}`extra_select` target and the config settings for the extras. + + Args: + name: {type}`str` the name of the {obj}`extra_select` target. + targets: {type}`dict[str, str]` the normalized extras to the labels of + the targets that include the extra. + default: {type}`str` the label used when the requested extra is not + provided by the package, which follows `pip` that only warns about + unknown extras. + distribution: {type}`str` the name and the version of the package + used in the warning about the unknown extras. + native: the native struct for overriding in tests. + rules: a struct with the `extra_select` rule for overriding in tests. + visibility: {type}`list[str]` the visibility of the target. + """ + rules = rules or struct(extra_select = extra_select) + deps = {} + for extra, label in sorted(targets.items()): + setting = "{}_{}".format(name, extra) + native.config_setting( + name = setting, + flag_values = {REQUESTED_EXTRA: extra}, + visibility = ["//visibility:private"], + ) + deps[":" + setting] = [label] + + rules.extra_select( + name = name, + deps = select(deps | {"//conditions:default": [default]}) if deps else [default], + distribution = distribution, + extras = sorted(targets), + visibility = visibility, + ) diff --git a/tests/pypi/extra_dep/BUILD.bazel b/tests/pypi/extra_dep/BUILD.bazel new file mode 100644 index 0000000000..1236a6ad28 --- /dev/null +++ b/tests/pypi/extra_dep/BUILD.bazel @@ -0,0 +1,3 @@ +load(":extra_dep_tests.bzl", "extra_dep_test_suite") + +extra_dep_test_suite(name = "extra_dep_tests") diff --git a/tests/pypi/extra_dep/extra_dep_tests.bzl b/tests/pypi/extra_dep/extra_dep_tests.bzl new file mode 100644 index 0000000000..13adcadd3e --- /dev/null +++ b/tests/pypi/extra_dep/extra_dep_tests.bzl @@ -0,0 +1,446 @@ +"""Tests for the extra_dep and extra_select rules.""" + +load("@rules_testing//lib:analysis_test.bzl", "analysis_test") +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("@rules_testing//lib:util.bzl", rt_util = "util") +load("//python:py_info.bzl", "PyInfo") +load("//python:py_library.bzl", "py_library") +load("//python/private:common_labels.bzl", "labels") # buildifier: disable=bzl-visibility +load("//python/private:py_cc_link_params_info.bzl", "PyCcLinkParamsInfo") # buildifier: disable=bzl-visibility +load("//python/private/pypi:env_marker_setting.bzl", "env_marker_setting") # buildifier: disable=bzl-visibility +load("//python/private/pypi:extra_dep.bzl", "extra_dep", "extra_select_targets") # buildifier: disable=bzl-visibility + +_tests = [] +_basic_tests = [] + +def _py_library(name, **kwargs): + """A py_library with a generated source to check the output configuration.""" + rt_util.helper_target( + native.genrule, + name = name + "_gen", + outs = [name + ".py"], + cmd = "touch $@", + ) + rt_util.helper_target( + py_library, + name = name, + srcs = [name + ".py"], + **kwargs + ) + +def _pkg(name, extras = {}): + """Create the targets of a package like the whl repositories do. + + Args: + name: the name of the package, the `_extras` target is created + for the packages that depend on its extras. + extras: the extras to the dependencies of the extra. + """ + _py_library(name) + for extra, deps in extras.items(): + rt_util.helper_target( + py_library, + name = "{}[{}]".format(name, extra), + deps = [name] + deps, + ) + + extra_select_targets( + name = name + "_extras", + targets = {extra: "{}[{}]".format(name, extra) for extra in extras}, + default = name, + distribution = name + " 1.0", + visibility = ["//visibility:private"], + ) + +def _extra_dep(name, pkg, extra): + rt_util.helper_target( + extra_dep, + name = name, + deps = [pkg + "_extras"], + extra = extra, + ) + +def _sources(target): + return sorted([f.basename for f in target[PyInfo].transitive_sources.to_list()]) + +def _test_extra(name): + _py_library(name + "_dep_a") + _pkg(name + "_foo", {"a": [name + "_dep_a"]}) + _extra_dep(name + "_subject", name + "_foo", "a") + analysis_test( + name = name, + impl = _test_extra_impl, + target = name + "_subject", + ) + +def _test_extra_impl(env, target): + env.expect.that_collection(_sources(target)).contains_exactly([ + "test_extra_dep_a.py", + "test_extra_foo.py", + ]) + + # The setting is reset, so the targets of the extra are analyzed in the + # same configuration as the target that depends on the extra. + env.expect.that_collection({ + f.root.path: None + for f in target[PyInfo].transitive_sources.to_list() + }.keys()).contains_exactly([env.ctx.bin_dir.path]) + +_tests.append(_test_extra) + +def _test_unknown_extra(name): + _py_library(name + "_dep_a") + _pkg(name + "_foo", {"a": [name + "_dep_a"]}) + _extra_dep(name + "_subject", name + "_foo", "unknown") + analysis_test( + name = name, + impl = _test_unknown_extra_impl, + target = name + "_subject", + ) + +def _test_unknown_extra_impl(env, target): + # Like `pip`, the extras that the package does not provide are ignored and + # a warning is printed. + env.expect.that_collection(_sources(target)).contains_exactly([ + "test_unknown_extra_foo.py", + ]) + +_tests.append(_test_unknown_extra) + +def _test_no_extras(name): + _pkg(name + "_foo") + _extra_dep(name + "_subject", name + "_foo", "a") + analysis_test( + name = name, + impl = _test_no_extras_impl, + target = name + "_subject", + ) + +def _test_no_extras_impl(env, target): + env.expect.that_collection(_sources(target)).contains_exactly([ + "test_no_extras_foo.py", + ]) + +_tests.append(_test_no_extras) + +def _test_different_extras_of_the_same_package(name): + # Both `p` and `q` depend on `c`, but each of them gets only the + # dependencies of the extra that it requests. + _py_library(name + "_tomli") + _py_library(name + "_other_dep") + _pkg(name + "_c", { + "other": [name + "_other_dep"], + "toml": [name + "_tomli"], + }) + _extra_dep(name + "_p_c[toml]", name + "_c", "toml") + _extra_dep(name + "_q_c[other]", name + "_c", "other") + _py_library(name + "_p", deps = [name + "_p_c[toml]"]) + _py_library(name + "_q", deps = [name + "_q_c[other]"]) + analysis_test( + name = name, + impl = _test_different_extras_of_the_same_package_impl, + targets = { + "c": name + "_c", + "p": name + "_p", + "q": name + "_q", + }, + ) + +def _test_different_extras_of_the_same_package_impl(env, targets): + prefix = "test_different_extras_of_the_same_package_" + env.expect.that_collection(_sources(targets.p)).contains_exactly([ + prefix + "c.py", + prefix + "p.py", + prefix + "tomli.py", + ]) + env.expect.that_collection(_sources(targets.q)).contains_exactly([ + prefix + "c.py", + prefix + "other_dep.py", + prefix + "q.py", + ]) + + # The package without extras does not get the dependencies of the extras. + env.expect.that_collection(_sources(targets.c)).contains_exactly([ + prefix + "c.py", + ]) + +_tests.append(_test_different_extras_of_the_same_package) + +def _test_env_markers_of_the_extra(name): + # This is `coverage[toml]`, which requires + # `tomli; python_full_version <= "3.11.0a6" and extra == "toml"`, and + # `pytest-cov` that requires `coverage[toml]`. + _py_library(name + "_tomli") + rt_util.helper_target( + env_marker_setting, + name = name + "_include_tomli", + expression = "python_full_version <= \"3.11.0a6\"", + ) + _pkg(name + "_coverage", { + "toml": select({ + ":is_{}_include_tomli_true".format(name): [name + "_tomli"], + "//conditions:default": [], + }), + }) + _extra_dep(name + "_dep_coverage[toml]", name + "_coverage", "toml") + _py_library(name + "_pytest_cov", deps = [name + "_dep_coverage[toml]"]) + + for version, want in { + "3.10": ["coverage.py", "pytest_cov.py", "tomli.py"], + "3.12": ["coverage.py", "pytest_cov.py"], + }.items(): + analysis_test( + name = "{}_{}".format(name, version.replace(".", "_")), + impl = _test_env_markers_of_the_extra_impl, + target = name + "_pytest_cov", + config_settings = { + labels.PYTHON_VERSION: version, + }, + attr_values = { + "want": [name + "_" + w for w in want], + }, + attrs = { + "want": attr.string_list(), + }, + ) + + native.test_suite( + name = name, + tests = [ + name + "_3_10", + name + "_3_12", + ], + ) + +def _test_env_markers_of_the_extra_impl(env, target): + # The setting of the python version is kept when the setting of the extra + # is set and reset. + env.expect.that_collection(_sources(target)).contains_exactly(env.ctx.attr.want) + +_tests.append(_test_env_markers_of_the_extra) + +_ProvidersInfo = provider( + doc = "The providers of a target to compare them in the tests", + fields = ["target"], +) + +def _providers_impl(ctx): + return [_ProvidersInfo(target = ctx.attr.target)] + +_providers = rule( + implementation = _providers_impl, + attrs = {"target": attr.label()}, +) + +def _test_providers_are_forwarded(name): + _py_library(name + "_generated", data = [name + "_data.txt"]) + rt_util.helper_target( + py_library, + name = name + "_dep_a", + srcs = ["instrumented.py"], + deps = [name + "_generated"], + ) + rt_util.helper_target( + native.genrule, + name = name + "_data", + outs = [name + "_data.txt"], + cmd = "touch $@", + ) + _pkg(name + "_foo", {"a": [name + "_dep_a"]}) + _extra_dep(name + "_subject", name + "_foo", "a") + rt_util.helper_target( + _providers, + name = name + "_want", + target = name + "_foo[a]", + ) + analysis_test( + name = name, + impl = _test_providers_are_forwarded_impl, + targets = { + "subject": name + "_subject", + "want": name + "_want", + }, + config_settings = { + "//command_line_option:collect_code_coverage": True, + "//command_line_option:instrumentation_filter": "//tests/pypi/extra_dep[:/]", + }, + ) + +def _test_providers_are_forwarded_impl(env, targets): + got = targets.subject + want = targets.want[_ProvidersInfo].target + + for provider in [PyInfo, PyCcLinkParamsInfo, InstrumentedFilesInfo, OutputGroupInfo]: + env.expect.where(provider = str(provider)).that_bool(provider in got).equals(True) + + env.expect.that_bool(got[PyInfo] == want[PyInfo]).equals(True) + env.expect.that_bool(got[PyCcLinkParamsInfo] == want[PyCcLinkParamsInfo]).equals(True) + + # The output groups of the target. + env.expect.that_collection(dir(got[OutputGroupInfo])).contains_exactly(dir(want[OutputGroupInfo])) + for group in dir(want[OutputGroupInfo]): + env.expect.where(group = group).that_depset_of_files( + getattr(got[OutputGroupInfo], group), + ).contains_exactly([ + f.short_path + for f in getattr(want[OutputGroupInfo], group).to_list() + ]) + + # The files and the runfiles of the target. + env.expect.that_depset_of_files(got[DefaultInfo].files).contains_exactly([ + f.short_path + for f in want[DefaultInfo].files.to_list() + ]) + for runfiles in ["default_runfiles", "data_runfiles"]: + got_files = sorted([f.short_path for f in getattr(got[DefaultInfo], runfiles).files.to_list()]) + want_files = sorted([f.short_path for f in getattr(want[DefaultInfo], runfiles).files.to_list()]) + env.expect.where(runfiles = runfiles).that_collection(got_files).contains_exactly(want_files) + env.expect.where(runfiles = runfiles).that_collection(got_files).contains( + "tests/pypi/extra_dep/test_providers_are_forwarded_data.txt", + ) + + # The coverage is collected for the sources of the extra. + got_instrumented = sorted([f.short_path for f in got[InstrumentedFilesInfo].instrumented_files.to_list()]) + env.expect.that_collection(got_instrumented).contains_exactly([ + f.short_path + for f in want[InstrumentedFilesInfo].instrumented_files.to_list() + ]) + env.expect.that_collection(got_instrumented).contains( + "tests/pypi/extra_dep/instrumented.py", + ) + +_tests.append(_test_providers_are_forwarded) + +_VisitedInfo = provider( + doc = "The targets visited by an aspect that propagates along the `deps`", + fields = ["labels"], +) + +def _deps_aspect_impl(target, ctx): + return [_VisitedInfo(labels = depset( + [target.label.name], + transitive = [ + dep[_VisitedInfo].labels + for dep in getattr(ctx.rule.attr, "deps", []) + if _VisitedInfo in dep + ], + ))] + +_deps_aspect = aspect( + implementation = _deps_aspect_impl, + attr_aspects = ["deps"], +) + +def _with_deps_aspect_impl(ctx): + return [ctx.attr.target[_VisitedInfo]] + +_with_deps_aspect = rule( + implementation = _with_deps_aspect_impl, + attrs = {"target": attr.label(aspects = [_deps_aspect])}, +) + +def _test_aspects_propagate_along_deps(name): + _py_library(name + "_dep_a") + _pkg(name + "_foo", {"a": [name + "_dep_a"]}) + _extra_dep(name + "_dep_foo[a]", name + "_foo", "a") + _py_library(name + "_consumer", deps = [name + "_dep_foo[a]"]) + rt_util.helper_target( + _with_deps_aspect, + name = name + "_subject", + target = name + "_consumer", + ) + analysis_test( + name = name, + impl = _test_aspects_propagate_along_deps_impl, + target = name + "_subject", + ) + +def _test_aspects_propagate_along_deps_impl(env, target): + prefix = "test_aspects_propagate_along_deps_" + env.expect.that_collection(target[_VisitedInfo].labels.to_list()).contains_exactly([ + prefix + "consumer", + prefix + "dep_foo[a]", + prefix + "foo_extras", + prefix + "foo[a]", + prefix + "foo", + prefix + "dep_a", + ]) + +_tests.append(_test_aspects_propagate_along_deps) + +def _test_extra_select_targets(env): + config_settings = [] + extra_selects = [] + + extra_select_targets( + name = "_extras", + targets = {"a": ":[a]", "b": ":[b]"}, + default = ":[]", + distribution = "foo 1.0", + native = struct(config_setting = lambda **kwargs: config_settings.append(kwargs)), + rules = struct(extra_select = lambda **kwargs: extra_selects.append(kwargs)), + ) + + env.expect.that_collection(config_settings).contains_exactly([ + { + "name": "_extras_a", + "flag_values": {str(Label("//python/private/pypi:requested_extra")): "a"}, + "visibility": ["//visibility:private"], + }, + { + "name": "_extras_b", + "flag_values": {str(Label("//python/private/pypi:requested_extra")): "b"}, + "visibility": ["//visibility:private"], + }, + ]) # buildifier: @unsorted-dict-items + env.expect.that_collection(extra_selects).contains_exactly([ + { + "name": "_extras", + "deps": select({ + ":_extras_a": [":[a]"], + ":_extras_b": [":[b]"], + "//conditions:default": [":[]"], + }), + "distribution": "foo 1.0", + "extras": ["a", "b"], + "visibility": None, + }, + ]) # buildifier: @unsorted-dict-items + +_basic_tests.append(_test_extra_select_targets) + +def _test_extra_select_targets_no_extras(env): + extra_selects = [] + + extra_select_targets( + name = "_extras", + targets = {}, + default = ":[]", + native = struct(), + rules = struct(extra_select = lambda **kwargs: extra_selects.append(kwargs)), + visibility = ["//visibility:public"], + ) + + env.expect.that_collection(extra_selects).contains_exactly([ + { + "name": "_extras", + "deps": [":[]"], + "distribution": "", + "extras": [], + "visibility": ["//visibility:public"], + }, + ]) # buildifier: @unsorted-dict-items + +_basic_tests.append(_test_extra_select_targets_no_extras) + +def extra_dep_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite( + name = name, + tests = _tests, + basic_tests = _basic_tests, + ) diff --git a/tests/pypi/extra_dep/instrumented.py b/tests/pypi/extra_dep/instrumented.py new file mode 100644 index 0000000000..8667111328 --- /dev/null +++ b/tests/pypi/extra_dep/instrumented.py @@ -0,0 +1 @@ +"""A source file for the tests, only the source files are instrumented."""