From 3ca76891996ffff61b064d02d101285c21dc8d69 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Thu, 19 Nov 2026 20:05:32 +0900 Subject: [PATCH 1/2] feat(pypi): change the dependencies and tags of a package with pip.override The dependencies of a package come from its METADATA and cannot be changed without patching the wheel, e.g. to drop a dependency that is only needed for optional features or to add a missing one. Add 'ignore_dependencies', 'extra_dependencies' (PEP 508 requirements with markers) and 'tags' to pip.override(package = ...). They are passed to the whl repositories as attributes that only change the generated BUILD file, so they do not prevent sharing the extracted wheels. --- docs/conf.py | 1 + docs/pypi/index.md | 2 +- docs/pypi/{patch.md => override.md} | 29 +++++++- docs/pypi/rules-pycross-comparison.md | 8 +- news/pip-override-deps.added.md | 5 ++ python/private/pypi/BUILD.bazel | 7 +- python/private/pypi/extension.bzl | 29 +++++++- .../pypi/generate_whl_library_build_bazel.bzl | 20 ++++- python/private/pypi/hub_builder.bzl | 5 ++ python/private/pypi/patch_and_extract_whl.bzl | 3 + python/private/pypi/whl_deps_repo.bzl | 12 +++ python/private/pypi/whl_overrides.bzl | 73 ++++++++++++++++++- ...generate_whl_library_build_bazel_tests.bzl | 41 +++++++++++ .../whl_overrides/whl_overrides_tests.bzl | 63 +++++++++++++++- 14 files changed, 283 insertions(+), 15 deletions(-) rename docs/pypi/{patch.md => override.md} (53%) create mode 100644 news/pip-override-deps.added.md diff --git a/docs/conf.py b/docs/conf.py index bc5808235f..e259250983 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -96,6 +96,7 @@ "api/sphinxdocs/inventories/index": "/api/sphinxdocs/sphinxdocs/inventories/index.html", "pip.html": "pypi/index.html", "pypi-dependencies.html": "pypi/index.html", + "pypi/patch": "override.html", } exclude_patterns = ["_includes/*"] diff --git a/docs/pypi/index.md b/docs/pypi/index.md index f877532871..af3a225bad 100644 --- a/docs/pypi/index.md +++ b/docs/pypi/index.md @@ -28,7 +28,7 @@ use :maxdepth: 1 circular-dependencies -patch +override ../howto/multi-platform-pypi-deps rules-pycross-comparison ``` diff --git a/docs/pypi/patch.md b/docs/pypi/override.md similarity index 53% rename from docs/pypi/patch.md rename to docs/pypi/override.md index 57320702a4..e31352a211 100644 --- a/docs/pypi/patch.md +++ b/docs/pypi/override.md @@ -1,7 +1,12 @@ :::{default-domain} bzl ::: -# Patching wheels +(pypi-override)= +# Per-package overrides + +The {obj}`pip.override` tag class changes how the packages of a hub are set up. + +## Patching wheels Sometimes the wheels have to be patched to: * Workaround the lack of a standard `site-packages` layout ({gh-issue}`2156`). @@ -37,3 +42,25 @@ pip.override( The patches are applied with `patch_strip = 0` by default, so use `patch_strip = 1` for patches created with `git diff`. ::: + +## Changing the dependencies and tags + +The dependencies of a package come from the `Requires-Dist` lines of its +`METADATA`. They can be changed with {attr}`pip.override.ignore_dependencies`, +e.g. to drop an optional dependency that is listed as a required one, and +{attr}`pip.override.extra_dependencies`, e.g. to add a missing dependency. The +extra dependencies are PEP 508 requirements, so they can have extras and +markers. Both need to refer to packages in the same hub. + +{attr}`pip.override.tags` adds tags to the targets of the package, e.g. to +avoid building it remotely. + +```starlark +pip.override( + package = "foo", + hub_name = "my_deps", + ignore_dependencies = ["bar"], + extra_dependencies = ["baz[socks]; sys_platform == 'linux'"], + tags = ["no-remote"], +) +``` diff --git a/docs/pypi/rules-pycross-comparison.md b/docs/pypi/rules-pycross-comparison.md index 54a30406b9..95fa54ce78 100644 --- a/docs/pypi/rules-pycross-comparison.md +++ b/docs/pypi/rules-pycross-comparison.md @@ -111,12 +111,12 @@ The `BUILD.bazel` files keep using `@uv_deps//ipython`, | `build_mode` | `"never"`: {attr}`pip.parse.download_only` for the whole hub; `"auto"` is the default | partial, per-package: planned | | `build_target` | none | planned: replace a package with your own target | | `build_backend`, `extra_build_tools`, `build_tools_repo`, `site_hooks`, `pre_build_patches` | none | planned: an extension point for an sdist builder, such as rules_pycross | -| `ignore_dependencies` | none | planned | -| `extra_dependencies` | {attr}`pip.whl_mods.data` (runfiles only) | partial, planned | +| `ignore_dependencies` | {attr}`pip.override.ignore_dependencies` | have | +| `extra_dependencies` | {attr}`pip.override.extra_dependencies`, which also accepts markers | have | | `install_exclude_globs` | {attr}`pip.whl_mods.srcs_exclude_glob`, {attr}`pip.whl_mods.data_exclude_glob`, {attr}`pip.parse.pip_data_exclude` | have; the globs start with `site-packages/` | | `post_install_patches` | {attr}`pip.override.package` with {attr}`pip.override.patches` | have; the wheel is patched before it is extracted | | `site_paths`, `bin_paths`, `data_paths`, `include_paths` | a fixed `site-packages`, `bin`, `data`, `include` layout | not needed | -| `wheel_library_tags` | none | planned | +| `wheel_library_tags` | {attr}`pip.override.tags` | have | | `precompile` | the {flag}`--precompile` flag for all packages | partial | :::{note} @@ -207,7 +207,7 @@ module. Do not depend on the same distribution from both in one binary. | dependency groups, `transitive` | {obj}`pip.configure`, requires {gh-pr}`4395` | have | | `;testonly` | none, see {gh-pr}`4428` | planned | | extras targets | requires {gh-pr}`4490` and {gh-pr}`4523` | have | -| `ignore_dependencies`, `extra_dependencies`, `wheel_library_tags` | none | planned | +| `ignore_dependencies`, `extra_dependencies`, `wheel_library_tags` | {obj}`pip.override` | have | | per-package `precompile` | global flags | partial | | `post_install_patches` | {attr}`pip.override.package` | have | | `install_exclude_globs` | {obj}`pip.whl_mods`, {attr}`pip.parse.pip_data_exclude` | have | diff --git a/news/pip-override-deps.added.md b/news/pip-override-deps.added.md new file mode 100644 index 0000000000..ee035f071b --- /dev/null +++ b/news/pip-override-deps.added.md @@ -0,0 +1,5 @@ +(pypi) The {obj}`pip.override` tag class now accepts +{attr}`pip.override.ignore_dependencies`, {attr}`pip.override.extra_dependencies` +and {attr}`pip.override.tags` to change the dependencies and the tags of the +targets of a {attr}`pip.override.package` (bzlmod only). The `patch` docs page +is now `override`. diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 7f67eb9fbc..f61bb8d6f8 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -202,7 +202,11 @@ bzl_library( bzl_library( name = "generate_whl_library_build_bazel", srcs = ["generate_whl_library_build_bazel.bzl"], - deps = ["//python/private:text_util"], + deps = [ + ":pep508_requirement", + "//python/private:normalize_name", + "//python/private:text_util", + ], ) bzl_library( @@ -579,6 +583,7 @@ bzl_library( srcs = ["whl_overrides.bzl"], deps = [ ":parse_whl_name", + ":pep508_requirement", "//python/private:normalize_name", ], ) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index d064d71aa5..41f64676ee 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -33,7 +33,7 @@ load(":pypi_cache.bzl", "pypi_cache") load(":simpleapi_download.bzl", "simpleapi_download") load(":unified_hub_repo.bzl", "unified_hub_repo") load(":whl_library.bzl", "whl_library") -load(":whl_overrides.bzl", "parse_whl_overrides", "unmatched_whl_overrides", "whl_overrides_for_hub") +load(":whl_overrides.bzl", "package_settings_for_hub", "parse_whl_overrides", "unmatched_whl_overrides", "whl_overrides_for_hub") def _whl_mods_impl(whl_mods_dict): """Implementation of the pip.whl_mods tag class. @@ -278,6 +278,7 @@ You cannot use both the additive_build_content and additive_build_content_file a module_name = mod.name, config = config, whl_overrides = whl_overrides_for_hub(whl_overrides, pip_attr.hub_name), + package_settings = package_settings_for_hub(whl_overrides, pip_attr.hub_name), simpleapi_download_fn = simpleapi_download, simpleapi_cache = simpleapi_cache, # TODO @aignas 2025-09-06: do not use kwargs @@ -1051,6 +1052,15 @@ cannot have a child module that uses the same `hub_name`. # 'archive_override', 'git_override' bzlmod functions. _override_tag = tag_class( attrs = { + "extra_dependencies": attr.string_list( + doc = """\ +Extra dependencies of the {attr}`pip.override.package` as PEP 508 requirements, e.g. +`["bar", "baz[extra]; sys_platform == 'linux'"]`. They are added to the +`Requires-Dist` of the package and need to be in the same hub. + +:::{versionadded} VERSION_NEXT_FEATURE +:::""", + ), "file": attr.string( doc = """\ The Python distribution file name which needs to be patched. This will be @@ -1069,6 +1079,15 @@ wheels of a package. The hub that the {attr}`pip.override.package` override applies to. If empty, it applies to all of the hubs. +:::{versionadded} VERSION_NEXT_FEATURE +:::""", + ), + "ignore_dependencies": attr.string_list( + doc = """\ +The names of the dependencies of the {attr}`pip.override.package` to ignore, e.g. to drop a +dependency that is not needed or to break a dependency cycle. They need to be +in the same hub. + :::{versionadded} VERSION_NEXT_FEATURE :::""", ), @@ -1096,7 +1115,13 @@ patches.""", doc = """\ A list of patches to apply to the repository *after* 'whl_library' is extracted and BUILD.bazel file is generated.""", - mandatory = True, + ), + "tags": attr.string_list( + doc = """\ +Extra tags to set on the targets of the {attr}`pip.override.package`, e.g. `["no-remote"]`. + +:::{versionadded} VERSION_NEXT_FEATURE +:::""", ), }, doc = """\ diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index bb8faa71dc..55759c9bcc 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -14,7 +14,9 @@ """Generate the BUILD.bazel contents for a repo defined by a whl_library.""" +load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:text_util.bzl", "render") +load(":pep508_requirement.bzl", "requirement") _RENDER = { "copy_executables": render.dict, @@ -59,6 +61,9 @@ def generate_whl_library_build_bazel( config_load, purl = None, requires_dist = [], + ignore_dependencies = [], + extra_dependencies = [], + extra_tags = [], **kwargs): """Generate a BUILD file for an unzipped Wheel @@ -68,6 +73,10 @@ def generate_whl_library_build_bazel( config_load: {type}`str` The location from where to load the config. purl: The purl. requires_dist: {type}`list[str]` The list of dependencies from the METADATA file. + ignore_dependencies: {type}`list[str]` The names of the dependencies + to remove from `requires_dist`. + extra_dependencies: {type}`list[str]` The extra `Requires-Dist` lines. + extra_tags: {type}`list[str]` The extra tags of the targets. **kwargs: Extra args serialized to be passed to the {obj}`whl_library_targets`. @@ -87,9 +96,18 @@ def generate_whl_library_build_bazel( tags = [ "pypi_name={}".format(kwargs.get("metadata_name")), "pypi_version={}".format(metadata_version), - ] + ] + extra_tags kwargs["tags"] = tags + if ignore_dependencies: + ignored = {normalize_name(name): None for name in ignore_dependencies} + requires_dist = [ + req + for req in requires_dist + if requirement(req).name_ not in ignored + ] + requires_dist = requires_dist + extra_dependencies + if not requires_dist: # no deps, we can leave the extra loads out pass diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 408d4727d7..3279b9f3a0 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -29,6 +29,7 @@ def hub_builder( module_name, config, whl_overrides, + package_settings = {}, minor_mapping, available_interpreters, simpleapi_download_fn, @@ -41,6 +42,8 @@ def hub_builder( module_name: {type}`str`, the module name that has created the hub. config: The platform configuration. whl_overrides: {type}`dict[str, struct]` - per-wheel overrides. + package_settings: {type}`dict[str, dict]` - extra `whl_library` args + for each package. minor_mapping: {type}`dict[str, str]` the mapping between minor and full versions. available_interpreters: {type}`dict[str, Label]` The dictionary of available interpreters that have been registered using the `python` bzlmod extension. @@ -98,6 +101,7 @@ def hub_builder( # Instance constants passed in by callers _config = config, _whl_overrides = whl_overrides, + _package_settings = package_settings, _logger = logger, _minor_mapping = minor_mapping, _available_interpreters = available_interpreters, @@ -770,6 +774,7 @@ def _whl_library_args(self, *, whl, whl_modifications): p: json.encode(args) for p, args in self._whl_overrides.get(whl.name, {}).items() }, + **self._package_settings.get(whl.name, {}) ) whl_library_args.update({k: v for k, v in maybe_args.items() if v}) diff --git a/python/private/pypi/patch_and_extract_whl.bzl b/python/private/pypi/patch_and_extract_whl.bzl index cfacb4adf4..965bb441de 100644 --- a/python/private/pypi/patch_and_extract_whl.bzl +++ b/python/private/pypi/patch_and_extract_whl.bzl @@ -133,6 +133,9 @@ def patch_and_extract_whl(rctx, *, whl_path, logger, sdist_filename = None, whl_ data_exclude = rctx.attr.pip_data_exclude, group_deps = rctx.attr.group_deps, group_name = rctx.attr.group_name, + ignore_dependencies = rctx.attr.ignore_dependencies, + extra_dependencies = rctx.attr.extra_dependencies, + extra_tags = rctx.attr.extra_tags, namespace_package_files = namespace_package_files, provides_extra = metadata.provides_extra, entry_points = entry_points, diff --git a/python/private/pypi/whl_deps_repo.bzl b/python/private/pypi/whl_deps_repo.bzl index 73a0051f11..455cc852ee 100644 --- a/python/private/pypi/whl_deps_repo.bzl +++ b/python/private/pypi/whl_deps_repo.bzl @@ -18,6 +18,12 @@ For example if your whl depends on `numpy` and your Python package repo is named `pip` so that you would normally do `@pip//numpy`, then this should be: `@pip//{name}`. """, ), + "extra_dependencies": attr.string_list( + doc = "Extra `Requires-Dist` lines (PEP 508 requirements) to add to the dependencies.", + ), + "extra_tags": attr.string_list( + doc = "Extra tags to set on the generated targets.", + ), "group_deps": attr.string_list( doc = "List of dependencies to skip in order to break the cycles within a dependency group.", default = [], @@ -25,6 +31,9 @@ For example if your whl depends on `numpy` and your Python package repo is named "group_name": attr.string( doc = "Name of the group, if any.", ), + "ignore_dependencies": attr.string_list( + doc = "The names of the dependencies to remove from the `Requires-Dist` lines.", + ), } def _whl_deps_repo_impl(rctx): @@ -54,6 +63,9 @@ def _whl_deps_repo_impl(rctx): requires_dist = metadata.requires_dist, group_deps = rctx.attr.group_deps, group_name = rctx.attr.group_name, + ignore_dependencies = rctx.attr.ignore_dependencies, + extra_dependencies = rctx.attr.extra_dependencies, + extra_tags = rctx.attr.extra_tags, repo = rctx.attr.repo or ( str(rctx.attr.metadata_file) if rctx.attr.metadata_file else None ), diff --git a/python/private/pypi/whl_overrides.bzl b/python/private/pypi/whl_overrides.bzl index 83e31fdcea..78d4584af8 100644 --- a/python/private/pypi/whl_overrides.bzl +++ b/python/private/pypi/whl_overrides.bzl @@ -2,6 +2,7 @@ load("//python/private:normalize_name.bzl", "normalize_name") load(":parse_whl_name.bzl", "parse_whl_name") +load(":pep508_requirement.bzl", "requirement") def parse_whl_overrides(modules, *, _fail = fail): """Collect the `pip.override` tags of the root module. @@ -17,6 +18,7 @@ def parse_whl_overrides(modules, *, _fail = fail): """ files = {} packages = {} + settings = {} seen_files = {} seen_packages = {} for module in modules: @@ -31,6 +33,14 @@ def parse_whl_overrides(modules, *, _fail = fail): continue if attr.file: + if attr.ignore_dependencies or attr.extra_dependencies or attr.tags: + _fail("pip.override(file = '{}') only supports 'patches', use 'package' for the other settings".format(attr.file)) + continue + + if not attr.patches: + _fail("pip.override(file = '{}') needs 'patches'".format(attr.file)) + continue + if attr.hub_name: _fail("pip.override(file = '{}') applies to all hubs and does not support 'hub_name'".format(attr.file)) continue @@ -71,6 +81,20 @@ def parse_whl_overrides(modules, *, _fail = fail): continue seen_packages[key] = None + if attr.ignore_dependencies or attr.extra_dependencies or attr.tags: + if version: + _fail("pip.override(package = '{}') cannot set 'ignore_dependencies', 'extra_dependencies' or 'tags' for a single version".format(attr.package)) + continue + + settings.setdefault(attr.hub_name, {})[whl_name] = struct( + ignore_dependencies = attr.ignore_dependencies, + extra_dependencies = attr.extra_dependencies, + extra_tags = attr.tags, + ) + elif not attr.patches: + _fail("pip.override(package = '{}') does not set anything".format(attr.package)) + continue + patches = packages.setdefault(attr.hub_name, {}).setdefault(whl_name, {}) for patch in attr.patches: if patch in patches and patches[patch].patch_strip != attr.patch_strip: @@ -84,8 +108,33 @@ def parse_whl_overrides(modules, *, _fail = fail): return struct( files = files, packages = packages, + settings = settings, ) +def package_settings_for_hub(overrides, hub_name): + """Get the settings of the packages in a hub. + + The settings for the hub replace the settings for all hubs field by field. + + Args: + overrides: {type}`struct` the result of {func}`parse_whl_overrides`. + hub_name: {type}`str` the name of the hub. + + Returns: + {type}`dict[str, dict]` the `whl_library` args for each package name. + """ + all_hubs = overrides.settings.get("", {}) + this_hub = overrides.settings.get(hub_name, {}) + ret = {} + for whl_name in all_hubs.keys() + this_hub.keys(): + if whl_name in ret: + continue + ret[whl_name] = { + field: getattr(this_hub.get(whl_name), field, None) or getattr(all_hubs.get(whl_name), field, None) or [] + for field in ["ignore_dependencies", "extra_dependencies", "extra_tags"] + } + return ret + def whl_overrides_for_hub(overrides, hub_name, *, _fail = fail): """Get the patches of the packages in a hub. @@ -138,7 +187,11 @@ def unmatched_whl_overrides(overrides, hub_packages): {type}`list[str]` the error messages, empty if all overrides match. """ errors = [] - for hub_name, packages in sorted(overrides.packages.items()): + by_hub = {} + for hub_name, packages in overrides.packages.items() + overrides.settings.items(): + by_hub.setdefault(hub_name, {}).update({name: None for name in packages}) + + for hub_name, packages in sorted(by_hub.items()): if hub_name and hub_name not in hub_packages: errors.append("pip.override(hub_name = '{}') refers to a hub that does not exist".format(hub_name)) continue @@ -151,4 +204,22 @@ def unmatched_whl_overrides(overrides, hub_packages): " in hub '{}'".format(hub_name) if hub_name else "", )) + for hub_name, packages in sorted(overrides.settings.items()): + hubs = [hub_name] if hub_name else hub_packages.keys() + for whl_name, setting in sorted(packages.items()): + for field, values in [ + ("ignore_dependencies", setting.ignore_dependencies), + ("extra_dependencies", setting.extra_dependencies), + ]: + for value in values: + dep = normalize_name(requirement(value).name) + for h in hubs: + if h in hub_packages and whl_name in hub_packages[h] and dep not in hub_packages[h]: + errors.append("pip.override(package = '{}').{} refers to '{}', which is not in hub '{}'".format( + whl_name, + field, + value, + h, + )) + return errors diff --git a/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl index d3a390d9bb..771a9b57e9 100644 --- a/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl +++ b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl @@ -269,6 +269,47 @@ whl_library_deps_targets( _tests.append(_test_deps_targets) +def _test_deps_overrides(env): + want = """\ +load("@package_metadata//rules:package_metadata.bzl", "package_metadata") +load("@rules_python//python/private/pypi:whl_library_deps_targets.bzl", "whl_library_deps_targets") + +package(default_visibility = ["//visibility:public"]) + +whl_library_deps_targets( + metadata_name = "foo", + name = "foo.whl", + repo = "@some_repo", + requires_dist = [ + "baz", + "qux; sys_platform == 'linux'", + ], + tags = [ + "pypi_name=foo", + "pypi_version=0", + "no-remote", + ], +) +""" + actual = generate_whl_library_build_bazel( + metadata_version = "0", + metadata_name = "foo", + name = "foo.whl", + repo = "@some_repo", + config_load = "", + requires_dist = [ + "Bar.Baz>=1", + "bar_baz[extra]; extra == 'x'", + "baz", + ], + ignore_dependencies = ["bar-baz"], + extra_dependencies = ["qux; sys_platform == 'linux'"], + extra_tags = ["no-remote"], + ) + env.expect.that_str(actual.replace("@@", "@")).equals(want) + +_tests.append(_test_deps_overrides) + def generate_whl_library_build_bazel_test_suite(name): """Create the test suite. diff --git a/tests/pypi/whl_overrides/whl_overrides_tests.bzl b/tests/pypi/whl_overrides/whl_overrides_tests.bzl index 092be75aa4..6196b63950 100644 --- a/tests/pypi/whl_overrides/whl_overrides_tests.bzl +++ b/tests/pypi/whl_overrides/whl_overrides_tests.bzl @@ -1,17 +1,29 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private/pypi:whl_overrides.bzl", "parse_whl_overrides", "unmatched_whl_overrides", "whl_overrides_for_hub") # buildifier: disable=bzl-visibility +load("//python/private/pypi:whl_overrides.bzl", "package_settings_for_hub", "parse_whl_overrides", "unmatched_whl_overrides", "whl_overrides_for_hub") # buildifier: disable=bzl-visibility _tests = [] -def _override(*, file = "", package = "", hub_name = "", patches = [], patch_strip = 0): +def _override( + *, + file = "", + package = "", + hub_name = "", + patches = [], + patch_strip = 0, + ignore_dependencies = [], + extra_dependencies = [], + tags = []): return struct( + extra_dependencies = extra_dependencies, file = file, hub_name = hub_name, + ignore_dependencies = ignore_dependencies, package = package, patch_strip = patch_strip, patches = patches, + tags = tags, ) def _mod(*overrides, is_root = True): @@ -79,8 +91,12 @@ def _test_errors(env): for overrides, want in [ ([_override(patches = ["//:a.patch"])], "pip.override needs exactly one of 'file' or 'package'"), ([_override(file = "foo-1.0-py3-none-any.whl", package = "foo")], "pip.override needs exactly one of 'file' or 'package'"), - ([_override(file = "foo-1.0-py3-none-any.whl", hub_name = "pypi")], "does not support 'hub_name'"), - ([_override(file = "foo-1.0.tar.gz")], "Only whl overrides are supported"), + ([_override(file = "foo-1.0-py3-none-any.whl", hub_name = "pypi", patches = ["//:a.patch"])], "does not support 'hub_name'"), + ([_override(file = "foo-1.0.tar.gz", patches = ["//:a.patch"])], "Only whl overrides are supported"), + ([_override(file = "foo-1.0-py3-none-any.whl")], "needs 'patches'"), + ([_override(file = "foo-1.0-py3-none-any.whl", patches = ["//:a.patch"], tags = ["a"])], "only supports 'patches'"), + ([_override(package = "foo")], "does not set anything"), + ([_override(package = "foo@1.0", tags = ["a"])], "cannot set 'ignore_dependencies', 'extra_dependencies' or 'tags' for a single version"), ([_override(package = "foo@", patches = ["//:a.patch"])], "needs to be a package name or '@'"), ([_override(package = "foo==1.0", patches = ["//:a.patch"])], "needs to be a package name or '@'"), ( @@ -112,6 +128,45 @@ def _test_errors(env): _tests.append(_test_errors) +def _test_package_settings(env): + got, failures = _parse(_mod( + _override(package = "foo", ignore_dependencies = ["Bar"], tags = ["all"]), + _override(package = "foo", hub_name = "pypi", extra_dependencies = ["baz[x]; sys_platform == 'linux'"]), + _override(package = "qux", hub_name = "other", ignore_dependencies = ["missing"]), + )) + env.expect.that_collection(failures).contains_exactly([]) + + # The hub settings replace the settings for all hubs field by field + env.expect.that_dict(package_settings_for_hub(got, "pypi")).contains_exactly({ + "foo": { + "extra_dependencies": ["baz[x]; sys_platform == 'linux'"], + "extra_tags": ["all"], + "ignore_dependencies": ["Bar"], + }, + }) + env.expect.that_dict(package_settings_for_hub(got, "other")).contains_exactly({ + "foo": { + "extra_dependencies": [], + "extra_tags": ["all"], + "ignore_dependencies": ["Bar"], + }, + "qux": { + "extra_dependencies": [], + "extra_tags": [], + "ignore_dependencies": ["missing"], + }, + }) + + env.expect.that_collection(unmatched_whl_overrides(got, { + "other": ["foo", "bar", "qux"], + "pypi": ["foo", "bar"], + })).contains_exactly([ + "pip.override(package = 'foo').extra_dependencies refers to 'baz[x]; sys_platform == 'linux'', which is not in hub 'pypi'", + "pip.override(package = 'qux').ignore_dependencies refers to 'missing', which is not in hub 'other'", + ]) + +_tests.append(_test_package_settings) + def whl_overrides_test_suite(name): """Create the test suite. From e1a49200f6c86aba80b72f8bf133dd6e75cf019a Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Thu, 19 Nov 2026 20:09:30 +0900 Subject: [PATCH 2/2] fix(pypi): validate the ignored dependency names and add the hub settings Only accept package names in ignore_dependencies, so that e.g. 'bar[x]' does not silently ignore nothing, and add the settings for a hub to the settings for all hubs, like the patches. --- docs/pypi/override.md | 4 +-- python/private/pypi/extension.bzl | 13 +++++---- python/private/pypi/whl_overrides.bzl | 29 +++++++++++-------- .../whl_overrides/whl_overrides_tests.bzl | 4 ++- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/docs/pypi/override.md b/docs/pypi/override.md index e31352a211..fe5930c890 100644 --- a/docs/pypi/override.md +++ b/docs/pypi/override.md @@ -52,8 +52,8 @@ e.g. to drop an optional dependency that is listed as a required one, and extra dependencies are PEP 508 requirements, so they can have extras and markers. Both need to refer to packages in the same hub. -{attr}`pip.override.tags` adds tags to the targets of the package, e.g. to -avoid building it remotely. +{attr}`pip.override.tags` adds tags to the generated `py_library` targets of +the package. ```starlark pip.override( diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 41f64676ee..ae7d68a750 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -1077,16 +1077,18 @@ wheels of a package. "hub_name": attr.string( doc = """\ The hub that the {attr}`pip.override.package` override applies to. If empty, -it applies to all of the hubs. +it applies to all of the hubs. The settings for a hub are added to the settings +for all of the hubs. :::{versionadded} VERSION_NEXT_FEATURE :::""", ), "ignore_dependencies": attr.string_list( doc = """\ -The names of the dependencies of the {attr}`pip.override.package` to ignore, e.g. to drop a -dependency that is not needed or to break a dependency cycle. They need to be -in the same hub. +The names of the dependencies of the {attr}`pip.override.package` to ignore, +e.g. to drop a dependency that is not needed or to break a dependency cycle. +All of the `Requires-Dist` lines of these packages are removed, including the +ones for the extras. They need to be package names in the same hub. :::{versionadded} VERSION_NEXT_FEATURE :::""", @@ -1118,7 +1120,8 @@ and BUILD.bazel file is generated.""", ), "tags": attr.string_list( doc = """\ -Extra tags to set on the targets of the {attr}`pip.override.package`, e.g. `["no-remote"]`. +Extra tags to set on the generated `py_library` targets of the +{attr}`pip.override.package`. :::{versionadded} VERSION_NEXT_FEATURE :::""", diff --git a/python/private/pypi/whl_overrides.bzl b/python/private/pypi/whl_overrides.bzl index 78d4584af8..3c775bb245 100644 --- a/python/private/pypi/whl_overrides.bzl +++ b/python/private/pypi/whl_overrides.bzl @@ -86,6 +86,15 @@ def parse_whl_overrides(modules, *, _fail = fail): _fail("pip.override(package = '{}') cannot set 'ignore_dependencies', 'extra_dependencies' or 'tags' for a single version".format(attr.package)) continue + bad = [ + dep + for dep in attr.ignore_dependencies + if requirement(dep).name_ != normalize_name(dep) or normalize_name(dep) == whl_name + ] + if bad: + _fail("pip.override(package = '{}').ignore_dependencies needs the names of other packages, got: {}".format(attr.package, bad)) + continue + settings.setdefault(attr.hub_name, {})[whl_name] = struct( ignore_dependencies = attr.ignore_dependencies, extra_dependencies = attr.extra_dependencies, @@ -114,7 +123,7 @@ def parse_whl_overrides(modules, *, _fail = fail): def package_settings_for_hub(overrides, hub_name): """Get the settings of the packages in a hub. - The settings for the hub replace the settings for all hubs field by field. + The settings for the hub are added to the settings for all hubs. Args: overrides: {type}`struct` the result of {func}`parse_whl_overrides`. @@ -129,10 +138,10 @@ def package_settings_for_hub(overrides, hub_name): for whl_name in all_hubs.keys() + this_hub.keys(): if whl_name in ret: continue - ret[whl_name] = { - field: getattr(this_hub.get(whl_name), field, None) or getattr(all_hubs.get(whl_name), field, None) or [] - for field in ["ignore_dependencies", "extra_dependencies", "extra_tags"] - } + ret[whl_name] = {} + for field in ["ignore_dependencies", "extra_dependencies", "extra_tags"]: + values = getattr(all_hubs.get(whl_name), field, []) + getattr(this_hub.get(whl_name), field, []) + ret[whl_name][field] = {v: None for v in values}.keys() return ret def whl_overrides_for_hub(overrides, hub_name, *, _fail = fail): @@ -187,11 +196,7 @@ def unmatched_whl_overrides(overrides, hub_packages): {type}`list[str]` the error messages, empty if all overrides match. """ errors = [] - by_hub = {} - for hub_name, packages in overrides.packages.items() + overrides.settings.items(): - by_hub.setdefault(hub_name, {}).update({name: None for name in packages}) - - for hub_name, packages in sorted(by_hub.items()): + for hub_name, packages in sorted(overrides.packages.items()): if hub_name and hub_name not in hub_packages: errors.append("pip.override(hub_name = '{}') refers to a hub that does not exist".format(hub_name)) continue @@ -212,13 +217,13 @@ def unmatched_whl_overrides(overrides, hub_packages): ("extra_dependencies", setting.extra_dependencies), ]: for value in values: - dep = normalize_name(requirement(value).name) + dep = requirement(value).name_ for h in hubs: if h in hub_packages and whl_name in hub_packages[h] and dep not in hub_packages[h]: errors.append("pip.override(package = '{}').{} refers to '{}', which is not in hub '{}'".format( whl_name, field, - value, + dep, h, )) diff --git a/tests/pypi/whl_overrides/whl_overrides_tests.bzl b/tests/pypi/whl_overrides/whl_overrides_tests.bzl index 6196b63950..935a0c3371 100644 --- a/tests/pypi/whl_overrides/whl_overrides_tests.bzl +++ b/tests/pypi/whl_overrides/whl_overrides_tests.bzl @@ -96,6 +96,8 @@ def _test_errors(env): ([_override(file = "foo-1.0-py3-none-any.whl")], "needs 'patches'"), ([_override(file = "foo-1.0-py3-none-any.whl", patches = ["//:a.patch"], tags = ["a"])], "only supports 'patches'"), ([_override(package = "foo")], "does not set anything"), + ([_override(package = "foo", ignore_dependencies = ["bar[x]"])], "ignore_dependencies needs the names of other packages, got: [\"bar[x]\"]"), + ([_override(package = "foo", ignore_dependencies = ["Foo"])], "ignore_dependencies needs the names of other packages, got: [\"Foo\"]"), ([_override(package = "foo@1.0", tags = ["a"])], "cannot set 'ignore_dependencies', 'extra_dependencies' or 'tags' for a single version"), ([_override(package = "foo@", patches = ["//:a.patch"])], "needs to be a package name or '@'"), ([_override(package = "foo==1.0", patches = ["//:a.patch"])], "needs to be a package name or '@'"), @@ -161,7 +163,7 @@ def _test_package_settings(env): "other": ["foo", "bar", "qux"], "pypi": ["foo", "bar"], })).contains_exactly([ - "pip.override(package = 'foo').extra_dependencies refers to 'baz[x]; sys_platform == 'linux'', which is not in hub 'pypi'", + "pip.override(package = 'foo').extra_dependencies refers to 'baz', which is not in hub 'pypi'", "pip.override(package = 'qux').ignore_dependencies refers to 'missing', which is not in hub 'other'", ])