diff --git a/docs/pypi/circular-dependencies.md b/docs/pypi/circular-dependencies.md index 2d510316e7..0766b83de4 100644 --- a/docs/pypi/circular-dependencies.md +++ b/docs/pypi/circular-dependencies.md @@ -106,3 +106,29 @@ Alternatively, one could use `rules_python`'s patching features to remove one leg of the dependency manually, for instance, by making `apache-airflow-providers-postgres` not explicitly depend on `apache-airflow` or perhaps `apache-airflow-providers-common-sql`. + +## Grouping the cycles automatically + +With a {attr}`pip.parse.uv_lock` or a {attr}`pip.parse.pylock` that records the +dependencies, set {attr}`pip.parse.experimental_automatic_requirement_cycles` +to find the cycles in the dependency graph of the lock file and group them +automatically: + +```starlark +pip.parse( + hub_name = "pypi", + python_version = "3.13", + uv_lock = "//:uv.lock", + experimental_automatic_requirement_cycles = True, +) +``` + +The groups are named `auto_`, e.g. `auto_sphinx`; the members +can be the extras of the packages, e.g. `foo[bar]`, when a cycle goes through +an extra. The groups in {attr}`pip.parse.experimental_requirement_cycles` take +in the cycles that they overlap with. + +The graph of all of the lock files of a hub is used, so the groups are the +same for all of the Python versions, and the markers of the dependencies are +not used, so a cycle is grouped on all of the platforms. A `pylock.toml` needs +to record the dependencies of the packages, which not all tools do. diff --git a/docs/pypi/rules-pycross-comparison.md b/docs/pypi/rules-pycross-comparison.md index fb81bca16a..bd216ccd59 100644 --- a/docs/pypi/rules-pycross-comparison.md +++ b/docs/pypi/rules-pycross-comparison.md @@ -186,8 +186,10 @@ all of them. You can define more platforms with {obj}`pip.default`. * Packages that are not available on every target platform are left out of `all_requirements`. rules_pycross includes them for the platforms where they are available. -* Dependency cycles have to be listed in - {attr}`pip.parse.experimental_requirement_cycles`. +* Dependency cycles are grouped automatically with + {attr}`pip.parse.experimental_automatic_requirement_cycles` or listed in + {attr}`pip.parse.experimental_requirement_cycles`; a whole cycle is one + group, without the marker-gated dependencies of rules_pycross. * `rules_python` does not install wheels in build actions; it extracts them in repository rules. @@ -222,7 +224,7 @@ module. Do not depend on the same distribution from both in one binary. | one file with two hashes | files are looked up by the lock file hash | have | | glibc, musl and macOS version flags | {obj}`pip.default` platforms | partial | | `unavailable_package_mode` | {attr}`pip.parse.unavailable_package_mode`; no `fail_at_execution` | partial | -| automatic dependency cycles | {attr}`pip.parse.experimental_requirement_cycles` | planned | +| automatic dependency cycles | {attr}`pip.parse.experimental_automatic_requirement_cycles` | have | | uv `conflicts` | an error; use a separate hub for each conflicting group | gap | | per-repo transitions | none; the {flag}`--venv` flag selects a hub for the unified `@pypi` hub | gap | | `modules_mapping` without wheels | Gazelle `modules_mapping` | partial | diff --git a/news/pip-parse-automatic-cycles.added.md b/news/pip-parse-automatic-cycles.added.md new file mode 100644 index 0000000000..ce319dd1a0 --- /dev/null +++ b/news/pip-parse-automatic-cycles.added.md @@ -0,0 +1,3 @@ +(pypi) The {attr}`pip.parse.experimental_automatic_requirement_cycles` +attribute groups the dependency cycles in the `uv.lock` or `pylock.toml` +automatically (bzlmod only). diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 88478d1a15..67f3a36101 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -112,6 +112,15 @@ bzl_library( deps = ["@bazel_skylib//lib:selects"], ) +bzl_library( + name = "cycles", + srcs = ["cycles.bzl"], + deps = [ + ":labels", + "//python/private:normalize_name", + ], +) + bzl_library( name = "deps", srcs = ["deps.bzl"], @@ -155,6 +164,7 @@ bzl_library( name = "extension", srcs = ["extension.bzl"], deps = [ + ":cycles", ":default_platforms", ":hub_builder", ":hub_exposure", @@ -214,6 +224,7 @@ bzl_library( srcs = ["hub_builder.bzl"], deps = [ ":attrs", + ":cycles", ":hash", ":labels", ":parse_requirements", diff --git a/python/private/pypi/cycles.bzl b/python/private/pypi/cycles.bzl new file mode 100644 index 0000000000..d96032c9ca --- /dev/null +++ b/python/private/pypi/cycles.bzl @@ -0,0 +1,215 @@ +"""Find the dependency cycles in the lock files to group the packages automatically.""" + +load("//python/private:normalize_name.bzl", "normalize_name") +load(":labels.bzl", "group_member") + +def _add_edge(graph, node, dep): + edges = graph.setdefault(node, []) + if dep not in edges: + edges.append(dep) + +def _dep_nodes(dep): + """The nodes of a dependency entry, e.g. `foo` and `foo[bar]`.""" + name = normalize_name(dep["name"]) + extras = dep.get("extra", []) + if type(extras) == type(""): + extras = [extras] + return [name] + [group_member("{}[{}]".format(name, e)) for e in extras] + +def uv_lock_graph(uv_lock): + """Get the dependency graph of a uv.lock file. + + The nodes are the packages, e.g. `foo`, and their extras, e.g. `foo[bar]`, + like the targets of the hub. An extra depends on its package and on its + optional dependencies. The development dependencies and the workspace + members (the `virtual` and `editable` sources), which are not in the hub, + are not used. + + Args: + uv_lock: {type}`dict` the decoded uv.lock contents. + + Returns: + {type}`dict[str, list[str]]` the dependencies of each node. + """ + roots = { + normalize_name(pkg["name"]): None + for pkg in uv_lock.get("package", []) + if "virtual" in pkg.get("source", {}) or "editable" in pkg.get("source", {}) + } + graph = {} + for pkg in uv_lock.get("package", []): + name = normalize_name(pkg["name"]) + if name in roots: + continue + graph.setdefault(name, []) + for dep in pkg.get("dependencies", []): + if normalize_name(dep["name"]) in roots: + continue + for node in _dep_nodes(dep): + _add_edge(graph, name, node) + for extra, deps in pkg.get("optional-dependencies", {}).items(): + node = group_member("{}[{}]".format(name, extra)) + _add_edge(graph, node, name) + for dep in deps: + if normalize_name(dep["name"]) in roots: + continue + for dep_node in _dep_nodes(dep): + _add_edge(graph, node, dep_node) + return graph + +def pylock_graph(pylock): + """Get the dependency graph of a pylock.toml file, if it records one. + + Args: + pylock: {type}`dict` the decoded pylock.toml contents. + + Returns: + {type}`dict[str, list[str]]` the normalized dependency names of each + normalized package name. + """ + graph = {} + for pkg in pylock.get("packages", []): + if "directory" in pkg: + continue + name = normalize_name(pkg["name"]) + graph.setdefault(name, []) + for dep in pkg.get("dependencies", []): + if "name" not in dep: + continue + _add_edge(graph, name, normalize_name(dep["name"])) + return graph + +def strongly_connected_components(graph): + """Find the strongly connected components with Tarjan's algorithm. + + The algorithm is iterative, because Starlark does not support recursion. + + Args: + graph: {type}`dict[str, list[str]]` the edges of each node. + + Returns: + {type}`list[list[str]]` the sorted components with more than one node, + sorted by their first node. + """ + index = {} + low = {} + on_stack = {} + stack = [] + components = [] + counter = 0 + + # Each node is visited at most once and each edge is followed at most + # once, so this is enough iterations. + max_steps = 1 + for edges in graph.values(): + max_steps += 2 * (1 + len(edges)) + + for root in sorted(graph): + if root in index: + continue + + # The work stack has (node, next edge index) items. + work = [(root, 0)] + index[root] = counter + low[root] = counter + counter += 1 + stack.append(root) + on_stack[root] = True + + for _ in range(max_steps): + if not work: + break + node, i = work[-1] + edges = graph.get(node, []) + if i < len(edges): + work[-1] = (node, i + 1) + dep = edges[i] + if dep not in graph: + continue + if dep not in index: + index[dep] = counter + low[dep] = counter + counter += 1 + stack.append(dep) + on_stack[dep] = True + work.append((dep, 0)) + elif on_stack.get(dep): + low[node] = min(low[node], index[dep]) + continue + + work.pop() + if work: + parent = work[-1][0] + low[parent] = min(low[parent], low[node]) + + if low[node] == index[node]: + component = [] + for _ in range(len(stack)): + member = stack.pop() + on_stack[member] = False + component.append(member) + if member == node: + break + if len(component) > 1: + components.append(sorted(component)) + + if work: + fail("BUG: the strongly connected components were not found in {} steps".format(max_steps)) + + return sorted(components) + +def automatic_requirement_cycles(graphs, requirement_cycles = {}): + """Group the packages that form dependency cycles. + + Args: + graphs: {type}`list[dict[str, list[str]]]` the dependency graphs, e.g. + one for each lock file of a hub, which are merged. + requirement_cycles: {type}`dict[str, list[str]]` the groups that the + user defined, which take in the automatic groups that they + overlap with. + + Returns: + {type}`dict[str, list[str]]` the groups, the user groups and the + `auto_` groups. + """ + graph = {} + for g in graphs: + for name, edges in g.items(): + graph.setdefault(name, []) + for dep in edges: + _add_edge(graph, name, dep) + + ret = { + name: [group_member(m) for m in members] + for name, members in requirement_cycles.items() + } + group_by_member = { + member: name + for name, members in ret.items() + for member in members + } + for component in strongly_connected_components(graph): + user_groups = sorted({ + group_by_member[m]: None + for m in component + if m in group_by_member + }) + if not user_groups: + name = "auto_" + component[0].replace("[", "_").replace("]", "") + if name in ret: + fail("The automatic group '{}' clashes with a group in 'experimental_requirement_cycles'".format(name)) + ret[name] = component + continue + + # A cycle needs to be in a single group, so the first user group takes + # in the cycle and the other user groups that it overlaps with. + target = ret[user_groups[0]] + for other in user_groups[1:]: + target.extend([m for m in ret.pop(other) if m not in target]) + for m in component: + if m not in target: + target.append(m) + for m in target: + group_by_member[m] = user_groups[0] + + return ret diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 47e87a37ed..5189321fd9 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -22,6 +22,7 @@ load("//python/private:auth.bzl", "AUTH_ATTRS") load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:pyproject_utils.bzl", "read_pyproject", "version_from_requires_python") load("//python/private:repo_utils.bzl", "repo_utils") +load(":cycles.bzl", "pylock_graph", "uv_lock_graph") load(":default_platforms.bzl", "default_platforms") load(":hub_builder.bzl", "hub_builder") load(":hub_exposure.bzl", "add_extras", "hub_exposure") @@ -225,6 +226,31 @@ You cannot use both the additive_build_content and additive_build_content_file a simpleapi_cache = pypi_cache(mctx = module_ctx) is_pypi_hub_reserved = module_ctx.getenv("RULES_PYTHON_PYPI_HUB_RESERVED", "0") == "1" + + # The dependency graphs of the lock files of each hub, so that the + # dependency cycles are grouped the same way for all of the Python + # versions of a hub. + cycle_graphs = {} + for mod in module_ctx.modules: + for pip_attr in mod.tags.parse: + if not pip_attr.experimental_automatic_requirement_cycles: + continue + graphs = cycle_graphs.setdefault((mod.name, pip_attr.hub_name), {}) + lock = pip_attr.uv_lock or pip_attr.pylock + if lock in graphs: + continue + graph = None + if pip_attr.uv_lock: + graph = uv_lock_graph(toml.decode(module_ctx.read(pip_attr.uv_lock))) + elif pip_attr.pylock: + graph = pylock_graph(toml.decode(module_ctx.read(pip_attr.pylock))) + if not graph or not [edges for edges in graph.values() if edges]: + print(( + "WARNING: pip.parse(hub_name = '{}', experimental_automatic_requirement_cycles = True) " + + "has no dependency graph, it needs a 'uv_lock' or a 'pylock' with the dependencies" + ).format(pip_attr.hub_name)) # buildifier: disable=print + continue + graphs[lock] = graph renamed_default_hub = None # The hub names as declared by the users, before the reserved `pypi` hub is @@ -280,6 +306,7 @@ You cannot use both the additive_build_content and additive_build_content_file a whl_overrides = whl_overrides_for_hub(whl_overrides, pip_attr.hub_name), package_settings = package_settings_for_hub(whl_overrides, pip_attr.hub_name), build_mode = build_mode_for_hub(whl_overrides, pip_attr.hub_name), + cycle_graphs = cycle_graphs[(mod.name, pip_attr.hub_name)].values() if (mod.name, pip_attr.hub_name) in cycle_graphs else None, simpleapi_download_fn = simpleapi_download, simpleapi_cache = simpleapi_cache, # TODO @aignas 2025-09-06: do not use kwargs @@ -802,6 +829,21 @@ def _pip_parse_ext_attrs(**kwargs): A dict of attributes. """ attrs = dict({ + "experimental_automatic_requirement_cycles": attr.bool( + doc = """\ +Group the packages that form dependency cycles automatically, so that they do +not need to be listed in {attr}`experimental_requirement_cycles`. The cycles +are found in the dependency graph of the {attr}`uv_lock` or the {attr}`pylock` +(if it records the dependencies) of all of the `pip.parse` calls of the hub, +using the dependencies and the extras of the packages. The groups are named +`auto_`; the groups in {attr}`experimental_requirement_cycles` +take in the cycles that they overlap with. Setting it on one of the +`pip.parse` calls of a hub enables it for the hub. The markers of the +dependencies are not used, so a cycle is grouped on all of the platforms. + +:::{versionadded} VERSION_NEXT_FEATURE +:::""", + ), "experimental_extra_index_urls": attr.string_list( doc = """\ May be removed in future releases. diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index b681380de4..22037ed6b8 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -8,6 +8,7 @@ load("//python/private:text_util.bzl", "render") load("//python/private:version.bzl", "version") load("//python/private:version_label.bzl", "version_label") load(":attrs.bzl", "use_isolated") +load(":cycles.bzl", "automatic_requirement_cycles") load(":hash.bzl", "hash") load(":labels.bzl", "group_member") load(":parse_requirements.bzl", "parse_requirements") @@ -31,6 +32,7 @@ def hub_builder( whl_overrides, package_settings = {}, build_mode = None, + cycle_graphs = None, minor_mapping, available_interpreters, simpleapi_download_fn, @@ -47,6 +49,8 @@ def hub_builder( for each package. build_mode: {type}`callable | None` - returns the build mode of a normalized package name. + cycle_graphs: {type}`list[dict] | None` - the dependency graphs to + group the dependency cycles automatically. minor_mapping: {type}`dict[str, str]` the mapping between minor and full versions. available_interpreters: {type}`dict[str, Label]` The dictionary of available interpreters that have been registered using the `python` bzlmod extension. @@ -107,6 +111,7 @@ def hub_builder( _whl_overrides = whl_overrides, _package_settings = package_settings, _build_mode = build_mode or (lambda _: ""), + _cycle_graphs = cycle_graphs, _logger = logger, _minor_mapping = minor_mapping, _available_interpreters = available_interpreters, @@ -141,7 +146,12 @@ def _build(self): whl_map = whl_map, # Maps a wheel to a list of groups # dict[str group_name, list[str]] - group_map = self._group_map, + group_map = { + # The automatic groups may have the packages that the hub does + # not have, e.g. the ones for the other Python versions. + name: [m for m in members if m.partition("[")[0] in whl_map] + for name, members in self._group_map.items() + }, # The per-package aliases for the hub to create. # dict[str package, list[str]] extra_aliases = { @@ -231,7 +241,10 @@ def _pip_parse(self, module_ctx, pip_attr, default_envsubst = [], python_version config = self._config, target_platforms = pip_attr.target_platforms or ["{os}_{arch}"], ) - _add_group_map(self, pip_attr.experimental_requirement_cycles) + requirement_cycles = pip_attr.experimental_requirement_cycles + if self._cycle_graphs != None: + requirement_cycles = automatic_requirement_cycles(self._cycle_graphs, requirement_cycles) + _add_group_map(self, requirement_cycles) _add_extra_aliases(self, pip_attr.extra_hub_aliases) _create_whl_repos( self, diff --git a/tests/pypi/cycles/BUILD.bazel b/tests/pypi/cycles/BUILD.bazel new file mode 100644 index 0000000000..d6b14eee43 --- /dev/null +++ b/tests/pypi/cycles/BUILD.bazel @@ -0,0 +1,3 @@ +load(":cycles_tests.bzl", "cycles_test_suite") + +cycles_test_suite(name = "cycles_tests") diff --git a/tests/pypi/cycles/cycles_tests.bzl b/tests/pypi/cycles/cycles_tests.bzl new file mode 100644 index 0000000000..950cfab396 --- /dev/null +++ b/tests/pypi/cycles/cycles_tests.bzl @@ -0,0 +1,101 @@ +"" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:cycles.bzl", "automatic_requirement_cycles", "pylock_graph", "strongly_connected_components", "uv_lock_graph") # buildifier: disable=bzl-visibility + +_tests = [] + +def _test_scc(env): + env.expect.that_collection(strongly_connected_components({ + "a": ["b"], + "b": ["c", "missing"], + "c": ["a"], + "d": ["d"], # a self loop is not a group + "e": ["f"], + "f": ["g"], + "g": ["f", "e"], + "h": ["a"], + })).contains_exactly([ + ["a", "b", "c"], + ["e", "f", "g"], + ]) + + # A long chain does not need recursion + chain = {"n{}".format(i): ["n{}".format(i + 1)] for i in range(500)} + chain["n500"] = ["n0"] + got = strongly_connected_components(chain) + env.expect.that_int(len(got)).equals(1) + env.expect.that_int(len(got[0])).equals(501) + +_tests.append(_test_scc) + +def _test_graphs(env): + env.expect.that_dict(uv_lock_graph({"package": [ + # The workspace members are not in the hub + {"dependencies": [{"name": "sphinx"}], "name": "my-project", "source": {"virtual": "."}}, + { + "dependencies": [{"name": "sphinxcontrib-serializinghtml"}, {"name": "my-project"}], + # The development dependencies are not used + "dev-dependencies": {"dev": [{"name": "bar"}]}, + "name": "Sphinx", + "optional-dependencies": {"Docs": [{"extra": ["bar"], "name": "Foo"}]}, + }, + {"dependencies": [{"name": "sphinx"}], "name": "sphinxcontrib-serializinghtml"}, + ]})).contains_exactly({ + "sphinx": ["sphinxcontrib_serializinghtml"], + "sphinx[docs]": ["sphinx", "foo", "foo[bar]"], + "sphinxcontrib_serializinghtml": ["sphinx"], + }) + env.expect.that_dict(pylock_graph({"packages": [ + {"dependencies": [{"name": "b"}, {"url": "https://example.org/x.whl"}], "name": "a"}, + {"name": "b"}, + {"directory": {"path": "."}, "name": "my-project"}, + ]})).contains_exactly({ + "a": ["b"], + "b": [], + }) + +_tests.append(_test_graphs) + +def _test_automatic_requirement_cycles(env): + graphs = [ + {"a": ["b"], "b": ["a"], "x": ["y"], "y": []}, + # The graphs of the other lock files are merged + {"p": ["q"], "q": ["p", "r"], "r": ["q"], "x": ["y"], "y": ["x"]}, + ] + + env.expect.that_dict(automatic_requirement_cycles(graphs)).contains_exactly({ + "auto_a": ["a", "b"], + "auto_p": ["p", "q", "r"], + "auto_x": ["x", "y"], + }) + + # The user groups take in the overlapping cycles. The members are matched + # exactly, so `p` is not the same as `p[extra]`. + env.expect.that_dict(automatic_requirement_cycles(graphs, { + "mine": ["P[extra]", "z"], + "other": ["r"], + "third": ["q"], + })).contains_exactly({ + "auto_a": ["a", "b"], + "auto_x": ["x", "y"], + "mine": ["p[extra]", "z"], + "other": ["r", "q", "p"], + }) + + # A cycle through an extra groups the extra + env.expect.that_dict(automatic_requirement_cycles([ + {"a": [], "a[x]": ["a", "b"], "b": ["a[x]"]}, + ])).contains_exactly({ + "auto_a_x": ["a[x]", "b"], + }) + +_tests.append(_test_automatic_requirement_cycles) + +def cycles_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) diff --git a/tests/pypi/extension/pip_parse.bzl b/tests/pypi/extension/pip_parse.bzl index 3670282a92..6582585fc2 100644 --- a/tests/pypi/extension/pip_parse.bzl +++ b/tests/pypi/extension/pip_parse.bzl @@ -23,6 +23,7 @@ def pip_parse( environment = {}, envsubst = [], experimental_extra_index_urls = [], + experimental_automatic_requirement_cycles = False, experimental_index_url = "", experimental_requirement_cycles = {}, extra_hub_aliases = {}, @@ -31,6 +32,7 @@ def pip_parse( netrc = None, parse_all_requirements_files = True, pip_data_exclude = None, + pylock = None, pyproject_toml = None, python_interpreter = None, python_interpreter_target = None, @@ -77,6 +79,8 @@ def pip_parse( requirements_windows = requirements_windows, timeout = timeout, uv_lock = uv_lock, + pylock = pylock, + experimental_automatic_requirement_cycles = experimental_automatic_requirement_cycles, whl_modifications = whl_modifications, parallel_download = False, experimental_index_url_overrides = {}, diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index d6ee9add71..a08e9ee4cc 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -53,6 +53,7 @@ def hub_builder( minor_mapping = {}, whl_overrides = {}, build_mode = None, + cycle_graphs = None, simpleapi_download_fn = None, log_printer = None, available_interpreters = {}): @@ -89,6 +90,7 @@ def hub_builder( ), whl_overrides = whl_overrides, build_mode = build_mode, + cycle_graphs = cycle_graphs, minor_mapping = minor_mapping or {"3.15": "3.15.19"}, available_interpreters = available_interpreters or { "python_3_15_host": "unit_test_interpreter_target", @@ -312,6 +314,32 @@ def _test_unavailable_package_mode_mismatch(env): _tests.append(_test_unavailable_package_mode_mismatch) +def _test_automatic_requirement_cycles(env): + builder = hub_builder(env, cycle_graphs = [{"bar": ["foo"], "foo": ["bar"]}]) + builder.pip_parse( + _mock_mctx( + os_name = "osx", + arch_name = "aarch64", + mock_files = { + "uv.lock": """{"package":[ +{"name":"foo","source":{"registry":"https://pypi.org/simple"},"version":"1.0","dependencies":[{"name":"bar"}],"wheels":[{"hash":"sha256:deadbeef","url":"https://files.pythonhosted.org/packages/foo-1.0-py3-none-any.whl"}]}, +{"name":"bar","source":{"registry":"https://pypi.org/simple"},"version":"1.0","dependencies":[{"name":"foo"}],"wheels":[{"hash":"sha256:deadbaaf","url":"https://files.pythonhosted.org/packages/bar-1.0-py3-none-any.whl"}]} +]}""", + }, + ), + _parse( + hub_name = "pypi", + python_version = "3.15", + requirements_lock = None, + uv_lock = "uv.lock", + ), + ) + pypi = builder.build() + + pypi.group_map().contains_exactly({"auto_bar": ["bar", "foo"]}) + +_tests.append(_test_automatic_requirement_cycles) + def _test_uv_lock_conflicting_versions(env): logs = {} builder = hub_builder(