From 2ccc178d42daa7d45a5afca78bb92d6e040e6b4c Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Thu, 19 Nov 2026 19:43:33 +0900 Subject: [PATCH] feat(pypi): evaluate the extras and dependency_groups markers PEP 751 adds the list valued 'extras' and 'dependency_groups' marker variables, which are used as '"" in '. Evaluate them with normalized names, so that the markers in pylock.toml files can be evaluated. The callers that evaluate the lock files need to set them to an empty list when there are none; other uses of them fail. --- python/private/pypi/BUILD.bazel | 1 + python/private/pypi/pep508_evaluate.bzl | 49 +++++++++++++++++++-- tests/pypi/pep508/evaluate_tests.bzl | 57 ++++++++++++++++++++++++- 3 files changed, 103 insertions(+), 4 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 44661524aa..adc748b74b 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -325,6 +325,7 @@ bzl_library( srcs = ["pep508_evaluate.bzl"], deps = [ "//python/private:enum", + "//python/private:normalize_name", "//python/private:version", ], ) diff --git a/python/private/pypi/pep508_evaluate.bzl b/python/private/pypi/pep508_evaluate.bzl index 61e461a54f..2c31168208 100644 --- a/python/private/pypi/pep508_evaluate.bzl +++ b/python/private/pypi/pep508_evaluate.bzl @@ -16,6 +16,7 @@ """ load("//python/private:enum.bzl", "enum") +load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:version.bzl", "version") # The expression parsing and resolution for the PEP508 is below @@ -41,6 +42,13 @@ _NON_VERSION_VAR_NAMES = [ "sys_platform", "extra", ] + +# The variables that hold a list of names, see PEP 751. These can only be +# used as `"" in ` or `"" not in `. +_SET_VAR_NAMES = [ + "dependency_groups", + "extras", +] _AND = "and" _OR = "or" _NOT = "not" @@ -271,7 +279,7 @@ def _value(self): fail("BUG: invalid state: {}".format(self.tree)) -def marker_expr(left, op, right, *, env, strict = True): +def marker_expr(left, op, right, *, env, strict = True, _fail = fail): """Evaluate a marker expression Args: @@ -280,17 +288,22 @@ def marker_expr(left, op, right, *, env, strict = True): right: {type}`str` the env identifier or a value quoted in `"`. strict: {type}`bool` if false, only evaluates the values that are present in the environment, otherwise returns the original expression. - env: {type}`dict[str, str]` the `env` to substitute `env` identifiers in + env: {type}`dict[str, str | list[str]]` the `env` to substitute `env` identifiers in the ` ` expression. Note, if `env` has a key "_aliases", then we will do normalization so that we can ensure that e.g. `aarch64` evaluation in the `platform_machine` works the same way irrespective if the marker uses `arm64` or `aarch64` value - in the expression. + in the expression. The `extras` and `dependency_groups` values + are lists of names. + _fail: {type}`callable` the function to call on errors, for tests. Returns: {type}`bool` if the expression evaluation result or {type}`str` if the expression could not be evaluated. """ + if left in _SET_VAR_NAMES or right in _SET_VAR_NAMES: + return _set_expr(left, op, right, env = env, strict = strict, _fail = _fail) + var_name = None if right not in env and left not in env and not strict: return "{} {} {}".format(left, op, right) @@ -322,6 +335,36 @@ def marker_expr(left, op, right, *, env, strict = True): # Do not fail here, just evaluate the expression to False. return False +def _set_expr(left, op, right, *, env, strict, _fail): + """Evaluate a `"" in ` expression for a list valued variable. + + The names are normalized, so `"Foo.Bar" in extras` matches the `foo-bar` + extra. The callers evaluating lock files (PEP 751) should set the values + to an empty list if there are none. + """ + if left[0] != '"' or right not in _SET_VAR_NAMES or op not in ["in", "not in"]: + return _fail("unsupported marker: '{} {} {}', the {} can only be used as '\"\" in ' or '\"\" not in '".format( + left, + op, + right, + " and ".join(_SET_VAR_NAMES), + )) + + if right not in env: + if not strict: + return "{} {} {}".format(left, op, right) + return _fail("the marker '{} {} {}' needs the '{}' value in the env".format(left, op, right, right)) + + values = env[right] + if type(values) != type([]): + return _fail("the '{}' value in the env must be a list, got: {}".format(right, values)) + + name = normalize_name(left.strip("\"")) + names = [normalize_name(n) for n in values] + if op == "in": + return name in names + return name not in names + def _env_expr(left, op, right): """Evaluate a string comparison expression""" if op == "==": diff --git a/tests/pypi/pep508/evaluate_tests.bzl b/tests/pypi/pep508/evaluate_tests.bzl index 7c7b5d8173..6e589b8c68 100644 --- a/tests/pypi/pep508/evaluate_tests.bzl +++ b/tests/pypi/pep508/evaluate_tests.bzl @@ -15,7 +15,7 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("//python/private/pypi:pep508_env.bzl", pep508_env = "env") # buildifier: disable=bzl-visibility -load("//python/private/pypi:pep508_evaluate.bzl", "evaluate", "tokenize") # buildifier: disable=bzl-visibility +load("//python/private/pypi:pep508_evaluate.bzl", "evaluate", "marker_expr", "tokenize") # buildifier: disable=bzl-visibility _tests = [] @@ -329,6 +329,61 @@ def _misc_expressions(env): _tests.append(_misc_expressions) +def _evaluate_set_values(env): + marker_env = { + "dependency_groups": ["dev", "Docs.Extra"], + "extras": ["db-backend"], + } + + for input, want in { + "'dev' in dependency_groups": True, + "'dev' not in dependency_groups": False, + "'docs-extra' in dependency_groups": True, + "'test' in dependency_groups": False, + "'test' not in dependency_groups": True, + "'db_backend' in extras": True, + "'DB.Backend' in extras": True, + "'dev' in extras": False, + "'dev' in dependency_groups and 'db-backend' in extras": True, + "'test' in dependency_groups or 'db-backend' in extras": True, + }.items(): # buildifier: @unsorted-dict-items + _check_evaluate(env, input, want, marker_env) + _check_evaluate(env, input, want, marker_env, strict = False) + + # A missing value is not evaluated in non-strict mode, the callers need to + # set the values to an empty list if there are none. + for input in [ + "'dev' in dependency_groups", + "'foo' not in extras", + ]: + _check_evaluate(env, input, input.replace("'", '"'), {}, strict = False) + + # The other values are still evaluated partially + for input, want in { + "os_name == 'osx' and 'dev' in dependency_groups": "os_name == \"osx\"", + "os_name == 'osx' and 'docs' in dependency_groups": False, + "os_name == 'osx' or 'dev' in dependency_groups": "", + "not 'dev' in dependency_groups": False, + }.items(): # buildifier: @unsorted-dict-items + _check_evaluate(env, input, want, {"dependency_groups": ["dev"]}, strict = False) + +_tests.append(_evaluate_set_values) + +def _evaluate_set_values_errors(env): + for left, op, right, marker_env, want in [ + ("extras", "in", "\"a\"", {"extras": []}, "unsupported marker: 'extras in \"a\"'"), + ("\"a\"", "==", "extras", {"extras": []}, "unsupported marker: '\"a\" == extras'"), + ("extras", "in", "dependency_groups", {"extras": []}, "unsupported marker: 'extras in dependency_groups'"), + ("\"a\"", "in", "extras", {}, "the marker '\"a\" in extras' needs the 'extras' value in the env"), + ("\"a\"", "in", "extras", {"extras": "a"}, "the 'extras' value in the env must be a list, got: a"), + ]: + failures = [] + marker_expr(left, op, right, env = marker_env, _fail = failures.append) + env.expect.where(marker = (left, op, right)).that_int(len(failures)).equals(1) + env.expect.that_str(failures[0]).contains(want) + +_tests.append(_evaluate_set_values_errors) + def evaluate_test_suite(name): # buildifier: disable=function-docstring test_suite( name = name,