From 3ef53b2098194d6754702dffa6defc24b0fd5566 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:07:15 +0900 Subject: [PATCH 01/32] refactor(pypi): select packages from the pyproject.toml dependency groups Read the dependencies, the optional dependencies and the PEP 735 dependency groups from the pyproject.toml and add a helper to select the packages using dependency group specifiers. --- python/private/pypi/BUILD.bazel | 9 ++ python/private/pypi/dependency_groups.bzl | 142 ++++++++++++++++++ python/private/pyproject_utils.bzl | 14 +- tests/pypi/dependency_groups/BUILD.bazel | 3 + .../dependency_groups_tests.bzl | 86 +++++++++++ 5 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 python/private/pypi/dependency_groups.bzl create mode 100644 tests/pypi/dependency_groups/BUILD.bazel create mode 100644 tests/pypi/dependency_groups/dependency_groups_tests.bzl diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index f94d24594f..af005f706e 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -632,6 +632,15 @@ bzl_library( ], ) +bzl_library( + name = "dependency_groups", + srcs = ["dependency_groups.bzl"], + deps = [ + ":pep508_requirement", + "//python/private:normalize_name", + ], +) + bzl_library( name = "argparse", srcs = ["argparse.bzl"], diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl new file mode 100644 index 0000000000..2ddadee3dd --- /dev/null +++ b/python/private/pypi/dependency_groups.bzl @@ -0,0 +1,142 @@ +"""Select PyPI packages using pyproject.toml dependency groups.""" + +load("//python/private:normalize_name.bzl", "normalize_name") +load(":pep508_requirement.bzl", "requirement") + +def select_dependency_groups(pyproject, specs, _fail = fail): + """Select the packages named by the given dependency group specifiers. + + Supported specifiers are: + + * `"default"`: the `[project] dependencies`. + * `"optional:"`: a `[project.optional-dependencies]` entry. + * `"group:"`: a PEP 735 `[dependency-groups]` entry. + * `"optional:*"` and `"group:*"`: all entries of the table. + * `"*"`: all of the above. + * `""`: a dependency group, or an optional dependency group if + there is no dependency group with that name. + * `"transitive"`: only supported together with `"*"` and selects all + of the packages, including the transitive dependencies. + + Args: + pyproject: {type}`struct` returned by `read_pyproject`. + specs: {type}`list[str]` the specifiers. + _fail: {type}`callable` the failure function, mainly for testing. + + Returns: + {type}`struct` with the attributes: + * `packages`: {type}`list[str]` the sorted normalized package names. + * `all`: {type}`bool` whether all of the packages including the + transitive dependencies should be selected, i.e. `"*"` and + `"transitive"` were given. + """ + groups = {"default": pyproject.dependencies} + for name, reqs in pyproject.optional_dependencies.items(): + groups["optional:" + normalize_name(name)] = reqs + for name, reqs in _expand_dependency_groups(pyproject.dependency_groups, _fail).items(): + groups["group:" + name] = reqs + + if "transitive" in specs: + if "*" not in specs: + _fail("The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: {}".format(specs)) + return struct(packages = [], all = True) + + selected = {} + for spec in specs: + keys = _match(spec, groups) + if not keys: + _fail("Unknown dependency group '{}'. Available groups: {}".format( + spec, + ", ".join(sorted(groups)), + )) + for key in keys: + selected[key] = None + + # A self reference like `my-project[test]` selects the optional + # dependencies, but only one level deep. + project = normalize_name(pyproject.name or "") + packages = {} + for key in selected: + for req in groups[key]: + req = requirement(req) + names = [req.name_] + if req.name_ == project: + names = [ + requirement(r).name_ + for extra in req.extras + for r in groups.get("optional:" + extra, []) + ] + for name in names: + if name: + packages[name] = None + + return struct(packages = sorted(packages), all = False) + +def _match(spec, groups): + if spec == "*": + return groups.keys() + if spec in ("optional:*", "group:*"): + prefix = spec[:-1] + return [key for key in groups if key.startswith(prefix)] + + kind, _, name = spec.rpartition(":") + name = normalize_name(name) + if kind: + candidates = [kind + ":" + name] + elif name == "default": + candidates = ["default"] + else: + candidates = ["group:" + name, "optional:" + name] + + for key in candidates: + if key in groups: + return [key] + return [] + +def _expand_dependency_groups(dependency_groups, _fail): + """Expand the PEP 735 `include-group` entries. + + Args: + dependency_groups: {type}`dict[str, list]` the `[dependency-groups]` + table. + _fail: {type}`callable` the failure function. + + Returns: + {type}`dict[str, list[str]]` normalized group names mapped to their + requirement strings. + """ + groups = { + normalize_name(name): items + for name, items in dependency_groups.items() + } + + # Each round expands at least one group unless there is a cycle. + expanded = {} + for _ in range(len(groups)): + for name, items in groups.items(): + if name in expanded: + continue + + reqs = [] + for item in items: + if type(item) == type(""): + reqs.append(item) + continue + + include = normalize_name(item.get("include-group", "")) + if include not in groups: + _fail("Unknown dependency group '{}' included from '{}'".format(include, name)) + return {} + if include not in expanded: + reqs = None + break + reqs.extend(expanded[include]) + + if reqs != None: + expanded[name] = reqs + + if len(expanded) != len(groups): + _fail("Cyclic dependency group includes: {}".format( + ", ".join(sorted([name for name in groups if name not in expanded])), + )) + return expanded diff --git a/python/private/pyproject_utils.bzl b/python/private/pyproject_utils.bzl index e96ffd2023..a9494b8ac2 100644 --- a/python/private/pyproject_utils.bzl +++ b/python/private/pyproject_utils.bzl @@ -18,10 +18,22 @@ def read_pyproject(module_ctx, pyproject): {type}`struct` with the attributes: * `requires_python`: {type}`str | None` the raw `requires-python` value (e.g. `"==3.13.9"`), or `None` if it is not set. + * `name`: {type}`str | None` the `[project] name`. + * `dependencies`: {type}`list[str]` the `[project] dependencies`. + * `optional_dependencies`: {type}`dict[str, list[str]]` the + `[project.optional-dependencies]` table. + * `dependency_groups`: {type}`dict[str, list]` the PEP 735 + `[dependency-groups]` table. Items are either requirement strings + or `{include-group = ""}` tables. """ data = toml.decode(module_ctx.read(module_ctx.path(pyproject), watch = "yes")) + project = data.get("project", {}) return struct( - requires_python = data.get("project", {}).get("requires-python"), + requires_python = project.get("requires-python"), + name = project.get("name"), + dependencies = project.get("dependencies") or [], + optional_dependencies = project.get("optional-dependencies") or {}, + dependency_groups = data.get("dependency-groups") or {}, ) def version_from_requires_python(requires_python): diff --git a/tests/pypi/dependency_groups/BUILD.bazel b/tests/pypi/dependency_groups/BUILD.bazel new file mode 100644 index 0000000000..ba6877e124 --- /dev/null +++ b/tests/pypi/dependency_groups/BUILD.bazel @@ -0,0 +1,3 @@ +load(":dependency_groups_tests.bzl", "dependency_groups_test_suite") + +dependency_groups_test_suite(name = "dependency_groups_tests") diff --git a/tests/pypi/dependency_groups/dependency_groups_tests.bzl b/tests/pypi/dependency_groups/dependency_groups_tests.bzl new file mode 100644 index 0000000000..ee23aec1fb --- /dev/null +++ b/tests/pypi/dependency_groups/dependency_groups_tests.bzl @@ -0,0 +1,86 @@ +"""dependency_groups tests""" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:dependency_groups.bzl", "select_dependency_groups") # buildifier: disable=bzl-visibility + +_tests = [] + +_PYPROJECT = struct( + name = "my-project", + dependencies = ["main-pkg>=1.0"], + optional_dependencies = { + "Docs": ["docs-pkg"], + "all": ["My_Project[docs, grpc]"], + "grpc": ["grpc-pkg"], + }, + dependency_groups = { + "dev": ["dev-pkg", "shared-pkg", {"include-group": "Test"}], + "test": ["test_pkg", "shared-pkg"], + }, +) + +def _select(specs, pyproject = _PYPROJECT): + failures = [] + got = select_dependency_groups(pyproject, specs, _fail = failures.append) + return got, failures + +def _test_select(env): + for specs, want in [ + (["default"], ["main_pkg"]), + (["optional:docs"], ["docs_pkg"]), + (["group:test"], ["shared_pkg", "test_pkg"]), + (["dev"], ["dev_pkg", "shared_pkg", "test_pkg"]), + (["grpc"], ["grpc_pkg"]), + (["all"], ["docs_pkg", "grpc_pkg"]), + (["optional:*"], ["docs_pkg", "grpc_pkg"]), + (["group:*"], ["dev_pkg", "shared_pkg", "test_pkg"]), + (["default", "group:test"], ["main_pkg", "shared_pkg", "test_pkg"]), + (["*"], ["dev_pkg", "docs_pkg", "grpc_pkg", "main_pkg", "shared_pkg", "test_pkg"]), + ]: + got, failures = _select(specs) + env.expect.where(specs = specs).that_collection(failures).contains_exactly([]) + env.expect.where(specs = specs).that_collection(got.packages).contains_exactly(want).in_order() + env.expect.where(specs = specs).that_bool(got.all).equals(False) + +_tests.append(_test_select) + +def _test_select_all(env): + got, failures = _select(["*", "transitive"]) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_bool(got.all).equals(True) + env.expect.that_collection(got.packages).contains_exactly([]) + +_tests.append(_test_select_all) + +def _test_select_errors(env): + for specs, want in [ + (["missing"], "Unknown dependency group 'missing'. Available groups: default, group:dev, group:test, optional:all, optional:docs, optional:grpc"), + (["default", "transitive"], "The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: [\"default\", \"transitive\"]"), + ]: + _, failures = _select(specs) + env.expect.where(specs = specs).that_collection(failures).contains_exactly([want]) + +_tests.append(_test_select_errors) + +def _test_include_group_errors(env): + for groups, want in [ + ({"a": [{"include-group": "missing"}]}, "Unknown dependency group 'missing' included from 'a'"), + ({"a": [{"include-group": "b"}], "b": [{"include-group": "a"}]}, "Cyclic dependency group includes: a, b"), + ]: + _, failures = _select(["*"], pyproject = struct( + name = None, + dependencies = [], + optional_dependencies = {}, + dependency_groups = groups, + )) + env.expect.that_collection(failures).contains_exactly([want]) + +_tests.append(_test_include_group_errors) + +def dependency_groups_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) From 755737072827673c74afba95147995086a579543 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:07:32 +0900 Subject: [PATCH 02/32] refactor(pypi): support hiding the unexposed packages of a hub Add the `hide_unexposed_packages` attribute to the hub repository. When set, the aliases of the packages that are not exposed are only visible to the wheel repositories that depend on them. --- python/private/pypi/hub_repository.bzl | 13 +++- python/private/pypi/render_pkg_aliases.bzl | 45 +++++++++++++- .../render_pkg_aliases_test.bzl | 62 +++++++++++++++++++ 3 files changed, 115 insertions(+), 5 deletions(-) diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index f915aa1c77..d421f3cc09 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -26,12 +26,16 @@ exports_files(["requirements.bzl"]) """ def _impl(rctx): - bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() + if rctx.attr.hide_unexposed_packages: + bzl_packages = rctx.attr.packages + else: + bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() aliases = render_multiplatform_pkg_aliases( aliases = { key: _whl_config_settings_from_json(values) for key, values in rctx.attr.whl_map.items() }, + exposed_packages = bzl_packages if rctx.attr.hide_unexposed_packages else None, extra_hub_aliases = rctx.attr.extra_hub_aliases, requirement_cycles = rctx.attr.groups, platform_config_settings = rctx.attr.platform_config_settings, @@ -78,10 +82,15 @@ hub_repository = repository_rule( "groups": attr.string_list_dict( mandatory = False, ), + "hide_unexposed_packages": attr.bool( + default = False, + doc = "Make the aliases of the packages that are not in `packages` visible only to the wheel repositories.", + ), "packages": attr.string_list( mandatory = False, doc = """\ -The list of packages that will be exposed via all_*requirements macros. Defaults to whl_map keys. +The list of packages that will be exposed via all_*requirements macros. Defaults to whl_map keys +unless `hide_unexposed_packages` is set. """, ), "platform_config_settings": attr.string_list_dict( diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 0a1c328491..5582cc00c9 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -65,7 +65,7 @@ def _repr_actual(aliases): else: return render.dict(aliases, key_repr = _repr_config_setting) -def _render_common_aliases(*, name, aliases, **kwargs): +def _render_common_aliases(*, name, aliases, visibility, **kwargs): pkg_aliases = render.call( "pkg_aliases", name = repr(name), @@ -80,14 +80,21 @@ def _render_common_aliases(*, name, aliases, **kwargs): return """\ load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") {extra_loads} -package(default_visibility = ["//visibility:public"]) +package(default_visibility = {visibility}) {aliases}""".format( aliases = pkg_aliases, extra_loads = extra_loads, + visibility = render.list(visibility), ) -def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases = {}, **kwargs): +def render_pkg_aliases( + *, + aliases, + requirement_cycles = None, + extra_hub_aliases = {}, + exposed_packages = None, + **kwargs): """Create alias declarations for each PyPI package. The aliases should be appended to the pip_repository BUILD.bazel file. These aliases @@ -100,6 +107,9 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases requirement_cycles: any package groups to also add. extra_hub_aliases: The list of extra aliases for each whl to be added in addition to the default ones. + exposed_packages: The packages that should be public. When set, the + aliases of the other packages are only visible to the wheel + repositories. **kwargs: Extra kwargs to pass to the rules. Returns: @@ -124,12 +134,18 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases for whl_name in group_whls } + public = ["//visibility:public"] + if exposed_packages != None: + exposed_packages = {normalize_name(name): None for name in exposed_packages} + internal_visibility = _internal_visibility(aliases, has_groups = bool(requirement_cycles)) + files = { "{}/BUILD.bazel".format(normalize_name(name)): _render_common_aliases( name = normalize_name(name), aliases = pkg_aliases, extra_aliases = extra_hub_aliases.get(normalize_name(name), []), group_name = whl_group_mapping.get(normalize_name(name)), + visibility = public if exposed_packages == None or normalize_name(name) in exposed_packages else internal_visibility, **kwargs ).strip() for name, pkg_aliases in aliases.items() @@ -139,6 +155,29 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases files["_groups/BUILD.bazel"] = generate_group_library_build_bazel("", requirement_cycles) return files +def _internal_visibility(aliases, has_groups = False): + """The visibility of the packages that are not exposed. + + The wheel repositories depend on the aliases of the hub, so the aliases + have to be visible to them. + """ + repo_names = {} + for pkg_aliases in aliases.values(): + if type(pkg_aliases) == type(""): + repo_names[pkg_aliases] = None + continue + + for repo_name in pkg_aliases.values(): + repo_names[repo_name] = None + + visibility = [ + "@{}//:__pkg__".format(repo_name) + for repo_name in sorted(repo_names) + ] + if has_groups: + visibility.append("//_groups:__pkg__") + return visibility + def _major_minor(python_version): major, _, tail = python_version.partition(".") minor, _, _ = tail.partition(".") diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index 9114f59279..13bee38ff5 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -58,6 +58,68 @@ pkg_aliases( _tests.append(_test_legacy_aliases) +def _test_hidden_aliases(env): + actual = render_pkg_aliases( + aliases = { + "bar": "pypi_bar", + "foo": "pypi_foo", + }, + exposed_packages = ["foo"], + ) + + env.expect.that_str(actual["bar/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = [ + "@pypi_bar//:__pkg__", + "@pypi_foo//:__pkg__", +]) + +pkg_aliases( + name = "bar", + actual = "pypi_bar", +)""") + env.expect.that_str(actual["foo/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = ["//visibility:public"]) + +pkg_aliases( + name = "foo", + actual = "pypi_foo", +)""") + +_tests.append(_test_hidden_aliases) + +def _test_hidden_aliases_in_groups(env): + actual = render_pkg_aliases( + aliases = { + "bar": "pypi_bar", + "baz": "pypi_baz", + }, + requirement_cycles = { + "group": ["bar", "baz"], + }, + exposed_packages = ["baz"], + ) + + env.expect.that_str(actual["bar/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = [ + "@pypi_bar//:__pkg__", + "@pypi_baz//:__pkg__", + "//_groups:__pkg__", +]) + +pkg_aliases( + name = "bar", + actual = "pypi_bar", + group_name = "group", +)""") + +_tests.append(_test_hidden_aliases_in_groups) + def _test_bzlmod_aliases(env): # Use this function as it is used in pip_repository actual = render_multiplatform_pkg_aliases( From 23127e5b1bea7fd8a1be0f3dfbab80b4134b8399 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:08:09 +0900 Subject: [PATCH 03/32] feat(pypi): add pip.configure to select the exposed packages of a hub Add the `pip.configure` tag class that limits the packages that a hub exposes to the ones listed in the dependency groups of a `pyproject.toml` file. The other packages of the hub are still used by the wheels that need them, but their aliases are not public. --- docs/pypi/download.md | 50 ++++++ news/4395.added.md | 2 + python/private/pypi/BUILD.bazel | 10 ++ python/private/pypi/extension.bzl | 108 +++++++++++++ python/private/pypi/hub_exposure.bzl | 84 ++++++++++ tests/integration/unified_pypi/BUILD.bazel | 16 ++ tests/integration/unified_pypi/MODULE.bazel | 14 ++ .../unified_pypi/pyproject_filtered.toml | 4 + tests/integration/unified_pypi_test.py | 13 ++ tests/pypi/extension/extension_tests.bzl | 92 ++++++++++- tests/pypi/extension/pip_parse.bzl | 12 ++ tests/pypi/hub_exposure/BUILD.bazel | 3 + .../pypi/hub_exposure/hub_exposure_tests.bzl | 153 ++++++++++++++++++ 13 files changed, 559 insertions(+), 2 deletions(-) create mode 100644 news/4395.added.md create mode 100644 python/private/pypi/hub_exposure.bzl create mode 100644 tests/integration/unified_pypi/pyproject_filtered.toml create mode 100644 tests/pypi/hub_exposure/BUILD.bazel create mode 100644 tests/pypi/hub_exposure/hub_exposure_tests.bzl diff --git a/docs/pypi/download.md b/docs/pypi/download.md index cc8b7890aa..5f16606d24 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -134,6 +134,56 @@ rename, and `requirement("numpy")` routes through the active hub like fixed at loading time, before the active hub is known, so load them from a concrete hub instead. Prefer `@pypi//` labels in new code. +(pip-configure)= +### Configuring the Exposed Packages (pip.configure) + +By default a hub exposes all of the packages from its requirements files or +`uv.lock` file, including the transitive dependencies. A `pip.configure` tag +can limit the hub to the packages listed in a `pyproject.toml` file: the +`[project] dependencies`, `[project.optional-dependencies]` and PEP 735 +`[dependency-groups]` tables. + +```toml +[project] +dependencies = ["numpy"] + +[dependency-groups] +dev = ["pytest"] +``` + +```starlark +pip.parse( + hub_name = "pypi_hub", + pyproject_toml = "//:pyproject.toml", + requirements_lock = "//:requirements_lock.txt", +) + +pip.configure( + hub_name = "pypi_hub", + dependency_groups = ["default", "group:dev"], +) +``` + +Here `@pypi_hub//numpy` and `@pypi_hub//pytest` can be used, but the +dependencies of `numpy` are only used through `@pypi_hub//numpy`. The +`all_requirements` lists in the hub's `requirements.bzl` and the +[Unified @pypi Hub](unified-pypi-hub) only contain the exposed packages. + +The {attr}`pip.configure.dependency_groups` attribute defaults to `["*"]`, +which selects every group. Use `["*", "transitive"]` to expose all of the +packages of the hub, including the transitive dependencies. Selecting the +transitive dependencies of only some of the groups is not supported. + +Multiple tags for the same hub are combined. Only the root module and the +module that created the hub can configure it; if both do, the root module +wins. A self reference such as `my-project[test]` in the `pyproject.toml` +selects the optional dependencies of the project. The packages that are not +part of the hub are ignored, so a single `pyproject.toml` can list tools that +are missing from the lock file. + +:::{versionadded} VERSION_NEXT_FEATURE +::: + ### Declaring Abstract Dependencies (pip.dep) :::{versionadded} 2.2.0 diff --git a/news/4395.added.md b/news/4395.added.md new file mode 100644 index 0000000000..62d7f10880 --- /dev/null +++ b/news/4395.added.md @@ -0,0 +1,2 @@ +(pypi) Added {obj}`pip.configure` to limit the packages that a hub exposes to +the ones listed in the dependency groups of a `pyproject.toml` file. diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index af005f706e..caf482b67f 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -129,6 +129,7 @@ bzl_library( deps = [ ":default_platforms", ":hub_builder", + ":hub_exposure", ":hub_repository", ":parse_whl_name", ":pep508_env", @@ -641,6 +642,15 @@ bzl_library( ], ) +bzl_library( + name = "hub_exposure", + srcs = ["hub_exposure.bzl"], + deps = [ + ":dependency_groups", + "//python/private:pyproject_utils", + ], +) + bzl_library( name = "argparse", srcs = ["argparse.bzl"], diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index f7a1dadaf7..e64e856489 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -24,6 +24,7 @@ load("//python/private:pyproject_utils.bzl", "read_pyproject", "version_from_req load("//python/private:repo_utils.bzl", "repo_utils") load(":default_platforms.bzl", "default_platforms") load(":hub_builder.bzl", "hub_builder") +load(":hub_exposure.bzl", "hub_exposure") load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") load(":parse_whl_name.bzl", "parse_whl_name") load(":pep508_env.bzl", "env") @@ -89,6 +90,7 @@ def build_config( unified_hub_requirements_bzl = False defaults = { "platforms": default_platforms(), + "pyproject_toml": None, "python_version": None, } for mod in module_ctx.modules: @@ -105,6 +107,8 @@ def build_config( unified_hub_requirements_bzl = True pyproject_toml = tag.pyproject_toml if pyproject_toml: + if mod.is_root: + defaults["pyproject_toml"] = pyproject_toml pyproject = read_pyproject(module_ctx, pyproject_toml) if pyproject.requires_python: defaults["python_version"] = version_from_requires_python(pyproject.requires_python) @@ -151,6 +155,7 @@ def build_config( for name, values in defaults["platforms"].items() }, enable_pipstar_extract = enable_pipstar_extract, + pyproject_toml = defaults["pyproject_toml"], toml_decode = toml.decode, unified_hub_requirements_bzl = unified_hub_requirements_bzl, ) @@ -243,6 +248,7 @@ You cannot use both the additive_build_content and additive_build_content_file a # dict[str repo, HubBuilder] # See `hub_builder.bzl%hub_builder()` for `HubBuilder` pip_hub_map = {} + hub_pyproject = {} simpleapi_cache = pypi_cache(mctx = module_ctx) is_pypi_hub_reserved = module_ctx.getenv("RULES_PYTHON_PYPI_HUB_RESERVED", "0") == "1" @@ -319,6 +325,9 @@ You cannot use both the additive_build_content and additive_build_content_file a else: builder = pip_hub_map[hub_name] + if pip_attr.pyproject_toml and hub_name not in hub_pyproject: + hub_pyproject[hub_name] = pip_attr.pyproject_toml + builder.pip_parse( module_ctx, pip_attr = pip_attr, @@ -342,6 +351,7 @@ You cannot use both the additive_build_content and additive_build_content_file a exposed_packages = {} extra_aliases = {} whl_libraries = {} + hubs = {} for hub in pip_hub_map.values(): out = hub.build() @@ -355,8 +365,25 @@ You cannot use both the additive_build_content and additive_build_content_file a extra_aliases[hub.name] = out.extra_aliases hub_group_map[hub.name] = out.group_map hub_whl_map[hub.name] = out.whl_map + hubs[hub.name] = struct( + module_name = hub.module_name, + pyproject = hub_pyproject.get(hub.name), + packages = out.whl_map.keys(), + exposed_packages = out.exposed_packages, + ) + + configured_hubs = hub_exposure( + module_ctx, + hubs = hubs, + default_pyproject = config.pyproject_toml, + _fail = _fail, + ) + if configured_hubs == None: + return None + exposed_packages.update(configured_hubs) return struct( + configured_hubs = configured_hubs, config = config, declared_deps = declared_deps, default_hub = config.default_hub or renamed_default_hub, @@ -511,6 +538,8 @@ def _pip_impl(module_ctx): packages = mods.exposed_packages.get(hub_name, []), platform_config_settings = mods.platform_config_settings.get(hub_name, {}), groups = mods.hub_group_map.get(hub_name), + # The hubs configured with `pip.configure` hide the other packages. + hide_unexposed_packages = hub_name in mods.configured_hubs, ) _create_unified_hub_repo(mods) @@ -638,6 +667,10 @@ This is designed to work with dependency management tools like Renovate. :::{versionadded} 2.3.0 ::: + +:::{versionchanged} VERSION_NEXT_FEATURE +Also used by {obj}`pip.configure` for the hubs of the root module. +::: """, ), "unified_hub_requirements_bzl": attr.bool( @@ -841,6 +874,10 @@ The version must be specified as `==X.Y.Z` (exact version with full semver). :::{versionadded} 2.3.0 ::: + +:::{versionchanged} VERSION_NEXT_FEATURE +Also used by {obj}`pip.configure` to select the packages that the hub exposes. +::: """, ), "python_version": attr.string( @@ -1026,6 +1063,72 @@ execution-phase error when built. """, ) +_configure_tag = tag_class( + attrs = { + "dependency_groups": attr.string_list( + default = ["*"], + doc = """\ +The `pyproject.toml` dependency groups whose packages the hub exposes. + +Each item is one of: + +* `"default"`: the `[project] dependencies`. +* `"optional:"`: the `[project.optional-dependencies]` entry ``. +* `"group:"`: the PEP 735 `[dependency-groups]` entry ``. +* `"optional:*"` or `"group:*"`: all entries of the table. +* `"*"`: all of the above. +* `""`: the dependency group `` if it exists, otherwise the + optional dependency group ``. +* `"transitive"`: only supported together with `"*"`, exposes all of the + packages of the hub including the transitive dependencies, i.e. all of the + packages from the requirements files or the `uv.lock` file. + +See [Configuring the Exposed Packages](#pip-configure) for examples. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + "hub_name": attr.string( + mandatory = True, + doc = """\ +The name of the hub to configure. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + "pyproject_toml": attr.label( + mandatory = False, + doc = """\ +The `pyproject.toml` file that defines the dependency groups. Defaults to the +{attr}`pip.parse.pyproject_toml` of the hub, then to +{attr}`pip.default.pyproject_toml` if the hub is defined by the root module. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + }, + doc = """\ +Configure the packages exposed by a hub. + +The packages are selected from a `pyproject.toml` file using the +`[project] dependencies`, `[project.optional-dependencies]` and PEP 735 +`[dependency-groups]` tables. The other packages of the hub, e.g. the +transitive dependencies, are still used by the packages that need them, but +they are not public. + +Only the root module and the module that created the hub can configure it. If +both do, the root module wins. Multiple tags for the same hub are combined. + +See [Configuring the Exposed Packages](#pip-configure) for examples. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", +) + pypi = module_extension( environ = ["RULES_PYTHON_PYPI_HUB_RESERVED"], doc = """\ @@ -1049,12 +1152,17 @@ Environment Variables: - `RULES_PYTHON_PYPI_HUB_RESERVED`: Enable fallback renaming for reserved hub name collisions. See the {envvar}`RULES_PYTHON_PYPI_HUB_RESERVED` documentation for details. +pip.configure: +This tag class configures the packages exposed by a hub using the dependency +groups from a pyproject.toml file. + pip.whl_mods: This tag class is used to help create JSON files to describe modifications to the BUILD files for wheels. """, implementation = _pip_impl, tag_classes = { + "configure": _configure_tag, "default": tag_class( attrs = _default_attrs, doc = """\ diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl new file mode 100644 index 0000000000..6ca9315a81 --- /dev/null +++ b/python/private/pypi/hub_exposure.bzl @@ -0,0 +1,84 @@ +"""Compute the packages exposed by the hubs configured with `pip.configure`.""" + +load("//python/private:pyproject_utils.bzl", "read_pyproject") +load(":dependency_groups.bzl", "select_dependency_groups") + +def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): + """Compute the packages exposed by the hubs configured with `pip.configure`. + + Args: + module_ctx: {type}`module_ctx` the module context. + hubs: {type}`dict[str, struct]` the hubs by name. Each struct has: + * `module_name`: {type}`str` the module that created the hub. + * `pyproject`: {type}`Label | None` from `pip.parse`. + * `packages`: {type}`list[str]` the normalized names of all + packages in the hub. + * `exposed_packages`: {type}`list[str]` the packages that the hub + exposes when it is not configured. + default_pyproject: {type}`Label | None` from `pip.default`, only used + for the hubs of the root module. + _fail: {type}`callable` the failure function, mainly for testing. + + Returns: + {type}`dict[str, list[str]] | None` the configured hubs mapped to the + sorted packages that they expose. The hubs that are not configured + are not included. Returns `None` on failure. + """ + + # The root module wins over the owner of the hub when both configure it. + root_modules = [mod for mod in module_ctx.modules if mod.is_root] + root_name = root_modules[0].name if root_modules else None + root_configured = { + tag.hub_name: None + for mod in root_modules + for tag in mod.tags.configure + } + + configured = {} + for mod in module_ctx.modules: + for tag in mod.tags.configure: + hub = tag.hub_name + if hub not in hubs: + _fail("pip.configure() refers to unknown hub '{}'. Available hubs: {}".format( + hub, + ", ".join(sorted(hubs)), + )) + return None + + info = hubs[hub] + if not mod.is_root and (mod.name != info.module_name or hub in root_configured): + # Only the root module and the module that created the hub can + # configure it. + continue + + pyproject = ( + tag.pyproject_toml or + info.pyproject or + (default_pyproject if info.module_name == root_name else None) + ) + if not pyproject: + _fail("pip.configure() for hub '{}' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`".format(hub)) + return None + + selection = select_dependency_groups( + read_pyproject(module_ctx, pyproject), + tag.dependency_groups, + _fail = _fail, + ) + if selection.all: + packages = info.exposed_packages + else: + packages = selection.packages + + # Multiple tags for the same hub are combined. + exposed = configured.setdefault(hub, {}) + for name in packages: + # The pyproject.toml may list packages that are not in the hub, + # e.g. tools that are not locked. + if name in info.packages: + exposed[name] = None + + return { + hub: sorted(packages) + for hub, packages in configured.items() + } diff --git a/tests/integration/unified_pypi/BUILD.bazel b/tests/integration/unified_pypi/BUILD.bazel index 13d188d087..f3a9513d1f 100644 --- a/tests/integration/unified_pypi/BUILD.bazel +++ b/tests/integration/unified_pypi/BUILD.bazel @@ -95,3 +95,19 @@ platform( "@platforms//os:windows", ], ) + +# The pypi_filtered hub only makes its pyproject.toml packages public. +py_test( + name = "test_filtered_direct", + srcs = ["test_default.py"], + main = "test_default.py", + deps = ["@pypi_filtered//colorama"], +) + +py_binary( + name = "bin_filtered_six", + srcs = ["bin_six_a.py"], + main = "bin_six_a.py", + tags = ["manual"], + deps = ["@pypi_filtered//six"], +) diff --git a/tests/integration/unified_pypi/MODULE.bazel b/tests/integration/unified_pypi/MODULE.bazel index ee8f622cb4..70e01ccdac 100644 --- a/tests/integration/unified_pypi/MODULE.bazel +++ b/tests/integration/unified_pypi/MODULE.bazel @@ -53,6 +53,20 @@ pip.parse( ) use_repo(pip, "pypi_b") +# `pypi_filtered` only exposes the `[project] dependencies` of the +# `pyproject.toml`, the other packages are not public. +pip.parse( + hub_name = "pypi_filtered", + pyproject_toml = "//:pyproject_filtered.toml", + python_version = "3.11", + requirements_lock = "//:requirements_b.txt", +) +pip.configure( + dependency_groups = ["default"], + hub_name = "pypi_filtered", +) +use_repo(pip, "pypi_filtered") + pip.default(default_hub = "pypi_b") pip.default(unified_hub_requirements_bzl = True) pip.dep( diff --git a/tests/integration/unified_pypi/pyproject_filtered.toml b/tests/integration/unified_pypi/pyproject_filtered.toml new file mode 100644 index 0000000000..632f72fe22 --- /dev/null +++ b/tests/integration/unified_pypi/pyproject_filtered.toml @@ -0,0 +1,4 @@ +[project] +name = "unified-pypi" +version = "0.0.0" +dependencies = ["colorama"] diff --git a/tests/integration/unified_pypi_test.py b/tests/integration/unified_pypi_test.py index 9ce41fafb7..5ac2f7df8b 100644 --- a/tests/integration/unified_pypi_test.py +++ b/tests/integration/unified_pypi_test.py @@ -7,6 +7,19 @@ class UnifiedPypiTest(runner.TestCase): + def test_configure_hides_unexposed_packages(self): + self.run_bazel("test", "//:test_filtered_direct") + result = self.run_bazel("build", "//:bin_filtered_six", check=False) + self.assertNotEqual( + result.exit_code, + 0, + "Expected the alias of the unexposed package to be hidden", + ) + self.assert_result_matches( + result, + r"alias '[^']*//six:six'.*is not visible from", + ) + def test_default_fallback_hub(self): self.run_bazel("test", "//:test_default") diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index ee8e58dfe4..17f8d6741e 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -21,7 +21,7 @@ load("//python/private/pypi:extension.bzl", "build_config", "parse_modules") # load("//python/private/pypi:platform.bzl", _plat = "platform") # buildifier: disable=bzl-visibility load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # buildifier: disable=bzl-visibility load("//tests/support/mocks:mocks.bzl", "mocks") -load(":pip_parse.bzl", _parse = "pip_parse") +load(":pip_parse.bzl", _configure = "configure", _parse = "pip_parse") _tests = [] @@ -101,10 +101,11 @@ def _dep(*, name, extra_targets = []): extra_targets = extra_targets, ) -def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], dep = [], is_root = True): +def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], dep = [], configure = [], is_root = True): return struct( name = name, tags = struct( + configure = configure, parse = parse, override = override, whl_mods = whl_mods, @@ -118,6 +119,7 @@ def _parse_modules(env, **kwargs): return env.expect.that_struct( parse_modules(**kwargs), attrs = dict( + configured_hubs = subjects.dict, declared_deps = subjects.dict, default_hub = subjects.str, exposed_packages = subjects.dict, @@ -766,6 +768,92 @@ simple==0.0.1 \ _tests.append(_test_default_index_setting) +_CONFIGURE_REQUIREMENTS = """\ +simple==0.0.1 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +other-pkg==0.0.2 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +transitive-pkg==0.0.3 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +""" + +_CONFIGURE_PYPROJECT = """\ +[project] +dependencies = ["simple"] + +[dependency-groups] +dev = ["other-pkg"] +test = ["simple"] +""" + +def _configure_parse_modules(env, *modules): + return _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + os_name = "linux", + arch_name = "x86_64", + mock_files = { + "pyproject.toml": _CONFIGURE_PYPROJECT, + "requirements.txt": _CONFIGURE_REQUIREMENTS, + }, + *modules + ), + available_interpreters = { + "python_3_15_host": "unit_test_interpreter_target", + }, + minor_mapping = {"3.15": "3.15.19"}, + ) + +def _configure_parse(*, hub_name = "pypi_a", pyproject_toml = "pyproject.toml"): + return _parse( + hub_name = hub_name, + pyproject_toml = pyproject_toml, + python_version = "3.15", + simpleapi_skip = ["simple", "other-pkg", "transitive-pkg"], + requirements_lock = "requirements.txt", + ) + +def _test_configure(env): + for groups, want in [ + (["*"], ["other_pkg", "simple"]), + (["test"], ["simple"]), + (["dev"], ["other_pkg"]), + (["*", "transitive"], ["other_pkg", "simple", "transitive_pkg"]), + ]: + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [_configure_parse()], + configure = [_configure(hub_name = "pypi_a", dependency_groups = groups)], + ), + ) + + pypi.exposed_packages().contains_exactly({"pypi_a": want}) + pypi.configured_hubs().contains_exactly({"pypi_a": want}) + +_tests.append(_test_configure) + +def _test_not_configured_exposes_all(env): + # The pyproject.toml does not filter the hub without `pip.configure`. + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [_configure_parse()], + ), + ) + + pypi.exposed_packages().contains_exactly({ + "pypi_a": ["other_pkg", "simple", "transitive_pkg"], + }) + pypi.configured_hubs().contains_exactly({}) + +_tests.append(_test_not_configured_exposes_all) + def extension_test_suite(name): """Create the test suite. diff --git a/tests/pypi/extension/pip_parse.bzl b/tests/pypi/extension/pip_parse.bzl index 7b5bdfdfd6..505c249169 100644 --- a/tests/pypi/extension/pip_parse.bzl +++ b/tests/pypi/extension/pip_parse.bzl @@ -1,5 +1,17 @@ """A simple test helper""" +def configure( + *, + dependency_groups = ["*"], + hub_name, + pyproject_toml = None): + """A simple helper for testing to simulate the PyPI extension configure tag class""" + return struct( + dependency_groups = dependency_groups, + hub_name = hub_name, + pyproject_toml = pyproject_toml, + ) + def pip_parse( *, hub_name, diff --git a/tests/pypi/hub_exposure/BUILD.bazel b/tests/pypi/hub_exposure/BUILD.bazel new file mode 100644 index 0000000000..187361e1ce --- /dev/null +++ b/tests/pypi/hub_exposure/BUILD.bazel @@ -0,0 +1,3 @@ +load(":hub_exposure_tests.bzl", "hub_exposure_test_suite") + +hub_exposure_test_suite(name = "hub_exposure_tests") diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl new file mode 100644 index 0000000000..cafbb229b5 --- /dev/null +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -0,0 +1,153 @@ +"""hub_exposure tests""" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:hub_exposure.bzl", "hub_exposure") # buildifier: disable=bzl-visibility +load("//tests/support/mocks:mocks.bzl", "mocks") + +_tests = [] + +_PYPROJECT = """\ +[project] +dependencies = ["simple"] + +[dependency-groups] +dev = ["other-pkg"] +test = ["simple", "not-in-hub"] +""" + +def _hub(*, module_name = "root", pyproject = None): + return struct( + module_name = module_name, + pyproject = pyproject, + packages = ["other_pkg", "simple", "transitive_pkg"], + exposed_packages = ["other_pkg", "simple", "transitive_pkg"], + ) + +def _configure(hub_name = "hub", dependency_groups = ["*"], pyproject_toml = None): + return mocks.tag( + hub_name = hub_name, + dependency_groups = dependency_groups, + pyproject_toml = pyproject_toml, + ) + +def _module(name = "root", is_root = True, configure = []): + return mocks.module(name = name, is_root = is_root, configure = configure) + +def _exposure(*modules, hubs = None, default_pyproject = None): + failures = [] + got = hub_exposure( + mocks.mctx( + modules = list(modules), + mock_files = { + "other.toml": "[project]\ndependencies = [\"other-pkg\"]\n", + "pyproject.toml": _PYPROJECT, + }, + ), + hubs = hubs or {"hub": _hub(pyproject = "pyproject.toml")}, + default_pyproject = default_pyproject, + _fail = failures.append, + ) + return got, failures + +def _test_not_configured(env): + got, failures = _exposure(_module()) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({}) + +_tests.append(_test_not_configured) + +def _test_configure(env): + for groups, want in [ + (["*"], ["other_pkg", "simple"]), + (["default"], ["simple"]), + (["dev"], ["other_pkg"]), + # `not-in-hub` is ignored + (["group:test"], ["simple"]), + (["*", "transitive"], ["other_pkg", "simple", "transitive_pkg"]), + ]: + got, failures = _exposure(_module(configure = [_configure(dependency_groups = groups)])) + env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) + env.expect.where(groups = groups).that_dict(got).contains_exactly({"hub": want}) + +_tests.append(_test_configure) + +def _test_configure_combines_tags(env): + got, failures = _exposure(_module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(dependency_groups = ["dev"]), + ])) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg", "simple"]}) + +_tests.append(_test_configure_combines_tags) + +def _test_configure_pyproject_precedence(env): + # The tag wins over the `pip.parse` value. + got, _ = _exposure(_module(configure = [ + _configure(pyproject_toml = "other.toml"), + ])) + env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) + + # The `pip.default` value is used for the hubs of the root module. + got, failures = _exposure( + _module(configure = [_configure(dependency_groups = ["default"])]), + hubs = {"hub": _hub()}, + default_pyproject = "pyproject.toml", + ) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({"hub": ["simple"]}) + +_tests.append(_test_configure_pyproject_precedence) + +def _test_configure_module_precedence(env): + hubs = {"hub": _hub(module_name = "dep", pyproject = "pyproject.toml")} + + # The owner of the hub can configure it. + got, _ = _exposure( + _module(), + _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + hubs = hubs, + ) + env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) + + # The root module wins over the owner. + got, _ = _exposure( + _module(configure = [_configure(dependency_groups = ["default"])]), + _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + hubs = hubs, + ) + env.expect.that_dict(got).contains_exactly({"hub": ["simple"]}) + + # Other modules are ignored. + got, _ = _exposure( + _module(), + _module(name = "other", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + hubs = hubs, + ) + env.expect.that_dict(got).contains_exactly({}) + +_tests.append(_test_configure_module_precedence) + +def _test_configure_errors(env): + _, failures = _exposure(_module(configure = [_configure(hub_name = "missing")])) + env.expect.that_collection(failures).contains_exactly([ + "pip.configure() refers to unknown hub 'missing'. Available hubs: hub", + ]) + + _, failures = _exposure( + _module(configure = [_configure()]), + hubs = {"hub": _hub()}, + ) + env.expect.that_collection(failures).contains_exactly([ + "pip.configure() for hub 'hub' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`", + ]) + +_tests.append(_test_configure_errors) + +def hub_exposure_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) From 6bb7f8379a68c92c78b12acb4a0fad3b8c99a36a Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:11:22 +0900 Subject: [PATCH 04/32] fix(pypi): only add the exposed packages of a configured hub to @pypi Otherwise the unified hub would point to the hidden aliases of the hub and fail with a visibility error instead of the missing package error. --- python/private/pypi/extension.bzl | 4 +++- tests/integration/unified_pypi/BUILD.bazel | 12 ++++++++++++ tests/integration/unified_pypi_test.py | 13 +++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index e64e856489..ce5e95492d 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -416,7 +416,9 @@ def _create_unified_hub_repo(mods): extra_aliases = {} for hub_name in hubs: - for pkg_name in mods.hub_whl_map[hub_name]: + # The hubs configured with `pip.configure` only expose some packages. + hub_packages = mods.configured_hubs.get(hub_name, mods.hub_whl_map[hub_name]) + for pkg_name in hub_packages: norm_pkg = normalize_name(pkg_name) if norm_pkg not in packages: packages[norm_pkg] = [] diff --git a/tests/integration/unified_pypi/BUILD.bazel b/tests/integration/unified_pypi/BUILD.bazel index f3a9513d1f..78f9a857ee 100644 --- a/tests/integration/unified_pypi/BUILD.bazel +++ b/tests/integration/unified_pypi/BUILD.bazel @@ -104,6 +104,18 @@ py_test( deps = ["@pypi_filtered//colorama"], ) +# six is not exposed by pypi_filtered, so it is missing in the unified hub. +py_binary( + name = "bin_filtered_six_unified", + srcs = ["bin_six_a.py"], + config_settings = { + "@rules_python//python/config_settings:venv": "pypi_filtered", + }, + main = "bin_six_a.py", + tags = ["manual"], + deps = ["@pypi//six"], +) + py_binary( name = "bin_filtered_six", srcs = ["bin_six_a.py"], diff --git a/tests/integration/unified_pypi_test.py b/tests/integration/unified_pypi_test.py index 5ac2f7df8b..f59a19fb42 100644 --- a/tests/integration/unified_pypi_test.py +++ b/tests/integration/unified_pypi_test.py @@ -20,6 +20,19 @@ def test_configure_hides_unexposed_packages(self): r"alias '[^']*//six:six'.*is not visible from", ) + def test_configure_hides_unexposed_packages_in_unified_hub(self): + self.run_bazel("cquery", "//:bin_filtered_six_unified") + result = self.run_bazel("build", "//:bin_filtered_six_unified", check=False) + self.assertNotEqual( + result.exit_code, + 0, + "Expected build to fail during execution phase", + ) + self.assert_result_matches( + result, + 'ERROR: PyPI package "six" is not available when building under PyPI hub "pypi_filtered"\\. Try adding it to the requirements of this hub', + ) + def test_default_fallback_hub(self): self.run_bazel("test", "//:test_default") From b31a734fd273538849adba68184df7b801212fce Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:05:33 +0900 Subject: [PATCH 05/32] fix(pypi): treat exposing all packages as not configuring the hub pip.configure with ["*", "transitive"] used the packages that are available on all platforms, which made the platform-specific packages private. Treat it as if the hub was not configured instead, which keeps these packages public and out of all_requirements as before. --- docs/pypi/download.md | 5 ++-- python/private/pypi/extension.bzl | 1 - python/private/pypi/hub_exposure.bzl | 24 +++++++++---------- .../pypi/hub_exposure/hub_exposure_tests.bzl | 14 +++++++++-- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/docs/pypi/download.md b/docs/pypi/download.md index 5f16606d24..654e156921 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -171,8 +171,9 @@ dependencies of `numpy` are only used through `@pypi_hub//numpy`. The The {attr}`pip.configure.dependency_groups` attribute defaults to `["*"]`, which selects every group. Use `["*", "transitive"]` to expose all of the -packages of the hub, including the transitive dependencies. Selecting the -transitive dependencies of only some of the groups is not supported. +packages of the hub, including the transitive dependencies, which is the same +as not configuring the hub. Selecting the transitive dependencies of only some +of the groups is not supported. Multiple tags for the same hub are combined. Only the root module and the module that created the hub can configure it; if both do, the root module diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index ce5e95492d..10e0c456a2 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -369,7 +369,6 @@ You cannot use both the additive_build_content and additive_build_content_file a module_name = hub.module_name, pyproject = hub_pyproject.get(hub.name), packages = out.whl_map.keys(), - exposed_packages = out.exposed_packages, ) configured_hubs = hub_exposure( diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl index 6ca9315a81..fe9e586153 100644 --- a/python/private/pypi/hub_exposure.bzl +++ b/python/private/pypi/hub_exposure.bzl @@ -13,28 +13,27 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): * `pyproject`: {type}`Label | None` from `pip.parse`. * `packages`: {type}`list[str]` the normalized names of all packages in the hub. - * `exposed_packages`: {type}`list[str]` the packages that the hub - exposes when it is not configured. default_pyproject: {type}`Label | None` from `pip.default`, only used for the hubs of the root module. _fail: {type}`callable` the failure function, mainly for testing. Returns: {type}`dict[str, list[str]] | None` the configured hubs mapped to the - sorted packages that they expose. The hubs that are not configured - are not included. Returns `None` on failure. + sorted packages that they expose. The hubs that are not configured or + that expose all of their packages are not included, so that they + behave the same. Returns `None` on failure. """ # The root module wins over the owner of the hub when both configure it. root_modules = [mod for mod in module_ctx.modules if mod.is_root] root_name = root_modules[0].name if root_modules else None - root_configured = { - tag.hub_name: None - for mod in root_modules - for tag in mod.tags.configure - } + root_configured = {} + for mod in root_modules: + for tag in mod.tags.configure: + root_configured[tag.hub_name] = None configured = {} + exposes_all = {} for mod in module_ctx.modules: for tag in mod.tags.configure: hub = tag.hub_name @@ -66,13 +65,11 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): _fail = _fail, ) if selection.all: - packages = info.exposed_packages - else: - packages = selection.packages + exposes_all[hub] = None # Multiple tags for the same hub are combined. exposed = configured.setdefault(hub, {}) - for name in packages: + for name in selection.packages: # The pyproject.toml may list packages that are not in the hub, # e.g. tools that are not locked. if name in info.packages: @@ -81,4 +78,5 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): return { hub: sorted(packages) for hub, packages in configured.items() + if hub not in exposes_all } diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl index cafbb229b5..5d84a351b5 100644 --- a/tests/pypi/hub_exposure/hub_exposure_tests.bzl +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -20,7 +20,6 @@ def _hub(*, module_name = "root", pyproject = None): module_name = module_name, pyproject = pyproject, packages = ["other_pkg", "simple", "transitive_pkg"], - exposed_packages = ["other_pkg", "simple", "transitive_pkg"], ) def _configure(hub_name = "hub", dependency_groups = ["*"], pyproject_toml = None): @@ -63,7 +62,6 @@ def _test_configure(env): (["dev"], ["other_pkg"]), # `not-in-hub` is ignored (["group:test"], ["simple"]), - (["*", "transitive"], ["other_pkg", "simple", "transitive_pkg"]), ]: got, failures = _exposure(_module(configure = [_configure(dependency_groups = groups)])) env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) @@ -71,6 +69,18 @@ def _test_configure(env): _tests.append(_test_configure) +def _test_configure_all_packages(env): + # Exposing all of the packages is the same as not configuring the hub, so + # that e.g. the platform-specific packages stay public. + got, failures = _exposure(_module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(dependency_groups = ["*", "transitive"]), + ])) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({}) + +_tests.append(_test_configure_all_packages) + def _test_configure_combines_tags(env): got, failures = _exposure(_module(configure = [ _configure(dependency_groups = ["default"]), From 9d97360242b44f6a73472793d17a17287cabb1f3 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:05:56 +0900 Subject: [PATCH 06/32] refactor(pypi): use explicit loops and simpler conditions in pip.configure Replace the nested comprehensions and the long inline conditions with explicit loops and use types.is_string. --- python/private/pypi/BUILD.bazel | 1 + python/private/pypi/dependency_groups.bzl | 19 ++++++++----------- python/private/pypi/hub_repository.bzl | 4 +++- python/private/pypi/render_pkg_aliases.bzl | 19 +++++++++++-------- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index caf482b67f..fe28b17413 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -639,6 +639,7 @@ bzl_library( deps = [ ":pep508_requirement", "//python/private:normalize_name", + "@bazel_skylib//lib:types", ], ) diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl index 2ddadee3dd..4e40c7e856 100644 --- a/python/private/pypi/dependency_groups.bzl +++ b/python/private/pypi/dependency_groups.bzl @@ -1,5 +1,6 @@ """Select PyPI packages using pyproject.toml dependency groups.""" +load("@bazel_skylib//lib:types.bzl", "types") load("//python/private:normalize_name.bzl", "normalize_name") load(":pep508_requirement.bzl", "requirement") @@ -59,16 +60,12 @@ def select_dependency_groups(pyproject, specs, _fail = fail): for key in selected: for req in groups[key]: req = requirement(req) - names = [req.name_] - if req.name_ == project: - names = [ - requirement(r).name_ - for extra in req.extras - for r in groups.get("optional:" + extra, []) - ] - for name in names: - if name: - packages[name] = None + if req.name_ != project: + packages[req.name_] = None + continue + for extra in req.extras: + for r in groups.get("optional:" + extra, []): + packages[requirement(r).name_] = None return struct(packages = sorted(packages), all = False) @@ -119,7 +116,7 @@ def _expand_dependency_groups(dependency_groups, _fail): reqs = [] for item in items: - if type(item) == type(""): + if types.is_string(item): reqs.append(item) continue diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index d421f3cc09..ff5311c6ea 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -28,14 +28,16 @@ exports_files(["requirements.bzl"]) def _impl(rctx): if rctx.attr.hide_unexposed_packages: bzl_packages = rctx.attr.packages + exposed_packages = bzl_packages else: bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() + exposed_packages = None aliases = render_multiplatform_pkg_aliases( aliases = { key: _whl_config_settings_from_json(values) for key, values in rctx.attr.whl_map.items() }, - exposed_packages = bzl_packages if rctx.attr.hide_unexposed_packages else None, + exposed_packages = exposed_packages, extra_hub_aliases = rctx.attr.extra_hub_aliases, requirement_cycles = rctx.attr.groups, platform_config_settings = rctx.attr.platform_config_settings, diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 5582cc00c9..2c86155876 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -139,17 +139,20 @@ def render_pkg_aliases( exposed_packages = {normalize_name(name): None for name in exposed_packages} internal_visibility = _internal_visibility(aliases, has_groups = bool(requirement_cycles)) - files = { - "{}/BUILD.bazel".format(normalize_name(name)): _render_common_aliases( - name = normalize_name(name), + files = {} + for name, pkg_aliases in aliases.items(): + name = normalize_name(name) + visibility = public + if exposed_packages != None and name not in exposed_packages: + visibility = internal_visibility + files["{}/BUILD.bazel".format(name)] = _render_common_aliases( + name = name, aliases = pkg_aliases, - extra_aliases = extra_hub_aliases.get(normalize_name(name), []), - group_name = whl_group_mapping.get(normalize_name(name)), - visibility = public if exposed_packages == None or normalize_name(name) in exposed_packages else internal_visibility, + extra_aliases = extra_hub_aliases.get(name, []), + group_name = whl_group_mapping.get(name), + visibility = visibility, **kwargs ).strip() - for name, pkg_aliases in aliases.items() - } if requirement_cycles: files["_groups/BUILD.bazel"] = generate_group_library_build_bazel("", requirement_cycles) From 35c066052cb0bea531aba78784149420d8056aaa Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:06:00 +0900 Subject: [PATCH 07/32] test(pypi): test the pip.default pyproject_toml with pip.configure Keep a single wiring test for pip.configure in the extension tests, as the selection is tested in hub_exposure_tests, and test that the pip.default pyproject_toml is used. Also link the issue in the news entry. --- news/4395.added.md | 3 +- tests/pypi/extension/extension_tests.bzl | 49 +++++++++++++++--------- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/news/4395.added.md b/news/4395.added.md index 62d7f10880..3e633cc997 100644 --- a/news/4395.added.md +++ b/news/4395.added.md @@ -1,2 +1,3 @@ (pypi) Added {obj}`pip.configure` to limit the packages that a hub exposes to -the ones listed in the dependency groups of a `pyproject.toml` file. +the ones listed in the dependency groups of a `pyproject.toml` file +([#3413](https://github.com/bazel-contrib/rules_python/issues/3413)). diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 17f8d6741e..680853a0d0 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -807,36 +807,47 @@ def _configure_parse_modules(env, *modules): minor_mapping = {"3.15": "3.15.19"}, ) -def _configure_parse(*, hub_name = "pypi_a", pyproject_toml = "pyproject.toml"): +def _configure_parse(**kwargs): return _parse( - hub_name = hub_name, - pyproject_toml = pyproject_toml, + hub_name = "pypi_a", python_version = "3.15", simpleapi_skip = ["simple", "other-pkg", "transitive-pkg"], requirements_lock = "requirements.txt", + **kwargs ) def _test_configure(env): - for groups, want in [ - (["*"], ["other_pkg", "simple"]), - (["test"], ["simple"]), - (["dev"], ["other_pkg"]), - (["*", "transitive"], ["other_pkg", "simple", "transitive_pkg"]), - ]: - pypi = _configure_parse_modules( - env, - _mod( - name = "rules_python", - parse = [_configure_parse()], - configure = [_configure(hub_name = "pypi_a", dependency_groups = groups)], - ), - ) + # The selection itself is tested in `hub_exposure_tests`. + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [_configure_parse(pyproject_toml = "pyproject.toml")], + configure = [_configure(hub_name = "pypi_a", dependency_groups = ["test"])], + ), + ) - pypi.exposed_packages().contains_exactly({"pypi_a": want}) - pypi.configured_hubs().contains_exactly({"pypi_a": want}) + pypi.exposed_packages().contains_exactly({"pypi_a": ["simple"]}) + pypi.configured_hubs().contains_exactly({"pypi_a": ["simple"]}) _tests.append(_test_configure) +def _test_configure_default_pyproject(env): + # The `pip.default` pyproject.toml is used for the hubs of the root module. + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + default = _default_tags_default + [_default(pyproject_toml = "pyproject.toml")], + parse = [_configure_parse()], + configure = [_configure(hub_name = "pypi_a", dependency_groups = ["dev"])], + ), + ) + + pypi.configured_hubs().contains_exactly({"pypi_a": ["other_pkg"]}) + +_tests.append(_test_configure_default_pyproject) + def _test_not_configured_exposes_all(env): # The pyproject.toml does not filter the hub without `pip.configure`. pypi = _configure_parse_modules( From 2873e71749076be9aff3492a0437f7cfd271d889 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:50:18 +0900 Subject: [PATCH 08/32] feat(pypi): add strip_extras to remove the extras from requirement lines --- python/private/pypi/pep508_requirement.bzl | 19 +++++++++++++++++++ tests/pypi/pep508/requirement_tests.bzl | 15 ++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/python/private/pypi/pep508_requirement.bzl b/python/private/pypi/pep508_requirement.bzl index 7552642572..8aaaa29e37 100644 --- a/python/private/pypi/pep508_requirement.bzl +++ b/python/private/pypi/pep508_requirement.bzl @@ -56,3 +56,22 @@ def requirement(spec): extras = extras, version = version, ) + +def strip_extras(requirement_line): + """Remove the extras from the requirement line. + + Args: + requirement_line: {type}`str` the requirement line, e.g. + `foo[bar]==1.0 --hash=sha256:...`. + + Returns: + The requirement line without the extras, e.g. `foo==1.0 --hash=sha256:...`. + """ + for i, c in enumerate(requirement_line.elems()): + if c in "=<>!~;@( ": + break + if c == "[": + _, _, tail = requirement_line[i:].partition("]") + return requirement_line[:i] + tail + + return requirement_line diff --git a/tests/pypi/pep508/requirement_tests.bzl b/tests/pypi/pep508/requirement_tests.bzl index 2ce45922da..c027209aa1 100644 --- a/tests/pypi/pep508/requirement_tests.bzl +++ b/tests/pypi/pep508/requirement_tests.bzl @@ -14,7 +14,7 @@ """Tests for parsing the requirement specifier.""" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private/pypi:pep508_requirement.bzl", "requirement") # buildifier: disable=bzl-visibility +load("//python/private/pypi:pep508_requirement.bzl", "requirement", "strip_extras") # buildifier: disable=bzl-visibility _tests = [] @@ -42,6 +42,19 @@ def _test_requirement_line_parsing(env): _tests.append(_test_requirement_line_parsing) +def _test_strip_extras(env): + for line, want in { + "foo": "foo", + "foo @ https://example.org/foo[1].whl": "foo @ https://example.org/foo[1].whl", + "foo==1.0 ; python_version < '3.12' --hash=sha256:deadbeef": "foo==1.0 ; python_version < '3.12' --hash=sha256:deadbeef", + "foo[bar,baz] @ https://example.org/foo.whl": "foo @ https://example.org/foo.whl", + "foo[bar]": "foo", + "foo[bar]==1.0 --hash=sha256:deadbeef": "foo==1.0 --hash=sha256:deadbeef", + }.items(): + env.expect.where(line = line).that_str(strip_extras(line)).equals(want) + +_tests.append(_test_strip_extras) + def requirement_test_suite(name): # buildifier: disable=function-docstring test_suite( name = name, From 2f92d4e4c2cf2457f84fef0b3249cb4ea518a1d6 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:50:44 +0900 Subject: [PATCH 09/32] feat(pypi): add keep_extras and dependency excludes to pep508_deps --- python/private/pypi/pep508_deps.bzl | 31 ++++++++++------- tests/pypi/pep508/deps_tests.bzl | 54 +++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 12 deletions(-) diff --git a/python/private/pypi/pep508_deps.bzl b/python/private/pypi/pep508_deps.bzl index e36ceadbd5..da42fbe5e9 100644 --- a/python/private/pypi/pep508_deps.bzl +++ b/python/private/pypi/pep508_deps.bzl @@ -26,23 +26,32 @@ def deps( requires_dist, extras = [], excludes = [], - include = []): + include = [], + keep_extras = False): """Parse the RequiresDist from wheel METADATA Args: name: {type}`str` the name of the wheel. requires_dist: {type}`list[str]` the list of RequiresDist lines from the METADATA file. - excludes: {type}`list[str]` what packages should we exclude. + excludes: {type}`list[str]` what dependencies should we exclude, e.g. + `foo` or `foo[bar]`, in the same format as the returned values. include: {type}`list[str]` what packages should we exclude. If it is not specified, then we will include all deps from `requires_dist`. extras: {type}`list[str]` the requested extras to generate targets for. + keep_extras: {type}`bool` keep the extras of the dependencies, if + `True`, the dependencies with extras are returned as `foo[bar]`. Returns: A struct with attributes: * deps: {type}`list[str]` dependencies to include unconditionally. * deps_select: {type}`dict[str, list[str]]` dependencies to include on particular subset of target platforms. + + The dependencies are normalized names, e.g. `foo`, or normalized names + with a single extra, e.g. `foo[bar]`, if the dependency is requested + with extras and `keep_extras` is set. A dependency with multiple + extras, e.g. `foo[bar,baz]`, is returned as `foo[bar]` and `foo[baz]`. """ if not requires_dist: return struct( @@ -60,25 +69,25 @@ def deps( want_extras = _resolve_extras(name, reqs, extras) include = [normalize_name(n) for n in include] - # drop self edges - excludes = [name] + [normalize_name(x) for x in excludes] - - reqs_by_name = {} + reqs_by_dep = {} for req in reqs: - if req.name_ in excludes: + # drop self edges + if req.name_ == name: continue if include and req.name_ not in include: continue - reqs_by_name.setdefault(req.name, []).append(req) + for dep in ["{}[{}]".format(req.name_, e) for e in req.extras if keep_extras] or [req.name_]: + if dep not in excludes: + reqs_by_dep.setdefault(dep, []).append(req) - for name, reqs in reqs_by_name.items(): + for dep, reqs in reqs_by_dep.items(): _add_reqs( deps, deps_select, - normalize_name(name), + dep, reqs, extras = want_extras, ) @@ -92,8 +101,6 @@ def deps( ) def _add(deps, deps_select, dep, markers = None): - dep = normalize_name(dep) - if not markers: deps[dep] = True elif len(markers) == 1: diff --git a/tests/pypi/pep508/deps_tests.bzl b/tests/pypi/pep508/deps_tests.bzl index 0d171d8e95..6e291e5a6c 100644 --- a/tests/pypi/pep508/deps_tests.bzl +++ b/tests/pypi/pep508/deps_tests.bzl @@ -320,6 +320,60 @@ def test_extras_with_hyphens_are_normalized(env): _tests.append(test_extras_with_hyphens_are_normalized) +def test_deps_with_extras(env): + got = deps( + "foo", + requires_dist = [ + "bar[Socks]", + "baz[a,b]; sys_platform == 'linux'", + "qux", + "qux[c]; extra == 'feat'", + ], + extras = ["feat"], + keep_extras = True, + ) + + # The extras of the dependencies are kept so that the targets with the + # extras can be used. + env.expect.that_collection(got.deps).contains_exactly(["bar[socks]", "qux", "qux[c]"]) + env.expect.that_dict(got.deps_select).contains_exactly({ + "baz[a]": "sys_platform == \"linux\"", + "baz[b]": "sys_platform == \"linux\"", + }) + +_tests.append(test_deps_with_extras) + +def test_deps_with_extras_excludes_and_includes(env): + got = deps( + "foo", + requires_dist = ["bar", "bar[socks]", "baz[a]", "baz[b]", "qux[b]"], + # The excludes are dependencies and the includes are packages + excludes = ["bar", "baz[a]"], + include = ["bar", "baz"], + keep_extras = True, + ) + + env.expect.that_collection(got.deps).contains_exactly(["bar[socks]", "baz[b]"]) + +_tests.append(test_deps_with_extras_excludes_and_includes) + +def test_deps_without_extras(env): + got = deps( + "foo", + requires_dist = [ + "bar[socks]", + "baz[a]; sys_platform == 'linux'", + "baz[b]; sys_platform == 'darwin'", + ], + ) + + env.expect.that_collection(got.deps).contains_exactly(["bar"]) + env.expect.that_dict(got.deps_select).contains_exactly({ + "baz": "(sys_platform == \"darwin\") or (sys_platform == \"linux\")", + }) + +_tests.append(test_deps_without_extras) + def deps_test_suite(name): # buildifier: disable=function-docstring test_suite( name = name, From efc64f051062f5f44216fb0acc96e33599e0e185 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:51:20 +0900 Subject: [PATCH 10/32] feat(pypi): add extra_label and group_member label helpers --- python/private/pypi/BUILD.bazel | 11 ++++++----- python/private/pypi/labels.bzl | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index f94d24594f..bea420c835 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -632,6 +632,12 @@ bzl_library( ], ) +bzl_library( + name = "labels", + srcs = ["labels.bzl"], + deps = ["//python/private:normalize_name"], +) + bzl_library( name = "argparse", srcs = ["argparse.bzl"], @@ -657,11 +663,6 @@ bzl_library( srcs = ["hash.bzl"], ) -bzl_library( - name = "labels", - srcs = ["labels.bzl"], -) - bzl_library( name = "package_annotation", srcs = ["package_annotation.bzl"], diff --git a/python/private/pypi/labels.bzl b/python/private/pypi/labels.bzl index c77167b593..f6257106e9 100644 --- a/python/private/pypi/labels.bzl +++ b/python/private/pypi/labels.bzl @@ -14,6 +14,8 @@ """Constants used by parts of pip_repository for naming libraries and wheels.""" +load("//python/private:normalize_name.bzl", "normalize_name") + EXTRACTED_WHEEL_FILES = "extracted_whl_files" WHEEL_FILE_PUBLIC_LABEL = "whl" WHEEL_FILE_IMPL_LABEL = "_whl" @@ -24,3 +26,30 @@ PY_SRCS_LABEL = "srcs" DATA_LABEL = "data" DIST_INFO_LABEL = "dist_info" NODEPS_LABEL = "no_deps" + +def extra_label(extra): + """Return the label name for the target that includes an extra. + + Args: + extra: {type}`str` the normalized name of the extra, an empty string + means the base package without any extras. + + Returns: + The target name, e.g. `[extra]`. + """ + return "[{}]".format(extra) + +def group_member(spec): + """Normalize a member of a dependency group. + + Args: + spec: {type}`str` the package name, e.g. `Foo`, or the package name + with a single extra, e.g. `Foo[Bar]`. + + Returns: + The normalized member, e.g. `foo` or `foo[bar]`. + """ + name, _, extra = spec.partition("[") + name = normalize_name(name.strip()) + extra = normalize_name(extra.partition("]")[0].strip()) if extra else "" + return "{}[{}]".format(name, extra) if extra else name From 552d8f167c6845de0deb77f3fff5f65236fdd138 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:52:36 +0900 Subject: [PATCH 11/32] feat(pypi): create a target for every provided extra in the whl repositories --- python/private/internal_dev_deps.bzl | 2 +- python/private/pypi/BUILD.bazel | 2 - .../pypi/generate_whl_library_build_bazel.bzl | 2 +- python/private/pypi/patch_and_extract_whl.bzl | 3 +- python/private/pypi/pkg_aliases.bzl | 4 - python/private/pypi/whl_deps_repo.bzl | 3 +- .../private/pypi/whl_library_deps_targets.bzl | 188 ++++++++----- python/private/pypi/whl_library_targets.bzl | 6 +- tests/pypi/pkg_aliases/pkg_aliases_test.bzl | 1 - tests/pypi/whl_library/BUILD.bazel | 2 +- .../whl_library_deps_targets_tests.bzl | 266 +++++++++++++++++- 11 files changed, 382 insertions(+), 97 deletions(-) diff --git a/python/private/internal_dev_deps.bzl b/python/private/internal_dev_deps.bzl index 5bbee232f9..111b839adf 100644 --- a/python/private/internal_dev_deps.bzl +++ b/python/private/internal_dev_deps.bzl @@ -117,7 +117,7 @@ def _internal_dev_deps_impl(mctx): name = "whl_library_extras_direct_dep", root = "//tests/pypi/whl_library/testdata/pkg:BUILD.bazel", output = "pkg-1.0-any-none-any.whl", - requirement = "pkg[optional]", + requirement = "pkg", # The following is necessary to enable pipstar and make tests faster config_load = "@rules_python//tests/pypi/whl_library/testdata:packages.bzl", dep_template = "@whl_library_extras_{name}//:{target}", diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index bea420c835..1e33b366db 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -586,7 +586,6 @@ bzl_library( srcs = ["whl_deps_repo.bzl"], deps = [ ":generate_whl_library_build_bazel", - ":pep508_requirement", "//python/private:repo_utils", ], ) @@ -597,7 +596,6 @@ bzl_library( deps = [ ":generate_whl_library_build_bazel", ":patch_whl", - ":pep508_requirement", ":pypi_repo_utils", ":whl_extract", ":whl_metadata", diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index bec45bb08f..bb8faa71dc 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -22,9 +22,9 @@ _RENDER = { "data": render.list, "data_exclude": render.list, "entry_points": render.dict_dict, - "extras": render.list, "group_deps": render.list, "include": str, + "provides_extra": render.list, "repo": lambda maybe_label: repr(str(maybe_label)), "requires_dist": render.list, "srcs_exclude": render.list, diff --git a/python/private/pypi/patch_and_extract_whl.bzl b/python/private/pypi/patch_and_extract_whl.bzl index 37c90e3221..cfacb4adf4 100644 --- a/python/private/pypi/patch_and_extract_whl.bzl +++ b/python/private/pypi/patch_and_extract_whl.bzl @@ -4,7 +4,6 @@ load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:repo_utils.bzl", "repo_utils") load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") load(":patch_whl.bzl", "get_whl_patches", "patch_whl") -load(":pep508_requirement.bzl", "requirement") load(":pypi_repo_utils.bzl", "pypi_repo_utils") load(":whl_extract.bzl", "whl_extract") load(":whl_metadata.bzl", "parse_entry_points", "whl_metadata") @@ -135,7 +134,7 @@ def patch_and_extract_whl(rctx, *, whl_path, logger, sdist_filename = None, whl_ group_deps = rctx.attr.group_deps, group_name = rctx.attr.group_name, namespace_package_files = namespace_package_files, - extras = requirement(rctx.attr.requirement).extras, + provides_extra = metadata.provides_extra, entry_points = entry_points, purl = _to_purl( index = rctx.attr.index_url, diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index 227e9bf10f..308d9a21fa 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -83,7 +83,6 @@ def pkg_aliases( group_name = None, extra_aliases = None, requires_dist = [], - extras = [], include = [], group_deps = [], **kwargs): @@ -103,8 +102,6 @@ def pkg_aliases( extra_aliases: {type}`list[str]` The extra aliases to be created. requires_dist: {type}`list[str]` The list of dependencies. Comes from METADATA or a lock file. - extras: {type}`list[str]` The extras for which we should add extra - dependencies when parsing the requires_dist. Comes from METADATA or a lock file. include: {type}`list[str]` The subset of packages to include. Comes from `//:config.bzl#packages` **kwargs: extra kwargs to pass to {bzl:obj}`get_config_settings`. @@ -129,7 +126,6 @@ def pkg_aliases( aliases = {}, # not none metadata_name = name, requires_dist = requires_dist, - extras = extras, include = include, group_deps = group_deps, group_name = group_name, diff --git a/python/private/pypi/whl_deps_repo.bzl b/python/private/pypi/whl_deps_repo.bzl index 20891da397..75e5a3771d 100644 --- a/python/private/pypi/whl_deps_repo.bzl +++ b/python/private/pypi/whl_deps_repo.bzl @@ -2,7 +2,6 @@ load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") -load(":pep508_requirement.bzl", "requirement") # Reusable common attributes for generating BUILD.bazel from Requires-Dist in the wheel METADATA. whl_deps_attrs = { @@ -62,7 +61,7 @@ def _whl_deps_repo_impl(rctx): repo = rctx.attr.repo or ( str(rctx.attr.metadata_file) if rctx.attr.metadata_file else None ), - extras = requirement(rctx.attr.requirement).extras, + provides_extra = getattr(metadata, "provides_extra", []), ) rctx.file("BUILD.bazel", build_file_contents) diff --git a/python/private/pypi/whl_library_deps_targets.bzl b/python/private/pypi/whl_library_deps_targets.bzl index e549f52a47..42de802267 100644 --- a/python/private/pypi/whl_library_deps_targets.bzl +++ b/python/private/pypi/whl_library_deps_targets.bzl @@ -8,12 +8,12 @@ load( "DATA_LABEL", "DIST_INFO_LABEL", "EXTRACTED_WHEEL_FILES", - "PY_LIBRARY_IMPL_LABEL", "PY_LIBRARY_PUBLIC_LABEL", "PY_SRCS_LABEL", "WHEEL_FILE", - "WHEEL_FILE_IMPL_LABEL", "WHEEL_FILE_PUBLIC_LABEL", + "extra_label", + "group_member", ) load(":pep508_deps.bzl", "deps") @@ -24,7 +24,7 @@ def whl_library_deps_targets( aliases = None, metadata_name, requires_dist = [], - extras = [], + provides_extra = [], include = [], group_deps = [], group_name = None, @@ -48,10 +48,9 @@ def whl_library_deps_targets( group rules which wrap this library and its fellows together. requires_dist: {type}`list[str]` The list of `Requires-Dist` values from the whl `METADATA`. Optional because some packages don't have them. - extras: {type}`list[str]` The list of requested extras. This essentially - includes extra transitive dependencies in the final targets - depending on the wheel `METADATA`. Optional because some packages - don't request them. + provides_extra: {type}`list[str]` The list of extras that the package + provides. For each extra the `[]` target is created + that includes the package and the dependencies of that extra. include: {type}`list[str]` The list of packages to include. group_name: {type}`str | None` name of the dependency group (if any). dep_template: {type}`str | None` The dep_template to use. @@ -75,27 +74,33 @@ def whl_library_deps_targets( DATA_LABEL: repo_label(DATA_LABEL), } + # The members of the group are the packages, e.g. `foo`, or the extras of + # the packages, e.g. `foo[bar]`. Only the targets of the members are part + # of the group and they do not depend on the other members to avoid cycles. + self_name = normalize_name(metadata_name) + members = {group_member(m): None for m in group_deps} if group_name else {} + is_member = lambda extra: ( + "{}[{}]".format(self_name, extra) if extra else self_name + ) in members + # If this library is a member of a group, its public label aliases need to # point to the group implementation rule not the implementation rules. We # also need to mark the implementation rules as visible to the group # implementation. + group_label = None if group_name and "//:" in dep_template: # This is the legacy behaviour where the group library is outside the hub repo # # It is expected to disappear when we drop WORKSPACE or drop the vendoring of # pip_parse `requirements.bzl` in WORKSPACE. The alternative would be to add # another argument to the macro, but it is already full of arguments. - label_tmpl = dep_template.format( + group_label = dep_template.format( name = "_config", target = normalize_name(group_name) + "_{}", ).replace( "//:", "//_groups:", ) - aliases = aliases | { - PY_LIBRARY_PUBLIC_LABEL: label_tmpl.format(PY_LIBRARY_PUBLIC_LABEL), - WHEEL_FILE_PUBLIC_LABEL: label_tmpl.format(WHEEL_FILE_PUBLIC_LABEL), - } impl_vis = [dep_template.format( name = "_config", target = "__pkg__", @@ -103,25 +108,40 @@ def whl_library_deps_targets( "//:", "//_groups:", )] - - py_library_label = PY_LIBRARY_IMPL_LABEL - whl_file_label = WHEEL_FILE_IMPL_LABEL + elif group_name: + impl_vis = [dep_template.format(name = "", target = "__subpackages__")] else: - py_library_label = PY_LIBRARY_PUBLIC_LABEL - whl_file_label = WHEEL_FILE_PUBLIC_LABEL - if group_name: - impl_vis = [dep_template.format(name = "", target = "__subpackages__")] - else: - impl_vis = visibility + impl_vis = visibility + + def _name(extra, public): + # In the legacy mode the targets of the members are implementation + # targets and the public targets are aliases to the group. + prefix = "_" if group_label and is_member(extra) else "" + return prefix + (extra_label(extra) if extra else public) + + def _visibility(extra): + # In the legacy mode the targets that are not members are public. + return visibility if group_label and not is_member(extra) else impl_vis + + # Create targets for every extra that the package provides, so that the hub + # can select only the extras that are requested on each platform without + # passing the extras to this repository. + extras_targets = [""] + sorted({normalize_name(e): None for e in provides_extra}) + whl_file_label = _name("", WHEEL_FILE_PUBLIC_LABEL) + + if group_label: + for extra in extras_targets: + if is_member(extra): + aliases[extra_label(extra) if extra else PY_LIBRARY_PUBLIC_LABEL] = group_label.format(PY_LIBRARY_PUBLIC_LABEL) + if is_member(""): + aliases[WHEEL_FILE_PUBLIC_LABEL] = group_label.format(WHEEL_FILE_PUBLIC_LABEL) if not requires_dist: - # If the package is in a group but has no deps, we still need the public labels to - # point at the srcs targets so that the group implementation can use them. We don't - # need any of the extra targets, so just create the aliases. - aliases = aliases | { - py_library_label: repo_label(PY_SRCS_LABEL), - whl_file_label: repo_label(WHEEL_FILE), - } + # If the package is in a group but has no deps, we still need the labels to point at + # the srcs targets so that the group implementation can use them. + aliases[whl_file_label] = repo_label(WHEEL_FILE) + for extra in extras_targets: + aliases[_name(extra, PY_LIBRARY_PUBLIC_LABEL)] = repo_label(PY_SRCS_LABEL) for alias, actual in aliases.items(): native.alias( @@ -134,37 +154,53 @@ def whl_library_deps_targets( # If there are extras, then they will be visible in requires_dist. return - package_deps = _parse_requires_dist( - name = metadata_name, - requires_dist = requires_dist, - excludes = group_deps, - extras = extras, - include = include, - ) + # The settings for the deps that are included conditionally keyed by the + # dep and the expression. + settings = {} + + def _pkg_label(dep): + dep_name, _, extra = dep.partition("[") + if not extra: + return dep_template.format(name = dep_name, target = PY_LIBRARY_PUBLIC_LABEL) - _config_settings( - dependencies_with_markers = package_deps.deps_select, - rules = rules, - visibility = ["//visibility:private"], - ) + return dep_template.format(name = dep_name, target = extra_label(extra.rstrip("]"))) + + whl_label = lambda dep: dep_template.format(name = dep, target = WHEEL_FILE_PUBLIC_LABEL) if hasattr(native, "filegroup"): # We include the whl file as srcs so that `$(location :whl)` expands to the whl file. - # The transitive dependencies are available via the `data` attribute. + # The transitive dependencies are available via the `data` attribute. The extras + # are not used for the whl targets, so only the packages without extras can be + # members of the group for the whl targets. + whl_deps = deps( + name = self_name, + requires_dist = requires_dist, + include = include, + excludes = members if is_member("") else [], + ) native.filegroup( name = whl_file_label, srcs = [repo_label(WHEEL_FILE)], data = _deps( deps = [], - package_deps = package_deps, - tmpl = dep_template.format(name = "{}", target = WHEEL_FILE_PUBLIC_LABEL), + package_deps = whl_deps, + conditions = _conditions(settings, whl_deps), + label = whl_label, ), - visibility = impl_vis, + visibility = _visibility(""), ) - if hasattr(rules, "py_library"): + for extra in extras_targets if hasattr(rules, "py_library") else []: + package_deps = deps( + name = self_name, + requires_dist = requires_dist, + include = include, + excludes = members if is_member(extra) else [], + extras = [extra] if extra else [], + keep_extras = True, + ) rules.py_library( - name = py_library_label, + name = _name(extra, PY_LIBRARY_PUBLIC_LABEL), # We include as srcs to ensure that the (locations :pkg) works as expected. srcs = [repo_label(PY_SRCS_LABEL)], deps = _deps( @@ -173,52 +209,48 @@ def whl_library_deps_targets( # `pyi_srcs`, see `tests/base_rules/py_library`. deps = [repo_label(PY_SRCS_LABEL)], package_deps = package_deps, - tmpl = dep_template.format(name = "{}", target = PY_LIBRARY_PUBLIC_LABEL), + conditions = _conditions(settings, package_deps), + label = _pkg_label, ), # Disable precompilation on this wrapper target to prevent duplicate # pyc generation; the underlying PY_SRCS_LABEL target handles it. precompile = "disabled", tags = tags, - visibility = impl_vis, + visibility = _visibility(extra), ) -def _parse_requires_dist( - *, - name, - requires_dist, - excludes, - include, - extras): - return deps( - name = normalize_name(name), - requires_dist = requires_dist, - excludes = excludes, - include = include, - extras = extras, - ) - -def _config_settings(dependencies_with_markers, rules, **kwargs): - """Generate config settings for the targets. - - Args: - dependencies_with_markers: {type}`dict[str, str]` The markers to evaluate by - each dep. - rules: used for testing - **kwargs: Extra kwargs to pass to the rule. - """ - for dep, expression in dependencies_with_markers.items(): + for (_, expression), name in settings.items(): rules.env_marker_setting( - name = "include_{}".format(dep), + name = name, expression = expression, - **kwargs + visibility = ["//visibility:private"], ) -def _deps(deps, package_deps, tmpl): - deps = [] + deps + [tmpl.format(d) for d in sorted(package_deps.deps)] +def _conditions(settings, package_deps): + return { + dep: _setting_name(settings, dep, expression) + for dep, expression in package_deps.deps_select.items() + } + +def _setting_name(settings, dep, expression): + """Return a unique name for the setting evaluating the expression for the dep. + + The first expression for a dep is named `include_` and any different + expressions for the same dep get a numeric suffix. + """ + key = (dep, expression) + if key not in settings: + n = len([d for d, _ in settings if d == dep]) + settings[key] = "include_{}".format(dep) + ("_{}".format(n + 1) if n else "") + + return settings[key] + +def _deps(deps, package_deps, conditions, label): + deps = [] + deps + [label(d) for d in sorted(package_deps.deps)] for dep in package_deps.deps_select: deps = deps + select({ - ":is_include_{}_true".format(dep): [tmpl.format(dep)], + ":is_{}_true".format(conditions[dep]): [label(dep)], "//conditions:default": [], }) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index a1cc727460..50be70c532 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -26,7 +26,7 @@ def whl_library_targets( whl_file = None, metadata_name = "", requires_dist = [], - extras = [], + provides_extra = [], entry_points = {}, include = [], group_deps = [], @@ -55,7 +55,7 @@ def whl_library_targets( group rules which wrap this library and its fellows together. requires_dist: {type}`list[str]` The list of `Requires-Dist` values from the whl `METADATA`. - extras: {type}`list[str]` The list of requested extras. This essentially includes extra transitive dependencies in the final targets depending on the wheel `METADATA`. + provides_extra: {type}`list[str]` The list of extras that the package provides. entry_points: {type}`list[dict]` A list of parsed entry point definitions. include: {type}`list[str]` The list of packages to include. group_name: {type}`str | None` name of the dependency group (if any). @@ -97,7 +97,7 @@ def whl_library_targets( dep_template = dep_template, # only needed if requires_dist or group_name is present group_deps = group_deps, # only needed if group_name is present group_name = group_name, # must specify group_deps together - extras = extras, # only needed if requires_dist is present + provides_extra = provides_extra, include = include, # only needed if requires_dist is present repo = None, # set aliases in the same repo aliases = {}, diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index 2d54bd34db..4343a7db1a 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -308,7 +308,6 @@ def _test_deps_and_aliases(env): want_deps_targets = { "aliases": {}, "dep_template": "//{name}:{target}", - "extras": [], "group_deps": ["bar", "foo"], "group_name": "my_group", "include": [], diff --git a/tests/pypi/whl_library/BUILD.bazel b/tests/pypi/whl_library/BUILD.bazel index cade0d2b8e..427780c9f2 100644 --- a/tests/pypi/whl_library/BUILD.bazel +++ b/tests/pypi/whl_library/BUILD.bazel @@ -6,6 +6,6 @@ py_test( srcs = ["whl_library_extras_test.py"], target_compatible_with = SUPPORTS_BZLMOD, deps = [ - "@whl_library_extras_direct_dep//:pkg", + "@whl_library_extras_direct_dep//:[optional]", ], ) diff --git a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl index e8659ff642..788bf44f7f 100644 --- a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl +++ b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl @@ -33,7 +33,6 @@ def _test_whl_library_deps_targets(env): # Overrides for testing repo = None, aliases = None, - extras = [], native = struct( filegroup = lambda **kwargs: filegroup_calls.append(kwargs), alias = lambda **kwargs: None, @@ -105,7 +104,6 @@ def _test_whl_library_deps_targets_no_deps(env): group_name = "qux", repo = None, aliases = {}, - extras = [], native = struct( filegroup = lambda **kwargs: filegroup_calls.append(kwargs), alias = lambda **kwargs: alias_calls.append(kwargs), @@ -139,6 +137,270 @@ def _test_whl_library_deps_targets_no_deps(env): _tests.append(_test_whl_library_deps_targets_no_deps) +def _test_extras(env): + filegroup_calls = [] + py_library_calls = [] + env_marker_setting_calls = [] + + whl_library_deps_targets( + name = "foo-0-py3-none-any.whl", + metadata_name = "Foo", + dep_template = "@pypi//{name}:{target}", + requires_dist = [ + "bar", + "grpc; extra == 'grpc'", + "aiohttp; extra == 'async' and python_version < '3.12'", + "uvloop; extra == 'async'", + "foo[grpc]; extra == 'all'", + ], + provides_extra = ["async", "grpc", "all"], + include = ["bar", "grpc", "aiohttp", "uvloop"], + repo = None, + aliases = {}, + native = struct( + filegroup = lambda **kwargs: filegroup_calls.append(kwargs), + alias = lambda **kwargs: None, + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + env_marker_setting = lambda **kwargs: env_marker_setting_calls.append(kwargs), + ), + ) + + # The `pkg` target has no extras because none were requested and there is + # a target for each extra that the package provides. The `whl` target does + # not use the extras. + env.expect.that_collection([ + (c["name"], c["deps"]) + for c in py_library_calls + ]).contains_exactly([ + ("pkg", ["srcs", "@pypi//bar:pkg"]), + ("[all]", ["srcs", "@pypi//bar:pkg", "@pypi//grpc:pkg"]), + ("[async]", ["srcs", "@pypi//bar:pkg", "@pypi//uvloop:pkg"] + select({ + ":is_include_aiohttp_true": ["@pypi//aiohttp:pkg"], + "//conditions:default": [], + })), + ("[grpc]", ["srcs", "@pypi//bar:pkg", "@pypi//grpc:pkg"]), + ]) + env.expect.that_collection([ + (c["name"], c["data"]) + for c in filegroup_calls + ]).contains_exactly([ + ("whl", ["@pypi//bar:whl"]), + ]) + env.expect.that_collection(env_marker_setting_calls).contains_exactly([ + { + "name": "include_aiohttp", + "expression": "python_version < \"3.12\"", + "visibility": ["//visibility:private"], + }, + ]) # buildifier: @unsorted-dict-items + +_tests.append(_test_extras) + +def _test_extras_with_different_markers(env): + py_library_calls = [] + env_marker_setting_calls = [] + + whl_library_deps_targets( + name = "foo-0-py3-none-any.whl", + metadata_name = "foo", + dep_template = "@pypi//{name}:{target}", + requires_dist = [ + "bar; sys_platform == 'win32'", + "bar; extra == 'a' and sys_platform == 'linux'", + ], + provides_extra = ["a"], + repo = None, + aliases = {}, + native = struct( + filegroup = lambda **kwargs: None, + alias = lambda **kwargs: None, + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + env_marker_setting = lambda **kwargs: env_marker_setting_calls.append(kwargs), + ), + ) + + env.expect.that_collection([ + (c["name"], c["deps"]) + for c in py_library_calls + ]).contains_exactly([ + ("pkg", ["srcs"] + select({ + ":is_include_bar_true": ["@pypi//bar:pkg"], + "//conditions:default": [], + })), + ("[a]", ["srcs"] + select({ + ":is_include_bar_2_true": ["@pypi//bar:pkg"], + "//conditions:default": [], + })), + ]) + env.expect.that_collection([ + (c["name"], c["expression"]) + for c in env_marker_setting_calls + ]).contains_exactly([ + ("include_bar", "sys_platform == \"win32\""), + ("include_bar_2", "(sys_platform == \"linux\") or (sys_platform == \"win32\")"), + ]) + +_tests.append(_test_extras_with_different_markers) + +def _test_extras_no_deps(env): + alias_calls = [] + + whl_library_deps_targets( + name = "foo-0-py3-none-any.whl", + metadata_name = "foo", + dep_template = "@pypi//{name}:{target}", + requires_dist = [], + provides_extra = ["A"], + repo = None, + aliases = {}, + native = struct( + alias = lambda **kwargs: alias_calls.append(kwargs), + ), + rules = struct(), + ) + + env.expect.that_collection([ + (c["name"], c["actual"]) + for c in alias_calls + ]).contains_exactly([ + ("pkg", "srcs"), + ("whl", "whl_file"), + ("[a]", "srcs"), + ]) + +_tests.append(_test_extras_no_deps) + +def _test_extras_in_group(env): + py_library_calls = [] + filegroup_calls = [] + + whl_library_deps_targets( + name = "foo-0-py3-none-any.whl", + metadata_name = "foo", + dep_template = "@pypi//{name}:{target}", + requires_dist = ["baz", "bar; extra == 'a'", "qux; extra == 'a'"], + provides_extra = ["a"], + group_name = "group", + # Only the `a` extra is in a cycle with `qux`. + group_deps = ["Foo[A]", "qux"], + repo = None, + aliases = {}, + native = struct( + filegroup = lambda **kwargs: filegroup_calls.append(kwargs), + alias = lambda **kwargs: None, + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + env_marker_setting = lambda **kwargs: None, + ), + ) + + # Only the member targets exclude the other members and the targets are + # visible to the hub that creates the group library. + env.expect.that_collection([ + (c["name"], c["deps"], c["visibility"]) + for c in py_library_calls + ]).contains_exactly([ + ("pkg", ["srcs", "@pypi//baz:pkg"], ["@pypi//:__subpackages__"]), + ("[a]", ["srcs", "@pypi//bar:pkg", "@pypi//baz:pkg"], ["@pypi//:__subpackages__"]), + ]) + env.expect.that_collection([ + (c["name"], c["data"]) + for c in filegroup_calls + ]).contains_exactly([ + ("whl", ["@pypi//baz:whl"]), + ]) + +_tests.append(_test_extras_in_group) + +def _test_extras_in_group_legacy(env): + alias_calls = [] + py_library_calls = [] + filegroup_calls = [] + + whl_library_deps_targets( + name = "foo-0-py3-none-any.whl", + metadata_name = "foo", + dep_template = "@pip_{name}//:{target}", + requires_dist = ["qux", "bar; extra == 'a'"], + provides_extra = ["a", "b"], + group_name = "group", + group_deps = ["foo", "foo[a]", "qux"], + repo = None, + aliases = {}, + native = struct( + filegroup = lambda **kwargs: filegroup_calls.append(kwargs), + alias = lambda **kwargs: alias_calls.append(kwargs), + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + env_marker_setting = lambda **kwargs: None, + ), + ) + + # The member targets are implementation targets and the public targets + # point to the group library in the config repo. + env.expect.that_collection([ + (c["name"], c["actual"]) + for c in alias_calls + ]).contains_exactly([ + ("pkg", "@pip__config//_groups:group_pkg"), + ("[a]", "@pip__config//_groups:group_pkg"), + ("whl", "@pip__config//_groups:group_whl"), + ]) + env.expect.that_collection([ + (c["name"], c["deps"], c["visibility"]) + for c in py_library_calls + ]).contains_exactly([ + ("_pkg", ["srcs"], ["@pip__config//_groups:__pkg__"]), + ("_[a]", ["srcs", "@pip_bar//:pkg"], ["@pip__config//_groups:__pkg__"]), + ("[b]", ["srcs", "@pip_qux//:pkg"], ["//visibility:public"]), + ]) + env.expect.that_collection([ + (c["name"], c["data"], c["visibility"]) + for c in filegroup_calls + ]).contains_exactly([ + ("_whl", [], ["@pip__config//_groups:__pkg__"]), + ]) + +_tests.append(_test_extras_in_group_legacy) + +def _test_deps_with_extras_without_hub_info(env): + py_library_calls = [] + + # When the dependencies refer to the whl repositories directly, then there + # is no information about the available extras and the targets with the + # extras in the dependency repositories are used. + whl_library_deps_targets( + name = "foo-0-py3-none-any.whl", + metadata_name = "foo", + dep_template = "@pypi_{name}//:{target}", + requires_dist = ["bar[socks]"], + repo = None, + aliases = {}, + native = struct( + filegroup = lambda **kwargs: None, + alias = lambda **kwargs: None, + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + env_marker_setting = lambda **kwargs: None, + ), + ) + + env.expect.that_collection([ + (c["name"], c["deps"]) + for c in py_library_calls + ]).contains_exactly([ + ("pkg", ["srcs", "@pypi_bar//:[socks]"]), + ]) + +_tests.append(_test_deps_with_extras_without_hub_info) + def whl_library_deps_targets_test_suite(name): """create the test suite. From ace0065e66d321e512acbb6871d0abc69f720212 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:53:21 +0900 Subject: [PATCH 12/32] feat(pypi): record the requested extras in whl_config_setting --- python/private/pypi/hub_repository.bzl | 2 ++ python/private/pypi/render_pkg_aliases.bzl | 3 ++- python/private/pypi/whl_config_setting.bzl | 5 ++++- .../render_pkg_aliases_test.bzl | 22 +++++++++++++++++++ 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index f915aa1c77..27b324f465 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -142,6 +142,8 @@ def whl_config_settings_to_json(repo_mapping): def _whl_config_setting_dict(a): ret = {} + if a.extras: + ret["extras"] = a.extras if a.target_platforms: ret["target_platforms"] = a.target_platforms if a.version: diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 0a1c328491..271a12ddfb 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -46,10 +46,11 @@ def _repr_dict(*, value_repr = repr, **kwargs): return {k: value_repr(v) for k, v in kwargs.items() if v} def _repr_config_setting(alias): - if alias.target_platforms: + if alias.target_platforms or alias.extras: return render.call( "whl_config_setting", **_repr_dict( + extras = alias.extras, target_platforms = alias.target_platforms, version = alias.version, ) diff --git a/python/private/pypi/whl_config_setting.bzl b/python/private/pypi/whl_config_setting.bzl index 1d868b1b65..7c2c595770 100644 --- a/python/private/pypi/whl_config_setting.bzl +++ b/python/private/pypi/whl_config_setting.bzl @@ -14,7 +14,7 @@ "A small function to create an alias for a whl distribution" -def whl_config_setting(*, version = None, target_platforms = None): +def whl_config_setting(*, version = None, target_platforms = None, extras = None): """The bzl_packages value used by by the render_pkg_aliases function. This contains the minimum amount of information required to generate correct @@ -27,6 +27,8 @@ def whl_config_setting(*, version = None, target_platforms = None): is no match found during a select. target_platforms: {type}`list[str] | None` the list of target_platforms for this distribution. + extras: {type}`list[str] | None` the extras of the distribution that + are requested on the target platforms. Returns: a struct with the validated and parsed values. @@ -55,4 +57,5 @@ def whl_config_setting(*, version = None, target_platforms = None): # Make the struct hashable target_platforms = tuple(target_platforms) if target_platforms else None, version = version, + extras = tuple(sorted(extras)) if extras else None, ) diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index 9114f59279..807c4728fd 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -247,6 +247,28 @@ def _test_get_flag_versions_from_alias_target_platforms(env): _tests.append(_test_get_flag_versions_from_alias_target_platforms) +def _test_aliases_with_extras(env): + actual = render_pkg_aliases( + aliases = { + "foo": { + whl_config_setting(version = "3.2", extras = ["b", "a"]): "pypi_32_foo", + }, + }, + ) + + env.expect.that_str(actual["foo/BUILD.bazel"]).contains("""\ +pkg_aliases( + name = "foo", + actual = { + whl_config_setting( + extras = ("a", "b"), + version = "3.2", + ): "pypi_32_foo", + }, +)""") + +_tests.append(_test_aliases_with_extras) + def render_pkg_aliases_test_suite(name): """Create the test suite. From c0bd9cc3abb789636f5d8fb4334b37ddda008a45 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:54:03 +0900 Subject: [PATCH 13/32] feat(pypi): add the per-platform extra targets to the hub packages --- python/private/pypi/BUILD.bazel | 1 + python/private/pypi/pkg_aliases.bzl | 150 ++++++++++++-- tests/pypi/pkg_aliases/pkg_aliases_test.bzl | 216 +++++++++++++++++++- 3 files changed, 349 insertions(+), 18 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 1e33b366db..8cee5b5210 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -353,6 +353,7 @@ bzl_library( deps = [ ":labels", ":whl_library_deps_targets", + "//python:py_library", "//python/private:common_labels", "//python/private:text_util", "@bazel_skylib//lib:selects", diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index 308d9a21fa..56157ca9a6 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -32,6 +32,7 @@ setting maps to and their precedence, refer to documentation on that page. """ load("@bazel_skylib//lib:selects.bzl", "selects") +load("//python:py_library.bzl", "py_library") load("//python/private:common_labels.bzl", "labels") load("//python/private:text_util.bzl", "render") load( @@ -39,12 +40,12 @@ load( "DATA_LABEL", "DIST_INFO_LABEL", "EXTRACTED_WHEEL_FILES", - "PY_LIBRARY_IMPL_LABEL", "PY_LIBRARY_PUBLIC_LABEL", "PY_SRCS_LABEL", "WHEEL_FILE", "WHEEL_FILE_IMPL_LABEL", "WHEEL_FILE_PUBLIC_LABEL", + "extra_label", ) load(":whl_library_deps_targets.bzl", "whl_library_deps_targets") @@ -75,6 +76,7 @@ _LABEL_NONE = labels.NONE _LABEL_CURRENT_CONFIG = Label("//python/config_settings:current_config") _LABEL_CURRENT_CONFIG_NO_MATCH = Label("//python/config_settings:is_not_matching_current_config") _INCOMPATIBLE = "_no_matching_repository" +_DEFAULT = "//conditions:default" def pkg_aliases( *, @@ -83,6 +85,7 @@ def pkg_aliases( group_name = None, extra_aliases = None, requires_dist = [], + extras = [], include = [], group_deps = [], **kwargs): @@ -102,12 +105,20 @@ def pkg_aliases( extra_aliases: {type}`list[str]` The extra aliases to be created. requires_dist: {type}`list[str]` The list of dependencies. Comes from METADATA or a lock file. + extras: {type}`list[str]` The requested extras of the package when + `actual` is the name of a single repo. When `actual` is a dict, the + requested extras come from the {obj}`whl_config_setting` keys. include: {type}`list[str]` The subset of packages to include. Comes from `//:config.bzl#packages` **kwargs: extra kwargs to pass to {bzl:obj}`get_config_settings`. """ + + # The package without extras is the member of the group. + group_extras = [""] if group_name else [] + _native = kwargs.pop("native", native) rules = kwargs.pop("rules", struct( + py_library = py_library, whl_library_deps_targets = whl_library_deps_targets, )) select = kwargs.pop("select", selects.with_or) @@ -136,16 +147,11 @@ def pkg_aliases( ) target_names = {} else: - if group_name: - py_library_target = PY_LIBRARY_IMPL_LABEL - whl_target = WHEEL_FILE_IMPL_LABEL - else: - py_library_target = PY_LIBRARY_PUBLIC_LABEL - whl_target = WHEEL_FILE_PUBLIC_LABEL - + # The `pkg` targets are created by `_pkg_targets` below. The whl + # targets do not use the extras, so only the package without extras + # can be a member of the group. target_names = { - PY_LIBRARY_PUBLIC_LABEL: py_library_target, - WHEEL_FILE_PUBLIC_LABEL: whl_target, + WHEEL_FILE_PUBLIC_LABEL: WHEEL_FILE_IMPL_LABEL if "" in group_extras else WHEEL_FILE_PUBLIC_LABEL, } target_names = target_names | { @@ -159,6 +165,7 @@ def pkg_aliases( for x in extra_aliases or [] } + extras_by_setting = _extras_by_setting(actual, extras) actual = multiplatform_whl_aliases(aliases = actual, **kwargs) if type(actual) == type({}) and "//conditions:default" not in actual: _native.alias( @@ -212,16 +219,129 @@ def pkg_aliases( **kwargs ) - if group_name: - _native.alias( - name = PY_LIBRARY_PUBLIC_LABEL, - actual = "//_groups:{}_pkg".format(group_name), - ) + if "" in group_extras: _native.alias( name = WHEEL_FILE_PUBLIC_LABEL, actual = "//_groups:{}_whl".format(group_name), ) + if requires_dist: + if group_name: + _native.alias( + name = PY_LIBRARY_PUBLIC_LABEL, + actual = "//_groups:{}_pkg".format(group_name), + ) + else: + _pkg_targets( + actual = actual, + extras_by_setting = extras_by_setting, + group_name = group_name, + group_extras = group_extras, + native = _native, + rules = rules, + select = select, + ) + +def _extras_by_setting(aliases, extras): + """Return the requested extras for each config setting label.""" + if type(aliases) == type(""): + # A single repo is used for all of the platforms, e.g. in `WORKSPACE`. + return {_DEFAULT: tuple(sorted(extras))} + + return { + "//_config" + setting: alias.extras + for alias in aliases + if type(alias) == "struct" and alias.extras + for setting in get_config_settings( + target_platforms = alias.target_platforms, + python_version = alias.version, + ) + } + +def _pkg_targets(*, actual, extras_by_setting, group_name, group_extras, native, rules, select): + """Create the `pkg`, `[]` and `[]` targets. + + The whl repositories have an `[]` target for every extra that the + package provides and the hub exposes only the extras requested on each + platform. The `pkg` target includes the extras requested on each platform. + + Args: + actual: {type}`str | dict[str, str]` the repo name or the config + setting labels to repo names. + extras_by_setting: {type}`dict[str, tuple[str]]` the config setting + labels to the extras requested on that platform. + group_name: {type}`str | None` the group name that the pkg belongs to. + group_extras: {type}`list[str]` the members of the group in this + package, where an empty string is the package without extras. + native: the native struct. + rules: the rules struct. + select: the select function. + """ + if type(actual) == type(""): + actual = {_DEFAULT: actual} + + def _select(values): + return values[_DEFAULT] if values.keys() == [_DEFAULT] else select(values) + + def _target(repo, name): + return repo if repo == _INCOMPATIBLE else "@{}//:{}".format(repo, name) + + group_label = "//_groups:{}_{}".format(group_name, PY_LIBRARY_PUBLIC_LABEL) + + def _hub_label(extra): + return group_label if extra in group_extras else ":" + extra_label(extra) + + all_extras = sorted({ + extra: None + for extras in list(extras_by_setting.values()) + [group_extras] + for extra in extras + if extra + }) + for extra in [""] + all_extras: + # The platforms that do not request the extra get the package without + # extras. + targets = _select({ + setting: _target( + repo, + extra_label(extra) if extra in extras_by_setting.get(setting, ()) else PY_LIBRARY_PUBLIC_LABEL, + ) + for setting, repo in actual.items() + }) + + if extra not in group_extras: + native.alias(name = extra_label(extra), actual = targets) + continue + + # The group library uses the implementation targets of the members. + native.alias( + name = "_" + (extra_label(extra) if extra else PY_LIBRARY_PUBLIC_LABEL), + actual = targets, + visibility = ["//_groups:__subpackages__"], + ) + native.alias(name = extra_label(extra), actual = group_label) + + # The `[]` and `[]` targets have the no match error, so we do not + # need to handle the incompatible platforms here. + targets = { + setting: sorted({ + _hub_label(e): None + for e in extras_by_setting.get(setting, ()) or [""] + }) + for setting in actual + } + unique = {label: None for t in targets.values() for label in t} + if len(unique) == 1: + native.alias(name = PY_LIBRARY_PUBLIC_LABEL, actual = unique.keys()[0]) + elif all([len(t) == 1 for t in targets.values()]): + native.alias( + name = PY_LIBRARY_PUBLIC_LABEL, + actual = _select({setting: t[0] for setting, t in targets.items()}), + ) + else: + # Some platforms request more than one extra, so we need to combine + # the targets. + rules.py_library(name = PY_LIBRARY_PUBLIC_LABEL, deps = _select(targets)) + def multiplatform_whl_aliases( *, aliases = []): diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index 4343a7db1a..0cbbfb88d9 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -41,7 +41,8 @@ def _test_legacy_aliases(env): # buildifier: disable=unsorted-dict-items want = { "foo": ":pkg", - "pkg": "@repo//:pkg", + "pkg": ":[]", + "[]": "@repo//:pkg", "srcs": "@repo//:srcs", "whl_file": "@repo//:whl_file", "whl": "@repo//:whl", @@ -79,7 +80,8 @@ def _test_config_setting_aliases(env): # buildifier: disable=unsorted-dict-items want = { - "pkg": { + "pkg": ":[]", + "[]": { "//:my_config_setting": "@bar_baz_repo//:pkg", "//conditions:default": "_no_matching_repository", }, @@ -180,7 +182,8 @@ def _test_multiplatform_whl_aliases(env): # buildifier: disable=unsorted-dict-items want = { - "pkg": { + "pkg": ":[]", + "[]": { "//:my_config_setting": "@bzlmod_repo//:pkg", "//_config:is_cp39": "@version_repo//:pkg", "//_config:is_cp39_linux_x86_64": "@version_platform_repo//:pkg", @@ -240,6 +243,10 @@ def _test_group_aliases(env): "name": "extracted_whl_files", "actual": "@repo//:extracted_whl_files", }, + { + "name": "[]", + "actual": "//_groups:my_group_pkg", + }, { "name": "pkg", "actual": "//_groups:my_group_pkg", @@ -520,6 +527,209 @@ def _test_config_settings_exist_legacy(env): _tests.append(_test_config_settings_exist_legacy) +_LINUX = "//_config:is_cp312_linux_x86_64" +_OSX = "//_config:is_cp312_osx_aarch64" +_WINDOWS = "//_config:is_cp312_windows_x86_64" + +def _extras_actual(extras_by_platform): + return { + whl_config_setting( + version = "3.12", + target_platforms = ["cp312_" + platform], + extras = extras, + ): repo + for platform, (repo, extras) in extras_by_platform.items() + } + +def _test_extras_aliases(env): + got = {} + py_library_calls = [] + pkg_aliases( + name = "foo", + actual = _extras_actual({ + "linux_x86_64": ("repo", ["bar"]), + "osx_aarch64": ("repo", ["baz"]), + "windows_x86_64": ("other_repo", []), + }), + native = struct( + alias = lambda *, name, actual, visibility = None, tags = None: got.update({name: actual}), + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + ), + select = lambda value, no_match_error = None: value, + ) + + # buildifier: disable=unsorted-dict-items + want = { + "foo": ":pkg", + "pkg": { + _LINUX: ":[bar]", + _OSX: ":[baz]", + _WINDOWS: ":[]", + # The `[]` target has the no match error. + "//conditions:default": ":[]", + }, + "whl": { + _LINUX: "@repo//:whl", + _OSX: "@repo//:whl", + _WINDOWS: "@other_repo//:whl", + "//conditions:default": "_no_matching_repository", + }, + "[]": { + _LINUX: "@repo//:pkg", + _OSX: "@repo//:pkg", + _WINDOWS: "@other_repo//:pkg", + "//conditions:default": "_no_matching_repository", + }, + "[bar]": { + _LINUX: "@repo//:[bar]", + _OSX: "@repo//:pkg", + _WINDOWS: "@other_repo//:pkg", + "//conditions:default": "_no_matching_repository", + }, + "[baz]": { + _LINUX: "@repo//:pkg", + _OSX: "@repo//:[baz]", + _WINDOWS: "@other_repo//:pkg", + "//conditions:default": "_no_matching_repository", + }, + } + env.expect.that_dict(got).contains_at_least(want) + env.expect.that_collection(py_library_calls).contains_exactly([]) + +_tests.append(_test_extras_aliases) + +def _test_extras_aliases_union(env): + got = {} + py_library_calls = [] + pkg_aliases( + name = "foo", + actual = _extras_actual({ + "linux_x86_64": ("repo", ["a", "b"]), + "osx_aarch64": ("repo", ["c"]), + "windows_x86_64": ("repo", []), + }), + native = struct( + alias = lambda *, name, actual, visibility = None, tags = None: got.update({name: actual}), + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + ), + select = lambda value, no_match_error = None: value, + ) + + # buildifier: disable=unsorted-dict-items + env.expect.that_collection(py_library_calls).contains_exactly([{ + "name": "pkg", + "deps": { + _LINUX: [":[a]", ":[b]"], + _OSX: [":[c]"], + _WINDOWS: [":[]"], + "//conditions:default": [":[]"], + }, + }]) + env.expect.that_collection(got.keys()).not_contains("pkg") + + # The whl target does not use the extras. + # buildifier: disable=unsorted-dict-items + env.expect.that_dict(got).contains_at_least({ + "whl": { + _LINUX: "@repo//:whl", + _OSX: "@repo//:whl", + _WINDOWS: "@repo//:whl", + "//conditions:default": "_no_matching_repository", + }, + }) + +_tests.append(_test_extras_aliases_union) + +def _test_extras_aliases_group(env): + got = {} + got_visibility = {} + + def _alias(*, name, actual, visibility = None, **_): + got[name] = actual + if visibility: + got_visibility[name] = visibility + + pkg_aliases( + name = "foo", + actual = _extras_actual({ + "linux_x86_64": ("repo", ["bar"]), + "osx_aarch64": ("repo", []), + }), + group_name = "group", + native = struct(alias = _alias), + select = lambda value, no_match_error = None: value, + ) + + # The package without extras is the member of the group by default, the + # `[bar]` extra is not a member of the group. + # buildifier: disable=unsorted-dict-items + env.expect.that_dict(got).contains_at_least({ + "pkg": { + _LINUX: ":[bar]", + _OSX: "//_groups:group_pkg", + "//conditions:default": "//_groups:group_pkg", + }, + "[]": "//_groups:group_pkg", + "_pkg": { + _LINUX: "@repo//:pkg", + _OSX: "@repo//:pkg", + "//conditions:default": "_no_matching_repository", + }, + "[bar]": { + _LINUX: "@repo//:[bar]", + _OSX: "@repo//:pkg", + "//conditions:default": "_no_matching_repository", + }, + "whl": "//_groups:group_whl", + "_whl": { + _LINUX: "@repo//:whl", + _OSX: "@repo//:whl", + "//conditions:default": "_no_matching_repository", + }, + }) + env.expect.that_dict(got_visibility).contains_at_least({ + "_pkg": ["//_groups:__subpackages__"], + "_whl": ["//_groups:__subpackages__"], + }) + +_tests.append(_test_extras_aliases_group) + +def _test_extras_aliases_single_repo(env): + got = {} + py_library_calls = [] + + # This is how `WORKSPACE` uses a single repo for each package. + pkg_aliases( + name = "foo", + actual = "repo", + extras = ["b", "a"], + native = struct( + alias = lambda *, name, actual, visibility = None, tags = None: got.update({name: actual}), + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + ), + select = lambda value, no_match_error = None: value, + ) + + # buildifier: disable=unsorted-dict-items + env.expect.that_dict(got).contains_at_least({ + "[]": "@repo//:pkg", + "[a]": "@repo//:[a]", + "[b]": "@repo//:[b]", + "whl": "@repo//:whl", + }) + env.expect.that_collection(py_library_calls).contains_exactly([{ + "deps": [":[a]", ":[b]"], + "name": "pkg", + }]) + +_tests.append(_test_extras_aliases_single_repo) + def pkg_aliases_test_suite(name): """Create the test suite. From 998146caaf9d1b7e97cb1e80111378a90cccbfc1 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:54:32 +0900 Subject: [PATCH 14/32] feat(pypi): use a single whl repository for the different extras of a distribution --- python/private/pypi/BUILD.bazel | 1 + python/private/pypi/hub_builder.bzl | 109 ++++++++++++++++--- tests/pypi/hub_builder/hub_builder_tests.bzl | 108 ++++++++++++------ 3 files changed, 167 insertions(+), 51 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 8cee5b5210..87885b18cc 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -185,6 +185,7 @@ bzl_library( ":parse_requirements", ":pep508_env", ":pep508_evaluate", + ":pep508_requirement", ":python_tag", ":requirements_files_by_platform", ":whl_config_setting", diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index ef6a7247c1..b6809660fd 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -12,6 +12,7 @@ load(":hash.bzl", "hash") load(":parse_requirements.bzl", "parse_requirements") load(":pep508_env.bzl", "env") load(":pep508_evaluate.bzl", "evaluate") +load(":pep508_requirement.bzl", "requirement", "strip_extras") load(":python_tag.bzl", "python_tag") load(":requirements_files_by_platform.bzl", "requirements_files_by_platform") load(":whl_config_setting.bzl", "whl_config_setting") @@ -362,16 +363,16 @@ def _add_whl_library(self, *, python_version, whl, repo): self._whl_libraries[repo_name] = repo.args mapping = self._whl_map.setdefault(whl.name, {}) - if repo.config_setting in mapping and mapping[repo.config_setting] != repo_name: - fail( - "attempting to override an existing repo '{}' for config setting '{}' with a new repo '{}'".format( - mapping[repo.config_setting], - repo.config_setting, - repo_name, - ), - ) - else: - mapping[repo.config_setting] = repo_name + for config_setting in repo.config_settings: + if config_setting in mapping and mapping[config_setting] != repo_name: + fail( + "attempting to override an existing repo '{}' for config setting '{}' with a new repo '{}'".format( + mapping[config_setting], + config_setting, + repo_name, + ), + ) + mapping[config_setting] = repo_name ### end of setters, below we have various functions to implement the public methods @@ -606,7 +607,8 @@ def _create_whl_repos( whl = whl, whl_modifications = whl_modifications, ) - for src in whl.srcs: + srcs = _merge_extras(whl.srcs) + for src in srcs: repo = _whl_repo( src = src, index_url = whl.index_url, @@ -616,7 +618,11 @@ def _create_whl_repos( use_downloader = src.url and _use_downloader(self, python_version, whl.name), auth_patterns = self._config.auth_patterns or pip_attr.auth_patterns, python_version = _major_minor_version(python_version), - is_multiple_versions = whl.is_multiple_versions, + # Different extras for the same distribution are not different versions. + is_multiple_versions = whl.is_multiple_versions and len({ + s.requirement_line: None + for s in srcs + }) > 1, interpreter = interpreter, enable_pipstar_extract = enable_pipstar_extract, ) @@ -627,6 +633,77 @@ def _create_whl_repos( repo = repo, ) +def _merge_extras(srcs): + """Merge the sources that differ only in the requested extras. + + The whl repositories create targets for all of the extras that a package + provides, so the extras are not passed to them and the same distribution + requested with different extras on different platforms uses a single whl + repository. The extras of each platform are recorded so that the hub can + expose only the requested extras for each platform. + + Args: + srcs: {type}`list[struct]` the sources returned by `parse_requirements`. + + Returns: + The list of sources without the extras in the requirement line and + with an extra `extras_by_platform` attribute. + """ + merged = {} + for src in srcs: + line = strip_extras(src.requirement_line) + entry = merged.setdefault( + (src.filename, src.url, line, tuple(src.extra_pip_args)), + struct( + src = src, + line = line, + extras_by_platform = {}, + target_platforms = [], + ), + ) + extras = sorted(requirement(src.requirement_line).extras) + for p in src.target_platforms: + if p not in entry.target_platforms: + entry.target_platforms.append(p) + entry.extras_by_platform[p] = extras + + return [ + struct( + distribution = entry.src.distribution, + extra_pip_args = entry.src.extra_pip_args, + requirement_line = entry.line, + target_platforms = entry.target_platforms, + extras_by_platform = entry.extras_by_platform, + filename = entry.src.filename, + digest = entry.src.digest, + url = entry.src.url, + yanked = entry.src.yanked, + ) + for entry in merged.values() + ] + +def _config_settings(*, src, version, target_platforms): + """Return the config settings for each set of extras requested by the platforms.""" + plats_by_extras = {} + for p in src.target_platforms: + plats_by_extras.setdefault(tuple(src.extras_by_platform.get(p, [])), []).append(p) + + if len(plats_by_extras) == 1: + return [whl_config_setting( + version = version, + target_platforms = target_platforms or None, + extras = plats_by_extras.keys()[0], + )] + + return [ + whl_config_setting( + version = version, + target_platforms = plats, + extras = extras, + ) + for extras, plats in plats_by_extras.items() + ] + def _common_args(self, module_ctx, *, pip_attr): # Construct args separately so that the lock file can be smaller and does not include unused # attrs. @@ -725,9 +802,10 @@ def _whl_repo( *target_platforms ), args = args, - config_setting = whl_config_setting( + config_settings = _config_settings( + src = src, version = python_version, - target_platforms = target_platforms or None, + target_platforms = target_platforms, ), ) @@ -758,7 +836,8 @@ def _whl_repo( return struct( repo_name = whl_repo_name(src.filename, src.digest, *target_platforms), args = args, - config_setting = whl_config_setting( + config_settings = _config_settings( + src = src, version = python_version, target_platforms = src.target_platforms, ), diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index 073ccc0f58..216dca27f7 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -176,6 +176,51 @@ def _test_uv_lock_only(env): _tests.append(_test_uv_lock_only) +def _test_uv_lock_extras(env): + builder = hub_builder(env) + builder.pip_parse( + _mock_mctx( + os_name = "osx", + arch_name = "aarch64", + mock_files = { + "uv.lock": """{"package":[{"name":"simple","provides-extras":["async-rest","grpc"],"source":{"registry":"https://pypi.org/simple"},"version":"0.0.1","wheels":[{"hash":"sha256:deadbeef","url":"https://files.pythonhosted.org/packages/simple-0.0.1-py3-none-any.whl"}]}]}""", + }, + ), + _parse( + hub_name = "pypi", + python_version = "3.15", + requirements_lock = None, + uv_lock = "uv.lock", + target_platforms = ["osx_aarch64", "linux_x86_64"], + ), + ) + pypi = builder.build() + + pypi.whl_map().contains_exactly({ + "simple": { + "pypi_315_simple_py3_none_any_deadbeef": [ + whl_config_setting( + version = "3.15", + target_platforms = ["cp315_linux_x86_64", "cp315_osx_aarch64"], + extras = ["async_rest", "grpc"], + ), + ], + }, + }) + pypi.whl_libraries().contains_at_least({ + "pypi_315_simple_py3_none_any_deadbeef": { + "config_load": "@pypi//:config.bzl", + "dep_template": "@pypi//{name}:{target}", + "filename": "simple-0.0.1-py3-none-any.whl", + "index_url": "https://pypi.org/simple/simple", + "integrity": "sha256-3q2+7w==", + "requirement": "simple==0.0.1", + "urls": ["https://files.pythonhosted.org/packages/simple-0.0.1-py3-none-any.whl"], + }, + }) + +_tests.append(_test_uv_lock_extras) + def _test_simple_multiple_requirements(env): sub_tests = { ("osx", "aarch64"): "simple==0.0.2 --hash=sha256:deadb00f", @@ -224,10 +269,10 @@ _tests.append(_test_simple_multiple_requirements) def _test_simple_extras_vs_no_extras(env): sub_tests = { - ("osx", "aarch64"): "simple[foo]==0.0.1 --hash=sha256:deadbeef", - ("windows", "aarch64"): "simple==0.0.1 --hash=sha256:deadbeef", + ("osx", "aarch64"): ("simple==0.0.1 --hash=sha256:deadbeef", ["foo"]), + ("windows", "aarch64"): ("simple==0.0.1 --hash=sha256:deadbeef", None), } - for (host_os, host_arch), want_requirement in sub_tests.items(): + for (host_os, host_arch), (want_requirement, want_extras) in sub_tests.items(): builder = hub_builder(env) builder.pip_parse( mocks.mctx( @@ -252,7 +297,7 @@ def _test_simple_extras_vs_no_extras(env): pypi.whl_map().contains_exactly({ "simple": { "pypi_315_simple": [ - whl_config_setting(version = "3.15"), + whl_config_setting(version = "3.15", extras = want_extras), ], }, }) @@ -314,37 +359,30 @@ def _test_simple_extras_vs_no_extras_simpleapi(env): pypi.exposed_packages().contains_exactly(["simple"]) pypi.group_map().contains_exactly({}) + + # The same wheel is used on both platforms, so a single repository has the + # extras for all of the platforms and the hub selects the extras per platform. pypi.whl_map().contains_exactly({ "simple": { - "pypi_315_simple_py3_none_any_deadbeef_osx_aarch64": [ + "pypi_315_simple_py3_none_any_deadbeef": [ whl_config_setting( target_platforms = [ - "cp315_osx_aarch64", + "cp315_windows_aarch64", ], version = "3.15", ), - ], - "pypi_315_simple_py3_none_any_deadbeef_windows_aarch64": [ whl_config_setting( target_platforms = [ - "cp315_windows_aarch64", + "cp315_osx_aarch64", ], version = "3.15", + extras = ["foo"], ), ], }, }) pypi.whl_libraries().contains_exactly({ - "pypi_315_simple_py3_none_any_deadbeef_osx_aarch64": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", - "filename": "simple-0.0.1-py3-none-any.whl", - "index_url": "https://example.com/simple/", - "integrity": "sha256-3q2+7w==", - "requirement": "simple[foo]==0.0.1", - "urls": ["/simple-0.0.1-py3-none-any.whl"], - }, - "pypi_315_simple_py3_none_any_deadbeef_windows_aarch64": { + "pypi_315_simple_py3_none_any_deadbeef": { "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "filename": "simple-0.0.1-py3-none-any.whl", @@ -1416,10 +1454,10 @@ _tests.append(_test_simple_get_index) def _test_optimum_sys_platform_extra(env): sub_tests = { - ("osx", "aarch64"): "optimum[onnxruntime]==1.17.1", - ("linux", "aarch64"): "optimum[onnxruntime-gpu]==1.17.1", + ("osx", "aarch64"): ("optimum==1.17.1", ["onnxruntime"]), + ("linux", "aarch64"): ("optimum==1.17.1", ["onnxruntime_gpu"]), } - for (host_os, host_arch), want_requirement in sub_tests.items(): + for (host_os, host_arch), (want_requirement, want_extras) in sub_tests.items(): builder = hub_builder( env, ) @@ -1447,7 +1485,7 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' pypi.whl_map().contains_exactly({ "optimum": { "pypi_315_optimum": [ - whl_config_setting(version = "3.15"), + whl_config_setting(version = "3.15", extras = want_extras), ], }, }) @@ -1509,38 +1547,36 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' pypi.exposed_packages().contains_exactly(["optimum"]) pypi.group_map().contains_exactly({}) + + # The same distribution is requested with different extras on different + # platforms, so a single repository has all of the extras and the hub + # selects the extras per platform. pypi.whl_map().contains_exactly({ "optimum": { - "pypi_315_optimum_mylinuxx86_64": [ + "pypi_315_optimum": [ whl_config_setting( version = "3.15", target_platforms = [ "cp315_mylinuxx86_64", ], + extras = ["onnxruntime_gpu"], ), - ], - "pypi_315_optimum_myosxaarch64": [ whl_config_setting( version = "3.15", target_platforms = [ "cp315_myosxaarch64", ], + extras = ["onnxruntime"], ), ], }, }) pypi.whl_libraries().contains_exactly({ - "pypi_315_optimum_mylinuxx86_64": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", - "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "optimum[onnxruntime-gpu]==1.17.1", - }, - "pypi_315_optimum_myosxaarch64": { + "pypi_315_optimum": { "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "optimum[onnxruntime]==1.17.1", + "requirement": "optimum==1.17.1", }, }) pypi.extra_aliases().contains_exactly({}) @@ -1598,7 +1634,7 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' pypi.whl_map().contains_exactly({ "optimum": { "pypi_315_optimum": [ - whl_config_setting(version = "3.15"), + whl_config_setting(version = "3.15", extras = ["onnxruntime_gpu"]), ], }, }) @@ -1607,7 +1643,7 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "optimum[onnxruntime-gpu]==1.17.1", + "requirement": "optimum==1.17.1", }, }) pypi.extra_aliases().contains_exactly({}) From dfaa556ea997b8c1cbfe1593dd5c18ba22d4bd80 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:54:49 +0900 Subject: [PATCH 15/32] feat(pypi): support extras in the requirement macro and add the requirements macro --- python/private/pypi/labels.bzl | 55 +++++++++++++++++++ .../private/pypi/requirements.bzl.tmpl.bzlmod | 24 +++++++- tests/pypi/labels/BUILD.bazel | 5 ++ tests/pypi/labels/labels_tests.bzl | 53 ++++++++++++++++++ 4 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 tests/pypi/labels/BUILD.bazel create mode 100644 tests/pypi/labels/labels_tests.bzl diff --git a/python/private/pypi/labels.bzl b/python/private/pypi/labels.bzl index f6257106e9..3414d586a7 100644 --- a/python/private/pypi/labels.bzl +++ b/python/private/pypi/labels.bzl @@ -39,6 +39,61 @@ def extra_label(extra): """ return "[{}]".format(extra) +def _labels(tmpl, requirement): + name, bracket, extras = requirement.partition("[") + name = normalize_name(name.strip()) + if not bracket: + return [tmpl.format(name, PY_LIBRARY_PUBLIC_LABEL)] + + extras, _, _ = extras.partition("]") + extras = sorted([ + normalize_name(extra.strip()) + for extra in extras.split(",") + if extra.strip() + ]) + return [tmpl.format(name, extra_label(extra)) for extra in extras or [""]] + +def requirement_label(tmpl, requirement, *, fail = fail): + """Return the label for a requirement in a hub repository. + + Args: + tmpl: {type}`str` the label template with two `{}` placeholders for + the package name and the target name. + requirement: {type}`str` the name of the package with at most one + extra, e.g. `foo`, `foo[]` or `foo[bar]`. + fail: the function to call on errors, used in tests. + + Returns: + {type}`str` the label of the `pkg` target if no extras are given, the + label of the `[]` target if empty extras are given, or the label of + the `[extra]` target if one extra is given. + """ + labels = _labels(tmpl, requirement) + if len(labels) > 1: + fail("Only a single extra is supported by 'requirement', use 'requirements(\"{}\")' instead to get a list of labels.".format(requirement)) + + return labels[0] + +def requirement_labels(tmpl, *requirements): + """Return the labels for requirements in a hub repository. + + Args: + tmpl: {type}`str` the label template with two `{}` placeholders for + the package name and the target name. + *requirements: {type}`str` the names of the packages with optional + extras, e.g. `foo`, `foo[]`, `foo[bar]` or `foo[bar,baz]`. + + Returns: + {type}`list[str]` the labels for each requirement, with an `[extra]` + label for each extra of a requirement. Each `[extra]` target includes + the package. + """ + return [ + label + for requirement in requirements + for label in _labels(tmpl, requirement) + ] + def group_member(spec): """Normalize a member of a dependency group. diff --git a/python/private/pypi/requirements.bzl.tmpl.bzlmod b/python/private/pypi/requirements.bzl.tmpl.bzlmod index ba227aeb2d..1b6a6c02dd 100644 --- a/python/private/pypi/requirements.bzl.tmpl.bzlmod +++ b/python/private/pypi/requirements.bzl.tmpl.bzlmod @@ -4,6 +4,7 @@ """ load("@rules_python//python:pip.bzl", "pip_utils") +load("@rules_python//python/private/pypi:labels.bzl", "requirement_label", "requirement_labels") all_requirements = %%ALL_REQUIREMENTS%% @@ -14,7 +15,28 @@ all_whl_requirements = all_whl_requirements_by_package.values() all_data_requirements = %%ALL_DATA_REQUIREMENTS%% def requirement(name): - return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "pkg") + """Return the label for the package. + + Args: + name: The name of the package with at most one extra, e.g. `foo`, + `foo[]` or `foo[bar]`. Use `requirements` for multiple extras. + + Returns: + The label of the package. + """ + return requirement_label("%%MACRO_TMPL%%", name) + +def requirements(*names): + """Return the labels for the packages. + + Args: + *names: The names of the packages with optional extras, e.g. `foo`, + `foo[]`, `foo[bar]` or `foo[bar,baz]`. + + Returns: + The list of labels, with a label for each extra of a package. + """ + return requirement_labels("%%MACRO_TMPL%%", *names) def whl_requirement(name): return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "whl") diff --git a/tests/pypi/labels/BUILD.bazel b/tests/pypi/labels/BUILD.bazel new file mode 100644 index 0000000000..ed55e66eae --- /dev/null +++ b/tests/pypi/labels/BUILD.bazel @@ -0,0 +1,5 @@ +load(":labels_tests.bzl", "labels_test_suite") + +labels_test_suite( + name = "labels_tests", +) diff --git a/tests/pypi/labels/labels_tests.bzl b/tests/pypi/labels/labels_tests.bzl new file mode 100644 index 0000000000..b26e6db471 --- /dev/null +++ b/tests/pypi/labels/labels_tests.bzl @@ -0,0 +1,53 @@ +"" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:labels.bzl", "requirement_label", "requirement_labels") # buildifier: disable=bzl-visibility + +_tests = [] + +_TMPL = "@@pypi//{}:{}" + +def _test_requirement_label(env): + for requirement, want in { + "Foo.Bar": "@@pypi//foo_bar:pkg", + "foo[ grpc ]": "@@pypi//foo:[grpc]", + "foo[Async-Rest]": "@@pypi//foo:[async_rest]", + "foo[]": "@@pypi//foo:[]", + "foo[grpc]": "@@pypi//foo:[grpc]", + }.items(): + env.expect.where(requirement = requirement).that_str(requirement_label(_TMPL, requirement)).equals(want) + +_tests.append(_test_requirement_label) + +def _test_requirement_label_multiple_extras_fails(env): + errors = [] + requirement_label( + _TMPL, + "google-api-core[grpc,async_rest]", + fail = lambda msg: errors.append(msg), + ) + env.expect.that_collection(errors).has_size(1) + env.expect.that_str(errors[0]).contains("requirements(\"google-api-core[grpc,async_rest]\")") + +_tests.append(_test_requirement_label_multiple_extras_fails) + +def _test_requirement_labels(env): + env.expect.that_collection( + requirement_labels(_TMPL, "google-api-core[grpc, async_rest]", "foobar[baz]", "foo", "bar[]"), + ).contains_exactly([ + "@@pypi//google_api_core:[async_rest]", + "@@pypi//google_api_core:[grpc]", + "@@pypi//foobar:[baz]", + "@@pypi//foo:pkg", + "@@pypi//bar:[]", + ]).in_order() + +_tests.append(_test_requirement_labels) + +def labels_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) From 6e7744866029e968d2440fdaf29b07072892c00f Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:55:41 +0900 Subject: [PATCH 16/32] refactor(pypi): extract the failing action of missing_package_error --- python/private/pypi/missing_package.bzl | 28 ++++++++++++++----------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/python/private/pypi/missing_package.bzl b/python/private/pypi/missing_package.bzl index 46823e813b..4b750caf45 100644 --- a/python/private/pypi/missing_package.bzl +++ b/python/private/pypi/missing_package.bzl @@ -3,23 +3,14 @@ load("//python/private:py_info.bzl", "PyInfo") load("//python/private:reexports.bzl", "BuiltinPyInfo") -def _missing_package_error_impl(ctx): +def _error(ctx, message): out = ctx.actions.declare_file(ctx.label.name + ".error") - if ctx.attr.hub_name: - hub_clause = ' when building under PyPI hub "{hub}". Try adding it to the requirements of this hub (e.g. requirements_lock or requirements_by_platform in pip.parse)'.format( - hub = ctx.attr.hub_name, - ) - else: - hub_clause = ' because no default PyPI hub was configured. Try designating a default hub via pip.default(default_hub = "...") or select a hub using --@rules_python//python/config_settings:venv' - # Register an action that fails when Bazel attempts to stage/build this file ctx.actions.run_shell( outputs = [out], - command = "echo 'ERROR: PyPI package \"{pkg}\" is not available{hub_clause}.' >&2 && exit 1".format( - pkg = ctx.attr.package_name, - hub_clause = hub_clause, - ), + command = "echo \"$1\" >&2 && exit 1", + arguments = ["ERROR: " + message], ) maybe_builtin = [BuiltinPyInfo(transitive_sources = depset([out]))] if BuiltinPyInfo != None else [] @@ -34,6 +25,19 @@ def _missing_package_error_impl(ctx): ), ] + maybe_builtin +def _missing_package_error_impl(ctx): + if ctx.attr.hub_name: + hub_clause = ' when building under PyPI hub "{hub}". Try adding it to the requirements of this hub (e.g. requirements_lock or requirements_by_platform in pip.parse)'.format( + hub = ctx.attr.hub_name, + ) + else: + hub_clause = ' because no default PyPI hub was configured. Try designating a default hub via pip.default(default_hub = "...") or select a hub using --@rules_python//python/config_settings:venv' + + return _error(ctx, 'PyPI package "{pkg}" is not available{hub_clause}.'.format( + pkg = ctx.attr.package_name, + hub_clause = hub_clause, + )) + missing_package_error = rule( implementation = _missing_package_error_impl, attrs = { From 3284430bbc15079e71218906de55dc7825ad4e16 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:55:51 +0900 Subject: [PATCH 17/32] feat(pypi): fail the build when a required extra of a dependency is missing --- python/private/pypi/BUILD.bazel | 1 + python/private/pypi/hub_repository.bzl | 15 ++-- python/private/pypi/missing_package.bzl | 24 +++++- python/private/pypi/render_pkg_aliases.bzl | 19 +++++ .../private/pypi/whl_library_deps_targets.bzl | 28 ++++++- .../render_pkg_aliases_test.bzl | 19 +++++ .../whl_library_deps_targets_tests.bzl | 84 +++++++++++++++++++ 7 files changed, 181 insertions(+), 9 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 87885b18cc..57dd8ad343 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -612,6 +612,7 @@ bzl_library( deps = [ ":env_marker_setting", ":labels", + ":missing_package", ":pep508_deps", "//python:py_library", "//python/private:normalize_name", diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index 27b324f465..7389c50964 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -15,7 +15,7 @@ "" load("//python/private:text_util.bzl", "render") -load(":render_pkg_aliases.bzl", "render_multiplatform_pkg_aliases") +load(":render_pkg_aliases.bzl", "available_extras", "render_multiplatform_pkg_aliases") load(":whl_config_setting.bzl", "whl_config_setting") _BUILD_FILE_CONTENTS = """\ @@ -27,11 +27,12 @@ exports_files(["requirements.bzl"]) def _impl(rctx): bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() + whl_map = { + key: _whl_config_settings_from_json(values) + for key, values in rctx.attr.whl_map.items() + } aliases = render_multiplatform_pkg_aliases( - aliases = { - key: _whl_config_settings_from_json(values) - for key, values in rctx.attr.whl_map.items() - }, + aliases = whl_map, extra_hub_aliases = rctx.attr.extra_hub_aliases, requirement_cycles = rctx.attr.groups, platform_config_settings = rctx.attr.platform_config_settings, @@ -50,7 +51,9 @@ def _impl(rctx): "config.bzl", rctx.attr._config_template, substitutions = { - "%%PACKAGES%%": render.dict(rctx.attr.whl_map, value_repr = lambda x: "None"), + # The available extras of each package for the spoke repositories + # to check if the extras of their dependencies are available. + "%%PACKAGES%%": render.dict(available_extras(whl_map), value_repr = render.list), }, ) rctx.template("requirements.bzl", rctx.attr._requirements_bzl_template, substitutions = { diff --git a/python/private/pypi/missing_package.bzl b/python/private/pypi/missing_package.bzl index 4b750caf45..e5647d5786 100644 --- a/python/private/pypi/missing_package.bzl +++ b/python/private/pypi/missing_package.bzl @@ -1,4 +1,4 @@ -"""Rule for generating an execution-phase action failure when a PyPI package is missing.""" +"""Rules for generating an execution-phase action failure when a PyPI package or its extra is missing.""" load("//python/private:py_info.bzl", "PyInfo") load("//python/private:reexports.bzl", "BuiltinPyInfo") @@ -45,3 +45,25 @@ missing_package_error = rule( "package_name": attr.string(mandatory = True), }, ) + +def _missing_extra_error_impl(ctx): + return _error(ctx, ( + 'The PyPI package "{pkg}" requires "{dep}[{extra}]", but the extra "{extra}" of ' + + '"{dep}" is not available in the hub. Add "{dep}[{extra}]" to the requirements or ' + + "the lock file of the hub, e.g. by compiling the requirements with the extras kept " + + "(`--no-strip-extras`) or by adding the extra to the project dependencies." + ).format( + pkg = ctx.attr.required_by, + dep = ctx.attr.package_name, + extra = ctx.attr.extra, + )) + +missing_extra_error = rule( + implementation = _missing_extra_error_impl, + doc = "A target that fails the build with a helpful message when an extra of a PyPI package is missing.", + attrs = { + "extra": attr.string(mandatory = True), + "package_name": attr.string(mandatory = True), + "required_by": attr.string(mandatory = True), + }, +) diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 271a12ddfb..44b80f0d28 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -243,3 +243,22 @@ def _non_versioned_platform(p, *, strict = False): return p else: fail("Expected to always have a platform in the form '{{abi}}_{{os}}_{{arch}}', got: {}".format(p)) + +def available_extras(aliases): + """Return the extras that are available in the hub for each package. + + Args: + aliases: {type}`dict[str, dict[struct, str]]` the packages to the + {obj}`whl_config_setting` instances and the repo names. + + Returns: + {type}`dict[str, list[str]]` the sorted extras for each package. + """ + return { + pkg: sorted({ + extra: None + for setting in settings + for extra in setting.extras or [] + }) + for pkg, settings in aliases.items() + } diff --git a/python/private/pypi/whl_library_deps_targets.bzl b/python/private/pypi/whl_library_deps_targets.bzl index 42de802267..5cce4e870f 100644 --- a/python/private/pypi/whl_library_deps_targets.bzl +++ b/python/private/pypi/whl_library_deps_targets.bzl @@ -15,6 +15,7 @@ load( "extra_label", "group_member", ) +load(":missing_package.bzl", "missing_extra_error") load(":pep508_deps.bzl", "deps") def whl_library_deps_targets( @@ -35,6 +36,7 @@ def whl_library_deps_targets( rules = struct( py_library = py_library, env_marker_setting = env_marker_setting, + missing_extra_error = missing_extra_error, )): """Create all of the whl_library targets. @@ -51,7 +53,11 @@ def whl_library_deps_targets( provides_extra: {type}`list[str]` The list of extras that the package provides. For each extra the `[]` target is created that includes the package and the dependencies of that extra. - include: {type}`list[str]` The list of packages to include. + include: {type}`list[str] | dict[str, list[str] | None]` The list + of packages to include. If it is a dict, then the values are the + extras of each package that are available in the hub and the + dependencies on the extras that are not available will fail the + build with a helpful message. group_name: {type}`str | None` name of the dependency group (if any). dep_template: {type}`str | None` The dep_template to use. tags: {type}`list[str]` The tags set on the targets. @@ -158,12 +164,21 @@ def whl_library_deps_targets( # dep and the expression. settings = {} + # The dependencies on extras that are not available in the hub. + missing_extras = {} + def _pkg_label(dep): dep_name, _, extra = dep.partition("[") if not extra: return dep_template.format(name = dep_name, target = PY_LIBRARY_PUBLIC_LABEL) - return dep_template.format(name = dep_name, target = extra_label(extra.rstrip("]"))) + extra = extra.rstrip("]") + available = include.get(dep_name) if type(include) == type({}) else None + if available != None and extra not in available: + missing_extras[dep] = (dep_name, extra) + return ":_missing_" + dep + + return dep_template.format(name = dep_name, target = extra_label(extra)) whl_label = lambda dep: dep_template.format(name = dep, target = WHEEL_FILE_PUBLIC_LABEL) @@ -226,6 +241,15 @@ def whl_library_deps_targets( visibility = ["//visibility:private"], ) + for dep, (dep_name, extra) in sorted(missing_extras.items()): + rules.missing_extra_error( + name = "_missing_" + dep, + package_name = dep_name, + extra = extra, + required_by = metadata_name, + visibility = ["//visibility:private"], + ) + def _conditions(settings, package_deps): return { dep: _setting_name(settings, dep, expression) diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index 807c4728fd..6786d27040 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -17,6 +17,7 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load( "//python/private/pypi:render_pkg_aliases.bzl", + "available_extras", "get_whl_flag_versions", "render_multiplatform_pkg_aliases", "render_pkg_aliases", @@ -269,6 +270,24 @@ pkg_aliases( _tests.append(_test_aliases_with_extras) +def _test_available_extras(env): + got = available_extras({ + "bar": { + whl_config_setting(version = "3.2"): "pypi_32_bar", + }, + "foo": { + whl_config_setting(version = "3.2", target_platforms = ["cp32_linux_x86_64"], extras = ["b"]): "pypi_32_foo", + whl_config_setting(version = "3.2", target_platforms = ["cp32_osx_aarch64"], extras = ["a", "b"]): "pypi_32_foo", + }, + }) + + env.expect.that_dict(got).contains_exactly({ + "bar": [], + "foo": ["a", "b"], + }) + +_tests.append(_test_available_extras) + def render_pkg_aliases_test_suite(name): """Create the test suite. diff --git a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl index 788bf44f7f..420efbdbcd 100644 --- a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl +++ b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl @@ -369,6 +369,90 @@ def _test_extras_in_group_legacy(env): _tests.append(_test_extras_in_group_legacy) +def _test_deps_with_extras(env): + filegroup_calls = [] + py_library_calls = [] + env_marker_setting_calls = [] + missing_calls = [] + + whl_library_deps_targets( + name = "foo-0-py3-none-any.whl", + metadata_name = "foo", + dep_template = "@pypi//{name}:{target}", + requires_dist = [ + "bar[socks]", + "baz[a,b]; sys_platform == 'linux'", + "baz; sys_platform == 'darwin'", + "qux[missing]", + ], + # The values are the extras that are available in the hub + include = { + "bar": ["socks"], + "baz": ["a", "b"], + "qux": [], + }, + repo = None, + aliases = {}, + native = struct( + filegroup = lambda **kwargs: filegroup_calls.append(kwargs), + alias = lambda **kwargs: None, + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + env_marker_setting = lambda **kwargs: env_marker_setting_calls.append(kwargs), + missing_extra_error = lambda **kwargs: missing_calls.append(kwargs), + ), + ) + + # The dependencies use the targets with the extras, the missing extras use + # a target that fails the build with a helpful message. + env.expect.that_collection([ + (c["name"], c["deps"]) + for c in py_library_calls + ]).contains_exactly([ + ("pkg", ["srcs", "@pypi//bar:[socks]", ":_missing_qux[missing]"] + select({ + ":is_include_baz_2_true": ["@pypi//baz:pkg"], + "//conditions:default": [], + }) + select({ + ":is_include_baz[a]_true": ["@pypi//baz:[a]"], + "//conditions:default": [], + }) + select({ + ":is_include_baz[b]_true": ["@pypi//baz:[b]"], + "//conditions:default": [], + })), + ]) + env.expect.that_collection(missing_calls).contains_exactly([ + { + "name": "_missing_qux[missing]", + "package_name": "qux", + "extra": "missing", + "required_by": "foo", + "visibility": ["//visibility:private"], + }, + ]) # buildifier: @unsorted-dict-items + + # The whl targets do not use the extras. + env.expect.that_collection([ + (c["name"], c["data"]) + for c in filegroup_calls + ]).contains_exactly([ + ("whl", ["@pypi//bar:whl", "@pypi//qux:whl"] + select({ + ":is_include_baz_true": ["@pypi//baz:whl"], + "//conditions:default": [], + })), + ]) + env.expect.that_collection([ + (c["name"], c["expression"]) + for c in env_marker_setting_calls + ]).contains_exactly([ + ("include_baz", "(sys_platform == \"darwin\") or (sys_platform == \"linux\")"), + ("include_baz_2", "sys_platform == \"darwin\""), + ("include_baz[a]", "sys_platform == \"linux\""), + ("include_baz[b]", "sys_platform == \"linux\""), + ]) + +_tests.append(_test_deps_with_extras) + def _test_deps_with_extras_without_hub_info(env): py_library_calls = [] From cca8d1bc28a8e5f492fcb4cdd7d00816a753024b Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:56:55 +0900 Subject: [PATCH 18/32] feat(pypi): support the extra targets in WORKSPACE --- python/private/pypi/BUILD.bazel | 1 + python/private/pypi/pip_repository.bzl | 19 +++++++++++---- python/private/pypi/render_pkg_aliases.bzl | 5 +++- .../pypi/requirements.bzl.tmpl.workspace | 23 ++++++++++++++++++- .../render_pkg_aliases_test.bzl | 17 ++++++++++++++ 5 files changed, 59 insertions(+), 6 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 57dd8ad343..b295f12a29 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -329,6 +329,7 @@ bzl_library( ":hash", ":parse_requirements", ":pep508_env", + ":pep508_requirement", ":pip_repository_attrs", ":pypi_repo_utils", ":render_pkg_aliases", diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index d02c62eae1..2b6721fc6f 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -24,6 +24,7 @@ load(":default_platforms.bzl", "default_platforms") load(":hash.bzl", "hash") load(":parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement") load(":pep508_env.bzl", "env") +load(":pep508_requirement.bzl", "strip_extras", parse_requirement = "requirement") load(":pip_repository_attrs.bzl", "ATTRS") load(":pypi_repo_utils.bzl", "pypi_repo_utils") load(":render_pkg_aliases.bzl", "render_pkg_aliases") @@ -150,6 +151,7 @@ def _pip_repository_impl(rctx): logger = logger, ) selected_requirements = {} + selected_extras = {} selected_srcs = {} options = None repository_platform = host_platform(rctx) @@ -161,7 +163,11 @@ def _pip_repository_impl(rctx): if not requirement: continue options = options or requirement.extra_pip_args - selected_requirements[whl.name] = requirement.requirement_line + + # The whl repositories have targets for all of the extras that a + # package provides, so the extras are exposed only via the hub. + selected_requirements[whl.name] = strip_extras(requirement.requirement_line) + selected_extras[whl.name] = parse_requirement(requirement.requirement_line).extras if not requirement.url: continue @@ -183,7 +189,7 @@ def _pip_repository_impl(rctx): # ensure that only the locked distributions can be used. digests = sorted({src.digest: None for src in whl.srcs}) if all(digests): - selected_requirements[whl.name] = " ".join([requirement.requirement_line] + [ + selected_requirements[whl.name] = " ".join([selected_requirements[whl.name]] + [ "--hash={}".format(d) for d in digests ]) @@ -224,6 +230,7 @@ def _pip_repository_impl(rctx): # NOTE: Maintain the order consistent with `buildifier` 'load("@rules_python//python:pip.bzl", "pip_utils")', 'load("@rules_python//python/pip_install:pip_repository.bzl", "whl_config_repo", "whl_library")', + 'load("@rules_python//python/private/pypi:labels.bzl", "requirement_label", "requirement_labels") # buildifier: disable=bzl-visibility', ] annotations = {} @@ -261,6 +268,7 @@ def _pip_repository_impl(rctx): for pkg in bzl_packages }, extra_hub_aliases = rctx.attr.extra_hub_aliases, + extras = selected_extras, requirement_cycles = requirement_cycles, ) for path, contents in aliases.items(): @@ -272,10 +280,13 @@ def _pip_repository_impl(rctx): "config.bzl", rctx.attr._config_template, substitutions = { + # The available extras of each package for the spoke + # repositories to check if the extras of their dependencies + # are available. "%%PACKAGES%%": render.dict({ - pkg: None + pkg: sorted(selected_extras.get(pkg, [])) for pkg in bzl_packages - }, value_repr = lambda x: "None"), + }, value_repr = render.list), }, ) config_repo_template = repr(rctx.attr.name) diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 44b80f0d28..9894f555c7 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -88,7 +88,7 @@ package(default_visibility = ["//visibility:public"]) extra_loads = extra_loads, ) -def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases = {}, **kwargs): +def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases = {}, extras = {}, **kwargs): """Create alias declarations for each PyPI package. The aliases should be appended to the pip_repository BUILD.bazel file. These aliases @@ -101,6 +101,8 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases requirement_cycles: any package groups to also add. extra_hub_aliases: The list of extra aliases for each whl to be added in addition to the default ones. + extras: {type}`dict[str, list[str]]` The requested extras for each + whl when the aliases are repo names. **kwargs: Extra kwargs to pass to the rules. Returns: @@ -130,6 +132,7 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases name = normalize_name(name), aliases = pkg_aliases, extra_aliases = extra_hub_aliases.get(normalize_name(name), []), + extras = extras.get(normalize_name(name), []), group_name = whl_group_mapping.get(normalize_name(name)), **kwargs ).strip() diff --git a/python/private/pypi/requirements.bzl.tmpl.workspace b/python/private/pypi/requirements.bzl.tmpl.workspace index faf774e7ed..65d9938cef 100644 --- a/python/private/pypi/requirements.bzl.tmpl.workspace +++ b/python/private/pypi/requirements.bzl.tmpl.workspace @@ -19,7 +19,28 @@ _config = %%CONFIG%% _annotations = %%ANNOTATIONS%% def requirement(name): - return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "pkg") + """Return the label for the package. + + Args: + name: The name of the package with at most one extra, e.g. `foo`, + `foo[]` or `foo[bar]`. Use `requirements` for multiple extras. + + Returns: + The label of the package. + """ + return requirement_label("%%MACRO_TMPL%%", name) + +def requirements(*names): + """Return the labels for the packages. + + Args: + *names: The names of the packages with optional extras, e.g. `foo`, + `foo[]`, `foo[bar]` or `foo[bar,baz]`. + + Returns: + The list of labels, with a label for each extra of a package. + """ + return requirement_labels("%%MACRO_TMPL%%", *names) def whl_requirement(name): return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "whl") diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index 6786d27040..40438134ef 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -288,6 +288,23 @@ def _test_available_extras(env): _tests.append(_test_available_extras) +def _test_aliases_with_extras_for_repo_names(env): + actual = render_pkg_aliases( + aliases = { + "foo": "pypi_foo", + }, + extras = {"foo": ["a"]}, + ) + + env.expect.that_str(actual["foo/BUILD.bazel"]).contains("""\ +pkg_aliases( + name = "foo", + actual = "pypi_foo", + extras = ["a"], +)""") + +_tests.append(_test_aliases_with_extras_for_repo_names) + def render_pkg_aliases_test_suite(name): """Create the test suite. From 181ddc9dbfa534c7989669b743d947572a29da9d Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:57:31 +0900 Subject: [PATCH 19/32] feat(pypi): support extras as members of the requirement cycle groups --- python/private/pypi/BUILD.bazel | 3 ++ .../generate_group_library_build_bazel.bzl | 47 ++++++++++--------- python/private/pypi/hub_builder.bzl | 13 +++-- python/private/pypi/pip_repository.bzl | 7 ++- python/private/pypi/pkg_aliases.bzl | 12 +++-- python/private/pypi/render_pkg_aliases.bzl | 22 ++++++--- .../pypi/requirements.bzl.tmpl.workspace | 4 +- ...nerate_group_library_build_bazel_tests.bzl | 27 +++++++++++ tests/pypi/hub_builder/hub_builder_tests.bzl | 40 ++++++++++++++++ tests/pypi/pkg_aliases/pkg_aliases_test.bzl | 29 ++++++++++++ .../render_pkg_aliases_test.bzl | 22 +++++++++ 11 files changed, 187 insertions(+), 39 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index b295f12a29..1678a5d94d 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -182,6 +182,7 @@ bzl_library( deps = [ ":attrs", ":hash", + ":labels", ":parse_requirements", ":pep508_env", ":pep508_evaluate", @@ -327,6 +328,7 @@ bzl_library( deps = [ ":default_platforms", ":hash", + ":labels", ":parse_requirements", ":pep508_env", ":pep508_requirement", @@ -389,6 +391,7 @@ bzl_library( srcs = ["render_pkg_aliases.bzl"], deps = [ ":generate_group_library_build_bazel", + ":labels", "//python/private:normalize_name", "//python/private:text_util", ], diff --git a/python/private/pypi/generate_group_library_build_bazel.bzl b/python/private/pypi/generate_group_library_build_bazel.bzl index 571cfd6b3f..114f96d41e 100644 --- a/python/private/pypi/generate_group_library_build_bazel.bzl +++ b/python/private/pypi/generate_group_library_build_bazel.bzl @@ -22,6 +22,8 @@ load( "PY_LIBRARY_PUBLIC_LABEL", "WHEEL_FILE_IMPL_LABEL", "WHEEL_FILE_PUBLIC_LABEL", + "extra_label", + "group_member", ) _PRELUDE = """\ @@ -60,35 +62,36 @@ def _generate_group_libraries(repo_prefix, group_name, group_members): repo_prefix: str; the pip_parse repo prefix. group_name: str; the name which the user provided for the dep group. group_members: list[str]; the names of the _packages_ (not repositories) - which make up the group. + which make up the group, optionally with a single extra, e.g. `foo[bar]`. """ - group_members = sorted(group_members) + # The members are the packages, e.g. `foo`, or the extras of the packages, + # e.g. `foo[bar]`. The whl targets do not use the extras, so only the + # packages are members for the whl targets. + members = sorted([group_member(d) for d in group_members]) + lib_targets = [] + whl_targets = [] + for member in members: + name, _, extra = member.partition("[") + if extra: + lib_targets.append((name, "_" + extra_label(extra.rstrip("]")))) + else: + lib_targets.append((name, PY_LIBRARY_IMPL_LABEL)) + whl_targets.append((name, WHEEL_FILE_IMPL_LABEL)) if repo_prefix: - lib_dependencies = [ - "@%s%s//:%s" % (repo_prefix, normalize_name(d), PY_LIBRARY_IMPL_LABEL) - for d in group_members - ] - whl_file_deps = [ - "@%s%s//:%s" % (repo_prefix, normalize_name(d), WHEEL_FILE_IMPL_LABEL) - for d in group_members - ] - visibility = [ - "@%s%s//:__pkg__" % (repo_prefix, normalize_name(d)) - for d in group_members - ] + label_tmpl = "@" + repo_prefix + "{}//:{}" + visibility = sorted({ + "@%s%s//:__pkg__" % (repo_prefix, name): None + for name, _ in lib_targets + }) else: - lib_dependencies = [ - "//%s:%s" % (normalize_name(d), PY_LIBRARY_IMPL_LABEL) - for d in group_members - ] - whl_file_deps = [ - "//%s:%s" % (normalize_name(d), WHEEL_FILE_IMPL_LABEL) - for d in group_members - ] + label_tmpl = "//{}:{}" visibility = ["//:__subpackages__"] + lib_dependencies = [label_tmpl.format(name, target) for name, target in lib_targets] + whl_file_deps = [label_tmpl.format(name, target) for name, target in whl_targets] + return _GROUP_TEMPLATE.format( name = normalize_name(group_name), whl_public_label = WHEEL_FILE_PUBLIC_LABEL, diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index b6809660fd..841f012242 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -9,6 +9,7 @@ load("//python/private:version.bzl", "version") load("//python/private:version_label.bzl", "version_label") load(":attrs.bzl", "use_isolated") load(":hash.bzl", "hash") +load(":labels.bzl", "group_member") load(":parse_requirements.bzl", "parse_requirements") load(":pep508_env.bzl", "env") load(":pep508_evaluate.bzl", "evaluate") @@ -258,18 +259,20 @@ def _add_group_map(self, group_map): # cycles for different abis/oses? For now we will need the users to # assume the same groups across all versions/platforms until we start # using an alternative cycle resolution strategy. + # The members are the packages, e.g. `foo`, or the extras of the packages, + # e.g. `foo[bar]`. group_map = { - name: [normalize_name(whl_name) for whl_name in whls] - for name, whls in group_map.items() + name: [group_member(member) for member in members] + for name, members in group_map.items() } self._group_map.clear() self._group_name_by_whl.clear() self._group_map.update(group_map) self._group_name_by_whl.update({ - whl_name: group_name - for group_name, group_whls in self._group_map.items() - for whl_name in group_whls + member.partition("[")[0]: group_name + for group_name, members in self._group_map.items() + for member in members }) def _add_extra_aliases(self, extra_hub_aliases): diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index 2b6721fc6f..67bbe1e945 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -22,6 +22,7 @@ load("//python/private:text_util.bzl", "render") load("//python/private:version.bzl", "version") load(":default_platforms.bzl", "default_platforms") load(":hash.bzl", "hash") +load(":labels.bzl", "group_member") load(":parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement") load(":pep508_env.bzl", "env") load(":pep508_requirement.bzl", "strip_extras", parse_requirement = "requirement") @@ -222,7 +223,11 @@ def _pip_repository_impl(rctx): # label referencing a package which may not be installed on the current # platform. requirement_cycles = { - normalize_name(name): sorted([normalize_name(d) for d in group if normalize_name(d) in bzl_packages]) + normalize_name(name): sorted([ + group_member(d) + for d in group + if group_member(d).partition("[")[0] in bzl_packages + ]) for name, group in requirement_cycles.items() } diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index 56157ca9a6..1920e2b971 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -83,6 +83,7 @@ def pkg_aliases( name, actual, group_name = None, + group_extras = None, extra_aliases = None, requires_dist = [], extras = [], @@ -100,6 +101,10 @@ def pkg_aliases( the aliases to point to mapping to repositories. The keys are passed to bazel skylib's `selects.with_or`, so they can be tuples as well. group_name: {type}`str` The group name that the pkg belongs to. + group_extras: {type}`list[str] | None` The members of the group in + this package, where an empty string is the package without extras + and the other values are the extras, e.g. `["", "bar"]`. Defaults + to the package without extras if `group_name` is set. group_deps: {type}`list[str]` The packages that are in the given group. Comes as an arg through the hub repository. extra_aliases: {type}`list[str]` The extra aliases to be created. @@ -112,9 +117,10 @@ def pkg_aliases( Comes from `//:config.bzl#packages` **kwargs: extra kwargs to pass to {bzl:obj}`get_config_settings`. """ - - # The package without extras is the member of the group. - group_extras = [""] if group_name else [] + if not group_name: + group_extras = [] + elif group_extras == None: + group_extras = [""] _native = kwargs.pop("native", native) rules = kwargs.pop("rules", struct( diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 9894f555c7..cd6c387fb1 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -22,6 +22,7 @@ load( ":generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel", ) # buildifier: disable=bzl-visibility +load(":labels.bzl", "group_member") NO_MATCH_ERROR_MESSAGE_TEMPLATE = """\ No matching wheel for current configuration's Python version. @@ -115,17 +116,23 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases fail("The aliases need to be provided as a dict, got: {}".format(type(aliases))) whl_group_mapping = {} + group_extras = {} if requirement_cycles: + # The members are the packages, e.g. `foo`, or the extras of the + # packages, e.g. `foo[bar]`. requirement_cycles = { - name: [normalize_name(whl_name) for whl_name in whls] - for name, whls in requirement_cycles.items() + name: [group_member(member) for member in members] + for name, members in requirement_cycles.items() } - whl_group_mapping = { - whl_name: group_name - for group_name, group_whls in requirement_cycles.items() - for whl_name in group_whls - } + for group_name, members in requirement_cycles.items(): + for member in members: + whl_name, _, extra = member.partition("[") + whl_group_mapping[whl_name] = group_name + group_extras.setdefault(whl_name, []).append(extra.rstrip("]")) + + # The package without extras is the default member. + group_extras = {k: v for k, v in group_extras.items() if v != [""]} files = { "{}/BUILD.bazel".format(normalize_name(name)): _render_common_aliases( @@ -134,6 +141,7 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases extra_aliases = extra_hub_aliases.get(normalize_name(name), []), extras = extras.get(normalize_name(name), []), group_name = whl_group_mapping.get(normalize_name(name)), + group_extras = group_extras.get(normalize_name(name)), **kwargs ).strip() for name, pkg_aliases in aliases.items() diff --git a/python/private/pypi/requirements.bzl.tmpl.workspace b/python/private/pypi/requirements.bzl.tmpl.workspace index 65d9938cef..61c191a3e7 100644 --- a/python/private/pypi/requirements.bzl.tmpl.workspace +++ b/python/private/pypi/requirements.bzl.tmpl.workspace @@ -68,8 +68,10 @@ def install_deps(**whl_library_kwargs): # Set up the requirement groups all_requirement_groups = %%ALL_REQUIREMENT_GROUPS%% + # The members are the packages, e.g. `foo`, or the extras of the packages, + # e.g. `foo[bar]`. requirement_group_mapping = { - requirement: group_name + requirement.partition("[")[0]: group_name for group_name, group_requirements in all_requirement_groups.items() for requirement in group_requirements } diff --git a/tests/pypi/generate_group_library_build_bazel/generate_group_library_build_bazel_tests.bzl b/tests/pypi/generate_group_library_build_bazel/generate_group_library_build_bazel_tests.bzl index a91f861a36..2fe22eb487 100644 --- a/tests/pypi/generate_group_library_build_bazel/generate_group_library_build_bazel_tests.bzl +++ b/tests/pypi/generate_group_library_build_bazel/generate_group_library_build_bazel_tests.bzl @@ -95,6 +95,33 @@ py_library( _tests.append(_test_in_hub) +def _test_in_hub_with_extras(env): + actual = generate_group_library_build_bazel( + repo_prefix = "", + groups = {"vbap": ["Pcodedmp", "oletools[Full]"]}, + ) + + # The extras are members only for the py_library targets + env.expect.that_str(actual).contains("""\ +filegroup( + name = "vbap_whl", + srcs = [], + data = ["//pcodedmp:_whl"], + visibility = ["//:__subpackages__"], +)""") + env.expect.that_str(actual).contains("""\ +py_library( + name = "vbap_pkg", + srcs = [], + deps = [ + "//oletools:_[full]", + "//pcodedmp:_pkg", + ], + visibility = ["//:__subpackages__"], +)""") + +_tests.append(_test_in_hub_with_extras) + def generate_group_library_build_bazel_test_suite(name): """Create the test suite. diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index 216dca27f7..88ef33e74a 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -1713,6 +1713,46 @@ Attempting to create a duplicate library pypi_315_foo for foo with different arg _tests.append(_test_err_duplicate_repos) +def _test_groups_with_extras(env): + builder = hub_builder(env) + builder.pip_parse( + _mock_mctx( + os_name = "osx", + arch_name = "aarch64", + mock_files = { + "requirements.txt": """\ +simple==0.0.1 --hash=sha256:deadbeef +other==0.0.1 --hash=sha256:deadbaaf +""", + }, + ), + _parse( + hub_name = "pypi", + python_version = "3.15", + requirements_lock = "requirements.txt", + experimental_requirement_cycles = { + "group": ["Simple[Foo]", "other"], + }, + ), + ) + pypi = builder.build() + + pypi.group_map().contains_exactly({ + "group": ["simple[foo]", "other"], + }) + pypi.whl_libraries().contains_at_least({ + "pypi_315_simple": { + "config_load": "@pypi//:config.bzl", + "dep_template": "@pypi//{name}:{target}", + "group_deps": ["simple[foo]", "other"], + "group_name": "group", + "python_interpreter_target": "unit_test_interpreter_target", + "requirement": "simple==0.0.1 --hash=sha256:deadbeef", + }, + }) + +_tests.append(_test_groups_with_extras) + def hub_builder_test_suite(name): """Create the test suite. diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index 0cbbfb88d9..8cdfc37a7a 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -698,6 +698,35 @@ def _test_extras_aliases_group(env): _tests.append(_test_extras_aliases_group) +def _test_extras_aliases_group_with_extras(env): + got = {} + + pkg_aliases( + name = "foo", + actual = "repo", + extras = ["bar"], + group_name = "group", + # Only the `bar` extra is the member of the group. + group_extras = ["bar"], + native = struct( + alias = lambda *, name, actual, visibility = None, tags = None: got.update({name: actual}), + ), + select = lambda value, no_match_error = None: value, + ) + + # buildifier: disable=unsorted-dict-items + env.expect.that_dict(got).contains_at_least({ + "pkg": "//_groups:group_pkg", + "[]": "@repo//:pkg", + "[bar]": "//_groups:group_pkg", + "_[bar]": "@repo//:[bar]", + "whl": "@repo//:whl", + }) + env.expect.that_collection(got.keys()).not_contains("_pkg") + env.expect.that_collection(got.keys()).not_contains("_whl") + +_tests.append(_test_extras_aliases_group_with_extras) + def _test_extras_aliases_single_repo(env): got = {} py_library_calls = [] diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index 40438134ef..07f00bb155 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -305,6 +305,28 @@ pkg_aliases( _tests.append(_test_aliases_with_extras_for_repo_names) +def _test_groups_with_extras(env): + actual = render_pkg_aliases( + aliases = { + "bar": {whl_config_setting(version = "3.1"): "pypi_31_bar"}, + "foo": {whl_config_setting(version = "3.1"): "pypi_31_foo"}, + }, + requirement_cycles = { + "group": ["Bar", "foo[Extra]"], + }, + ) + + env.expect.that_str(actual["foo/BUILD.bazel"]).contains("""\ + group_name = "group", + group_extras = ["extra"], +""") + env.expect.that_str(actual["bar/BUILD.bazel"]).contains("""\ + group_name = "group", +""") + env.expect.that_bool("group_extras" in actual["bar/BUILD.bazel"]).equals(False) + +_tests.append(_test_groups_with_extras) + def render_pkg_aliases_test_suite(name): """Create the test suite. From cf3b4b6d280c2c06b945d1645d2d4e47d0210a44 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:57:57 +0900 Subject: [PATCH 20/32] refactor(pypi): pass the requirement line only to the repositories using pip --- python/private/internal_dev_deps.bzl | 10 ---------- python/private/pypi/hub_builder.bzl | 5 ++++- python/private/pypi/pip_archive.bzl | 5 +++++ python/private/pypi/whl_deps_repo.bzl | 4 ---- python/private/pypi/whl_file_repo.bzl | 5 ++--- tests/pypi/extension/extension_tests.bzl | 1 - tests/pypi/hub_builder/hub_builder_tests.bzl | 13 ------------- 7 files changed, 11 insertions(+), 32 deletions(-) diff --git a/python/private/internal_dev_deps.bzl b/python/private/internal_dev_deps.bzl index 111b839adf..a708ea9a1b 100644 --- a/python/private/internal_dev_deps.bzl +++ b/python/private/internal_dev_deps.bzl @@ -39,7 +39,6 @@ def _internal_dev_deps_impl(mctx): whl_library( name = "somepkg_with_build_files", whl_file = "@whl_with_build_files//:somepkg-1.0-any-none-any.whl", - requirement = "somepkg", ) # Setup for //tests/implicit_namespace_packages @@ -51,7 +50,6 @@ def _internal_dev_deps_impl(mctx): whl_library( name = "implicit_namespace_ns_sub1", whl_file = "@implicit_namespace_ns_sub1_whl//:ns_sub1-1.0-any-none-any.whl", - requirement = "ns-sub1", enable_implicit_namespace_pkgs = False, ) @@ -63,7 +61,6 @@ def _internal_dev_deps_impl(mctx): whl_library( name = "implicit_namespace_ns_sub2", whl_file = "@implicit_namespace_ns_sub2_whl//:ns_sub2-1.0-any-none-any.whl", - requirement = "ns-sub2", enable_implicit_namespace_pkgs = False, ) @@ -75,7 +72,6 @@ def _internal_dev_deps_impl(mctx): whl_library( name = "pkgutil_nspkg1", whl_file = "@pkgutil_nspkg1_whl//:pkgutil_nspkg1-1.0-any-none-any.whl", - requirement = "pkgutil_nspkg1", enable_implicit_namespace_pkgs = False, ) @@ -87,7 +83,6 @@ def _internal_dev_deps_impl(mctx): whl_library( name = "pkgutil_nspkg2", whl_file = "@pkgutil_nspkg2_whl//:pkgutil_nspkg2-1.0-any-none-any.whl", - requirement = "pkgutil_nspkg2", enable_implicit_namespace_pkgs = False, ) @@ -99,7 +94,6 @@ def _internal_dev_deps_impl(mctx): whl_library( name = "whl_with_data1", whl_file = "@whl_with_data1_whl//:whl_with_data1-1.0-any-none-any.whl", - requirement = "whl-with-data1", ) whl_from_dir_repo( @@ -110,14 +104,12 @@ def _internal_dev_deps_impl(mctx): whl_library( name = "whl_with_data2", whl_file = "@whl_with_data2_whl//:whl_with_data2-1.0-any-none-any.whl", - requirement = "whl-with-data2", ) _whl_library_from_dir( name = "whl_library_extras_direct_dep", root = "//tests/pypi/whl_library/testdata/pkg:BUILD.bazel", output = "pkg-1.0-any-none-any.whl", - requirement = "pkg", # The following is necessary to enable pipstar and make tests faster config_load = "@rules_python//tests/pypi/whl_library/testdata:packages.bzl", dep_template = "@whl_library_extras_{name}//:{target}", @@ -126,7 +118,6 @@ def _internal_dev_deps_impl(mctx): name = "whl_library_extras_optional_dep", root = "//tests/pypi/whl_library/testdata/optional_dep:BUILD.bazel", output = "optional_dep-1.0-any-none-any.whl", - requirement = "optional_dep", # The following is necessary to enable pipstar and make tests faster config_load = "@rules_python//tests/pypi/whl_library/testdata:packages.bzl", ) @@ -140,7 +131,6 @@ def _internal_dev_deps_impl(mctx): whl_library( name = "patch_whl_pkg", whl_file = "@patch_whl_pkg_whl//:pkg-1.0-any-none-any.whl", - requirement = "pkg", whl_patches = { "//tests/pypi/patch_whl/testdata/patches:modify_pkg.patch": json.encode({ "patch_strip": 0, diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 841f012242..443212b1e3 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -775,8 +775,11 @@ def _whl_repo( interpreter, enable_pipstar_extract = False): args = dict(whl_library_args) - args["requirement"] = src.requirement_line is_whl = src.filename.endswith(".whl") + if not (src.url and is_whl): + # The requirement line is only needed for `pip` to download or build + # the wheel. + args["requirement"] = src.requirement_line if src.extra_pip_args and not is_whl: # pip is not used to download wheels and the python diff --git a/python/private/pypi/pip_archive.bzl b/python/private/pypi/pip_archive.bzl index 1764abe70d..9a641bd142 100644 --- a/python/private/pypi/pip_archive.bzl +++ b/python/private/pypi/pip_archive.bzl @@ -358,6 +358,11 @@ pip_archive_attrs = { "quiet", "timeout", ] +} | { + "requirement": attr.string( + mandatory = True, + doc = "Python requirement string describing the package to download or build with `pip`. The extras are not needed because the targets for all of the extras are created.", + ), } pip_archive = repository_rule( diff --git a/python/private/pypi/whl_deps_repo.bzl b/python/private/pypi/whl_deps_repo.bzl index 75e5a3771d..73a0051f11 100644 --- a/python/private/pypi/whl_deps_repo.bzl +++ b/python/private/pypi/whl_deps_repo.bzl @@ -25,10 +25,6 @@ For example if your whl depends on `numpy` and your Python package repo is named "group_name": attr.string( doc = "Name of the group, if any.", ), - "requirement": attr.string( - mandatory = True, - doc = "Python requirement string describing the package to make available, if 'urls' or 'whl_file' is given, then this only needs to include foo[any_extras] as a bare minimum.", - ), } def _whl_deps_repo_impl(rctx): diff --git a/python/private/pypi/whl_file_repo.bzl b/python/private/pypi/whl_file_repo.bzl index 40d79ac10b..0118402d11 100644 --- a/python/private/pypi/whl_file_repo.bzl +++ b/python/private/pypi/whl_file_repo.bzl @@ -36,8 +36,8 @@ def download_whl(rctx, *, logger): auth = get_auth(rctx, urls), ) if not rctx.attr.sha256 and not rctx.attr.integrity: - logger.warn("Please update the requirement line to include the hash:\n{} \\\n --hash=sha256:{}".format( - rctx.attr.requirement, + logger.warn("Please update the requirement line of '{}' to include the hash:\n --hash=sha256:{}".format( + filename, result.sha256, )) if not result.success: @@ -50,7 +50,6 @@ whl_file_repo_attrs = { "index_url": attr.string(doc = "The index URL used to resolve relative wheel URLs."), "integrity": attr.string(doc = "The expected SRI checksum, used when sha256 is unset."), "patches": attr.label_keyed_string_dict(doc = "Patch labels mapped to their strip levels."), - "requirement": attr.string(doc = "The requirement used in download diagnostics."), "sha256": attr.string(doc = "The expected SHA256 checksum."), "urls": attr.string_list(doc = "The wheel download URLs."), "whl_file": attr.label(doc = "A caller-owned wheel to use instead of downloading."), diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index b14e6feffb..02e45ee24a 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -808,7 +808,6 @@ simple==0.0.1 \ "filename": "simple-0.0.1-py3-none-any.whl", "index_url": test.want_index_url, "integrity": "sha256-3q2+7w==", - "requirement": "simple==0.0.1", "urls": ["file://simple-0.0.1-py3-none-any.whl"], }, }) diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index 88ef33e74a..aa441c21e6 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -214,7 +214,6 @@ def _test_uv_lock_extras(env): "filename": "simple-0.0.1-py3-none-any.whl", "index_url": "https://pypi.org/simple/simple", "integrity": "sha256-3q2+7w==", - "requirement": "simple==0.0.1", "urls": ["https://files.pythonhosted.org/packages/simple-0.0.1-py3-none-any.whl"], }, }) @@ -388,7 +387,6 @@ def _test_simple_extras_vs_no_extras_simpleapi(env): "filename": "simple-0.0.1-py3-none-any.whl", "index_url": "https://example.com/simple/", "integrity": "sha256-3q2+7w==", - "requirement": "simple==0.0.1", "urls": ["/simple-0.0.1-py3-none-any.whl"], }, }) @@ -460,7 +458,6 @@ def _test_simple_sha512_simpleapi(env): "filename": "simple-0.0.1-py3-none-any.whl", "index_url": "https://example.com/simple/", "integrity": "sha512-3q2+7w==", - "requirement": "simple==0.0.1", "urls": ["/simple-0.0.1-py3-none-any.whl"], }, }) @@ -851,7 +848,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", "index_url": "https://torch.index/torch/", "integrity": "sha256-iADe7wAmAR1QLAwlbMS2fQAjR/Y8OjjNjkXx9EXGE2Q=", - "requirement": "torch==2.4.1+cpu", "urls": ["/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-linux_x86_64.whl"], }, "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432_linux_aarch64": { @@ -860,7 +856,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", "index_url": "https://torch.index/torch/", "integrity": "sha256-NhCUMrEL1xY8mzDOiW88LMobhrl2X5VqFZTw/0MJHio=", - "requirement": "torch==2.4.1", "urls": ["/whl/cpu/torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"], }, "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c_windows_x86_64": { @@ -869,7 +864,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", "index_url": "https://torch.index/torch/", "integrity": "sha256-OlcOXFU0Fc293+Z5IHMns6OAayHGreoU+6d2hNFhnpc=", - "requirement": "torch==2.4.1+cpu", "urls": ["/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-win_amd64.whl"], }, "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5_osx_aarch64": { @@ -878,7 +872,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", "index_url": "https://torch.index/torch/", "integrity": "sha256-crSE1bbOwac1vz+locSIPQF0hpjF6c/b60/6t8eYfg0=", - "requirement": "torch==2.4.1", "urls": ["/whl/cpu/torch-2.4.1-cp312-none-macosx_11_0_arm64.whl"], }, }) @@ -1021,7 +1014,6 @@ simple==0.0.1 --hash=sha256:deadb00f "filename": "simple-0.0.1-py3-none-any.whl", "index_url": test.expect_index_url, "integrity": "sha256-3q2wDw==", - "requirement": "simple==0.0.1", "urls": [test.expect_url], } if getattr(test, "envsubst", []): @@ -1374,7 +1366,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "dep_template": "@pypi//{name}:{target}", "filename": "direct_without_sha-0.0.1-py3-none-any.whl", "integrity": "", - "requirement": "direct_without_sha==0.0.1", "urls": ["example-direct.org/direct_without_sha-0.0.1-py3-none-any.whl"], "whl_patches": {"my_patch": "1"}, }, @@ -1398,7 +1389,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "filename": "plat-pkg-0.0.4-py3-none-linux_x86_64.whl", "index_url": "https://pypi.org/simple", "integrity": "sha256-3q20Tw==", - "requirement": "plat_pkg==0.0.4", "urls": ["example2.org/index/plat_pkg/"], }, "pypi_315_simple_py3_none_any_deadb00f": { @@ -1407,7 +1397,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "filename": "simple-0.0.1-py3-none-any.whl", "index_url": "https://pypi.org/simple", "integrity": "sha256-3q2wDw==", - "requirement": "simple==0.0.1", "urls": ["example2.org"], }, "pypi_315_some_pkg_py3_none_any_deadbaaf": { @@ -1415,7 +1404,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "dep_template": "@pypi//{name}:{target}", "filename": "some_pkg-0.0.1-py3-none-any.whl", "integrity": "sha256-3q26rw==", - "requirement": "some_pkg==0.0.1", "urls": ["example-direct.org/some_pkg-0.0.1-py3-none-any.whl"], }, "pypi_315_some_py3_none_any_deadb33f": { @@ -1424,7 +1412,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "filename": "some-other-pkg-0.0.1-py3-none-any.whl", "index_url": "https://with_index_url", "integrity": "sha256-3q2zPw==", - "requirement": "some_other_pkg==0.0.1", "urls": ["example2.org/index/some_other_pkg/"], }, }) From be422b2bdd2eb30a92d974c6102b8eb6e2af3bee Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:58:08 +0900 Subject: [PATCH 21/32] test(pypi): check the extras of the dependencies from the METADATA --- MODULE.bazel | 1 + python/private/internal_dev_deps.bzl | 16 ++++++++++++++++ tests/pypi/whl_library/BUILD.bazel | 10 ++++++++++ .../whl_library/testdata/consumer/BUILD.bazel | 0 .../consumer/consumer-1.0.dist-info/METADATA | 3 +++ .../consumer/consumer-1.0.dist-info/RECORD | 0 .../consumer/consumer-1.0.dist-info/WHEEL | 1 + .../whl_library/testdata/consumer/consumer.py | 3 +++ tests/pypi/whl_library/testdata/packages.bzl | 1 + .../whl_library_transitive_extras_test.py | 12 ++++++++++++ 10 files changed, 47 insertions(+) create mode 100644 tests/pypi/whl_library/testdata/consumer/BUILD.bazel create mode 100644 tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA create mode 100644 tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/RECORD create mode 100644 tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/WHEEL create mode 100644 tests/pypi/whl_library/testdata/consumer/consumer.py create mode 100644 tests/pypi/whl_library/whl_library_transitive_extras_test.py diff --git a/MODULE.bazel b/MODULE.bazel index b7301c7291..d15a0ff231 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -335,6 +335,7 @@ use_repo( "pkgutil_nspkg2", "rules_python_runtime_env_tc_info", "somepkg_with_build_files", + "whl_library_extras_consumer", "whl_library_extras_direct_dep", "whl_with_build_files", "whl_with_data1", diff --git a/python/private/internal_dev_deps.bzl b/python/private/internal_dev_deps.bzl index a708ea9a1b..1c5d15e848 100644 --- a/python/private/internal_dev_deps.bzl +++ b/python/private/internal_dev_deps.bzl @@ -114,6 +114,22 @@ def _internal_dev_deps_impl(mctx): config_load = "@rules_python//tests/pypi/whl_library/testdata:packages.bzl", dep_template = "@whl_library_extras_{name}//:{target}", ) + _whl_library_from_dir( + name = "whl_library_extras_pkg", + root = "//tests/pypi/whl_library/testdata/pkg:BUILD.bazel", + output = "pkg-1.0-any-none-any.whl", + # The following is necessary to enable pipstar and make tests faster + config_load = "@rules_python//tests/pypi/whl_library/testdata:packages.bzl", + dep_template = "@whl_library_extras_{name}//:{target}", + ) + _whl_library_from_dir( + name = "whl_library_extras_consumer", + root = "//tests/pypi/whl_library/testdata/consumer:BUILD.bazel", + output = "consumer-1.0-any-none-any.whl", + # The following is necessary to enable pipstar and make tests faster + config_load = "@rules_python//tests/pypi/whl_library/testdata:packages.bzl", + dep_template = "@whl_library_extras_{name}//:{target}", + ) _whl_library_from_dir( name = "whl_library_extras_optional_dep", root = "//tests/pypi/whl_library/testdata/optional_dep:BUILD.bazel", diff --git a/tests/pypi/whl_library/BUILD.bazel b/tests/pypi/whl_library/BUILD.bazel index 427780c9f2..af2c4c24bf 100644 --- a/tests/pypi/whl_library/BUILD.bazel +++ b/tests/pypi/whl_library/BUILD.bazel @@ -9,3 +9,13 @@ py_test( "@whl_library_extras_direct_dep//:[optional]", ], ) + +py_test( + name = "whl_library_transitive_extras_test", + srcs = ["whl_library_transitive_extras_test.py"], + target_compatible_with = SUPPORTS_BZLMOD, + deps = [ + # The METADATA of the consumer requires `pkg[optional]`. + "@whl_library_extras_consumer//:pkg", + ], +) diff --git a/tests/pypi/whl_library/testdata/consumer/BUILD.bazel b/tests/pypi/whl_library/testdata/consumer/BUILD.bazel new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA b/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA new file mode 100644 index 0000000000..9a723c958a --- /dev/null +++ b/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA @@ -0,0 +1,3 @@ +Name: consumer +Version: 1.0 +Requires-Dist: pkg[optional] diff --git a/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/RECORD b/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/RECORD new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/WHEEL b/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/WHEEL new file mode 100644 index 0000000000..a64521a1cc --- /dev/null +++ b/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/WHEEL @@ -0,0 +1 @@ +Wheel-Version: 1.0 diff --git a/tests/pypi/whl_library/testdata/consumer/consumer.py b/tests/pypi/whl_library/testdata/consumer/consumer.py new file mode 100644 index 0000000000..20da50d156 --- /dev/null +++ b/tests/pypi/whl_library/testdata/consumer/consumer.py @@ -0,0 +1,3 @@ +import pkg + +WITH_EXTRAS = pkg.WITH_EXTRAS diff --git a/tests/pypi/whl_library/testdata/packages.bzl b/tests/pypi/whl_library/testdata/packages.bzl index e4a9b0af4c..fd12c500ca 100644 --- a/tests/pypi/whl_library/testdata/packages.bzl +++ b/tests/pypi/whl_library/testdata/packages.bzl @@ -1,6 +1,7 @@ """A list of packages that this logical testdata hub repo contains.""" packages = [ + "consumer", "optional_dep", "pkg", ] diff --git a/tests/pypi/whl_library/whl_library_transitive_extras_test.py b/tests/pypi/whl_library/whl_library_transitive_extras_test.py new file mode 100644 index 0000000000..9aa7a80f04 --- /dev/null +++ b/tests/pypi/whl_library/whl_library_transitive_extras_test.py @@ -0,0 +1,12 @@ +import unittest + + +class TransitiveExtrasTest(unittest.TestCase): + def test_extras_from_metadata_propagated(self): + import consumer + + self.assertEqual(consumer.WITH_EXTRAS, True) + + +if __name__ == "__main__": + unittest.main() From 646f2701e8520b2fe5591445e882e44d43106e75 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:58:14 +0900 Subject: [PATCH 22/32] docs(pypi): document the extra targets --- docs/pypi/circular-dependencies.md | 26 ++++++++++ docs/pypi/use.md | 80 ++++++++++++++++++++++++++++-- news/2786.added.md | 17 +++++++ 3 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 news/2786.added.md diff --git a/docs/pypi/circular-dependencies.md b/docs/pypi/circular-dependencies.md index 62613f489e..2d510316e7 100644 --- a/docs/pypi/circular-dependencies.md +++ b/docs/pypi/circular-dependencies.md @@ -65,6 +65,32 @@ be a part of the `airflow` cycle. For instance: ) ``` +Each extra of a package has its own target, so a cycle that goes through an +extra back to the package without extras, e.g. `foo[bar]` depending on `baz` +which depends on `foo`, is not a cycle in Bazel and does not need a group. If +the cycle includes the extra itself, e.g. `baz` depends on `foo[bar]`, then +only the extra needs to be a member of the group: + +```starlark + ... + experimental_requirement_cycles = { + "foo": [ + "foo[bar]", + "baz", + ] + } +) +``` + +Only the targets of the members, `@pypi//foo:[bar]` and `@pypi//baz`, are +then part of the group and the other targets of `foo`, e.g. `@pypi//foo:[]`, +can be used without the group. The `whl` targets do not use the extras, so +only the packages without extras are part of the group for the `whl` targets. + +:::{versionadded} VERSION_NEXT_FEATURE +Support for the extras as members of the groups. +::: + Alternatively, one could resolve the cycle by removing one leg of it. For example, while `apache-airflow-providers-sqlite` is "baked into" the Airflow diff --git a/docs/pypi/use.md b/docs/pypi/use.md index 9d0c54c4ab..2660749052 100644 --- a/docs/pypi/use.md +++ b/docs/pypi/use.md @@ -57,11 +57,85 @@ which can help you create a `py_binary` target for a particular console script e [whl_ep]: https://packaging.python.org/specifications/entry-points/ +{#extras} ## 'Extras' dependencies -Any "extras" specified in the requirements lock file will be automatically added -as transitive dependencies of the package. In the example above, you'd just put -`requirement("useful_dep")` or `@pypi//useful_dep`. +Any "extras" specified in the requirements lock file or the `uv.lock` file will +be automatically added as transitive dependencies of the package. In the +example above, you'd just put `requirement("useful_dep")` or +`@pypi//useful_dep`. + +If you need only some of the extras, the hub repo also has a target for the +package without any extras and a target for each extra used in the lock file. +For example, if the lock file has `google-api-core[async_rest,grpc]`, then the +hub has: + +``` +@pypi//google_api_core # The package with all of the extras +@pypi//google_api_core:[] # The package without any extras +@pypi//google_api_core:[async_rest] # The package with the async_rest extra +@pypi//google_api_core:[grpc] # The package with the grpc extra +``` + +Extra names are normalized in the same way as the package names, so +`async-rest` becomes `async_rest`. + +The `requirement` helper supports the same syntax: + +```starlark +load("@pypi//:requirements.bzl", "requirement") + +py_library( + name = "my_lib", + deps = [ + requirement("google-api-core[grpc]"), + ], +) +``` + +There are no targets for combinations of extras, because each `[extra]` target +already includes the package. The `requirement` helper accepts at most one +extra. To request more than one extra, use the `requirements` helper, which +accepts any number of packages and returns a list with a label for each +package and each extra: + +```starlark +load("@pypi//:requirements.bzl", "requirements") + +py_library( + name = "my_lib", + deps = [ + ":other_lib", + ] + requirements( + "google-api-core[async_rest,grpc]", + "numpy", + ), +) +``` + +If the lock file requests different extras for a package on different +platforms, for example `foo[bar]` on Linux and `foo[baz]` on macOS, then +`@pypi//foo` includes only the extras requested for the target platform and +`@pypi//foo:[bar]` on macOS is the same as `@pypi//foo:[]`. + +The dependencies of the packages use the extras written in the wheel +`METADATA`, so if `foo` depends on `bar[socks]`, then `@pypi//foo` depends on +`@pypi//bar:[socks]`. If the extra is not in the lock file, then building +`@pypi//foo` fails with a message asking you to add `bar[socks]` to the +requirements or the lock file. For example, when compiling the requirements +with `uv pip compile`, pass `--no-strip-extras` to keep the extras in the +output. + +The `whl` targets, e.g. `@pypi//foo:whl`, do not use the extras and include +only the wheels of the package and its dependencies without extras. + +The packages in a dependency group can also have only some of their extras in +the group, see the [circular dependencies](./circular-dependencies) docs. + +:::{versionadded} VERSION_NEXT_FEATURE +The `[]` and `[extra]` targets, the support for extras in the `requirement` +helper and the `requirements` helper were added. +::: ## Consuming Wheel Dists Directly diff --git a/news/2786.added.md b/news/2786.added.md new file mode 100644 index 0000000000..4c00959a7d --- /dev/null +++ b/news/2786.added.md @@ -0,0 +1,17 @@ +(pypi) The hub repositories now have a `[]` target for each package without +any extras and an `[extra]` target for each extra used in the lock file, e.g. +`@pypi//google_api_core:[grpc]`, in both bzlmod and `WORKSPACE`. The +`requirement` macro accepts a single extra, e.g. +`requirement("google-api-core[grpc]")`, and the new `requirements` macro +returns a list of labels for any number of packages and extras, e.g. +`requirements("google-api-core[async_rest,grpc]", "numpy")`. The whl +repositories now create the targets for every extra that the package provides +and the requested extras are no longer passed to them, so changing the +requested extras does not refetch them. The dependencies use the extras from +the wheel `METADATA` and the build fails with a helpful message if an extra is +missing in the lock file. The `whl` targets no longer include the dependencies +of the extras. The members of `experimental_requirement_cycles` can now be the +extras of the packages, e.g. `foo[bar]`. See +[the docs](https://rules-python.readthedocs.io/en/latest/pypi/use.html#extras) +([#2786](https://github.com/bazel-contrib/rules_python/issues/2786), +[#4168](https://github.com/bazel-contrib/rules_python/issues/4168)). From dafa8381190740bb40f47bd4fa0e24b196842741 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:58:14 +0900 Subject: [PATCH 23/32] chore(examples): update the vendored requirements.bzl --- examples/pip_parse_vendored/requirements.bzl | 28 ++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/examples/pip_parse_vendored/requirements.bzl b/examples/pip_parse_vendored/requirements.bzl index 82af35372d..0a308ac19b 100644 --- a/examples/pip_parse_vendored/requirements.bzl +++ b/examples/pip_parse_vendored/requirements.bzl @@ -5,6 +5,7 @@ load("@rules_python//python:pip.bzl", "pip_utils") load("@rules_python//python/pip_install:pip_repository.bzl", "whl_config_repo", "whl_library") +load("@rules_python//python/private/pypi:labels.bzl", "requirement_label", "requirement_labels") # buildifier: disable=bzl-visibility all_requirements = [ "@my_project_pip_deps_vendored_certifi//:pkg", @@ -79,7 +80,28 @@ _config = { _annotations = {} def requirement(name): - return "@my_project_pip_deps_vendored_{}//:{}".format(pip_utils.normalize_name(name), "pkg") + """Return the label for the package. + + Args: + name: The name of the package with at most one extra, e.g. `foo`, + `foo[]` or `foo[bar]`. Use `requirements` for multiple extras. + + Returns: + The label of the package. + """ + return requirement_label("@my_project_pip_deps_vendored_{}//:{}", name) + +def requirements(*names): + """Return the labels for the packages. + + Args: + *names: The names of the packages with optional extras, e.g. `foo`, + `foo[]`, `foo[bar]` or `foo[bar,baz]`. + + Returns: + The list of labels, with a label for each extra of a package. + """ + return requirement_labels("@my_project_pip_deps_vendored_{}//:{}", *names) def whl_requirement(name): return "@my_project_pip_deps_vendored_{}//:{}".format(pip_utils.normalize_name(name), "whl") @@ -107,8 +129,10 @@ def install_deps(**whl_library_kwargs): # Set up the requirement groups all_requirement_groups = {} + # The members are the packages, e.g. `foo`, or the extras of the packages, + # e.g. `foo[bar]`. requirement_group_mapping = { - requirement: group_name + requirement.partition("[")[0]: group_name for group_name, group_requirements in all_requirement_groups.items() for requirement in group_requirements } From 6431f8bfe5ac402bd81943b0f9a0439bb8c0f75c Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:01:38 +0900 Subject: [PATCH 24/32] fix(pypi): parse the whl METADATA with CRLF line endings --- python/private/pypi/whl_metadata.bzl | 6 +++--- .../pypi/whl_metadata/whl_metadata_tests.bzl | 21 +++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/python/private/pypi/whl_metadata.bzl b/python/private/pypi/whl_metadata.bzl index 2981a5d92f..203caafc58 100644 --- a/python/private/pypi/whl_metadata.bzl +++ b/python/private/pypi/whl_metadata.bzl @@ -56,7 +56,7 @@ def parse_whl_metadata(contents): "requires_dist": [], "version": "", } - for line in contents.strip().split("\n"): + for line in contents.strip().splitlines(): if not line: # Stop parsing on first empty line, which marks the end of the # headers containing the metadata. @@ -70,10 +70,10 @@ def parse_whl_metadata(contents): parsed["version"] = value.strip() elif line.startswith(_REQUIRES_DIST): _, _, value = line.partition(_REQUIRES_DIST) - parsed["requires_dist"].append(value.strip(" ")) + parsed["requires_dist"].append(value.strip()) elif line.startswith(_PROVIDES_EXTRA): _, _, value = line.partition(_PROVIDES_EXTRA) - parsed["provides_extra"].append(value.strip(" ")) + parsed["provides_extra"].append(value.strip()) return struct( name = parsed["name"], diff --git a/tests/pypi/whl_metadata/whl_metadata_tests.bzl b/tests/pypi/whl_metadata/whl_metadata_tests.bzl index 8131b0f452..8aca204146 100644 --- a/tests/pypi/whl_metadata/whl_metadata_tests.bzl +++ b/tests/pypi/whl_metadata/whl_metadata_tests.bzl @@ -246,6 +246,27 @@ FOOBAR = guimod:main _tests.append(_test_parse_entry_points_deduplicate) +def _test_parse_metadata_crlf(env): + got = _parse_whl_metadata( + env, + contents = "\r\n".join([ + "Name: foo", + "Version: 0.0.1", + "Requires-Dist: bar; extra == \"all\"", + "Provides-Extra: all", + "", + "Requires-Dist: this will be ignored", + ]), + ) + got.name().equals("foo") + got.version().equals("0.0.1") + got.requires_dist().contains_exactly([ + "bar; extra == \"all\"", + ]) + got.provides_extra().contains_exactly(["all"]) + +_tests.append(_test_parse_metadata_crlf) + def whl_metadata_test_suite(name): # buildifier: disable=function-docstring test_suite( name = name, From c2e07df71d78c17df418fdc577b51329b4f5ad1f Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:49:34 +0900 Subject: [PATCH 25/32] test(pypi): drop the requirement from the whl repositories in the integration test --- tests/integration/whl_library/MODULE.bazel | 2 -- tests/integration/whl_library/wheels.bzl | 3 --- 2 files changed, 5 deletions(-) diff --git a/tests/integration/whl_library/MODULE.bazel b/tests/integration/whl_library/MODULE.bazel index 9059879187..1cafbb2024 100644 --- a/tests/integration/whl_library/MODULE.bazel +++ b/tests/integration/whl_library/MODULE.bazel @@ -44,7 +44,6 @@ whl_archive( dep_template = DEP_TEMPLATE, filename = "requests-2.34.2-py3-none-any.whl", # https://pypi.org/project/requests/#requests-2.34.2-py3-none-any.whl - requirement = "requests", sha256 = "2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", urls = [ "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", @@ -57,7 +56,6 @@ whl_deps_repo( name = "whl_deps_repo", dep_template = DEP_TEMPLATE, metadata_file = "@whl_archive//:metadata.json", - requirement = "requests", ) wheels = use_extension("//:wheels.bzl", "wheels") diff --git a/tests/integration/whl_library/wheels.bzl b/tests/integration/whl_library/wheels.bzl index b82f0c0479..c690c7f027 100644 --- a/tests/integration/whl_library/wheels.bzl +++ b/tests/integration/whl_library/wheels.bzl @@ -16,7 +16,6 @@ def _wheels_impl(mrctx): common = { "envsubst": ["WHEEL_TEST_ADDRESS"], "filename": _FILENAME, - "requirement": "storage-pkg", "sha256": mrctx.read(Label("//:wheel.sha256")).strip(), "urls": ["http://${WHEEL_TEST_ADDRESS}/" + _FILENAME], } @@ -25,12 +24,10 @@ def _wheels_impl(mrctx): whl_library(name = "nonmatching", whl_patches = _patches("other.whl"), **common) whl_library( name = "provided", - requirement = "storage-pkg", whl_file = Label("//:" + _FILENAME), ) whl_library( name = "provided_patched", - requirement = "storage-pkg", whl_file = Label("//:" + _FILENAME), whl_patches = _patches(_FILENAME), ) From 9f9fc9f707128d0ed835758ba2da520fc14287af Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:28:54 +0900 Subject: [PATCH 26/32] refactor(pypi): keep the original hub_builder group map and repo mapping code --- python/private/pypi/hub_builder.bzl | 47 ++++++++++++---------- python/private/pypi/render_pkg_aliases.bzl | 4 +- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 443212b1e3..6ccda0a184 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -262,17 +262,17 @@ def _add_group_map(self, group_map): # The members are the packages, e.g. `foo`, or the extras of the packages, # e.g. `foo[bar]`. group_map = { - name: [group_member(member) for member in members] - for name, members in group_map.items() + name: [group_member(whl_name) for whl_name in whls] + for name, whls in group_map.items() } self._group_map.clear() self._group_name_by_whl.clear() self._group_map.update(group_map) self._group_name_by_whl.update({ - member.partition("[")[0]: group_name - for group_name, members in self._group_map.items() - for member in members + whl_name.partition("[")[0]: group_name + for group_name, group_whls in self._group_map.items() + for whl_name in group_whls }) def _add_extra_aliases(self, extra_hub_aliases): @@ -366,16 +366,16 @@ def _add_whl_library(self, *, python_version, whl, repo): self._whl_libraries[repo_name] = repo.args mapping = self._whl_map.setdefault(whl.name, {}) - for config_setting in repo.config_settings: - if config_setting in mapping and mapping[config_setting] != repo_name: - fail( - "attempting to override an existing repo '{}' for config setting '{}' with a new repo '{}'".format( - mapping[config_setting], - config_setting, - repo_name, - ), - ) - mapping[config_setting] = repo_name + if repo.config_setting in mapping and mapping[repo.config_setting] != repo_name: + fail( + "attempting to override an existing repo '{}' for config setting '{}' with a new repo '{}'".format( + mapping[repo.config_setting], + repo.config_setting, + repo_name, + ), + ) + else: + mapping[repo.config_setting] = repo_name ### end of setters, below we have various functions to implement the public methods @@ -629,12 +629,17 @@ def _create_whl_repos( interpreter = interpreter, enable_pipstar_extract = enable_pipstar_extract, ) - _add_whl_library( - self, - python_version = python_version, - whl = whl, - repo = repo, - ) + for config_setting in repo.config_settings if repo else []: + _add_whl_library( + self, + python_version = python_version, + whl = whl, + repo = struct( + repo_name = repo.repo_name, + args = repo.args, + config_setting = config_setting, + ), + ) def _merge_extras(srcs): """Merge the sources that differ only in the requested extras. diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index cd6c387fb1..1753f557ea 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -121,8 +121,8 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases # The members are the packages, e.g. `foo`, or the extras of the # packages, e.g. `foo[bar]`. requirement_cycles = { - name: [group_member(member) for member in members] - for name, members in requirement_cycles.items() + name: [group_member(whl_name) for whl_name in whls] + for name, whls in requirement_cycles.items() } for group_name, members in requirement_cycles.items(): From a24fac65e723af55a9e81b6e35a4f2b7f136f8d9 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:29:02 +0900 Subject: [PATCH 27/32] refactor(pypi): minimize the changes to the group library generation --- .../generate_group_library_build_bazel.bzl | 54 +++++++++++-------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/python/private/pypi/generate_group_library_build_bazel.bzl b/python/private/pypi/generate_group_library_build_bazel.bzl index 114f96d41e..92cf8f9870 100644 --- a/python/private/pypi/generate_group_library_build_bazel.bzl +++ b/python/private/pypi/generate_group_library_build_bazel.bzl @@ -65,33 +65,36 @@ def _generate_group_libraries(repo_prefix, group_name, group_members): which make up the group, optionally with a single extra, e.g. `foo[bar]`. """ - # The members are the packages, e.g. `foo`, or the extras of the packages, - # e.g. `foo[bar]`. The whl targets do not use the extras, so only the - # packages are members for the whl targets. - members = sorted([group_member(d) for d in group_members]) - lib_targets = [] - whl_targets = [] - for member in members: - name, _, extra = member.partition("[") - if extra: - lib_targets.append((name, "_" + extra_label(extra.rstrip("]")))) - else: - lib_targets.append((name, PY_LIBRARY_IMPL_LABEL)) - whl_targets.append((name, WHEEL_FILE_IMPL_LABEL)) + # The whl targets do not use the extras, so only the packages without + # extras are members for the whl targets. + group_members = sorted([group_member(d) for d in group_members]) + whl_members = [d for d in group_members if "[" not in d] + packages = sorted({_package(d): None for d in group_members}) if repo_prefix: - label_tmpl = "@" + repo_prefix + "{}//:{}" - visibility = sorted({ - "@%s%s//:__pkg__" % (repo_prefix, name): None - for name, _ in lib_targets - }) + lib_dependencies = [ + "@%s%s//:%s" % (repo_prefix, _package(d), _lib_impl_label(d)) + for d in group_members + ] + whl_file_deps = [ + "@%s%s//:%s" % (repo_prefix, normalize_name(d), WHEEL_FILE_IMPL_LABEL) + for d in whl_members + ] + visibility = [ + "@%s%s//:__pkg__" % (repo_prefix, normalize_name(d)) + for d in packages + ] else: - label_tmpl = "//{}:{}" + lib_dependencies = [ + "//%s:%s" % (_package(d), _lib_impl_label(d)) + for d in group_members + ] + whl_file_deps = [ + "//%s:%s" % (normalize_name(d), WHEEL_FILE_IMPL_LABEL) + for d in whl_members + ] visibility = ["//:__subpackages__"] - lib_dependencies = [label_tmpl.format(name, target) for name, target in lib_targets] - whl_file_deps = [label_tmpl.format(name, target) for name, target in whl_targets] - return _GROUP_TEMPLATE.format( name = normalize_name(group_name), whl_public_label = WHEEL_FILE_PUBLIC_LABEL, @@ -101,6 +104,13 @@ def _generate_group_libraries(repo_prefix, group_name, group_members): visibility = render.indent(render.list(visibility)).lstrip(), ) +def _package(member): + return member.partition("[")[0] + +def _lib_impl_label(member): + _, _, extra = member.partition("[") + return "_" + extra_label(extra.rstrip("]")) if extra else PY_LIBRARY_IMPL_LABEL + def generate_group_library_build_bazel( repo_prefix, groups): From 8f57a4feabb04b6b9345a88f9d1ce01bdcc10656 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:29:04 +0900 Subject: [PATCH 28/32] test(pypi): remove the overlapping extras tests --- tests/pypi/pep508/deps_tests.bzl | 17 --- tests/pypi/pkg_aliases/pkg_aliases_test.bzl | 51 +------ .../whl_library_deps_targets_tests.bzl | 127 ++---------------- 3 files changed, 14 insertions(+), 181 deletions(-) diff --git a/tests/pypi/pep508/deps_tests.bzl b/tests/pypi/pep508/deps_tests.bzl index 6e291e5a6c..b7a58a45a0 100644 --- a/tests/pypi/pep508/deps_tests.bzl +++ b/tests/pypi/pep508/deps_tests.bzl @@ -357,23 +357,6 @@ def test_deps_with_extras_excludes_and_includes(env): _tests.append(test_deps_with_extras_excludes_and_includes) -def test_deps_without_extras(env): - got = deps( - "foo", - requires_dist = [ - "bar[socks]", - "baz[a]; sys_platform == 'linux'", - "baz[b]; sys_platform == 'darwin'", - ], - ) - - env.expect.that_collection(got.deps).contains_exactly(["bar"]) - env.expect.that_dict(got.deps_select).contains_exactly({ - "baz": "(sys_platform == \"darwin\") or (sys_platform == \"linux\")", - }) - -_tests.append(test_deps_without_extras) - def deps_test_suite(name): # buildifier: disable=function-docstring test_suite( name = name, diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index 8cdfc37a7a..8cd4773612 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -631,17 +631,6 @@ def _test_extras_aliases_union(env): }]) env.expect.that_collection(got.keys()).not_contains("pkg") - # The whl target does not use the extras. - # buildifier: disable=unsorted-dict-items - env.expect.that_dict(got).contains_at_least({ - "whl": { - _LINUX: "@repo//:whl", - _OSX: "@repo//:whl", - _WINDOWS: "@repo//:whl", - "//conditions:default": "_no_matching_repository", - }, - }) - _tests.append(_test_extras_aliases_union) def _test_extras_aliases_group(env): @@ -684,16 +673,9 @@ def _test_extras_aliases_group(env): _OSX: "@repo//:pkg", "//conditions:default": "_no_matching_repository", }, - "whl": "//_groups:group_whl", - "_whl": { - _LINUX: "@repo//:whl", - _OSX: "@repo//:whl", - "//conditions:default": "_no_matching_repository", - }, }) env.expect.that_dict(got_visibility).contains_at_least({ "_pkg": ["//_groups:__subpackages__"], - "_whl": ["//_groups:__subpackages__"], }) _tests.append(_test_extras_aliases_group) @@ -701,6 +683,7 @@ _tests.append(_test_extras_aliases_group) def _test_extras_aliases_group_with_extras(env): got = {} + # This is how `WORKSPACE` uses a single repo for each package. pkg_aliases( name = "foo", actual = "repo", @@ -727,38 +710,6 @@ def _test_extras_aliases_group_with_extras(env): _tests.append(_test_extras_aliases_group_with_extras) -def _test_extras_aliases_single_repo(env): - got = {} - py_library_calls = [] - - # This is how `WORKSPACE` uses a single repo for each package. - pkg_aliases( - name = "foo", - actual = "repo", - extras = ["b", "a"], - native = struct( - alias = lambda *, name, actual, visibility = None, tags = None: got.update({name: actual}), - ), - rules = struct( - py_library = lambda **kwargs: py_library_calls.append(kwargs), - ), - select = lambda value, no_match_error = None: value, - ) - - # buildifier: disable=unsorted-dict-items - env.expect.that_dict(got).contains_at_least({ - "[]": "@repo//:pkg", - "[a]": "@repo//:[a]", - "[b]": "@repo//:[b]", - "whl": "@repo//:whl", - }) - env.expect.that_collection(py_library_calls).contains_exactly([{ - "deps": [":[a]", ":[b]"], - "name": "pkg", - }]) - -_tests.append(_test_extras_aliases_single_repo) - def pkg_aliases_test_suite(name): """Create the test suite. diff --git a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl index 420efbdbcd..1c520c148a 100644 --- a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl +++ b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl @@ -101,6 +101,7 @@ def _test_whl_library_deps_targets_no_deps(env): metadata_name = "Foo", dep_template = "@pypi//{name}:{target}", requires_dist = [], + provides_extra = ["A"], group_name = "qux", repo = None, aliases = {}, @@ -129,6 +130,11 @@ def _test_whl_library_deps_targets_no_deps(env): "actual": "whl_file", "visibility": ["//visibility:public"], }, + { + "name": "[a]", + "actual": "srcs", + "visibility": ["//visibility:public"], + }, ]) # buildifier: @unsorted-dict-items env.expect.that_collection(filegroup_calls).contains_exactly([]) @@ -147,7 +153,7 @@ def _test_extras(env): metadata_name = "Foo", dep_template = "@pypi//{name}:{target}", requires_dist = [ - "bar", + "bar[socks]", "grpc; extra == 'grpc'", "aiohttp; extra == 'async' and python_version < '3.12'", "uvloop; extra == 'async'", @@ -169,18 +175,19 @@ def _test_extras(env): # The `pkg` target has no extras because none were requested and there is # a target for each extra that the package provides. The `whl` target does - # not use the extras. + # not use the extras. Without the extras available in the hub, the + # dependencies use the targets with the extras directly. env.expect.that_collection([ (c["name"], c["deps"]) for c in py_library_calls ]).contains_exactly([ - ("pkg", ["srcs", "@pypi//bar:pkg"]), - ("[all]", ["srcs", "@pypi//bar:pkg", "@pypi//grpc:pkg"]), - ("[async]", ["srcs", "@pypi//bar:pkg", "@pypi//uvloop:pkg"] + select({ + ("pkg", ["srcs", "@pypi//bar:[socks]"]), + ("[all]", ["srcs", "@pypi//bar:[socks]", "@pypi//grpc:pkg"]), + ("[async]", ["srcs", "@pypi//bar:[socks]", "@pypi//uvloop:pkg"] + select({ ":is_include_aiohttp_true": ["@pypi//aiohttp:pkg"], "//conditions:default": [], })), - ("[grpc]", ["srcs", "@pypi//bar:pkg", "@pypi//grpc:pkg"]), + ("[grpc]", ["srcs", "@pypi//bar:[socks]", "@pypi//grpc:pkg"]), ]) env.expect.that_collection([ (c["name"], c["data"]) @@ -198,82 +205,6 @@ def _test_extras(env): _tests.append(_test_extras) -def _test_extras_with_different_markers(env): - py_library_calls = [] - env_marker_setting_calls = [] - - whl_library_deps_targets( - name = "foo-0-py3-none-any.whl", - metadata_name = "foo", - dep_template = "@pypi//{name}:{target}", - requires_dist = [ - "bar; sys_platform == 'win32'", - "bar; extra == 'a' and sys_platform == 'linux'", - ], - provides_extra = ["a"], - repo = None, - aliases = {}, - native = struct( - filegroup = lambda **kwargs: None, - alias = lambda **kwargs: None, - ), - rules = struct( - py_library = lambda **kwargs: py_library_calls.append(kwargs), - env_marker_setting = lambda **kwargs: env_marker_setting_calls.append(kwargs), - ), - ) - - env.expect.that_collection([ - (c["name"], c["deps"]) - for c in py_library_calls - ]).contains_exactly([ - ("pkg", ["srcs"] + select({ - ":is_include_bar_true": ["@pypi//bar:pkg"], - "//conditions:default": [], - })), - ("[a]", ["srcs"] + select({ - ":is_include_bar_2_true": ["@pypi//bar:pkg"], - "//conditions:default": [], - })), - ]) - env.expect.that_collection([ - (c["name"], c["expression"]) - for c in env_marker_setting_calls - ]).contains_exactly([ - ("include_bar", "sys_platform == \"win32\""), - ("include_bar_2", "(sys_platform == \"linux\") or (sys_platform == \"win32\")"), - ]) - -_tests.append(_test_extras_with_different_markers) - -def _test_extras_no_deps(env): - alias_calls = [] - - whl_library_deps_targets( - name = "foo-0-py3-none-any.whl", - metadata_name = "foo", - dep_template = "@pypi//{name}:{target}", - requires_dist = [], - provides_extra = ["A"], - repo = None, - aliases = {}, - native = struct( - alias = lambda **kwargs: alias_calls.append(kwargs), - ), - rules = struct(), - ) - - env.expect.that_collection([ - (c["name"], c["actual"]) - for c in alias_calls - ]).contains_exactly([ - ("pkg", "srcs"), - ("whl", "whl_file"), - ("[a]", "srcs"), - ]) - -_tests.append(_test_extras_no_deps) - def _test_extras_in_group(env): py_library_calls = [] filegroup_calls = [] @@ -453,38 +384,6 @@ def _test_deps_with_extras(env): _tests.append(_test_deps_with_extras) -def _test_deps_with_extras_without_hub_info(env): - py_library_calls = [] - - # When the dependencies refer to the whl repositories directly, then there - # is no information about the available extras and the targets with the - # extras in the dependency repositories are used. - whl_library_deps_targets( - name = "foo-0-py3-none-any.whl", - metadata_name = "foo", - dep_template = "@pypi_{name}//:{target}", - requires_dist = ["bar[socks]"], - repo = None, - aliases = {}, - native = struct( - filegroup = lambda **kwargs: None, - alias = lambda **kwargs: None, - ), - rules = struct( - py_library = lambda **kwargs: py_library_calls.append(kwargs), - env_marker_setting = lambda **kwargs: None, - ), - ) - - env.expect.that_collection([ - (c["name"], c["deps"]) - for c in py_library_calls - ]).contains_exactly([ - ("pkg", ["srcs", "@pypi_bar//:[socks]"]), - ]) - -_tests.append(_test_deps_with_extras_without_hub_info) - def whl_library_deps_targets_test_suite(name): """create the test suite. From df890f2fbcc4412958a04a9d638cc54f6654832c Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Wed, 18 Nov 2026 01:15:54 +0900 Subject: [PATCH 29/32] feat(pypi): collect the extras of the selected dependency groups select_dependency_groups now also returns the extras that the selected requirements request, e.g. requests[socks], so that pip.configure can expose them through the hub. --- python/private/pypi/dependency_groups.bzl | 32 ++++++++++++++---- .../dependency_groups_tests.bzl | 33 +++++++++++++++++++ 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl index 4e40c7e856..60c48eac83 100644 --- a/python/private/pypi/dependency_groups.bzl +++ b/python/private/pypi/dependency_groups.bzl @@ -30,6 +30,10 @@ def select_dependency_groups(pyproject, specs, _fail = fail): * `all`: {type}`bool` whether all of the packages including the transitive dependencies should be selected, i.e. `"*"` and `"transitive"` were given. + * `extras`: {type}`dict[str, list[str]]` the normalized package + names mapped to the sorted normalized extras that the selected + requirements request, e.g. `requests[socks]`. When `all` is set, + the extras of all of the groups are returned. """ groups = {"default": pyproject.dependencies} for name, reqs in pyproject.optional_dependencies.items(): @@ -37,10 +41,12 @@ def select_dependency_groups(pyproject, specs, _fail = fail): for name, reqs in _expand_dependency_groups(pyproject.dependency_groups, _fail).items(): groups["group:" + name] = reqs - if "transitive" in specs: + select_all = "transitive" in specs + if select_all: if "*" not in specs: _fail("The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: {}".format(specs)) - return struct(packages = [], all = True) + return struct(packages = [], all = True, extras = {}) + specs = ["*"] selected = {} for spec in specs: @@ -56,18 +62,30 @@ def select_dependency_groups(pyproject, specs, _fail = fail): # A self reference like `my-project[test]` selects the optional # dependencies, but only one level deep. project = normalize_name(pyproject.name or "") - packages = {} + reqs = [] for key in selected: for req in groups[key]: req = requirement(req) if req.name_ != project: - packages[req.name_] = None + reqs.append(req) continue for extra in req.extras: - for r in groups.get("optional:" + extra, []): - packages[requirement(r).name_] = None + reqs.extend([requirement(r) for r in groups.get("optional:" + extra, [])]) + + packages = {} + extras = {} + for req in reqs: + packages[req.name_] = None + if req.extras: + pkg_extras = extras.setdefault(req.name_, {}) + for extra in req.extras: + pkg_extras[extra] = None - return struct(packages = sorted(packages), all = False) + return struct( + packages = [] if select_all else sorted(packages), + all = select_all, + extras = {name: sorted(pkg_extras) for name, pkg_extras in extras.items()}, + ) def _match(spec, groups): if spec == "*": diff --git a/tests/pypi/dependency_groups/dependency_groups_tests.bzl b/tests/pypi/dependency_groups/dependency_groups_tests.bzl index ee23aec1fb..60c5a4efbb 100644 --- a/tests/pypi/dependency_groups/dependency_groups_tests.bzl +++ b/tests/pypi/dependency_groups/dependency_groups_tests.bzl @@ -52,6 +52,39 @@ def _test_select_all(env): _tests.append(_test_select_all) +def _test_select_extras(env): + pyproject = struct( + name = "my-project", + dependencies = ["main-pkg[Foo_Bar, baz]>=1.0", "plain-pkg"], + optional_dependencies = { + "all": ["my-project[docs]"], + "docs": ["docs-pkg[extra]", "main-pkg[docs]"], + }, + dependency_groups = { + "dev": ["dev-pkg[cli]", {"include-group": "test"}], + "test": ["test-pkg[cov]; sys_platform == 'linux'"], + }, + ) + for specs, want in [ + (["default"], {"main_pkg": ["baz", "foo_bar"]}), + (["dev"], {"dev_pkg": ["cli"], "test_pkg": ["cov"]}), + # The extras of the self reference are the optional dependencies of + # the project, not the extras of a package. + (["all"], {"docs_pkg": ["extra"], "main_pkg": ["docs"]}), + (["default", "docs"], {"docs_pkg": ["extra"], "main_pkg": ["baz", "docs", "foo_bar"]}), + (["*", "transitive"], { + "dev_pkg": ["cli"], + "docs_pkg": ["extra"], + "main_pkg": ["baz", "docs", "foo_bar"], + "test_pkg": ["cov"], + }), + ]: + got, failures = _select(specs, pyproject = pyproject) + env.expect.where(specs = specs).that_collection(failures).contains_exactly([]) + env.expect.where(specs = specs).that_dict(got.extras).contains_exactly(want) + +_tests.append(_test_select_extras) + def _test_select_errors(env): for specs, want in [ (["missing"], "Unknown dependency group 'missing'. Available groups: default, group:dev, group:test, optional:all, optional:docs, optional:grpc"), From c7b28a6acd19fe80c72e9a1e2f4816d56e8f3507 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Wed, 18 Nov 2026 01:16:14 +0900 Subject: [PATCH 30/32] feat(pypi): expose the pyproject.toml extras with pip.configure The extras requested in the selected pyproject.toml dependency groups are added to the hub, so @pypi//requests:[socks] exists and @pypi//requests includes the extra even if the lock file does not request it. hub_exposure now returns the extras together with the exposed packages and add_extras adds them to the config settings of the hub whl_map. --- python/private/pypi/BUILD.bazel | 1 + python/private/pypi/extension.bzl | 19 +++- python/private/pypi/hub_exposure.bzl | 74 ++++++++++++-- tests/pypi/extension/extension_tests.bzl | 42 ++++++++ .../pypi/hub_exposure/hub_exposure_tests.bzl | 98 +++++++++++++++++-- 5 files changed, 212 insertions(+), 22 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 05fc644c30..62f8c99125 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -650,6 +650,7 @@ bzl_library( srcs = ["hub_exposure.bzl"], deps = [ ":dependency_groups", + ":whl_config_setting", "//python/private:pyproject_utils", ], ) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index dcc7aa1a94..8ddc3c5847 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -24,7 +24,7 @@ load("//python/private:pyproject_utils.bzl", "read_pyproject", "version_from_req load("//python/private:repo_utils.bzl", "repo_utils") load(":default_platforms.bzl", "default_platforms") load(":hub_builder.bzl", "hub_builder") -load(":hub_exposure.bzl", "hub_exposure") +load(":hub_exposure.bzl", "add_extras", "hub_exposure") load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") load(":parse_whl_name.bzl", "parse_whl_name") load(":pep508_env.bzl", "env") @@ -374,16 +374,22 @@ You cannot use both the additive_build_content and additive_build_content_file a packages = out.whl_map.keys(), ) - configured_hubs = hub_exposure( + exposure = hub_exposure( module_ctx, hubs = hubs, default_pyproject = config.pyproject_toml, _fail = _fail, ) - if configured_hubs == None: + if exposure == None: return None + configured_hubs = exposure.packages exposed_packages.update(configured_hubs) + # The extras requested in the `pyproject.toml`, e.g. `requests[socks]`, + # are exposed by the hub even if the lock file does not have them. + for hub_name, extras in exposure.extras.items(): + hub_whl_map[hub_name] = add_extras(hub_whl_map[hub_name], extras) + return struct( configured_hubs = configured_hubs, config = config, @@ -1132,6 +1138,10 @@ Each item is one of: packages of the hub including the transitive dependencies, i.e. all of the packages from the requirements files or the `uv.lock` file. +The extras of the selected requirements, e.g. `requests[socks]`, are also +exposed by the hub, see [Configuring the Exposed Packages](#pip-configure). +With `"transitive"` the extras of all of the groups are exposed. + See [Configuring the Exposed Packages](#pip-configure) for examples. :::{versionadded} VERSION_NEXT_FEATURE @@ -1166,7 +1176,8 @@ The packages are selected from a `pyproject.toml` file using the `[project] dependencies`, `[project.optional-dependencies]` and PEP 735 `[dependency-groups]` tables. The other packages of the hub, e.g. the transitive dependencies, are still used by the packages that need them, but -they are not public. +they are not public. The extras requested by the selected requirements, e.g. +`requests[socks]`, are exposed even if the lock file does not request them. Only the root module and the module that created the hub can configure it. If both do, the root module wins. Multiple tags for the same hub are combined. diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl index fe9e586153..e2e441ccde 100644 --- a/python/private/pypi/hub_exposure.bzl +++ b/python/private/pypi/hub_exposure.bzl @@ -1,7 +1,8 @@ -"""Compute the packages exposed by the hubs configured with `pip.configure`.""" +"""Compute the packages and extras exposed by the hubs configured with `pip.configure`.""" load("//python/private:pyproject_utils.bzl", "read_pyproject") load(":dependency_groups.bzl", "select_dependency_groups") +load(":whl_config_setting.bzl", "whl_config_setting") def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): """Compute the packages exposed by the hubs configured with `pip.configure`. @@ -18,10 +19,15 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): _fail: {type}`callable` the failure function, mainly for testing. Returns: - {type}`dict[str, list[str]] | None` the configured hubs mapped to the - sorted packages that they expose. The hubs that are not configured or - that expose all of their packages are not included, so that they - behave the same. Returns `None` on failure. + {type}`struct | None` with the attributes below or `None` on failure: + * `packages`: {type}`dict[str, list[str]]` the configured hubs + mapped to the sorted packages that they expose. The hubs that are + not configured or that expose all of their packages are not + included, so that they behave the same. + * `extras`: {type}`dict[str, dict[str, list[str]]]` the configured + hubs mapped to the packages and the sorted extras that the + selected dependency groups request for them, e.g. + `{"hub": {"requests": ["socks"]}}`. """ # The root module wins over the owner of the hub when both configure it. @@ -34,6 +40,7 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): configured = {} exposes_all = {} + extras = {} for mod in module_ctx.modules: for tag in mod.tags.configure: hub = tag.hub_name @@ -75,8 +82,59 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): if name in info.packages: exposed[name] = None + hub_extras = extras.setdefault(hub, {}) + for name, pkg_extras in selection.extras.items(): + if name in info.packages: + for extra in pkg_extras: + hub_extras.setdefault(name, {})[extra] = None + + return struct( + packages = { + hub: sorted(packages) + for hub, packages in configured.items() + if hub not in exposes_all + }, + extras = { + hub: {name: sorted(pkg_extras) for name, pkg_extras in hub_extras.items()} + for hub, hub_extras in extras.items() + if hub_extras + }, + ) + +def add_extras(whl_map, extras): + """Add the extras requested by `pip.configure` to the hub `whl_map`. + + The extras are added to every config setting of the package, so the hub + exposes the `[]` targets and the package target includes them on + all of the platforms. + + Args: + whl_map: {type}`dict[str, dict[str, list[struct]]]` the packages + mapped to the repo names and their {obj}`whl_config_setting` + values. + extras: {type}`dict[str, list[str]]` the packages mapped to the + normalized extras to add. + + Returns: + {type}`dict[str, dict[str, list[struct]]]` the new `whl_map`. + """ + if not extras: + return whl_map + return { - hub: sorted(packages) - for hub, packages in configured.items() - if hub not in exposes_all + name: { + repo: [ + whl_config_setting( + version = setting.version, + target_platforms = setting.target_platforms, + extras = { + extra: None + for extra in list(setting.extras or []) + extras[name] + }.keys(), + ) + for setting in settings + ] + for repo, settings in repos.items() + } if name in extras else repos + for name, repos in whl_map.items() } diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index b888ea847e..ab547f7fa2 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -900,6 +900,48 @@ def _test_configure_default_pyproject(env): _tests.append(_test_configure_default_pyproject) +def _test_configure_extras(env): + # The extras requested in the pyproject.toml are added to the hub even if + # the lock file does not request them. + pypi = _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + _mod( + name = "rules_python", + parse = [_configure_parse(pyproject_toml = "pyproject.toml")], + configure = [_configure(hub_name = "pypi_a", dependency_groups = ["default"])], + ), + os_name = "linux", + arch_name = "x86_64", + mock_files = { + "pyproject.toml": """\ +[project] +dependencies = ["simple[foo]", "other-pkg"] +""", + "requirements.txt": _CONFIGURE_REQUIREMENTS, + }, + ), + available_interpreters = { + "python_3_15_host": "unit_test_interpreter_target", + }, + minor_mapping = {"3.15": "3.15.19"}, + ) + + pypi.configured_hubs().contains_exactly({"pypi_a": ["other_pkg", "simple"]}) + pypi.hub_whl_map().contains_exactly({"pypi_a": { + "other_pkg": { + "pypi_a_315_other_pkg": [whl_config_setting(version = "3.15")], + }, + "simple": { + "pypi_a_315_simple": [whl_config_setting(version = "3.15", extras = ["foo"])], + }, + "transitive_pkg": { + "pypi_a_315_transitive_pkg": [whl_config_setting(version = "3.15")], + }, + }}) + +_tests.append(_test_configure_extras) + def _test_not_configured_exposes_all(env): # The pyproject.toml does not filter the hub without `pip.configure`. pypi = _configure_parse_modules( diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl index 5d84a351b5..eeb57ccdbb 100644 --- a/tests/pypi/hub_exposure/hub_exposure_tests.bzl +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -1,7 +1,8 @@ """hub_exposure tests""" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private/pypi:hub_exposure.bzl", "hub_exposure") # buildifier: disable=bzl-visibility +load("//python/private/pypi:hub_exposure.bzl", "add_extras", "hub_exposure") # buildifier: disable=bzl-visibility +load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # buildifier: disable=bzl-visibility load("//tests/support/mocks:mocks.bzl", "mocks") _tests = [] @@ -15,6 +16,14 @@ dev = ["other-pkg"] test = ["simple", "not-in-hub"] """ +_EXTRAS_PYPROJECT = """\ +[project] +dependencies = ["simple[foo]", "simple[Bar]; sys_platform == 'linux'"] + +[dependency-groups] +dev = ["other-pkg[baz]", "simple[foo]", "not-in-hub[qux]"] +""" + def _hub(*, module_name = "root", pyproject = None): return struct( module_name = module_name, @@ -38,6 +47,7 @@ def _exposure(*modules, hubs = None, default_pyproject = None): mocks.mctx( modules = list(modules), mock_files = { + "extras.toml": _EXTRAS_PYPROJECT, "other.toml": "[project]\ndependencies = [\"other-pkg\"]\n", "pyproject.toml": _PYPROJECT, }, @@ -51,7 +61,7 @@ def _exposure(*modules, hubs = None, default_pyproject = None): def _test_not_configured(env): got, failures = _exposure(_module()) env.expect.that_collection(failures).contains_exactly([]) - env.expect.that_dict(got).contains_exactly({}) + env.expect.that_dict(got.packages).contains_exactly({}) _tests.append(_test_not_configured) @@ -65,7 +75,7 @@ def _test_configure(env): ]: got, failures = _exposure(_module(configure = [_configure(dependency_groups = groups)])) env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) - env.expect.where(groups = groups).that_dict(got).contains_exactly({"hub": want}) + env.expect.where(groups = groups).that_dict(got.packages).contains_exactly({"hub": want}) _tests.append(_test_configure) @@ -77,7 +87,7 @@ def _test_configure_all_packages(env): _configure(dependency_groups = ["*", "transitive"]), ])) env.expect.that_collection(failures).contains_exactly([]) - env.expect.that_dict(got).contains_exactly({}) + env.expect.that_dict(got.packages).contains_exactly({}) _tests.append(_test_configure_all_packages) @@ -87,7 +97,7 @@ def _test_configure_combines_tags(env): _configure(dependency_groups = ["dev"]), ])) env.expect.that_collection(failures).contains_exactly([]) - env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg", "simple"]}) + env.expect.that_dict(got.packages).contains_exactly({"hub": ["other_pkg", "simple"]}) _tests.append(_test_configure_combines_tags) @@ -96,7 +106,7 @@ def _test_configure_pyproject_precedence(env): got, _ = _exposure(_module(configure = [ _configure(pyproject_toml = "other.toml"), ])) - env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) + env.expect.that_dict(got.packages).contains_exactly({"hub": ["other_pkg"]}) # The `pip.default` value is used for the hubs of the root module. got, failures = _exposure( @@ -105,7 +115,7 @@ def _test_configure_pyproject_precedence(env): default_pyproject = "pyproject.toml", ) env.expect.that_collection(failures).contains_exactly([]) - env.expect.that_dict(got).contains_exactly({"hub": ["simple"]}) + env.expect.that_dict(got.packages).contains_exactly({"hub": ["simple"]}) _tests.append(_test_configure_pyproject_precedence) @@ -118,7 +128,7 @@ def _test_configure_module_precedence(env): _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), hubs = hubs, ) - env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) + env.expect.that_dict(got.packages).contains_exactly({"hub": ["other_pkg"]}) # The root module wins over the owner. got, _ = _exposure( @@ -126,7 +136,7 @@ def _test_configure_module_precedence(env): _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), hubs = hubs, ) - env.expect.that_dict(got).contains_exactly({"hub": ["simple"]}) + env.expect.that_dict(got.packages).contains_exactly({"hub": ["simple"]}) # Other modules are ignored. got, _ = _exposure( @@ -134,10 +144,78 @@ def _test_configure_module_precedence(env): _module(name = "other", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), hubs = hubs, ) - env.expect.that_dict(got).contains_exactly({}) + env.expect.that_dict(got.packages).contains_exactly({}) _tests.append(_test_configure_module_precedence) +def _test_configure_extras(env): + hubs = {"hub": _hub(pyproject = "extras.toml")} + for groups, want in [ + (["default"], {"hub": {"simple": ["bar", "foo"]}}), + # The packages that are not in the hub are ignored. + (["dev"], {"hub": {"other_pkg": ["baz"], "simple": ["foo"]}}), + # The extras are also exposed when all of the packages are exposed. + (["*", "transitive"], {"hub": {"other_pkg": ["baz"], "simple": ["bar", "foo"]}}), + ]: + got, failures = _exposure( + _module(configure = [_configure(dependency_groups = groups)]), + hubs = hubs, + ) + env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) + env.expect.where(groups = groups).that_dict(got.extras).contains_exactly(want) + + # The extras of multiple tags are combined. + got, _ = _exposure( + _module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(dependency_groups = ["dev"]), + ]), + hubs = hubs, + ) + env.expect.that_dict(got.extras).contains_exactly({ + "hub": {"other_pkg": ["baz"], "simple": ["bar", "foo"]}, + }) + + # No extras are requested. + got, _ = _exposure(_module(configure = [_configure()])) + env.expect.that_dict(got.extras).contains_exactly({}) + +_tests.append(_test_configure_extras) + +def _test_add_extras(env): + whl_map = { + "other_pkg": {"repo_other": [whl_config_setting(version = "3.15")]}, + "simple": { + "repo_linux": [whl_config_setting( + version = "3.15", + target_platforms = ["cp315_linux_x86_64"], + extras = ["bar"], + )], + "repo_osx": [whl_config_setting( + version = "3.15", + target_platforms = ["cp315_osx_aarch64"], + )], + }, + } + got = add_extras(whl_map, {"simple": ["bar", "foo"]}) + env.expect.that_dict(got).contains_exactly({ + "other_pkg": {"repo_other": [whl_config_setting(version = "3.15")]}, + "simple": { + "repo_linux": [whl_config_setting( + version = "3.15", + target_platforms = ["cp315_linux_x86_64"], + extras = ["bar", "foo"], + )], + "repo_osx": [whl_config_setting( + version = "3.15", + target_platforms = ["cp315_osx_aarch64"], + extras = ["bar", "foo"], + )], + }, + }) + +_tests.append(_test_add_extras) + def _test_configure_errors(env): _, failures = _exposure(_module(configure = [_configure(hub_name = "missing")])) env.expect.that_collection(failures).contains_exactly([ From 45c1c82c7c0f33fc716f82175a787b3ed0159172 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Wed, 18 Nov 2026 01:16:18 +0900 Subject: [PATCH 31/32] docs(pypi): document the pip.configure extras --- docs/pypi/download.md | 15 +++++++++++++++ news/pip-configure-extras.added.md | 5 +++++ 2 files changed, 20 insertions(+) create mode 100644 news/pip-configure-extras.added.md diff --git a/docs/pypi/download.md b/docs/pypi/download.md index 654e156921..a29b1387f7 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -182,6 +182,21 @@ selects the optional dependencies of the project. The packages that are not part of the hub are ignored, so a single `pyproject.toml` can list tools that are missing from the lock file. +The extras requested in the selected groups are exposed by the hub even if the +lock file does not request them, for example because the lock file was +generated with `--strip-extras`. With the following `pyproject.toml`: + +```toml +[project] +dependencies = ["requests[socks]"] +``` + +the hub has the `@pypi_hub//requests:[socks]` target and +`@pypi_hub//requests` includes the `socks` extra on all of the platforms. The +environment markers of the requirements are ignored when selecting the extras. +The dependencies of the extras, such as `pysocks`, still have to be in the lock +file. See [extras](./use.md#extras) for the targets of the extras. + :::{versionadded} VERSION_NEXT_FEATURE ::: diff --git a/news/pip-configure-extras.added.md b/news/pip-configure-extras.added.md new file mode 100644 index 0000000000..a722e3f9eb --- /dev/null +++ b/news/pip-configure-extras.added.md @@ -0,0 +1,5 @@ +(pypi) {obj}`pip.configure` now exposes the extras requested in the selected +`pyproject.toml` dependency groups, e.g. `requests[socks]`, as the +`@pypi//requests:[socks]` target and includes them in `@pypi//requests`, even +if the lock file does not request them +([#2786](https://github.com/bazel-contrib/rules_python/issues/2786)). From b514331f6d618ec2aa13e052a0c7696e4744dcab Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Wed, 18 Nov 2026 01:21:01 +0900 Subject: [PATCH 32/32] docs(pypi): name the news entry after the PR --- news/{pip-configure-extras.added.md => 4522.added.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename news/{pip-configure-extras.added.md => 4522.added.md} (100%) diff --git a/news/pip-configure-extras.added.md b/news/4522.added.md similarity index 100% rename from news/pip-configure-extras.added.md rename to news/4522.added.md