From 3ef53b2098194d6754702dffa6defc24b0fd5566 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:07:15 +0900 Subject: [PATCH 01/18] refactor(pypi): select packages from the pyproject.toml dependency groups Read the dependencies, the optional dependencies and the PEP 735 dependency groups from the pyproject.toml and add a helper to select the packages using dependency group specifiers. --- python/private/pypi/BUILD.bazel | 9 ++ python/private/pypi/dependency_groups.bzl | 142 ++++++++++++++++++ python/private/pyproject_utils.bzl | 14 +- tests/pypi/dependency_groups/BUILD.bazel | 3 + .../dependency_groups_tests.bzl | 86 +++++++++++ 5 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 python/private/pypi/dependency_groups.bzl create mode 100644 tests/pypi/dependency_groups/BUILD.bazel create mode 100644 tests/pypi/dependency_groups/dependency_groups_tests.bzl diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index f94d24594f..af005f706e 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -632,6 +632,15 @@ bzl_library( ], ) +bzl_library( + name = "dependency_groups", + srcs = ["dependency_groups.bzl"], + deps = [ + ":pep508_requirement", + "//python/private:normalize_name", + ], +) + bzl_library( name = "argparse", srcs = ["argparse.bzl"], diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl new file mode 100644 index 0000000000..2ddadee3dd --- /dev/null +++ b/python/private/pypi/dependency_groups.bzl @@ -0,0 +1,142 @@ +"""Select PyPI packages using pyproject.toml dependency groups.""" + +load("//python/private:normalize_name.bzl", "normalize_name") +load(":pep508_requirement.bzl", "requirement") + +def select_dependency_groups(pyproject, specs, _fail = fail): + """Select the packages named by the given dependency group specifiers. + + Supported specifiers are: + + * `"default"`: the `[project] dependencies`. + * `"optional:"`: a `[project.optional-dependencies]` entry. + * `"group:"`: a PEP 735 `[dependency-groups]` entry. + * `"optional:*"` and `"group:*"`: all entries of the table. + * `"*"`: all of the above. + * `""`: a dependency group, or an optional dependency group if + there is no dependency group with that name. + * `"transitive"`: only supported together with `"*"` and selects all + of the packages, including the transitive dependencies. + + Args: + pyproject: {type}`struct` returned by `read_pyproject`. + specs: {type}`list[str]` the specifiers. + _fail: {type}`callable` the failure function, mainly for testing. + + Returns: + {type}`struct` with the attributes: + * `packages`: {type}`list[str]` the sorted normalized package names. + * `all`: {type}`bool` whether all of the packages including the + transitive dependencies should be selected, i.e. `"*"` and + `"transitive"` were given. + """ + groups = {"default": pyproject.dependencies} + for name, reqs in pyproject.optional_dependencies.items(): + groups["optional:" + normalize_name(name)] = reqs + for name, reqs in _expand_dependency_groups(pyproject.dependency_groups, _fail).items(): + groups["group:" + name] = reqs + + if "transitive" in specs: + if "*" not in specs: + _fail("The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: {}".format(specs)) + return struct(packages = [], all = True) + + selected = {} + for spec in specs: + keys = _match(spec, groups) + if not keys: + _fail("Unknown dependency group '{}'. Available groups: {}".format( + spec, + ", ".join(sorted(groups)), + )) + for key in keys: + selected[key] = None + + # A self reference like `my-project[test]` selects the optional + # dependencies, but only one level deep. + project = normalize_name(pyproject.name or "") + packages = {} + for key in selected: + for req in groups[key]: + req = requirement(req) + names = [req.name_] + if req.name_ == project: + names = [ + requirement(r).name_ + for extra in req.extras + for r in groups.get("optional:" + extra, []) + ] + for name in names: + if name: + packages[name] = None + + return struct(packages = sorted(packages), all = False) + +def _match(spec, groups): + if spec == "*": + return groups.keys() + if spec in ("optional:*", "group:*"): + prefix = spec[:-1] + return [key for key in groups if key.startswith(prefix)] + + kind, _, name = spec.rpartition(":") + name = normalize_name(name) + if kind: + candidates = [kind + ":" + name] + elif name == "default": + candidates = ["default"] + else: + candidates = ["group:" + name, "optional:" + name] + + for key in candidates: + if key in groups: + return [key] + return [] + +def _expand_dependency_groups(dependency_groups, _fail): + """Expand the PEP 735 `include-group` entries. + + Args: + dependency_groups: {type}`dict[str, list]` the `[dependency-groups]` + table. + _fail: {type}`callable` the failure function. + + Returns: + {type}`dict[str, list[str]]` normalized group names mapped to their + requirement strings. + """ + groups = { + normalize_name(name): items + for name, items in dependency_groups.items() + } + + # Each round expands at least one group unless there is a cycle. + expanded = {} + for _ in range(len(groups)): + for name, items in groups.items(): + if name in expanded: + continue + + reqs = [] + for item in items: + if type(item) == type(""): + reqs.append(item) + continue + + include = normalize_name(item.get("include-group", "")) + if include not in groups: + _fail("Unknown dependency group '{}' included from '{}'".format(include, name)) + return {} + if include not in expanded: + reqs = None + break + reqs.extend(expanded[include]) + + if reqs != None: + expanded[name] = reqs + + if len(expanded) != len(groups): + _fail("Cyclic dependency group includes: {}".format( + ", ".join(sorted([name for name in groups if name not in expanded])), + )) + return expanded diff --git a/python/private/pyproject_utils.bzl b/python/private/pyproject_utils.bzl index e96ffd2023..a9494b8ac2 100644 --- a/python/private/pyproject_utils.bzl +++ b/python/private/pyproject_utils.bzl @@ -18,10 +18,22 @@ def read_pyproject(module_ctx, pyproject): {type}`struct` with the attributes: * `requires_python`: {type}`str | None` the raw `requires-python` value (e.g. `"==3.13.9"`), or `None` if it is not set. + * `name`: {type}`str | None` the `[project] name`. + * `dependencies`: {type}`list[str]` the `[project] dependencies`. + * `optional_dependencies`: {type}`dict[str, list[str]]` the + `[project.optional-dependencies]` table. + * `dependency_groups`: {type}`dict[str, list]` the PEP 735 + `[dependency-groups]` table. Items are either requirement strings + or `{include-group = ""}` tables. """ data = toml.decode(module_ctx.read(module_ctx.path(pyproject), watch = "yes")) + project = data.get("project", {}) return struct( - requires_python = data.get("project", {}).get("requires-python"), + requires_python = project.get("requires-python"), + name = project.get("name"), + dependencies = project.get("dependencies") or [], + optional_dependencies = project.get("optional-dependencies") or {}, + dependency_groups = data.get("dependency-groups") or {}, ) def version_from_requires_python(requires_python): diff --git a/tests/pypi/dependency_groups/BUILD.bazel b/tests/pypi/dependency_groups/BUILD.bazel new file mode 100644 index 0000000000..ba6877e124 --- /dev/null +++ b/tests/pypi/dependency_groups/BUILD.bazel @@ -0,0 +1,3 @@ +load(":dependency_groups_tests.bzl", "dependency_groups_test_suite") + +dependency_groups_test_suite(name = "dependency_groups_tests") diff --git a/tests/pypi/dependency_groups/dependency_groups_tests.bzl b/tests/pypi/dependency_groups/dependency_groups_tests.bzl new file mode 100644 index 0000000000..ee23aec1fb --- /dev/null +++ b/tests/pypi/dependency_groups/dependency_groups_tests.bzl @@ -0,0 +1,86 @@ +"""dependency_groups tests""" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:dependency_groups.bzl", "select_dependency_groups") # buildifier: disable=bzl-visibility + +_tests = [] + +_PYPROJECT = struct( + name = "my-project", + dependencies = ["main-pkg>=1.0"], + optional_dependencies = { + "Docs": ["docs-pkg"], + "all": ["My_Project[docs, grpc]"], + "grpc": ["grpc-pkg"], + }, + dependency_groups = { + "dev": ["dev-pkg", "shared-pkg", {"include-group": "Test"}], + "test": ["test_pkg", "shared-pkg"], + }, +) + +def _select(specs, pyproject = _PYPROJECT): + failures = [] + got = select_dependency_groups(pyproject, specs, _fail = failures.append) + return got, failures + +def _test_select(env): + for specs, want in [ + (["default"], ["main_pkg"]), + (["optional:docs"], ["docs_pkg"]), + (["group:test"], ["shared_pkg", "test_pkg"]), + (["dev"], ["dev_pkg", "shared_pkg", "test_pkg"]), + (["grpc"], ["grpc_pkg"]), + (["all"], ["docs_pkg", "grpc_pkg"]), + (["optional:*"], ["docs_pkg", "grpc_pkg"]), + (["group:*"], ["dev_pkg", "shared_pkg", "test_pkg"]), + (["default", "group:test"], ["main_pkg", "shared_pkg", "test_pkg"]), + (["*"], ["dev_pkg", "docs_pkg", "grpc_pkg", "main_pkg", "shared_pkg", "test_pkg"]), + ]: + got, failures = _select(specs) + env.expect.where(specs = specs).that_collection(failures).contains_exactly([]) + env.expect.where(specs = specs).that_collection(got.packages).contains_exactly(want).in_order() + env.expect.where(specs = specs).that_bool(got.all).equals(False) + +_tests.append(_test_select) + +def _test_select_all(env): + got, failures = _select(["*", "transitive"]) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_bool(got.all).equals(True) + env.expect.that_collection(got.packages).contains_exactly([]) + +_tests.append(_test_select_all) + +def _test_select_errors(env): + for specs, want in [ + (["missing"], "Unknown dependency group 'missing'. Available groups: default, group:dev, group:test, optional:all, optional:docs, optional:grpc"), + (["default", "transitive"], "The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: [\"default\", \"transitive\"]"), + ]: + _, failures = _select(specs) + env.expect.where(specs = specs).that_collection(failures).contains_exactly([want]) + +_tests.append(_test_select_errors) + +def _test_include_group_errors(env): + for groups, want in [ + ({"a": [{"include-group": "missing"}]}, "Unknown dependency group 'missing' included from 'a'"), + ({"a": [{"include-group": "b"}], "b": [{"include-group": "a"}]}, "Cyclic dependency group includes: a, b"), + ]: + _, failures = _select(["*"], pyproject = struct( + name = None, + dependencies = [], + optional_dependencies = {}, + dependency_groups = groups, + )) + env.expect.that_collection(failures).contains_exactly([want]) + +_tests.append(_test_include_group_errors) + +def dependency_groups_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) From 755737072827673c74afba95147995086a579543 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:07:32 +0900 Subject: [PATCH 02/18] refactor(pypi): support hiding the unexposed packages of a hub Add the `hide_unexposed_packages` attribute to the hub repository. When set, the aliases of the packages that are not exposed are only visible to the wheel repositories that depend on them. --- python/private/pypi/hub_repository.bzl | 13 +++- python/private/pypi/render_pkg_aliases.bzl | 45 +++++++++++++- .../render_pkg_aliases_test.bzl | 62 +++++++++++++++++++ 3 files changed, 115 insertions(+), 5 deletions(-) diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index f915aa1c77..d421f3cc09 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -26,12 +26,16 @@ exports_files(["requirements.bzl"]) """ def _impl(rctx): - bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() + if rctx.attr.hide_unexposed_packages: + bzl_packages = rctx.attr.packages + else: + bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() aliases = render_multiplatform_pkg_aliases( aliases = { key: _whl_config_settings_from_json(values) for key, values in rctx.attr.whl_map.items() }, + exposed_packages = bzl_packages if rctx.attr.hide_unexposed_packages else None, extra_hub_aliases = rctx.attr.extra_hub_aliases, requirement_cycles = rctx.attr.groups, platform_config_settings = rctx.attr.platform_config_settings, @@ -78,10 +82,15 @@ hub_repository = repository_rule( "groups": attr.string_list_dict( mandatory = False, ), + "hide_unexposed_packages": attr.bool( + default = False, + doc = "Make the aliases of the packages that are not in `packages` visible only to the wheel repositories.", + ), "packages": attr.string_list( mandatory = False, doc = """\ -The list of packages that will be exposed via all_*requirements macros. Defaults to whl_map keys. +The list of packages that will be exposed via all_*requirements macros. Defaults to whl_map keys +unless `hide_unexposed_packages` is set. """, ), "platform_config_settings": attr.string_list_dict( diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 0a1c328491..5582cc00c9 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -65,7 +65,7 @@ def _repr_actual(aliases): else: return render.dict(aliases, key_repr = _repr_config_setting) -def _render_common_aliases(*, name, aliases, **kwargs): +def _render_common_aliases(*, name, aliases, visibility, **kwargs): pkg_aliases = render.call( "pkg_aliases", name = repr(name), @@ -80,14 +80,21 @@ def _render_common_aliases(*, name, aliases, **kwargs): return """\ load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") {extra_loads} -package(default_visibility = ["//visibility:public"]) +package(default_visibility = {visibility}) {aliases}""".format( aliases = pkg_aliases, extra_loads = extra_loads, + visibility = render.list(visibility), ) -def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases = {}, **kwargs): +def render_pkg_aliases( + *, + aliases, + requirement_cycles = None, + extra_hub_aliases = {}, + exposed_packages = None, + **kwargs): """Create alias declarations for each PyPI package. The aliases should be appended to the pip_repository BUILD.bazel file. These aliases @@ -100,6 +107,9 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases requirement_cycles: any package groups to also add. extra_hub_aliases: The list of extra aliases for each whl to be added in addition to the default ones. + exposed_packages: The packages that should be public. When set, the + aliases of the other packages are only visible to the wheel + repositories. **kwargs: Extra kwargs to pass to the rules. Returns: @@ -124,12 +134,18 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases for whl_name in group_whls } + public = ["//visibility:public"] + if exposed_packages != None: + exposed_packages = {normalize_name(name): None for name in exposed_packages} + internal_visibility = _internal_visibility(aliases, has_groups = bool(requirement_cycles)) + files = { "{}/BUILD.bazel".format(normalize_name(name)): _render_common_aliases( name = normalize_name(name), aliases = pkg_aliases, extra_aliases = extra_hub_aliases.get(normalize_name(name), []), group_name = whl_group_mapping.get(normalize_name(name)), + visibility = public if exposed_packages == None or normalize_name(name) in exposed_packages else internal_visibility, **kwargs ).strip() for name, pkg_aliases in aliases.items() @@ -139,6 +155,29 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases files["_groups/BUILD.bazel"] = generate_group_library_build_bazel("", requirement_cycles) return files +def _internal_visibility(aliases, has_groups = False): + """The visibility of the packages that are not exposed. + + The wheel repositories depend on the aliases of the hub, so the aliases + have to be visible to them. + """ + repo_names = {} + for pkg_aliases in aliases.values(): + if type(pkg_aliases) == type(""): + repo_names[pkg_aliases] = None + continue + + for repo_name in pkg_aliases.values(): + repo_names[repo_name] = None + + visibility = [ + "@{}//:__pkg__".format(repo_name) + for repo_name in sorted(repo_names) + ] + if has_groups: + visibility.append("//_groups:__pkg__") + return visibility + def _major_minor(python_version): major, _, tail = python_version.partition(".") minor, _, _ = tail.partition(".") diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index 9114f59279..13bee38ff5 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -58,6 +58,68 @@ pkg_aliases( _tests.append(_test_legacy_aliases) +def _test_hidden_aliases(env): + actual = render_pkg_aliases( + aliases = { + "bar": "pypi_bar", + "foo": "pypi_foo", + }, + exposed_packages = ["foo"], + ) + + env.expect.that_str(actual["bar/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = [ + "@pypi_bar//:__pkg__", + "@pypi_foo//:__pkg__", +]) + +pkg_aliases( + name = "bar", + actual = "pypi_bar", +)""") + env.expect.that_str(actual["foo/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = ["//visibility:public"]) + +pkg_aliases( + name = "foo", + actual = "pypi_foo", +)""") + +_tests.append(_test_hidden_aliases) + +def _test_hidden_aliases_in_groups(env): + actual = render_pkg_aliases( + aliases = { + "bar": "pypi_bar", + "baz": "pypi_baz", + }, + requirement_cycles = { + "group": ["bar", "baz"], + }, + exposed_packages = ["baz"], + ) + + env.expect.that_str(actual["bar/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = [ + "@pypi_bar//:__pkg__", + "@pypi_baz//:__pkg__", + "//_groups:__pkg__", +]) + +pkg_aliases( + name = "bar", + actual = "pypi_bar", + group_name = "group", +)""") + +_tests.append(_test_hidden_aliases_in_groups) + def _test_bzlmod_aliases(env): # Use this function as it is used in pip_repository actual = render_multiplatform_pkg_aliases( From 23127e5b1bea7fd8a1be0f3dfbab80b4134b8399 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:08:09 +0900 Subject: [PATCH 03/18] feat(pypi): add pip.configure to select the exposed packages of a hub Add the `pip.configure` tag class that limits the packages that a hub exposes to the ones listed in the dependency groups of a `pyproject.toml` file. The other packages of the hub are still used by the wheels that need them, but their aliases are not public. --- docs/pypi/download.md | 50 ++++++ news/4395.added.md | 2 + python/private/pypi/BUILD.bazel | 10 ++ python/private/pypi/extension.bzl | 108 +++++++++++++ python/private/pypi/hub_exposure.bzl | 84 ++++++++++ tests/integration/unified_pypi/BUILD.bazel | 16 ++ tests/integration/unified_pypi/MODULE.bazel | 14 ++ .../unified_pypi/pyproject_filtered.toml | 4 + tests/integration/unified_pypi_test.py | 13 ++ tests/pypi/extension/extension_tests.bzl | 92 ++++++++++- tests/pypi/extension/pip_parse.bzl | 12 ++ tests/pypi/hub_exposure/BUILD.bazel | 3 + .../pypi/hub_exposure/hub_exposure_tests.bzl | 153 ++++++++++++++++++ 13 files changed, 559 insertions(+), 2 deletions(-) create mode 100644 news/4395.added.md create mode 100644 python/private/pypi/hub_exposure.bzl create mode 100644 tests/integration/unified_pypi/pyproject_filtered.toml create mode 100644 tests/pypi/hub_exposure/BUILD.bazel create mode 100644 tests/pypi/hub_exposure/hub_exposure_tests.bzl diff --git a/docs/pypi/download.md b/docs/pypi/download.md index cc8b7890aa..5f16606d24 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -134,6 +134,56 @@ rename, and `requirement("numpy")` routes through the active hub like fixed at loading time, before the active hub is known, so load them from a concrete hub instead. Prefer `@pypi//` labels in new code. +(pip-configure)= +### Configuring the Exposed Packages (pip.configure) + +By default a hub exposes all of the packages from its requirements files or +`uv.lock` file, including the transitive dependencies. A `pip.configure` tag +can limit the hub to the packages listed in a `pyproject.toml` file: the +`[project] dependencies`, `[project.optional-dependencies]` and PEP 735 +`[dependency-groups]` tables. + +```toml +[project] +dependencies = ["numpy"] + +[dependency-groups] +dev = ["pytest"] +``` + +```starlark +pip.parse( + hub_name = "pypi_hub", + pyproject_toml = "//:pyproject.toml", + requirements_lock = "//:requirements_lock.txt", +) + +pip.configure( + hub_name = "pypi_hub", + dependency_groups = ["default", "group:dev"], +) +``` + +Here `@pypi_hub//numpy` and `@pypi_hub//pytest` can be used, but the +dependencies of `numpy` are only used through `@pypi_hub//numpy`. The +`all_requirements` lists in the hub's `requirements.bzl` and the +[Unified @pypi Hub](unified-pypi-hub) only contain the exposed packages. + +The {attr}`pip.configure.dependency_groups` attribute defaults to `["*"]`, +which selects every group. Use `["*", "transitive"]` to expose all of the +packages of the hub, including the transitive dependencies. Selecting the +transitive dependencies of only some of the groups is not supported. + +Multiple tags for the same hub are combined. Only the root module and the +module that created the hub can configure it; if both do, the root module +wins. A self reference such as `my-project[test]` in the `pyproject.toml` +selects the optional dependencies of the project. The packages that are not +part of the hub are ignored, so a single `pyproject.toml` can list tools that +are missing from the lock file. + +:::{versionadded} VERSION_NEXT_FEATURE +::: + ### Declaring Abstract Dependencies (pip.dep) :::{versionadded} 2.2.0 diff --git a/news/4395.added.md b/news/4395.added.md new file mode 100644 index 0000000000..62d7f10880 --- /dev/null +++ b/news/4395.added.md @@ -0,0 +1,2 @@ +(pypi) Added {obj}`pip.configure` to limit the packages that a hub exposes to +the ones listed in the dependency groups of a `pyproject.toml` file. diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index af005f706e..caf482b67f 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -129,6 +129,7 @@ bzl_library( deps = [ ":default_platforms", ":hub_builder", + ":hub_exposure", ":hub_repository", ":parse_whl_name", ":pep508_env", @@ -641,6 +642,15 @@ bzl_library( ], ) +bzl_library( + name = "hub_exposure", + srcs = ["hub_exposure.bzl"], + deps = [ + ":dependency_groups", + "//python/private:pyproject_utils", + ], +) + bzl_library( name = "argparse", srcs = ["argparse.bzl"], diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index f7a1dadaf7..e64e856489 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -24,6 +24,7 @@ load("//python/private:pyproject_utils.bzl", "read_pyproject", "version_from_req load("//python/private:repo_utils.bzl", "repo_utils") load(":default_platforms.bzl", "default_platforms") load(":hub_builder.bzl", "hub_builder") +load(":hub_exposure.bzl", "hub_exposure") load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") load(":parse_whl_name.bzl", "parse_whl_name") load(":pep508_env.bzl", "env") @@ -89,6 +90,7 @@ def build_config( unified_hub_requirements_bzl = False defaults = { "platforms": default_platforms(), + "pyproject_toml": None, "python_version": None, } for mod in module_ctx.modules: @@ -105,6 +107,8 @@ def build_config( unified_hub_requirements_bzl = True pyproject_toml = tag.pyproject_toml if pyproject_toml: + if mod.is_root: + defaults["pyproject_toml"] = pyproject_toml pyproject = read_pyproject(module_ctx, pyproject_toml) if pyproject.requires_python: defaults["python_version"] = version_from_requires_python(pyproject.requires_python) @@ -151,6 +155,7 @@ def build_config( for name, values in defaults["platforms"].items() }, enable_pipstar_extract = enable_pipstar_extract, + pyproject_toml = defaults["pyproject_toml"], toml_decode = toml.decode, unified_hub_requirements_bzl = unified_hub_requirements_bzl, ) @@ -243,6 +248,7 @@ You cannot use both the additive_build_content and additive_build_content_file a # dict[str repo, HubBuilder] # See `hub_builder.bzl%hub_builder()` for `HubBuilder` pip_hub_map = {} + hub_pyproject = {} simpleapi_cache = pypi_cache(mctx = module_ctx) is_pypi_hub_reserved = module_ctx.getenv("RULES_PYTHON_PYPI_HUB_RESERVED", "0") == "1" @@ -319,6 +325,9 @@ You cannot use both the additive_build_content and additive_build_content_file a else: builder = pip_hub_map[hub_name] + if pip_attr.pyproject_toml and hub_name not in hub_pyproject: + hub_pyproject[hub_name] = pip_attr.pyproject_toml + builder.pip_parse( module_ctx, pip_attr = pip_attr, @@ -342,6 +351,7 @@ You cannot use both the additive_build_content and additive_build_content_file a exposed_packages = {} extra_aliases = {} whl_libraries = {} + hubs = {} for hub in pip_hub_map.values(): out = hub.build() @@ -355,8 +365,25 @@ You cannot use both the additive_build_content and additive_build_content_file a extra_aliases[hub.name] = out.extra_aliases hub_group_map[hub.name] = out.group_map hub_whl_map[hub.name] = out.whl_map + hubs[hub.name] = struct( + module_name = hub.module_name, + pyproject = hub_pyproject.get(hub.name), + packages = out.whl_map.keys(), + exposed_packages = out.exposed_packages, + ) + + configured_hubs = hub_exposure( + module_ctx, + hubs = hubs, + default_pyproject = config.pyproject_toml, + _fail = _fail, + ) + if configured_hubs == None: + return None + exposed_packages.update(configured_hubs) return struct( + configured_hubs = configured_hubs, config = config, declared_deps = declared_deps, default_hub = config.default_hub or renamed_default_hub, @@ -511,6 +538,8 @@ def _pip_impl(module_ctx): packages = mods.exposed_packages.get(hub_name, []), platform_config_settings = mods.platform_config_settings.get(hub_name, {}), groups = mods.hub_group_map.get(hub_name), + # The hubs configured with `pip.configure` hide the other packages. + hide_unexposed_packages = hub_name in mods.configured_hubs, ) _create_unified_hub_repo(mods) @@ -638,6 +667,10 @@ This is designed to work with dependency management tools like Renovate. :::{versionadded} 2.3.0 ::: + +:::{versionchanged} VERSION_NEXT_FEATURE +Also used by {obj}`pip.configure` for the hubs of the root module. +::: """, ), "unified_hub_requirements_bzl": attr.bool( @@ -841,6 +874,10 @@ The version must be specified as `==X.Y.Z` (exact version with full semver). :::{versionadded} 2.3.0 ::: + +:::{versionchanged} VERSION_NEXT_FEATURE +Also used by {obj}`pip.configure` to select the packages that the hub exposes. +::: """, ), "python_version": attr.string( @@ -1026,6 +1063,72 @@ execution-phase error when built. """, ) +_configure_tag = tag_class( + attrs = { + "dependency_groups": attr.string_list( + default = ["*"], + doc = """\ +The `pyproject.toml` dependency groups whose packages the hub exposes. + +Each item is one of: + +* `"default"`: the `[project] dependencies`. +* `"optional:"`: the `[project.optional-dependencies]` entry ``. +* `"group:"`: the PEP 735 `[dependency-groups]` entry ``. +* `"optional:*"` or `"group:*"`: all entries of the table. +* `"*"`: all of the above. +* `""`: the dependency group `` if it exists, otherwise the + optional dependency group ``. +* `"transitive"`: only supported together with `"*"`, exposes all of the + packages of the hub including the transitive dependencies, i.e. all of the + packages from the requirements files or the `uv.lock` file. + +See [Configuring the Exposed Packages](#pip-configure) for examples. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + "hub_name": attr.string( + mandatory = True, + doc = """\ +The name of the hub to configure. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + "pyproject_toml": attr.label( + mandatory = False, + doc = """\ +The `pyproject.toml` file that defines the dependency groups. Defaults to the +{attr}`pip.parse.pyproject_toml` of the hub, then to +{attr}`pip.default.pyproject_toml` if the hub is defined by the root module. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + }, + doc = """\ +Configure the packages exposed by a hub. + +The packages are selected from a `pyproject.toml` file using the +`[project] dependencies`, `[project.optional-dependencies]` and PEP 735 +`[dependency-groups]` tables. The other packages of the hub, e.g. the +transitive dependencies, are still used by the packages that need them, but +they are not public. + +Only the root module and the module that created the hub can configure it. If +both do, the root module wins. Multiple tags for the same hub are combined. + +See [Configuring the Exposed Packages](#pip-configure) for examples. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", +) + pypi = module_extension( environ = ["RULES_PYTHON_PYPI_HUB_RESERVED"], doc = """\ @@ -1049,12 +1152,17 @@ Environment Variables: - `RULES_PYTHON_PYPI_HUB_RESERVED`: Enable fallback renaming for reserved hub name collisions. See the {envvar}`RULES_PYTHON_PYPI_HUB_RESERVED` documentation for details. +pip.configure: +This tag class configures the packages exposed by a hub using the dependency +groups from a pyproject.toml file. + pip.whl_mods: This tag class is used to help create JSON files to describe modifications to the BUILD files for wheels. """, implementation = _pip_impl, tag_classes = { + "configure": _configure_tag, "default": tag_class( attrs = _default_attrs, doc = """\ diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl new file mode 100644 index 0000000000..6ca9315a81 --- /dev/null +++ b/python/private/pypi/hub_exposure.bzl @@ -0,0 +1,84 @@ +"""Compute the packages exposed by the hubs configured with `pip.configure`.""" + +load("//python/private:pyproject_utils.bzl", "read_pyproject") +load(":dependency_groups.bzl", "select_dependency_groups") + +def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): + """Compute the packages exposed by the hubs configured with `pip.configure`. + + Args: + module_ctx: {type}`module_ctx` the module context. + hubs: {type}`dict[str, struct]` the hubs by name. Each struct has: + * `module_name`: {type}`str` the module that created the hub. + * `pyproject`: {type}`Label | None` from `pip.parse`. + * `packages`: {type}`list[str]` the normalized names of all + packages in the hub. + * `exposed_packages`: {type}`list[str]` the packages that the hub + exposes when it is not configured. + default_pyproject: {type}`Label | None` from `pip.default`, only used + for the hubs of the root module. + _fail: {type}`callable` the failure function, mainly for testing. + + Returns: + {type}`dict[str, list[str]] | None` the configured hubs mapped to the + sorted packages that they expose. The hubs that are not configured + are not included. Returns `None` on failure. + """ + + # The root module wins over the owner of the hub when both configure it. + root_modules = [mod for mod in module_ctx.modules if mod.is_root] + root_name = root_modules[0].name if root_modules else None + root_configured = { + tag.hub_name: None + for mod in root_modules + for tag in mod.tags.configure + } + + configured = {} + for mod in module_ctx.modules: + for tag in mod.tags.configure: + hub = tag.hub_name + if hub not in hubs: + _fail("pip.configure() refers to unknown hub '{}'. Available hubs: {}".format( + hub, + ", ".join(sorted(hubs)), + )) + return None + + info = hubs[hub] + if not mod.is_root and (mod.name != info.module_name or hub in root_configured): + # Only the root module and the module that created the hub can + # configure it. + continue + + pyproject = ( + tag.pyproject_toml or + info.pyproject or + (default_pyproject if info.module_name == root_name else None) + ) + if not pyproject: + _fail("pip.configure() for hub '{}' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`".format(hub)) + return None + + selection = select_dependency_groups( + read_pyproject(module_ctx, pyproject), + tag.dependency_groups, + _fail = _fail, + ) + if selection.all: + packages = info.exposed_packages + else: + packages = selection.packages + + # Multiple tags for the same hub are combined. + exposed = configured.setdefault(hub, {}) + for name in packages: + # The pyproject.toml may list packages that are not in the hub, + # e.g. tools that are not locked. + if name in info.packages: + exposed[name] = None + + return { + hub: sorted(packages) + for hub, packages in configured.items() + } diff --git a/tests/integration/unified_pypi/BUILD.bazel b/tests/integration/unified_pypi/BUILD.bazel index 13d188d087..f3a9513d1f 100644 --- a/tests/integration/unified_pypi/BUILD.bazel +++ b/tests/integration/unified_pypi/BUILD.bazel @@ -95,3 +95,19 @@ platform( "@platforms//os:windows", ], ) + +# The pypi_filtered hub only makes its pyproject.toml packages public. +py_test( + name = "test_filtered_direct", + srcs = ["test_default.py"], + main = "test_default.py", + deps = ["@pypi_filtered//colorama"], +) + +py_binary( + name = "bin_filtered_six", + srcs = ["bin_six_a.py"], + main = "bin_six_a.py", + tags = ["manual"], + deps = ["@pypi_filtered//six"], +) diff --git a/tests/integration/unified_pypi/MODULE.bazel b/tests/integration/unified_pypi/MODULE.bazel index ee8f622cb4..70e01ccdac 100644 --- a/tests/integration/unified_pypi/MODULE.bazel +++ b/tests/integration/unified_pypi/MODULE.bazel @@ -53,6 +53,20 @@ pip.parse( ) use_repo(pip, "pypi_b") +# `pypi_filtered` only exposes the `[project] dependencies` of the +# `pyproject.toml`, the other packages are not public. +pip.parse( + hub_name = "pypi_filtered", + pyproject_toml = "//:pyproject_filtered.toml", + python_version = "3.11", + requirements_lock = "//:requirements_b.txt", +) +pip.configure( + dependency_groups = ["default"], + hub_name = "pypi_filtered", +) +use_repo(pip, "pypi_filtered") + pip.default(default_hub = "pypi_b") pip.default(unified_hub_requirements_bzl = True) pip.dep( diff --git a/tests/integration/unified_pypi/pyproject_filtered.toml b/tests/integration/unified_pypi/pyproject_filtered.toml new file mode 100644 index 0000000000..632f72fe22 --- /dev/null +++ b/tests/integration/unified_pypi/pyproject_filtered.toml @@ -0,0 +1,4 @@ +[project] +name = "unified-pypi" +version = "0.0.0" +dependencies = ["colorama"] diff --git a/tests/integration/unified_pypi_test.py b/tests/integration/unified_pypi_test.py index 9ce41fafb7..5ac2f7df8b 100644 --- a/tests/integration/unified_pypi_test.py +++ b/tests/integration/unified_pypi_test.py @@ -7,6 +7,19 @@ class UnifiedPypiTest(runner.TestCase): + def test_configure_hides_unexposed_packages(self): + self.run_bazel("test", "//:test_filtered_direct") + result = self.run_bazel("build", "//:bin_filtered_six", check=False) + self.assertNotEqual( + result.exit_code, + 0, + "Expected the alias of the unexposed package to be hidden", + ) + self.assert_result_matches( + result, + r"alias '[^']*//six:six'.*is not visible from", + ) + def test_default_fallback_hub(self): self.run_bazel("test", "//:test_default") diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index ee8e58dfe4..17f8d6741e 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -21,7 +21,7 @@ load("//python/private/pypi:extension.bzl", "build_config", "parse_modules") # load("//python/private/pypi:platform.bzl", _plat = "platform") # buildifier: disable=bzl-visibility load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # buildifier: disable=bzl-visibility load("//tests/support/mocks:mocks.bzl", "mocks") -load(":pip_parse.bzl", _parse = "pip_parse") +load(":pip_parse.bzl", _configure = "configure", _parse = "pip_parse") _tests = [] @@ -101,10 +101,11 @@ def _dep(*, name, extra_targets = []): extra_targets = extra_targets, ) -def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], dep = [], is_root = True): +def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], dep = [], configure = [], is_root = True): return struct( name = name, tags = struct( + configure = configure, parse = parse, override = override, whl_mods = whl_mods, @@ -118,6 +119,7 @@ def _parse_modules(env, **kwargs): return env.expect.that_struct( parse_modules(**kwargs), attrs = dict( + configured_hubs = subjects.dict, declared_deps = subjects.dict, default_hub = subjects.str, exposed_packages = subjects.dict, @@ -766,6 +768,92 @@ simple==0.0.1 \ _tests.append(_test_default_index_setting) +_CONFIGURE_REQUIREMENTS = """\ +simple==0.0.1 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +other-pkg==0.0.2 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +transitive-pkg==0.0.3 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +""" + +_CONFIGURE_PYPROJECT = """\ +[project] +dependencies = ["simple"] + +[dependency-groups] +dev = ["other-pkg"] +test = ["simple"] +""" + +def _configure_parse_modules(env, *modules): + return _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + os_name = "linux", + arch_name = "x86_64", + mock_files = { + "pyproject.toml": _CONFIGURE_PYPROJECT, + "requirements.txt": _CONFIGURE_REQUIREMENTS, + }, + *modules + ), + available_interpreters = { + "python_3_15_host": "unit_test_interpreter_target", + }, + minor_mapping = {"3.15": "3.15.19"}, + ) + +def _configure_parse(*, hub_name = "pypi_a", pyproject_toml = "pyproject.toml"): + return _parse( + hub_name = hub_name, + pyproject_toml = pyproject_toml, + python_version = "3.15", + simpleapi_skip = ["simple", "other-pkg", "transitive-pkg"], + requirements_lock = "requirements.txt", + ) + +def _test_configure(env): + for groups, want in [ + (["*"], ["other_pkg", "simple"]), + (["test"], ["simple"]), + (["dev"], ["other_pkg"]), + (["*", "transitive"], ["other_pkg", "simple", "transitive_pkg"]), + ]: + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [_configure_parse()], + configure = [_configure(hub_name = "pypi_a", dependency_groups = groups)], + ), + ) + + pypi.exposed_packages().contains_exactly({"pypi_a": want}) + pypi.configured_hubs().contains_exactly({"pypi_a": want}) + +_tests.append(_test_configure) + +def _test_not_configured_exposes_all(env): + # The pyproject.toml does not filter the hub without `pip.configure`. + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [_configure_parse()], + ), + ) + + pypi.exposed_packages().contains_exactly({ + "pypi_a": ["other_pkg", "simple", "transitive_pkg"], + }) + pypi.configured_hubs().contains_exactly({}) + +_tests.append(_test_not_configured_exposes_all) + def extension_test_suite(name): """Create the test suite. diff --git a/tests/pypi/extension/pip_parse.bzl b/tests/pypi/extension/pip_parse.bzl index 7b5bdfdfd6..505c249169 100644 --- a/tests/pypi/extension/pip_parse.bzl +++ b/tests/pypi/extension/pip_parse.bzl @@ -1,5 +1,17 @@ """A simple test helper""" +def configure( + *, + dependency_groups = ["*"], + hub_name, + pyproject_toml = None): + """A simple helper for testing to simulate the PyPI extension configure tag class""" + return struct( + dependency_groups = dependency_groups, + hub_name = hub_name, + pyproject_toml = pyproject_toml, + ) + def pip_parse( *, hub_name, diff --git a/tests/pypi/hub_exposure/BUILD.bazel b/tests/pypi/hub_exposure/BUILD.bazel new file mode 100644 index 0000000000..187361e1ce --- /dev/null +++ b/tests/pypi/hub_exposure/BUILD.bazel @@ -0,0 +1,3 @@ +load(":hub_exposure_tests.bzl", "hub_exposure_test_suite") + +hub_exposure_test_suite(name = "hub_exposure_tests") diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl new file mode 100644 index 0000000000..cafbb229b5 --- /dev/null +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -0,0 +1,153 @@ +"""hub_exposure tests""" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:hub_exposure.bzl", "hub_exposure") # buildifier: disable=bzl-visibility +load("//tests/support/mocks:mocks.bzl", "mocks") + +_tests = [] + +_PYPROJECT = """\ +[project] +dependencies = ["simple"] + +[dependency-groups] +dev = ["other-pkg"] +test = ["simple", "not-in-hub"] +""" + +def _hub(*, module_name = "root", pyproject = None): + return struct( + module_name = module_name, + pyproject = pyproject, + packages = ["other_pkg", "simple", "transitive_pkg"], + exposed_packages = ["other_pkg", "simple", "transitive_pkg"], + ) + +def _configure(hub_name = "hub", dependency_groups = ["*"], pyproject_toml = None): + return mocks.tag( + hub_name = hub_name, + dependency_groups = dependency_groups, + pyproject_toml = pyproject_toml, + ) + +def _module(name = "root", is_root = True, configure = []): + return mocks.module(name = name, is_root = is_root, configure = configure) + +def _exposure(*modules, hubs = None, default_pyproject = None): + failures = [] + got = hub_exposure( + mocks.mctx( + modules = list(modules), + mock_files = { + "other.toml": "[project]\ndependencies = [\"other-pkg\"]\n", + "pyproject.toml": _PYPROJECT, + }, + ), + hubs = hubs or {"hub": _hub(pyproject = "pyproject.toml")}, + default_pyproject = default_pyproject, + _fail = failures.append, + ) + return got, failures + +def _test_not_configured(env): + got, failures = _exposure(_module()) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({}) + +_tests.append(_test_not_configured) + +def _test_configure(env): + for groups, want in [ + (["*"], ["other_pkg", "simple"]), + (["default"], ["simple"]), + (["dev"], ["other_pkg"]), + # `not-in-hub` is ignored + (["group:test"], ["simple"]), + (["*", "transitive"], ["other_pkg", "simple", "transitive_pkg"]), + ]: + got, failures = _exposure(_module(configure = [_configure(dependency_groups = groups)])) + env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) + env.expect.where(groups = groups).that_dict(got).contains_exactly({"hub": want}) + +_tests.append(_test_configure) + +def _test_configure_combines_tags(env): + got, failures = _exposure(_module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(dependency_groups = ["dev"]), + ])) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg", "simple"]}) + +_tests.append(_test_configure_combines_tags) + +def _test_configure_pyproject_precedence(env): + # The tag wins over the `pip.parse` value. + got, _ = _exposure(_module(configure = [ + _configure(pyproject_toml = "other.toml"), + ])) + env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) + + # The `pip.default` value is used for the hubs of the root module. + got, failures = _exposure( + _module(configure = [_configure(dependency_groups = ["default"])]), + hubs = {"hub": _hub()}, + default_pyproject = "pyproject.toml", + ) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({"hub": ["simple"]}) + +_tests.append(_test_configure_pyproject_precedence) + +def _test_configure_module_precedence(env): + hubs = {"hub": _hub(module_name = "dep", pyproject = "pyproject.toml")} + + # The owner of the hub can configure it. + got, _ = _exposure( + _module(), + _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + hubs = hubs, + ) + env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) + + # The root module wins over the owner. + got, _ = _exposure( + _module(configure = [_configure(dependency_groups = ["default"])]), + _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + hubs = hubs, + ) + env.expect.that_dict(got).contains_exactly({"hub": ["simple"]}) + + # Other modules are ignored. + got, _ = _exposure( + _module(), + _module(name = "other", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + hubs = hubs, + ) + env.expect.that_dict(got).contains_exactly({}) + +_tests.append(_test_configure_module_precedence) + +def _test_configure_errors(env): + _, failures = _exposure(_module(configure = [_configure(hub_name = "missing")])) + env.expect.that_collection(failures).contains_exactly([ + "pip.configure() refers to unknown hub 'missing'. Available hubs: hub", + ]) + + _, failures = _exposure( + _module(configure = [_configure()]), + hubs = {"hub": _hub()}, + ) + env.expect.that_collection(failures).contains_exactly([ + "pip.configure() for hub 'hub' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`", + ]) + +_tests.append(_test_configure_errors) + +def hub_exposure_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) From 6bb7f8379a68c92c78b12acb4a0fad3b8c99a36a Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:11:22 +0900 Subject: [PATCH 04/18] fix(pypi): only add the exposed packages of a configured hub to @pypi Otherwise the unified hub would point to the hidden aliases of the hub and fail with a visibility error instead of the missing package error. --- python/private/pypi/extension.bzl | 4 +++- tests/integration/unified_pypi/BUILD.bazel | 12 ++++++++++++ tests/integration/unified_pypi_test.py | 13 +++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index e64e856489..ce5e95492d 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -416,7 +416,9 @@ def _create_unified_hub_repo(mods): extra_aliases = {} for hub_name in hubs: - for pkg_name in mods.hub_whl_map[hub_name]: + # The hubs configured with `pip.configure` only expose some packages. + hub_packages = mods.configured_hubs.get(hub_name, mods.hub_whl_map[hub_name]) + for pkg_name in hub_packages: norm_pkg = normalize_name(pkg_name) if norm_pkg not in packages: packages[norm_pkg] = [] diff --git a/tests/integration/unified_pypi/BUILD.bazel b/tests/integration/unified_pypi/BUILD.bazel index f3a9513d1f..78f9a857ee 100644 --- a/tests/integration/unified_pypi/BUILD.bazel +++ b/tests/integration/unified_pypi/BUILD.bazel @@ -104,6 +104,18 @@ py_test( deps = ["@pypi_filtered//colorama"], ) +# six is not exposed by pypi_filtered, so it is missing in the unified hub. +py_binary( + name = "bin_filtered_six_unified", + srcs = ["bin_six_a.py"], + config_settings = { + "@rules_python//python/config_settings:venv": "pypi_filtered", + }, + main = "bin_six_a.py", + tags = ["manual"], + deps = ["@pypi//six"], +) + py_binary( name = "bin_filtered_six", srcs = ["bin_six_a.py"], diff --git a/tests/integration/unified_pypi_test.py b/tests/integration/unified_pypi_test.py index 5ac2f7df8b..f59a19fb42 100644 --- a/tests/integration/unified_pypi_test.py +++ b/tests/integration/unified_pypi_test.py @@ -20,6 +20,19 @@ def test_configure_hides_unexposed_packages(self): r"alias '[^']*//six:six'.*is not visible from", ) + def test_configure_hides_unexposed_packages_in_unified_hub(self): + self.run_bazel("cquery", "//:bin_filtered_six_unified") + result = self.run_bazel("build", "//:bin_filtered_six_unified", check=False) + self.assertNotEqual( + result.exit_code, + 0, + "Expected build to fail during execution phase", + ) + self.assert_result_matches( + result, + 'ERROR: PyPI package "six" is not available when building under PyPI hub "pypi_filtered"\\. Try adding it to the requirements of this hub', + ) + def test_default_fallback_hub(self): self.run_bazel("test", "//:test_default") From fcaac6659712438657768a81294119cb865ffae7 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:22:46 +0900 Subject: [PATCH 05/18] refactor(pypi): support testonly packages in the hub repositories Bazel does not enforce `testonly` on aliases, so the main public targets of a testonly package are testonly wrappers around the aliases. The testonly packages are listed in `all_testonly_requirements` instead of the `all_*requirements` lists of the hub's requirements.bzl. --- python/private/pypi/BUILD.bazel | 1 + python/private/pypi/hub_repository.bzl | 19 ++++- python/private/pypi/pkg_aliases.bzl | 68 +++++++++++++++++- python/private/pypi/render_pkg_aliases.bzl | 8 +++ .../private/pypi/requirements.bzl.tmpl.bzlmod | 2 + python/private/pypi/unified_hub_repo.bzl | 10 +++ python/private/pypi/unified_hub_setup.bzl | 5 +- tests/pypi/pkg_aliases/pkg_aliases_test.bzl | 70 +++++++++++++++++++ .../render_pkg_aliases_test.bzl | 31 ++++++++ 9 files changed, 207 insertions(+), 7 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index caf482b67f..c0294975cd 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -354,6 +354,7 @@ bzl_library( deps = [ ":labels", ":whl_library_deps_targets", + "//python:py_library", "//python/private:common_labels", "//python/private:text_util", "@bazel_skylib//lib:selects", diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index d421f3cc09..91f2c2d0e4 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -30,6 +30,7 @@ def _impl(rctx): bzl_packages = rctx.attr.packages else: bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() + testonly_packages = rctx.attr.testonly_packages aliases = render_multiplatform_pkg_aliases( aliases = { key: _whl_config_settings_from_json(values) @@ -39,6 +40,7 @@ def _impl(rctx): extra_hub_aliases = rctx.attr.extra_hub_aliases, requirement_cycles = rctx.attr.groups, platform_config_settings = rctx.attr.platform_config_settings, + testonly_packages = testonly_packages, ) for path, contents in aliases.items(): rctx.file(path, contents) @@ -57,18 +59,25 @@ def _impl(rctx): "%%PACKAGES%%": render.dict(rctx.attr.whl_map, value_repr = lambda x: "None"), }, ) + + # The testonly packages are only listed in `all_testonly_requirements`. + all_packages = [p for p in bzl_packages if p not in testonly_packages] rctx.template("requirements.bzl", rctx.attr._requirements_bzl_template, substitutions = { "%%ALL_DATA_REQUIREMENTS%%": render.list([ macro_tmpl.format(p, "data") - for p in bzl_packages + for p in all_packages ]), "%%ALL_REQUIREMENTS%%": render.list([ macro_tmpl.format(p, "pkg") - for p in bzl_packages + for p in all_packages + ]), + "%%ALL_TESTONLY_REQUIREMENTS%%": render.list([ + macro_tmpl.format(p, "pkg") + for p in testonly_packages ]), "%%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%": render.dict({ p: macro_tmpl.format(p, "whl") - for p in bzl_packages + for p in all_packages }), "%%MACRO_TMPL%%": macro_tmpl, }) @@ -101,6 +110,10 @@ unless `hide_unexposed_packages` is set. mandatory = True, doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", ), + "testonly_packages": attr.string_list( + default = [], + doc = "The packages whose aliases should be testonly.", + ), "whl_map": attr.string_dict( mandatory = True, doc = """\ diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index 227e9bf10f..b838434088 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -32,6 +32,7 @@ setting maps to and their precedence, refer to documentation on that page. """ load("@bazel_skylib//lib:selects.bzl", "selects") +load("//python:py_library.bzl", "py_library") load("//python/private:common_labels.bzl", "labels") load("//python/private:text_util.bzl", "render") load( @@ -86,6 +87,7 @@ def pkg_aliases( extras = [], include = [], group_deps = [], + testonly = False, **kwargs): """Create aliases for an actual package. @@ -107,10 +109,14 @@ def pkg_aliases( dependencies when parsing the requires_dist. Comes from METADATA or a lock file. include: {type}`list[str]` The subset of packages to include. Comes from `//:config.bzl#packages` + testonly: {type}`bool` Whether the public targets should be testonly. + Bazel does not enforce `testonly` on aliases, so the main public + targets are testonly wrappers around the aliases. **kwargs: extra kwargs to pass to {bzl:obj}`get_config_settings`. """ _native = kwargs.pop("native", native) rules = kwargs.pop("rules", struct( + py_library = py_library, whl_library_deps_targets = whl_library_deps_targets, )) select = kwargs.pop("select", selects.with_or) @@ -118,9 +124,13 @@ def pkg_aliases( _native.alias( name = name, actual = ":" + PY_LIBRARY_PUBLIC_LABEL, + **({"testonly": True} if testonly else {}) ) if requires_dist: + if testonly: + fail("testonly is not supported together with requires_dist") + # if it has a group_name set, then it will create the impl label target in the # current macro, we still need to do the aliases to the actual groups package as # per below @@ -210,20 +220,72 @@ def pkg_aliases( elif target_name in (WHEEL_FILE, PY_SRCS_LABEL): kwargs["visibility"] = ["//visibility:private"] - _native.alias( + # The private targets are used by the non-testonly `//_groups` libraries. + _alias( + _native, + rules, name = target_name, actual = _actual, + testonly = testonly and not target_name.startswith("_"), **kwargs ) if group_name: - _native.alias( + _alias( + _native, + rules, name = PY_LIBRARY_PUBLIC_LABEL, actual = "//_groups:{}_pkg".format(group_name), + testonly = testonly, ) - _native.alias( + _alias( + _native, + rules, name = WHEEL_FILE_PUBLIC_LABEL, actual = "//_groups:{}_whl".format(group_name), + testonly = testonly, + ) + +# The public targets that are wrapped in testonly targets, because Bazel does +# not enforce `testonly` on aliases. +_TESTONLY_WRAPPED = [ + PY_LIBRARY_PUBLIC_LABEL, + WHEEL_FILE_PUBLIC_LABEL, + DATA_LABEL, + DIST_INFO_LABEL, + EXTRACTED_WHEEL_FILES, +] + +def _alias(_native, rules, *, name, actual, testonly, **kwargs): + if not testonly: + _native.alias(name = name, actual = actual, **kwargs) + return + + if name not in _TESTONLY_WRAPPED: + # E.g. the extra aliases, which may point to executables. Best effort, + # because Bazel does not enforce `testonly` on aliases. + _native.alias(name = name, actual = actual, testonly = True, **kwargs) + return + + impl = "_testonly_" + name + _native.alias( + name = impl, + actual = actual, + visibility = ["//visibility:private"], + ) + if name == PY_LIBRARY_PUBLIC_LABEL: + rules.py_library( + name = name, + deps = [":" + impl], + testonly = True, + **kwargs + ) + else: + _native.filegroup( + name = name, + srcs = [":" + impl], + testonly = True, + **kwargs ) def multiplatform_whl_aliases( diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 5582cc00c9..8271387418 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -94,6 +94,7 @@ def render_pkg_aliases( requirement_cycles = None, extra_hub_aliases = {}, exposed_packages = None, + testonly_packages = [], **kwargs): """Create alias declarations for each PyPI package. @@ -110,6 +111,7 @@ def render_pkg_aliases( exposed_packages: The packages that should be public. When set, the aliases of the other packages are only visible to the wheel repositories. + testonly_packages: The packages whose aliases should be testonly. **kwargs: Extra kwargs to pass to the rules. Returns: @@ -134,6 +136,11 @@ def render_pkg_aliases( for whl_name in group_whls } + testonly_packages = { + normalize_name(name): None + for name in testonly_packages + } + public = ["//visibility:public"] if exposed_packages != None: exposed_packages = {normalize_name(name): None for name in exposed_packages} @@ -146,6 +153,7 @@ def render_pkg_aliases( extra_aliases = extra_hub_aliases.get(normalize_name(name), []), group_name = whl_group_mapping.get(normalize_name(name)), visibility = public if exposed_packages == None or normalize_name(name) in exposed_packages else internal_visibility, + testonly = normalize_name(name) in testonly_packages, **kwargs ).strip() for name, pkg_aliases in aliases.items() diff --git a/python/private/pypi/requirements.bzl.tmpl.bzlmod b/python/private/pypi/requirements.bzl.tmpl.bzlmod index ba227aeb2d..7904ed5578 100644 --- a/python/private/pypi/requirements.bzl.tmpl.bzlmod +++ b/python/private/pypi/requirements.bzl.tmpl.bzlmod @@ -7,6 +7,8 @@ load("@rules_python//python:pip.bzl", "pip_utils") all_requirements = %%ALL_REQUIREMENTS%% +all_testonly_requirements = %%ALL_TESTONLY_REQUIREMENTS%% + all_whl_requirements_by_package = %%ALL_WHL_REQUIREMENTS_BY_PACKAGE%% all_whl_requirements = all_whl_requirements_by_package.values() diff --git a/python/private/pypi/unified_hub_repo.bzl b/python/private/pypi/unified_hub_repo.bzl index ddd15c0d86..984c3cacbe 100644 --- a/python/private/pypi/unified_hub_repo.bzl +++ b/python/private/pypi/unified_hub_repo.bzl @@ -25,6 +25,7 @@ define_pypi_package_targets( extra_aliases = {extra_aliases}, hubs = {hubs}, pkg_hubs = {pkg_hubs}, + testonly = {testonly}, ) """ @@ -79,6 +80,10 @@ def _unified_hub_repo_impl(rctx): extra_aliases_by_pkg.setdefault(pkg, {})[alias] = alias_hubs # 3. Generate package subpackages + testonly_packages = { + pkg_name: None + for pkg_name in rctx.attr.testonly_packages + } for pkg_name, pkg_hubs in rctx.attr.packages.items(): extra_aliases = extra_aliases_by_pkg.get(pkg_name, {}) rctx.file( @@ -89,6 +94,7 @@ def _unified_hub_repo_impl(rctx): hubs = hubs, pkg_hubs = pkg_hubs, pkg_name = pkg_name, + testonly = pkg_name in testonly_packages, ), ) @@ -130,6 +136,10 @@ unified_hub_repo = repository_rule( default = False, doc = "Whether to generate a requirements.bzl with the per-package `requirement` macros.", ), + "testonly_packages": attr.string_list( + default = [], + doc = "Packages that are testonly in at least one hub.", + ), }, doc = "Private repository rule creating the automatic Unified PyPI Hub.", ) diff --git a/python/private/pypi/unified_hub_setup.bzl b/python/private/pypi/unified_hub_setup.bzl index b19410fd4b..3ee8ae25d3 100644 --- a/python/private/pypi/unified_hub_setup.bzl +++ b/python/private/pypi/unified_hub_setup.bzl @@ -31,7 +31,7 @@ _STANDARD_ALIASES = [ EXTRACTED_WHEEL_FILES, ] -def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub = None): +def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub = None, testonly = False): """Define the targets for a PyPI package in the unified PyPI hub. Args: @@ -40,6 +40,7 @@ def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub extra_aliases: dict mapping extra alias names to lists of hubs that support them. hubs: list of all concrete hub names. default_hub: the hub to use by default. + testonly: whether all targets should be marked as testonly. """ pkg_name = name @@ -47,6 +48,7 @@ def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub native.alias( name = pkg_name, actual = ":pkg", + testonly = testonly, ) all_aliases = _STANDARD_ALIASES + sorted(extra_aliases.keys()) @@ -100,6 +102,7 @@ def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub native.alias( name = alias_name, actual = select(select_map), + testonly = testonly, ) # Generate missing package error targets diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index 2d54bd34db..fdec8877fc 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -55,6 +55,76 @@ def _test_legacy_aliases(env): _tests.append(_test_legacy_aliases) +def _testonly_mocks(): + got = {} + + def _add(kind): + return lambda name, **kwargs: got.setdefault(name, (kind, kwargs.get("testonly", False))) + + return got, struct( + native = struct( + alias = _add("alias"), + filegroup = _add("filegroup"), + ), + rules = struct( + py_library = _add("py_library"), + ), + ) + +def _test_testonly_aliases(env): + got, mocks = _testonly_mocks() + pkg_aliases( + name = "foo", + actual = "repo", + testonly = True, + native = mocks.native, + rules = mocks.rules, + ) + + # Bazel does not enforce `testonly` on aliases, so the public targets are + # wrapped. + env.expect.that_dict(got).contains_exactly({ + "data": ("filegroup", True), + "dist_info": ("filegroup", True), + "extracted_whl_files": ("filegroup", True), + "foo": ("alias", True), + "pkg": ("py_library", True), + "srcs": ("alias", True), + "whl": ("filegroup", True), + "whl_file": ("alias", True), + "_testonly_data": ("alias", False), + "_testonly_dist_info": ("alias", False), + "_testonly_extracted_whl_files": ("alias", False), + "_testonly_pkg": ("alias", False), + "_testonly_whl": ("alias", False), + }) + +_tests.append(_test_testonly_aliases) + +def _test_testonly_group_aliases(env): + got, mocks = _testonly_mocks() + pkg_aliases( + name = "foo", + actual = "repo", + group_name = "my_group", + testonly = True, + native = mocks.native, + rules = mocks.rules, + ) + + # The private targets are used by the non-testonly group libraries. + env.expect.that_dict(got).contains_at_least({ + "foo": ("alias", True), + "pkg": ("py_library", True), + "whl": ("filegroup", True), + "_pkg": ("alias", False), + "_testonly_pkg": ("alias", False), + "_testonly_whl": ("alias", False), + "_whl": ("alias", False), + }) + +_tests.append(_test_testonly_group_aliases) + def _test_config_setting_aliases(env): # Use this function as it is used in pip_repository got = {} diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index 13bee38ff5..38699f130b 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -58,6 +58,37 @@ pkg_aliases( _tests.append(_test_legacy_aliases) +def _test_testonly_aliases(env): + actual = render_pkg_aliases( + aliases = { + "bar": "pypi_bar", + "foo": "pypi_foo", + }, + testonly_packages = ["Foo"], + ) + + env.expect.that_str(actual["foo/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = ["//visibility:public"]) + +pkg_aliases( + name = "foo", + actual = "pypi_foo", + testonly = True, +)""") + env.expect.that_str(actual["bar/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = ["//visibility:public"]) + +pkg_aliases( + name = "bar", + actual = "pypi_bar", +)""") + +_tests.append(_test_testonly_aliases) + def _test_hidden_aliases(env): actual = render_pkg_aliases( aliases = { From 06306138419466b0d7a0faa8041110a298256efb Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:23:01 +0900 Subject: [PATCH 06/18] feat(pypi): add pip.configure.testonly to make packages testonly The packages selected by a pip.configure tag with `testonly = True` are testonly, unless they are also selected by a tag without `testonly`. --- docs/pypi/download.md | 28 +++++++++++++ news/pip-configure-testonly.added.md | 2 + python/private/pypi/extension.bzl | 29 ++++++++++++- python/private/pypi/hub_exposure.bzl | 20 ++++++--- tests/integration/unified_pypi/BUILD.bazel | 42 +++++++++++++++++++ tests/integration/unified_pypi/MODULE.bazel | 14 +++++++ .../unified_pypi/pyproject_testonly.toml | 6 +++ .../unified_pypi/requirements_testonly.txt | 3 ++ .../unified_pypi/test_iniconfig.py | 3 ++ tests/integration/unified_pypi_test.py | 19 +++++++++ tests/pypi/extension/extension_tests.bzl | 28 ++++++++++++- tests/pypi/extension/pip_parse.bzl | 4 +- .../pypi/hub_exposure/hub_exposure_tests.bzl | 36 ++++++++++++---- 13 files changed, 218 insertions(+), 16 deletions(-) create mode 100644 news/pip-configure-testonly.added.md create mode 100644 tests/integration/unified_pypi/pyproject_testonly.toml create mode 100644 tests/integration/unified_pypi/requirements_testonly.txt create mode 100644 tests/integration/unified_pypi/test_iniconfig.py diff --git a/docs/pypi/download.md b/docs/pypi/download.md index 5f16606d24..52f9ca60a4 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -181,6 +181,34 @@ selects the optional dependencies of the project. The packages that are not part of the hub are ignored, so a single `pyproject.toml` can list tools that are missing from the lock file. +#### Testonly Packages + +Set {attr}`pip.configure.testonly` to make the selected packages testonly, so +that they can only be used by tests and other testonly targets: + +```starlark +pip.configure( + hub_name = "pypi_hub", + dependency_groups = ["default"], +) +pip.configure( + hub_name = "pypi_hub", + dependency_groups = ["group:test"], + testonly = True, +) +``` + +A package that is also selected by a tag without `testonly` is not testonly. +The testonly packages are left out of the `all_requirements`, +`all_whl_requirements` and `all_data_requirements` lists in the hub's +`requirements.bzl` and are listed in `all_testonly_requirements` instead. In +the [Unified @pypi Hub](unified-pypi-hub), a package is testonly if it is +testonly in any of the hubs. + +The other packages of the hub depend on the testonly packages through the same +targets, so a package that a non-testonly package depends on can't be +testonly. + :::{versionadded} VERSION_NEXT_FEATURE ::: diff --git a/news/pip-configure-testonly.added.md b/news/pip-configure-testonly.added.md new file mode 100644 index 0000000000..ebc0594f7e --- /dev/null +++ b/news/pip-configure-testonly.added.md @@ -0,0 +1,2 @@ +(pypi) Added {attr}`pip.configure.testonly` to make the packages selected by +{obj}`pip.configure` testonly. diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index ce5e95492d..7448f6b5c2 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -380,7 +380,8 @@ You cannot use both the additive_build_content and additive_build_content_file a ) if configured_hubs == None: return None - exposed_packages.update(configured_hubs) + for hub_name, info in configured_hubs.items(): + exposed_packages[hub_name] = info.packages return struct( configured_hubs = configured_hubs, @@ -417,7 +418,7 @@ def _create_unified_hub_repo(mods): for hub_name in hubs: # The hubs configured with `pip.configure` only expose some packages. - hub_packages = mods.configured_hubs.get(hub_name, mods.hub_whl_map[hub_name]) + hub_packages = mods.configured_hubs[hub_name].packages if hub_name in mods.configured_hubs else mods.hub_whl_map[hub_name] for pkg_name in hub_packages: norm_pkg = normalize_name(pkg_name) if norm_pkg not in packages: @@ -449,6 +450,13 @@ def _create_unified_hub_repo(mods): hubs = hubs, packages = packages, requirements_bzl = mods.unified_hub_requirements_bzl, + # An alias can't be less testonly than what it points to, so a package + # is testonly if it is testonly in any hub. + testonly_packages = sorted({ + pkg: None + for hub_name in hubs + for pkg in (mods.configured_hubs[hub_name].testonly_packages if hub_name in mods.configured_hubs else []) + }), ) def _pip_impl(module_ctx): @@ -542,6 +550,7 @@ def _pip_impl(module_ctx): groups = mods.hub_group_map.get(hub_name), # The hubs configured with `pip.configure` hide the other packages. hide_unexposed_packages = hub_name in mods.configured_hubs, + testonly_packages = mods.configured_hubs[hub_name].testonly_packages if hub_name in mods.configured_hubs else [], ) _create_unified_hub_repo(mods) @@ -1107,6 +1116,22 @@ The `pyproject.toml` file that defines the dependency groups. Defaults to the {attr}`pip.parse.pyproject_toml` of the hub, then to {attr}`pip.default.pyproject_toml` if the hub is defined by the root module. +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + "testonly": attr.bool( + default = False, + doc = """\ +Make the aliases of the selected packages testonly. A package that is also +selected by a `pip.configure` tag without `testonly` is not testonly. + +The testonly packages are left out of the `all_requirements`, +`all_whl_requirements` and `all_data_requirements` lists in the hub's +`requirements.bzl` and listed in `all_testonly_requirements` instead. In the +[Unified @pypi Hub](unified-pypi-hub), a package is testonly if it is testonly +in any of the hubs. + :::{versionadded} VERSION_NEXT_FEATURE ::: """, diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl index 6ca9315a81..5d5bfae8e9 100644 --- a/python/private/pypi/hub_exposure.bzl +++ b/python/private/pypi/hub_exposure.bzl @@ -20,9 +20,16 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): _fail: {type}`callable` the failure function, mainly for testing. Returns: - {type}`dict[str, list[str]] | None` the configured hubs mapped to the - sorted packages that they expose. The hubs that are not configured - are not included. Returns `None` on failure. + {type}`dict[str, struct] | None` the configured hubs mapped to a + struct with: + * `packages`: {type}`list[str]` the sorted packages that the hub + exposes. + * `testonly_packages`: {type}`list[str]` the sorted packages whose + aliases are testonly. A package is testonly only if all of the + tags that select it are testonly. + + The hubs that are not configured are not included. Returns `None` on + failure. """ # The root module wins over the owner of the hub when both configure it. @@ -76,9 +83,12 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): # The pyproject.toml may list packages that are not in the hub, # e.g. tools that are not locked. if name in info.packages: - exposed[name] = None + exposed[name] = exposed.get(name, True) and tag.testonly return { - hub: sorted(packages) + hub: struct( + packages = sorted(packages), + testonly_packages = sorted([name for name, testonly in packages.items() if testonly]), + ) for hub, packages in configured.items() } diff --git a/tests/integration/unified_pypi/BUILD.bazel b/tests/integration/unified_pypi/BUILD.bazel index 78f9a857ee..e7bbca4f9d 100644 --- a/tests/integration/unified_pypi/BUILD.bazel +++ b/tests/integration/unified_pypi/BUILD.bazel @@ -1,4 +1,5 @@ load("@pypi//:requirements.bzl", "requirement") +load("@pypi_testonly//:requirements.bzl", "all_requirements", "all_testonly_requirements") load("@rules_python//python:py_binary.bzl", "py_binary") load("@rules_python//python:py_test.bzl", "py_test") @@ -123,3 +124,44 @@ py_binary( tags = ["manual"], deps = ["@pypi_filtered//six"], ) + +# The pypi_testonly hub only exposes testonly packages. +py_test( + name = "test_testonly", + srcs = ["test_iniconfig.py"], + main = "test_iniconfig.py", + deps = ["@pypi_testonly//iniconfig"], +) + +py_test( + name = "test_testonly_unified", + srcs = ["test_iniconfig.py"], + config_settings = { + "@rules_python//python/config_settings:venv": "pypi_testonly", + }, + main = "test_iniconfig.py", + deps = ["@pypi//iniconfig"], +) + +py_test( + name = "test_all_testonly_requirements", + srcs = ["test_iniconfig.py"], + main = "test_iniconfig.py", + deps = all_testonly_requirements, +) + +# The testonly packages are not in `all_requirements`. +py_binary( + name = "bin_all_requirements", + srcs = ["test_default.py"], + main = "test_default.py", + deps = ["@pypi//colorama"] + all_requirements, +) + +py_binary( + name = "bin_testonly", + srcs = ["test_iniconfig.py"], + main = "test_iniconfig.py", + tags = ["manual"], + deps = ["@pypi_testonly//iniconfig"], +) diff --git a/tests/integration/unified_pypi/MODULE.bazel b/tests/integration/unified_pypi/MODULE.bazel index 70e01ccdac..dfb19bc749 100644 --- a/tests/integration/unified_pypi/MODULE.bazel +++ b/tests/integration/unified_pypi/MODULE.bazel @@ -67,6 +67,20 @@ pip.configure( ) use_repo(pip, "pypi_filtered") +# `pypi_testonly` only exposes `iniconfig` and it is testonly. +pip.parse( + hub_name = "pypi_testonly", + pyproject_toml = "//:pyproject_testonly.toml", + python_version = "3.11", + requirements_lock = "//:requirements_testonly.txt", +) +pip.configure( + testonly = True, + dependency_groups = ["test"], + hub_name = "pypi_testonly", +) +use_repo(pip, "pypi_testonly") + pip.default(default_hub = "pypi_b") pip.default(unified_hub_requirements_bzl = True) pip.dep( diff --git a/tests/integration/unified_pypi/pyproject_testonly.toml b/tests/integration/unified_pypi/pyproject_testonly.toml new file mode 100644 index 0000000000..0541ccc0b4 --- /dev/null +++ b/tests/integration/unified_pypi/pyproject_testonly.toml @@ -0,0 +1,6 @@ +[project] +name = "unified-pypi" +version = "0.0.0" + +[dependency-groups] +test = ["iniconfig"] diff --git a/tests/integration/unified_pypi/requirements_testonly.txt b/tests/integration/unified_pypi/requirements_testonly.txt new file mode 100644 index 0000000000..862092a324 --- /dev/null +++ b/tests/integration/unified_pypi/requirements_testonly.txt @@ -0,0 +1,3 @@ +iniconfig==2.1.0 \ + --hash=sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7 \ + --hash=sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760 diff --git a/tests/integration/unified_pypi/test_iniconfig.py b/tests/integration/unified_pypi/test_iniconfig.py new file mode 100644 index 0000000000..45bc1a5008 --- /dev/null +++ b/tests/integration/unified_pypi/test_iniconfig.py @@ -0,0 +1,3 @@ +import iniconfig + +assert iniconfig.IniConfig diff --git a/tests/integration/unified_pypi_test.py b/tests/integration/unified_pypi_test.py index f59a19fb42..ea81f3814d 100644 --- a/tests/integration/unified_pypi_test.py +++ b/tests/integration/unified_pypi_test.py @@ -33,6 +33,25 @@ def test_configure_hides_unexposed_packages_in_unified_hub(self): 'ERROR: PyPI package "six" is not available when building under PyPI hub "pypi_filtered"\\. Try adding it to the requirements of this hub', ) + def test_configure_testonly(self): + self.run_bazel( + "test", + "//:test_testonly", + "//:test_testonly_unified", + "//:test_all_testonly_requirements", + ) + self.run_bazel("build", "//:bin_all_requirements") + result = self.run_bazel("build", "//:bin_testonly", check=False) + self.assertNotEqual( + result.exit_code, + 0, + "Expected the testonly package to be rejected in a binary", + ) + self.assert_result_matches( + result, + r"non-test target '//:bin_testonly' depends on testonly", + ) + def test_default_fallback_hub(self): self.run_bazel("test", "//:test_default") diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 17f8d6741e..c9272f4eaf 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -833,10 +833,36 @@ def _test_configure(env): ) pypi.exposed_packages().contains_exactly({"pypi_a": want}) - pypi.configured_hubs().contains_exactly({"pypi_a": want}) + pypi.configured_hubs().contains_exactly({"pypi_a": struct( + packages = want, + testonly_packages = [], + )}) _tests.append(_test_configure) +def _test_configure_testonly(env): + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [_configure_parse()], + configure = [ + _configure(hub_name = "pypi_a", dependency_groups = ["default"]), + _configure(hub_name = "pypi_a", dependency_groups = ["dev", "test"], testonly = True), + ], + ), + ) + + pypi.exposed_packages().contains_exactly({"pypi_a": ["other_pkg", "simple"]}) + + # `simple` is also selected by the non-testonly tag. + pypi.configured_hubs().contains_exactly({"pypi_a": struct( + packages = ["other_pkg", "simple"], + testonly_packages = ["other_pkg"], + )}) + +_tests.append(_test_configure_testonly) + def _test_not_configured_exposes_all(env): # The pyproject.toml does not filter the hub without `pip.configure`. pypi = _configure_parse_modules( diff --git a/tests/pypi/extension/pip_parse.bzl b/tests/pypi/extension/pip_parse.bzl index 505c249169..e28d20875c 100644 --- a/tests/pypi/extension/pip_parse.bzl +++ b/tests/pypi/extension/pip_parse.bzl @@ -4,12 +4,14 @@ def configure( *, dependency_groups = ["*"], hub_name, - pyproject_toml = None): + pyproject_toml = None, + testonly = False): """A simple helper for testing to simulate the PyPI extension configure tag class""" return struct( dependency_groups = dependency_groups, hub_name = hub_name, pyproject_toml = pyproject_toml, + testonly = testonly, ) def pip_parse( diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl index cafbb229b5..2a765b4f89 100644 --- a/tests/pypi/hub_exposure/hub_exposure_tests.bzl +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -23,11 +23,12 @@ def _hub(*, module_name = "root", pyproject = None): exposed_packages = ["other_pkg", "simple", "transitive_pkg"], ) -def _configure(hub_name = "hub", dependency_groups = ["*"], pyproject_toml = None): +def _configure(hub_name = "hub", dependency_groups = ["*"], pyproject_toml = None, testonly = False): return mocks.tag( hub_name = hub_name, dependency_groups = dependency_groups, pyproject_toml = pyproject_toml, + testonly = testonly, ) def _module(name = "root", is_root = True, configure = []): @@ -49,6 +50,9 @@ def _exposure(*modules, hubs = None, default_pyproject = None): ) return got, failures +def _packages(got): + return {hub: info.packages for hub, info in got.items()} + def _test_not_configured(env): got, failures = _exposure(_module()) env.expect.that_collection(failures).contains_exactly([]) @@ -67,7 +71,7 @@ def _test_configure(env): ]: got, failures = _exposure(_module(configure = [_configure(dependency_groups = groups)])) env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) - env.expect.where(groups = groups).that_dict(got).contains_exactly({"hub": want}) + env.expect.where(groups = groups).that_dict(_packages(got)).contains_exactly({"hub": want}) _tests.append(_test_configure) @@ -77,7 +81,7 @@ def _test_configure_combines_tags(env): _configure(dependency_groups = ["dev"]), ])) env.expect.that_collection(failures).contains_exactly([]) - env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg", "simple"]}) + env.expect.that_dict(_packages(got)).contains_exactly({"hub": ["other_pkg", "simple"]}) _tests.append(_test_configure_combines_tags) @@ -86,7 +90,7 @@ def _test_configure_pyproject_precedence(env): got, _ = _exposure(_module(configure = [ _configure(pyproject_toml = "other.toml"), ])) - env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) + env.expect.that_dict(_packages(got)).contains_exactly({"hub": ["other_pkg"]}) # The `pip.default` value is used for the hubs of the root module. got, failures = _exposure( @@ -95,7 +99,7 @@ def _test_configure_pyproject_precedence(env): default_pyproject = "pyproject.toml", ) env.expect.that_collection(failures).contains_exactly([]) - env.expect.that_dict(got).contains_exactly({"hub": ["simple"]}) + env.expect.that_dict(_packages(got)).contains_exactly({"hub": ["simple"]}) _tests.append(_test_configure_pyproject_precedence) @@ -108,7 +112,7 @@ def _test_configure_module_precedence(env): _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), hubs = hubs, ) - env.expect.that_dict(got).contains_exactly({"hub": ["other_pkg"]}) + env.expect.that_dict(_packages(got)).contains_exactly({"hub": ["other_pkg"]}) # The root module wins over the owner. got, _ = _exposure( @@ -116,7 +120,7 @@ def _test_configure_module_precedence(env): _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), hubs = hubs, ) - env.expect.that_dict(got).contains_exactly({"hub": ["simple"]}) + env.expect.that_dict(_packages(got)).contains_exactly({"hub": ["simple"]}) # Other modules are ignored. got, _ = _exposure( @@ -144,6 +148,24 @@ def _test_configure_errors(env): _tests.append(_test_configure_errors) +def _test_configure_testonly(env): + got, failures = _exposure(_module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(dependency_groups = ["dev", "test"], testonly = True), + ])) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(_packages(got)).contains_exactly({"hub": ["other_pkg", "simple"]}) + + # `simple` is also selected by the non-testonly tag. + env.expect.that_collection(got["hub"].testonly_packages).contains_exactly(["other_pkg"]) + + got, _ = _exposure(_module(configure = [ + _configure(dependency_groups = ["*"], testonly = True), + ])) + env.expect.that_collection(got["hub"].testonly_packages).contains_exactly(["other_pkg", "simple"]) + +_tests.append(_test_configure_testonly) + def hub_exposure_test_suite(name): """Create the test suite. From 0abc939ce2448dc8d08f60ec8190ad5ddf97d679 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:27:54 +0900 Subject: [PATCH 07/18] refactor(pypi): support venvs in the unified hub A venv is an extra value of the `venv` flag that resolves the packages to a hub. The hubs and venvs that resolve to the same target share a `config_setting_group` so that the selects stay small. --- python/private/pypi/BUILD.bazel | 1 + python/private/pypi/hub_repository.bzl | 5 +++ python/private/pypi/render_pkg_aliases.bzl | 5 ++- python/private/pypi/unified_hub_repo.bzl | 11 +++++-- python/private/pypi/unified_hub_setup.bzl | 36 ++++++++++++++++++---- 5 files changed, 49 insertions(+), 9 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index c0294975cd..c5f75c8cfb 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -437,6 +437,7 @@ bzl_library( name = "unified_hub_setup", srcs = ["unified_hub_setup.bzl"], deps = [ + "@bazel_skylib//lib:selects", "@rules_python//python/private/pypi:labels", "@rules_python//python/private/pypi:missing_package", ], diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index 91f2c2d0e4..e29615736a 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -37,6 +37,7 @@ def _impl(rctx): for key, values in rctx.attr.whl_map.items() }, exposed_packages = bzl_packages if rctx.attr.hide_unexposed_packages else None, + extra_internal_visibility = rctx.attr.extra_internal_visibility, extra_hub_aliases = rctx.attr.extra_hub_aliases, requirement_cycles = rctx.attr.groups, platform_config_settings = rctx.attr.platform_config_settings, @@ -88,6 +89,10 @@ hub_repository = repository_rule( doc = "Extra aliases to make for specific wheels in the hub repo.", mandatory = True, ), + "extra_internal_visibility": attr.string_list( + default = [], + doc = "The extra visibility of the packages that are not exposed, e.g. for the unified hub.", + ), "groups": attr.string_list_dict( mandatory = False, ), diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 8271387418..439571fc0c 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -95,6 +95,7 @@ def render_pkg_aliases( extra_hub_aliases = {}, exposed_packages = None, testonly_packages = [], + extra_internal_visibility = [], **kwargs): """Create alias declarations for each PyPI package. @@ -112,6 +113,8 @@ def render_pkg_aliases( aliases of the other packages are only visible to the wheel repositories. testonly_packages: The packages whose aliases should be testonly. + extra_internal_visibility: The extra visibility of the packages that + are not exposed. **kwargs: Extra kwargs to pass to the rules. Returns: @@ -144,7 +147,7 @@ def render_pkg_aliases( public = ["//visibility:public"] if exposed_packages != None: exposed_packages = {normalize_name(name): None for name in exposed_packages} - internal_visibility = _internal_visibility(aliases, has_groups = bool(requirement_cycles)) + internal_visibility = _internal_visibility(aliases, has_groups = bool(requirement_cycles)) + extra_internal_visibility files = { "{}/BUILD.bazel".format(normalize_name(name)): _render_common_aliases( diff --git a/python/private/pypi/unified_hub_repo.bzl b/python/private/pypi/unified_hub_repo.bzl index 984c3cacbe..187a1d0f3f 100644 --- a/python/private/pypi/unified_hub_repo.bzl +++ b/python/private/pypi/unified_hub_repo.bzl @@ -25,7 +25,7 @@ define_pypi_package_targets( extra_aliases = {extra_aliases}, hubs = {hubs}, pkg_hubs = {pkg_hubs}, - testonly = {testonly}, + testonly = {testonly},{venvs} ) """ @@ -59,7 +59,9 @@ def dist_info_requirement(name): ''' def _unified_hub_repo_impl(rctx): - hubs = rctx.attr.hubs + # The venvs are selected with the same flag as the hubs. + venvs = rctx.attr.venvs + hubs = rctx.attr.hubs + [venv for venv in sorted(venvs) if venv not in rctx.attr.hubs] default_hub = rctx.attr.default_hub or None # 1. Generate Root BUILD.bazel with shared config settings @@ -95,6 +97,7 @@ def _unified_hub_repo_impl(rctx): pkg_hubs = pkg_hubs, pkg_name = pkg_name, testonly = pkg_name in testonly_packages, + venvs = "\n venvs = {},".format(render.dict(dict(sorted(venvs.items())), key_repr = repr).replace("\n", "\n ")) if venvs else "", ), ) @@ -140,6 +143,10 @@ unified_hub_repo = repository_rule( default = [], doc = "Packages that are testonly in at least one hub.", ), + "venvs": attr.string_dict( + default = {}, + doc = "The venvs mapped to the hubs that provide their packages.", + ), }, doc = "Private repository rule creating the automatic Unified PyPI Hub.", ) diff --git a/python/private/pypi/unified_hub_setup.bzl b/python/private/pypi/unified_hub_setup.bzl index 3ee8ae25d3..28605cbf34 100644 --- a/python/private/pypi/unified_hub_setup.bzl +++ b/python/private/pypi/unified_hub_setup.bzl @@ -1,5 +1,6 @@ """Helper functions for setting up targets within the Unified PyPI Hub repository.""" +load("@bazel_skylib//lib:selects.bzl", "selects") load( "@rules_python//python/private/pypi:labels.bzl", "DATA_LABEL", @@ -31,16 +32,19 @@ _STANDARD_ALIASES = [ EXTRACTED_WHEEL_FILES, ] -def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub = None, testonly = False): +def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub = None, testonly = False, venvs = {}): """Define the targets for a PyPI package in the unified PyPI hub. Args: name: normalized PyPI package name, serving as the main target name. pkg_hubs: list of hubs that contain this package. extra_aliases: dict mapping extra alias names to lists of hubs that support them. - hubs: list of all concrete hub names. + hubs: list of all concrete hub and venv names. default_hub: the hub to use by default. testonly: whether all targets should be marked as testonly. + venvs: dict mapping the venv names to the hubs that provide their + packages. The hubs and venvs that resolve to the same target + share a `config_setting_group`. """ pkg_name = name @@ -53,9 +57,11 @@ def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub all_aliases = _STANDARD_ALIASES + sorted(extra_aliases.keys()) missing_errors = {} + groups = {} for alias_name in all_aliases: - select_map = {} + # The target mapped to the hubs and venvs that resolve to it. + targets = {} for hub in hubs: is_supported = ( (alias_name in _STANDARD_ALIASES and hub in pkg_hubs) or @@ -63,8 +69,8 @@ def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub ) if is_supported: - select_map["//:_is_venv_" + hub] = "@{hub}//{pkg}:{alias}".format( - hub = hub, + target = "@{hub}//{pkg}:{alias}".format( + hub = venvs.get(hub, hub), pkg = pkg_name, alias = alias_name, ) @@ -75,7 +81,18 @@ def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub "hub_name": hub, "package_name": pkg_name if alias_name in _STANDARD_ALIASES else (pkg_name + ":" + alias_name), } - select_map["//:_is_venv_" + hub] = ":{}".format(err_target) + target = ":{}".format(err_target) + targets.setdefault(target, []).append(hub) + + select_map = {} + for target, target_hubs in targets.items(): + if len(target_hubs) == 1: + select_map["//:_is_venv_" + target_hubs[0]] = target + continue + + group = "_is_venv_" + "_or_".join(target_hubs) + groups[group] = target_hubs + select_map[":" + group] = target # //conditions:default fallback default_supported = ( @@ -105,6 +122,13 @@ def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub testonly = testonly, ) + for group, group_hubs in groups.items(): + selects.config_setting_group( + name = group, + match_any = ["//:_is_venv_" + hub for hub in group_hubs], + visibility = ["//visibility:private"], + ) + # Generate missing package error targets for err_name, err_args in missing_errors.items(): missing_package_error( From c96fadabb753809e1d9d111b0a1a44e002103e26 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 02:28:00 +0900 Subject: [PATCH 08/18] feat(pypi): add pip.configure.venv to create named package selections A venv is a named selection of the packages of a hub that is used through the unified @pypi hub by setting the `venv` flag. It does not create a new repository and it does not change the hub. --- docs/pypi/download.md | 23 ++++++++ news/pip-configure-venvs.added.md | 3 ++ python/private/pypi/extension.bzl | 52 ++++++++++++++++--- python/private/pypi/hub_exposure.bzl | 48 +++++++++++++---- tests/integration/unified_pypi/BUILD.bazel | 33 ++++++++++++ tests/integration/unified_pypi/MODULE.bazel | 14 +++++ .../unified_pypi/pyproject_venvs.toml | 7 +++ tests/integration/unified_pypi_test.py | 16 ++++++ tests/pypi/extension/extension_tests.bzl | 28 ++++++++++ tests/pypi/extension/pip_parse.bzl | 4 +- .../pypi/hub_exposure/hub_exposure_tests.bzl | 38 +++++++++++++- 11 files changed, 248 insertions(+), 18 deletions(-) create mode 100644 news/pip-configure-venvs.added.md create mode 100644 tests/integration/unified_pypi/pyproject_venvs.toml diff --git a/docs/pypi/download.md b/docs/pypi/download.md index 52f9ca60a4..c4f975ddfe 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -209,6 +209,29 @@ The other packages of the hub depend on the testonly packages through the same targets, so a package that a non-testonly package depends on can't be testonly. +#### Venvs + +Set {attr}`pip.configure.venv` to create a venv, a named selection of the +packages of a hub, instead of changing the hub: + +```starlark +pip.configure( + hub_name = "pypi_hub", + venv = "pypi_hub_docs", + dependency_groups = ["default", "group:docs"], +) +``` + +The venv is used through the [Unified @pypi Hub](unified-pypi-hub) by setting +`--@rules_python//python/config_settings:venv=pypi_hub_docs`. It resolves the +`@pypi//` labels to the packages of `pypi_hub`, so nothing is downloaded +twice, and the packages that are not selected are reported as missing. A venv +does not create a new repository and does not change the hub. + +A venv name must not collide with a hub and a venv can't be used with two +different hubs. The packages of a venv are testonly if they are testonly in +the hub. + :::{versionadded} VERSION_NEXT_FEATURE ::: diff --git a/news/pip-configure-venvs.added.md b/news/pip-configure-venvs.added.md new file mode 100644 index 0000000000..add79e6689 --- /dev/null +++ b/news/pip-configure-venvs.added.md @@ -0,0 +1,3 @@ +(pypi) Added {attr}`pip.configure.venv` to create a named selection of the +packages of a hub that can be used through the unified `@pypi` hub with the +{flag}`--venv` flag. diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 7448f6b5c2..bd04a08ce8 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -372,19 +372,26 @@ You cannot use both the additive_build_content and additive_build_content_file a exposed_packages = out.exposed_packages, ) - configured_hubs = hub_exposure( + configured = hub_exposure( module_ctx, hubs = hubs, default_pyproject = config.pyproject_toml, _fail = _fail, ) - if configured_hubs == None: + if configured == None: return None - for hub_name, info in configured_hubs.items(): - exposed_packages[hub_name] = info.packages + configured_hubs = {} + venvs = {} + for name, info in configured.items(): + if name == info.hub: + configured_hubs[name] = info + exposed_packages[name] = info.packages + else: + venvs[name] = info return struct( configured_hubs = configured_hubs, + venvs = venvs, config = config, declared_deps = declared_deps, default_hub = config.default_hub or renamed_default_hub, @@ -416,9 +423,17 @@ def _create_unified_hub_repo(mods): packages = {} extra_aliases = {} - for hub_name in hubs: + # The venvs are named selections of the packages of a hub, which can be + # selected with the same `venv` flag as the hubs. + venvs = {name: info.hub for name, info in mods.venvs.items()} + for hub_name in hubs + sorted(venvs): # The hubs configured with `pip.configure` only expose some packages. - hub_packages = mods.configured_hubs[hub_name].packages if hub_name in mods.configured_hubs else mods.hub_whl_map[hub_name] + if hub_name in venvs: + hub_packages = mods.venvs[hub_name].packages + elif hub_name in mods.configured_hubs: + hub_packages = mods.configured_hubs[hub_name].packages + else: + hub_packages = mods.hub_whl_map[hub_name] for pkg_name in hub_packages: norm_pkg = normalize_name(pkg_name) if norm_pkg not in packages: @@ -426,7 +441,7 @@ def _create_unified_hub_repo(mods): if hub_name not in packages[norm_pkg]: packages[norm_pkg].append(hub_name) - extra = mods.extra_aliases.get(hub_name, {}).get(norm_pkg, []) + extra = mods.extra_aliases.get(venvs.get(hub_name, hub_name), {}).get(norm_pkg, []) for alias_name in extra: qual_alias = "%s:%s" % (norm_pkg, alias_name) if qual_alias not in extra_aliases: @@ -450,6 +465,7 @@ def _create_unified_hub_repo(mods): hubs = hubs, packages = packages, requirements_bzl = mods.unified_hub_requirements_bzl, + venvs = venvs, # An alias can't be less testonly than what it points to, so a package # is testonly if it is testonly in any hub. testonly_packages = sorted({ @@ -550,6 +566,8 @@ def _pip_impl(module_ctx): groups = mods.hub_group_map.get(hub_name), # The hubs configured with `pip.configure` hide the other packages. hide_unexposed_packages = hub_name in mods.configured_hubs, + # The venvs may select the packages that the hub does not expose. + extra_internal_visibility = ["@pypi//:__subpackages__"] if hub_name in [v.hub for v in mods.venvs.values()] else [], testonly_packages = mods.configured_hubs[hub_name].testonly_packages if hub_name in mods.configured_hubs else [], ) @@ -1132,6 +1150,26 @@ The testonly packages are left out of the `all_requirements`, [Unified @pypi Hub](unified-pypi-hub), a package is testonly if it is testonly in any of the hubs. +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + "venv": attr.string( + mandatory = False, + doc = """\ +Create a venv instead of configuring the hub. Defaults to {attr}`hub_name`. + +A venv is a named selection of the packages of the hub that is used through +the [Unified @pypi Hub](unified-pypi-hub) by setting +{flag}`--@rules_python//python/config_settings:venv` to its name. It does not +create a new repository and the hub itself is not changed. The packages that +are not selected are reported as missing, like the packages that are not in a +hub. + +The name must not collide with a hub and a venv can't be used with two +different hubs. Multiple tags with the same `venv` are combined. `testonly` is +not supported for venvs. + :::{versionadded} VERSION_NEXT_FEATURE ::: """, diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl index 5d5bfae8e9..c465eb0e21 100644 --- a/python/private/pypi/hub_exposure.bzl +++ b/python/private/pypi/hub_exposure.bzl @@ -20,8 +20,10 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): _fail: {type}`callable` the failure function, mainly for testing. Returns: - {type}`dict[str, struct] | None` the configured hubs mapped to a - struct with: + {type}`dict[str, struct] | None` the configured hubs and venvs mapped + to a struct with: + * `hub`: {type}`str` the hub that provides the packages. For a venv + this is different from the key. * `packages`: {type}`list[str]` the sorted packages that the hub exposes. * `testonly_packages`: {type}`list[str]` the sorted packages whose @@ -30,6 +32,9 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): The hubs that are not configured are not included. Returns `None` on failure. + + A venv is a named selection of the packages of a hub that is used + through the unified `@pypi` hub. """ # The root module wins over the owner of the hub when both configure it. @@ -39,6 +44,7 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): tag.hub_name: None for mod in root_modules for tag in mod.tags.configure + if tag.venv in ("", tag.hub_name) } configured = {} @@ -53,10 +59,18 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): return None info = hubs[hub] - if not mod.is_root and (mod.name != info.module_name or hub in root_configured): + venv = tag.venv or hub + if not mod.is_root and mod.name != info.module_name: # Only the root module and the module that created the hub can # configure it. continue + if venv == hub and not mod.is_root and hub in root_configured: + continue + if venv != hub: + err = _venv_error(venv, hub, hubs = hubs, configured = configured, testonly = tag.testonly) + if err: + _fail(err) + return None pyproject = ( tag.pyproject_toml or @@ -77,8 +91,8 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): else: packages = selection.packages - # Multiple tags for the same hub are combined. - exposed = configured.setdefault(hub, {}) + # Multiple tags for the same hub or venv are combined. + exposed = configured.setdefault(venv, struct(hub = hub, packages = {})).packages for name in packages: # The pyproject.toml may list packages that are not in the hub, # e.g. tools that are not locked. @@ -86,9 +100,25 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): exposed[name] = exposed.get(name, True) and tag.testonly return { - hub: struct( - packages = sorted(packages), - testonly_packages = sorted([name for name, testonly in packages.items() if testonly]), + name: struct( + hub = info.hub, + packages = sorted(info.packages), + testonly_packages = sorted([pkg for pkg, testonly in info.packages.items() if testonly]), ) - for hub, packages in configured.items() + for name, info in configured.items() } + +def _venv_error(venv, hub, *, hubs, configured, testonly): + if venv in hubs: + return "pip.configure() venv '{}' conflicts with the hub of the same name".format(venv) + if venv in ("pypi", "auto"): + return "pip.configure() venv name '{}' is reserved".format(venv) + if venv in configured and configured[venv].hub != hub: + return "pip.configure() venv '{}' is used with both hub '{}' and hub '{}'".format( + venv, + configured[venv].hub, + hub, + ) + if testonly: + return "pip.configure() does not support `testonly` for the venv '{}', the packages are testonly only if they are testonly in the hub '{}'".format(venv, hub) + return None diff --git a/tests/integration/unified_pypi/BUILD.bazel b/tests/integration/unified_pypi/BUILD.bazel index e7bbca4f9d..d57b615787 100644 --- a/tests/integration/unified_pypi/BUILD.bazel +++ b/tests/integration/unified_pypi/BUILD.bazel @@ -165,3 +165,36 @@ py_binary( tags = ["manual"], deps = ["@pypi_testonly//iniconfig"], ) + +# The venvs of pypi_b resolve to the pypi_b packages. +py_test( + name = "test_venv_colorama", + srcs = ["test_default.py"], + config_settings = { + "@rules_python//python/config_settings:venv": "pypi_b_colorama", + }, + main = "test_default.py", + deps = ["@pypi//colorama"], +) + +py_test( + name = "test_venv_dev", + srcs = ["test_default.py"], + config_settings = { + "@rules_python//python/config_settings:venv": "pypi_b_dev", + }, + main = "test_default.py", + deps = ["@pypi//colorama"], +) + +# six is not selected by the pypi_b_colorama venv: +py_binary( + name = "bin_venv_six", + srcs = ["bin_six_a.py"], + config_settings = { + "@rules_python//python/config_settings:venv": "pypi_b_colorama", + }, + main = "bin_six_a.py", + tags = ["manual"], + deps = ["@pypi//six"], +) diff --git a/tests/integration/unified_pypi/MODULE.bazel b/tests/integration/unified_pypi/MODULE.bazel index dfb19bc749..26b02f9f25 100644 --- a/tests/integration/unified_pypi/MODULE.bazel +++ b/tests/integration/unified_pypi/MODULE.bazel @@ -53,6 +53,20 @@ pip.parse( ) use_repo(pip, "pypi_b") +# The venvs are selections of the `pypi_b` packages that are used through the +# unified hub. +pip.configure( + dependency_groups = ["default"], + hub_name = "pypi_b", + pyproject_toml = "//:pyproject_venvs.toml", + venv = "pypi_b_colorama", +) +pip.configure( + hub_name = "pypi_b", + pyproject_toml = "//:pyproject_venvs.toml", + venv = "pypi_b_dev", +) + # `pypi_filtered` only exposes the `[project] dependencies` of the # `pyproject.toml`, the other packages are not public. pip.parse( diff --git a/tests/integration/unified_pypi/pyproject_venvs.toml b/tests/integration/unified_pypi/pyproject_venvs.toml new file mode 100644 index 0000000000..55420aa1b5 --- /dev/null +++ b/tests/integration/unified_pypi/pyproject_venvs.toml @@ -0,0 +1,7 @@ +[project] +name = "unified-pypi" +version = "0.0.0" +dependencies = ["colorama"] + +[dependency-groups] +dev = ["six"] diff --git a/tests/integration/unified_pypi_test.py b/tests/integration/unified_pypi_test.py index ea81f3814d..6a394a20c5 100644 --- a/tests/integration/unified_pypi_test.py +++ b/tests/integration/unified_pypi_test.py @@ -52,6 +52,22 @@ def test_configure_testonly(self): r"non-test target '//:bin_testonly' depends on testonly", ) + def test_configure_venvs(self): + self.run_bazel( + "test", "//:test_venv_colorama", "//:test_venv_dev", "//:test_default" + ) + self.run_bazel("cquery", "//:bin_venv_six") + result = self.run_bazel("build", "//:bin_venv_six", check=False) + self.assertNotEqual( + result.exit_code, + 0, + "Expected build to fail during execution phase", + ) + self.assert_result_matches( + result, + 'ERROR: PyPI package "six" is not available when building under PyPI hub "pypi_b_colorama"\\. Try adding it to the requirements of this hub', + ) + def test_default_fallback_hub(self): self.run_bazel("test", "//:test_default") diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index c9272f4eaf..771c6593fe 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -120,6 +120,7 @@ def _parse_modules(env, **kwargs): parse_modules(**kwargs), attrs = dict( configured_hubs = subjects.dict, + venvs = subjects.dict, declared_deps = subjects.dict, default_hub = subjects.str, exposed_packages = subjects.dict, @@ -834,6 +835,7 @@ def _test_configure(env): pypi.exposed_packages().contains_exactly({"pypi_a": want}) pypi.configured_hubs().contains_exactly({"pypi_a": struct( + hub = "pypi_a", packages = want, testonly_packages = [], )}) @@ -857,6 +859,7 @@ def _test_configure_testonly(env): # `simple` is also selected by the non-testonly tag. pypi.configured_hubs().contains_exactly({"pypi_a": struct( + hub = "pypi_a", packages = ["other_pkg", "simple"], testonly_packages = ["other_pkg"], )}) @@ -880,6 +883,31 @@ def _test_not_configured_exposes_all(env): _tests.append(_test_not_configured_exposes_all) +def _test_configure_venvs(env): + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [_configure_parse()], + configure = [ + _configure(hub_name = "pypi_a", venv = "venv_dev", dependency_groups = ["dev"]), + ], + ), + ) + + # The hub itself is not configured. + pypi.exposed_packages().contains_exactly({ + "pypi_a": ["other_pkg", "simple", "transitive_pkg"], + }) + pypi.configured_hubs().contains_exactly({}) + pypi.venvs().contains_exactly({"venv_dev": struct( + hub = "pypi_a", + packages = ["other_pkg"], + testonly_packages = [], + )}) + +_tests.append(_test_configure_venvs) + def extension_test_suite(name): """Create the test suite. diff --git a/tests/pypi/extension/pip_parse.bzl b/tests/pypi/extension/pip_parse.bzl index e28d20875c..1c6a66727e 100644 --- a/tests/pypi/extension/pip_parse.bzl +++ b/tests/pypi/extension/pip_parse.bzl @@ -5,13 +5,15 @@ def configure( dependency_groups = ["*"], hub_name, pyproject_toml = None, - testonly = False): + testonly = False, + venv = ""): """A simple helper for testing to simulate the PyPI extension configure tag class""" return struct( dependency_groups = dependency_groups, hub_name = hub_name, pyproject_toml = pyproject_toml, testonly = testonly, + venv = venv, ) def pip_parse( diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl index 2a765b4f89..b760ec9393 100644 --- a/tests/pypi/hub_exposure/hub_exposure_tests.bzl +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -23,12 +23,13 @@ def _hub(*, module_name = "root", pyproject = None): exposed_packages = ["other_pkg", "simple", "transitive_pkg"], ) -def _configure(hub_name = "hub", dependency_groups = ["*"], pyproject_toml = None, testonly = False): +def _configure(hub_name = "hub", dependency_groups = ["*"], pyproject_toml = None, testonly = False, venv = ""): return mocks.tag( hub_name = hub_name, dependency_groups = dependency_groups, pyproject_toml = pyproject_toml, testonly = testonly, + venv = venv, ) def _module(name = "root", is_root = True, configure = []): @@ -166,6 +167,41 @@ def _test_configure_testonly(env): _tests.append(_test_configure_testonly) +def _test_configure_venvs(env): + got, failures = _exposure(_module(configure = [ + _configure(venv = "dev", dependency_groups = ["dev"]), + _configure(venv = "dev", dependency_groups = ["default"]), + _configure(venv = "main", dependency_groups = ["default"]), + ])) + env.expect.that_collection(failures).contains_exactly([]) + + # The hub itself is not configured. + env.expect.that_dict(_packages(got)).contains_exactly({ + "dev": ["other_pkg", "simple"], + "main": ["simple"], + }) + env.expect.that_str(got["dev"].hub).equals("hub") + env.expect.that_str(got["main"].hub).equals("hub") + +_tests.append(_test_configure_venvs) + +def _test_configure_venv_errors(env): + hubs = { + "hub": _hub(pyproject = "pyproject.toml"), + "other": _hub(pyproject = "pyproject.toml"), + } + for configure, want in [ + ([_configure(venv = "other")], "pip.configure() venv 'other' conflicts with the hub of the same name"), + ([_configure(venv = "pypi")], "pip.configure() venv name 'pypi' is reserved"), + ([_configure(venv = "auto")], "pip.configure() venv name 'auto' is reserved"), + ([_configure(venv = "dev"), _configure(hub_name = "other", venv = "dev")], "pip.configure() venv 'dev' is used with both hub 'hub' and hub 'other'"), + ([_configure(venv = "dev", testonly = True)], "pip.configure() does not support `testonly` for the venv 'dev', the packages are testonly only if they are testonly in the hub 'hub'"), + ]: + _, failures = _exposure(_module(configure = configure), hubs = hubs) + env.expect.that_collection(failures).contains_exactly([want]) + +_tests.append(_test_configure_venv_errors) + def hub_exposure_test_suite(name): """Create the test suite. From b31a734fd273538849adba68184df7b801212fce Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:05:33 +0900 Subject: [PATCH 09/18] fix(pypi): treat exposing all packages as not configuring the hub pip.configure with ["*", "transitive"] used the packages that are available on all platforms, which made the platform-specific packages private. Treat it as if the hub was not configured instead, which keeps these packages public and out of all_requirements as before. --- docs/pypi/download.md | 5 ++-- python/private/pypi/extension.bzl | 1 - python/private/pypi/hub_exposure.bzl | 24 +++++++++---------- .../pypi/hub_exposure/hub_exposure_tests.bzl | 14 +++++++++-- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/docs/pypi/download.md b/docs/pypi/download.md index 5f16606d24..654e156921 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -171,8 +171,9 @@ dependencies of `numpy` are only used through `@pypi_hub//numpy`. The The {attr}`pip.configure.dependency_groups` attribute defaults to `["*"]`, which selects every group. Use `["*", "transitive"]` to expose all of the -packages of the hub, including the transitive dependencies. Selecting the -transitive dependencies of only some of the groups is not supported. +packages of the hub, including the transitive dependencies, which is the same +as not configuring the hub. Selecting the transitive dependencies of only some +of the groups is not supported. Multiple tags for the same hub are combined. Only the root module and the module that created the hub can configure it; if both do, the root module diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index ce5e95492d..10e0c456a2 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -369,7 +369,6 @@ You cannot use both the additive_build_content and additive_build_content_file a module_name = hub.module_name, pyproject = hub_pyproject.get(hub.name), packages = out.whl_map.keys(), - exposed_packages = out.exposed_packages, ) configured_hubs = hub_exposure( diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl index 6ca9315a81..fe9e586153 100644 --- a/python/private/pypi/hub_exposure.bzl +++ b/python/private/pypi/hub_exposure.bzl @@ -13,28 +13,27 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): * `pyproject`: {type}`Label | None` from `pip.parse`. * `packages`: {type}`list[str]` the normalized names of all packages in the hub. - * `exposed_packages`: {type}`list[str]` the packages that the hub - exposes when it is not configured. default_pyproject: {type}`Label | None` from `pip.default`, only used for the hubs of the root module. _fail: {type}`callable` the failure function, mainly for testing. Returns: {type}`dict[str, list[str]] | None` the configured hubs mapped to the - sorted packages that they expose. The hubs that are not configured - are not included. Returns `None` on failure. + sorted packages that they expose. The hubs that are not configured or + that expose all of their packages are not included, so that they + behave the same. Returns `None` on failure. """ # The root module wins over the owner of the hub when both configure it. root_modules = [mod for mod in module_ctx.modules if mod.is_root] root_name = root_modules[0].name if root_modules else None - root_configured = { - tag.hub_name: None - for mod in root_modules - for tag in mod.tags.configure - } + root_configured = {} + for mod in root_modules: + for tag in mod.tags.configure: + root_configured[tag.hub_name] = None configured = {} + exposes_all = {} for mod in module_ctx.modules: for tag in mod.tags.configure: hub = tag.hub_name @@ -66,13 +65,11 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): _fail = _fail, ) if selection.all: - packages = info.exposed_packages - else: - packages = selection.packages + exposes_all[hub] = None # Multiple tags for the same hub are combined. exposed = configured.setdefault(hub, {}) - for name in packages: + for name in selection.packages: # The pyproject.toml may list packages that are not in the hub, # e.g. tools that are not locked. if name in info.packages: @@ -81,4 +78,5 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): return { hub: sorted(packages) for hub, packages in configured.items() + if hub not in exposes_all } diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl index cafbb229b5..5d84a351b5 100644 --- a/tests/pypi/hub_exposure/hub_exposure_tests.bzl +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -20,7 +20,6 @@ def _hub(*, module_name = "root", pyproject = None): module_name = module_name, pyproject = pyproject, packages = ["other_pkg", "simple", "transitive_pkg"], - exposed_packages = ["other_pkg", "simple", "transitive_pkg"], ) def _configure(hub_name = "hub", dependency_groups = ["*"], pyproject_toml = None): @@ -63,7 +62,6 @@ def _test_configure(env): (["dev"], ["other_pkg"]), # `not-in-hub` is ignored (["group:test"], ["simple"]), - (["*", "transitive"], ["other_pkg", "simple", "transitive_pkg"]), ]: got, failures = _exposure(_module(configure = [_configure(dependency_groups = groups)])) env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) @@ -71,6 +69,18 @@ def _test_configure(env): _tests.append(_test_configure) +def _test_configure_all_packages(env): + # Exposing all of the packages is the same as not configuring the hub, so + # that e.g. the platform-specific packages stay public. + got, failures = _exposure(_module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(dependency_groups = ["*", "transitive"]), + ])) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({}) + +_tests.append(_test_configure_all_packages) + def _test_configure_combines_tags(env): got, failures = _exposure(_module(configure = [ _configure(dependency_groups = ["default"]), From 9d97360242b44f6a73472793d17a17287cabb1f3 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:05:56 +0900 Subject: [PATCH 10/18] refactor(pypi): use explicit loops and simpler conditions in pip.configure Replace the nested comprehensions and the long inline conditions with explicit loops and use types.is_string. --- python/private/pypi/BUILD.bazel | 1 + python/private/pypi/dependency_groups.bzl | 19 ++++++++----------- python/private/pypi/hub_repository.bzl | 4 +++- python/private/pypi/render_pkg_aliases.bzl | 19 +++++++++++-------- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index caf482b67f..fe28b17413 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -639,6 +639,7 @@ bzl_library( deps = [ ":pep508_requirement", "//python/private:normalize_name", + "@bazel_skylib//lib:types", ], ) diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl index 2ddadee3dd..4e40c7e856 100644 --- a/python/private/pypi/dependency_groups.bzl +++ b/python/private/pypi/dependency_groups.bzl @@ -1,5 +1,6 @@ """Select PyPI packages using pyproject.toml dependency groups.""" +load("@bazel_skylib//lib:types.bzl", "types") load("//python/private:normalize_name.bzl", "normalize_name") load(":pep508_requirement.bzl", "requirement") @@ -59,16 +60,12 @@ def select_dependency_groups(pyproject, specs, _fail = fail): for key in selected: for req in groups[key]: req = requirement(req) - names = [req.name_] - if req.name_ == project: - names = [ - requirement(r).name_ - for extra in req.extras - for r in groups.get("optional:" + extra, []) - ] - for name in names: - if name: - packages[name] = None + if req.name_ != project: + packages[req.name_] = None + continue + for extra in req.extras: + for r in groups.get("optional:" + extra, []): + packages[requirement(r).name_] = None return struct(packages = sorted(packages), all = False) @@ -119,7 +116,7 @@ def _expand_dependency_groups(dependency_groups, _fail): reqs = [] for item in items: - if type(item) == type(""): + if types.is_string(item): reqs.append(item) continue diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index d421f3cc09..ff5311c6ea 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -28,14 +28,16 @@ exports_files(["requirements.bzl"]) def _impl(rctx): if rctx.attr.hide_unexposed_packages: bzl_packages = rctx.attr.packages + exposed_packages = bzl_packages else: bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() + exposed_packages = None aliases = render_multiplatform_pkg_aliases( aliases = { key: _whl_config_settings_from_json(values) for key, values in rctx.attr.whl_map.items() }, - exposed_packages = bzl_packages if rctx.attr.hide_unexposed_packages else None, + exposed_packages = exposed_packages, extra_hub_aliases = rctx.attr.extra_hub_aliases, requirement_cycles = rctx.attr.groups, platform_config_settings = rctx.attr.platform_config_settings, diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 5582cc00c9..2c86155876 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -139,17 +139,20 @@ def render_pkg_aliases( exposed_packages = {normalize_name(name): None for name in exposed_packages} internal_visibility = _internal_visibility(aliases, has_groups = bool(requirement_cycles)) - files = { - "{}/BUILD.bazel".format(normalize_name(name)): _render_common_aliases( - name = normalize_name(name), + files = {} + for name, pkg_aliases in aliases.items(): + name = normalize_name(name) + visibility = public + if exposed_packages != None and name not in exposed_packages: + visibility = internal_visibility + files["{}/BUILD.bazel".format(name)] = _render_common_aliases( + name = name, aliases = pkg_aliases, - extra_aliases = extra_hub_aliases.get(normalize_name(name), []), - group_name = whl_group_mapping.get(normalize_name(name)), - visibility = public if exposed_packages == None or normalize_name(name) in exposed_packages else internal_visibility, + extra_aliases = extra_hub_aliases.get(name, []), + group_name = whl_group_mapping.get(name), + visibility = visibility, **kwargs ).strip() - for name, pkg_aliases in aliases.items() - } if requirement_cycles: files["_groups/BUILD.bazel"] = generate_group_library_build_bazel("", requirement_cycles) From 35c066052cb0bea531aba78784149420d8056aaa Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:06:00 +0900 Subject: [PATCH 11/18] test(pypi): test the pip.default pyproject_toml with pip.configure Keep a single wiring test for pip.configure in the extension tests, as the selection is tested in hub_exposure_tests, and test that the pip.default pyproject_toml is used. Also link the issue in the news entry. --- news/4395.added.md | 3 +- tests/pypi/extension/extension_tests.bzl | 49 +++++++++++++++--------- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/news/4395.added.md b/news/4395.added.md index 62d7f10880..3e633cc997 100644 --- a/news/4395.added.md +++ b/news/4395.added.md @@ -1,2 +1,3 @@ (pypi) Added {obj}`pip.configure` to limit the packages that a hub exposes to -the ones listed in the dependency groups of a `pyproject.toml` file. +the ones listed in the dependency groups of a `pyproject.toml` file +([#3413](https://github.com/bazel-contrib/rules_python/issues/3413)). diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 17f8d6741e..680853a0d0 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -807,36 +807,47 @@ def _configure_parse_modules(env, *modules): minor_mapping = {"3.15": "3.15.19"}, ) -def _configure_parse(*, hub_name = "pypi_a", pyproject_toml = "pyproject.toml"): +def _configure_parse(**kwargs): return _parse( - hub_name = hub_name, - pyproject_toml = pyproject_toml, + hub_name = "pypi_a", python_version = "3.15", simpleapi_skip = ["simple", "other-pkg", "transitive-pkg"], requirements_lock = "requirements.txt", + **kwargs ) def _test_configure(env): - for groups, want in [ - (["*"], ["other_pkg", "simple"]), - (["test"], ["simple"]), - (["dev"], ["other_pkg"]), - (["*", "transitive"], ["other_pkg", "simple", "transitive_pkg"]), - ]: - pypi = _configure_parse_modules( - env, - _mod( - name = "rules_python", - parse = [_configure_parse()], - configure = [_configure(hub_name = "pypi_a", dependency_groups = groups)], - ), - ) + # The selection itself is tested in `hub_exposure_tests`. + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [_configure_parse(pyproject_toml = "pyproject.toml")], + configure = [_configure(hub_name = "pypi_a", dependency_groups = ["test"])], + ), + ) - pypi.exposed_packages().contains_exactly({"pypi_a": want}) - pypi.configured_hubs().contains_exactly({"pypi_a": want}) + pypi.exposed_packages().contains_exactly({"pypi_a": ["simple"]}) + pypi.configured_hubs().contains_exactly({"pypi_a": ["simple"]}) _tests.append(_test_configure) +def _test_configure_default_pyproject(env): + # The `pip.default` pyproject.toml is used for the hubs of the root module. + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + default = _default_tags_default + [_default(pyproject_toml = "pyproject.toml")], + parse = [_configure_parse()], + configure = [_configure(hub_name = "pypi_a", dependency_groups = ["dev"])], + ), + ) + + pypi.configured_hubs().contains_exactly({"pypi_a": ["other_pkg"]}) + +_tests.append(_test_configure_default_pyproject) + def _test_not_configured_exposes_all(env): # The pyproject.toml does not filter the hub without `pip.configure`. pypi = _configure_parse_modules( From 003655c8ab6329907b1953ece8094c24241bed2e Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:08:42 +0900 Subject: [PATCH 12/18] refactor(pypi): drop the unreachable testonly check in pkg_aliases --- news/{pip-configure-testonly.added.md => 4428.added.md} | 0 python/private/pypi/pkg_aliases.bzl | 5 +---- 2 files changed, 1 insertion(+), 4 deletions(-) rename news/{pip-configure-testonly.added.md => 4428.added.md} (100%) diff --git a/news/pip-configure-testonly.added.md b/news/4428.added.md similarity index 100% rename from news/pip-configure-testonly.added.md rename to news/4428.added.md diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index b838434088..f165e5dffe 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -124,13 +124,10 @@ def pkg_aliases( _native.alias( name = name, actual = ":" + PY_LIBRARY_PUBLIC_LABEL, - **({"testonly": True} if testonly else {}) + testonly = testonly, ) if requires_dist: - if testonly: - fail("testonly is not supported together with requires_dist") - # if it has a group_name set, then it will create the impl label target in the # current macro, we still need to do the aliases to the actual groups package as # per below From 5e4c8098726042175cd265302bff4df5e4c6697d Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:08:46 +0900 Subject: [PATCH 13/18] refactor(pypi): compute the testonly packages once in parse_modules Compute the testonly packages of each hub and of the unified hub in parse_modules instead of using long inline conditions, and test that a package is testonly in the unified hub if it is testonly in any hub. --- python/private/pypi/extension.bzl | 24 ++++++++++++-------- tests/pypi/extension/extension_tests.bzl | 29 ++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index ddbb43ebc4..9479d08eaf 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -379,8 +379,16 @@ You cannot use both the additive_build_content and additive_build_content_file a ) if configured_hubs == None: return None + testonly_packages = {} + unified_testonly_packages = {} for hub_name, info in configured_hubs.items(): exposed_packages[hub_name] = info.packages + testonly_packages[hub_name] = info.testonly_packages + + # An alias can't be less testonly than what it points to, so a package + # is testonly in the unified hub if it is testonly in any hub. + for pkg in info.testonly_packages: + unified_testonly_packages[pkg] = None return struct( configured_hubs = configured_hubs, @@ -392,7 +400,9 @@ You cannot use both the additive_build_content and additive_build_content_file a facts = simpleapi_cache.get_facts(), hub_group_map = hub_group_map, hub_whl_map = hub_whl_map, + testonly_packages = testonly_packages, unified_hub_requirements_bzl = config.unified_hub_requirements_bzl, + unified_testonly_packages = sorted(unified_testonly_packages), whl_libraries = whl_libraries, whl_mods = whl_mods, platform_config_settings = { @@ -417,7 +427,9 @@ def _create_unified_hub_repo(mods): for hub_name in hubs: # The hubs configured with `pip.configure` only expose some packages. - hub_packages = mods.configured_hubs[hub_name].packages if hub_name in mods.configured_hubs else mods.hub_whl_map[hub_name] + hub_packages = mods.hub_whl_map[hub_name] + if hub_name in mods.configured_hubs: + hub_packages = mods.configured_hubs[hub_name].packages for pkg_name in hub_packages: norm_pkg = normalize_name(pkg_name) if norm_pkg not in packages: @@ -449,13 +461,7 @@ def _create_unified_hub_repo(mods): hubs = hubs, packages = packages, requirements_bzl = mods.unified_hub_requirements_bzl, - # An alias can't be less testonly than what it points to, so a package - # is testonly if it is testonly in any hub. - testonly_packages = sorted({ - pkg: None - for hub_name in hubs - for pkg in (mods.configured_hubs[hub_name].testonly_packages if hub_name in mods.configured_hubs else []) - }), + testonly_packages = mods.unified_testonly_packages, ) def _pip_impl(module_ctx): @@ -549,7 +555,7 @@ def _pip_impl(module_ctx): groups = mods.hub_group_map.get(hub_name), # The hubs configured with `pip.configure` hide the other packages. hide_unexposed_packages = hub_name in mods.configured_hubs, - testonly_packages = mods.configured_hubs[hub_name].testonly_packages if hub_name in mods.configured_hubs else [], + testonly_packages = mods.testonly_packages.get(hub_name, []), ) _create_unified_hub_repo(mods) diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 93651937df..590be1cbb2 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -125,6 +125,8 @@ def _parse_modules(env, **kwargs): exposed_packages = subjects.dict, hub_group_map = subjects.dict, hub_whl_map = subjects.dict, + testonly_packages = subjects.dict, + unified_testonly_packages = subjects.collection, unified_hub_requirements_bzl = subjects.bool, whl_libraries = subjects.dict, whl_mods = subjects.dict, @@ -807,9 +809,9 @@ def _configure_parse_modules(env, *modules): minor_mapping = {"3.15": "3.15.19"}, ) -def _configure_parse(**kwargs): +def _configure_parse(hub_name = "pypi_a", **kwargs): return _parse( - hub_name = "pypi_a", + hub_name = hub_name, python_version = "3.15", simpleapi_skip = ["simple", "other-pkg", "transitive-pkg"], requirements_lock = "requirements.txt", @@ -858,6 +860,29 @@ def _test_configure_testonly(env): _tests.append(_test_configure_testonly) +def _test_configure_testonly_unified(env): + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [ + _configure_parse(hub_name = "pypi_a", pyproject_toml = "pyproject.toml"), + _configure_parse(hub_name = "pypi_b", pyproject_toml = "pyproject.toml"), + ], + configure = [ + _configure(hub_name = "pypi_a", dependency_groups = ["default"], testonly = True), + _configure(hub_name = "pypi_b", dependency_groups = ["default"]), + ], + ), + ) + + pypi.testonly_packages().contains_exactly({"pypi_a": ["simple"], "pypi_b": []}) + + # A package is testonly in the unified hub if it is testonly in any hub. + pypi.unified_testonly_packages().contains_exactly(["simple"]) + +_tests.append(_test_configure_testonly_unified) + def _test_configure_default_pyproject(env): # The `pip.default` pyproject.toml is used for the hubs of the root module. pypi = _configure_parse_modules( From 41a0664a8e26d19089cdc0269861e1432c46b7f4 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:08:48 +0900 Subject: [PATCH 14/18] news: name the pip.configure.testonly entry after the PR --- news/4428.added.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/news/4428.added.md b/news/4428.added.md index ebc0594f7e..179e302535 100644 --- a/news/4428.added.md +++ b/news/4428.added.md @@ -1,2 +1,3 @@ (pypi) Added {attr}`pip.configure.testonly` to make the packages selected by -{obj}`pip.configure` testonly. +{obj}`pip.configure` testonly +([#3413](https://github.com/bazel-contrib/rules_python/issues/3413)). From 5a97b921f88c028b82e646fd8c2de24eb57ab181 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:11:39 +0900 Subject: [PATCH 15/18] refactor(pypi): map the venvs to their hubs without config_setting groups A select may map multiple keys to the same target, so map each venv to the target of its hub directly. Also always render the venvs in the unified hub and drop the unreachable hub name collision check. --- ...configure-venvs.added.md => 4429.added.md} | 0 python/private/pypi/BUILD.bazel | 1 - python/private/pypi/unified_hub_repo.bzl | 7 +++-- python/private/pypi/unified_hub_setup.bzl | 30 +++---------------- 4 files changed, 8 insertions(+), 30 deletions(-) rename news/{pip-configure-venvs.added.md => 4429.added.md} (100%) diff --git a/news/pip-configure-venvs.added.md b/news/4429.added.md similarity index 100% rename from news/pip-configure-venvs.added.md rename to news/4429.added.md diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index d8fa184690..bd3bcc387b 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -437,7 +437,6 @@ bzl_library( name = "unified_hub_setup", srcs = ["unified_hub_setup.bzl"], deps = [ - "@bazel_skylib//lib:selects", "@rules_python//python/private/pypi:labels", "@rules_python//python/private/pypi:missing_package", ], diff --git a/python/private/pypi/unified_hub_repo.bzl b/python/private/pypi/unified_hub_repo.bzl index 187a1d0f3f..a9afde1757 100644 --- a/python/private/pypi/unified_hub_repo.bzl +++ b/python/private/pypi/unified_hub_repo.bzl @@ -25,7 +25,8 @@ define_pypi_package_targets( extra_aliases = {extra_aliases}, hubs = {hubs}, pkg_hubs = {pkg_hubs}, - testonly = {testonly},{venvs} + testonly = {testonly}, + venvs = {venvs}, ) """ @@ -61,7 +62,7 @@ def dist_info_requirement(name): def _unified_hub_repo_impl(rctx): # The venvs are selected with the same flag as the hubs. venvs = rctx.attr.venvs - hubs = rctx.attr.hubs + [venv for venv in sorted(venvs) if venv not in rctx.attr.hubs] + hubs = rctx.attr.hubs + sorted(venvs) default_hub = rctx.attr.default_hub or None # 1. Generate Root BUILD.bazel with shared config settings @@ -97,7 +98,7 @@ def _unified_hub_repo_impl(rctx): pkg_hubs = pkg_hubs, pkg_name = pkg_name, testonly = pkg_name in testonly_packages, - venvs = "\n venvs = {},".format(render.dict(dict(sorted(venvs.items())), key_repr = repr).replace("\n", "\n ")) if venvs else "", + venvs = dict(sorted(venvs.items())), ), ) diff --git a/python/private/pypi/unified_hub_setup.bzl b/python/private/pypi/unified_hub_setup.bzl index 28605cbf34..884f7029a6 100644 --- a/python/private/pypi/unified_hub_setup.bzl +++ b/python/private/pypi/unified_hub_setup.bzl @@ -1,6 +1,5 @@ """Helper functions for setting up targets within the Unified PyPI Hub repository.""" -load("@bazel_skylib//lib:selects.bzl", "selects") load( "@rules_python//python/private/pypi:labels.bzl", "DATA_LABEL", @@ -43,8 +42,7 @@ def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub default_hub: the hub to use by default. testonly: whether all targets should be marked as testonly. venvs: dict mapping the venv names to the hubs that provide their - packages. The hubs and venvs that resolve to the same target - share a `config_setting_group`. + packages. """ pkg_name = name @@ -57,11 +55,9 @@ def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub all_aliases = _STANDARD_ALIASES + sorted(extra_aliases.keys()) missing_errors = {} - groups = {} for alias_name in all_aliases: - # The target mapped to the hubs and venvs that resolve to it. - targets = {} + select_map = {} for hub in hubs: is_supported = ( (alias_name in _STANDARD_ALIASES and hub in pkg_hubs) or @@ -69,7 +65,7 @@ def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub ) if is_supported: - target = "@{hub}//{pkg}:{alias}".format( + select_map["//:_is_venv_" + hub] = "@{hub}//{pkg}:{alias}".format( hub = venvs.get(hub, hub), pkg = pkg_name, alias = alias_name, @@ -81,18 +77,7 @@ def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub "hub_name": hub, "package_name": pkg_name if alias_name in _STANDARD_ALIASES else (pkg_name + ":" + alias_name), } - target = ":{}".format(err_target) - targets.setdefault(target, []).append(hub) - - select_map = {} - for target, target_hubs in targets.items(): - if len(target_hubs) == 1: - select_map["//:_is_venv_" + target_hubs[0]] = target - continue - - group = "_is_venv_" + "_or_".join(target_hubs) - groups[group] = target_hubs - select_map[":" + group] = target + select_map["//:_is_venv_" + hub] = ":{}".format(err_target) # //conditions:default fallback default_supported = ( @@ -122,13 +107,6 @@ def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub testonly = testonly, ) - for group, group_hubs in groups.items(): - selects.config_setting_group( - name = group, - match_any = ["//:_is_venv_" + hub for hub in group_hubs], - visibility = ["//visibility:private"], - ) - # Generate missing package error targets for err_name, err_args in missing_errors.items(): missing_package_error( From 7db87445e974085022f8ab55a48b9222b4534106 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:11:52 +0900 Subject: [PATCH 16/18] refactor(pypi): compute the hubs used by venvs once Also document the reserved venv names and test the extra internal visibility of the hidden packages. --- news/4429.added.md | 3 ++- python/private/pypi/extension.bzl | 8 ++++-- python/private/pypi/hub_exposure.bzl | 3 +++ .../render_pkg_aliases_test.bzl | 26 +++++++++++++++++++ 4 files changed, 37 insertions(+), 3 deletions(-) diff --git a/news/4429.added.md b/news/4429.added.md index add79e6689..c2b0d5d902 100644 --- a/news/4429.added.md +++ b/news/4429.added.md @@ -1,3 +1,4 @@ (pypi) Added {attr}`pip.configure.venv` to create a named selection of the packages of a hub that can be used through the unified `@pypi` hub with the -{flag}`--venv` flag. +{flag}`--venv` flag +([#3413](https://github.com/bazel-contrib/rules_python/issues/3413)). diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 8c78362f97..0236dfdb49 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -556,7 +556,12 @@ def _pip_impl(module_ctx): for name, args in mods.whl_libraries.items(): whl_library(name = name, **args) + # The venvs may select the packages that their hub does not expose. + venv_hubs = {info.hub: None for info in mods.venvs.values()} for hub_name, whl_map in mods.hub_whl_map.items(): + extra_internal_visibility = [] + if hub_name in venv_hubs: + extra_internal_visibility = ["@pypi//:__subpackages__"] hub_repository( name = hub_name, repo_name = hub_name, @@ -570,8 +575,7 @@ def _pip_impl(module_ctx): groups = mods.hub_group_map.get(hub_name), # The hubs configured with `pip.configure` hide the other packages. hide_unexposed_packages = hub_name in mods.configured_hubs, - # The venvs may select the packages that the hub does not expose. - extra_internal_visibility = ["@pypi//:__subpackages__"] if hub_name in [v.hub for v in mods.venvs.values()] else [], + extra_internal_visibility = extra_internal_visibility, testonly_packages = mods.testonly_packages.get(hub_name, []), ) diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl index d7909db688..55871e7045 100644 --- a/python/private/pypi/hub_exposure.bzl +++ b/python/private/pypi/hub_exposure.bzl @@ -115,6 +115,9 @@ def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): def _venv_error(venv, hub, *, hubs, configured, testonly): if venv in hubs: return "pip.configure() venv '{}' conflicts with the hub of the same name".format(venv) + + # `pypi` is the unified hub and `auto` is the default value of the + # `//python/config_settings:venv` flag. if venv in ("pypi", "auto"): return "pip.configure() venv name '{}' is reserved".format(venv) if venv in configured and configured[venv].hub != hub: diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index 38699f130b..2aaafc80eb 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -151,6 +151,32 @@ pkg_aliases( _tests.append(_test_hidden_aliases_in_groups) +def _test_hidden_aliases_extra_internal_visibility(env): + actual = render_pkg_aliases( + aliases = { + "bar": "pypi_bar", + "foo": "pypi_foo", + }, + exposed_packages = ["foo"], + extra_internal_visibility = ["@pypi//:__subpackages__"], + ) + + env.expect.that_str(actual["bar/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = [ + "@pypi_bar//:__pkg__", + "@pypi_foo//:__pkg__", + "@pypi//:__subpackages__", +]) + +pkg_aliases( + name = "bar", + actual = "pypi_bar", +)""") + +_tests.append(_test_hidden_aliases_extra_internal_visibility) + def _test_bzlmod_aliases(env): # Use this function as it is used in pip_repository actual = render_multiplatform_pkg_aliases( From 71be0c06b8f42f8e68b0d6f2f3c2c5fbb52765de Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:12:30 +0900 Subject: [PATCH 17/18] fix(pypi): only pass testonly to the package alias when it is set The existing tests mock the alias without a testonly argument. --- python/private/pypi/pkg_aliases.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index f165e5dffe..d4fac5b269 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -124,7 +124,7 @@ def pkg_aliases( _native.alias( name = name, actual = ":" + PY_LIBRARY_PUBLIC_LABEL, - testonly = testonly, + **({"testonly": True} if testonly else {}) ) if requires_dist: From 8cb72564cbdbc82ad5c78940a24c2eeeee2f54ec Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:13:05 +0900 Subject: [PATCH 18/18] test(pypi): pass the pyproject.toml in the venv extension test --- tests/pypi/extension/extension_tests.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index f9aeb34eea..681e9ba9e0 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -928,7 +928,7 @@ def _test_configure_venvs(env): env, _mod( name = "rules_python", - parse = [_configure_parse()], + parse = [_configure_parse(pyproject_toml = "pyproject.toml")], configure = [ _configure(hub_name = "pypi_a", venv = "venv_dev", dependency_groups = ["dev"]), ],