diff --git a/docs/pypi/download.md b/docs/pypi/download.md index cc8b7890aa..fc26cfd712 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -134,6 +134,85 @@ rename, and `requirement("numpy")` routes through the active hub like fixed at loading time, before the active hub is known, so load them from a concrete hub instead. Prefer `@pypi//` labels in new code. +(pip-configure)= +### Configuring the Exposed Packages (pip.configure) + +By default a hub exposes all of the packages from its requirements files or +`uv.lock` file, including the transitive dependencies. A `pip.configure` tag +can limit the hub to the packages listed in a `pyproject.toml` file: the +`[project] dependencies`, `[project.optional-dependencies]` and PEP 735 +`[dependency-groups]` tables. + +```toml +[project] +dependencies = ["numpy"] + +[dependency-groups] +dev = ["pytest"] +``` + +```starlark +pip.parse( + hub_name = "pypi_hub", + pyproject_toml = "//:pyproject.toml", + requirements_lock = "//:requirements_lock.txt", +) + +pip.configure( + hub_name = "pypi_hub", + dependency_groups = ["default", "group:dev"], +) +``` + +Here `@pypi_hub//numpy` and `@pypi_hub//pytest` can be used, but the +dependencies of `numpy` are only used through `@pypi_hub//numpy`. The +`all_requirements` lists in the hub's `requirements.bzl` and the +[Unified @pypi Hub](unified-pypi-hub) only contain the exposed packages. + +The {attr}`pip.configure.dependency_groups` attribute defaults to `["*"]`, +which selects every group. Use `["*", "transitive"]` to expose all of the +packages of the hub, including the transitive dependencies, which is the same +as not configuring the hub. Selecting the transitive dependencies of only some +of the groups is not supported. + +Multiple tags for the same hub are combined. Only the root module and the +module that created the hub can configure it; if both do, the root module +wins. A self reference such as `my-project[test]` in the `pyproject.toml` +selects the optional dependencies of the project. The packages that are not +part of the hub are ignored, so a single `pyproject.toml` can list tools that +are missing from the lock file. + +#### Testonly Packages + +Set {attr}`pip.configure.testonly` to make the selected packages testonly, so +that they can only be used by tests and other testonly targets: + +```starlark +pip.configure( + hub_name = "pypi_hub", + dependency_groups = ["default"], +) +pip.configure( + hub_name = "pypi_hub", + dependency_groups = ["group:test"], + testonly = True, +) +``` + +A package that is also selected by a tag without `testonly` is not testonly. +The testonly packages are left out of the `all_requirements`, +`all_whl_requirements` and `all_data_requirements` lists in the hub's +`requirements.bzl` and are listed in `all_testonly_requirements` instead. In +the [Unified @pypi Hub](unified-pypi-hub), a package is testonly if it is +testonly in any of the hubs. + +The other packages of the hub depend on the testonly packages through the same +targets, so a package that a non-testonly package depends on can't be +testonly. + +:::{versionadded} VERSION_NEXT_FEATURE +::: + ### Declaring Abstract Dependencies (pip.dep) :::{versionadded} 2.2.0 diff --git a/news/4395.added.md b/news/4395.added.md new file mode 100644 index 0000000000..3e633cc997 --- /dev/null +++ b/news/4395.added.md @@ -0,0 +1,3 @@ +(pypi) Added {obj}`pip.configure` to limit the packages that a hub exposes to +the ones listed in the dependency groups of a `pyproject.toml` file +([#3413](https://github.com/bazel-contrib/rules_python/issues/3413)). diff --git a/news/4428.added.md b/news/4428.added.md new file mode 100644 index 0000000000..179e302535 --- /dev/null +++ b/news/4428.added.md @@ -0,0 +1,3 @@ +(pypi) Added {attr}`pip.configure.testonly` to make the packages selected by +{obj}`pip.configure` testonly +([#3413](https://github.com/bazel-contrib/rules_python/issues/3413)). diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 03a9e113d5..63c5167757 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -129,6 +129,7 @@ bzl_library( deps = [ ":default_platforms", ":hub_builder", + ":hub_exposure", ":hub_repository", ":parse_whl_name", ":pep508_env", @@ -352,6 +353,7 @@ bzl_library( deps = [ ":labels", ":whl_library_deps_targets", + "//python:py_library", "//python/private:common_labels", "//python/private:text_util", "@bazel_skylib//lib:selects", @@ -629,6 +631,25 @@ bzl_library( ], ) +bzl_library( + name = "dependency_groups", + srcs = ["dependency_groups.bzl"], + deps = [ + ":pep508_requirement", + "//python/private:normalize_name", + "@bazel_skylib//lib:types", + ], +) + +bzl_library( + name = "hub_exposure", + srcs = ["hub_exposure.bzl"], + deps = [ + ":dependency_groups", + "//python/private:pyproject_utils", + ], +) + bzl_library( name = "argparse", srcs = ["argparse.bzl"], diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl new file mode 100644 index 0000000000..4e40c7e856 --- /dev/null +++ b/python/private/pypi/dependency_groups.bzl @@ -0,0 +1,139 @@ +"""Select PyPI packages using pyproject.toml dependency groups.""" + +load("@bazel_skylib//lib:types.bzl", "types") +load("//python/private:normalize_name.bzl", "normalize_name") +load(":pep508_requirement.bzl", "requirement") + +def select_dependency_groups(pyproject, specs, _fail = fail): + """Select the packages named by the given dependency group specifiers. + + Supported specifiers are: + + * `"default"`: the `[project] dependencies`. + * `"optional:"`: a `[project.optional-dependencies]` entry. + * `"group:"`: a PEP 735 `[dependency-groups]` entry. + * `"optional:*"` and `"group:*"`: all entries of the table. + * `"*"`: all of the above. + * `""`: a dependency group, or an optional dependency group if + there is no dependency group with that name. + * `"transitive"`: only supported together with `"*"` and selects all + of the packages, including the transitive dependencies. + + Args: + pyproject: {type}`struct` returned by `read_pyproject`. + specs: {type}`list[str]` the specifiers. + _fail: {type}`callable` the failure function, mainly for testing. + + Returns: + {type}`struct` with the attributes: + * `packages`: {type}`list[str]` the sorted normalized package names. + * `all`: {type}`bool` whether all of the packages including the + transitive dependencies should be selected, i.e. `"*"` and + `"transitive"` were given. + """ + groups = {"default": pyproject.dependencies} + for name, reqs in pyproject.optional_dependencies.items(): + groups["optional:" + normalize_name(name)] = reqs + for name, reqs in _expand_dependency_groups(pyproject.dependency_groups, _fail).items(): + groups["group:" + name] = reqs + + if "transitive" in specs: + if "*" not in specs: + _fail("The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: {}".format(specs)) + return struct(packages = [], all = True) + + selected = {} + for spec in specs: + keys = _match(spec, groups) + if not keys: + _fail("Unknown dependency group '{}'. Available groups: {}".format( + spec, + ", ".join(sorted(groups)), + )) + for key in keys: + selected[key] = None + + # A self reference like `my-project[test]` selects the optional + # dependencies, but only one level deep. + project = normalize_name(pyproject.name or "") + packages = {} + for key in selected: + for req in groups[key]: + req = requirement(req) + if req.name_ != project: + packages[req.name_] = None + continue + for extra in req.extras: + for r in groups.get("optional:" + extra, []): + packages[requirement(r).name_] = None + + return struct(packages = sorted(packages), all = False) + +def _match(spec, groups): + if spec == "*": + return groups.keys() + if spec in ("optional:*", "group:*"): + prefix = spec[:-1] + return [key for key in groups if key.startswith(prefix)] + + kind, _, name = spec.rpartition(":") + name = normalize_name(name) + if kind: + candidates = [kind + ":" + name] + elif name == "default": + candidates = ["default"] + else: + candidates = ["group:" + name, "optional:" + name] + + for key in candidates: + if key in groups: + return [key] + return [] + +def _expand_dependency_groups(dependency_groups, _fail): + """Expand the PEP 735 `include-group` entries. + + Args: + dependency_groups: {type}`dict[str, list]` the `[dependency-groups]` + table. + _fail: {type}`callable` the failure function. + + Returns: + {type}`dict[str, list[str]]` normalized group names mapped to their + requirement strings. + """ + groups = { + normalize_name(name): items + for name, items in dependency_groups.items() + } + + # Each round expands at least one group unless there is a cycle. + expanded = {} + for _ in range(len(groups)): + for name, items in groups.items(): + if name in expanded: + continue + + reqs = [] + for item in items: + if types.is_string(item): + reqs.append(item) + continue + + include = normalize_name(item.get("include-group", "")) + if include not in groups: + _fail("Unknown dependency group '{}' included from '{}'".format(include, name)) + return {} + if include not in expanded: + reqs = None + break + reqs.extend(expanded[include]) + + if reqs != None: + expanded[name] = reqs + + if len(expanded) != len(groups): + _fail("Cyclic dependency group includes: {}".format( + ", ".join(sorted([name for name in groups if name not in expanded])), + )) + return expanded diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 21d5ba509c..706022fe0e 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -24,6 +24,7 @@ load("//python/private:pyproject_utils.bzl", "read_pyproject", "version_from_req load("//python/private:repo_utils.bzl", "repo_utils") load(":default_platforms.bzl", "default_platforms") load(":hub_builder.bzl", "hub_builder") +load(":hub_exposure.bzl", "hub_exposure") load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") load(":parse_whl_name.bzl", "parse_whl_name") load(":pep508_env.bzl", "env") @@ -89,6 +90,7 @@ def build_config( unified_hub_requirements_bzl = False defaults = { "platforms": default_platforms(), + "pyproject_toml": None, "python_version": None, } for mod in module_ctx.modules: @@ -105,6 +107,8 @@ def build_config( unified_hub_requirements_bzl = True pyproject_toml = tag.pyproject_toml if pyproject_toml: + if mod.is_root: + defaults["pyproject_toml"] = pyproject_toml pyproject = read_pyproject(module_ctx, pyproject_toml) if pyproject.requires_python: defaults["python_version"] = version_from_requires_python(pyproject.requires_python) @@ -153,6 +157,7 @@ def build_config( for name, values in defaults["platforms"].items() }, enable_pipstar_extract = enable_pipstar_extract, + pyproject_toml = defaults["pyproject_toml"], toml_decode = toml.decode, unified_hub_requirements_bzl = unified_hub_requirements_bzl, ) @@ -245,6 +250,7 @@ You cannot use both the additive_build_content and additive_build_content_file a # dict[str repo, HubBuilder] # See `hub_builder.bzl%hub_builder()` for `HubBuilder` pip_hub_map = {} + hub_pyproject = {} simpleapi_cache = pypi_cache(mctx = module_ctx) is_pypi_hub_reserved = module_ctx.getenv("RULES_PYTHON_PYPI_HUB_RESERVED", "0") == "1" @@ -321,6 +327,9 @@ You cannot use both the additive_build_content and additive_build_content_file a else: builder = pip_hub_map[hub_name] + if pip_attr.pyproject_toml and hub_name not in hub_pyproject: + hub_pyproject[hub_name] = pip_attr.pyproject_toml + builder.pip_parse( module_ctx, pip_attr = pip_attr, @@ -345,6 +354,7 @@ You cannot use both the additive_build_content and additive_build_content_file a exposed_packages = {} extra_aliases = {} whl_libraries = {} + hubs = {} for hub in pip_hub_map.values(): out = hub.build() @@ -358,8 +368,33 @@ You cannot use both the additive_build_content and additive_build_content_file a extra_aliases[hub.name] = out.extra_aliases hub_group_map[hub.name] = out.group_map hub_whl_map[hub.name] = out.whl_map + hubs[hub.name] = struct( + module_name = hub.module_name, + pyproject = hub_pyproject.get(hub.name), + packages = out.whl_map.keys(), + ) + + configured_hubs = hub_exposure( + module_ctx, + hubs = hubs, + default_pyproject = config.pyproject_toml, + _fail = _fail, + ) + if configured_hubs == None: + return None + testonly_packages = {} + unified_testonly_packages = {} + for hub_name, info in configured_hubs.items(): + exposed_packages[hub_name] = info.packages + testonly_packages[hub_name] = info.testonly_packages + + # An alias can't be less testonly than what it points to, so a package + # is testonly in the unified hub if it is testonly in any hub. + for pkg in info.testonly_packages: + unified_testonly_packages[pkg] = None return struct( + configured_hubs = configured_hubs, config = config, declared_deps = declared_deps, default_hub = config.default_hub or renamed_default_hub, @@ -368,7 +403,9 @@ You cannot use both the additive_build_content and additive_build_content_file a facts = simpleapi_cache.get_facts(), hub_group_map = hub_group_map, hub_whl_map = hub_whl_map, + testonly_packages = testonly_packages, unified_hub_requirements_bzl = config.unified_hub_requirements_bzl, + unified_testonly_packages = sorted(unified_testonly_packages), whl_libraries = whl_libraries, whl_mods = whl_mods, platform_config_settings = { @@ -392,7 +429,11 @@ def _create_unified_hub_repo(mods): extra_aliases = {} for hub_name in hubs: - for pkg_name in mods.hub_whl_map[hub_name]: + # The hubs configured with `pip.configure` only expose some packages. + hub_packages = mods.hub_whl_map[hub_name] + if hub_name in mods.configured_hubs: + hub_packages = mods.configured_hubs[hub_name].packages + for pkg_name in hub_packages: norm_pkg = normalize_name(pkg_name) if norm_pkg not in packages: packages[norm_pkg] = [] @@ -423,6 +464,7 @@ def _create_unified_hub_repo(mods): hubs = hubs, packages = packages, requirements_bzl = mods.unified_hub_requirements_bzl, + testonly_packages = mods.unified_testonly_packages, ) def _pip_impl(module_ctx): @@ -514,6 +556,9 @@ def _pip_impl(module_ctx): packages = mods.exposed_packages.get(hub_name, []), platform_config_settings = mods.platform_config_settings.get(hub_name, {}), groups = mods.hub_group_map.get(hub_name), + # The hubs configured with `pip.configure` hide the other packages. + hide_unexposed_packages = hub_name in mods.configured_hubs, + testonly_packages = mods.testonly_packages.get(hub_name, []), ) _create_unified_hub_repo(mods) @@ -656,6 +701,10 @@ This is designed to work with dependency management tools like Renovate. :::{versionadded} 2.3.0 ::: + +:::{versionchanged} VERSION_NEXT_FEATURE +Also used by {obj}`pip.configure` for the hubs of the root module. +::: """, ), "unified_hub_requirements_bzl": attr.bool( @@ -875,6 +924,10 @@ The version must be specified as `==X.Y.Z` (exact version with full semver). :::{versionadded} 2.3.0 ::: + +:::{versionchanged} VERSION_NEXT_FEATURE +Also used by {obj}`pip.configure` to select the packages that the hub exposes. +::: """, ), "python_version": attr.string( @@ -1074,6 +1127,89 @@ execution-phase error when built. """, ) +_configure_tag = tag_class( + attrs = { + "dependency_groups": attr.string_list( + default = ["*"], + doc = """\ +The `pyproject.toml` dependency groups whose packages the hub exposes. + +Each item is one of: + +* `"default"`: the `[project] dependencies`. +* `"optional:"`: the `[project.optional-dependencies]` entry ``. +* `"group:"`: the PEP 735 `[dependency-groups]` entry ``. +* `"optional:*"` or `"group:*"`: all entries of the table. +* `"*"`: all of the above. +* `""`: the dependency group `` if it exists, otherwise the + optional dependency group ``. +* `"transitive"`: only supported together with `"*"`, exposes all of the + packages of the hub including the transitive dependencies, i.e. all of the + packages from the requirements files or the `uv.lock` file. + +See [Configuring the Exposed Packages](#pip-configure) for examples. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + "hub_name": attr.string( + mandatory = True, + doc = """\ +The name of the hub to configure. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + "pyproject_toml": attr.label( + mandatory = False, + doc = """\ +The `pyproject.toml` file that defines the dependency groups. Defaults to the +{attr}`pip.parse.pyproject_toml` of the hub, then to +{attr}`pip.default.pyproject_toml` if the hub is defined by the root module. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + "testonly": attr.bool( + default = False, + doc = """\ +Make the aliases of the selected packages testonly. A package that is also +selected by a `pip.configure` tag without `testonly` is not testonly. It can't +be used with `["*", "transitive"]`, which exposes all of the packages. + +The testonly packages are left out of the `all_requirements`, +`all_whl_requirements` and `all_data_requirements` lists in the hub's +`requirements.bzl` and listed in `all_testonly_requirements` instead. In the +[Unified @pypi Hub](unified-pypi-hub), a package is testonly if it is testonly +in any of the hubs. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + }, + doc = """\ +Configure the packages exposed by a hub. + +The packages are selected from a `pyproject.toml` file using the +`[project] dependencies`, `[project.optional-dependencies]` and PEP 735 +`[dependency-groups]` tables. The other packages of the hub, e.g. the +transitive dependencies, are still used by the packages that need them, but +they are not public. + +Only the root module and the module that created the hub can configure it. If +both do, the root module wins. Multiple tags for the same hub are combined. + +See [Configuring the Exposed Packages](#pip-configure) for examples. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", +) + pypi = module_extension( environ = ["RULES_PYTHON_PYPI_HUB_RESERVED"], doc = """\ @@ -1097,12 +1233,17 @@ Environment Variables: - `RULES_PYTHON_PYPI_HUB_RESERVED`: Enable fallback renaming for reserved hub name collisions. See the {envvar}`RULES_PYTHON_PYPI_HUB_RESERVED` documentation for details. +pip.configure: +This tag class configures the packages exposed by a hub using the dependency +groups from a pyproject.toml file. + pip.whl_mods: This tag class is used to help create JSON files to describe modifications to the BUILD files for wheels. """, implementation = _pip_impl, tag_classes = { + "configure": _configure_tag, "default": tag_class( attrs = _default_attrs, doc = """\ diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl new file mode 100644 index 0000000000..e79de7556a --- /dev/null +++ b/python/private/pypi/hub_exposure.bzl @@ -0,0 +1,95 @@ +"""Compute the packages exposed by the hubs configured with `pip.configure`.""" + +load("//python/private:pyproject_utils.bzl", "read_pyproject") +load(":dependency_groups.bzl", "select_dependency_groups") + +def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): + """Compute the packages exposed by the hubs configured with `pip.configure`. + + Args: + module_ctx: {type}`module_ctx` the module context. + hubs: {type}`dict[str, struct]` the hubs by name. Each struct has: + * `module_name`: {type}`str` the module that created the hub. + * `pyproject`: {type}`Label | None` from `pip.parse`. + * `packages`: {type}`list[str]` the normalized names of all + packages in the hub. + default_pyproject: {type}`Label | None` from `pip.default`, only used + for the hubs of the root module. + _fail: {type}`callable` the failure function, mainly for testing. + + Returns: + {type}`dict[str, struct] | None` the configured hubs mapped to a + struct with: + * `packages`: {type}`list[str]` the sorted packages that the hub + exposes. + * `testonly_packages`: {type}`list[str]` the sorted packages whose + aliases are testonly. A package is testonly only if all of the + tags that select it are testonly. + + The hubs that are not configured or that expose all of their packages + are not included, so that they behave the same. Returns `None` on + failure. + """ + + # The root module wins over the owner of the hub when both configure it. + root_modules = [mod for mod in module_ctx.modules if mod.is_root] + root_name = root_modules[0].name if root_modules else None + root_configured = {} + for mod in root_modules: + for tag in mod.tags.configure: + root_configured[tag.hub_name] = None + + configured = {} + exposes_all = {} + for mod in module_ctx.modules: + for tag in mod.tags.configure: + hub = tag.hub_name + if hub not in hubs: + _fail("pip.configure() refers to unknown hub '{}'. Available hubs: {}".format( + hub, + ", ".join(sorted(hubs)), + )) + return None + + info = hubs[hub] + if not mod.is_root and (mod.name != info.module_name or hub in root_configured): + # Only the root module and the module that created the hub can + # configure it. + continue + + pyproject = ( + tag.pyproject_toml or + info.pyproject or + (default_pyproject if info.module_name == root_name else None) + ) + if not pyproject: + _fail("pip.configure() for hub '{}' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`".format(hub)) + return None + + selection = select_dependency_groups( + read_pyproject(module_ctx, pyproject), + tag.dependency_groups, + _fail = _fail, + ) + if selection.all: + if tag.testonly: + _fail("pip.configure() for hub '{}' cannot use `testonly` when exposing all of the packages with {}".format(hub, tag.dependency_groups)) + return None + exposes_all[hub] = None + + # Multiple tags for the same hub are combined. + exposed = configured.setdefault(hub, {}) + for name in selection.packages: + # The pyproject.toml may list packages that are not in the hub, + # e.g. tools that are not locked. + if name in info.packages: + exposed[name] = exposed.get(name, True) and tag.testonly + + return { + hub: struct( + packages = sorted(packages), + testonly_packages = sorted([name for name, testonly in packages.items() if testonly]), + ) + for hub, packages in configured.items() + if hub not in exposes_all + } diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index f915aa1c77..8764951851 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -26,15 +26,23 @@ exports_files(["requirements.bzl"]) """ def _impl(rctx): - bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() + if rctx.attr.hide_unexposed_packages: + bzl_packages = rctx.attr.packages + exposed_packages = bzl_packages + else: + bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() + exposed_packages = None + testonly_packages = rctx.attr.testonly_packages aliases = render_multiplatform_pkg_aliases( aliases = { key: _whl_config_settings_from_json(values) for key, values in rctx.attr.whl_map.items() }, + exposed_packages = exposed_packages, extra_hub_aliases = rctx.attr.extra_hub_aliases, requirement_cycles = rctx.attr.groups, platform_config_settings = rctx.attr.platform_config_settings, + testonly_packages = testonly_packages, ) for path, contents in aliases.items(): rctx.file(path, contents) @@ -53,18 +61,25 @@ def _impl(rctx): "%%PACKAGES%%": render.dict(rctx.attr.whl_map, value_repr = lambda x: "None"), }, ) + + # The testonly packages are only listed in `all_testonly_requirements`. + all_packages = [p for p in bzl_packages if p not in testonly_packages] rctx.template("requirements.bzl", rctx.attr._requirements_bzl_template, substitutions = { "%%ALL_DATA_REQUIREMENTS%%": render.list([ macro_tmpl.format(p, "data") - for p in bzl_packages + for p in all_packages ]), "%%ALL_REQUIREMENTS%%": render.list([ macro_tmpl.format(p, "pkg") - for p in bzl_packages + for p in all_packages + ]), + "%%ALL_TESTONLY_REQUIREMENTS%%": render.list([ + macro_tmpl.format(p, "pkg") + for p in testonly_packages ]), "%%ALL_WHL_REQUIREMENTS_BY_PACKAGE%%": render.dict({ p: macro_tmpl.format(p, "whl") - for p in bzl_packages + for p in all_packages }), "%%MACRO_TMPL%%": macro_tmpl, }) @@ -78,10 +93,15 @@ hub_repository = repository_rule( "groups": attr.string_list_dict( mandatory = False, ), + "hide_unexposed_packages": attr.bool( + default = False, + doc = "Make the aliases of the packages that are not in `packages` visible only to the wheel repositories.", + ), "packages": attr.string_list( mandatory = False, doc = """\ -The list of packages that will be exposed via all_*requirements macros. Defaults to whl_map keys. +The list of packages that will be exposed via all_*requirements macros. Defaults to whl_map keys +unless `hide_unexposed_packages` is set. """, ), "platform_config_settings": attr.string_list_dict( @@ -92,6 +112,10 @@ The list of packages that will be exposed via all_*requirements macros. Defaults mandatory = True, doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.", ), + "testonly_packages": attr.string_list( + default = [], + doc = "The packages whose aliases should be testonly.", + ), "whl_map": attr.string_dict( mandatory = True, doc = """\ diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index 227e9bf10f..d4fac5b269 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -32,6 +32,7 @@ setting maps to and their precedence, refer to documentation on that page. """ load("@bazel_skylib//lib:selects.bzl", "selects") +load("//python:py_library.bzl", "py_library") load("//python/private:common_labels.bzl", "labels") load("//python/private:text_util.bzl", "render") load( @@ -86,6 +87,7 @@ def pkg_aliases( extras = [], include = [], group_deps = [], + testonly = False, **kwargs): """Create aliases for an actual package. @@ -107,10 +109,14 @@ def pkg_aliases( dependencies when parsing the requires_dist. Comes from METADATA or a lock file. include: {type}`list[str]` The subset of packages to include. Comes from `//:config.bzl#packages` + testonly: {type}`bool` Whether the public targets should be testonly. + Bazel does not enforce `testonly` on aliases, so the main public + targets are testonly wrappers around the aliases. **kwargs: extra kwargs to pass to {bzl:obj}`get_config_settings`. """ _native = kwargs.pop("native", native) rules = kwargs.pop("rules", struct( + py_library = py_library, whl_library_deps_targets = whl_library_deps_targets, )) select = kwargs.pop("select", selects.with_or) @@ -118,6 +124,7 @@ def pkg_aliases( _native.alias( name = name, actual = ":" + PY_LIBRARY_PUBLIC_LABEL, + **({"testonly": True} if testonly else {}) ) if requires_dist: @@ -210,20 +217,72 @@ def pkg_aliases( elif target_name in (WHEEL_FILE, PY_SRCS_LABEL): kwargs["visibility"] = ["//visibility:private"] - _native.alias( + # The private targets are used by the non-testonly `//_groups` libraries. + _alias( + _native, + rules, name = target_name, actual = _actual, + testonly = testonly and not target_name.startswith("_"), **kwargs ) if group_name: - _native.alias( + _alias( + _native, + rules, name = PY_LIBRARY_PUBLIC_LABEL, actual = "//_groups:{}_pkg".format(group_name), + testonly = testonly, ) - _native.alias( + _alias( + _native, + rules, name = WHEEL_FILE_PUBLIC_LABEL, actual = "//_groups:{}_whl".format(group_name), + testonly = testonly, + ) + +# The public targets that are wrapped in testonly targets, because Bazel does +# not enforce `testonly` on aliases. +_TESTONLY_WRAPPED = [ + PY_LIBRARY_PUBLIC_LABEL, + WHEEL_FILE_PUBLIC_LABEL, + DATA_LABEL, + DIST_INFO_LABEL, + EXTRACTED_WHEEL_FILES, +] + +def _alias(_native, rules, *, name, actual, testonly, **kwargs): + if not testonly: + _native.alias(name = name, actual = actual, **kwargs) + return + + if name not in _TESTONLY_WRAPPED: + # E.g. the extra aliases, which may point to executables. Best effort, + # because Bazel does not enforce `testonly` on aliases. + _native.alias(name = name, actual = actual, testonly = True, **kwargs) + return + + impl = "_testonly_" + name + _native.alias( + name = impl, + actual = actual, + visibility = ["//visibility:private"], + ) + if name == PY_LIBRARY_PUBLIC_LABEL: + rules.py_library( + name = name, + deps = [":" + impl], + testonly = True, + **kwargs + ) + else: + _native.filegroup( + name = name, + srcs = [":" + impl], + testonly = True, + **kwargs ) def multiplatform_whl_aliases( diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 0a1c328491..d9650fc079 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -65,7 +65,7 @@ def _repr_actual(aliases): else: return render.dict(aliases, key_repr = _repr_config_setting) -def _render_common_aliases(*, name, aliases, **kwargs): +def _render_common_aliases(*, name, aliases, visibility, **kwargs): pkg_aliases = render.call( "pkg_aliases", name = repr(name), @@ -80,14 +80,22 @@ def _render_common_aliases(*, name, aliases, **kwargs): return """\ load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") {extra_loads} -package(default_visibility = ["//visibility:public"]) +package(default_visibility = {visibility}) {aliases}""".format( aliases = pkg_aliases, extra_loads = extra_loads, + visibility = render.list(visibility), ) -def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases = {}, **kwargs): +def render_pkg_aliases( + *, + aliases, + requirement_cycles = None, + extra_hub_aliases = {}, + exposed_packages = None, + testonly_packages = [], + **kwargs): """Create alias declarations for each PyPI package. The aliases should be appended to the pip_repository BUILD.bazel file. These aliases @@ -100,6 +108,10 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases requirement_cycles: any package groups to also add. extra_hub_aliases: The list of extra aliases for each whl to be added in addition to the default ones. + exposed_packages: The packages that should be public. When set, the + aliases of the other packages are only visible to the wheel + repositories. + testonly_packages: The packages whose aliases should be testonly. **kwargs: Extra kwargs to pass to the rules. Returns: @@ -124,21 +136,59 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases for whl_name in group_whls } - files = { - "{}/BUILD.bazel".format(normalize_name(name)): _render_common_aliases( - name = normalize_name(name), + testonly_packages = { + normalize_name(name): None + for name in testonly_packages + } + + public = ["//visibility:public"] + if exposed_packages != None: + exposed_packages = {normalize_name(name): None for name in exposed_packages} + internal_visibility = _internal_visibility(aliases, has_groups = bool(requirement_cycles)) + + files = {} + for name, pkg_aliases in aliases.items(): + name = normalize_name(name) + visibility = public + if exposed_packages != None and name not in exposed_packages: + visibility = internal_visibility + files["{}/BUILD.bazel".format(name)] = _render_common_aliases( + name = name, aliases = pkg_aliases, - extra_aliases = extra_hub_aliases.get(normalize_name(name), []), - group_name = whl_group_mapping.get(normalize_name(name)), + extra_aliases = extra_hub_aliases.get(name, []), + group_name = whl_group_mapping.get(name), + visibility = visibility, + testonly = name in testonly_packages, **kwargs ).strip() - for name, pkg_aliases in aliases.items() - } if requirement_cycles: files["_groups/BUILD.bazel"] = generate_group_library_build_bazel("", requirement_cycles) return files +def _internal_visibility(aliases, has_groups = False): + """The visibility of the packages that are not exposed. + + The wheel repositories depend on the aliases of the hub, so the aliases + have to be visible to them. + """ + repo_names = {} + for pkg_aliases in aliases.values(): + if type(pkg_aliases) == type(""): + repo_names[pkg_aliases] = None + continue + + for repo_name in pkg_aliases.values(): + repo_names[repo_name] = None + + visibility = [ + "@{}//:__pkg__".format(repo_name) + for repo_name in sorted(repo_names) + ] + if has_groups: + visibility.append("//_groups:__pkg__") + return visibility + def _major_minor(python_version): major, _, tail = python_version.partition(".") minor, _, _ = tail.partition(".") diff --git a/python/private/pypi/requirements.bzl.tmpl.bzlmod b/python/private/pypi/requirements.bzl.tmpl.bzlmod index ba227aeb2d..7904ed5578 100644 --- a/python/private/pypi/requirements.bzl.tmpl.bzlmod +++ b/python/private/pypi/requirements.bzl.tmpl.bzlmod @@ -7,6 +7,8 @@ load("@rules_python//python:pip.bzl", "pip_utils") all_requirements = %%ALL_REQUIREMENTS%% +all_testonly_requirements = %%ALL_TESTONLY_REQUIREMENTS%% + all_whl_requirements_by_package = %%ALL_WHL_REQUIREMENTS_BY_PACKAGE%% all_whl_requirements = all_whl_requirements_by_package.values() diff --git a/python/private/pypi/unified_hub_repo.bzl b/python/private/pypi/unified_hub_repo.bzl index ddd15c0d86..984c3cacbe 100644 --- a/python/private/pypi/unified_hub_repo.bzl +++ b/python/private/pypi/unified_hub_repo.bzl @@ -25,6 +25,7 @@ define_pypi_package_targets( extra_aliases = {extra_aliases}, hubs = {hubs}, pkg_hubs = {pkg_hubs}, + testonly = {testonly}, ) """ @@ -79,6 +80,10 @@ def _unified_hub_repo_impl(rctx): extra_aliases_by_pkg.setdefault(pkg, {})[alias] = alias_hubs # 3. Generate package subpackages + testonly_packages = { + pkg_name: None + for pkg_name in rctx.attr.testonly_packages + } for pkg_name, pkg_hubs in rctx.attr.packages.items(): extra_aliases = extra_aliases_by_pkg.get(pkg_name, {}) rctx.file( @@ -89,6 +94,7 @@ def _unified_hub_repo_impl(rctx): hubs = hubs, pkg_hubs = pkg_hubs, pkg_name = pkg_name, + testonly = pkg_name in testonly_packages, ), ) @@ -130,6 +136,10 @@ unified_hub_repo = repository_rule( default = False, doc = "Whether to generate a requirements.bzl with the per-package `requirement` macros.", ), + "testonly_packages": attr.string_list( + default = [], + doc = "Packages that are testonly in at least one hub.", + ), }, doc = "Private repository rule creating the automatic Unified PyPI Hub.", ) diff --git a/python/private/pypi/unified_hub_setup.bzl b/python/private/pypi/unified_hub_setup.bzl index b19410fd4b..3ee8ae25d3 100644 --- a/python/private/pypi/unified_hub_setup.bzl +++ b/python/private/pypi/unified_hub_setup.bzl @@ -31,7 +31,7 @@ _STANDARD_ALIASES = [ EXTRACTED_WHEEL_FILES, ] -def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub = None): +def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub = None, testonly = False): """Define the targets for a PyPI package in the unified PyPI hub. Args: @@ -40,6 +40,7 @@ def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub extra_aliases: dict mapping extra alias names to lists of hubs that support them. hubs: list of all concrete hub names. default_hub: the hub to use by default. + testonly: whether all targets should be marked as testonly. """ pkg_name = name @@ -47,6 +48,7 @@ def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub native.alias( name = pkg_name, actual = ":pkg", + testonly = testonly, ) all_aliases = _STANDARD_ALIASES + sorted(extra_aliases.keys()) @@ -100,6 +102,7 @@ def define_pypi_package_targets(name, pkg_hubs, extra_aliases, hubs, default_hub native.alias( name = alias_name, actual = select(select_map), + testonly = testonly, ) # Generate missing package error targets diff --git a/python/private/pyproject_utils.bzl b/python/private/pyproject_utils.bzl index e96ffd2023..a9494b8ac2 100644 --- a/python/private/pyproject_utils.bzl +++ b/python/private/pyproject_utils.bzl @@ -18,10 +18,22 @@ def read_pyproject(module_ctx, pyproject): {type}`struct` with the attributes: * `requires_python`: {type}`str | None` the raw `requires-python` value (e.g. `"==3.13.9"`), or `None` if it is not set. + * `name`: {type}`str | None` the `[project] name`. + * `dependencies`: {type}`list[str]` the `[project] dependencies`. + * `optional_dependencies`: {type}`dict[str, list[str]]` the + `[project.optional-dependencies]` table. + * `dependency_groups`: {type}`dict[str, list]` the PEP 735 + `[dependency-groups]` table. Items are either requirement strings + or `{include-group = ""}` tables. """ data = toml.decode(module_ctx.read(module_ctx.path(pyproject), watch = "yes")) + project = data.get("project", {}) return struct( - requires_python = data.get("project", {}).get("requires-python"), + requires_python = project.get("requires-python"), + name = project.get("name"), + dependencies = project.get("dependencies") or [], + optional_dependencies = project.get("optional-dependencies") or {}, + dependency_groups = data.get("dependency-groups") or {}, ) def version_from_requires_python(requires_python): diff --git a/tests/integration/unified_pypi/BUILD.bazel b/tests/integration/unified_pypi/BUILD.bazel index 13d188d087..e7bbca4f9d 100644 --- a/tests/integration/unified_pypi/BUILD.bazel +++ b/tests/integration/unified_pypi/BUILD.bazel @@ -1,4 +1,5 @@ load("@pypi//:requirements.bzl", "requirement") +load("@pypi_testonly//:requirements.bzl", "all_requirements", "all_testonly_requirements") load("@rules_python//python:py_binary.bzl", "py_binary") load("@rules_python//python:py_test.bzl", "py_test") @@ -95,3 +96,72 @@ platform( "@platforms//os:windows", ], ) + +# The pypi_filtered hub only makes its pyproject.toml packages public. +py_test( + name = "test_filtered_direct", + srcs = ["test_default.py"], + main = "test_default.py", + deps = ["@pypi_filtered//colorama"], +) + +# six is not exposed by pypi_filtered, so it is missing in the unified hub. +py_binary( + name = "bin_filtered_six_unified", + srcs = ["bin_six_a.py"], + config_settings = { + "@rules_python//python/config_settings:venv": "pypi_filtered", + }, + main = "bin_six_a.py", + tags = ["manual"], + deps = ["@pypi//six"], +) + +py_binary( + name = "bin_filtered_six", + srcs = ["bin_six_a.py"], + main = "bin_six_a.py", + tags = ["manual"], + deps = ["@pypi_filtered//six"], +) + +# The pypi_testonly hub only exposes testonly packages. +py_test( + name = "test_testonly", + srcs = ["test_iniconfig.py"], + main = "test_iniconfig.py", + deps = ["@pypi_testonly//iniconfig"], +) + +py_test( + name = "test_testonly_unified", + srcs = ["test_iniconfig.py"], + config_settings = { + "@rules_python//python/config_settings:venv": "pypi_testonly", + }, + main = "test_iniconfig.py", + deps = ["@pypi//iniconfig"], +) + +py_test( + name = "test_all_testonly_requirements", + srcs = ["test_iniconfig.py"], + main = "test_iniconfig.py", + deps = all_testonly_requirements, +) + +# The testonly packages are not in `all_requirements`. +py_binary( + name = "bin_all_requirements", + srcs = ["test_default.py"], + main = "test_default.py", + deps = ["@pypi//colorama"] + all_requirements, +) + +py_binary( + name = "bin_testonly", + srcs = ["test_iniconfig.py"], + main = "test_iniconfig.py", + tags = ["manual"], + deps = ["@pypi_testonly//iniconfig"], +) diff --git a/tests/integration/unified_pypi/MODULE.bazel b/tests/integration/unified_pypi/MODULE.bazel index ee8f622cb4..dfb19bc749 100644 --- a/tests/integration/unified_pypi/MODULE.bazel +++ b/tests/integration/unified_pypi/MODULE.bazel @@ -53,6 +53,34 @@ pip.parse( ) use_repo(pip, "pypi_b") +# `pypi_filtered` only exposes the `[project] dependencies` of the +# `pyproject.toml`, the other packages are not public. +pip.parse( + hub_name = "pypi_filtered", + pyproject_toml = "//:pyproject_filtered.toml", + python_version = "3.11", + requirements_lock = "//:requirements_b.txt", +) +pip.configure( + dependency_groups = ["default"], + hub_name = "pypi_filtered", +) +use_repo(pip, "pypi_filtered") + +# `pypi_testonly` only exposes `iniconfig` and it is testonly. +pip.parse( + hub_name = "pypi_testonly", + pyproject_toml = "//:pyproject_testonly.toml", + python_version = "3.11", + requirements_lock = "//:requirements_testonly.txt", +) +pip.configure( + testonly = True, + dependency_groups = ["test"], + hub_name = "pypi_testonly", +) +use_repo(pip, "pypi_testonly") + pip.default(default_hub = "pypi_b") pip.default(unified_hub_requirements_bzl = True) pip.dep( diff --git a/tests/integration/unified_pypi/pyproject_filtered.toml b/tests/integration/unified_pypi/pyproject_filtered.toml new file mode 100644 index 0000000000..632f72fe22 --- /dev/null +++ b/tests/integration/unified_pypi/pyproject_filtered.toml @@ -0,0 +1,4 @@ +[project] +name = "unified-pypi" +version = "0.0.0" +dependencies = ["colorama"] diff --git a/tests/integration/unified_pypi/pyproject_testonly.toml b/tests/integration/unified_pypi/pyproject_testonly.toml new file mode 100644 index 0000000000..0541ccc0b4 --- /dev/null +++ b/tests/integration/unified_pypi/pyproject_testonly.toml @@ -0,0 +1,6 @@ +[project] +name = "unified-pypi" +version = "0.0.0" + +[dependency-groups] +test = ["iniconfig"] diff --git a/tests/integration/unified_pypi/requirements_testonly.txt b/tests/integration/unified_pypi/requirements_testonly.txt new file mode 100644 index 0000000000..862092a324 --- /dev/null +++ b/tests/integration/unified_pypi/requirements_testonly.txt @@ -0,0 +1,3 @@ +iniconfig==2.1.0 \ + --hash=sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7 \ + --hash=sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760 diff --git a/tests/integration/unified_pypi/test_iniconfig.py b/tests/integration/unified_pypi/test_iniconfig.py new file mode 100644 index 0000000000..45bc1a5008 --- /dev/null +++ b/tests/integration/unified_pypi/test_iniconfig.py @@ -0,0 +1,3 @@ +import iniconfig + +assert iniconfig.IniConfig diff --git a/tests/integration/unified_pypi_test.py b/tests/integration/unified_pypi_test.py index 9ce41fafb7..ea81f3814d 100644 --- a/tests/integration/unified_pypi_test.py +++ b/tests/integration/unified_pypi_test.py @@ -7,6 +7,51 @@ class UnifiedPypiTest(runner.TestCase): + def test_configure_hides_unexposed_packages(self): + self.run_bazel("test", "//:test_filtered_direct") + result = self.run_bazel("build", "//:bin_filtered_six", check=False) + self.assertNotEqual( + result.exit_code, + 0, + "Expected the alias of the unexposed package to be hidden", + ) + self.assert_result_matches( + result, + r"alias '[^']*//six:six'.*is not visible from", + ) + + def test_configure_hides_unexposed_packages_in_unified_hub(self): + self.run_bazel("cquery", "//:bin_filtered_six_unified") + result = self.run_bazel("build", "//:bin_filtered_six_unified", check=False) + self.assertNotEqual( + result.exit_code, + 0, + "Expected build to fail during execution phase", + ) + self.assert_result_matches( + result, + 'ERROR: PyPI package "six" is not available when building under PyPI hub "pypi_filtered"\\. Try adding it to the requirements of this hub', + ) + + def test_configure_testonly(self): + self.run_bazel( + "test", + "//:test_testonly", + "//:test_testonly_unified", + "//:test_all_testonly_requirements", + ) + self.run_bazel("build", "//:bin_all_requirements") + result = self.run_bazel("build", "//:bin_testonly", check=False) + self.assertNotEqual( + result.exit_code, + 0, + "Expected the testonly package to be rejected in a binary", + ) + self.assert_result_matches( + result, + r"non-test target '//:bin_testonly' depends on testonly", + ) + def test_default_fallback_hub(self): self.run_bazel("test", "//:test_default") diff --git a/tests/pypi/dependency_groups/BUILD.bazel b/tests/pypi/dependency_groups/BUILD.bazel new file mode 100644 index 0000000000..ba6877e124 --- /dev/null +++ b/tests/pypi/dependency_groups/BUILD.bazel @@ -0,0 +1,3 @@ +load(":dependency_groups_tests.bzl", "dependency_groups_test_suite") + +dependency_groups_test_suite(name = "dependency_groups_tests") diff --git a/tests/pypi/dependency_groups/dependency_groups_tests.bzl b/tests/pypi/dependency_groups/dependency_groups_tests.bzl new file mode 100644 index 0000000000..ee23aec1fb --- /dev/null +++ b/tests/pypi/dependency_groups/dependency_groups_tests.bzl @@ -0,0 +1,86 @@ +"""dependency_groups tests""" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:dependency_groups.bzl", "select_dependency_groups") # buildifier: disable=bzl-visibility + +_tests = [] + +_PYPROJECT = struct( + name = "my-project", + dependencies = ["main-pkg>=1.0"], + optional_dependencies = { + "Docs": ["docs-pkg"], + "all": ["My_Project[docs, grpc]"], + "grpc": ["grpc-pkg"], + }, + dependency_groups = { + "dev": ["dev-pkg", "shared-pkg", {"include-group": "Test"}], + "test": ["test_pkg", "shared-pkg"], + }, +) + +def _select(specs, pyproject = _PYPROJECT): + failures = [] + got = select_dependency_groups(pyproject, specs, _fail = failures.append) + return got, failures + +def _test_select(env): + for specs, want in [ + (["default"], ["main_pkg"]), + (["optional:docs"], ["docs_pkg"]), + (["group:test"], ["shared_pkg", "test_pkg"]), + (["dev"], ["dev_pkg", "shared_pkg", "test_pkg"]), + (["grpc"], ["grpc_pkg"]), + (["all"], ["docs_pkg", "grpc_pkg"]), + (["optional:*"], ["docs_pkg", "grpc_pkg"]), + (["group:*"], ["dev_pkg", "shared_pkg", "test_pkg"]), + (["default", "group:test"], ["main_pkg", "shared_pkg", "test_pkg"]), + (["*"], ["dev_pkg", "docs_pkg", "grpc_pkg", "main_pkg", "shared_pkg", "test_pkg"]), + ]: + got, failures = _select(specs) + env.expect.where(specs = specs).that_collection(failures).contains_exactly([]) + env.expect.where(specs = specs).that_collection(got.packages).contains_exactly(want).in_order() + env.expect.where(specs = specs).that_bool(got.all).equals(False) + +_tests.append(_test_select) + +def _test_select_all(env): + got, failures = _select(["*", "transitive"]) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_bool(got.all).equals(True) + env.expect.that_collection(got.packages).contains_exactly([]) + +_tests.append(_test_select_all) + +def _test_select_errors(env): + for specs, want in [ + (["missing"], "Unknown dependency group 'missing'. Available groups: default, group:dev, group:test, optional:all, optional:docs, optional:grpc"), + (["default", "transitive"], "The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: [\"default\", \"transitive\"]"), + ]: + _, failures = _select(specs) + env.expect.where(specs = specs).that_collection(failures).contains_exactly([want]) + +_tests.append(_test_select_errors) + +def _test_include_group_errors(env): + for groups, want in [ + ({"a": [{"include-group": "missing"}]}, "Unknown dependency group 'missing' included from 'a'"), + ({"a": [{"include-group": "b"}], "b": [{"include-group": "a"}]}, "Cyclic dependency group includes: a, b"), + ]: + _, failures = _select(["*"], pyproject = struct( + name = None, + dependencies = [], + optional_dependencies = {}, + dependency_groups = groups, + )) + env.expect.that_collection(failures).contains_exactly([want]) + +_tests.append(_test_include_group_errors) + +def dependency_groups_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 91bf3c76d1..90025682ae 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -21,7 +21,7 @@ load("//python/private/pypi:extension.bzl", "build_config", "parse_modules") # load("//python/private/pypi:platform.bzl", _plat = "platform") # buildifier: disable=bzl-visibility load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # buildifier: disable=bzl-visibility load("//tests/support/mocks:mocks.bzl", "mocks") -load(":pip_parse.bzl", _parse = "pip_parse") +load(":pip_parse.bzl", _configure = "configure", _parse = "pip_parse") _tests = [] @@ -107,10 +107,11 @@ def _dep(*, name, extra_targets = []): extra_targets = extra_targets, ) -def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], dep = [], is_root = True): +def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], dep = [], configure = [], is_root = True): return struct( name = name, tags = struct( + configure = configure, parse = parse, override = override, whl_mods = whl_mods, @@ -124,11 +125,14 @@ def _parse_modules(env, **kwargs): return env.expect.that_struct( parse_modules(**kwargs), attrs = dict( + configured_hubs = subjects.dict, declared_deps = subjects.dict, default_hub = subjects.str, exposed_packages = subjects.dict, hub_group_map = subjects.dict, hub_whl_map = subjects.dict, + testonly_packages = subjects.dict, + unified_testonly_packages = subjects.collection, unified_hub_requirements_bzl = subjects.bool, whl_libraries = subjects.dict, whl_mods = subjects.dict, @@ -819,6 +823,155 @@ simple==0.0.1 \ _tests.append(_test_default_index_setting) +_CONFIGURE_REQUIREMENTS = """\ +simple==0.0.1 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +other-pkg==0.0.2 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +transitive-pkg==0.0.3 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +""" + +_CONFIGURE_PYPROJECT = """\ +[project] +dependencies = ["simple"] + +[dependency-groups] +dev = ["other-pkg"] +test = ["simple"] +""" + +def _configure_parse_modules(env, *modules): + return _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + os_name = "linux", + arch_name = "x86_64", + mock_files = { + "pyproject.toml": _CONFIGURE_PYPROJECT, + "requirements.txt": _CONFIGURE_REQUIREMENTS, + }, + *modules + ), + available_interpreters = { + "python_3_15_host": "unit_test_interpreter_target", + }, + minor_mapping = {"3.15": "3.15.19"}, + ) + +def _configure_parse(hub_name = "pypi_a", **kwargs): + return _parse( + hub_name = hub_name, + python_version = "3.15", + simpleapi_skip = ["simple", "other-pkg", "transitive-pkg"], + requirements_lock = "requirements.txt", + **kwargs + ) + +def _test_configure(env): + # The selection itself is tested in `hub_exposure_tests`. + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [_configure_parse(pyproject_toml = "pyproject.toml")], + configure = [_configure(hub_name = "pypi_a", dependency_groups = ["test"])], + ), + ) + + pypi.exposed_packages().contains_exactly({"pypi_a": ["simple"]}) + pypi.configured_hubs().contains_exactly({"pypi_a": struct( + packages = ["simple"], + testonly_packages = [], + )}) + +_tests.append(_test_configure) + +def _test_configure_testonly(env): + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [_configure_parse(pyproject_toml = "pyproject.toml")], + configure = [ + _configure(hub_name = "pypi_a", dependency_groups = ["default"]), + _configure(hub_name = "pypi_a", dependency_groups = ["dev", "test"], testonly = True), + ], + ), + ) + + pypi.exposed_packages().contains_exactly({"pypi_a": ["other_pkg", "simple"]}) + + # `simple` is also selected by the non-testonly tag. + pypi.configured_hubs().contains_exactly({"pypi_a": struct( + packages = ["other_pkg", "simple"], + testonly_packages = ["other_pkg"], + )}) + +_tests.append(_test_configure_testonly) + +def _test_configure_testonly_unified(env): + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [ + _configure_parse(hub_name = "pypi_a", pyproject_toml = "pyproject.toml"), + _configure_parse(hub_name = "pypi_b", pyproject_toml = "pyproject.toml"), + ], + configure = [ + _configure(hub_name = "pypi_a", dependency_groups = ["default"], testonly = True), + _configure(hub_name = "pypi_b", dependency_groups = ["default"]), + ], + ), + ) + + pypi.testonly_packages().contains_exactly({"pypi_a": ["simple"], "pypi_b": []}) + + # A package is testonly in the unified hub if it is testonly in any hub. + pypi.unified_testonly_packages().contains_exactly(["simple"]) + +_tests.append(_test_configure_testonly_unified) + +def _test_configure_default_pyproject(env): + # The `pip.default` pyproject.toml is used for the hubs of the root module. + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + default = _default_tags_default + [_default(pyproject_toml = "pyproject.toml")], + parse = [_configure_parse()], + configure = [_configure(hub_name = "pypi_a", dependency_groups = ["dev"])], + ), + ) + + pypi.configured_hubs().contains_exactly({"pypi_a": struct( + packages = ["other_pkg"], + testonly_packages = [], + )}) + +_tests.append(_test_configure_default_pyproject) + +def _test_not_configured_exposes_all(env): + # The pyproject.toml does not filter the hub without `pip.configure`. + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [_configure_parse()], + ), + ) + + pypi.exposed_packages().contains_exactly({ + "pypi_a": ["other_pkg", "simple", "transitive_pkg"], + }) + pypi.configured_hubs().contains_exactly({}) + +_tests.append(_test_not_configured_exposes_all) + def extension_test_suite(name): """Create the test suite. diff --git a/tests/pypi/extension/pip_parse.bzl b/tests/pypi/extension/pip_parse.bzl index 638836e588..d6d6d4aa79 100644 --- a/tests/pypi/extension/pip_parse.bzl +++ b/tests/pypi/extension/pip_parse.bzl @@ -1,5 +1,19 @@ """A simple test helper""" +def configure( + *, + dependency_groups = ["*"], + hub_name, + pyproject_toml = None, + testonly = False): + """A simple helper for testing to simulate the PyPI extension configure tag class""" + return struct( + dependency_groups = dependency_groups, + hub_name = hub_name, + pyproject_toml = pyproject_toml, + testonly = testonly, + ) + def pip_parse( *, hub_name, diff --git a/tests/pypi/hub_exposure/BUILD.bazel b/tests/pypi/hub_exposure/BUILD.bazel new file mode 100644 index 0000000000..187361e1ce --- /dev/null +++ b/tests/pypi/hub_exposure/BUILD.bazel @@ -0,0 +1,3 @@ +load(":hub_exposure_tests.bzl", "hub_exposure_test_suite") + +hub_exposure_test_suite(name = "hub_exposure_tests") diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl new file mode 100644 index 0000000000..dd005c6133 --- /dev/null +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -0,0 +1,194 @@ +"""hub_exposure tests""" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:hub_exposure.bzl", "hub_exposure") # buildifier: disable=bzl-visibility +load("//tests/support/mocks:mocks.bzl", "mocks") + +_tests = [] + +_PYPROJECT = """\ +[project] +dependencies = ["simple"] + +[dependency-groups] +dev = ["other-pkg"] +test = ["simple", "not-in-hub"] +""" + +def _hub(*, module_name = "root", pyproject = None): + return struct( + module_name = module_name, + pyproject = pyproject, + packages = ["other_pkg", "simple", "transitive_pkg"], + ) + +def _configure(hub_name = "hub", dependency_groups = ["*"], pyproject_toml = None, testonly = False): + return mocks.tag( + hub_name = hub_name, + dependency_groups = dependency_groups, + pyproject_toml = pyproject_toml, + testonly = testonly, + ) + +def _module(name = "root", is_root = True, configure = []): + return mocks.module(name = name, is_root = is_root, configure = configure) + +def _exposure(*modules, hubs = None, default_pyproject = None): + failures = [] + got = hub_exposure( + mocks.mctx( + modules = list(modules), + mock_files = { + "other.toml": "[project]\ndependencies = [\"other-pkg\"]\n", + "pyproject.toml": _PYPROJECT, + }, + ), + hubs = hubs or {"hub": _hub(pyproject = "pyproject.toml")}, + default_pyproject = default_pyproject, + _fail = failures.append, + ) + return got, failures + +def _packages(got): + return {hub: info.packages for hub, info in got.items()} + +def _test_not_configured(env): + got, failures = _exposure(_module()) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({}) + +_tests.append(_test_not_configured) + +def _test_configure(env): + for groups, want in [ + (["*"], ["other_pkg", "simple"]), + (["default"], ["simple"]), + (["dev"], ["other_pkg"]), + # `not-in-hub` is ignored + (["group:test"], ["simple"]), + ]: + got, failures = _exposure(_module(configure = [_configure(dependency_groups = groups)])) + env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) + env.expect.where(groups = groups).that_dict(_packages(got)).contains_exactly({"hub": want}) + +_tests.append(_test_configure) + +def _test_configure_all_packages(env): + # Exposing all of the packages is the same as not configuring the hub, so + # that e.g. the platform-specific packages stay public. + got, failures = _exposure(_module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(dependency_groups = ["*", "transitive"]), + ])) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got).contains_exactly({}) + +_tests.append(_test_configure_all_packages) + +def _test_configure_combines_tags(env): + got, failures = _exposure(_module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(dependency_groups = ["dev"]), + ])) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(_packages(got)).contains_exactly({"hub": ["other_pkg", "simple"]}) + +_tests.append(_test_configure_combines_tags) + +def _test_configure_pyproject_precedence(env): + # The tag wins over the `pip.parse` value. + got, _ = _exposure(_module(configure = [ + _configure(pyproject_toml = "other.toml"), + ])) + env.expect.that_dict(_packages(got)).contains_exactly({"hub": ["other_pkg"]}) + + # The `pip.default` value is used for the hubs of the root module. + got, failures = _exposure( + _module(configure = [_configure(dependency_groups = ["default"])]), + hubs = {"hub": _hub()}, + default_pyproject = "pyproject.toml", + ) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(_packages(got)).contains_exactly({"hub": ["simple"]}) + +_tests.append(_test_configure_pyproject_precedence) + +def _test_configure_module_precedence(env): + hubs = {"hub": _hub(module_name = "dep", pyproject = "pyproject.toml")} + + # The owner of the hub can configure it. + got, _ = _exposure( + _module(), + _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + hubs = hubs, + ) + env.expect.that_dict(_packages(got)).contains_exactly({"hub": ["other_pkg"]}) + + # The root module wins over the owner. + got, _ = _exposure( + _module(configure = [_configure(dependency_groups = ["default"])]), + _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + hubs = hubs, + ) + env.expect.that_dict(_packages(got)).contains_exactly({"hub": ["simple"]}) + + # Other modules are ignored. + got, _ = _exposure( + _module(), + _module(name = "other", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + hubs = hubs, + ) + env.expect.that_dict(got).contains_exactly({}) + +_tests.append(_test_configure_module_precedence) + +def _test_configure_errors(env): + _, failures = _exposure(_module(configure = [_configure(hub_name = "missing")])) + env.expect.that_collection(failures).contains_exactly([ + "pip.configure() refers to unknown hub 'missing'. Available hubs: hub", + ]) + + _, failures = _exposure( + _module(configure = [_configure()]), + hubs = {"hub": _hub()}, + ) + env.expect.that_collection(failures).contains_exactly([ + "pip.configure() for hub 'hub' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`", + ]) + +_tests.append(_test_configure_errors) + +def _test_configure_testonly(env): + got, failures = _exposure(_module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(dependency_groups = ["dev", "test"], testonly = True), + ])) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(_packages(got)).contains_exactly({"hub": ["other_pkg", "simple"]}) + + # `simple` is also selected by the non-testonly tag. + env.expect.that_collection(got["hub"].testonly_packages).contains_exactly(["other_pkg"]) + + got, _ = _exposure(_module(configure = [ + _configure(dependency_groups = ["*"], testonly = True), + ])) + env.expect.that_collection(got["hub"].testonly_packages).contains_exactly(["other_pkg", "simple"]) + + # The hub that exposes all of the packages behaves as if it was not + # configured, so the packages can't be made testonly. + got, failures = _exposure(_module(configure = [ + _configure(dependency_groups = ["*", "transitive"], testonly = True), + ])) + env.expect.that_collection(failures).contains_exactly([ + "pip.configure() for hub 'hub' cannot use `testonly` when exposing all of the packages with [\"*\", \"transitive\"]", + ]) + +_tests.append(_test_configure_testonly) + +def hub_exposure_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index 2d54bd34db..fdec8877fc 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -55,6 +55,76 @@ def _test_legacy_aliases(env): _tests.append(_test_legacy_aliases) +def _testonly_mocks(): + got = {} + + def _add(kind): + return lambda name, **kwargs: got.setdefault(name, (kind, kwargs.get("testonly", False))) + + return got, struct( + native = struct( + alias = _add("alias"), + filegroup = _add("filegroup"), + ), + rules = struct( + py_library = _add("py_library"), + ), + ) + +def _test_testonly_aliases(env): + got, mocks = _testonly_mocks() + pkg_aliases( + name = "foo", + actual = "repo", + testonly = True, + native = mocks.native, + rules = mocks.rules, + ) + + # Bazel does not enforce `testonly` on aliases, so the public targets are + # wrapped. + env.expect.that_dict(got).contains_exactly({ + "data": ("filegroup", True), + "dist_info": ("filegroup", True), + "extracted_whl_files": ("filegroup", True), + "foo": ("alias", True), + "pkg": ("py_library", True), + "srcs": ("alias", True), + "whl": ("filegroup", True), + "whl_file": ("alias", True), + "_testonly_data": ("alias", False), + "_testonly_dist_info": ("alias", False), + "_testonly_extracted_whl_files": ("alias", False), + "_testonly_pkg": ("alias", False), + "_testonly_whl": ("alias", False), + }) + +_tests.append(_test_testonly_aliases) + +def _test_testonly_group_aliases(env): + got, mocks = _testonly_mocks() + pkg_aliases( + name = "foo", + actual = "repo", + group_name = "my_group", + testonly = True, + native = mocks.native, + rules = mocks.rules, + ) + + # The private targets are used by the non-testonly group libraries. + env.expect.that_dict(got).contains_at_least({ + "foo": ("alias", True), + "pkg": ("py_library", True), + "whl": ("filegroup", True), + "_pkg": ("alias", False), + "_testonly_pkg": ("alias", False), + "_testonly_whl": ("alias", False), + "_whl": ("alias", False), + }) + +_tests.append(_test_testonly_group_aliases) + def _test_config_setting_aliases(env): # Use this function as it is used in pip_repository got = {} diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index 9114f59279..38699f130b 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -58,6 +58,99 @@ pkg_aliases( _tests.append(_test_legacy_aliases) +def _test_testonly_aliases(env): + actual = render_pkg_aliases( + aliases = { + "bar": "pypi_bar", + "foo": "pypi_foo", + }, + testonly_packages = ["Foo"], + ) + + env.expect.that_str(actual["foo/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = ["//visibility:public"]) + +pkg_aliases( + name = "foo", + actual = "pypi_foo", + testonly = True, +)""") + env.expect.that_str(actual["bar/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = ["//visibility:public"]) + +pkg_aliases( + name = "bar", + actual = "pypi_bar", +)""") + +_tests.append(_test_testonly_aliases) + +def _test_hidden_aliases(env): + actual = render_pkg_aliases( + aliases = { + "bar": "pypi_bar", + "foo": "pypi_foo", + }, + exposed_packages = ["foo"], + ) + + env.expect.that_str(actual["bar/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = [ + "@pypi_bar//:__pkg__", + "@pypi_foo//:__pkg__", +]) + +pkg_aliases( + name = "bar", + actual = "pypi_bar", +)""") + env.expect.that_str(actual["foo/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = ["//visibility:public"]) + +pkg_aliases( + name = "foo", + actual = "pypi_foo", +)""") + +_tests.append(_test_hidden_aliases) + +def _test_hidden_aliases_in_groups(env): + actual = render_pkg_aliases( + aliases = { + "bar": "pypi_bar", + "baz": "pypi_baz", + }, + requirement_cycles = { + "group": ["bar", "baz"], + }, + exposed_packages = ["baz"], + ) + + env.expect.that_str(actual["bar/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = [ + "@pypi_bar//:__pkg__", + "@pypi_baz//:__pkg__", + "//_groups:__pkg__", +]) + +pkg_aliases( + name = "bar", + actual = "pypi_bar", + group_name = "group", +)""") + +_tests.append(_test_hidden_aliases_in_groups) + def _test_bzlmod_aliases(env): # Use this function as it is used in pip_repository actual = render_multiplatform_pkg_aliases(