diff --git a/MODULE.bazel b/MODULE.bazel index 51139cbe6d..4d445a37bc 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -335,6 +335,7 @@ use_repo( "pkgutil_nspkg2", "rules_python_runtime_env_tc_info", "somepkg_with_build_files", + "whl_library_extras_consumer", "whl_library_extras_direct_dep", "whl_with_build_files", "whl_with_data1", diff --git a/docs/pypi/circular-dependencies.md b/docs/pypi/circular-dependencies.md index 62613f489e..2d510316e7 100644 --- a/docs/pypi/circular-dependencies.md +++ b/docs/pypi/circular-dependencies.md @@ -65,6 +65,32 @@ be a part of the `airflow` cycle. For instance: ) ``` +Each extra of a package has its own target, so a cycle that goes through an +extra back to the package without extras, e.g. `foo[bar]` depending on `baz` +which depends on `foo`, is not a cycle in Bazel and does not need a group. If +the cycle includes the extra itself, e.g. `baz` depends on `foo[bar]`, then +only the extra needs to be a member of the group: + +```starlark + ... + experimental_requirement_cycles = { + "foo": [ + "foo[bar]", + "baz", + ] + } +) +``` + +Only the targets of the members, `@pypi//foo:[bar]` and `@pypi//baz`, are +then part of the group and the other targets of `foo`, e.g. `@pypi//foo:[]`, +can be used without the group. The `whl` targets do not use the extras, so +only the packages without extras are part of the group for the `whl` targets. + +:::{versionadded} VERSION_NEXT_FEATURE +Support for the extras as members of the groups. +::: + Alternatively, one could resolve the cycle by removing one leg of it. For example, while `apache-airflow-providers-sqlite` is "baked into" the Airflow diff --git a/docs/pypi/download.md b/docs/pypi/download.md index cc8b7890aa..a29b1387f7 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -134,6 +134,72 @@ rename, and `requirement("numpy")` routes through the active hub like fixed at loading time, before the active hub is known, so load them from a concrete hub instead. Prefer `@pypi//` labels in new code. +(pip-configure)= +### Configuring the Exposed Packages (pip.configure) + +By default a hub exposes all of the packages from its requirements files or +`uv.lock` file, including the transitive dependencies. A `pip.configure` tag +can limit the hub to the packages listed in a `pyproject.toml` file: the +`[project] dependencies`, `[project.optional-dependencies]` and PEP 735 +`[dependency-groups]` tables. + +```toml +[project] +dependencies = ["numpy"] + +[dependency-groups] +dev = ["pytest"] +``` + +```starlark +pip.parse( + hub_name = "pypi_hub", + pyproject_toml = "//:pyproject.toml", + requirements_lock = "//:requirements_lock.txt", +) + +pip.configure( + hub_name = "pypi_hub", + dependency_groups = ["default", "group:dev"], +) +``` + +Here `@pypi_hub//numpy` and `@pypi_hub//pytest` can be used, but the +dependencies of `numpy` are only used through `@pypi_hub//numpy`. The +`all_requirements` lists in the hub's `requirements.bzl` and the +[Unified @pypi Hub](unified-pypi-hub) only contain the exposed packages. + +The {attr}`pip.configure.dependency_groups` attribute defaults to `["*"]`, +which selects every group. Use `["*", "transitive"]` to expose all of the +packages of the hub, including the transitive dependencies, which is the same +as not configuring the hub. Selecting the transitive dependencies of only some +of the groups is not supported. + +Multiple tags for the same hub are combined. Only the root module and the +module that created the hub can configure it; if both do, the root module +wins. A self reference such as `my-project[test]` in the `pyproject.toml` +selects the optional dependencies of the project. The packages that are not +part of the hub are ignored, so a single `pyproject.toml` can list tools that +are missing from the lock file. + +The extras requested in the selected groups are exposed by the hub even if the +lock file does not request them, for example because the lock file was +generated with `--strip-extras`. With the following `pyproject.toml`: + +```toml +[project] +dependencies = ["requests[socks]"] +``` + +the hub has the `@pypi_hub//requests:[socks]` target and +`@pypi_hub//requests` includes the `socks` extra on all of the platforms. The +environment markers of the requirements are ignored when selecting the extras. +The dependencies of the extras, such as `pysocks`, still have to be in the lock +file. See [extras](./use.md#extras) for the targets of the extras. + +:::{versionadded} VERSION_NEXT_FEATURE +::: + ### Declaring Abstract Dependencies (pip.dep) :::{versionadded} 2.2.0 diff --git a/docs/pypi/use.md b/docs/pypi/use.md index 9d0c54c4ab..2660749052 100644 --- a/docs/pypi/use.md +++ b/docs/pypi/use.md @@ -57,11 +57,85 @@ which can help you create a `py_binary` target for a particular console script e [whl_ep]: https://packaging.python.org/specifications/entry-points/ +{#extras} ## 'Extras' dependencies -Any "extras" specified in the requirements lock file will be automatically added -as transitive dependencies of the package. In the example above, you'd just put -`requirement("useful_dep")` or `@pypi//useful_dep`. +Any "extras" specified in the requirements lock file or the `uv.lock` file will +be automatically added as transitive dependencies of the package. In the +example above, you'd just put `requirement("useful_dep")` or +`@pypi//useful_dep`. + +If you need only some of the extras, the hub repo also has a target for the +package without any extras and a target for each extra used in the lock file. +For example, if the lock file has `google-api-core[async_rest,grpc]`, then the +hub has: + +``` +@pypi//google_api_core # The package with all of the extras +@pypi//google_api_core:[] # The package without any extras +@pypi//google_api_core:[async_rest] # The package with the async_rest extra +@pypi//google_api_core:[grpc] # The package with the grpc extra +``` + +Extra names are normalized in the same way as the package names, so +`async-rest` becomes `async_rest`. + +The `requirement` helper supports the same syntax: + +```starlark +load("@pypi//:requirements.bzl", "requirement") + +py_library( + name = "my_lib", + deps = [ + requirement("google-api-core[grpc]"), + ], +) +``` + +There are no targets for combinations of extras, because each `[extra]` target +already includes the package. The `requirement` helper accepts at most one +extra. To request more than one extra, use the `requirements` helper, which +accepts any number of packages and returns a list with a label for each +package and each extra: + +```starlark +load("@pypi//:requirements.bzl", "requirements") + +py_library( + name = "my_lib", + deps = [ + ":other_lib", + ] + requirements( + "google-api-core[async_rest,grpc]", + "numpy", + ), +) +``` + +If the lock file requests different extras for a package on different +platforms, for example `foo[bar]` on Linux and `foo[baz]` on macOS, then +`@pypi//foo` includes only the extras requested for the target platform and +`@pypi//foo:[bar]` on macOS is the same as `@pypi//foo:[]`. + +The dependencies of the packages use the extras written in the wheel +`METADATA`, so if `foo` depends on `bar[socks]`, then `@pypi//foo` depends on +`@pypi//bar:[socks]`. If the extra is not in the lock file, then building +`@pypi//foo` fails with a message asking you to add `bar[socks]` to the +requirements or the lock file. For example, when compiling the requirements +with `uv pip compile`, pass `--no-strip-extras` to keep the extras in the +output. + +The `whl` targets, e.g. `@pypi//foo:whl`, do not use the extras and include +only the wheels of the package and its dependencies without extras. + +The packages in a dependency group can also have only some of their extras in +the group, see the [circular dependencies](./circular-dependencies) docs. + +:::{versionadded} VERSION_NEXT_FEATURE +The `[]` and `[extra]` targets, the support for extras in the `requirement` +helper and the `requirements` helper were added. +::: ## Consuming Wheel Dists Directly diff --git a/examples/pip_parse_vendored/requirements.bzl b/examples/pip_parse_vendored/requirements.bzl index 95c4b985f9..6db1f21765 100644 --- a/examples/pip_parse_vendored/requirements.bzl +++ b/examples/pip_parse_vendored/requirements.bzl @@ -5,6 +5,7 @@ load("@rules_python//python:pip.bzl", "pip_utils") load("@rules_python//python/pip_install:pip_repository.bzl", "whl_config_repo", "whl_library") +load("@rules_python//python/private/pypi:labels.bzl", "requirement_label", "requirement_labels") # buildifier: disable=bzl-visibility all_requirements = [ "@my_project_pip_deps_vendored_certifi//:pkg", @@ -79,7 +80,28 @@ _config = { _annotations = {} def requirement(name): - return "@my_project_pip_deps_vendored_{}//:{}".format(pip_utils.normalize_name(name), "pkg") + """Return the label for the package. + + Args: + name: The name of the package with at most one extra, e.g. `foo`, + `foo[]` or `foo[bar]`. Use `requirements` for multiple extras. + + Returns: + The label of the package. + """ + return requirement_label("@my_project_pip_deps_vendored_{}//:{}", name) + +def requirements(*names): + """Return the labels for the packages. + + Args: + *names: The names of the packages with optional extras, e.g. `foo`, + `foo[]`, `foo[bar]` or `foo[bar,baz]`. + + Returns: + The list of labels, with a label for each extra of a package. + """ + return requirement_labels("@my_project_pip_deps_vendored_{}//:{}", *names) def whl_requirement(name): return "@my_project_pip_deps_vendored_{}//:{}".format(pip_utils.normalize_name(name), "whl") @@ -107,8 +129,10 @@ def install_deps(**whl_library_kwargs): # Set up the requirement groups all_requirement_groups = {} + # The members are the packages, e.g. `foo`, or the extras of the packages, + # e.g. `foo[bar]`. requirement_group_mapping = { - requirement: group_name + requirement.partition("[")[0]: group_name for group_name, group_requirements in all_requirement_groups.items() for requirement in group_requirements } diff --git a/news/2786.added.md b/news/2786.added.md new file mode 100644 index 0000000000..4c00959a7d --- /dev/null +++ b/news/2786.added.md @@ -0,0 +1,17 @@ +(pypi) The hub repositories now have a `[]` target for each package without +any extras and an `[extra]` target for each extra used in the lock file, e.g. +`@pypi//google_api_core:[grpc]`, in both bzlmod and `WORKSPACE`. The +`requirement` macro accepts a single extra, e.g. +`requirement("google-api-core[grpc]")`, and the new `requirements` macro +returns a list of labels for any number of packages and extras, e.g. +`requirements("google-api-core[async_rest,grpc]", "numpy")`. The whl +repositories now create the targets for every extra that the package provides +and the requested extras are no longer passed to them, so changing the +requested extras does not refetch them. The dependencies use the extras from +the wheel `METADATA` and the build fails with a helpful message if an extra is +missing in the lock file. The `whl` targets no longer include the dependencies +of the extras. The members of `experimental_requirement_cycles` can now be the +extras of the packages, e.g. `foo[bar]`. See +[the docs](https://rules-python.readthedocs.io/en/latest/pypi/use.html#extras) +([#2786](https://github.com/bazel-contrib/rules_python/issues/2786), +[#4168](https://github.com/bazel-contrib/rules_python/issues/4168)). diff --git a/news/4395.added.md b/news/4395.added.md new file mode 100644 index 0000000000..3e633cc997 --- /dev/null +++ b/news/4395.added.md @@ -0,0 +1,3 @@ +(pypi) Added {obj}`pip.configure` to limit the packages that a hub exposes to +the ones listed in the dependency groups of a `pyproject.toml` file +([#3413](https://github.com/bazel-contrib/rules_python/issues/3413)). diff --git a/news/4522.added.md b/news/4522.added.md new file mode 100644 index 0000000000..a722e3f9eb --- /dev/null +++ b/news/4522.added.md @@ -0,0 +1,5 @@ +(pypi) {obj}`pip.configure` now exposes the extras requested in the selected +`pyproject.toml` dependency groups, e.g. `requests[socks]`, as the +`@pypi//requests:[socks]` target and includes them in `@pypi//requests`, even +if the lock file does not request them +([#2786](https://github.com/bazel-contrib/rules_python/issues/2786)). diff --git a/python/private/internal_dev_deps.bzl b/python/private/internal_dev_deps.bzl index 5bbee232f9..1c5d15e848 100644 --- a/python/private/internal_dev_deps.bzl +++ b/python/private/internal_dev_deps.bzl @@ -39,7 +39,6 @@ def _internal_dev_deps_impl(mctx): whl_library( name = "somepkg_with_build_files", whl_file = "@whl_with_build_files//:somepkg-1.0-any-none-any.whl", - requirement = "somepkg", ) # Setup for //tests/implicit_namespace_packages @@ -51,7 +50,6 @@ def _internal_dev_deps_impl(mctx): whl_library( name = "implicit_namespace_ns_sub1", whl_file = "@implicit_namespace_ns_sub1_whl//:ns_sub1-1.0-any-none-any.whl", - requirement = "ns-sub1", enable_implicit_namespace_pkgs = False, ) @@ -63,7 +61,6 @@ def _internal_dev_deps_impl(mctx): whl_library( name = "implicit_namespace_ns_sub2", whl_file = "@implicit_namespace_ns_sub2_whl//:ns_sub2-1.0-any-none-any.whl", - requirement = "ns-sub2", enable_implicit_namespace_pkgs = False, ) @@ -75,7 +72,6 @@ def _internal_dev_deps_impl(mctx): whl_library( name = "pkgutil_nspkg1", whl_file = "@pkgutil_nspkg1_whl//:pkgutil_nspkg1-1.0-any-none-any.whl", - requirement = "pkgutil_nspkg1", enable_implicit_namespace_pkgs = False, ) @@ -87,7 +83,6 @@ def _internal_dev_deps_impl(mctx): whl_library( name = "pkgutil_nspkg2", whl_file = "@pkgutil_nspkg2_whl//:pkgutil_nspkg2-1.0-any-none-any.whl", - requirement = "pkgutil_nspkg2", enable_implicit_namespace_pkgs = False, ) @@ -99,7 +94,6 @@ def _internal_dev_deps_impl(mctx): whl_library( name = "whl_with_data1", whl_file = "@whl_with_data1_whl//:whl_with_data1-1.0-any-none-any.whl", - requirement = "whl-with-data1", ) whl_from_dir_repo( @@ -110,14 +104,28 @@ def _internal_dev_deps_impl(mctx): whl_library( name = "whl_with_data2", whl_file = "@whl_with_data2_whl//:whl_with_data2-1.0-any-none-any.whl", - requirement = "whl-with-data2", ) _whl_library_from_dir( name = "whl_library_extras_direct_dep", root = "//tests/pypi/whl_library/testdata/pkg:BUILD.bazel", output = "pkg-1.0-any-none-any.whl", - requirement = "pkg[optional]", + # The following is necessary to enable pipstar and make tests faster + config_load = "@rules_python//tests/pypi/whl_library/testdata:packages.bzl", + dep_template = "@whl_library_extras_{name}//:{target}", + ) + _whl_library_from_dir( + name = "whl_library_extras_pkg", + root = "//tests/pypi/whl_library/testdata/pkg:BUILD.bazel", + output = "pkg-1.0-any-none-any.whl", + # The following is necessary to enable pipstar and make tests faster + config_load = "@rules_python//tests/pypi/whl_library/testdata:packages.bzl", + dep_template = "@whl_library_extras_{name}//:{target}", + ) + _whl_library_from_dir( + name = "whl_library_extras_consumer", + root = "//tests/pypi/whl_library/testdata/consumer:BUILD.bazel", + output = "consumer-1.0-any-none-any.whl", # The following is necessary to enable pipstar and make tests faster config_load = "@rules_python//tests/pypi/whl_library/testdata:packages.bzl", dep_template = "@whl_library_extras_{name}//:{target}", @@ -126,7 +134,6 @@ def _internal_dev_deps_impl(mctx): name = "whl_library_extras_optional_dep", root = "//tests/pypi/whl_library/testdata/optional_dep:BUILD.bazel", output = "optional_dep-1.0-any-none-any.whl", - requirement = "optional_dep", # The following is necessary to enable pipstar and make tests faster config_load = "@rules_python//tests/pypi/whl_library/testdata:packages.bzl", ) @@ -140,7 +147,6 @@ def _internal_dev_deps_impl(mctx): whl_library( name = "patch_whl_pkg", whl_file = "@patch_whl_pkg_whl//:pkg-1.0-any-none-any.whl", - requirement = "pkg", whl_patches = { "//tests/pypi/patch_whl/testdata/patches:modify_pkg.patch": json.encode({ "patch_strip": 0, diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 03a9e113d5..62f8c99125 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -129,6 +129,7 @@ bzl_library( deps = [ ":default_platforms", ":hub_builder", + ":hub_exposure", ":hub_repository", ":parse_whl_name", ":pep508_env", @@ -182,9 +183,11 @@ bzl_library( deps = [ ":attrs", ":hash", + ":labels", ":parse_requirements", ":pep508_env", ":pep508_evaluate", + ":pep508_requirement", ":python_tag", ":requirements_files_by_platform", ":whl_config_setting", @@ -325,8 +328,10 @@ bzl_library( deps = [ ":default_platforms", ":hash", + ":labels", ":parse_requirements", ":pep508_env", + ":pep508_requirement", ":pip_repository_attrs", ":pypi_repo_utils", ":render_pkg_aliases", @@ -352,6 +357,7 @@ bzl_library( deps = [ ":labels", ":whl_library_deps_targets", + "//python:py_library", "//python/private:common_labels", "//python/private:text_util", "@bazel_skylib//lib:selects", @@ -384,6 +390,7 @@ bzl_library( srcs = ["render_pkg_aliases.bzl"], deps = [ ":generate_group_library_build_bazel", + ":labels", "//python/private:normalize_name", "//python/private:text_util", ], @@ -583,7 +590,6 @@ bzl_library( srcs = ["whl_deps_repo.bzl"], deps = [ ":generate_whl_library_build_bazel", - ":pep508_requirement", "//python/private:repo_utils", ], ) @@ -594,7 +600,6 @@ bzl_library( deps = [ ":generate_whl_library_build_bazel", ":patch_whl", - ":pep508_requirement", ":pypi_repo_utils", ":whl_extract", ":whl_metadata", @@ -609,6 +614,7 @@ bzl_library( deps = [ ":env_marker_setting", ":labels", + ":missing_package", ":pep508_deps", "//python:py_library", "//python/private:normalize_name", @@ -629,6 +635,32 @@ bzl_library( ], ) +bzl_library( + name = "dependency_groups", + srcs = ["dependency_groups.bzl"], + deps = [ + ":pep508_requirement", + "//python/private:normalize_name", + "@bazel_skylib//lib:types", + ], +) + +bzl_library( + name = "hub_exposure", + srcs = ["hub_exposure.bzl"], + deps = [ + ":dependency_groups", + ":whl_config_setting", + "//python/private:pyproject_utils", + ], +) + +bzl_library( + name = "labels", + srcs = ["labels.bzl"], + deps = ["//python/private:normalize_name"], +) + bzl_library( name = "argparse", srcs = ["argparse.bzl"], @@ -654,11 +686,6 @@ bzl_library( srcs = ["hash.bzl"], ) -bzl_library( - name = "labels", - srcs = ["labels.bzl"], -) - bzl_library( name = "package_annotation", srcs = ["package_annotation.bzl"], diff --git a/python/private/pypi/dependency_groups.bzl b/python/private/pypi/dependency_groups.bzl new file mode 100644 index 0000000000..60c48eac83 --- /dev/null +++ b/python/private/pypi/dependency_groups.bzl @@ -0,0 +1,157 @@ +"""Select PyPI packages using pyproject.toml dependency groups.""" + +load("@bazel_skylib//lib:types.bzl", "types") +load("//python/private:normalize_name.bzl", "normalize_name") +load(":pep508_requirement.bzl", "requirement") + +def select_dependency_groups(pyproject, specs, _fail = fail): + """Select the packages named by the given dependency group specifiers. + + Supported specifiers are: + + * `"default"`: the `[project] dependencies`. + * `"optional:"`: a `[project.optional-dependencies]` entry. + * `"group:"`: a PEP 735 `[dependency-groups]` entry. + * `"optional:*"` and `"group:*"`: all entries of the table. + * `"*"`: all of the above. + * `""`: a dependency group, or an optional dependency group if + there is no dependency group with that name. + * `"transitive"`: only supported together with `"*"` and selects all + of the packages, including the transitive dependencies. + + Args: + pyproject: {type}`struct` returned by `read_pyproject`. + specs: {type}`list[str]` the specifiers. + _fail: {type}`callable` the failure function, mainly for testing. + + Returns: + {type}`struct` with the attributes: + * `packages`: {type}`list[str]` the sorted normalized package names. + * `all`: {type}`bool` whether all of the packages including the + transitive dependencies should be selected, i.e. `"*"` and + `"transitive"` were given. + * `extras`: {type}`dict[str, list[str]]` the normalized package + names mapped to the sorted normalized extras that the selected + requirements request, e.g. `requests[socks]`. When `all` is set, + the extras of all of the groups are returned. + """ + groups = {"default": pyproject.dependencies} + for name, reqs in pyproject.optional_dependencies.items(): + groups["optional:" + normalize_name(name)] = reqs + for name, reqs in _expand_dependency_groups(pyproject.dependency_groups, _fail).items(): + groups["group:" + name] = reqs + + select_all = "transitive" in specs + if select_all: + if "*" not in specs: + _fail("The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: {}".format(specs)) + return struct(packages = [], all = True, extras = {}) + specs = ["*"] + + selected = {} + for spec in specs: + keys = _match(spec, groups) + if not keys: + _fail("Unknown dependency group '{}'. Available groups: {}".format( + spec, + ", ".join(sorted(groups)), + )) + for key in keys: + selected[key] = None + + # A self reference like `my-project[test]` selects the optional + # dependencies, but only one level deep. + project = normalize_name(pyproject.name or "") + reqs = [] + for key in selected: + for req in groups[key]: + req = requirement(req) + if req.name_ != project: + reqs.append(req) + continue + for extra in req.extras: + reqs.extend([requirement(r) for r in groups.get("optional:" + extra, [])]) + + packages = {} + extras = {} + for req in reqs: + packages[req.name_] = None + if req.extras: + pkg_extras = extras.setdefault(req.name_, {}) + for extra in req.extras: + pkg_extras[extra] = None + + return struct( + packages = [] if select_all else sorted(packages), + all = select_all, + extras = {name: sorted(pkg_extras) for name, pkg_extras in extras.items()}, + ) + +def _match(spec, groups): + if spec == "*": + return groups.keys() + if spec in ("optional:*", "group:*"): + prefix = spec[:-1] + return [key for key in groups if key.startswith(prefix)] + + kind, _, name = spec.rpartition(":") + name = normalize_name(name) + if kind: + candidates = [kind + ":" + name] + elif name == "default": + candidates = ["default"] + else: + candidates = ["group:" + name, "optional:" + name] + + for key in candidates: + if key in groups: + return [key] + return [] + +def _expand_dependency_groups(dependency_groups, _fail): + """Expand the PEP 735 `include-group` entries. + + Args: + dependency_groups: {type}`dict[str, list]` the `[dependency-groups]` + table. + _fail: {type}`callable` the failure function. + + Returns: + {type}`dict[str, list[str]]` normalized group names mapped to their + requirement strings. + """ + groups = { + normalize_name(name): items + for name, items in dependency_groups.items() + } + + # Each round expands at least one group unless there is a cycle. + expanded = {} + for _ in range(len(groups)): + for name, items in groups.items(): + if name in expanded: + continue + + reqs = [] + for item in items: + if types.is_string(item): + reqs.append(item) + continue + + include = normalize_name(item.get("include-group", "")) + if include not in groups: + _fail("Unknown dependency group '{}' included from '{}'".format(include, name)) + return {} + if include not in expanded: + reqs = None + break + reqs.extend(expanded[include]) + + if reqs != None: + expanded[name] = reqs + + if len(expanded) != len(groups): + _fail("Cyclic dependency group includes: {}".format( + ", ".join(sorted([name for name in groups if name not in expanded])), + )) + return expanded diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 21d5ba509c..8ddc3c5847 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -24,6 +24,7 @@ load("//python/private:pyproject_utils.bzl", "read_pyproject", "version_from_req load("//python/private:repo_utils.bzl", "repo_utils") load(":default_platforms.bzl", "default_platforms") load(":hub_builder.bzl", "hub_builder") +load(":hub_exposure.bzl", "add_extras", "hub_exposure") load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") load(":parse_whl_name.bzl", "parse_whl_name") load(":pep508_env.bzl", "env") @@ -89,6 +90,7 @@ def build_config( unified_hub_requirements_bzl = False defaults = { "platforms": default_platforms(), + "pyproject_toml": None, "python_version": None, } for mod in module_ctx.modules: @@ -105,6 +107,8 @@ def build_config( unified_hub_requirements_bzl = True pyproject_toml = tag.pyproject_toml if pyproject_toml: + if mod.is_root: + defaults["pyproject_toml"] = pyproject_toml pyproject = read_pyproject(module_ctx, pyproject_toml) if pyproject.requires_python: defaults["python_version"] = version_from_requires_python(pyproject.requires_python) @@ -153,6 +157,7 @@ def build_config( for name, values in defaults["platforms"].items() }, enable_pipstar_extract = enable_pipstar_extract, + pyproject_toml = defaults["pyproject_toml"], toml_decode = toml.decode, unified_hub_requirements_bzl = unified_hub_requirements_bzl, ) @@ -245,6 +250,7 @@ You cannot use both the additive_build_content and additive_build_content_file a # dict[str repo, HubBuilder] # See `hub_builder.bzl%hub_builder()` for `HubBuilder` pip_hub_map = {} + hub_pyproject = {} simpleapi_cache = pypi_cache(mctx = module_ctx) is_pypi_hub_reserved = module_ctx.getenv("RULES_PYTHON_PYPI_HUB_RESERVED", "0") == "1" @@ -321,6 +327,9 @@ You cannot use both the additive_build_content and additive_build_content_file a else: builder = pip_hub_map[hub_name] + if pip_attr.pyproject_toml and hub_name not in hub_pyproject: + hub_pyproject[hub_name] = pip_attr.pyproject_toml + builder.pip_parse( module_ctx, pip_attr = pip_attr, @@ -345,6 +354,7 @@ You cannot use both the additive_build_content and additive_build_content_file a exposed_packages = {} extra_aliases = {} whl_libraries = {} + hubs = {} for hub in pip_hub_map.values(): out = hub.build() @@ -358,8 +368,30 @@ You cannot use both the additive_build_content and additive_build_content_file a extra_aliases[hub.name] = out.extra_aliases hub_group_map[hub.name] = out.group_map hub_whl_map[hub.name] = out.whl_map + hubs[hub.name] = struct( + module_name = hub.module_name, + pyproject = hub_pyproject.get(hub.name), + packages = out.whl_map.keys(), + ) + + exposure = hub_exposure( + module_ctx, + hubs = hubs, + default_pyproject = config.pyproject_toml, + _fail = _fail, + ) + if exposure == None: + return None + configured_hubs = exposure.packages + exposed_packages.update(configured_hubs) + + # The extras requested in the `pyproject.toml`, e.g. `requests[socks]`, + # are exposed by the hub even if the lock file does not have them. + for hub_name, extras in exposure.extras.items(): + hub_whl_map[hub_name] = add_extras(hub_whl_map[hub_name], extras) return struct( + configured_hubs = configured_hubs, config = config, declared_deps = declared_deps, default_hub = config.default_hub or renamed_default_hub, @@ -392,7 +424,9 @@ def _create_unified_hub_repo(mods): extra_aliases = {} for hub_name in hubs: - for pkg_name in mods.hub_whl_map[hub_name]: + # The hubs configured with `pip.configure` only expose some packages. + hub_packages = mods.configured_hubs.get(hub_name, mods.hub_whl_map[hub_name]) + for pkg_name in hub_packages: norm_pkg = normalize_name(pkg_name) if norm_pkg not in packages: packages[norm_pkg] = [] @@ -514,6 +548,8 @@ def _pip_impl(module_ctx): packages = mods.exposed_packages.get(hub_name, []), platform_config_settings = mods.platform_config_settings.get(hub_name, {}), groups = mods.hub_group_map.get(hub_name), + # The hubs configured with `pip.configure` hide the other packages. + hide_unexposed_packages = hub_name in mods.configured_hubs, ) _create_unified_hub_repo(mods) @@ -656,6 +692,10 @@ This is designed to work with dependency management tools like Renovate. :::{versionadded} 2.3.0 ::: + +:::{versionchanged} VERSION_NEXT_FEATURE +Also used by {obj}`pip.configure` for the hubs of the root module. +::: """, ), "unified_hub_requirements_bzl": attr.bool( @@ -875,6 +915,10 @@ The version must be specified as `==X.Y.Z` (exact version with full semver). :::{versionadded} 2.3.0 ::: + +:::{versionchanged} VERSION_NEXT_FEATURE +Also used by {obj}`pip.configure` to select the packages that the hub exposes. +::: """, ), "python_version": attr.string( @@ -1074,6 +1118,77 @@ execution-phase error when built. """, ) +_configure_tag = tag_class( + attrs = { + "dependency_groups": attr.string_list( + default = ["*"], + doc = """\ +The `pyproject.toml` dependency groups whose packages the hub exposes. + +Each item is one of: + +* `"default"`: the `[project] dependencies`. +* `"optional:"`: the `[project.optional-dependencies]` entry ``. +* `"group:"`: the PEP 735 `[dependency-groups]` entry ``. +* `"optional:*"` or `"group:*"`: all entries of the table. +* `"*"`: all of the above. +* `""`: the dependency group `` if it exists, otherwise the + optional dependency group ``. +* `"transitive"`: only supported together with `"*"`, exposes all of the + packages of the hub including the transitive dependencies, i.e. all of the + packages from the requirements files or the `uv.lock` file. + +The extras of the selected requirements, e.g. `requests[socks]`, are also +exposed by the hub, see [Configuring the Exposed Packages](#pip-configure). +With `"transitive"` the extras of all of the groups are exposed. + +See [Configuring the Exposed Packages](#pip-configure) for examples. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + "hub_name": attr.string( + mandatory = True, + doc = """\ +The name of the hub to configure. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + "pyproject_toml": attr.label( + mandatory = False, + doc = """\ +The `pyproject.toml` file that defines the dependency groups. Defaults to the +{attr}`pip.parse.pyproject_toml` of the hub, then to +{attr}`pip.default.pyproject_toml` if the hub is defined by the root module. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", + ), + }, + doc = """\ +Configure the packages exposed by a hub. + +The packages are selected from a `pyproject.toml` file using the +`[project] dependencies`, `[project.optional-dependencies]` and PEP 735 +`[dependency-groups]` tables. The other packages of the hub, e.g. the +transitive dependencies, are still used by the packages that need them, but +they are not public. The extras requested by the selected requirements, e.g. +`requests[socks]`, are exposed even if the lock file does not request them. + +Only the root module and the module that created the hub can configure it. If +both do, the root module wins. Multiple tags for the same hub are combined. + +See [Configuring the Exposed Packages](#pip-configure) for examples. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""", +) + pypi = module_extension( environ = ["RULES_PYTHON_PYPI_HUB_RESERVED"], doc = """\ @@ -1097,12 +1212,17 @@ Environment Variables: - `RULES_PYTHON_PYPI_HUB_RESERVED`: Enable fallback renaming for reserved hub name collisions. See the {envvar}`RULES_PYTHON_PYPI_HUB_RESERVED` documentation for details. +pip.configure: +This tag class configures the packages exposed by a hub using the dependency +groups from a pyproject.toml file. + pip.whl_mods: This tag class is used to help create JSON files to describe modifications to the BUILD files for wheels. """, implementation = _pip_impl, tag_classes = { + "configure": _configure_tag, "default": tag_class( attrs = _default_attrs, doc = """\ diff --git a/python/private/pypi/generate_group_library_build_bazel.bzl b/python/private/pypi/generate_group_library_build_bazel.bzl index 571cfd6b3f..92cf8f9870 100644 --- a/python/private/pypi/generate_group_library_build_bazel.bzl +++ b/python/private/pypi/generate_group_library_build_bazel.bzl @@ -22,6 +22,8 @@ load( "PY_LIBRARY_PUBLIC_LABEL", "WHEEL_FILE_IMPL_LABEL", "WHEEL_FILE_PUBLIC_LABEL", + "extra_label", + "group_member", ) _PRELUDE = """\ @@ -60,32 +62,36 @@ def _generate_group_libraries(repo_prefix, group_name, group_members): repo_prefix: str; the pip_parse repo prefix. group_name: str; the name which the user provided for the dep group. group_members: list[str]; the names of the _packages_ (not repositories) - which make up the group. + which make up the group, optionally with a single extra, e.g. `foo[bar]`. """ - group_members = sorted(group_members) + # The whl targets do not use the extras, so only the packages without + # extras are members for the whl targets. + group_members = sorted([group_member(d) for d in group_members]) + whl_members = [d for d in group_members if "[" not in d] + packages = sorted({_package(d): None for d in group_members}) if repo_prefix: lib_dependencies = [ - "@%s%s//:%s" % (repo_prefix, normalize_name(d), PY_LIBRARY_IMPL_LABEL) + "@%s%s//:%s" % (repo_prefix, _package(d), _lib_impl_label(d)) for d in group_members ] whl_file_deps = [ "@%s%s//:%s" % (repo_prefix, normalize_name(d), WHEEL_FILE_IMPL_LABEL) - for d in group_members + for d in whl_members ] visibility = [ "@%s%s//:__pkg__" % (repo_prefix, normalize_name(d)) - for d in group_members + for d in packages ] else: lib_dependencies = [ - "//%s:%s" % (normalize_name(d), PY_LIBRARY_IMPL_LABEL) + "//%s:%s" % (_package(d), _lib_impl_label(d)) for d in group_members ] whl_file_deps = [ "//%s:%s" % (normalize_name(d), WHEEL_FILE_IMPL_LABEL) - for d in group_members + for d in whl_members ] visibility = ["//:__subpackages__"] @@ -98,6 +104,13 @@ def _generate_group_libraries(repo_prefix, group_name, group_members): visibility = render.indent(render.list(visibility)).lstrip(), ) +def _package(member): + return member.partition("[")[0] + +def _lib_impl_label(member): + _, _, extra = member.partition("[") + return "_" + extra_label(extra.rstrip("]")) if extra else PY_LIBRARY_IMPL_LABEL + def generate_group_library_build_bazel( repo_prefix, groups): diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index bec45bb08f..bb8faa71dc 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -22,9 +22,9 @@ _RENDER = { "data": render.list, "data_exclude": render.list, "entry_points": render.dict_dict, - "extras": render.list, "group_deps": render.list, "include": str, + "provides_extra": render.list, "repo": lambda maybe_label: repr(str(maybe_label)), "requires_dist": render.list, "srcs_exclude": render.list, diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index df0c48794e..05def3daf4 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -9,9 +9,11 @@ load("//python/private:version.bzl", "version") load("//python/private:version_label.bzl", "version_label") load(":attrs.bzl", "use_isolated") load(":hash.bzl", "hash") +load(":labels.bzl", "group_member") load(":parse_requirements.bzl", "parse_requirements") load(":pep508_env.bzl", "env") load(":pep508_evaluate.bzl", "evaluate") +load(":pep508_requirement.bzl", "requirement", "strip_extras") load(":python_tag.bzl", "python_tag") load(":requirements_files_by_platform.bzl", "requirements_files_by_platform") load(":whl_config_setting.bzl", "whl_config_setting") @@ -257,8 +259,10 @@ def _add_group_map(self, group_map): # cycles for different abis/oses? For now we will need the users to # assume the same groups across all versions/platforms until we start # using an alternative cycle resolution strategy. + # The members are the packages, e.g. `foo`, or the extras of the packages, + # e.g. `foo[bar]`. group_map = { - name: [normalize_name(whl_name) for whl_name in whls] + name: [group_member(whl_name) for whl_name in whls] for name, whls in group_map.items() } self._group_map.clear() @@ -266,7 +270,7 @@ def _add_group_map(self, group_map): self._group_map.update(group_map) self._group_name_by_whl.update({ - whl_name: group_name + whl_name.partition("[")[0]: group_name for group_name, group_whls in self._group_map.items() for whl_name in group_whls }) @@ -610,7 +614,8 @@ def _create_whl_repos( whl = whl, whl_modifications = whl_modifications, ) - for src in whl.srcs: + srcs = _merge_extras(whl.srcs) + for src in srcs: repo = _whl_repo( src = src, index_url = whl.index_url, @@ -620,16 +625,96 @@ def _create_whl_repos( use_downloader = src.url and _use_downloader(self, python_version, whl.name), auth_patterns = self._config.auth_patterns or pip_attr.auth_patterns, python_version = _major_minor_version(python_version), - is_multiple_versions = whl.is_multiple_versions, + # Different extras for the same distribution are not different versions. + is_multiple_versions = whl.is_multiple_versions and len({ + s.requirement_line: None + for s in srcs + }) > 1, interpreter = interpreter, enable_pipstar_extract = enable_pipstar_extract, ) - _add_whl_library( - self, - python_version = python_version, - whl = whl, - repo = repo, - ) + for config_setting in repo.config_settings if repo else []: + _add_whl_library( + self, + python_version = python_version, + whl = whl, + repo = struct( + repo_name = repo.repo_name, + args = repo.args, + config_setting = config_setting, + ), + ) + +def _merge_extras(srcs): + """Merge the sources that differ only in the requested extras. + + The whl repositories create targets for all of the extras that a package + provides, so the extras are not passed to them and the same distribution + requested with different extras on different platforms uses a single whl + repository. The extras of each platform are recorded so that the hub can + expose only the requested extras for each platform. + + Args: + srcs: {type}`list[struct]` the sources returned by `parse_requirements`. + + Returns: + The list of sources without the extras in the requirement line and + with an extra `extras_by_platform` attribute. + """ + merged = {} + for src in srcs: + line = strip_extras(src.requirement_line) + entry = merged.setdefault( + (src.filename, src.url, line, tuple(src.extra_pip_args)), + struct( + src = src, + line = line, + extras_by_platform = {}, + target_platforms = [], + ), + ) + extras = sorted(requirement(src.requirement_line).extras) + for p in src.target_platforms: + if p not in entry.target_platforms: + entry.target_platforms.append(p) + entry.extras_by_platform[p] = extras + + return [ + struct( + distribution = entry.src.distribution, + extra_pip_args = entry.src.extra_pip_args, + requirement_line = entry.line, + target_platforms = entry.target_platforms, + extras_by_platform = entry.extras_by_platform, + filename = entry.src.filename, + digest = entry.src.digest, + url = entry.src.url, + yanked = entry.src.yanked, + ) + for entry in merged.values() + ] + +def _config_settings(*, src, version, target_platforms): + """Return the config settings for each set of extras requested by the platforms.""" + plats_by_extras = {} + for p in src.target_platforms: + plats_by_extras.setdefault(tuple(src.extras_by_platform.get(p, [])), []).append(p) + + if len(plats_by_extras) == 1: + return [whl_config_setting( + version = version, + target_platforms = target_platforms or None, + extras = plats_by_extras.keys()[0], + )] + + return [ + whl_config_setting( + version = version, + target_platforms = plats, + extras = extras, + ) + for extras, plats in plats_by_extras.items() + ] def _common_args(self, module_ctx, *, pip_attr): # Construct args separately so that the lock file can be smaller and does not include unused @@ -699,8 +784,11 @@ def _whl_repo( interpreter, enable_pipstar_extract = False): args = dict(whl_library_args) - args["requirement"] = src.requirement_line is_whl = src.filename.endswith(".whl") + if not (src.url and is_whl): + # The requirement line is only needed for `pip` to download or build + # the wheel. + args["requirement"] = src.requirement_line if src.extra_pip_args and not is_whl: # pip is not used to download wheels and the python @@ -729,9 +817,10 @@ def _whl_repo( *target_platforms ), args = args, - config_setting = whl_config_setting( + config_settings = _config_settings( + src = src, version = python_version, - target_platforms = target_platforms or None, + target_platforms = target_platforms, ), ) @@ -762,7 +851,8 @@ def _whl_repo( return struct( repo_name = whl_repo_name(src.filename, src.digest, *target_platforms), args = args, - config_setting = whl_config_setting( + config_settings = _config_settings( + src = src, version = python_version, target_platforms = src.target_platforms, ), diff --git a/python/private/pypi/hub_exposure.bzl b/python/private/pypi/hub_exposure.bzl new file mode 100644 index 0000000000..e2e441ccde --- /dev/null +++ b/python/private/pypi/hub_exposure.bzl @@ -0,0 +1,140 @@ +"""Compute the packages and extras exposed by the hubs configured with `pip.configure`.""" + +load("//python/private:pyproject_utils.bzl", "read_pyproject") +load(":dependency_groups.bzl", "select_dependency_groups") +load(":whl_config_setting.bzl", "whl_config_setting") + +def hub_exposure(module_ctx, *, hubs, default_pyproject = None, _fail = fail): + """Compute the packages exposed by the hubs configured with `pip.configure`. + + Args: + module_ctx: {type}`module_ctx` the module context. + hubs: {type}`dict[str, struct]` the hubs by name. Each struct has: + * `module_name`: {type}`str` the module that created the hub. + * `pyproject`: {type}`Label | None` from `pip.parse`. + * `packages`: {type}`list[str]` the normalized names of all + packages in the hub. + default_pyproject: {type}`Label | None` from `pip.default`, only used + for the hubs of the root module. + _fail: {type}`callable` the failure function, mainly for testing. + + Returns: + {type}`struct | None` with the attributes below or `None` on failure: + * `packages`: {type}`dict[str, list[str]]` the configured hubs + mapped to the sorted packages that they expose. The hubs that are + not configured or that expose all of their packages are not + included, so that they behave the same. + * `extras`: {type}`dict[str, dict[str, list[str]]]` the configured + hubs mapped to the packages and the sorted extras that the + selected dependency groups request for them, e.g. + `{"hub": {"requests": ["socks"]}}`. + """ + + # The root module wins over the owner of the hub when both configure it. + root_modules = [mod for mod in module_ctx.modules if mod.is_root] + root_name = root_modules[0].name if root_modules else None + root_configured = {} + for mod in root_modules: + for tag in mod.tags.configure: + root_configured[tag.hub_name] = None + + configured = {} + exposes_all = {} + extras = {} + for mod in module_ctx.modules: + for tag in mod.tags.configure: + hub = tag.hub_name + if hub not in hubs: + _fail("pip.configure() refers to unknown hub '{}'. Available hubs: {}".format( + hub, + ", ".join(sorted(hubs)), + )) + return None + + info = hubs[hub] + if not mod.is_root and (mod.name != info.module_name or hub in root_configured): + # Only the root module and the module that created the hub can + # configure it. + continue + + pyproject = ( + tag.pyproject_toml or + info.pyproject or + (default_pyproject if info.module_name == root_name else None) + ) + if not pyproject: + _fail("pip.configure() for hub '{}' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`".format(hub)) + return None + + selection = select_dependency_groups( + read_pyproject(module_ctx, pyproject), + tag.dependency_groups, + _fail = _fail, + ) + if selection.all: + exposes_all[hub] = None + + # Multiple tags for the same hub are combined. + exposed = configured.setdefault(hub, {}) + for name in selection.packages: + # The pyproject.toml may list packages that are not in the hub, + # e.g. tools that are not locked. + if name in info.packages: + exposed[name] = None + + hub_extras = extras.setdefault(hub, {}) + for name, pkg_extras in selection.extras.items(): + if name in info.packages: + for extra in pkg_extras: + hub_extras.setdefault(name, {})[extra] = None + + return struct( + packages = { + hub: sorted(packages) + for hub, packages in configured.items() + if hub not in exposes_all + }, + extras = { + hub: {name: sorted(pkg_extras) for name, pkg_extras in hub_extras.items()} + for hub, hub_extras in extras.items() + if hub_extras + }, + ) + +def add_extras(whl_map, extras): + """Add the extras requested by `pip.configure` to the hub `whl_map`. + + The extras are added to every config setting of the package, so the hub + exposes the `[]` targets and the package target includes them on + all of the platforms. + + Args: + whl_map: {type}`dict[str, dict[str, list[struct]]]` the packages + mapped to the repo names and their {obj}`whl_config_setting` + values. + extras: {type}`dict[str, list[str]]` the packages mapped to the + normalized extras to add. + + Returns: + {type}`dict[str, dict[str, list[struct]]]` the new `whl_map`. + """ + if not extras: + return whl_map + + return { + name: { + repo: [ + whl_config_setting( + version = setting.version, + target_platforms = setting.target_platforms, + extras = { + extra: None + for extra in list(setting.extras or []) + extras[name] + }.keys(), + ) + for setting in settings + ] + for repo, settings in repos.items() + } if name in extras else repos + for name, repos in whl_map.items() + } diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index f915aa1c77..c7895b5e96 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -15,7 +15,7 @@ "" load("//python/private:text_util.bzl", "render") -load(":render_pkg_aliases.bzl", "render_multiplatform_pkg_aliases") +load(":render_pkg_aliases.bzl", "available_extras", "render_multiplatform_pkg_aliases") load(":whl_config_setting.bzl", "whl_config_setting") _BUILD_FILE_CONTENTS = """\ @@ -26,12 +26,19 @@ exports_files(["requirements.bzl"]) """ def _impl(rctx): - bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() + if rctx.attr.hide_unexposed_packages: + bzl_packages = rctx.attr.packages + exposed_packages = bzl_packages + else: + bzl_packages = rctx.attr.packages or rctx.attr.whl_map.keys() + exposed_packages = None + whl_map = { + key: _whl_config_settings_from_json(values) + for key, values in rctx.attr.whl_map.items() + } aliases = render_multiplatform_pkg_aliases( - aliases = { - key: _whl_config_settings_from_json(values) - for key, values in rctx.attr.whl_map.items() - }, + aliases = whl_map, + exposed_packages = exposed_packages, extra_hub_aliases = rctx.attr.extra_hub_aliases, requirement_cycles = rctx.attr.groups, platform_config_settings = rctx.attr.platform_config_settings, @@ -50,7 +57,9 @@ def _impl(rctx): "config.bzl", rctx.attr._config_template, substitutions = { - "%%PACKAGES%%": render.dict(rctx.attr.whl_map, value_repr = lambda x: "None"), + # The available extras of each package for the spoke repositories + # to check if the extras of their dependencies are available. + "%%PACKAGES%%": render.dict(available_extras(whl_map), value_repr = render.list), }, ) rctx.template("requirements.bzl", rctx.attr._requirements_bzl_template, substitutions = { @@ -78,10 +87,15 @@ hub_repository = repository_rule( "groups": attr.string_list_dict( mandatory = False, ), + "hide_unexposed_packages": attr.bool( + default = False, + doc = "Make the aliases of the packages that are not in `packages` visible only to the wheel repositories.", + ), "packages": attr.string_list( mandatory = False, doc = """\ -The list of packages that will be exposed via all_*requirements macros. Defaults to whl_map keys. +The list of packages that will be exposed via all_*requirements macros. Defaults to whl_map keys +unless `hide_unexposed_packages` is set. """, ), "platform_config_settings": attr.string_list_dict( @@ -142,6 +156,8 @@ def whl_config_settings_to_json(repo_mapping): def _whl_config_setting_dict(a): ret = {} + if a.extras: + ret["extras"] = a.extras if a.target_platforms: ret["target_platforms"] = a.target_platforms if a.version: diff --git a/python/private/pypi/labels.bzl b/python/private/pypi/labels.bzl index c77167b593..3414d586a7 100644 --- a/python/private/pypi/labels.bzl +++ b/python/private/pypi/labels.bzl @@ -14,6 +14,8 @@ """Constants used by parts of pip_repository for naming libraries and wheels.""" +load("//python/private:normalize_name.bzl", "normalize_name") + EXTRACTED_WHEEL_FILES = "extracted_whl_files" WHEEL_FILE_PUBLIC_LABEL = "whl" WHEEL_FILE_IMPL_LABEL = "_whl" @@ -24,3 +26,85 @@ PY_SRCS_LABEL = "srcs" DATA_LABEL = "data" DIST_INFO_LABEL = "dist_info" NODEPS_LABEL = "no_deps" + +def extra_label(extra): + """Return the label name for the target that includes an extra. + + Args: + extra: {type}`str` the normalized name of the extra, an empty string + means the base package without any extras. + + Returns: + The target name, e.g. `[extra]`. + """ + return "[{}]".format(extra) + +def _labels(tmpl, requirement): + name, bracket, extras = requirement.partition("[") + name = normalize_name(name.strip()) + if not bracket: + return [tmpl.format(name, PY_LIBRARY_PUBLIC_LABEL)] + + extras, _, _ = extras.partition("]") + extras = sorted([ + normalize_name(extra.strip()) + for extra in extras.split(",") + if extra.strip() + ]) + return [tmpl.format(name, extra_label(extra)) for extra in extras or [""]] + +def requirement_label(tmpl, requirement, *, fail = fail): + """Return the label for a requirement in a hub repository. + + Args: + tmpl: {type}`str` the label template with two `{}` placeholders for + the package name and the target name. + requirement: {type}`str` the name of the package with at most one + extra, e.g. `foo`, `foo[]` or `foo[bar]`. + fail: the function to call on errors, used in tests. + + Returns: + {type}`str` the label of the `pkg` target if no extras are given, the + label of the `[]` target if empty extras are given, or the label of + the `[extra]` target if one extra is given. + """ + labels = _labels(tmpl, requirement) + if len(labels) > 1: + fail("Only a single extra is supported by 'requirement', use 'requirements(\"{}\")' instead to get a list of labels.".format(requirement)) + + return labels[0] + +def requirement_labels(tmpl, *requirements): + """Return the labels for requirements in a hub repository. + + Args: + tmpl: {type}`str` the label template with two `{}` placeholders for + the package name and the target name. + *requirements: {type}`str` the names of the packages with optional + extras, e.g. `foo`, `foo[]`, `foo[bar]` or `foo[bar,baz]`. + + Returns: + {type}`list[str]` the labels for each requirement, with an `[extra]` + label for each extra of a requirement. Each `[extra]` target includes + the package. + """ + return [ + label + for requirement in requirements + for label in _labels(tmpl, requirement) + ] + +def group_member(spec): + """Normalize a member of a dependency group. + + Args: + spec: {type}`str` the package name, e.g. `Foo`, or the package name + with a single extra, e.g. `Foo[Bar]`. + + Returns: + The normalized member, e.g. `foo` or `foo[bar]`. + """ + name, _, extra = spec.partition("[") + name = normalize_name(name.strip()) + extra = normalize_name(extra.partition("]")[0].strip()) if extra else "" + return "{}[{}]".format(name, extra) if extra else name diff --git a/python/private/pypi/missing_package.bzl b/python/private/pypi/missing_package.bzl index 46823e813b..e5647d5786 100644 --- a/python/private/pypi/missing_package.bzl +++ b/python/private/pypi/missing_package.bzl @@ -1,25 +1,16 @@ -"""Rule for generating an execution-phase action failure when a PyPI package is missing.""" +"""Rules for generating an execution-phase action failure when a PyPI package or its extra is missing.""" load("//python/private:py_info.bzl", "PyInfo") load("//python/private:reexports.bzl", "BuiltinPyInfo") -def _missing_package_error_impl(ctx): +def _error(ctx, message): out = ctx.actions.declare_file(ctx.label.name + ".error") - if ctx.attr.hub_name: - hub_clause = ' when building under PyPI hub "{hub}". Try adding it to the requirements of this hub (e.g. requirements_lock or requirements_by_platform in pip.parse)'.format( - hub = ctx.attr.hub_name, - ) - else: - hub_clause = ' because no default PyPI hub was configured. Try designating a default hub via pip.default(default_hub = "...") or select a hub using --@rules_python//python/config_settings:venv' - # Register an action that fails when Bazel attempts to stage/build this file ctx.actions.run_shell( outputs = [out], - command = "echo 'ERROR: PyPI package \"{pkg}\" is not available{hub_clause}.' >&2 && exit 1".format( - pkg = ctx.attr.package_name, - hub_clause = hub_clause, - ), + command = "echo \"$1\" >&2 && exit 1", + arguments = ["ERROR: " + message], ) maybe_builtin = [BuiltinPyInfo(transitive_sources = depset([out]))] if BuiltinPyInfo != None else [] @@ -34,6 +25,19 @@ def _missing_package_error_impl(ctx): ), ] + maybe_builtin +def _missing_package_error_impl(ctx): + if ctx.attr.hub_name: + hub_clause = ' when building under PyPI hub "{hub}". Try adding it to the requirements of this hub (e.g. requirements_lock or requirements_by_platform in pip.parse)'.format( + hub = ctx.attr.hub_name, + ) + else: + hub_clause = ' because no default PyPI hub was configured. Try designating a default hub via pip.default(default_hub = "...") or select a hub using --@rules_python//python/config_settings:venv' + + return _error(ctx, 'PyPI package "{pkg}" is not available{hub_clause}.'.format( + pkg = ctx.attr.package_name, + hub_clause = hub_clause, + )) + missing_package_error = rule( implementation = _missing_package_error_impl, attrs = { @@ -41,3 +45,25 @@ missing_package_error = rule( "package_name": attr.string(mandatory = True), }, ) + +def _missing_extra_error_impl(ctx): + return _error(ctx, ( + 'The PyPI package "{pkg}" requires "{dep}[{extra}]", but the extra "{extra}" of ' + + '"{dep}" is not available in the hub. Add "{dep}[{extra}]" to the requirements or ' + + "the lock file of the hub, e.g. by compiling the requirements with the extras kept " + + "(`--no-strip-extras`) or by adding the extra to the project dependencies." + ).format( + pkg = ctx.attr.required_by, + dep = ctx.attr.package_name, + extra = ctx.attr.extra, + )) + +missing_extra_error = rule( + implementation = _missing_extra_error_impl, + doc = "A target that fails the build with a helpful message when an extra of a PyPI package is missing.", + attrs = { + "extra": attr.string(mandatory = True), + "package_name": attr.string(mandatory = True), + "required_by": attr.string(mandatory = True), + }, +) diff --git a/python/private/pypi/patch_and_extract_whl.bzl b/python/private/pypi/patch_and_extract_whl.bzl index 37c90e3221..cfacb4adf4 100644 --- a/python/private/pypi/patch_and_extract_whl.bzl +++ b/python/private/pypi/patch_and_extract_whl.bzl @@ -4,7 +4,6 @@ load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:repo_utils.bzl", "repo_utils") load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") load(":patch_whl.bzl", "get_whl_patches", "patch_whl") -load(":pep508_requirement.bzl", "requirement") load(":pypi_repo_utils.bzl", "pypi_repo_utils") load(":whl_extract.bzl", "whl_extract") load(":whl_metadata.bzl", "parse_entry_points", "whl_metadata") @@ -135,7 +134,7 @@ def patch_and_extract_whl(rctx, *, whl_path, logger, sdist_filename = None, whl_ group_deps = rctx.attr.group_deps, group_name = rctx.attr.group_name, namespace_package_files = namespace_package_files, - extras = requirement(rctx.attr.requirement).extras, + provides_extra = metadata.provides_extra, entry_points = entry_points, purl = _to_purl( index = rctx.attr.index_url, diff --git a/python/private/pypi/pep508_deps.bzl b/python/private/pypi/pep508_deps.bzl index e36ceadbd5..da42fbe5e9 100644 --- a/python/private/pypi/pep508_deps.bzl +++ b/python/private/pypi/pep508_deps.bzl @@ -26,23 +26,32 @@ def deps( requires_dist, extras = [], excludes = [], - include = []): + include = [], + keep_extras = False): """Parse the RequiresDist from wheel METADATA Args: name: {type}`str` the name of the wheel. requires_dist: {type}`list[str]` the list of RequiresDist lines from the METADATA file. - excludes: {type}`list[str]` what packages should we exclude. + excludes: {type}`list[str]` what dependencies should we exclude, e.g. + `foo` or `foo[bar]`, in the same format as the returned values. include: {type}`list[str]` what packages should we exclude. If it is not specified, then we will include all deps from `requires_dist`. extras: {type}`list[str]` the requested extras to generate targets for. + keep_extras: {type}`bool` keep the extras of the dependencies, if + `True`, the dependencies with extras are returned as `foo[bar]`. Returns: A struct with attributes: * deps: {type}`list[str]` dependencies to include unconditionally. * deps_select: {type}`dict[str, list[str]]` dependencies to include on particular subset of target platforms. + + The dependencies are normalized names, e.g. `foo`, or normalized names + with a single extra, e.g. `foo[bar]`, if the dependency is requested + with extras and `keep_extras` is set. A dependency with multiple + extras, e.g. `foo[bar,baz]`, is returned as `foo[bar]` and `foo[baz]`. """ if not requires_dist: return struct( @@ -60,25 +69,25 @@ def deps( want_extras = _resolve_extras(name, reqs, extras) include = [normalize_name(n) for n in include] - # drop self edges - excludes = [name] + [normalize_name(x) for x in excludes] - - reqs_by_name = {} + reqs_by_dep = {} for req in reqs: - if req.name_ in excludes: + # drop self edges + if req.name_ == name: continue if include and req.name_ not in include: continue - reqs_by_name.setdefault(req.name, []).append(req) + for dep in ["{}[{}]".format(req.name_, e) for e in req.extras if keep_extras] or [req.name_]: + if dep not in excludes: + reqs_by_dep.setdefault(dep, []).append(req) - for name, reqs in reqs_by_name.items(): + for dep, reqs in reqs_by_dep.items(): _add_reqs( deps, deps_select, - normalize_name(name), + dep, reqs, extras = want_extras, ) @@ -92,8 +101,6 @@ def deps( ) def _add(deps, deps_select, dep, markers = None): - dep = normalize_name(dep) - if not markers: deps[dep] = True elif len(markers) == 1: diff --git a/python/private/pypi/pep508_requirement.bzl b/python/private/pypi/pep508_requirement.bzl index 7552642572..8aaaa29e37 100644 --- a/python/private/pypi/pep508_requirement.bzl +++ b/python/private/pypi/pep508_requirement.bzl @@ -56,3 +56,22 @@ def requirement(spec): extras = extras, version = version, ) + +def strip_extras(requirement_line): + """Remove the extras from the requirement line. + + Args: + requirement_line: {type}`str` the requirement line, e.g. + `foo[bar]==1.0 --hash=sha256:...`. + + Returns: + The requirement line without the extras, e.g. `foo==1.0 --hash=sha256:...`. + """ + for i, c in enumerate(requirement_line.elems()): + if c in "=<>!~;@( ": + break + if c == "[": + _, _, tail = requirement_line[i:].partition("]") + return requirement_line[:i] + tail + + return requirement_line diff --git a/python/private/pypi/pip_archive.bzl b/python/private/pypi/pip_archive.bzl index 1764abe70d..9a641bd142 100644 --- a/python/private/pypi/pip_archive.bzl +++ b/python/private/pypi/pip_archive.bzl @@ -358,6 +358,11 @@ pip_archive_attrs = { "quiet", "timeout", ] +} | { + "requirement": attr.string( + mandatory = True, + doc = "Python requirement string describing the package to download or build with `pip`. The extras are not needed because the targets for all of the extras are created.", + ), } pip_archive = repository_rule( diff --git a/python/private/pypi/pip_repository.bzl b/python/private/pypi/pip_repository.bzl index d02c62eae1..67bbe1e945 100644 --- a/python/private/pypi/pip_repository.bzl +++ b/python/private/pypi/pip_repository.bzl @@ -22,8 +22,10 @@ load("//python/private:text_util.bzl", "render") load("//python/private:version.bzl", "version") load(":default_platforms.bzl", "default_platforms") load(":hash.bzl", "hash") +load(":labels.bzl", "group_member") load(":parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement") load(":pep508_env.bzl", "env") +load(":pep508_requirement.bzl", "strip_extras", parse_requirement = "requirement") load(":pip_repository_attrs.bzl", "ATTRS") load(":pypi_repo_utils.bzl", "pypi_repo_utils") load(":render_pkg_aliases.bzl", "render_pkg_aliases") @@ -150,6 +152,7 @@ def _pip_repository_impl(rctx): logger = logger, ) selected_requirements = {} + selected_extras = {} selected_srcs = {} options = None repository_platform = host_platform(rctx) @@ -161,7 +164,11 @@ def _pip_repository_impl(rctx): if not requirement: continue options = options or requirement.extra_pip_args - selected_requirements[whl.name] = requirement.requirement_line + + # The whl repositories have targets for all of the extras that a + # package provides, so the extras are exposed only via the hub. + selected_requirements[whl.name] = strip_extras(requirement.requirement_line) + selected_extras[whl.name] = parse_requirement(requirement.requirement_line).extras if not requirement.url: continue @@ -183,7 +190,7 @@ def _pip_repository_impl(rctx): # ensure that only the locked distributions can be used. digests = sorted({src.digest: None for src in whl.srcs}) if all(digests): - selected_requirements[whl.name] = " ".join([requirement.requirement_line] + [ + selected_requirements[whl.name] = " ".join([selected_requirements[whl.name]] + [ "--hash={}".format(d) for d in digests ]) @@ -216,7 +223,11 @@ def _pip_repository_impl(rctx): # label referencing a package which may not be installed on the current # platform. requirement_cycles = { - normalize_name(name): sorted([normalize_name(d) for d in group if normalize_name(d) in bzl_packages]) + normalize_name(name): sorted([ + group_member(d) + for d in group + if group_member(d).partition("[")[0] in bzl_packages + ]) for name, group in requirement_cycles.items() } @@ -224,6 +235,7 @@ def _pip_repository_impl(rctx): # NOTE: Maintain the order consistent with `buildifier` 'load("@rules_python//python:pip.bzl", "pip_utils")', 'load("@rules_python//python/pip_install:pip_repository.bzl", "whl_config_repo", "whl_library")', + 'load("@rules_python//python/private/pypi:labels.bzl", "requirement_label", "requirement_labels") # buildifier: disable=bzl-visibility', ] annotations = {} @@ -261,6 +273,7 @@ def _pip_repository_impl(rctx): for pkg in bzl_packages }, extra_hub_aliases = rctx.attr.extra_hub_aliases, + extras = selected_extras, requirement_cycles = requirement_cycles, ) for path, contents in aliases.items(): @@ -272,10 +285,13 @@ def _pip_repository_impl(rctx): "config.bzl", rctx.attr._config_template, substitutions = { + # The available extras of each package for the spoke + # repositories to check if the extras of their dependencies + # are available. "%%PACKAGES%%": render.dict({ - pkg: None + pkg: sorted(selected_extras.get(pkg, [])) for pkg in bzl_packages - }, value_repr = lambda x: "None"), + }, value_repr = render.list), }, ) config_repo_template = repr(rctx.attr.name) diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index 227e9bf10f..1920e2b971 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -32,6 +32,7 @@ setting maps to and their precedence, refer to documentation on that page. """ load("@bazel_skylib//lib:selects.bzl", "selects") +load("//python:py_library.bzl", "py_library") load("//python/private:common_labels.bzl", "labels") load("//python/private:text_util.bzl", "render") load( @@ -39,12 +40,12 @@ load( "DATA_LABEL", "DIST_INFO_LABEL", "EXTRACTED_WHEEL_FILES", - "PY_LIBRARY_IMPL_LABEL", "PY_LIBRARY_PUBLIC_LABEL", "PY_SRCS_LABEL", "WHEEL_FILE", "WHEEL_FILE_IMPL_LABEL", "WHEEL_FILE_PUBLIC_LABEL", + "extra_label", ) load(":whl_library_deps_targets.bzl", "whl_library_deps_targets") @@ -75,12 +76,14 @@ _LABEL_NONE = labels.NONE _LABEL_CURRENT_CONFIG = Label("//python/config_settings:current_config") _LABEL_CURRENT_CONFIG_NO_MATCH = Label("//python/config_settings:is_not_matching_current_config") _INCOMPATIBLE = "_no_matching_repository" +_DEFAULT = "//conditions:default" def pkg_aliases( *, name, actual, group_name = None, + group_extras = None, extra_aliases = None, requires_dist = [], extras = [], @@ -98,19 +101,30 @@ def pkg_aliases( the aliases to point to mapping to repositories. The keys are passed to bazel skylib's `selects.with_or`, so they can be tuples as well. group_name: {type}`str` The group name that the pkg belongs to. + group_extras: {type}`list[str] | None` The members of the group in + this package, where an empty string is the package without extras + and the other values are the extras, e.g. `["", "bar"]`. Defaults + to the package without extras if `group_name` is set. group_deps: {type}`list[str]` The packages that are in the given group. Comes as an arg through the hub repository. extra_aliases: {type}`list[str]` The extra aliases to be created. requires_dist: {type}`list[str]` The list of dependencies. Comes from METADATA or a lock file. - extras: {type}`list[str]` The extras for which we should add extra - dependencies when parsing the requires_dist. Comes from METADATA or a lock file. + extras: {type}`list[str]` The requested extras of the package when + `actual` is the name of a single repo. When `actual` is a dict, the + requested extras come from the {obj}`whl_config_setting` keys. include: {type}`list[str]` The subset of packages to include. Comes from `//:config.bzl#packages` **kwargs: extra kwargs to pass to {bzl:obj}`get_config_settings`. """ + if not group_name: + group_extras = [] + elif group_extras == None: + group_extras = [""] + _native = kwargs.pop("native", native) rules = kwargs.pop("rules", struct( + py_library = py_library, whl_library_deps_targets = whl_library_deps_targets, )) select = kwargs.pop("select", selects.with_or) @@ -129,7 +143,6 @@ def pkg_aliases( aliases = {}, # not none metadata_name = name, requires_dist = requires_dist, - extras = extras, include = include, group_deps = group_deps, group_name = group_name, @@ -140,16 +153,11 @@ def pkg_aliases( ) target_names = {} else: - if group_name: - py_library_target = PY_LIBRARY_IMPL_LABEL - whl_target = WHEEL_FILE_IMPL_LABEL - else: - py_library_target = PY_LIBRARY_PUBLIC_LABEL - whl_target = WHEEL_FILE_PUBLIC_LABEL - + # The `pkg` targets are created by `_pkg_targets` below. The whl + # targets do not use the extras, so only the package without extras + # can be a member of the group. target_names = { - PY_LIBRARY_PUBLIC_LABEL: py_library_target, - WHEEL_FILE_PUBLIC_LABEL: whl_target, + WHEEL_FILE_PUBLIC_LABEL: WHEEL_FILE_IMPL_LABEL if "" in group_extras else WHEEL_FILE_PUBLIC_LABEL, } target_names = target_names | { @@ -163,6 +171,7 @@ def pkg_aliases( for x in extra_aliases or [] } + extras_by_setting = _extras_by_setting(actual, extras) actual = multiplatform_whl_aliases(aliases = actual, **kwargs) if type(actual) == type({}) and "//conditions:default" not in actual: _native.alias( @@ -216,16 +225,129 @@ def pkg_aliases( **kwargs ) - if group_name: - _native.alias( - name = PY_LIBRARY_PUBLIC_LABEL, - actual = "//_groups:{}_pkg".format(group_name), - ) + if "" in group_extras: _native.alias( name = WHEEL_FILE_PUBLIC_LABEL, actual = "//_groups:{}_whl".format(group_name), ) + if requires_dist: + if group_name: + _native.alias( + name = PY_LIBRARY_PUBLIC_LABEL, + actual = "//_groups:{}_pkg".format(group_name), + ) + else: + _pkg_targets( + actual = actual, + extras_by_setting = extras_by_setting, + group_name = group_name, + group_extras = group_extras, + native = _native, + rules = rules, + select = select, + ) + +def _extras_by_setting(aliases, extras): + """Return the requested extras for each config setting label.""" + if type(aliases) == type(""): + # A single repo is used for all of the platforms, e.g. in `WORKSPACE`. + return {_DEFAULT: tuple(sorted(extras))} + + return { + "//_config" + setting: alias.extras + for alias in aliases + if type(alias) == "struct" and alias.extras + for setting in get_config_settings( + target_platforms = alias.target_platforms, + python_version = alias.version, + ) + } + +def _pkg_targets(*, actual, extras_by_setting, group_name, group_extras, native, rules, select): + """Create the `pkg`, `[]` and `[]` targets. + + The whl repositories have an `[]` target for every extra that the + package provides and the hub exposes only the extras requested on each + platform. The `pkg` target includes the extras requested on each platform. + + Args: + actual: {type}`str | dict[str, str]` the repo name or the config + setting labels to repo names. + extras_by_setting: {type}`dict[str, tuple[str]]` the config setting + labels to the extras requested on that platform. + group_name: {type}`str | None` the group name that the pkg belongs to. + group_extras: {type}`list[str]` the members of the group in this + package, where an empty string is the package without extras. + native: the native struct. + rules: the rules struct. + select: the select function. + """ + if type(actual) == type(""): + actual = {_DEFAULT: actual} + + def _select(values): + return values[_DEFAULT] if values.keys() == [_DEFAULT] else select(values) + + def _target(repo, name): + return repo if repo == _INCOMPATIBLE else "@{}//:{}".format(repo, name) + + group_label = "//_groups:{}_{}".format(group_name, PY_LIBRARY_PUBLIC_LABEL) + + def _hub_label(extra): + return group_label if extra in group_extras else ":" + extra_label(extra) + + all_extras = sorted({ + extra: None + for extras in list(extras_by_setting.values()) + [group_extras] + for extra in extras + if extra + }) + for extra in [""] + all_extras: + # The platforms that do not request the extra get the package without + # extras. + targets = _select({ + setting: _target( + repo, + extra_label(extra) if extra in extras_by_setting.get(setting, ()) else PY_LIBRARY_PUBLIC_LABEL, + ) + for setting, repo in actual.items() + }) + + if extra not in group_extras: + native.alias(name = extra_label(extra), actual = targets) + continue + + # The group library uses the implementation targets of the members. + native.alias( + name = "_" + (extra_label(extra) if extra else PY_LIBRARY_PUBLIC_LABEL), + actual = targets, + visibility = ["//_groups:__subpackages__"], + ) + native.alias(name = extra_label(extra), actual = group_label) + + # The `[]` and `[]` targets have the no match error, so we do not + # need to handle the incompatible platforms here. + targets = { + setting: sorted({ + _hub_label(e): None + for e in extras_by_setting.get(setting, ()) or [""] + }) + for setting in actual + } + unique = {label: None for t in targets.values() for label in t} + if len(unique) == 1: + native.alias(name = PY_LIBRARY_PUBLIC_LABEL, actual = unique.keys()[0]) + elif all([len(t) == 1 for t in targets.values()]): + native.alias( + name = PY_LIBRARY_PUBLIC_LABEL, + actual = _select({setting: t[0] for setting, t in targets.items()}), + ) + else: + # Some platforms request more than one extra, so we need to combine + # the targets. + rules.py_library(name = PY_LIBRARY_PUBLIC_LABEL, deps = _select(targets)) + def multiplatform_whl_aliases( *, aliases = []): diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index 0a1c328491..4d1e47f40b 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -22,6 +22,7 @@ load( ":generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel", ) # buildifier: disable=bzl-visibility +load(":labels.bzl", "group_member") NO_MATCH_ERROR_MESSAGE_TEMPLATE = """\ No matching wheel for current configuration's Python version. @@ -46,10 +47,11 @@ def _repr_dict(*, value_repr = repr, **kwargs): return {k: value_repr(v) for k, v in kwargs.items() if v} def _repr_config_setting(alias): - if alias.target_platforms: + if alias.target_platforms or alias.extras: return render.call( "whl_config_setting", **_repr_dict( + extras = alias.extras, target_platforms = alias.target_platforms, version = alias.version, ) @@ -65,7 +67,7 @@ def _repr_actual(aliases): else: return render.dict(aliases, key_repr = _repr_config_setting) -def _render_common_aliases(*, name, aliases, **kwargs): +def _render_common_aliases(*, name, aliases, visibility, **kwargs): pkg_aliases = render.call( "pkg_aliases", name = repr(name), @@ -80,14 +82,22 @@ def _render_common_aliases(*, name, aliases, **kwargs): return """\ load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") {extra_loads} -package(default_visibility = ["//visibility:public"]) +package(default_visibility = {visibility}) {aliases}""".format( aliases = pkg_aliases, extra_loads = extra_loads, + visibility = render.list(visibility), ) -def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases = {}, **kwargs): +def render_pkg_aliases( + *, + aliases, + requirement_cycles = None, + extra_hub_aliases = {}, + extras = {}, + exposed_packages = None, + **kwargs): """Create alias declarations for each PyPI package. The aliases should be appended to the pip_repository BUILD.bazel file. These aliases @@ -100,6 +110,11 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases requirement_cycles: any package groups to also add. extra_hub_aliases: The list of extra aliases for each whl to be added in addition to the default ones. + extras: {type}`dict[str, list[str]]` The requested extras for each + whl when the aliases are repo names. + exposed_packages: The packages that should be public. When set, the + aliases of the other packages are only visible to the wheel + repositories. **kwargs: Extra kwargs to pass to the rules. Returns: @@ -112,33 +127,73 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases fail("The aliases need to be provided as a dict, got: {}".format(type(aliases))) whl_group_mapping = {} + group_extras = {} if requirement_cycles: + # The members are the packages, e.g. `foo`, or the extras of the + # packages, e.g. `foo[bar]`. requirement_cycles = { - name: [normalize_name(whl_name) for whl_name in whls] + name: [group_member(whl_name) for whl_name in whls] for name, whls in requirement_cycles.items() } - whl_group_mapping = { - whl_name: group_name - for group_name, group_whls in requirement_cycles.items() - for whl_name in group_whls - } - - files = { - "{}/BUILD.bazel".format(normalize_name(name)): _render_common_aliases( - name = normalize_name(name), + for group_name, members in requirement_cycles.items(): + for member in members: + whl_name, _, extra = member.partition("[") + whl_group_mapping[whl_name] = group_name + group_extras.setdefault(whl_name, []).append(extra.rstrip("]")) + + # The package without extras is the default member. + group_extras = {k: v for k, v in group_extras.items() if v != [""]} + + public = ["//visibility:public"] + if exposed_packages != None: + exposed_packages = {normalize_name(name): None for name in exposed_packages} + internal_visibility = _internal_visibility(aliases, has_groups = bool(requirement_cycles)) + + files = {} + for name, pkg_aliases in aliases.items(): + name = normalize_name(name) + visibility = public + if exposed_packages != None and name not in exposed_packages: + visibility = internal_visibility + files["{}/BUILD.bazel".format(name)] = _render_common_aliases( + name = name, aliases = pkg_aliases, - extra_aliases = extra_hub_aliases.get(normalize_name(name), []), - group_name = whl_group_mapping.get(normalize_name(name)), + extra_aliases = extra_hub_aliases.get(name, []), + extras = extras.get(name, []), + group_name = whl_group_mapping.get(name), + group_extras = group_extras.get(name), + visibility = visibility, **kwargs ).strip() - for name, pkg_aliases in aliases.items() - } if requirement_cycles: files["_groups/BUILD.bazel"] = generate_group_library_build_bazel("", requirement_cycles) return files +def _internal_visibility(aliases, has_groups = False): + """The visibility of the packages that are not exposed. + + The wheel repositories depend on the aliases of the hub, so the aliases + have to be visible to them. + """ + repo_names = {} + for pkg_aliases in aliases.values(): + if type(pkg_aliases) == type(""): + repo_names[pkg_aliases] = None + continue + + for repo_name in pkg_aliases.values(): + repo_names[repo_name] = None + + visibility = [ + "@{}//:__pkg__".format(repo_name) + for repo_name in sorted(repo_names) + ] + if has_groups: + visibility.append("//_groups:__pkg__") + return visibility + def _major_minor(python_version): major, _, tail = python_version.partition(".") minor, _, _ = tail.partition(".") @@ -242,3 +297,22 @@ def _non_versioned_platform(p, *, strict = False): return p else: fail("Expected to always have a platform in the form '{{abi}}_{{os}}_{{arch}}', got: {}".format(p)) + +def available_extras(aliases): + """Return the extras that are available in the hub for each package. + + Args: + aliases: {type}`dict[str, dict[struct, str]]` the packages to the + {obj}`whl_config_setting` instances and the repo names. + + Returns: + {type}`dict[str, list[str]]` the sorted extras for each package. + """ + return { + pkg: sorted({ + extra: None + for setting in settings + for extra in setting.extras or [] + }) + for pkg, settings in aliases.items() + } diff --git a/python/private/pypi/requirements.bzl.tmpl.bzlmod b/python/private/pypi/requirements.bzl.tmpl.bzlmod index ba227aeb2d..1b6a6c02dd 100644 --- a/python/private/pypi/requirements.bzl.tmpl.bzlmod +++ b/python/private/pypi/requirements.bzl.tmpl.bzlmod @@ -4,6 +4,7 @@ """ load("@rules_python//python:pip.bzl", "pip_utils") +load("@rules_python//python/private/pypi:labels.bzl", "requirement_label", "requirement_labels") all_requirements = %%ALL_REQUIREMENTS%% @@ -14,7 +15,28 @@ all_whl_requirements = all_whl_requirements_by_package.values() all_data_requirements = %%ALL_DATA_REQUIREMENTS%% def requirement(name): - return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "pkg") + """Return the label for the package. + + Args: + name: The name of the package with at most one extra, e.g. `foo`, + `foo[]` or `foo[bar]`. Use `requirements` for multiple extras. + + Returns: + The label of the package. + """ + return requirement_label("%%MACRO_TMPL%%", name) + +def requirements(*names): + """Return the labels for the packages. + + Args: + *names: The names of the packages with optional extras, e.g. `foo`, + `foo[]`, `foo[bar]` or `foo[bar,baz]`. + + Returns: + The list of labels, with a label for each extra of a package. + """ + return requirement_labels("%%MACRO_TMPL%%", *names) def whl_requirement(name): return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "whl") diff --git a/python/private/pypi/requirements.bzl.tmpl.workspace b/python/private/pypi/requirements.bzl.tmpl.workspace index faf774e7ed..61c191a3e7 100644 --- a/python/private/pypi/requirements.bzl.tmpl.workspace +++ b/python/private/pypi/requirements.bzl.tmpl.workspace @@ -19,7 +19,28 @@ _config = %%CONFIG%% _annotations = %%ANNOTATIONS%% def requirement(name): - return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "pkg") + """Return the label for the package. + + Args: + name: The name of the package with at most one extra, e.g. `foo`, + `foo[]` or `foo[bar]`. Use `requirements` for multiple extras. + + Returns: + The label of the package. + """ + return requirement_label("%%MACRO_TMPL%%", name) + +def requirements(*names): + """Return the labels for the packages. + + Args: + *names: The names of the packages with optional extras, e.g. `foo`, + `foo[]`, `foo[bar]` or `foo[bar,baz]`. + + Returns: + The list of labels, with a label for each extra of a package. + """ + return requirement_labels("%%MACRO_TMPL%%", *names) def whl_requirement(name): return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "whl") @@ -47,8 +68,10 @@ def install_deps(**whl_library_kwargs): # Set up the requirement groups all_requirement_groups = %%ALL_REQUIREMENT_GROUPS%% + # The members are the packages, e.g. `foo`, or the extras of the packages, + # e.g. `foo[bar]`. requirement_group_mapping = { - requirement: group_name + requirement.partition("[")[0]: group_name for group_name, group_requirements in all_requirement_groups.items() for requirement in group_requirements } diff --git a/python/private/pypi/whl_config_setting.bzl b/python/private/pypi/whl_config_setting.bzl index 1d868b1b65..7c2c595770 100644 --- a/python/private/pypi/whl_config_setting.bzl +++ b/python/private/pypi/whl_config_setting.bzl @@ -14,7 +14,7 @@ "A small function to create an alias for a whl distribution" -def whl_config_setting(*, version = None, target_platforms = None): +def whl_config_setting(*, version = None, target_platforms = None, extras = None): """The bzl_packages value used by by the render_pkg_aliases function. This contains the minimum amount of information required to generate correct @@ -27,6 +27,8 @@ def whl_config_setting(*, version = None, target_platforms = None): is no match found during a select. target_platforms: {type}`list[str] | None` the list of target_platforms for this distribution. + extras: {type}`list[str] | None` the extras of the distribution that + are requested on the target platforms. Returns: a struct with the validated and parsed values. @@ -55,4 +57,5 @@ def whl_config_setting(*, version = None, target_platforms = None): # Make the struct hashable target_platforms = tuple(target_platforms) if target_platforms else None, version = version, + extras = tuple(sorted(extras)) if extras else None, ) diff --git a/python/private/pypi/whl_deps_repo.bzl b/python/private/pypi/whl_deps_repo.bzl index 20891da397..73a0051f11 100644 --- a/python/private/pypi/whl_deps_repo.bzl +++ b/python/private/pypi/whl_deps_repo.bzl @@ -2,7 +2,6 @@ load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") -load(":pep508_requirement.bzl", "requirement") # Reusable common attributes for generating BUILD.bazel from Requires-Dist in the wheel METADATA. whl_deps_attrs = { @@ -26,10 +25,6 @@ For example if your whl depends on `numpy` and your Python package repo is named "group_name": attr.string( doc = "Name of the group, if any.", ), - "requirement": attr.string( - mandatory = True, - doc = "Python requirement string describing the package to make available, if 'urls' or 'whl_file' is given, then this only needs to include foo[any_extras] as a bare minimum.", - ), } def _whl_deps_repo_impl(rctx): @@ -62,7 +57,7 @@ def _whl_deps_repo_impl(rctx): repo = rctx.attr.repo or ( str(rctx.attr.metadata_file) if rctx.attr.metadata_file else None ), - extras = requirement(rctx.attr.requirement).extras, + provides_extra = getattr(metadata, "provides_extra", []), ) rctx.file("BUILD.bazel", build_file_contents) diff --git a/python/private/pypi/whl_file_repo.bzl b/python/private/pypi/whl_file_repo.bzl index 40d79ac10b..0118402d11 100644 --- a/python/private/pypi/whl_file_repo.bzl +++ b/python/private/pypi/whl_file_repo.bzl @@ -36,8 +36,8 @@ def download_whl(rctx, *, logger): auth = get_auth(rctx, urls), ) if not rctx.attr.sha256 and not rctx.attr.integrity: - logger.warn("Please update the requirement line to include the hash:\n{} \\\n --hash=sha256:{}".format( - rctx.attr.requirement, + logger.warn("Please update the requirement line of '{}' to include the hash:\n --hash=sha256:{}".format( + filename, result.sha256, )) if not result.success: @@ -50,7 +50,6 @@ whl_file_repo_attrs = { "index_url": attr.string(doc = "The index URL used to resolve relative wheel URLs."), "integrity": attr.string(doc = "The expected SRI checksum, used when sha256 is unset."), "patches": attr.label_keyed_string_dict(doc = "Patch labels mapped to their strip levels."), - "requirement": attr.string(doc = "The requirement used in download diagnostics."), "sha256": attr.string(doc = "The expected SHA256 checksum."), "urls": attr.string_list(doc = "The wheel download URLs."), "whl_file": attr.label(doc = "A caller-owned wheel to use instead of downloading."), diff --git a/python/private/pypi/whl_library_deps_targets.bzl b/python/private/pypi/whl_library_deps_targets.bzl index e549f52a47..5cce4e870f 100644 --- a/python/private/pypi/whl_library_deps_targets.bzl +++ b/python/private/pypi/whl_library_deps_targets.bzl @@ -8,13 +8,14 @@ load( "DATA_LABEL", "DIST_INFO_LABEL", "EXTRACTED_WHEEL_FILES", - "PY_LIBRARY_IMPL_LABEL", "PY_LIBRARY_PUBLIC_LABEL", "PY_SRCS_LABEL", "WHEEL_FILE", - "WHEEL_FILE_IMPL_LABEL", "WHEEL_FILE_PUBLIC_LABEL", + "extra_label", + "group_member", ) +load(":missing_package.bzl", "missing_extra_error") load(":pep508_deps.bzl", "deps") def whl_library_deps_targets( @@ -24,7 +25,7 @@ def whl_library_deps_targets( aliases = None, metadata_name, requires_dist = [], - extras = [], + provides_extra = [], include = [], group_deps = [], group_name = None, @@ -35,6 +36,7 @@ def whl_library_deps_targets( rules = struct( py_library = py_library, env_marker_setting = env_marker_setting, + missing_extra_error = missing_extra_error, )): """Create all of the whl_library targets. @@ -48,11 +50,14 @@ def whl_library_deps_targets( group rules which wrap this library and its fellows together. requires_dist: {type}`list[str]` The list of `Requires-Dist` values from the whl `METADATA`. Optional because some packages don't have them. - extras: {type}`list[str]` The list of requested extras. This essentially - includes extra transitive dependencies in the final targets - depending on the wheel `METADATA`. Optional because some packages - don't request them. - include: {type}`list[str]` The list of packages to include. + provides_extra: {type}`list[str]` The list of extras that the package + provides. For each extra the `[]` target is created + that includes the package and the dependencies of that extra. + include: {type}`list[str] | dict[str, list[str] | None]` The list + of packages to include. If it is a dict, then the values are the + extras of each package that are available in the hub and the + dependencies on the extras that are not available will fail the + build with a helpful message. group_name: {type}`str | None` name of the dependency group (if any). dep_template: {type}`str | None` The dep_template to use. tags: {type}`list[str]` The tags set on the targets. @@ -75,27 +80,33 @@ def whl_library_deps_targets( DATA_LABEL: repo_label(DATA_LABEL), } + # The members of the group are the packages, e.g. `foo`, or the extras of + # the packages, e.g. `foo[bar]`. Only the targets of the members are part + # of the group and they do not depend on the other members to avoid cycles. + self_name = normalize_name(metadata_name) + members = {group_member(m): None for m in group_deps} if group_name else {} + is_member = lambda extra: ( + "{}[{}]".format(self_name, extra) if extra else self_name + ) in members + # If this library is a member of a group, its public label aliases need to # point to the group implementation rule not the implementation rules. We # also need to mark the implementation rules as visible to the group # implementation. + group_label = None if group_name and "//:" in dep_template: # This is the legacy behaviour where the group library is outside the hub repo # # It is expected to disappear when we drop WORKSPACE or drop the vendoring of # pip_parse `requirements.bzl` in WORKSPACE. The alternative would be to add # another argument to the macro, but it is already full of arguments. - label_tmpl = dep_template.format( + group_label = dep_template.format( name = "_config", target = normalize_name(group_name) + "_{}", ).replace( "//:", "//_groups:", ) - aliases = aliases | { - PY_LIBRARY_PUBLIC_LABEL: label_tmpl.format(PY_LIBRARY_PUBLIC_LABEL), - WHEEL_FILE_PUBLIC_LABEL: label_tmpl.format(WHEEL_FILE_PUBLIC_LABEL), - } impl_vis = [dep_template.format( name = "_config", target = "__pkg__", @@ -103,25 +114,40 @@ def whl_library_deps_targets( "//:", "//_groups:", )] - - py_library_label = PY_LIBRARY_IMPL_LABEL - whl_file_label = WHEEL_FILE_IMPL_LABEL + elif group_name: + impl_vis = [dep_template.format(name = "", target = "__subpackages__")] else: - py_library_label = PY_LIBRARY_PUBLIC_LABEL - whl_file_label = WHEEL_FILE_PUBLIC_LABEL - if group_name: - impl_vis = [dep_template.format(name = "", target = "__subpackages__")] - else: - impl_vis = visibility + impl_vis = visibility + + def _name(extra, public): + # In the legacy mode the targets of the members are implementation + # targets and the public targets are aliases to the group. + prefix = "_" if group_label and is_member(extra) else "" + return prefix + (extra_label(extra) if extra else public) + + def _visibility(extra): + # In the legacy mode the targets that are not members are public. + return visibility if group_label and not is_member(extra) else impl_vis + + # Create targets for every extra that the package provides, so that the hub + # can select only the extras that are requested on each platform without + # passing the extras to this repository. + extras_targets = [""] + sorted({normalize_name(e): None for e in provides_extra}) + whl_file_label = _name("", WHEEL_FILE_PUBLIC_LABEL) + + if group_label: + for extra in extras_targets: + if is_member(extra): + aliases[extra_label(extra) if extra else PY_LIBRARY_PUBLIC_LABEL] = group_label.format(PY_LIBRARY_PUBLIC_LABEL) + if is_member(""): + aliases[WHEEL_FILE_PUBLIC_LABEL] = group_label.format(WHEEL_FILE_PUBLIC_LABEL) if not requires_dist: - # If the package is in a group but has no deps, we still need the public labels to - # point at the srcs targets so that the group implementation can use them. We don't - # need any of the extra targets, so just create the aliases. - aliases = aliases | { - py_library_label: repo_label(PY_SRCS_LABEL), - whl_file_label: repo_label(WHEEL_FILE), - } + # If the package is in a group but has no deps, we still need the labels to point at + # the srcs targets so that the group implementation can use them. + aliases[whl_file_label] = repo_label(WHEEL_FILE) + for extra in extras_targets: + aliases[_name(extra, PY_LIBRARY_PUBLIC_LABEL)] = repo_label(PY_SRCS_LABEL) for alias, actual in aliases.items(): native.alias( @@ -134,37 +160,62 @@ def whl_library_deps_targets( # If there are extras, then they will be visible in requires_dist. return - package_deps = _parse_requires_dist( - name = metadata_name, - requires_dist = requires_dist, - excludes = group_deps, - extras = extras, - include = include, - ) + # The settings for the deps that are included conditionally keyed by the + # dep and the expression. + settings = {} + + # The dependencies on extras that are not available in the hub. + missing_extras = {} + + def _pkg_label(dep): + dep_name, _, extra = dep.partition("[") + if not extra: + return dep_template.format(name = dep_name, target = PY_LIBRARY_PUBLIC_LABEL) - _config_settings( - dependencies_with_markers = package_deps.deps_select, - rules = rules, - visibility = ["//visibility:private"], - ) + extra = extra.rstrip("]") + available = include.get(dep_name) if type(include) == type({}) else None + if available != None and extra not in available: + missing_extras[dep] = (dep_name, extra) + return ":_missing_" + dep + + return dep_template.format(name = dep_name, target = extra_label(extra)) + + whl_label = lambda dep: dep_template.format(name = dep, target = WHEEL_FILE_PUBLIC_LABEL) if hasattr(native, "filegroup"): # We include the whl file as srcs so that `$(location :whl)` expands to the whl file. - # The transitive dependencies are available via the `data` attribute. + # The transitive dependencies are available via the `data` attribute. The extras + # are not used for the whl targets, so only the packages without extras can be + # members of the group for the whl targets. + whl_deps = deps( + name = self_name, + requires_dist = requires_dist, + include = include, + excludes = members if is_member("") else [], + ) native.filegroup( name = whl_file_label, srcs = [repo_label(WHEEL_FILE)], data = _deps( deps = [], - package_deps = package_deps, - tmpl = dep_template.format(name = "{}", target = WHEEL_FILE_PUBLIC_LABEL), + package_deps = whl_deps, + conditions = _conditions(settings, whl_deps), + label = whl_label, ), - visibility = impl_vis, + visibility = _visibility(""), ) - if hasattr(rules, "py_library"): + for extra in extras_targets if hasattr(rules, "py_library") else []: + package_deps = deps( + name = self_name, + requires_dist = requires_dist, + include = include, + excludes = members if is_member(extra) else [], + extras = [extra] if extra else [], + keep_extras = True, + ) rules.py_library( - name = py_library_label, + name = _name(extra, PY_LIBRARY_PUBLIC_LABEL), # We include as srcs to ensure that the (locations :pkg) works as expected. srcs = [repo_label(PY_SRCS_LABEL)], deps = _deps( @@ -173,52 +224,57 @@ def whl_library_deps_targets( # `pyi_srcs`, see `tests/base_rules/py_library`. deps = [repo_label(PY_SRCS_LABEL)], package_deps = package_deps, - tmpl = dep_template.format(name = "{}", target = PY_LIBRARY_PUBLIC_LABEL), + conditions = _conditions(settings, package_deps), + label = _pkg_label, ), # Disable precompilation on this wrapper target to prevent duplicate # pyc generation; the underlying PY_SRCS_LABEL target handles it. precompile = "disabled", tags = tags, - visibility = impl_vis, + visibility = _visibility(extra), ) -def _parse_requires_dist( - *, - name, - requires_dist, - excludes, - include, - extras): - return deps( - name = normalize_name(name), - requires_dist = requires_dist, - excludes = excludes, - include = include, - extras = extras, - ) - -def _config_settings(dependencies_with_markers, rules, **kwargs): - """Generate config settings for the targets. - - Args: - dependencies_with_markers: {type}`dict[str, str]` The markers to evaluate by - each dep. - rules: used for testing - **kwargs: Extra kwargs to pass to the rule. - """ - for dep, expression in dependencies_with_markers.items(): + for (_, expression), name in settings.items(): rules.env_marker_setting( - name = "include_{}".format(dep), + name = name, expression = expression, - **kwargs + visibility = ["//visibility:private"], ) -def _deps(deps, package_deps, tmpl): - deps = [] + deps + [tmpl.format(d) for d in sorted(package_deps.deps)] + for dep, (dep_name, extra) in sorted(missing_extras.items()): + rules.missing_extra_error( + name = "_missing_" + dep, + package_name = dep_name, + extra = extra, + required_by = metadata_name, + visibility = ["//visibility:private"], + ) + +def _conditions(settings, package_deps): + return { + dep: _setting_name(settings, dep, expression) + for dep, expression in package_deps.deps_select.items() + } + +def _setting_name(settings, dep, expression): + """Return a unique name for the setting evaluating the expression for the dep. + + The first expression for a dep is named `include_` and any different + expressions for the same dep get a numeric suffix. + """ + key = (dep, expression) + if key not in settings: + n = len([d for d, _ in settings if d == dep]) + settings[key] = "include_{}".format(dep) + ("_{}".format(n + 1) if n else "") + + return settings[key] + +def _deps(deps, package_deps, conditions, label): + deps = [] + deps + [label(d) for d in sorted(package_deps.deps)] for dep in package_deps.deps_select: deps = deps + select({ - ":is_include_{}_true".format(dep): [tmpl.format(dep)], + ":is_{}_true".format(conditions[dep]): [label(dep)], "//conditions:default": [], }) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index a1cc727460..50be70c532 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -26,7 +26,7 @@ def whl_library_targets( whl_file = None, metadata_name = "", requires_dist = [], - extras = [], + provides_extra = [], entry_points = {}, include = [], group_deps = [], @@ -55,7 +55,7 @@ def whl_library_targets( group rules which wrap this library and its fellows together. requires_dist: {type}`list[str]` The list of `Requires-Dist` values from the whl `METADATA`. - extras: {type}`list[str]` The list of requested extras. This essentially includes extra transitive dependencies in the final targets depending on the wheel `METADATA`. + provides_extra: {type}`list[str]` The list of extras that the package provides. entry_points: {type}`list[dict]` A list of parsed entry point definitions. include: {type}`list[str]` The list of packages to include. group_name: {type}`str | None` name of the dependency group (if any). @@ -97,7 +97,7 @@ def whl_library_targets( dep_template = dep_template, # only needed if requires_dist or group_name is present group_deps = group_deps, # only needed if group_name is present group_name = group_name, # must specify group_deps together - extras = extras, # only needed if requires_dist is present + provides_extra = provides_extra, include = include, # only needed if requires_dist is present repo = None, # set aliases in the same repo aliases = {}, diff --git a/python/private/pypi/whl_metadata.bzl b/python/private/pypi/whl_metadata.bzl index 2981a5d92f..203caafc58 100644 --- a/python/private/pypi/whl_metadata.bzl +++ b/python/private/pypi/whl_metadata.bzl @@ -56,7 +56,7 @@ def parse_whl_metadata(contents): "requires_dist": [], "version": "", } - for line in contents.strip().split("\n"): + for line in contents.strip().splitlines(): if not line: # Stop parsing on first empty line, which marks the end of the # headers containing the metadata. @@ -70,10 +70,10 @@ def parse_whl_metadata(contents): parsed["version"] = value.strip() elif line.startswith(_REQUIRES_DIST): _, _, value = line.partition(_REQUIRES_DIST) - parsed["requires_dist"].append(value.strip(" ")) + parsed["requires_dist"].append(value.strip()) elif line.startswith(_PROVIDES_EXTRA): _, _, value = line.partition(_PROVIDES_EXTRA) - parsed["provides_extra"].append(value.strip(" ")) + parsed["provides_extra"].append(value.strip()) return struct( name = parsed["name"], diff --git a/python/private/pyproject_utils.bzl b/python/private/pyproject_utils.bzl index e96ffd2023..a9494b8ac2 100644 --- a/python/private/pyproject_utils.bzl +++ b/python/private/pyproject_utils.bzl @@ -18,10 +18,22 @@ def read_pyproject(module_ctx, pyproject): {type}`struct` with the attributes: * `requires_python`: {type}`str | None` the raw `requires-python` value (e.g. `"==3.13.9"`), or `None` if it is not set. + * `name`: {type}`str | None` the `[project] name`. + * `dependencies`: {type}`list[str]` the `[project] dependencies`. + * `optional_dependencies`: {type}`dict[str, list[str]]` the + `[project.optional-dependencies]` table. + * `dependency_groups`: {type}`dict[str, list]` the PEP 735 + `[dependency-groups]` table. Items are either requirement strings + or `{include-group = ""}` tables. """ data = toml.decode(module_ctx.read(module_ctx.path(pyproject), watch = "yes")) + project = data.get("project", {}) return struct( - requires_python = data.get("project", {}).get("requires-python"), + requires_python = project.get("requires-python"), + name = project.get("name"), + dependencies = project.get("dependencies") or [], + optional_dependencies = project.get("optional-dependencies") or {}, + dependency_groups = data.get("dependency-groups") or {}, ) def version_from_requires_python(requires_python): diff --git a/tests/integration/unified_pypi/BUILD.bazel b/tests/integration/unified_pypi/BUILD.bazel index 13d188d087..78f9a857ee 100644 --- a/tests/integration/unified_pypi/BUILD.bazel +++ b/tests/integration/unified_pypi/BUILD.bazel @@ -95,3 +95,31 @@ platform( "@platforms//os:windows", ], ) + +# The pypi_filtered hub only makes its pyproject.toml packages public. +py_test( + name = "test_filtered_direct", + srcs = ["test_default.py"], + main = "test_default.py", + deps = ["@pypi_filtered//colorama"], +) + +# six is not exposed by pypi_filtered, so it is missing in the unified hub. +py_binary( + name = "bin_filtered_six_unified", + srcs = ["bin_six_a.py"], + config_settings = { + "@rules_python//python/config_settings:venv": "pypi_filtered", + }, + main = "bin_six_a.py", + tags = ["manual"], + deps = ["@pypi//six"], +) + +py_binary( + name = "bin_filtered_six", + srcs = ["bin_six_a.py"], + main = "bin_six_a.py", + tags = ["manual"], + deps = ["@pypi_filtered//six"], +) diff --git a/tests/integration/unified_pypi/MODULE.bazel b/tests/integration/unified_pypi/MODULE.bazel index ee8f622cb4..70e01ccdac 100644 --- a/tests/integration/unified_pypi/MODULE.bazel +++ b/tests/integration/unified_pypi/MODULE.bazel @@ -53,6 +53,20 @@ pip.parse( ) use_repo(pip, "pypi_b") +# `pypi_filtered` only exposes the `[project] dependencies` of the +# `pyproject.toml`, the other packages are not public. +pip.parse( + hub_name = "pypi_filtered", + pyproject_toml = "//:pyproject_filtered.toml", + python_version = "3.11", + requirements_lock = "//:requirements_b.txt", +) +pip.configure( + dependency_groups = ["default"], + hub_name = "pypi_filtered", +) +use_repo(pip, "pypi_filtered") + pip.default(default_hub = "pypi_b") pip.default(unified_hub_requirements_bzl = True) pip.dep( diff --git a/tests/integration/unified_pypi/pyproject_filtered.toml b/tests/integration/unified_pypi/pyproject_filtered.toml new file mode 100644 index 0000000000..632f72fe22 --- /dev/null +++ b/tests/integration/unified_pypi/pyproject_filtered.toml @@ -0,0 +1,4 @@ +[project] +name = "unified-pypi" +version = "0.0.0" +dependencies = ["colorama"] diff --git a/tests/integration/unified_pypi_test.py b/tests/integration/unified_pypi_test.py index 9ce41fafb7..f59a19fb42 100644 --- a/tests/integration/unified_pypi_test.py +++ b/tests/integration/unified_pypi_test.py @@ -7,6 +7,32 @@ class UnifiedPypiTest(runner.TestCase): + def test_configure_hides_unexposed_packages(self): + self.run_bazel("test", "//:test_filtered_direct") + result = self.run_bazel("build", "//:bin_filtered_six", check=False) + self.assertNotEqual( + result.exit_code, + 0, + "Expected the alias of the unexposed package to be hidden", + ) + self.assert_result_matches( + result, + r"alias '[^']*//six:six'.*is not visible from", + ) + + def test_configure_hides_unexposed_packages_in_unified_hub(self): + self.run_bazel("cquery", "//:bin_filtered_six_unified") + result = self.run_bazel("build", "//:bin_filtered_six_unified", check=False) + self.assertNotEqual( + result.exit_code, + 0, + "Expected build to fail during execution phase", + ) + self.assert_result_matches( + result, + 'ERROR: PyPI package "six" is not available when building under PyPI hub "pypi_filtered"\\. Try adding it to the requirements of this hub', + ) + def test_default_fallback_hub(self): self.run_bazel("test", "//:test_default") diff --git a/tests/integration/whl_library/MODULE.bazel b/tests/integration/whl_library/MODULE.bazel index 9059879187..1cafbb2024 100644 --- a/tests/integration/whl_library/MODULE.bazel +++ b/tests/integration/whl_library/MODULE.bazel @@ -44,7 +44,6 @@ whl_archive( dep_template = DEP_TEMPLATE, filename = "requests-2.34.2-py3-none-any.whl", # https://pypi.org/project/requests/#requests-2.34.2-py3-none-any.whl - requirement = "requests", sha256 = "2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", urls = [ "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", @@ -57,7 +56,6 @@ whl_deps_repo( name = "whl_deps_repo", dep_template = DEP_TEMPLATE, metadata_file = "@whl_archive//:metadata.json", - requirement = "requests", ) wheels = use_extension("//:wheels.bzl", "wheels") diff --git a/tests/integration/whl_library/wheels.bzl b/tests/integration/whl_library/wheels.bzl index b82f0c0479..c690c7f027 100644 --- a/tests/integration/whl_library/wheels.bzl +++ b/tests/integration/whl_library/wheels.bzl @@ -16,7 +16,6 @@ def _wheels_impl(mrctx): common = { "envsubst": ["WHEEL_TEST_ADDRESS"], "filename": _FILENAME, - "requirement": "storage-pkg", "sha256": mrctx.read(Label("//:wheel.sha256")).strip(), "urls": ["http://${WHEEL_TEST_ADDRESS}/" + _FILENAME], } @@ -25,12 +24,10 @@ def _wheels_impl(mrctx): whl_library(name = "nonmatching", whl_patches = _patches("other.whl"), **common) whl_library( name = "provided", - requirement = "storage-pkg", whl_file = Label("//:" + _FILENAME), ) whl_library( name = "provided_patched", - requirement = "storage-pkg", whl_file = Label("//:" + _FILENAME), whl_patches = _patches(_FILENAME), ) diff --git a/tests/pypi/dependency_groups/BUILD.bazel b/tests/pypi/dependency_groups/BUILD.bazel new file mode 100644 index 0000000000..ba6877e124 --- /dev/null +++ b/tests/pypi/dependency_groups/BUILD.bazel @@ -0,0 +1,3 @@ +load(":dependency_groups_tests.bzl", "dependency_groups_test_suite") + +dependency_groups_test_suite(name = "dependency_groups_tests") diff --git a/tests/pypi/dependency_groups/dependency_groups_tests.bzl b/tests/pypi/dependency_groups/dependency_groups_tests.bzl new file mode 100644 index 0000000000..60c5a4efbb --- /dev/null +++ b/tests/pypi/dependency_groups/dependency_groups_tests.bzl @@ -0,0 +1,119 @@ +"""dependency_groups tests""" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:dependency_groups.bzl", "select_dependency_groups") # buildifier: disable=bzl-visibility + +_tests = [] + +_PYPROJECT = struct( + name = "my-project", + dependencies = ["main-pkg>=1.0"], + optional_dependencies = { + "Docs": ["docs-pkg"], + "all": ["My_Project[docs, grpc]"], + "grpc": ["grpc-pkg"], + }, + dependency_groups = { + "dev": ["dev-pkg", "shared-pkg", {"include-group": "Test"}], + "test": ["test_pkg", "shared-pkg"], + }, +) + +def _select(specs, pyproject = _PYPROJECT): + failures = [] + got = select_dependency_groups(pyproject, specs, _fail = failures.append) + return got, failures + +def _test_select(env): + for specs, want in [ + (["default"], ["main_pkg"]), + (["optional:docs"], ["docs_pkg"]), + (["group:test"], ["shared_pkg", "test_pkg"]), + (["dev"], ["dev_pkg", "shared_pkg", "test_pkg"]), + (["grpc"], ["grpc_pkg"]), + (["all"], ["docs_pkg", "grpc_pkg"]), + (["optional:*"], ["docs_pkg", "grpc_pkg"]), + (["group:*"], ["dev_pkg", "shared_pkg", "test_pkg"]), + (["default", "group:test"], ["main_pkg", "shared_pkg", "test_pkg"]), + (["*"], ["dev_pkg", "docs_pkg", "grpc_pkg", "main_pkg", "shared_pkg", "test_pkg"]), + ]: + got, failures = _select(specs) + env.expect.where(specs = specs).that_collection(failures).contains_exactly([]) + env.expect.where(specs = specs).that_collection(got.packages).contains_exactly(want).in_order() + env.expect.where(specs = specs).that_bool(got.all).equals(False) + +_tests.append(_test_select) + +def _test_select_all(env): + got, failures = _select(["*", "transitive"]) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_bool(got.all).equals(True) + env.expect.that_collection(got.packages).contains_exactly([]) + +_tests.append(_test_select_all) + +def _test_select_extras(env): + pyproject = struct( + name = "my-project", + dependencies = ["main-pkg[Foo_Bar, baz]>=1.0", "plain-pkg"], + optional_dependencies = { + "all": ["my-project[docs]"], + "docs": ["docs-pkg[extra]", "main-pkg[docs]"], + }, + dependency_groups = { + "dev": ["dev-pkg[cli]", {"include-group": "test"}], + "test": ["test-pkg[cov]; sys_platform == 'linux'"], + }, + ) + for specs, want in [ + (["default"], {"main_pkg": ["baz", "foo_bar"]}), + (["dev"], {"dev_pkg": ["cli"], "test_pkg": ["cov"]}), + # The extras of the self reference are the optional dependencies of + # the project, not the extras of a package. + (["all"], {"docs_pkg": ["extra"], "main_pkg": ["docs"]}), + (["default", "docs"], {"docs_pkg": ["extra"], "main_pkg": ["baz", "docs", "foo_bar"]}), + (["*", "transitive"], { + "dev_pkg": ["cli"], + "docs_pkg": ["extra"], + "main_pkg": ["baz", "docs", "foo_bar"], + "test_pkg": ["cov"], + }), + ]: + got, failures = _select(specs, pyproject = pyproject) + env.expect.where(specs = specs).that_collection(failures).contains_exactly([]) + env.expect.where(specs = specs).that_dict(got.extras).contains_exactly(want) + +_tests.append(_test_select_extras) + +def _test_select_errors(env): + for specs, want in [ + (["missing"], "Unknown dependency group 'missing'. Available groups: default, group:dev, group:test, optional:all, optional:docs, optional:grpc"), + (["default", "transitive"], "The 'transitive' dependency group specifier is only supported together with '*', which selects all of the packages, got: [\"default\", \"transitive\"]"), + ]: + _, failures = _select(specs) + env.expect.where(specs = specs).that_collection(failures).contains_exactly([want]) + +_tests.append(_test_select_errors) + +def _test_include_group_errors(env): + for groups, want in [ + ({"a": [{"include-group": "missing"}]}, "Unknown dependency group 'missing' included from 'a'"), + ({"a": [{"include-group": "b"}], "b": [{"include-group": "a"}]}, "Cyclic dependency group includes: a, b"), + ]: + _, failures = _select(["*"], pyproject = struct( + name = None, + dependencies = [], + optional_dependencies = {}, + dependency_groups = groups, + )) + env.expect.that_collection(failures).contains_exactly([want]) + +_tests.append(_test_include_group_errors) + +def dependency_groups_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 91bf3c76d1..ab547f7fa2 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -21,7 +21,7 @@ load("//python/private/pypi:extension.bzl", "build_config", "parse_modules") # load("//python/private/pypi:platform.bzl", _plat = "platform") # buildifier: disable=bzl-visibility load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # buildifier: disable=bzl-visibility load("//tests/support/mocks:mocks.bzl", "mocks") -load(":pip_parse.bzl", _parse = "pip_parse") +load(":pip_parse.bzl", _configure = "configure", _parse = "pip_parse") _tests = [] @@ -107,10 +107,11 @@ def _dep(*, name, extra_targets = []): extra_targets = extra_targets, ) -def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], dep = [], is_root = True): +def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], dep = [], configure = [], is_root = True): return struct( name = name, tags = struct( + configure = configure, parse = parse, override = override, whl_mods = whl_mods, @@ -124,6 +125,7 @@ def _parse_modules(env, **kwargs): return env.expect.that_struct( parse_modules(**kwargs), attrs = dict( + configured_hubs = subjects.dict, declared_deps = subjects.dict, default_hub = subjects.str, exposed_packages = subjects.dict, @@ -812,13 +814,151 @@ simple==0.0.1 \ "filename": "simple-0.0.1-py3-none-any.whl", "index_url": test.want_index_url, "integrity": "sha256-3q2+7w==", - "requirement": "simple==0.0.1", "urls": ["file://simple-0.0.1-py3-none-any.whl"], }, }) _tests.append(_test_default_index_setting) +_CONFIGURE_REQUIREMENTS = """\ +simple==0.0.1 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +other-pkg==0.0.2 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +transitive-pkg==0.0.3 \ + --hash=sha256:deadbeef \ + --hash=sha256:deadbaaf +""" + +_CONFIGURE_PYPROJECT = """\ +[project] +dependencies = ["simple"] + +[dependency-groups] +dev = ["other-pkg"] +test = ["simple"] +""" + +def _configure_parse_modules(env, *modules): + return _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + os_name = "linux", + arch_name = "x86_64", + mock_files = { + "pyproject.toml": _CONFIGURE_PYPROJECT, + "requirements.txt": _CONFIGURE_REQUIREMENTS, + }, + *modules + ), + available_interpreters = { + "python_3_15_host": "unit_test_interpreter_target", + }, + minor_mapping = {"3.15": "3.15.19"}, + ) + +def _configure_parse(**kwargs): + return _parse( + hub_name = "pypi_a", + python_version = "3.15", + simpleapi_skip = ["simple", "other-pkg", "transitive-pkg"], + requirements_lock = "requirements.txt", + **kwargs + ) + +def _test_configure(env): + # The selection itself is tested in `hub_exposure_tests`. + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [_configure_parse(pyproject_toml = "pyproject.toml")], + configure = [_configure(hub_name = "pypi_a", dependency_groups = ["test"])], + ), + ) + + pypi.exposed_packages().contains_exactly({"pypi_a": ["simple"]}) + pypi.configured_hubs().contains_exactly({"pypi_a": ["simple"]}) + +_tests.append(_test_configure) + +def _test_configure_default_pyproject(env): + # The `pip.default` pyproject.toml is used for the hubs of the root module. + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + default = _default_tags_default + [_default(pyproject_toml = "pyproject.toml")], + parse = [_configure_parse()], + configure = [_configure(hub_name = "pypi_a", dependency_groups = ["dev"])], + ), + ) + + pypi.configured_hubs().contains_exactly({"pypi_a": ["other_pkg"]}) + +_tests.append(_test_configure_default_pyproject) + +def _test_configure_extras(env): + # The extras requested in the pyproject.toml are added to the hub even if + # the lock file does not request them. + pypi = _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + _mod( + name = "rules_python", + parse = [_configure_parse(pyproject_toml = "pyproject.toml")], + configure = [_configure(hub_name = "pypi_a", dependency_groups = ["default"])], + ), + os_name = "linux", + arch_name = "x86_64", + mock_files = { + "pyproject.toml": """\ +[project] +dependencies = ["simple[foo]", "other-pkg"] +""", + "requirements.txt": _CONFIGURE_REQUIREMENTS, + }, + ), + available_interpreters = { + "python_3_15_host": "unit_test_interpreter_target", + }, + minor_mapping = {"3.15": "3.15.19"}, + ) + + pypi.configured_hubs().contains_exactly({"pypi_a": ["other_pkg", "simple"]}) + pypi.hub_whl_map().contains_exactly({"pypi_a": { + "other_pkg": { + "pypi_a_315_other_pkg": [whl_config_setting(version = "3.15")], + }, + "simple": { + "pypi_a_315_simple": [whl_config_setting(version = "3.15", extras = ["foo"])], + }, + "transitive_pkg": { + "pypi_a_315_transitive_pkg": [whl_config_setting(version = "3.15")], + }, + }}) + +_tests.append(_test_configure_extras) + +def _test_not_configured_exposes_all(env): + # The pyproject.toml does not filter the hub without `pip.configure`. + pypi = _configure_parse_modules( + env, + _mod( + name = "rules_python", + parse = [_configure_parse()], + ), + ) + + pypi.exposed_packages().contains_exactly({ + "pypi_a": ["other_pkg", "simple", "transitive_pkg"], + }) + pypi.configured_hubs().contains_exactly({}) + +_tests.append(_test_not_configured_exposes_all) + def extension_test_suite(name): """Create the test suite. diff --git a/tests/pypi/extension/pip_parse.bzl b/tests/pypi/extension/pip_parse.bzl index 638836e588..3670282a92 100644 --- a/tests/pypi/extension/pip_parse.bzl +++ b/tests/pypi/extension/pip_parse.bzl @@ -1,5 +1,17 @@ """A simple test helper""" +def configure( + *, + dependency_groups = ["*"], + hub_name, + pyproject_toml = None): + """A simple helper for testing to simulate the PyPI extension configure tag class""" + return struct( + dependency_groups = dependency_groups, + hub_name = hub_name, + pyproject_toml = pyproject_toml, + ) + def pip_parse( *, hub_name, diff --git a/tests/pypi/generate_group_library_build_bazel/generate_group_library_build_bazel_tests.bzl b/tests/pypi/generate_group_library_build_bazel/generate_group_library_build_bazel_tests.bzl index a91f861a36..2fe22eb487 100644 --- a/tests/pypi/generate_group_library_build_bazel/generate_group_library_build_bazel_tests.bzl +++ b/tests/pypi/generate_group_library_build_bazel/generate_group_library_build_bazel_tests.bzl @@ -95,6 +95,33 @@ py_library( _tests.append(_test_in_hub) +def _test_in_hub_with_extras(env): + actual = generate_group_library_build_bazel( + repo_prefix = "", + groups = {"vbap": ["Pcodedmp", "oletools[Full]"]}, + ) + + # The extras are members only for the py_library targets + env.expect.that_str(actual).contains("""\ +filegroup( + name = "vbap_whl", + srcs = [], + data = ["//pcodedmp:_whl"], + visibility = ["//:__subpackages__"], +)""") + env.expect.that_str(actual).contains("""\ +py_library( + name = "vbap_pkg", + srcs = [], + deps = [ + "//oletools:_[full]", + "//pcodedmp:_pkg", + ], + visibility = ["//:__subpackages__"], +)""") + +_tests.append(_test_in_hub_with_extras) + def generate_group_library_build_bazel_test_suite(name): """Create the test suite. diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index dd7cd650b8..f9e2b0a905 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -181,6 +181,50 @@ def _test_uv_lock_only(env): _tests.append(_test_uv_lock_only) +def _test_uv_lock_extras(env): + builder = hub_builder(env) + builder.pip_parse( + _mock_mctx( + os_name = "osx", + arch_name = "aarch64", + mock_files = { + "uv.lock": """{"package":[{"name":"simple","provides-extras":["async-rest","grpc"],"source":{"registry":"https://pypi.org/simple"},"version":"0.0.1","wheels":[{"hash":"sha256:deadbeef","url":"https://files.pythonhosted.org/packages/simple-0.0.1-py3-none-any.whl"}]}]}""", + }, + ), + _parse( + hub_name = "pypi", + python_version = "3.15", + requirements_lock = None, + uv_lock = "uv.lock", + target_platforms = ["osx_aarch64", "linux_x86_64"], + ), + ) + pypi = builder.build() + + pypi.whl_map().contains_exactly({ + "simple": { + "pypi_315_simple_py3_none_any_deadbeef": [ + whl_config_setting( + version = "3.15", + target_platforms = ["cp315_linux_x86_64", "cp315_osx_aarch64"], + extras = ["async_rest", "grpc"], + ), + ], + }, + }) + pypi.whl_libraries().contains_at_least({ + "pypi_315_simple_py3_none_any_deadbeef": { + "config_load": "@pypi//:config.bzl", + "dep_template": "@pypi//{name}:{target}", + "filename": "simple-0.0.1-py3-none-any.whl", + "index_url": "https://pypi.org/simple/simple", + "integrity": "sha256-3q2+7w==", + "urls": ["https://files.pythonhosted.org/packages/simple-0.0.1-py3-none-any.whl"], + }, + }) + +_tests.append(_test_uv_lock_extras) + def _test_target_platforms_all(env): builder = hub_builder(env) builder.pip_parse( @@ -377,10 +421,10 @@ _tests.append(_test_simple_multiple_requirements) def _test_simple_extras_vs_no_extras(env): sub_tests = { - ("osx", "aarch64"): "simple[foo]==0.0.1 --hash=sha256:deadbeef", - ("windows", "aarch64"): "simple==0.0.1 --hash=sha256:deadbeef", + ("osx", "aarch64"): ("simple==0.0.1 --hash=sha256:deadbeef", ["foo"]), + ("windows", "aarch64"): ("simple==0.0.1 --hash=sha256:deadbeef", None), } - for (host_os, host_arch), want_requirement in sub_tests.items(): + for (host_os, host_arch), (want_requirement, want_extras) in sub_tests.items(): builder = hub_builder(env) builder.pip_parse( mocks.mctx( @@ -405,7 +449,7 @@ def _test_simple_extras_vs_no_extras(env): pypi.whl_map().contains_exactly({ "simple": { "pypi_315_simple": [ - whl_config_setting(version = "3.15"), + whl_config_setting(version = "3.15", extras = want_extras), ], }, }) @@ -467,43 +511,35 @@ def _test_simple_extras_vs_no_extras_simpleapi(env): pypi.exposed_packages().contains_exactly(["simple"]) pypi.group_map().contains_exactly({}) + + # The same wheel is used on both platforms, so a single repository has the + # extras for all of the platforms and the hub selects the extras per platform. pypi.whl_map().contains_exactly({ "simple": { - "pypi_315_simple_py3_none_any_deadbeef_osx_aarch64": [ + "pypi_315_simple_py3_none_any_deadbeef": [ whl_config_setting( target_platforms = [ - "cp315_osx_aarch64", + "cp315_windows_aarch64", ], version = "3.15", ), - ], - "pypi_315_simple_py3_none_any_deadbeef_windows_aarch64": [ whl_config_setting( target_platforms = [ - "cp315_windows_aarch64", + "cp315_osx_aarch64", ], version = "3.15", + extras = ["foo"], ), ], }, }) pypi.whl_libraries().contains_exactly({ - "pypi_315_simple_py3_none_any_deadbeef_osx_aarch64": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", - "filename": "simple-0.0.1-py3-none-any.whl", - "index_url": "https://example.com/simple/", - "integrity": "sha256-3q2+7w==", - "requirement": "simple[foo]==0.0.1", - "urls": ["/simple-0.0.1-py3-none-any.whl"], - }, - "pypi_315_simple_py3_none_any_deadbeef_windows_aarch64": { + "pypi_315_simple_py3_none_any_deadbeef": { "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "filename": "simple-0.0.1-py3-none-any.whl", "index_url": "https://example.com/simple/", "integrity": "sha256-3q2+7w==", - "requirement": "simple==0.0.1", "urls": ["/simple-0.0.1-py3-none-any.whl"], }, }) @@ -575,7 +611,6 @@ def _test_simple_sha512_simpleapi(env): "filename": "simple-0.0.1-py3-none-any.whl", "index_url": "https://example.com/simple/", "integrity": "sha512-3q2+7w==", - "requirement": "simple==0.0.1", "urls": ["/simple-0.0.1-py3-none-any.whl"], }, }) @@ -966,7 +1001,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", "index_url": "https://torch.index/torch/", "integrity": "sha256-iADe7wAmAR1QLAwlbMS2fQAjR/Y8OjjNjkXx9EXGE2Q=", - "requirement": "torch==2.4.1+cpu", "urls": ["/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-linux_x86_64.whl"], }, "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432_linux_aarch64": { @@ -975,7 +1009,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", "index_url": "https://torch.index/torch/", "integrity": "sha256-NhCUMrEL1xY8mzDOiW88LMobhrl2X5VqFZTw/0MJHio=", - "requirement": "torch==2.4.1", "urls": ["/whl/cpu/torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"], }, "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c_windows_x86_64": { @@ -984,7 +1017,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", "index_url": "https://torch.index/torch/", "integrity": "sha256-OlcOXFU0Fc293+Z5IHMns6OAayHGreoU+6d2hNFhnpc=", - "requirement": "torch==2.4.1+cpu", "urls": ["/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-win_amd64.whl"], }, "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5_osx_aarch64": { @@ -993,7 +1025,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", "index_url": "https://torch.index/torch/", "integrity": "sha256-crSE1bbOwac1vz+locSIPQF0hpjF6c/b60/6t8eYfg0=", - "requirement": "torch==2.4.1", "urls": ["/whl/cpu/torch-2.4.1-cp312-none-macosx_11_0_arm64.whl"], }, }) @@ -1136,7 +1167,6 @@ simple==0.0.1 --hash=sha256:deadb00f "filename": "simple-0.0.1-py3-none-any.whl", "index_url": test.expect_index_url, "integrity": "sha256-3q2wDw==", - "requirement": "simple==0.0.1", "urls": [test.expect_url], } if getattr(test, "envsubst", []): @@ -1489,7 +1519,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "dep_template": "@pypi//{name}:{target}", "filename": "direct_without_sha-0.0.1-py3-none-any.whl", "integrity": "", - "requirement": "direct_without_sha==0.0.1", "urls": ["example-direct.org/direct_without_sha-0.0.1-py3-none-any.whl"], "whl_patches": {"my_patch": "1"}, }, @@ -1513,7 +1542,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "filename": "plat-pkg-0.0.4-py3-none-linux_x86_64.whl", "index_url": "https://pypi.org/simple", "integrity": "sha256-3q20Tw==", - "requirement": "plat_pkg==0.0.4", "urls": ["example2.org/index/plat_pkg/"], }, "pypi_315_simple_py3_none_any_deadb00f": { @@ -1522,7 +1550,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "filename": "simple-0.0.1-py3-none-any.whl", "index_url": "https://pypi.org/simple", "integrity": "sha256-3q2wDw==", - "requirement": "simple==0.0.1", "urls": ["example2.org"], }, "pypi_315_some_pkg_py3_none_any_deadbaaf": { @@ -1530,7 +1557,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "dep_template": "@pypi//{name}:{target}", "filename": "some_pkg-0.0.1-py3-none-any.whl", "integrity": "sha256-3q26rw==", - "requirement": "some_pkg==0.0.1", "urls": ["example-direct.org/some_pkg-0.0.1-py3-none-any.whl"], }, "pypi_315_some_py3_none_any_deadb33f": { @@ -1539,7 +1565,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "filename": "some-other-pkg-0.0.1-py3-none-any.whl", "index_url": "https://with_index_url", "integrity": "sha256-3q2zPw==", - "requirement": "some_other_pkg==0.0.1", "urls": ["example2.org/index/some_other_pkg/"], }, }) @@ -1569,10 +1594,10 @@ _tests.append(_test_simple_get_index) def _test_optimum_sys_platform_extra(env): sub_tests = { - ("osx", "aarch64"): "optimum[onnxruntime]==1.17.1", - ("linux", "aarch64"): "optimum[onnxruntime-gpu]==1.17.1", + ("osx", "aarch64"): ("optimum==1.17.1", ["onnxruntime"]), + ("linux", "aarch64"): ("optimum==1.17.1", ["onnxruntime_gpu"]), } - for (host_os, host_arch), want_requirement in sub_tests.items(): + for (host_os, host_arch), (want_requirement, want_extras) in sub_tests.items(): builder = hub_builder( env, ) @@ -1600,7 +1625,7 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' pypi.whl_map().contains_exactly({ "optimum": { "pypi_315_optimum": [ - whl_config_setting(version = "3.15"), + whl_config_setting(version = "3.15", extras = want_extras), ], }, }) @@ -1662,38 +1687,36 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' pypi.exposed_packages().contains_exactly(["optimum"]) pypi.group_map().contains_exactly({}) + + # The same distribution is requested with different extras on different + # platforms, so a single repository has all of the extras and the hub + # selects the extras per platform. pypi.whl_map().contains_exactly({ "optimum": { - "pypi_315_optimum_mylinuxx86_64": [ + "pypi_315_optimum": [ whl_config_setting( version = "3.15", target_platforms = [ "cp315_mylinuxx86_64", ], + extras = ["onnxruntime_gpu"], ), - ], - "pypi_315_optimum_myosxaarch64": [ whl_config_setting( version = "3.15", target_platforms = [ "cp315_myosxaarch64", ], + extras = ["onnxruntime"], ), ], }, }) pypi.whl_libraries().contains_exactly({ - "pypi_315_optimum_mylinuxx86_64": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", - "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "optimum[onnxruntime-gpu]==1.17.1", - }, - "pypi_315_optimum_myosxaarch64": { + "pypi_315_optimum": { "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "optimum[onnxruntime]==1.17.1", + "requirement": "optimum==1.17.1", }, }) pypi.extra_aliases().contains_exactly({}) @@ -1751,7 +1774,7 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' pypi.whl_map().contains_exactly({ "optimum": { "pypi_315_optimum": [ - whl_config_setting(version = "3.15"), + whl_config_setting(version = "3.15", extras = ["onnxruntime_gpu"]), ], }, }) @@ -1760,7 +1783,7 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "optimum[onnxruntime-gpu]==1.17.1", + "requirement": "optimum==1.17.1", }, }) pypi.extra_aliases().contains_exactly({}) @@ -1830,6 +1853,46 @@ Attempting to create a duplicate library pypi_315_foo for foo with different arg _tests.append(_test_err_duplicate_repos) +def _test_groups_with_extras(env): + builder = hub_builder(env) + builder.pip_parse( + _mock_mctx( + os_name = "osx", + arch_name = "aarch64", + mock_files = { + "requirements.txt": """\ +simple==0.0.1 --hash=sha256:deadbeef +other==0.0.1 --hash=sha256:deadbaaf +""", + }, + ), + _parse( + hub_name = "pypi", + python_version = "3.15", + requirements_lock = "requirements.txt", + experimental_requirement_cycles = { + "group": ["Simple[Foo]", "other"], + }, + ), + ) + pypi = builder.build() + + pypi.group_map().contains_exactly({ + "group": ["simple[foo]", "other"], + }) + pypi.whl_libraries().contains_at_least({ + "pypi_315_simple": { + "config_load": "@pypi//:config.bzl", + "dep_template": "@pypi//{name}:{target}", + "group_deps": ["simple[foo]", "other"], + "group_name": "group", + "python_interpreter_target": "unit_test_interpreter_target", + "requirement": "simple==0.0.1 --hash=sha256:deadbeef", + }, + }) + +_tests.append(_test_groups_with_extras) + def hub_builder_test_suite(name): """Create the test suite. diff --git a/tests/pypi/hub_exposure/BUILD.bazel b/tests/pypi/hub_exposure/BUILD.bazel new file mode 100644 index 0000000000..187361e1ce --- /dev/null +++ b/tests/pypi/hub_exposure/BUILD.bazel @@ -0,0 +1,3 @@ +load(":hub_exposure_tests.bzl", "hub_exposure_test_suite") + +hub_exposure_test_suite(name = "hub_exposure_tests") diff --git a/tests/pypi/hub_exposure/hub_exposure_tests.bzl b/tests/pypi/hub_exposure/hub_exposure_tests.bzl new file mode 100644 index 0000000000..eeb57ccdbb --- /dev/null +++ b/tests/pypi/hub_exposure/hub_exposure_tests.bzl @@ -0,0 +1,241 @@ +"""hub_exposure tests""" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:hub_exposure.bzl", "add_extras", "hub_exposure") # buildifier: disable=bzl-visibility +load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # buildifier: disable=bzl-visibility +load("//tests/support/mocks:mocks.bzl", "mocks") + +_tests = [] + +_PYPROJECT = """\ +[project] +dependencies = ["simple"] + +[dependency-groups] +dev = ["other-pkg"] +test = ["simple", "not-in-hub"] +""" + +_EXTRAS_PYPROJECT = """\ +[project] +dependencies = ["simple[foo]", "simple[Bar]; sys_platform == 'linux'"] + +[dependency-groups] +dev = ["other-pkg[baz]", "simple[foo]", "not-in-hub[qux]"] +""" + +def _hub(*, module_name = "root", pyproject = None): + return struct( + module_name = module_name, + pyproject = pyproject, + packages = ["other_pkg", "simple", "transitive_pkg"], + ) + +def _configure(hub_name = "hub", dependency_groups = ["*"], pyproject_toml = None): + return mocks.tag( + hub_name = hub_name, + dependency_groups = dependency_groups, + pyproject_toml = pyproject_toml, + ) + +def _module(name = "root", is_root = True, configure = []): + return mocks.module(name = name, is_root = is_root, configure = configure) + +def _exposure(*modules, hubs = None, default_pyproject = None): + failures = [] + got = hub_exposure( + mocks.mctx( + modules = list(modules), + mock_files = { + "extras.toml": _EXTRAS_PYPROJECT, + "other.toml": "[project]\ndependencies = [\"other-pkg\"]\n", + "pyproject.toml": _PYPROJECT, + }, + ), + hubs = hubs or {"hub": _hub(pyproject = "pyproject.toml")}, + default_pyproject = default_pyproject, + _fail = failures.append, + ) + return got, failures + +def _test_not_configured(env): + got, failures = _exposure(_module()) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got.packages).contains_exactly({}) + +_tests.append(_test_not_configured) + +def _test_configure(env): + for groups, want in [ + (["*"], ["other_pkg", "simple"]), + (["default"], ["simple"]), + (["dev"], ["other_pkg"]), + # `not-in-hub` is ignored + (["group:test"], ["simple"]), + ]: + got, failures = _exposure(_module(configure = [_configure(dependency_groups = groups)])) + env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) + env.expect.where(groups = groups).that_dict(got.packages).contains_exactly({"hub": want}) + +_tests.append(_test_configure) + +def _test_configure_all_packages(env): + # Exposing all of the packages is the same as not configuring the hub, so + # that e.g. the platform-specific packages stay public. + got, failures = _exposure(_module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(dependency_groups = ["*", "transitive"]), + ])) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got.packages).contains_exactly({}) + +_tests.append(_test_configure_all_packages) + +def _test_configure_combines_tags(env): + got, failures = _exposure(_module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(dependency_groups = ["dev"]), + ])) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got.packages).contains_exactly({"hub": ["other_pkg", "simple"]}) + +_tests.append(_test_configure_combines_tags) + +def _test_configure_pyproject_precedence(env): + # The tag wins over the `pip.parse` value. + got, _ = _exposure(_module(configure = [ + _configure(pyproject_toml = "other.toml"), + ])) + env.expect.that_dict(got.packages).contains_exactly({"hub": ["other_pkg"]}) + + # The `pip.default` value is used for the hubs of the root module. + got, failures = _exposure( + _module(configure = [_configure(dependency_groups = ["default"])]), + hubs = {"hub": _hub()}, + default_pyproject = "pyproject.toml", + ) + env.expect.that_collection(failures).contains_exactly([]) + env.expect.that_dict(got.packages).contains_exactly({"hub": ["simple"]}) + +_tests.append(_test_configure_pyproject_precedence) + +def _test_configure_module_precedence(env): + hubs = {"hub": _hub(module_name = "dep", pyproject = "pyproject.toml")} + + # The owner of the hub can configure it. + got, _ = _exposure( + _module(), + _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + hubs = hubs, + ) + env.expect.that_dict(got.packages).contains_exactly({"hub": ["other_pkg"]}) + + # The root module wins over the owner. + got, _ = _exposure( + _module(configure = [_configure(dependency_groups = ["default"])]), + _module(name = "dep", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + hubs = hubs, + ) + env.expect.that_dict(got.packages).contains_exactly({"hub": ["simple"]}) + + # Other modules are ignored. + got, _ = _exposure( + _module(), + _module(name = "other", is_root = False, configure = [_configure(dependency_groups = ["dev"])]), + hubs = hubs, + ) + env.expect.that_dict(got.packages).contains_exactly({}) + +_tests.append(_test_configure_module_precedence) + +def _test_configure_extras(env): + hubs = {"hub": _hub(pyproject = "extras.toml")} + for groups, want in [ + (["default"], {"hub": {"simple": ["bar", "foo"]}}), + # The packages that are not in the hub are ignored. + (["dev"], {"hub": {"other_pkg": ["baz"], "simple": ["foo"]}}), + # The extras are also exposed when all of the packages are exposed. + (["*", "transitive"], {"hub": {"other_pkg": ["baz"], "simple": ["bar", "foo"]}}), + ]: + got, failures = _exposure( + _module(configure = [_configure(dependency_groups = groups)]), + hubs = hubs, + ) + env.expect.where(groups = groups).that_collection(failures).contains_exactly([]) + env.expect.where(groups = groups).that_dict(got.extras).contains_exactly(want) + + # The extras of multiple tags are combined. + got, _ = _exposure( + _module(configure = [ + _configure(dependency_groups = ["default"]), + _configure(dependency_groups = ["dev"]), + ]), + hubs = hubs, + ) + env.expect.that_dict(got.extras).contains_exactly({ + "hub": {"other_pkg": ["baz"], "simple": ["bar", "foo"]}, + }) + + # No extras are requested. + got, _ = _exposure(_module(configure = [_configure()])) + env.expect.that_dict(got.extras).contains_exactly({}) + +_tests.append(_test_configure_extras) + +def _test_add_extras(env): + whl_map = { + "other_pkg": {"repo_other": [whl_config_setting(version = "3.15")]}, + "simple": { + "repo_linux": [whl_config_setting( + version = "3.15", + target_platforms = ["cp315_linux_x86_64"], + extras = ["bar"], + )], + "repo_osx": [whl_config_setting( + version = "3.15", + target_platforms = ["cp315_osx_aarch64"], + )], + }, + } + got = add_extras(whl_map, {"simple": ["bar", "foo"]}) + env.expect.that_dict(got).contains_exactly({ + "other_pkg": {"repo_other": [whl_config_setting(version = "3.15")]}, + "simple": { + "repo_linux": [whl_config_setting( + version = "3.15", + target_platforms = ["cp315_linux_x86_64"], + extras = ["bar", "foo"], + )], + "repo_osx": [whl_config_setting( + version = "3.15", + target_platforms = ["cp315_osx_aarch64"], + extras = ["bar", "foo"], + )], + }, + }) + +_tests.append(_test_add_extras) + +def _test_configure_errors(env): + _, failures = _exposure(_module(configure = [_configure(hub_name = "missing")])) + env.expect.that_collection(failures).contains_exactly([ + "pip.configure() refers to unknown hub 'missing'. Available hubs: hub", + ]) + + _, failures = _exposure( + _module(configure = [_configure()]), + hubs = {"hub": _hub()}, + ) + env.expect.that_collection(failures).contains_exactly([ + "pip.configure() for hub 'hub' requires a pyproject.toml, set `pyproject_toml` on `pip.configure` or `pip.parse`", + ]) + +_tests.append(_test_configure_errors) + +def hub_exposure_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) diff --git a/tests/pypi/labels/BUILD.bazel b/tests/pypi/labels/BUILD.bazel new file mode 100644 index 0000000000..ed55e66eae --- /dev/null +++ b/tests/pypi/labels/BUILD.bazel @@ -0,0 +1,5 @@ +load(":labels_tests.bzl", "labels_test_suite") + +labels_test_suite( + name = "labels_tests", +) diff --git a/tests/pypi/labels/labels_tests.bzl b/tests/pypi/labels/labels_tests.bzl new file mode 100644 index 0000000000..b26e6db471 --- /dev/null +++ b/tests/pypi/labels/labels_tests.bzl @@ -0,0 +1,53 @@ +"" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private/pypi:labels.bzl", "requirement_label", "requirement_labels") # buildifier: disable=bzl-visibility + +_tests = [] + +_TMPL = "@@pypi//{}:{}" + +def _test_requirement_label(env): + for requirement, want in { + "Foo.Bar": "@@pypi//foo_bar:pkg", + "foo[ grpc ]": "@@pypi//foo:[grpc]", + "foo[Async-Rest]": "@@pypi//foo:[async_rest]", + "foo[]": "@@pypi//foo:[]", + "foo[grpc]": "@@pypi//foo:[grpc]", + }.items(): + env.expect.where(requirement = requirement).that_str(requirement_label(_TMPL, requirement)).equals(want) + +_tests.append(_test_requirement_label) + +def _test_requirement_label_multiple_extras_fails(env): + errors = [] + requirement_label( + _TMPL, + "google-api-core[grpc,async_rest]", + fail = lambda msg: errors.append(msg), + ) + env.expect.that_collection(errors).has_size(1) + env.expect.that_str(errors[0]).contains("requirements(\"google-api-core[grpc,async_rest]\")") + +_tests.append(_test_requirement_label_multiple_extras_fails) + +def _test_requirement_labels(env): + env.expect.that_collection( + requirement_labels(_TMPL, "google-api-core[grpc, async_rest]", "foobar[baz]", "foo", "bar[]"), + ).contains_exactly([ + "@@pypi//google_api_core:[async_rest]", + "@@pypi//google_api_core:[grpc]", + "@@pypi//foobar:[baz]", + "@@pypi//foo:pkg", + "@@pypi//bar:[]", + ]).in_order() + +_tests.append(_test_requirement_labels) + +def labels_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) diff --git a/tests/pypi/pep508/deps_tests.bzl b/tests/pypi/pep508/deps_tests.bzl index 0d171d8e95..b7a58a45a0 100644 --- a/tests/pypi/pep508/deps_tests.bzl +++ b/tests/pypi/pep508/deps_tests.bzl @@ -320,6 +320,43 @@ def test_extras_with_hyphens_are_normalized(env): _tests.append(test_extras_with_hyphens_are_normalized) +def test_deps_with_extras(env): + got = deps( + "foo", + requires_dist = [ + "bar[Socks]", + "baz[a,b]; sys_platform == 'linux'", + "qux", + "qux[c]; extra == 'feat'", + ], + extras = ["feat"], + keep_extras = True, + ) + + # The extras of the dependencies are kept so that the targets with the + # extras can be used. + env.expect.that_collection(got.deps).contains_exactly(["bar[socks]", "qux", "qux[c]"]) + env.expect.that_dict(got.deps_select).contains_exactly({ + "baz[a]": "sys_platform == \"linux\"", + "baz[b]": "sys_platform == \"linux\"", + }) + +_tests.append(test_deps_with_extras) + +def test_deps_with_extras_excludes_and_includes(env): + got = deps( + "foo", + requires_dist = ["bar", "bar[socks]", "baz[a]", "baz[b]", "qux[b]"], + # The excludes are dependencies and the includes are packages + excludes = ["bar", "baz[a]"], + include = ["bar", "baz"], + keep_extras = True, + ) + + env.expect.that_collection(got.deps).contains_exactly(["bar[socks]", "baz[b]"]) + +_tests.append(test_deps_with_extras_excludes_and_includes) + def deps_test_suite(name): # buildifier: disable=function-docstring test_suite( name = name, diff --git a/tests/pypi/pep508/requirement_tests.bzl b/tests/pypi/pep508/requirement_tests.bzl index 2ce45922da..c027209aa1 100644 --- a/tests/pypi/pep508/requirement_tests.bzl +++ b/tests/pypi/pep508/requirement_tests.bzl @@ -14,7 +14,7 @@ """Tests for parsing the requirement specifier.""" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private/pypi:pep508_requirement.bzl", "requirement") # buildifier: disable=bzl-visibility +load("//python/private/pypi:pep508_requirement.bzl", "requirement", "strip_extras") # buildifier: disable=bzl-visibility _tests = [] @@ -42,6 +42,19 @@ def _test_requirement_line_parsing(env): _tests.append(_test_requirement_line_parsing) +def _test_strip_extras(env): + for line, want in { + "foo": "foo", + "foo @ https://example.org/foo[1].whl": "foo @ https://example.org/foo[1].whl", + "foo==1.0 ; python_version < '3.12' --hash=sha256:deadbeef": "foo==1.0 ; python_version < '3.12' --hash=sha256:deadbeef", + "foo[bar,baz] @ https://example.org/foo.whl": "foo @ https://example.org/foo.whl", + "foo[bar]": "foo", + "foo[bar]==1.0 --hash=sha256:deadbeef": "foo==1.0 --hash=sha256:deadbeef", + }.items(): + env.expect.where(line = line).that_str(strip_extras(line)).equals(want) + +_tests.append(_test_strip_extras) + def requirement_test_suite(name): # buildifier: disable=function-docstring test_suite( name = name, diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index 2d54bd34db..8cd4773612 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -41,7 +41,8 @@ def _test_legacy_aliases(env): # buildifier: disable=unsorted-dict-items want = { "foo": ":pkg", - "pkg": "@repo//:pkg", + "pkg": ":[]", + "[]": "@repo//:pkg", "srcs": "@repo//:srcs", "whl_file": "@repo//:whl_file", "whl": "@repo//:whl", @@ -79,7 +80,8 @@ def _test_config_setting_aliases(env): # buildifier: disable=unsorted-dict-items want = { - "pkg": { + "pkg": ":[]", + "[]": { "//:my_config_setting": "@bar_baz_repo//:pkg", "//conditions:default": "_no_matching_repository", }, @@ -180,7 +182,8 @@ def _test_multiplatform_whl_aliases(env): # buildifier: disable=unsorted-dict-items want = { - "pkg": { + "pkg": ":[]", + "[]": { "//:my_config_setting": "@bzlmod_repo//:pkg", "//_config:is_cp39": "@version_repo//:pkg", "//_config:is_cp39_linux_x86_64": "@version_platform_repo//:pkg", @@ -240,6 +243,10 @@ def _test_group_aliases(env): "name": "extracted_whl_files", "actual": "@repo//:extracted_whl_files", }, + { + "name": "[]", + "actual": "//_groups:my_group_pkg", + }, { "name": "pkg", "actual": "//_groups:my_group_pkg", @@ -308,7 +315,6 @@ def _test_deps_and_aliases(env): want_deps_targets = { "aliases": {}, "dep_template": "//{name}:{target}", - "extras": [], "group_deps": ["bar", "foo"], "group_name": "my_group", "include": [], @@ -521,6 +527,189 @@ def _test_config_settings_exist_legacy(env): _tests.append(_test_config_settings_exist_legacy) +_LINUX = "//_config:is_cp312_linux_x86_64" +_OSX = "//_config:is_cp312_osx_aarch64" +_WINDOWS = "//_config:is_cp312_windows_x86_64" + +def _extras_actual(extras_by_platform): + return { + whl_config_setting( + version = "3.12", + target_platforms = ["cp312_" + platform], + extras = extras, + ): repo + for platform, (repo, extras) in extras_by_platform.items() + } + +def _test_extras_aliases(env): + got = {} + py_library_calls = [] + pkg_aliases( + name = "foo", + actual = _extras_actual({ + "linux_x86_64": ("repo", ["bar"]), + "osx_aarch64": ("repo", ["baz"]), + "windows_x86_64": ("other_repo", []), + }), + native = struct( + alias = lambda *, name, actual, visibility = None, tags = None: got.update({name: actual}), + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + ), + select = lambda value, no_match_error = None: value, + ) + + # buildifier: disable=unsorted-dict-items + want = { + "foo": ":pkg", + "pkg": { + _LINUX: ":[bar]", + _OSX: ":[baz]", + _WINDOWS: ":[]", + # The `[]` target has the no match error. + "//conditions:default": ":[]", + }, + "whl": { + _LINUX: "@repo//:whl", + _OSX: "@repo//:whl", + _WINDOWS: "@other_repo//:whl", + "//conditions:default": "_no_matching_repository", + }, + "[]": { + _LINUX: "@repo//:pkg", + _OSX: "@repo//:pkg", + _WINDOWS: "@other_repo//:pkg", + "//conditions:default": "_no_matching_repository", + }, + "[bar]": { + _LINUX: "@repo//:[bar]", + _OSX: "@repo//:pkg", + _WINDOWS: "@other_repo//:pkg", + "//conditions:default": "_no_matching_repository", + }, + "[baz]": { + _LINUX: "@repo//:pkg", + _OSX: "@repo//:[baz]", + _WINDOWS: "@other_repo//:pkg", + "//conditions:default": "_no_matching_repository", + }, + } + env.expect.that_dict(got).contains_at_least(want) + env.expect.that_collection(py_library_calls).contains_exactly([]) + +_tests.append(_test_extras_aliases) + +def _test_extras_aliases_union(env): + got = {} + py_library_calls = [] + pkg_aliases( + name = "foo", + actual = _extras_actual({ + "linux_x86_64": ("repo", ["a", "b"]), + "osx_aarch64": ("repo", ["c"]), + "windows_x86_64": ("repo", []), + }), + native = struct( + alias = lambda *, name, actual, visibility = None, tags = None: got.update({name: actual}), + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + ), + select = lambda value, no_match_error = None: value, + ) + + # buildifier: disable=unsorted-dict-items + env.expect.that_collection(py_library_calls).contains_exactly([{ + "name": "pkg", + "deps": { + _LINUX: [":[a]", ":[b]"], + _OSX: [":[c]"], + _WINDOWS: [":[]"], + "//conditions:default": [":[]"], + }, + }]) + env.expect.that_collection(got.keys()).not_contains("pkg") + +_tests.append(_test_extras_aliases_union) + +def _test_extras_aliases_group(env): + got = {} + got_visibility = {} + + def _alias(*, name, actual, visibility = None, **_): + got[name] = actual + if visibility: + got_visibility[name] = visibility + + pkg_aliases( + name = "foo", + actual = _extras_actual({ + "linux_x86_64": ("repo", ["bar"]), + "osx_aarch64": ("repo", []), + }), + group_name = "group", + native = struct(alias = _alias), + select = lambda value, no_match_error = None: value, + ) + + # The package without extras is the member of the group by default, the + # `[bar]` extra is not a member of the group. + # buildifier: disable=unsorted-dict-items + env.expect.that_dict(got).contains_at_least({ + "pkg": { + _LINUX: ":[bar]", + _OSX: "//_groups:group_pkg", + "//conditions:default": "//_groups:group_pkg", + }, + "[]": "//_groups:group_pkg", + "_pkg": { + _LINUX: "@repo//:pkg", + _OSX: "@repo//:pkg", + "//conditions:default": "_no_matching_repository", + }, + "[bar]": { + _LINUX: "@repo//:[bar]", + _OSX: "@repo//:pkg", + "//conditions:default": "_no_matching_repository", + }, + }) + env.expect.that_dict(got_visibility).contains_at_least({ + "_pkg": ["//_groups:__subpackages__"], + }) + +_tests.append(_test_extras_aliases_group) + +def _test_extras_aliases_group_with_extras(env): + got = {} + + # This is how `WORKSPACE` uses a single repo for each package. + pkg_aliases( + name = "foo", + actual = "repo", + extras = ["bar"], + group_name = "group", + # Only the `bar` extra is the member of the group. + group_extras = ["bar"], + native = struct( + alias = lambda *, name, actual, visibility = None, tags = None: got.update({name: actual}), + ), + select = lambda value, no_match_error = None: value, + ) + + # buildifier: disable=unsorted-dict-items + env.expect.that_dict(got).contains_at_least({ + "pkg": "//_groups:group_pkg", + "[]": "@repo//:pkg", + "[bar]": "//_groups:group_pkg", + "_[bar]": "@repo//:[bar]", + "whl": "@repo//:whl", + }) + env.expect.that_collection(got.keys()).not_contains("_pkg") + env.expect.that_collection(got.keys()).not_contains("_whl") + +_tests.append(_test_extras_aliases_group_with_extras) + def pkg_aliases_test_suite(name): """Create the test suite. diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index 9114f59279..4e67a51de5 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -17,6 +17,7 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load( "//python/private/pypi:render_pkg_aliases.bzl", + "available_extras", "get_whl_flag_versions", "render_multiplatform_pkg_aliases", "render_pkg_aliases", @@ -58,6 +59,68 @@ pkg_aliases( _tests.append(_test_legacy_aliases) +def _test_hidden_aliases(env): + actual = render_pkg_aliases( + aliases = { + "bar": "pypi_bar", + "foo": "pypi_foo", + }, + exposed_packages = ["foo"], + ) + + env.expect.that_str(actual["bar/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = [ + "@pypi_bar//:__pkg__", + "@pypi_foo//:__pkg__", +]) + +pkg_aliases( + name = "bar", + actual = "pypi_bar", +)""") + env.expect.that_str(actual["foo/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = ["//visibility:public"]) + +pkg_aliases( + name = "foo", + actual = "pypi_foo", +)""") + +_tests.append(_test_hidden_aliases) + +def _test_hidden_aliases_in_groups(env): + actual = render_pkg_aliases( + aliases = { + "bar": "pypi_bar", + "baz": "pypi_baz", + }, + requirement_cycles = { + "group": ["bar", "baz"], + }, + exposed_packages = ["baz"], + ) + + env.expect.that_str(actual["bar/BUILD.bazel"]).equals("""\ +load("@rules_python//python/private/pypi:pkg_aliases.bzl", "pkg_aliases") + +package(default_visibility = [ + "@pypi_bar//:__pkg__", + "@pypi_baz//:__pkg__", + "//_groups:__pkg__", +]) + +pkg_aliases( + name = "bar", + actual = "pypi_bar", + group_name = "group", +)""") + +_tests.append(_test_hidden_aliases_in_groups) + def _test_bzlmod_aliases(env): # Use this function as it is used in pip_repository actual = render_multiplatform_pkg_aliases( @@ -247,6 +310,85 @@ def _test_get_flag_versions_from_alias_target_platforms(env): _tests.append(_test_get_flag_versions_from_alias_target_platforms) +def _test_aliases_with_extras(env): + actual = render_pkg_aliases( + aliases = { + "foo": { + whl_config_setting(version = "3.2", extras = ["b", "a"]): "pypi_32_foo", + }, + }, + ) + + env.expect.that_str(actual["foo/BUILD.bazel"]).contains("""\ +pkg_aliases( + name = "foo", + actual = { + whl_config_setting( + extras = ("a", "b"), + version = "3.2", + ): "pypi_32_foo", + }, +)""") + +_tests.append(_test_aliases_with_extras) + +def _test_available_extras(env): + got = available_extras({ + "bar": { + whl_config_setting(version = "3.2"): "pypi_32_bar", + }, + "foo": { + whl_config_setting(version = "3.2", target_platforms = ["cp32_linux_x86_64"], extras = ["b"]): "pypi_32_foo", + whl_config_setting(version = "3.2", target_platforms = ["cp32_osx_aarch64"], extras = ["a", "b"]): "pypi_32_foo", + }, + }) + + env.expect.that_dict(got).contains_exactly({ + "bar": [], + "foo": ["a", "b"], + }) + +_tests.append(_test_available_extras) + +def _test_aliases_with_extras_for_repo_names(env): + actual = render_pkg_aliases( + aliases = { + "foo": "pypi_foo", + }, + extras = {"foo": ["a"]}, + ) + + env.expect.that_str(actual["foo/BUILD.bazel"]).contains("""\ +pkg_aliases( + name = "foo", + actual = "pypi_foo", + extras = ["a"], +)""") + +_tests.append(_test_aliases_with_extras_for_repo_names) + +def _test_groups_with_extras(env): + actual = render_pkg_aliases( + aliases = { + "bar": {whl_config_setting(version = "3.1"): "pypi_31_bar"}, + "foo": {whl_config_setting(version = "3.1"): "pypi_31_foo"}, + }, + requirement_cycles = { + "group": ["Bar", "foo[Extra]"], + }, + ) + + env.expect.that_str(actual["foo/BUILD.bazel"]).contains("""\ + group_name = "group", + group_extras = ["extra"], +""") + env.expect.that_str(actual["bar/BUILD.bazel"]).contains("""\ + group_name = "group", +""") + env.expect.that_bool("group_extras" in actual["bar/BUILD.bazel"]).equals(False) + +_tests.append(_test_groups_with_extras) + def render_pkg_aliases_test_suite(name): """Create the test suite. diff --git a/tests/pypi/whl_library/BUILD.bazel b/tests/pypi/whl_library/BUILD.bazel index cade0d2b8e..af2c4c24bf 100644 --- a/tests/pypi/whl_library/BUILD.bazel +++ b/tests/pypi/whl_library/BUILD.bazel @@ -6,6 +6,16 @@ py_test( srcs = ["whl_library_extras_test.py"], target_compatible_with = SUPPORTS_BZLMOD, deps = [ - "@whl_library_extras_direct_dep//:pkg", + "@whl_library_extras_direct_dep//:[optional]", + ], +) + +py_test( + name = "whl_library_transitive_extras_test", + srcs = ["whl_library_transitive_extras_test.py"], + target_compatible_with = SUPPORTS_BZLMOD, + deps = [ + # The METADATA of the consumer requires `pkg[optional]`. + "@whl_library_extras_consumer//:pkg", ], ) diff --git a/tests/pypi/whl_library/testdata/consumer/BUILD.bazel b/tests/pypi/whl_library/testdata/consumer/BUILD.bazel new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA b/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA new file mode 100644 index 0000000000..9a723c958a --- /dev/null +++ b/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/METADATA @@ -0,0 +1,3 @@ +Name: consumer +Version: 1.0 +Requires-Dist: pkg[optional] diff --git a/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/RECORD b/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/RECORD new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/WHEEL b/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/WHEEL new file mode 100644 index 0000000000..a64521a1cc --- /dev/null +++ b/tests/pypi/whl_library/testdata/consumer/consumer-1.0.dist-info/WHEEL @@ -0,0 +1 @@ +Wheel-Version: 1.0 diff --git a/tests/pypi/whl_library/testdata/consumer/consumer.py b/tests/pypi/whl_library/testdata/consumer/consumer.py new file mode 100644 index 0000000000..20da50d156 --- /dev/null +++ b/tests/pypi/whl_library/testdata/consumer/consumer.py @@ -0,0 +1,3 @@ +import pkg + +WITH_EXTRAS = pkg.WITH_EXTRAS diff --git a/tests/pypi/whl_library/testdata/packages.bzl b/tests/pypi/whl_library/testdata/packages.bzl index e4a9b0af4c..fd12c500ca 100644 --- a/tests/pypi/whl_library/testdata/packages.bzl +++ b/tests/pypi/whl_library/testdata/packages.bzl @@ -1,6 +1,7 @@ """A list of packages that this logical testdata hub repo contains.""" packages = [ + "consumer", "optional_dep", "pkg", ] diff --git a/tests/pypi/whl_library/whl_library_transitive_extras_test.py b/tests/pypi/whl_library/whl_library_transitive_extras_test.py new file mode 100644 index 0000000000..9aa7a80f04 --- /dev/null +++ b/tests/pypi/whl_library/whl_library_transitive_extras_test.py @@ -0,0 +1,12 @@ +import unittest + + +class TransitiveExtrasTest(unittest.TestCase): + def test_extras_from_metadata_propagated(self): + import consumer + + self.assertEqual(consumer.WITH_EXTRAS, True) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl index e8659ff642..1c520c148a 100644 --- a/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl +++ b/tests/pypi/whl_library_deps_targets/whl_library_deps_targets_tests.bzl @@ -33,7 +33,6 @@ def _test_whl_library_deps_targets(env): # Overrides for testing repo = None, aliases = None, - extras = [], native = struct( filegroup = lambda **kwargs: filegroup_calls.append(kwargs), alias = lambda **kwargs: None, @@ -102,10 +101,10 @@ def _test_whl_library_deps_targets_no_deps(env): metadata_name = "Foo", dep_template = "@pypi//{name}:{target}", requires_dist = [], + provides_extra = ["A"], group_name = "qux", repo = None, aliases = {}, - extras = [], native = struct( filegroup = lambda **kwargs: filegroup_calls.append(kwargs), alias = lambda **kwargs: alias_calls.append(kwargs), @@ -131,6 +130,11 @@ def _test_whl_library_deps_targets_no_deps(env): "actual": "whl_file", "visibility": ["//visibility:public"], }, + { + "name": "[a]", + "actual": "srcs", + "visibility": ["//visibility:public"], + }, ]) # buildifier: @unsorted-dict-items env.expect.that_collection(filegroup_calls).contains_exactly([]) @@ -139,6 +143,247 @@ def _test_whl_library_deps_targets_no_deps(env): _tests.append(_test_whl_library_deps_targets_no_deps) +def _test_extras(env): + filegroup_calls = [] + py_library_calls = [] + env_marker_setting_calls = [] + + whl_library_deps_targets( + name = "foo-0-py3-none-any.whl", + metadata_name = "Foo", + dep_template = "@pypi//{name}:{target}", + requires_dist = [ + "bar[socks]", + "grpc; extra == 'grpc'", + "aiohttp; extra == 'async' and python_version < '3.12'", + "uvloop; extra == 'async'", + "foo[grpc]; extra == 'all'", + ], + provides_extra = ["async", "grpc", "all"], + include = ["bar", "grpc", "aiohttp", "uvloop"], + repo = None, + aliases = {}, + native = struct( + filegroup = lambda **kwargs: filegroup_calls.append(kwargs), + alias = lambda **kwargs: None, + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + env_marker_setting = lambda **kwargs: env_marker_setting_calls.append(kwargs), + ), + ) + + # The `pkg` target has no extras because none were requested and there is + # a target for each extra that the package provides. The `whl` target does + # not use the extras. Without the extras available in the hub, the + # dependencies use the targets with the extras directly. + env.expect.that_collection([ + (c["name"], c["deps"]) + for c in py_library_calls + ]).contains_exactly([ + ("pkg", ["srcs", "@pypi//bar:[socks]"]), + ("[all]", ["srcs", "@pypi//bar:[socks]", "@pypi//grpc:pkg"]), + ("[async]", ["srcs", "@pypi//bar:[socks]", "@pypi//uvloop:pkg"] + select({ + ":is_include_aiohttp_true": ["@pypi//aiohttp:pkg"], + "//conditions:default": [], + })), + ("[grpc]", ["srcs", "@pypi//bar:[socks]", "@pypi//grpc:pkg"]), + ]) + env.expect.that_collection([ + (c["name"], c["data"]) + for c in filegroup_calls + ]).contains_exactly([ + ("whl", ["@pypi//bar:whl"]), + ]) + env.expect.that_collection(env_marker_setting_calls).contains_exactly([ + { + "name": "include_aiohttp", + "expression": "python_version < \"3.12\"", + "visibility": ["//visibility:private"], + }, + ]) # buildifier: @unsorted-dict-items + +_tests.append(_test_extras) + +def _test_extras_in_group(env): + py_library_calls = [] + filegroup_calls = [] + + whl_library_deps_targets( + name = "foo-0-py3-none-any.whl", + metadata_name = "foo", + dep_template = "@pypi//{name}:{target}", + requires_dist = ["baz", "bar; extra == 'a'", "qux; extra == 'a'"], + provides_extra = ["a"], + group_name = "group", + # Only the `a` extra is in a cycle with `qux`. + group_deps = ["Foo[A]", "qux"], + repo = None, + aliases = {}, + native = struct( + filegroup = lambda **kwargs: filegroup_calls.append(kwargs), + alias = lambda **kwargs: None, + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + env_marker_setting = lambda **kwargs: None, + ), + ) + + # Only the member targets exclude the other members and the targets are + # visible to the hub that creates the group library. + env.expect.that_collection([ + (c["name"], c["deps"], c["visibility"]) + for c in py_library_calls + ]).contains_exactly([ + ("pkg", ["srcs", "@pypi//baz:pkg"], ["@pypi//:__subpackages__"]), + ("[a]", ["srcs", "@pypi//bar:pkg", "@pypi//baz:pkg"], ["@pypi//:__subpackages__"]), + ]) + env.expect.that_collection([ + (c["name"], c["data"]) + for c in filegroup_calls + ]).contains_exactly([ + ("whl", ["@pypi//baz:whl"]), + ]) + +_tests.append(_test_extras_in_group) + +def _test_extras_in_group_legacy(env): + alias_calls = [] + py_library_calls = [] + filegroup_calls = [] + + whl_library_deps_targets( + name = "foo-0-py3-none-any.whl", + metadata_name = "foo", + dep_template = "@pip_{name}//:{target}", + requires_dist = ["qux", "bar; extra == 'a'"], + provides_extra = ["a", "b"], + group_name = "group", + group_deps = ["foo", "foo[a]", "qux"], + repo = None, + aliases = {}, + native = struct( + filegroup = lambda **kwargs: filegroup_calls.append(kwargs), + alias = lambda **kwargs: alias_calls.append(kwargs), + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + env_marker_setting = lambda **kwargs: None, + ), + ) + + # The member targets are implementation targets and the public targets + # point to the group library in the config repo. + env.expect.that_collection([ + (c["name"], c["actual"]) + for c in alias_calls + ]).contains_exactly([ + ("pkg", "@pip__config//_groups:group_pkg"), + ("[a]", "@pip__config//_groups:group_pkg"), + ("whl", "@pip__config//_groups:group_whl"), + ]) + env.expect.that_collection([ + (c["name"], c["deps"], c["visibility"]) + for c in py_library_calls + ]).contains_exactly([ + ("_pkg", ["srcs"], ["@pip__config//_groups:__pkg__"]), + ("_[a]", ["srcs", "@pip_bar//:pkg"], ["@pip__config//_groups:__pkg__"]), + ("[b]", ["srcs", "@pip_qux//:pkg"], ["//visibility:public"]), + ]) + env.expect.that_collection([ + (c["name"], c["data"], c["visibility"]) + for c in filegroup_calls + ]).contains_exactly([ + ("_whl", [], ["@pip__config//_groups:__pkg__"]), + ]) + +_tests.append(_test_extras_in_group_legacy) + +def _test_deps_with_extras(env): + filegroup_calls = [] + py_library_calls = [] + env_marker_setting_calls = [] + missing_calls = [] + + whl_library_deps_targets( + name = "foo-0-py3-none-any.whl", + metadata_name = "foo", + dep_template = "@pypi//{name}:{target}", + requires_dist = [ + "bar[socks]", + "baz[a,b]; sys_platform == 'linux'", + "baz; sys_platform == 'darwin'", + "qux[missing]", + ], + # The values are the extras that are available in the hub + include = { + "bar": ["socks"], + "baz": ["a", "b"], + "qux": [], + }, + repo = None, + aliases = {}, + native = struct( + filegroup = lambda **kwargs: filegroup_calls.append(kwargs), + alias = lambda **kwargs: None, + ), + rules = struct( + py_library = lambda **kwargs: py_library_calls.append(kwargs), + env_marker_setting = lambda **kwargs: env_marker_setting_calls.append(kwargs), + missing_extra_error = lambda **kwargs: missing_calls.append(kwargs), + ), + ) + + # The dependencies use the targets with the extras, the missing extras use + # a target that fails the build with a helpful message. + env.expect.that_collection([ + (c["name"], c["deps"]) + for c in py_library_calls + ]).contains_exactly([ + ("pkg", ["srcs", "@pypi//bar:[socks]", ":_missing_qux[missing]"] + select({ + ":is_include_baz_2_true": ["@pypi//baz:pkg"], + "//conditions:default": [], + }) + select({ + ":is_include_baz[a]_true": ["@pypi//baz:[a]"], + "//conditions:default": [], + }) + select({ + ":is_include_baz[b]_true": ["@pypi//baz:[b]"], + "//conditions:default": [], + })), + ]) + env.expect.that_collection(missing_calls).contains_exactly([ + { + "name": "_missing_qux[missing]", + "package_name": "qux", + "extra": "missing", + "required_by": "foo", + "visibility": ["//visibility:private"], + }, + ]) # buildifier: @unsorted-dict-items + + # The whl targets do not use the extras. + env.expect.that_collection([ + (c["name"], c["data"]) + for c in filegroup_calls + ]).contains_exactly([ + ("whl", ["@pypi//bar:whl", "@pypi//qux:whl"] + select({ + ":is_include_baz_true": ["@pypi//baz:whl"], + "//conditions:default": [], + })), + ]) + env.expect.that_collection([ + (c["name"], c["expression"]) + for c in env_marker_setting_calls + ]).contains_exactly([ + ("include_baz", "(sys_platform == \"darwin\") or (sys_platform == \"linux\")"), + ("include_baz_2", "sys_platform == \"darwin\""), + ("include_baz[a]", "sys_platform == \"linux\""), + ("include_baz[b]", "sys_platform == \"linux\""), + ]) + +_tests.append(_test_deps_with_extras) + def whl_library_deps_targets_test_suite(name): """create the test suite. diff --git a/tests/pypi/whl_metadata/whl_metadata_tests.bzl b/tests/pypi/whl_metadata/whl_metadata_tests.bzl index 8131b0f452..8aca204146 100644 --- a/tests/pypi/whl_metadata/whl_metadata_tests.bzl +++ b/tests/pypi/whl_metadata/whl_metadata_tests.bzl @@ -246,6 +246,27 @@ FOOBAR = guimod:main _tests.append(_test_parse_entry_points_deduplicate) +def _test_parse_metadata_crlf(env): + got = _parse_whl_metadata( + env, + contents = "\r\n".join([ + "Name: foo", + "Version: 0.0.1", + "Requires-Dist: bar; extra == \"all\"", + "Provides-Extra: all", + "", + "Requires-Dist: this will be ignored", + ]), + ) + got.name().equals("foo") + got.version().equals("0.0.1") + got.requires_dist().contains_exactly([ + "bar; extra == \"all\"", + ]) + got.provides_extra().contains_exactly(["all"]) + +_tests.append(_test_parse_metadata_crlf) + def whl_metadata_test_suite(name): # buildifier: disable=function-docstring test_suite( name = name,