From 341e55fb266790b7e2391c26759093b6045ce0b5 Mon Sep 17 00:00:00 2001 From: DN6 Date: Wed, 26 Aug 2026 10:34:36 +0530 Subject: [PATCH 01/24] [CI] Rewrite `tests_fetcher.py` as an AST-based import-graph selector with feature-split matrix output --- .github/workflows/pr_test_fetcher.yml | 32 +- utils/tests_fetcher.py | 1387 ++++++++----------------- 2 files changed, 424 insertions(+), 995 deletions(-) diff --git a/.github/workflows/pr_test_fetcher.yml b/.github/workflows/pr_test_fetcher.yml index 17789ec8a9cd..aef711f01cb2 100644 --- a/.github/workflows/pr_test_fetcher.yml +++ b/.github/workflows/pr_test_fetcher.yml @@ -28,7 +28,6 @@ jobs: shell: bash outputs: matrix: ${{ steps.set_matrix.outputs.matrix }} - test_map: ${{ steps.set_matrix.outputs.test_map }} steps: - name: Checkout diffusers uses: actions/checkout@v6 @@ -51,31 +50,26 @@ jobs: path: test_preparation.txt - id: set_matrix name: Create Test Matrix - # The `keys` is used as GitHub actions matrix for jobs, i.e. `models`, `pipelines`, etc. - # The `test_map` is used to get the actual identified test files under each key. - # If no test to run (so no `test_map.json` file), create a dummy map (empty matrix will fail) + # `test_map.json` is a list of matrix entries `{"name", "paths", "markers"}`, one job each. + # If no test to run (so no `test_map.json` file), emit an empty list and skip the test job. run: | if [ -f test_map.json ]; then - keys=$(python3 -c 'import json; fp = open("test_map.json"); test_map = json.load(fp); fp.close(); d = list(test_map.keys()); print(json.dumps(d))') - test_map=$(python3 -c 'import json; fp = open("test_map.json"); test_map = json.load(fp); fp.close(); print(json.dumps(test_map))') + matrix=$(python3 -c 'import json; print(json.dumps(json.load(open("test_map.json"))))') else - keys=$(python3 -c 'keys = ["dummy"]; print(keys)') - test_map=$(python3 -c 'test_map = {"dummy": []}; print(test_map)') + matrix='[]' fi - echo $keys - echo $test_map - echo "matrix=$keys" >> $GITHUB_OUTPUT - echo "test_map=$test_map" >> $GITHUB_OUTPUT + echo "matrix=$matrix" >> $GITHUB_OUTPUT + echo "Matrix: $matrix" run_pr_tests: - name: Run PR Tests + name: ${{ matrix.name }} needs: setup_pr_tests - if: contains(fromJson(needs.setup_pr_tests.outputs.matrix), 'dummy') != true + if: needs.setup_pr_tests.outputs.matrix != '[]' strategy: fail-fast: false max-parallel: 2 matrix: - modules: ${{ fromJson(needs.setup_pr_tests.outputs.matrix) }} + include: ${{ fromJson(needs.setup_pr_tests.outputs.matrix) }} runs-on: group: aws-general-8-plus container: @@ -101,20 +95,20 @@ jobs: - name: Run all selected tests on CPU run: | - pytest -n 2 --dist=loadfile -v --make-reports=${{ matrix.modules }}_tests_cpu ${{ fromJson(needs.setup_pr_tests.outputs.test_map)[matrix.modules] }} + pytest -n 2 --dist=loadfile -v -m "${{ matrix.markers }}" --make-reports=${{ matrix.name }}_tests_cpu ${{ matrix.paths }} - name: Failure short reports if: ${{ failure() }} continue-on-error: true run: | - cat reports/${{ matrix.modules }}_tests_cpu_stats.txt - cat reports/${{ matrix.modules }}_tests_cpu_failures_short.txt + cat reports/${{ matrix.name }}_tests_cpu_stats.txt + cat reports/${{ matrix.name }}_tests_cpu_failures_short.txt - name: Test suite reports artifacts if: ${{ always() }} uses: actions/upload-artifact@v6 with: - name: ${{ matrix.modules }}_test_reports + name: ${{ matrix.name }}_test_reports path: reports run_staging_tests: diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index d487efb40518..8afa2994fa44 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -14,1115 +14,550 @@ # limitations under the License. """ -Welcome to tests_fetcher V2. - -This util is designed to fetch tests to run on a PR so that only the tests impacted by the modifications are run, and -when too many models are being impacted, only run the tests of a subset of core models. It works like this. - -Stage 1: Identify the modified files. For jobs that run on the main branch, it's just the diff with the last commit. -On a PR, this takes all the files from the branching point to the current commit (so all modifications in a PR, not -just the last commit) but excludes modifications that are on docstrings or comments only. - -Stage 2: Extract the tests to run. This is done by looking at the imports in each module and test file: if module A -imports module B, then changing module B impacts module A, so the tests using module A should be run. We thus get the -dependencies of each model and then recursively builds the 'reverse' map of dependencies to get all modules and tests -impacted by a given file. We then only keep the tests (and only the core models tests if there are too many modules). - -Caveats: - - This module only filters tests by files (not individual tests) so it's better to have tests for different things - in different files. - - This module assumes inits are just importing things, not really building objects, so it's better to structure - them this way and move objects building in separate submodules. +Diffusers tests_fetcher (graph-based). + +For each PR, walk the AST of every modified Python file to extract diffusers-internal imports, build a +forward dependency graph for the repo, invert it to a reverse map (file → tests transitively depending on +it), and select the impacted tests. + +There is no automatic full-suite trigger. If a change is in territory the import graph can't see +correctly (dynamic dispatch via auto-mappings, lazy `_import_structure`, etc.), apply the `run-all-tests` +PR label or pass `--force_full_suite` to bypass selection. + +Pipeline-specific note: diffusers' `__init__.py` files use the `_import_structure = {...}` lazy-loading +pattern paired with an `if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT:` block containing real +`from .submodule import Class` statements. AST extraction sees the TYPE_CHECKING imports (since +`ast.walk` descends into `If` / `Try` blocks regardless of runtime conditions), so the import graph +mirrors the actual public API. + +Stage 1 — diff: list modified Python files (vs. merge-base with main, or the previous commit on main). + Docstring/comment-only changes are filtered out by content comparison. +Stage 2 — graph: parse every .py under `src/diffusers/` and `tests/` with `ast`, build the forward + dependency map, transitively close it, then invert to the reverse map. +Stage 3 — select: for each modified file, look up `reverse_map[file]` to get impacted tests. +Stage 4 — bucket: group tests by top-level `tests/` folder for the CI matrix. `tests/models` and + `tests/pipelines` are split further into one job per feature mixin (via the pytest markers the + mixins carry) so a large selection fans out instead of serialising in one job. Usage: -Base use to fetch the tests in a pull request - ```bash -python utils/tests_fetcher.py -``` - -Base use to fetch the tests on a the main branch (with diff from the last commit): - -```bash -python utils/tests_fetcher.py --diff_with_last_commit +python utils/tests_fetcher.py # PR mode: diff against main +python utils/tests_fetcher.py --diff_with_last_commit # main mode: diff against last commit +python utils/tests_fetcher.py --force_full_suite # bypass selection, run everything ``` """ import argparse +import ast import collections import json -import os -import re from contextlib import contextmanager from pathlib import Path -from typing import Dict, List, Tuple, Union +from typing import Dict, List, Optional, Tuple from git import Repo PATH_TO_REPO = Path(__file__).parent.parent.resolve() -PATH_TO_EXAMPLES = PATH_TO_REPO / "examples" PATH_TO_DIFFUSERS = PATH_TO_REPO / "src/diffusers" PATH_TO_TESTS = PATH_TO_REPO / "tests" -# Ignore fixtures in tests folder -# Ignore lora since they are always tested -MODULES_TO_IGNORE = ["fixtures", "lora"] - -IMPORTANT_PIPELINES = [ - "controlnet", - "stable_diffusion", - "stable_diffusion_2", - "stable_diffusion_xl", - "stable_video_diffusion", - "deepfloyd_if", - "kandinsky", - "kandinsky2_2", - "text_to_video_synthesis", - "wuerstchen", -] +# `tests/models` and `tests/pipelines` compose one test class per feature mixin, and each mixin carries a +# pytest marker (see the `is_*` decorators in tests/testing_utils.py). Each group below becomes its own +# matrix job, selected with `pytest -m `, but only when a selected file composes a mixin carrying +# one of the group's markers; `core` is everything no group and no CPU-skipped marker claims. +SPLIT_BY_FEATURE = {"models", "pipelines"} +FEATURE_GROUPS = { + "lora": ["lora"], + "attention": ["attention"], + "memory": ["memory", "cpu_offload", "group_offload"], + "cache": ["cache"], + "ip_adapter": ["ip_adapter"], +} +# Gated on an accelerator, multi-GPU, or an HF token, so every test skips on the CPU runner. Excluded +# from `core` and given no job rather than spinning up a runner to skip everything. +CPU_SKIPPED_MARKERS = ["quantization", "compile", "single_file", "training", "context_parallel", "tensor_parallel"] +FEATURE_MARKERS = [m for markers in FEATURE_GROUPS.values() for m in markers] +CORE_MARKERS = "not (" + " or ".join(FEATURE_MARKERS + CPU_SKIPPED_MARKERS) + ")" +# ============================================================ +# Generic helpers +# ============================================================ @contextmanager def checkout_commit(repo: Repo, commit_id: str): - """ - Context manager that checks out a given commit when entered, but gets back to the reference it was at on exit. - - Args: - repo (`git.Repo`): A git repository (for instance the Transformers repo). - commit_id (`str`): The commit reference to checkout inside the context manager. - """ + """Check out `commit_id` for the duration of the block, restoring the prior HEAD on exit.""" current_head = repo.head.commit if repo.head.is_detached else repo.head.ref - try: repo.git.checkout(commit_id) yield - finally: repo.git.checkout(current_head) -def clean_code(content: str) -> str: - """ - Remove docstrings, empty line or comments from some code (used to detect if a diff is real or only concern - comments or docstrings). - - Args: - content (`str`): The code to clean - - Returns: - `str`: The cleaned code. - """ - # We need to deactivate autoformatting here to write escaped triple quotes (we cannot use real triple quotes or - # this would mess up the result if this function applied to this particular file). - # fmt: off - # Remove docstrings by splitting on triple " then triple ': - splits = content.split('\"\"\"') - content = "".join(splits[::2]) - splits = content.split("\'\'\'") - # fmt: on - content = "".join(splits[::2]) - - # Remove empty lines and comments - lines_to_keep = [] - for line in content.split("\n"): - # remove anything that is after a # sign. - line = re.sub("#.*$", "", line) - # remove white lines - if len(line) != 0 and not line.isspace(): - lines_to_keep.append(line) - return "\n".join(lines_to_keep) - - -def keep_doc_examples_only(content: str) -> str: - """ - Remove everything from the code content except the doc examples (used to determined if a diff should trigger doc - tests or not). - - Args: - content (`str`): The code to clean - - Returns: - `str`: The cleaned code. - """ - # Keep doc examples only by splitting on triple "`" - splits = content.split("```") - # Add leading and trailing "```" so the navigation is easier when compared to the original input `content` - content = "```" + "```".join(splits[1::2]) + "```" - - # Remove empty lines and comments - lines_to_keep = [] - for line in content.split("\n"): - # remove anything that is after a # sign. - line = re.sub("#.*$", "", line) - # remove white lines - if len(line) != 0 and not line.isspace(): - lines_to_keep.append(line) - return "\n".join(lines_to_keep) - - -def get_all_tests() -> List[str]: - """ - Walks the `tests` folder to return a list of files/subfolders. This is used to split the tests to run when using - parallelism. The split is: +# ============================================================ +# Diff detection +# ============================================================ - - folders under `tests`: (`tokenization`, `pipelines`, etc) except the subfolder `models` is excluded. - - folders under `tests/models`: `bert`, `gpt2`, etc. - - test files under `tests`: `test_modeling_common.py`, `test_tokenization_common.py`, etc. - """ - - # test folders/files directly under `tests` folder - tests = os.listdir(PATH_TO_TESTS) - tests = [f"tests/{f}" for f in tests if "__pycache__" not in f] - tests = sorted([f for f in tests if (PATH_TO_REPO / f).is_dir() or f.startswith("tests/test_")]) - - return tests - - -def diff_is_docstring_only(repo: Repo, branching_point: str, filename: str) -> bool: - """ - Check if the diff is only in docstrings (or comments and whitespace) in a filename. - - Args: - repo (`git.Repo`): A git repository (for instance the Transformers repo). - branching_point (`str`): The commit reference of where to compare for the diff. - filename (`str`): The filename where we want to know if the diff isonly in docstrings/comments. - - Returns: - `bool`: Whether the diff is docstring/comments only or not. - """ - folder = Path(repo.working_dir) - with checkout_commit(repo, branching_point): - with open(folder / filename, "r", encoding="utf-8") as f: - old_content = f.read() - - with open(folder / filename, "r", encoding="utf-8") as f: - new_content = f.read() - - old_content_clean = clean_code(old_content) - new_content_clean = clean_code(new_content) - - return old_content_clean == new_content_clean - - -def diff_contains_doc_examples(repo: Repo, branching_point: str, filename: str) -> bool: - """ - Check if the diff is only in code examples of the doc in a filename. - Args: - repo (`git.Repo`): A git repository (for instance the Transformers repo). - branching_point (`str`): The commit reference of where to compare for the diff. - filename (`str`): The filename where we want to know if the diff is only in codes examples. +def _strip_comments_and_docstrings(source: str) -> str: + """Return source with all docstrings and comments removed via AST round-trip. - Returns: - `bool`: Whether the diff is only in code examples of the doc or not. + Used by `diff_is_docstring_only` to detect diffs that are purely cosmetic. """ - folder = Path(repo.working_dir) + try: + tree = ast.parse(source) + except SyntaxError: + return source + + for node in ast.walk(tree): + # Strip module/class/function docstrings. + if isinstance(node, (ast.Module, ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + body = node.body + if ( + body + and isinstance(body[0], ast.Expr) + and isinstance(body[0].value, ast.Constant) + and isinstance(body[0].value.value, str) + ): + body.pop(0) + return ast.unparse(tree) + + +def diff_is_docstring_only(repo: Repo, branching_point, filename: str) -> bool: + """True if the diff in `filename` between `branching_point` and HEAD only changes docstrings/comments.""" with checkout_commit(repo, branching_point): - with open(folder / filename, "r", encoding="utf-8") as f: - old_content = f.read() - - with open(folder / filename, "r", encoding="utf-8") as f: - new_content = f.read() + old_content = (PATH_TO_REPO / filename).read_text(encoding="utf-8") + new_content = (PATH_TO_REPO / filename).read_text(encoding="utf-8") + return _strip_comments_and_docstrings(old_content) == _strip_comments_and_docstrings(new_content) - old_content_clean = keep_doc_examples_only(old_content) - new_content_clean = keep_doc_examples_only(new_content) - return old_content_clean != new_content_clean - - -def get_diff(repo: Repo, base_commit: str, commits: List[str]) -> List[str]: - """ - Get the diff between a base commit and one or several commits. - - Args: - repo (`git.Repo`): - A git repository (for instance the Transformers repo). - base_commit (`str`): - The commit reference of where to compare for the diff. This is the current commit, not the branching point! - commits (`List[str]`): - The list of commits with which to compare the repo at `base_commit` (so the branching point). - - Returns: - `List[str]`: The list of Python files with a diff (files added, renamed or deleted are always returned, files - modified are returned if the diff in the file is not only in docstrings or comments, see - `diff_is_docstring_only`). - """ - print("\n### DIFF ###\n") +def get_diff(repo: Repo, base_commit, commits) -> List[str]: + """Return Python files changed between `commits` (branching point) and `base_commit` (HEAD).""" code_diff = [] for commit in commits: - for diff_obj in commit.diff(base_commit): - # We always add new python files - if diff_obj.change_type == "A" and diff_obj.b_path.endswith(".py"): - code_diff.append(diff_obj.b_path) - # We check that deleted python files won't break corresponding tests. - elif diff_obj.change_type == "D" and diff_obj.a_path.endswith(".py"): - code_diff.append(diff_obj.a_path) - # Now for modified files - elif diff_obj.change_type in ["M", "R"] and diff_obj.b_path.endswith(".py"): - # In case of renames, we'll look at the tests using both the old and new name. - if diff_obj.a_path != diff_obj.b_path: - code_diff.extend([diff_obj.a_path, diff_obj.b_path]) - else: - # Otherwise, we check modifications are in code and not docstrings. - if diff_is_docstring_only(repo, commit, diff_obj.b_path): - print(f"Ignoring diff in {diff_obj.b_path} as it only concerns docstrings or comments.") - else: - code_diff.append(diff_obj.a_path) - + for d in commit.diff(base_commit): + paths = [p for p in (d.a_path, d.b_path) if p and p.endswith(".py")] + if not paths: + continue + # Add/delete/rename: keep every changed path verbatim. Pure modification: skip if the diff is + # docstring/comment-only. + if d.change_type in ("A", "D") or d.a_path != d.b_path: + code_diff.extend(paths) + elif not diff_is_docstring_only(repo, commit, d.b_path): + code_diff.append(d.b_path) return code_diff def get_modified_python_files(diff_with_last_commit: bool = False) -> List[str]: - """ - Return a list of python files that have been modified between: - - - the current head and the main branch if `diff_with_last_commit=False` (default) - - the current head and its parent commit otherwise. - - Returns: - `List[str]`: The list of Python files with a diff (files added, renamed or deleted are always returned, files - modified are returned if the diff in the file is not only in docstrings or comments, see - `diff_is_docstring_only`). - """ + """List Python files modified between HEAD and either main (default) or the previous commit.""" repo = Repo(PATH_TO_REPO) - - if not diff_with_last_commit: - # Need to fetch refs for main using remotes when running with github actions. + if diff_with_last_commit: + base_label = "previous commit" + commits = repo.head.commit.parents + else: upstream_main = repo.remotes.origin.refs.main + base_label = f"merge-base with main ({upstream_main.commit})" + commits = repo.merge_base(upstream_main, repo.head) + print(f"Diffing HEAD ({repo.head.commit}) against {base_label}: {[str(c) for c in commits]}") + return get_diff(repo, repo.head.commit, commits) - print(f"main is at {upstream_main.commit}") - print(f"Current head is at {repo.head.commit}") - branching_commits = repo.merge_base(upstream_main, repo.head) - for commit in branching_commits: - print(f"Branching commit: {commit}") - return get_diff(repo, repo.head.commit, branching_commits) - else: - print(f"main is at {repo.head.commit}") - parent_commits = repo.head.commit.parents - for commit in parent_commits: - print(f"Parent commit: {commit}") - return get_diff(repo, repo.head.commit, parent_commits) +def get_all_tests() -> List[str]: + """Top-level entries under `tests/` (folders + `tests/test_*.py`), used to expand a full-suite selection.""" + return sorted( + f"tests/{p.name}" + for p in PATH_TO_TESTS.iterdir() + if "__pycache__" not in p.name and (p.is_dir() or p.name.startswith("test_")) + ) -def get_diff_for_doctesting(repo: Repo, base_commit: str, commits: List[str]) -> List[str]: - """ - Get the diff in doc examples between a base commit and one or several commits. +# ============================================================ +# AST-based import extraction +# ============================================================ - Args: - repo (`git.Repo`): - A git repository (for instance the Transformers repo). - base_commit (`str`): - The commit reference of where to compare for the diff. This is the current commit, not the branching point! - commits (`List[str]`): - The list of commits with which to compare the repo at `base_commit` (so the branching point). - Returns: - `List[str]`: The list of Python and Markdown files with a diff (files added or renamed are always returned, files - modified are returned if the diff in the file is only in doctest examples). - """ - print("\n### DIFF ###\n") - code_diff = [] - for commit in commits: - for diff_obj in commit.diff(base_commit): - # We only consider Python files and doc files. - if not diff_obj.b_path.endswith(".py") and not diff_obj.b_path.endswith(".md"): - continue - # We always add new python/md files - if diff_obj.change_type in ["A"]: - code_diff.append(diff_obj.b_path) - # Now for modified files - elif diff_obj.change_type in ["M", "R"]: - # In case of renames, we'll look at the tests using both the old and new name. - if diff_obj.a_path != diff_obj.b_path: - code_diff.extend([diff_obj.a_path, diff_obj.b_path]) - else: - # Otherwise, we check modifications contain some doc example(s). - if diff_contains_doc_examples(repo, commit, diff_obj.b_path): - code_diff.append(diff_obj.a_path) - else: - print(f"Ignoring diff in {diff_obj.b_path} as it doesn't contain any doc example.") +def _resolve_import(module: Optional[str], level: int, importer_pkg: List[str]) -> Optional[List[str]]: + """Resolve an `ImportFrom` node to a list of repo-rooted path parts. - return code_diff + Args: + module: the `X.Y` part of `from X.Y import Z` (None for `from . import Z`). + level: number of leading dots (0 for absolute, 1+ for relative). + importer_pkg: parts of the importing module's *package* (parent dir parts), e.g. + `["src", "diffusers", "pipelines", "flux"]` for `pipelines/flux/pipeline_flux.py`. + Returns: + Path parts like `["src", "diffusers", "pipelines", "flux", "pipeline_flux"]` (no extension), + or None if the import is external or can't be resolved. + """ + if level == 0: + if module is None or not (module == "diffusers" or module.startswith("diffusers.")): + return None + sub = module.split(".")[1:] + return ["src", "diffusers", *sub] + + if level > len(importer_pkg): + return None + base = importer_pkg[: len(importer_pkg) - level + 1] + if module: + return [*base, *module.split(".")] + return base + + +def _to_module_file(path_parts: List[str]) -> Optional[str]: + """Resolve `path_parts` to either `.py` or `/__init__.py`. Returns repo-relative path.""" + candidate = PATH_TO_REPO.joinpath(*path_parts).with_suffix(".py") + if candidate.is_file(): + return str(candidate.relative_to(PATH_TO_REPO)) + init = PATH_TO_REPO.joinpath(*path_parts) / "__init__.py" + if init.is_file(): + return str(init.relative_to(PATH_TO_REPO)) + return None + + +def _iter_module_level_imports(node): + """Yield `ImportFrom` nodes that execute at module load. + + Recurses into `If` / `Try` / `ClassDef` bodies (those run at import time) but stops at + `FunctionDef` / `AsyncFunctionDef` / `Lambda` boundaries — imports inside function bodies are + deferred runtime imports (e.g. lazy stubs in deprecation shims) and shouldn't count as dependencies. + """ + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + return + if isinstance(node, ast.ImportFrom): + yield node + for child in ast.iter_child_nodes(node): + yield from _iter_module_level_imports(child) -def get_all_doctest_files() -> List[str]: - """ - Return the complete list of python and Markdown files on which we run doctest. - At this moment, we restrict this to only take files from `src/` or `docs/source/en/` that are not in `utils/not_doctested.txt`. +def _extract_imports(module_file: str) -> List[Tuple[str, List[str]]]: + """Parse `module_file` and return [(target_file, [imported_symbols]), ...] for diffusers-internal imports. - Returns: - `List[str]`: The complete list of Python and Markdown files on which we run doctest. + Only module-level `from X import ...` statements are considered. Bare `import X`, `from X import *`, + external imports (transformers, torch, stdlib), and imports inside function bodies are skipped. """ - py_files = [str(x.relative_to(PATH_TO_REPO)) for x in PATH_TO_REPO.glob("**/*.py")] - md_files = [str(x.relative_to(PATH_TO_REPO)) for x in PATH_TO_REPO.glob("**/*.md")] - test_files_to_run = py_files + md_files - - # only include files in `src` or `docs/source/en/` - test_files_to_run = [x for x in test_files_to_run if x.startswith(("src/", "docs/source/en/"))] - # not include init files - test_files_to_run = [x for x in test_files_to_run if not x.endswith(("__init__.py",))] + abs_path = PATH_TO_REPO / module_file + try: + source = abs_path.read_text(encoding="utf-8") + tree = ast.parse(source) + except (SyntaxError, UnicodeDecodeError, FileNotFoundError): + return [] - # These are files not doctested yet. - with open("utils/not_doctested.txt") as fp: - not_doctested = {x.split(" ")[0] for x in fp.read().strip().split("\n")} + importer_pkg = list(Path(module_file).parts[:-1]) - # So far we don't have 100% coverage for doctest. This line will be removed once we achieve 100%. - test_files_to_run = [x for x in test_files_to_run if x not in not_doctested] + results: List[Tuple[str, List[str]]] = [] + for node in _iter_module_level_imports(tree): + names = [alias.name for alias in node.names if alias.name != "*"] + if not names: + continue + target_parts = _resolve_import(node.module, node.level, importer_pkg) + if target_parts is None: + continue + target_file = _to_module_file(target_parts) + if target_file is None: + continue + results.append((target_file, names)) - return sorted(test_files_to_run) + return results -def get_new_doctest_files(repo, base_commit, branching_commit) -> List[str]: - """ - Get the list of files that were removed from "utils/not_doctested.txt", between `base_commit` and - `branching_commit`. +# ============================================================ +# Dependency graph +# ============================================================ - Returns: - `List[str]`: List of files that were removed from "utils/not_doctested.txt". - """ - for diff_obj in branching_commit.diff(base_commit): - # Ignores all but the "utils/not_doctested.txt" file. - if diff_obj.a_path != "utils/not_doctested.txt": - continue - # Loads the two versions - folder = Path(repo.working_dir) - with checkout_commit(repo, branching_commit): - with open(folder / "utils/not_doctested.txt", "r", encoding="utf-8") as f: - old_content = f.read() - with open(folder / "utils/not_doctested.txt", "r", encoding="utf-8") as f: - new_content = f.read() - # Compute the removed lines and return them - removed_content = {x.split(" ")[0] for x in old_content.split("\n")} - { - x.split(" ")[0] for x in new_content.split("\n") - } - return sorted(removed_content) - return [] - - -def get_doctest_files(diff_with_last_commit: bool = False) -> List[str]: - """ - Return a list of python and Markdown files where doc example have been modified between: - - the current head and the main branch if `diff_with_last_commit=False` (default) - - the current head and its parent commit otherwise. +def get_module_dependencies(module_file: str, cache: Dict[str, List[Tuple[str, List[str]]]]) -> List[str]: + """Return source files `module_file` truly depends on, traversing inits to find the defining file. - Returns: - `List[str]`: The list of Python and Markdown files with a diff (files added or renamed are always returned, files - modified are returned if the diff in the file is only in doctest examples). + When an import lands on an `__init__.py`, walk its imports too, matching by symbol name to find the + actual submodule that re-exports each requested symbol. This collapses + `from diffusers import StableDiffusionPipeline` to `pipelines/stable_diffusion/pipeline_stable_diffusion.py` + instead of the root init (which would over-select to almost every test). """ - repo = Repo(PATH_TO_REPO) + if module_file not in cache: + cache[module_file] = _extract_imports(module_file) - test_files_to_run = [] # noqa - if not diff_with_last_commit: - upstream_main = repo.remotes.origin.refs.main - print(f"main is at {upstream_main.commit}") - print(f"Current head is at {repo.head.commit}") + dependencies: List[str] = [] + queue: List[Tuple[str, List[str]]] = list(cache[module_file]) + seen_inits: set = set() - branching_commits = repo.merge_base(upstream_main, repo.head) - for commit in branching_commits: - print(f"Branching commit: {commit}") - test_files_to_run = get_diff_for_doctesting(repo, repo.head.commit, branching_commits) - else: - print(f"main is at {repo.head.commit}") - parent_commits = repo.head.commit.parents - for commit in parent_commits: - print(f"Parent commit: {commit}") - test_files_to_run = get_diff_for_doctesting(repo, repo.head.commit, parent_commits) - - all_test_files_to_run = get_all_doctest_files() - - # Add to the test files to run any removed entry from "utils/not_doctested.txt". - new_test_files = get_new_doctest_files(repo, repo.head.commit, upstream_main.commit) - test_files_to_run = list(set(test_files_to_run + new_test_files)) - - # Do not run slow doctest tests on CircleCI - with open("utils/slow_documentation_tests.txt") as fp: - slow_documentation_tests = set(fp.read().strip().split("\n")) - test_files_to_run = [ - x for x in test_files_to_run if x in all_test_files_to_run and x not in slow_documentation_tests - ] + while queue: + target, symbols = queue.pop(0) - # Make sure we did not end up with a test file that was removed - test_files_to_run = [f for f in test_files_to_run if (PATH_TO_REPO / f).exists()] - - return sorted(test_files_to_run) - - -# (:?^|\n) -> Non-catching group for the beginning of the doc or a new line. -# \s*from\s+(\.+\S+)\s+import\s+([^\n]+) -> Line only contains from .xxx import yyy and we catch .xxx and yyy -# (?=\n) -> Look-ahead to a new line. We can't just put \n here or using find_all on this re will only catch every -# other import. -_re_single_line_relative_imports = re.compile(r"(?:^|\n)\s*from\s+(\.+\S+)\s+import\s+([^\n]+)(?=\n)") -# (:?^|\n) -> Non-catching group for the beginning of the doc or a new line. -# \s*from\s+(\.+\S+)\s+import\s+\(([^\)]+)\) -> Line continues with from .xxx import (yyy) and we catch .xxx and yyy -# yyy will take multiple lines otherwise there wouldn't be parenthesis. -_re_multi_line_relative_imports = re.compile(r"(?:^|\n)\s*from\s+(\.+\S+)\s+import\s+\(([^\)]+)\)") -# (:?^|\n) -> Non-catching group for the beginning of the doc or a new line. -# \s*from\s+transformers(\S*)\s+import\s+([^\n]+) -> Line only contains from transformers.xxx import yyy and we catch -# .xxx and yyy -# (?=\n) -> Look-ahead to a new line. We can't just put \n here or using find_all on this re will only catch every -# other import. -_re_single_line_direct_imports = re.compile(r"(?:^|\n)\s*from\s+diffusers(\S*)\s+import\s+([^\n]+)(?=\n)") -# (:?^|\n) -> Non-catching group for the beginning of the doc or a new line. -# \s*from\s+transformers(\S*)\s+import\s+\(([^\)]+)\) -> Line continues with from transformers.xxx import (yyy) and we -# catch .xxx and yyy. yyy will take multiple lines otherwise there wouldn't be parenthesis. -_re_multi_line_direct_imports = re.compile(r"(?:^|\n)\s*from\s+diffusers(\S*)\s+import\s+\(([^\)]+)\)") - - -def extract_imports(module_fname: str, cache: Dict[str, List[str]] = None) -> List[str]: - """ - Get the imports a given module makes. + if not target.endswith("__init__.py"): + dependencies.append(target) + continue - Args: - module_fname (`str`): - The name of the file of the module where we want to look at the imports (given relative to the root of - the repo). - cache (Dictionary `str` to `List[str]`, *optional*): - To speed up this function if it was previously called on `module_fname`, the cache of all previously - computed results. + # Avoid cycles through inits importing each other. + if target in seen_inits: + dependencies.append(target) + continue + seen_inits.add(target) - Returns: - `List[str]`: The list of module filenames imported in the input `module_fname` (a submodule we import from that - is a subfolder will give its init file). - """ - if cache is not None and module_fname in cache: - return cache[module_fname] - - with open(PATH_TO_REPO / module_fname, "r", encoding="utf-8") as f: - content = f.read() - - # Filter out all docstrings to not get imports in code examples. As before we need to deactivate formatting to - # keep this as escaped quotes and avoid this function failing on this file. - # fmt: off - splits = content.split('\"\"\"') - # fmt: on - content = "".join(splits[::2]) - - module_parts = str(module_fname).split(os.path.sep) - imported_modules = [] - - # Let's start with relative imports - relative_imports = _re_single_line_relative_imports.findall(content) - relative_imports = [ - (mod, imp) for mod, imp in relative_imports if "# tests_ignore" not in imp and imp.strip() != "(" - ] - multiline_relative_imports = _re_multi_line_relative_imports.findall(content) - relative_imports += [(mod, imp) for mod, imp in multiline_relative_imports if "# tests_ignore" not in imp] - - # We need to remove parts of the module name depending on the depth of the relative imports. - for module, imports in relative_imports: - level = 0 - while module.startswith("."): - module = module[1:] - level += 1 - - if len(module) > 0: - dep_parts = module_parts[: len(module_parts) - level] + module.split(".") - else: - dep_parts = module_parts[: len(module_parts) - level] - imported_module = os.path.sep.join(dep_parts) - imported_modules.append((imported_module, [imp.strip() for imp in imports.split(",")])) - - # Let's continue with direct imports - direct_imports = _re_single_line_direct_imports.findall(content) - direct_imports = [(mod, imp) for mod, imp in direct_imports if "# tests_ignore" not in imp and imp.strip() != "("] - multiline_direct_imports = _re_multi_line_direct_imports.findall(content) - direct_imports += [(mod, imp) for mod, imp in multiline_direct_imports if "# tests_ignore" not in imp] - - # We need to find the relative path of those imports. - for module, imports in direct_imports: - import_parts = module.split(".")[1:] # ignore the name of the repo since we add it below. - dep_parts = ["src", "diffusers"] + import_parts - imported_module = os.path.sep.join(dep_parts) - imported_modules.append((imported_module, [imp.strip() for imp in imports.split(",")])) - - result = [] - # Double check we get proper modules (either a python file or a folder with an init). - for module_file, imports in imported_modules: - if (PATH_TO_REPO / f"{module_file}.py").is_file(): - module_file = f"{module_file}.py" - elif (PATH_TO_REPO / module_file).is_dir() and (PATH_TO_REPO / module_file / "__init__.py").is_file(): - module_file = os.path.sep.join([module_file, "__init__.py"]) - imports = [imp for imp in imports if len(imp) > 0 and re.match("^[A-Za-z0-9_]*$", imp)] - if len(imports) > 0: - result.append((module_file, imports)) - - if cache is not None: - cache[module_fname] = result - - return result - - -def get_module_dependencies(module_fname: str, cache: Dict[str, List[str]] = None) -> List[str]: - """ - Refines the result of `extract_imports` to remove subfolders and get a proper list of module filenames: if a file - as an import `from utils import Foo, Bar`, with `utils` being a subfolder containing many files, this will traverse - the `utils` init file to check where those dependencies come from: for instance the files utils/foo.py and utils/bar.py. + if target not in cache: + cache[target] = _extract_imports(target) + init_imports = cache[target] - Warning: This presupposes that all intermediate inits are properly built (with imports from the respective - submodules) and work better if objects are defined in submodules and not the intermediate init (otherwise the - intermediate init is added, and inits usually have a lot of dependencies). + unresolved = list(symbols) + for sub_target, sub_names in init_imports: + matched = [s for s in unresolved if s in sub_names] + if matched: + queue.append((sub_target, matched)) + unresolved = [s for s in unresolved if s not in matched] - Args: - module_fname (`str`): - The name of the file of the module where we want to look at the imports (given relative to the root of - the repo). - cache (Dictionary `str` to `List[str]`, *optional*): - To speed up this function if it was previously called on `module_fname`, the cache of all previously - computed results. + if unresolved: + # Symbol(s) couldn't be resolved through the init's TYPE_CHECKING imports — likely lazy-loaded + # via `_import_structure` or defined directly in the init. Keep the init as the dep (coarse but + # correct: changes to the init will trigger this module). + dependencies.append(target) - Returns: - `List[str]`: The list of module filenames imported in the input `module_fname` (with submodule imports refined). - """ - dependencies = [] - imported_modules = extract_imports(module_fname, cache=cache) - # The while loop is to recursively traverse all inits we may encounter: we will add things as we go. - while len(imported_modules) > 0: - new_modules = [] - for module, imports in imported_modules: - # If we end up in an __init__ we are often not actually importing from this init (except in the case where - # the object is fully defined in the __init__) - if module.endswith("__init__.py"): - # So we get the imports from that init then try to find where our objects come from. - new_imported_modules = extract_imports(module, cache=cache) - for new_module, new_imports in new_imported_modules: - if any(i in new_imports for i in imports): - if new_module not in dependencies: - new_modules.append((new_module, [i for i in new_imports if i in imports])) - imports = [i for i in imports if i not in new_imports] - if len(imports) > 0: - # If there are any objects lefts, they may be a submodule - path_to_module = PATH_TO_REPO / module.replace("__init__.py", "") - dependencies.extend( - [ - os.path.join(module.replace("__init__.py", ""), f"{i}.py") - for i in imports - if (path_to_module / f"{i}.py").is_file() - ] - ) - imports = [i for i in imports if not (path_to_module / f"{i}.py").is_file()] - if len(imports) > 0: - # Then if there are still objects left, they are fully defined in the init, so we keep it as a - # dependency. - dependencies.append(module) - else: - dependencies.append(module) - - imported_modules = new_modules - - return dependencies - - -def create_reverse_dependency_tree() -> List[Tuple[str, str]]: - """ - Create a list of all edges (a, b) which mean that modifying a impacts b with a going over all module and test files. - """ - cache = {} - all_modules = list(PATH_TO_DIFFUSERS.glob("**/*.py")) + list(PATH_TO_TESTS.glob("**/*.py")) - all_modules = [str(mod.relative_to(PATH_TO_REPO)) for mod in all_modules] - edges = [(dep, mod) for mod in all_modules for dep in get_module_dependencies(mod, cache=cache)] + return list(set(dependencies)) - return list(set(edges)) +def _merged_nested_deps(m: str, direct_deps: Dict[str, List[str]]) -> bool: + """Pull each of m's deps' deps into m. Returns True if m grew. -def get_tree_starting_at(module: str, edges: List[Tuple[str, str]]) -> List[Union[str, List[str]]]: + Skips `__init__.py` targets — they re-export the entire package surface, so expanding through + them would pull in every diffusers symbol via the root init. """ - Returns the tree starting at a given module following all edges. - - Args: - module (`str`): The module that will be the root of the subtree we want. - edges (`List[Tuple[str, str]]`): The list of all edges of the tree. + merged = False + for d in list(direct_deps[m]): + if d.endswith("__init__.py"): + continue + new_deps = set(direct_deps[d]) - set(direct_deps[m]) + if new_deps: + direct_deps[m].extend(new_deps) + merged = True + return merged - Returns: - `List[Union[str, List[str]]]`: The tree to print in the following format: [module, [list of edges - starting at module], [list of edges starting at the preceding level], ...] - """ - vertices_seen = [module] - new_edges = [edge for edge in edges if edge[0] == module and edge[1] != module and "__init__.py" not in edge[1]] - tree = [module] - while len(new_edges) > 0: - tree.append(new_edges) - final_vertices = list({edge[1] for edge in new_edges}) - vertices_seen.extend(final_vertices) - new_edges = [ - edge - for edge in edges - if edge[0] in final_vertices and edge[1] not in vertices_seen and "__init__.py" not in edge[1] - ] - - return tree - - -def print_tree_deps_of(module, all_edges=None): - """ - Prints the tree of modules depending on a given module. - Args: - module (`str`): The module that will be the root of the subtree we want. - all_edges (`List[Tuple[str, str]]`, *optional*): - The list of all edges of the tree. Will be set to `create_reverse_dependency_tree()` if not passed. - """ - if all_edges is None: - all_edges = create_reverse_dependency_tree() - tree = get_tree_starting_at(module, all_edges) - - # The list of lines is a list of tuples (line_to_be_printed, module) - # Keeping the modules lets us know where to insert each new lines in the list. - lines = [(tree[0], tree[0])] - for index in range(1, len(tree)): - edges = tree[index] - start_edges = {edge[0] for edge in edges} - - for start in start_edges: - end_edges = {edge[1] for edge in edges if edge[0] == start} - # We will insert all those edges just after the line showing start. - pos = 0 - while lines[pos][1] != start: - pos += 1 - lines = lines[: pos + 1] + [(" " * (2 * index) + end, end) for end in end_edges] + lines[pos + 1 :] - - for line in lines: - # We don't print the refs that where just here to help build lines. - print(line[0]) - - -def init_test_examples_dependencies() -> Tuple[Dict[str, List[str]], List[str]]: - """ - The test examples do not import from the examples (which are just scripts, not modules) so we need some extra - care initializing the dependency map, which is the goal of this function. It initializes the dependency map for - example files by linking each example to the example test file for the example framework. +def create_reverse_dependency_map() -> Dict[str, List[str]]: + """Build the reverse dependency map: file → list of files that transitively depend on it. - Returns: - `Tuple[Dict[str, List[str]], List[str]]`: A tuple with two elements: the initialized dependency map which is a - dict test example file to list of example files potentially tested by that test file, and the list of all - example files (to avoid recomputing it later). + 1. Compute direct deps for every .py under `src/diffusers/` and `tests/`. + 2. Transitively close (skipping inits during recursion to avoid pulling in the universe via the root init). + 3. Invert. """ - test_example_deps = {} - all_examples = [] - for framework in ["pytorch", "tensorflow"]: - test_files = list((PATH_TO_EXAMPLES / framework).glob("test_*.py")) - all_examples.extend(test_files) - # Remove the files at the root of examples/framework since they are not proper examples (they are either utils - # or example test files). - examples = [ - f for f in (PATH_TO_EXAMPLES / framework).glob("**/*.py") if f.parent != PATH_TO_EXAMPLES / framework - ] - all_examples.extend(examples) - for test_file in test_files: - with open(test_file, "r", encoding="utf-8") as f: - content = f.read() - # Map all examples to the test files found in examples/framework. - test_example_deps[str(test_file.relative_to(PATH_TO_REPO))] = [ - str(e.relative_to(PATH_TO_REPO)) for e in examples if e.name in content - ] - # Also map the test files to themselves. - test_example_deps[str(test_file.relative_to(PATH_TO_REPO))].append( - str(test_file.relative_to(PATH_TO_REPO)) - ) - return test_example_deps, all_examples - - -def create_reverse_dependency_map() -> dict[str, List[str]]: - """ - Create the dependency map from module/test filename to the list of modules/tests that depend on it recursively. + cache: Dict[str, List[Tuple[str, List[str]]]] = {} + all_modules = [ + str(p.relative_to(PATH_TO_REPO)) + for p in list(PATH_TO_DIFFUSERS.glob("**/*.py")) + list(PATH_TO_TESTS.glob("**/*.py")) + ] + direct_deps: Dict[str, List[str]] = {m: get_module_dependencies(m, cache) for m in all_modules} - Returns: - `Dict[str, List[str]]`: The reverse dependency map as a dictionary mapping filenames to all the filenames - depending on it recursively. This way the tests impacted by a change in file A are the test files in the list - corresponding to key A in this result. - """ - cache = {} - # Start from the example deps init. - example_deps, examples = init_test_examples_dependencies() - # Add all modules and all tests to all examples - all_modules = list(PATH_TO_DIFFUSERS.glob("**/*.py")) + list(PATH_TO_TESTS.glob("**/*.py")) + examples - all_modules = [str(mod.relative_to(PATH_TO_REPO)) for mod in all_modules] - # Compute the direct dependencies of all modules. - direct_deps = {m: get_module_dependencies(m, cache=cache) for m in all_modules} - direct_deps.update(example_deps) - - # This recurses the dependencies - something_changed = True - while something_changed: - something_changed = False + # Each pass propagates dependency info one level deeper. Loop until a full pass adds nothing. + changed = True + while changed: + changed = False for m in all_modules: - for d in direct_deps[m]: - # We stop recursing at an init (cause we always end up in the main init and we don't want to add all - # files which the main init imports) - if d.endswith("__init__.py"): - continue - if d not in direct_deps: - raise ValueError(f"KeyError:{d}. From {m}") - new_deps = set(direct_deps[d]) - set(direct_deps[m]) - if len(new_deps) > 0: - direct_deps[m].extend(list(new_deps)) - something_changed = True - - # Finally we can build the reverse map. - reverse_map = collections.defaultdict(list) + if _merged_nested_deps(m, direct_deps): + changed = True + + reverse_map: Dict[str, List[str]] = collections.defaultdict(list) for m in all_modules: for d in direct_deps[m]: reverse_map[d].append(m) - # For inits, we don't do the reverse deps but the direct deps: if modifying an init, we want to make sure we test - # all the modules impacted by that init. - for m in [f for f in all_modules if f.endswith("__init__.py")]: - direct_deps = get_module_dependencies(m, cache=cache) - deps = sum([reverse_map[d] for d in direct_deps if not d.endswith("__init__.py")], direct_deps) - reverse_map[m] = list(set(deps) - {m}) - - return reverse_map - - -def create_module_to_test_map(reverse_map: Dict[str, List[str]] = None) -> dict[str, List[str]]: - """ - Extract the tests from the reverse_dependency_map and potentially filters the model tests. - - Args: - reverse_map (`Dict[str, List[str]]`, *optional*): - The reverse dependency map as created by `create_reverse_dependency_map`. Will default to the result of - that function if not provided. - filter_pipelines (`bool`, *optional*, defaults to `False`): - Whether or not to filter pipeline tests to only include core pipelines if a file impacts a lot of models. - - Returns: - `Dict[str, List[str]]`: A dictionary that maps each file to the tests to execute if that file was modified. - """ - if reverse_map is None: - reverse_map = create_reverse_dependency_map() - - # Utility that tells us if a given file is a test (taking test examples into account) - def is_test(fname): - if fname.startswith("tests"): - return True - if fname.startswith("examples") and fname.split(os.path.sep)[-1].startswith("test"): - return True - return False - - # Build the test map - test_map = {module: [f for f in deps if is_test(f)] for module, deps in reverse_map.items()} - - return test_map - - -def check_imports_all_exist(): - """ - Isn't used per se by the test fetcher but might be used later as a quality check. Putting this here for now so the - code is not lost. This checks all imports in a given file do exist. - """ - cache = {} - all_modules = list(PATH_TO_DIFFUSERS.glob("**/*.py")) + list(PATH_TO_TESTS.glob("**/*.py")) - all_modules = [str(mod.relative_to(PATH_TO_REPO)) for mod in all_modules] - direct_deps = {m: get_module_dependencies(m, cache=cache) for m in all_modules} - - for module, deps in direct_deps.items(): - for dep in deps: - if not (PATH_TO_REPO / dep).is_file(): - print(f"{module} has dependency on {dep} which does not exist.") - - -def _print_list(l) -> str: - """ - Pretty print a list of elements with one line per element and a - starting each line. - """ - return "\n".join([f"- {f}" for f in l]) - + # For inits, do the forward direction: editing an init impacts everything it re-exports. + for init in [m for m in all_modules if m.endswith("__init__.py")]: + deps = get_module_dependencies(init, cache) + impacted = set(deps) + for d in deps: + if not d.endswith("__init__.py"): + impacted.update(reverse_map.get(d, [])) + reverse_map[init] = sorted(impacted - {init}) -def update_test_map_with_core_pipelines(json_output_file: str): - print(f"\n### ADD CORE PIPELINE TESTS ###\n{_print_list(IMPORTANT_PIPELINES)}") - with open(json_output_file, "rb") as fp: - test_map = json.load(fp) + return dict(reverse_map) - # Add core pipelines as their own test group - test_map["core_pipelines"] = " ".join( - sorted([str(PATH_TO_TESTS / f"pipelines/{pipe}") for pipe in IMPORTANT_PIPELINES]) - ) - # If there are no existing pipeline tests save the map - if "pipelines" not in test_map: - with open(json_output_file, "w", encoding="UTF-8") as fp: - json.dump(test_map, fp, ensure_ascii=False) +# ============================================================ +# Test selection +# ============================================================ - pipeline_tests = test_map.pop("pipelines") - pipeline_tests = pipeline_tests.split(" ") - # Remove core pipeline tests from the fetched pipeline tests - updated_pipeline_tests = [] - for pipe in pipeline_tests: - if pipe == "tests/pipelines" or Path(pipe).parts[2] in IMPORTANT_PIPELINES: +def _bucket_for_matrix(test_paths: List[str]) -> Dict[str, List[str]]: + """Group test paths by top-level folder under `tests/`. Files directly under `tests/` go to `common`.""" + test_map: Dict[str, List[str]] = collections.defaultdict(list) + for p in test_paths: + parts = p.split("/") + if len(parts) < 2 or parts[0] != "tests": continue - updated_pipeline_tests.append(pipe) + bucket = "common" if len(parts) == 2 else parts[1] + test_map[bucket].append(p) + return {k: sorted(set(v)) for k, v in test_map.items()} - if len(updated_pipeline_tests) > 0: - test_map["pipelines"] = " ".join(sorted(updated_pipeline_tests)) - with open(json_output_file, "w", encoding="UTF-8") as fp: - json.dump(test_map, fp, ensure_ascii=False) - - -def create_json_map(test_files_to_run: List[str], json_output_file: str | None = None): - """ - Creates a map from a list of tests to run to easily split them by category, when running parallelism of slow tests. +def _base_names(node: ast.ClassDef) -> List[str]: + return [ + b.id if isinstance(b, ast.Name) else b.attr for b in node.bases if isinstance(b, (ast.Name, ast.Attribute)) + ] - Args: - test_files_to_run (`List[str]`): The list of tests to run. - json_output_file (`str`): The path where to store the built json map. - """ - if json_output_file is None: - return - test_map = {} - for test_file in test_files_to_run: - # `test_file` is a path to a test folder/file, starting with `tests/`. For example, - # - `tests/models/bert/test_modeling_bert.py` or `tests/models/bert` - # - `tests/trainer/test_trainer.py` or `tests/trainer` - # - `tests/test_modeling_common.py` - names = test_file.split(os.path.sep) - module = names[1] - if module in MODULES_TO_IGNORE: +def _marker_decorators() -> Dict[str, str]: + """Map decorator function name → pytest marker, from `def is_x(test_case): return pytest.mark.(test_case)`.""" + tree = ast.parse((PATH_TO_TESTS / "testing_utils.py").read_text(encoding="utf-8")) + decorators = {} + for node in ast.walk(tree): + if not isinstance(node, ast.FunctionDef) or not node.name.startswith("is_"): continue + for ret in ast.walk(node): + if isinstance(ret, ast.Return) and isinstance(ret.value, ast.Call): + func = ret.value.func + if ( + isinstance(func, ast.Attribute) + and isinstance(func.value, ast.Attribute) + and func.value.attr == "mark" + ): + decorators[node.name] = func.attr + return decorators + + +def _mixin_markers(bucket: str, decorators: Dict[str, str]) -> Dict[str, List[str]]: + """Map mixin class name → markers it carries, from the `@is_*` decorators on classes in + `tests//testing_utils/`. Markers are inherited through pytest, so a mixin also carries those of + any base mixin. Scoped per bucket because class names repeat across packages (the pipelines' legacy, + unmarked `IPAdapterTesterMixin` vs. the models' marked one).""" + bases: Dict[str, List[str]] = {} + markers: Dict[str, set] = {} + for module in (PATH_TO_TESTS / bucket / "testing_utils").glob("*.py"): + for node in ast.walk(ast.parse(module.read_text(encoding="utf-8"))): + if not isinstance(node, ast.ClassDef): + continue + bases[node.name] = _base_names(node) + markers[node.name] = { + decorators[d.id] for d in node.decorator_list if isinstance(d, ast.Name) and d.id in decorators + } + + changed = True + while changed: + changed = False + for cls, cls_bases in bases.items(): + inherited = set().union(*(markers.get(b, set()) for b in cls_bases)) + if not inherited <= markers[cls]: + markers[cls] |= inherited + changed = True + return {cls: sorted(m) for cls, m in markers.items() if m} + + +def _class_markers(test_file: str, mixin_markers: Dict[str, List[str]]) -> List[set]: + """Markers of every test class in `test_file`, derived from the marked mixins it composes.""" + tree = ast.parse((PATH_TO_REPO / test_file).read_text(encoding="utf-8")) + return [ + {m for base in _base_names(node) for m in mixin_markers.get(base, [])} + for node in ast.walk(tree) + if isinstance(node, ast.ClassDef) + ] - if len(names) > 2 or not test_file.endswith(".py"): - # test folders under `tests` or python files under them - # take the part like tokenization, `pipeline`, etc. for other test categories - key = os.path.sep.join(names[1:2]) - else: - # common test files directly under `tests/` - key = "common" - if key not in test_map: - test_map[key] = [] - test_map[key].append(test_file) +def _feature_groups_for(paths: List[str], mixin_markers: Dict[str, List[str]]) -> List[str]: + """Names of the feature groups (including `core`) that at least one test class in `paths` would land in.""" + class_markers = [m for p in paths for m in _class_markers(p, mixin_markers)] + non_core = set(FEATURE_MARKERS + CPU_SKIPPED_MARKERS) + groups = [] + if any(not m & non_core for m in class_markers): + groups.append("core") + for group, markers in FEATURE_GROUPS.items(): + if any(m & set(markers) for m in class_markers): + groups.append(group) + return groups + + +def _matrix_entries(test_map: Dict[str, List[str]]) -> List[Dict[str, str]]: + """Expand buckets into CI matrix entries: `{"name", "paths", "markers"}`, one job each.""" + decorators = _marker_decorators() + entries = [] + for bucket, paths in sorted(test_map.items()): + joined = " ".join(paths) + if bucket not in SPLIT_BY_FEATURE: + entries.append({"name": bucket, "paths": joined, "markers": ""}) + continue + for group in _feature_groups_for(paths, _mixin_markers(bucket, decorators)): + expr = CORE_MARKERS if group == "core" else " or ".join(FEATURE_GROUPS[group]) + entries.append({"name": f"{bucket}-{group}", "paths": joined, "markers": expr}) + return entries - # sort the keys & values - keys = sorted(test_map.keys()) - test_map = {k: " ".join(sorted(test_map[k])) for k in keys} +def _write_matrix(json_output_file: str, test_files: List[str]): with open(json_output_file, "w", encoding="UTF-8") as fp: - json.dump(test_map, fp, ensure_ascii=False) + json.dump(_matrix_entries(_bucket_for_matrix(test_files)), fp, ensure_ascii=False) -def infer_tests_to_run( - output_file: str, - diff_with_last_commit: bool = False, - json_output_file: str | None = None, -): - """ - The main function called by the test fetcher. Determines the tests to run from the diff. +def _is_test_file(path: str) -> bool: + """True if `path` is a `tests/.../test_*.py` file (the kind pytest collects).""" + return path.startswith("tests/") and Path(path).name.startswith("test_") - Args: - output_file (`str`): - The path where to store the summary of the test fetcher analysis. Other files will be stored in the same - folder: - - - examples_test_list.txt: The list of examples tests to run. - - test_repo_utils.txt: Will indicate if the repo utils tests should be run or not. - - doctest_list.txt: The list of doctests to run. - - diff_with_last_commit (`bool`, *optional*, defaults to `False`): - Whether to analyze the diff with the last commit (for use on the main branch after a PR is merged) or with - the branching point from main (for use on each PR). - filter_models (`bool`, *optional*, defaults to `True`): - Whether or not to filter the tests to core models only, when a file modified results in a lot of model - tests. - json_output_file (`str`, *optional*): - The path where to store the json file mapping categories of tests to tests to run (used for parallelism or - the slow tests). - """ + +def fetch_tests_to_run(json_output_file: str, diff_with_last_commit: bool): + """Determine the tests to run from the diff and write `test_map.json`.""" modified_files = get_modified_python_files(diff_with_last_commit=diff_with_last_commit) - print(f"\n### MODIFIED FILES ###\n{_print_list(modified_files)}") - # Create the map that will give us all impacted modules. reverse_map = create_reverse_dependency_map() - impacted_files = modified_files.copy() - for f in modified_files: - if f in reverse_map: - impacted_files.extend(reverse_map[f]) - # Remove duplicates - impacted_files = sorted(set(impacted_files)) - print(f"\n### IMPACTED FILES ###\n{_print_list(impacted_files)}") - - # Grab the corresponding test files: - if any(x in modified_files for x in ["setup.py"]): - test_files_to_run = ["tests", "examples"] - - # in order to trigger pipeline tests even if no code change at all - if "tests/utils/tiny_model_summary.json" in modified_files: - test_files_to_run = ["tests"] - any(f.split(os.path.sep)[0] == "utils" for f in modified_files) - else: - # All modified tests need to be run. - test_files_to_run = [ - f for f in modified_files if f.startswith("tests") and f.split(os.path.sep)[-1].startswith("test") - ] - # Then we grab the corresponding test files. - test_map = create_module_to_test_map(reverse_map=reverse_map) - for f in modified_files: - if f in test_map: - test_files_to_run.extend(test_map[f]) - test_files_to_run = sorted(set(test_files_to_run)) - # Make sure we did not end up with a test file that was removed - test_files_to_run = [f for f in test_files_to_run if (PATH_TO_REPO / f).exists()] - - any(f.split(os.path.sep)[0] == "utils" for f in modified_files) - - examples_tests_to_run = [f for f in test_files_to_run if f.startswith("examples")] - test_files_to_run = [f for f in test_files_to_run if not f.startswith("examples")] - print(f"\n### TEST TO RUN ###\n{_print_list(test_files_to_run)}") - if len(test_files_to_run) > 0: - with open(output_file, "w", encoding="utf-8") as f: - f.write(" ".join(test_files_to_run)) - - # Create a map that maps test categories to test files, i.e. `models/bert` -> [...test_modeling_bert.py, ...] - - # Get all test directories (and some common test files) under `tests` and `tests/models` if `test_files_to_run` - # contains `tests` (i.e. when `setup.py` is changed). - if "tests" in test_files_to_run: - test_files_to_run = get_all_tests() - - create_json_map(test_files_to_run, json_output_file) - - print(f"\n### EXAMPLES TEST TO RUN ###\n{_print_list(examples_tests_to_run)}") - if len(examples_tests_to_run) > 0: - # We use `all` in the case `commit_flags["test_all"]` as well as in `create_circleci_config.py` for processing - if examples_tests_to_run == ["examples"]: - examples_tests_to_run = ["all"] - example_file = Path(output_file).parent / "examples_test_list.txt" - with open(example_file, "w", encoding="utf-8") as f: - f.write(" ".join(examples_tests_to_run)) - - -def filter_tests(output_file: str, filters: List[str]): - """ - Reads the content of the output file and filters out all the tests in a list of given folders. - - Args: - output_file (`str` or `os.PathLike`): The path to the output file of the tests fetcher. - filters (`List[str]`): A list of folders to filter. - """ - if not os.path.isfile(output_file): - print("No test file found.") - return - with open(output_file, "r", encoding="utf-8") as f: - test_files = f.read().split(" ") + # Each modified file contributes itself (if it's a test) plus tests transitively impacted by it. + selected = set() + for f in modified_files: + if _is_test_file(f): + selected.add(f) + selected.update(t for t in reverse_map.get(f, []) if _is_test_file(t)) - if len(test_files) == 0 or test_files == [""]: - print("No tests to filter.") - return + test_files_to_run = sorted(p for p in selected if (PATH_TO_REPO / p).exists()) + _write_matrix(json_output_file, test_files_to_run) - if test_files == ["tests"]: - test_files = [os.path.join("tests", f) for f in os.listdir("tests") if f not in ["__init__.py"] + filters] - else: - test_files = [f for f in test_files if f.split(os.path.sep)[1] not in filters] - with open(output_file, "w", encoding="utf-8") as f: - f.write(" ".join(test_files)) +def _all_test_files() -> List[str]: + """Enumerate every `tests/.../test_*.py` (used for the full-suite path).""" + return sorted( + str(p.relative_to(PATH_TO_REPO)) for p in PATH_TO_TESTS.glob("**/test_*.py") if "__pycache__" not in p.parts + ) -def parse_commit_message(commit_message: str) -> dict[str, bool]: - """ - Parses the commit message to detect if a command is there to skip, force all or part of the CI. +def _write_full_suite(json_output_file: str): + """Schedule the entire test suite. Used by `--force_full_suite` and as exception fallback.""" + _write_matrix(json_output_file, _all_test_files()) - Args: - commit_message (`str`): The commit message of the current commit. - Returns: - `Dict[str, bool]`: A dictionary of strings to bools with keys the following keys: `"skip"`, - `"test_all_models"` and `"test_all"`. - """ - if commit_message is None: - return {"skip": False, "no_filter": False, "test_all": False} - - command_search = re.search(r"\[([^\]]*)\]", commit_message) - if command_search is not None: - command = command_search.groups()[0] - command = command.lower().replace("-", " ").replace("_", " ") - skip = command in ["ci skip", "skip ci", "circleci skip", "skip circleci"] - no_filter = set(command.split(" ")) == {"no", "filter"} - test_all = set(command.split(" ")) == {"test", "all"} - return {"skip": skip, "no_filter": no_filter, "test_all": test_all} - else: - return {"skip": False, "no_filter": False, "test_all": False} +# ============================================================ +# CLI +# ============================================================ if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument( - "--output_file", type=str, default="test_list.txt", help="Where to store the list of tests to run" - ) parser.add_argument( "--json_output_file", type=str, default="test_map.json", - help="Where to store the tests to run in a dictionary format mapping test categories to test files", + help="Where to store the list of matrix entries (name / paths / markers) consumed by CI.", ) parser.add_argument( "--diff_with_last_commit", action="store_true", - help="To fetch the tests between the current commit and the last commit", + help="Diff against the previous commit instead of main (use on main branch jobs)", ) parser.add_argument( - "--filter_tests", + "--force_full_suite", action="store_true", - help="Will filter the pipeline/repo utils tests outside of the generated list of tests.", - ) - parser.add_argument( - "--print_dependencies_of", - type=str, - help="Will only print the tree of modules depending on the file passed.", - default=None, - ) - parser.add_argument( - "--commit_message", - type=str, - help="The commit message (which could contain a command to force all tests or skip the CI).", - default=None, + help="Bypass selection and write outputs that schedule the entire test suite.", ) args = parser.parse_args() - if args.print_dependencies_of is not None: - print_tree_deps_of(args.print_dependencies_of) - else: - repo = Repo(PATH_TO_REPO) - commit_message = repo.head.commit.message - commit_flags = parse_commit_message(commit_message) - if commit_flags["skip"]: - print("Force-skipping the CI") - quit() - if commit_flags["no_filter"]: - print("Running all tests fetched without filtering.") - if commit_flags["test_all"]: - print("Force-launching all tests") - - diff_with_last_commit = args.diff_with_last_commit - if not diff_with_last_commit and not repo.head.is_detached and repo.head.ref == repo.refs.main: - print("main branch detected, fetching tests against last commit.") - diff_with_last_commit = True - - if not commit_flags["test_all"]: - try: - infer_tests_to_run( - args.output_file, - diff_with_last_commit=diff_with_last_commit, - json_output_file=args.json_output_file, - ) - filter_tests(args.output_file, ["repo_utils"]) - update_test_map_with_core_pipelines(json_output_file=args.json_output_file) - - except Exception as e: - print(f"\nError when trying to grab the relevant tests: {e}\n\nRunning all tests.") - commit_flags["test_all"] = True - - if commit_flags["test_all"]: - with open(args.output_file, "w", encoding="utf-8") as f: - f.write("tests") - example_file = Path(args.output_file).parent / "examples_test_list.txt" - with open(example_file, "w", encoding="utf-8") as f: - f.write("all") - - test_files_to_run = get_all_tests() - create_json_map(test_files_to_run, args.json_output_file) - update_test_map_with_core_pipelines(json_output_file=args.json_output_file) + + if args.force_full_suite: + print("Forcing full test suite.") + _write_full_suite(args.json_output_file) + raise SystemExit(0) + + repo = Repo(PATH_TO_REPO) + diff_with_last_commit = args.diff_with_last_commit + if not diff_with_last_commit and not repo.head.is_detached and repo.head.ref == repo.refs.main: + print("main branch detected, fetching tests against last commit.") + diff_with_last_commit = True + + try: + fetch_tests_to_run(args.json_output_file, diff_with_last_commit) + except Exception as e: + import traceback + + print(f"\nError when trying to grab the relevant tests: {e}\n") + traceback.print_exc() + print("\nRunning all tests.") + _write_full_suite(args.json_output_file) From c7c8cc41dcd1fcb7520ca5b57cac4a22baeb5bca Mon Sep 17 00:00:00 2001 From: DN6 Date: Wed, 26 Aug 2026 10:38:51 +0530 Subject: [PATCH 02/24] [tests] Import pipeline/model classes inside the tests that use them so the fetcher does not over-select through shared mixins --- tests/lora/test_lora_layers_sd.py | 8 ++- tests/models/test_modeling_common.py | 6 +- .../test_models_transformer_flux.py | 51 +--------------- tests/models/transformers/utils.py | 58 +++++++++++++++++++ tests/others/test_flashpack.py | 3 +- tests/pipelines/kandinsky3/test_kandinsky3.py | 6 +- .../kandinsky3/test_kandinsky3_img2img.py | 3 +- tests/pipelines/pag/test_pag_sd.py | 5 +- tests/pipelines/pag/test_pag_sd3_img2img.py | 5 +- tests/pipelines/pag/test_pag_sd_img2img.py | 5 +- tests/pipelines/pag/test_pag_sd_inpaint.py | 5 +- tests/pipelines/pag/test_pag_sdxl.py | 5 +- tests/pipelines/pag/test_pag_sdxl_img2img.py | 5 +- tests/pipelines/pag/test_pag_sdxl_inpaint.py | 5 +- tests/pipelines/test_pipelines_common.py | 2 +- 15 files changed, 107 insertions(+), 65 deletions(-) create mode 100644 tests/models/transformers/utils.py diff --git a/tests/lora/test_lora_layers_sd.py b/tests/lora/test_lora_layers_sd.py index 4662075cfd0a..51060abed97b 100644 --- a/tests/lora/test_lora_layers_sd.py +++ b/tests/lora/test_lora_layers_sd.py @@ -24,8 +24,6 @@ from transformers import CLIPTextModel, CLIPTokenizer from diffusers import ( - AutoPipelineForImage2Image, - AutoPipelineForText2Image, DDIMScheduler, DiffusionPipeline, LCMScheduler, @@ -641,6 +639,8 @@ def test_load_unload_load_kohya_lora(self): release_memory(pipe) def test_not_empty_state_dict(self): + from diffusers import AutoPipelineForText2Image + # Makes sure https://github.com/huggingface/diffusers/issues/7054 does not happen again pipe = AutoPipelineForText2Image.from_pretrained( "stable-diffusion-v1-5/stable-diffusion-v1-5", torch_dtype=torch.float16 @@ -655,6 +655,8 @@ def test_not_empty_state_dict(self): release_memory(pipe) def test_load_unload_load_state_dict(self): + from diffusers import AutoPipelineForText2Image + # Makes sure https://github.com/huggingface/diffusers/issues/7054 does not happen again pipe = AutoPipelineForText2Image.from_pretrained( "stable-diffusion-v1-5/stable-diffusion-v1-5", torch_dtype=torch.float16 @@ -705,6 +707,8 @@ def test_sdv1_5_lcm_lora(self): release_memory(pipe) def test_sdv1_5_lcm_lora_img2img(self): + from diffusers import AutoPipelineForImage2Image + pipe = AutoPipelineForImage2Image.from_pretrained( "stable-diffusion-v1-5/stable-diffusion-v1-5", torch_dtype=torch.float16 ) diff --git a/tests/models/test_modeling_common.py b/tests/models/test_modeling_common.py index 9968add19dd9..139111fe5c6f 100644 --- a/tests/models/test_modeling_common.py +++ b/tests/models/test_modeling_common.py @@ -26,7 +26,7 @@ from huggingface_hub import ModelCard, delete_repo, snapshot_download, try_to_load_from_cache from huggingface_hub.utils import HfHubHTTPError, is_jinja_available -from diffusers.models import FluxTransformer2DModel, SD3Transformer2DModel, UNet2DConditionModel +from diffusers.models import UNet2DConditionModel from ..others.test_utils import TOKEN, USER, is_staging_test from ..testing_utils import ( @@ -118,6 +118,8 @@ def test_cached_files_are_used_when_no_internet(self): assert False, "Parameters not the same!" def test_local_files_only_with_sharded_checkpoint(self): + from diffusers.models import FluxTransformer2DModel + repo_id = "hf-internal-testing/tiny-flux-sharded" error_response = mock.Mock( status_code=500, @@ -232,6 +234,8 @@ def test_keep_modules_in_fp32(self): A simple tests to check if the modules under `_keep_in_fp32_modules` are kept in fp32 when we load the model in fp16/bf16 Also ensures if inference works. """ + from diffusers.models import SD3Transformer2DModel + fp32_modules = SD3Transformer2DModel._keep_in_fp32_modules for torch_dtype in [torch.bfloat16, torch.float16]: diff --git a/tests/models/transformers/test_models_transformer_flux.py b/tests/models/transformers/test_models_transformer_flux.py index d60e34e2ea3f..27a80c4b9a02 100644 --- a/tests/models/transformers/test_models_transformer_flux.py +++ b/tests/models/transformers/test_models_transformer_flux.py @@ -23,7 +23,6 @@ import torch from diffusers import BitsAndBytesConfig, FluxTransformer2DModel, GGUFQuantizationConfig -from diffusers.models.embeddings import ImageProjection from diffusers.models.transformers.transformer_flux import FluxIPAdapterAttnProcessor from diffusers.utils.torch_utils import randn_tensor @@ -60,60 +59,12 @@ TorchCompileTesterMixin, TrainingTesterMixin, ) +from .utils import create_flux_ip_adapter_state_dict enable_full_determinism() -# TODO: This standalone function maintains backward compatibility with pipeline tests -# (tests/pipelines/test_pipelines_common.py) and will be refactored. -def create_flux_ip_adapter_state_dict(model) -> dict[str, dict[str, Any]]: - """Create a dummy IP Adapter state dict for Flux transformer testing.""" - ip_cross_attn_state_dict = {} - key_id = 0 - - for name in model.attn_processors.keys(): - if name.startswith("single_transformer_blocks"): - continue - - joint_attention_dim = model.config["joint_attention_dim"] - hidden_size = model.config["num_attention_heads"] * model.config["attention_head_dim"] - sd = FluxIPAdapterAttnProcessor( - hidden_size=hidden_size, cross_attention_dim=joint_attention_dim, scale=1.0 - ).state_dict() - ip_cross_attn_state_dict.update( - { - f"{key_id}.to_k_ip.weight": sd["to_k_ip.0.weight"], - f"{key_id}.to_v_ip.weight": sd["to_v_ip.0.weight"], - f"{key_id}.to_k_ip.bias": sd["to_k_ip.0.bias"], - f"{key_id}.to_v_ip.bias": sd["to_v_ip.0.bias"], - } - ) - key_id += 1 - - image_projection = ImageProjection( - cross_attention_dim=model.config["joint_attention_dim"], - image_embed_dim=( - model.config["pooled_projection_dim"] if "pooled_projection_dim" in model.config.keys() else 768 - ), - num_image_text_embeds=4, - ) - - ip_image_projection_state_dict = {} - sd = image_projection.state_dict() - ip_image_projection_state_dict.update( - { - "proj.weight": sd["image_embeds.weight"], - "proj.bias": sd["image_embeds.bias"], - "norm.weight": sd["norm.weight"], - "norm.bias": sd["norm.bias"], - } - ) - - del sd - return {"image_proj": ip_image_projection_state_dict, "ip_adapter": ip_cross_attn_state_dict} - - class FluxTransformerTesterConfig(BaseModelTesterConfig): @property def model_class(self): diff --git a/tests/models/transformers/utils.py b/tests/models/transformers/utils.py new file mode 100644 index 000000000000..2759b0550c2b --- /dev/null +++ b/tests/models/transformers/utils.py @@ -0,0 +1,58 @@ +"""Shared test helpers for transformer model tests. + +Diffusers imports inside helpers are deliberately deferred to function bodies so the test-fetcher's +module-level import graph doesn't propagate edits to specific transformer source files through this +file into every pipeline test that imports it. +""" + +from typing import Any + + +def create_flux_ip_adapter_state_dict(model) -> dict[str, dict[str, Any]]: + """Create a dummy IP Adapter state dict for Flux transformer testing.""" + from diffusers.models.embeddings import ImageProjection + from diffusers.models.transformers.transformer_flux import FluxIPAdapterAttnProcessor + + ip_cross_attn_state_dict = {} + key_id = 0 + + for name in model.attn_processors.keys(): + if name.startswith("single_transformer_blocks"): + continue + + joint_attention_dim = model.config["joint_attention_dim"] + hidden_size = model.config["num_attention_heads"] * model.config["attention_head_dim"] + sd = FluxIPAdapterAttnProcessor( + hidden_size=hidden_size, cross_attention_dim=joint_attention_dim, scale=1.0 + ).state_dict() + ip_cross_attn_state_dict.update( + { + f"{key_id}.to_k_ip.weight": sd["to_k_ip.0.weight"], + f"{key_id}.to_v_ip.weight": sd["to_v_ip.0.weight"], + f"{key_id}.to_k_ip.bias": sd["to_k_ip.0.bias"], + f"{key_id}.to_v_ip.bias": sd["to_v_ip.0.bias"], + } + ) + key_id += 1 + + image_projection = ImageProjection( + cross_attention_dim=model.config["joint_attention_dim"], + image_embed_dim=( + model.config["pooled_projection_dim"] if "pooled_projection_dim" in model.config.keys() else 768 + ), + num_image_text_embeds=4, + ) + + ip_image_projection_state_dict = {} + sd = image_projection.state_dict() + ip_image_projection_state_dict.update( + { + "proj.weight": sd["image_embeds.weight"], + "proj.bias": sd["image_embeds.bias"], + "norm.weight": sd["norm.weight"], + "norm.bias": sd["norm.bias"], + } + ) + + del sd + return {"image_proj": ip_image_projection_state_dict, "ip_adapter": ip_cross_attn_state_dict} diff --git a/tests/others/test_flashpack.py b/tests/others/test_flashpack.py index c20836df1c98..e7eefd6835ff 100644 --- a/tests/others/test_flashpack.py +++ b/tests/others/test_flashpack.py @@ -15,7 +15,6 @@ import pytest -from diffusers import AutoPipelineForText2Image from diffusers.models.auto_model import AutoModel from ..testing_utils import is_torch_available, require_flashpack, require_torch_gpu @@ -40,6 +39,8 @@ def test_save_load_model(self, tmp_path): @require_flashpack def test_save_load_pipeline(self, tmp_path): + from diffusers import AutoPipelineForText2Image + pipeline = AutoPipelineForText2Image.from_pretrained(self.model_id) pipeline.save_pretrained(tmp_path, use_flashpack=True) assert (tmp_path / "transformer" / "model.flashpack").exists() diff --git a/tests/pipelines/kandinsky3/test_kandinsky3.py b/tests/pipelines/kandinsky3/test_kandinsky3.py index 1a1aa4b9d9ca..8a7554a8430a 100644 --- a/tests/pipelines/kandinsky3/test_kandinsky3.py +++ b/tests/pipelines/kandinsky3/test_kandinsky3.py @@ -22,8 +22,6 @@ from transformers import AutoConfig, AutoTokenizer, T5EncoderModel from diffusers import ( - AutoPipelineForImage2Image, - AutoPipelineForText2Image, Kandinsky3Pipeline, Kandinsky3UNet, VQModel, @@ -185,6 +183,8 @@ def tearDown(self): backend_empty_cache(torch_device) def test_kandinskyV3(self): + from diffusers import AutoPipelineForText2Image + pipe = AutoPipelineForText2Image.from_pretrained( "kandinsky-community/kandinsky-3", variant="fp16", torch_dtype=torch.float16 ) @@ -211,6 +211,8 @@ def test_kandinskyV3(self): self.assertTrue(np.allclose(image_np, expected_image_np, atol=5e-2)) def test_kandinskyV3_img2img(self): + from diffusers import AutoPipelineForImage2Image + pipe = AutoPipelineForImage2Image.from_pretrained( "kandinsky-community/kandinsky-3", variant="fp16", torch_dtype=torch.float16 ) diff --git a/tests/pipelines/kandinsky3/test_kandinsky3_img2img.py b/tests/pipelines/kandinsky3/test_kandinsky3_img2img.py index d9f02326b1b5..d41e644a45ef 100644 --- a/tests/pipelines/kandinsky3/test_kandinsky3_img2img.py +++ b/tests/pipelines/kandinsky3/test_kandinsky3_img2img.py @@ -23,7 +23,6 @@ from transformers import AutoConfig, AutoTokenizer, T5EncoderModel from diffusers import ( - AutoPipelineForImage2Image, Kandinsky3Img2ImgPipeline, Kandinsky3UNet, VQModel, @@ -207,6 +206,8 @@ def tearDown(self): backend_empty_cache(torch_device) def test_kandinskyV3_img2img(self): + from diffusers import AutoPipelineForImage2Image + pipe = AutoPipelineForImage2Image.from_pretrained( "kandinsky-community/kandinsky-3", variant="fp16", torch_dtype=torch.float16 ) diff --git a/tests/pipelines/pag/test_pag_sd.py b/tests/pipelines/pag/test_pag_sd.py index 1dd3ef298fd0..707603d0d5ed 100644 --- a/tests/pipelines/pag/test_pag_sd.py +++ b/tests/pipelines/pag/test_pag_sd.py @@ -23,7 +23,6 @@ from diffusers import ( AutoencoderKL, - AutoPipelineForText2Image, DDIMScheduler, StableDiffusionPAGPipeline, StableDiffusionPipeline, @@ -315,6 +314,8 @@ def get_inputs(self, device, generator_device="cpu", seed=1, guidance_scale=7.0) return inputs def test_pag_cfg(self): + from diffusers import AutoPipelineForText2Image + pipeline = AutoPipelineForText2Image.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) @@ -333,6 +334,8 @@ def test_pag_cfg(self): ) def test_pag_uncond(self): + from diffusers import AutoPipelineForText2Image + pipeline = AutoPipelineForText2Image.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) diff --git a/tests/pipelines/pag/test_pag_sd3_img2img.py b/tests/pipelines/pag/test_pag_sd3_img2img.py index ede146915c55..be5c2e118850 100644 --- a/tests/pipelines/pag/test_pag_sd3_img2img.py +++ b/tests/pipelines/pag/test_pag_sd3_img2img.py @@ -16,7 +16,6 @@ from diffusers import ( AutoencoderKL, - AutoPipelineForImage2Image, FlowMatchEulerDiscreteScheduler, SD3Transformer2DModel, StableDiffusion3Img2ImgPipeline, @@ -240,6 +239,8 @@ def get_inputs( return inputs def test_pag_cfg(self): + from diffusers import AutoPipelineForImage2Image + pipeline = AutoPipelineForImage2Image.from_pretrained( self.repo_id, enable_pag=True, torch_dtype=torch.float16, pag_applied_layers=["blocks.17"] ) @@ -268,6 +269,8 @@ def test_pag_cfg(self): ) def test_pag_uncond(self): + from diffusers import AutoPipelineForImage2Image + pipeline = AutoPipelineForImage2Image.from_pretrained( self.repo_id, enable_pag=True, torch_dtype=torch.float16, pag_applied_layers=["blocks.(4|17)"] ) diff --git a/tests/pipelines/pag/test_pag_sd_img2img.py b/tests/pipelines/pag/test_pag_sd_img2img.py index 1e9b3c24c9ac..0afbd27c677e 100644 --- a/tests/pipelines/pag/test_pag_sd_img2img.py +++ b/tests/pipelines/pag/test_pag_sd_img2img.py @@ -25,7 +25,6 @@ from diffusers import ( AutoencoderKL, AutoencoderTiny, - AutoPipelineForImage2Image, EulerDiscreteScheduler, StableDiffusionImg2ImgPipeline, StableDiffusionPAGImg2ImgPipeline, @@ -254,6 +253,8 @@ def get_inputs(self, device, generator_device="cpu", dtype=torch.float32, seed=0 return inputs def test_pag_cfg(self): + from diffusers import AutoPipelineForImage2Image + pipeline = AutoPipelineForImage2Image.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) @@ -272,6 +273,8 @@ def test_pag_cfg(self): ) def test_pag_uncond(self): + from diffusers import AutoPipelineForImage2Image + pipeline = AutoPipelineForImage2Image.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) diff --git a/tests/pipelines/pag/test_pag_sd_inpaint.py b/tests/pipelines/pag/test_pag_sd_inpaint.py index 5a78ac6ade12..f351bd03b265 100644 --- a/tests/pipelines/pag/test_pag_sd_inpaint.py +++ b/tests/pipelines/pag/test_pag_sd_inpaint.py @@ -24,7 +24,6 @@ from diffusers import ( AutoencoderKL, - AutoPipelineForInpainting, PNDMScheduler, StableDiffusionPAGInpaintPipeline, UNet2DConditionModel, @@ -289,6 +288,8 @@ def get_inputs(self, device, generator_device="cpu", seed=0, guidance_scale=7.0) return inputs def test_pag_cfg(self): + from diffusers import AutoPipelineForInpainting + pipeline = AutoPipelineForInpainting.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) @@ -307,6 +308,8 @@ def test_pag_cfg(self): ) def test_pag_uncond(self): + from diffusers import AutoPipelineForInpainting + pipeline = AutoPipelineForInpainting.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) diff --git a/tests/pipelines/pag/test_pag_sdxl.py b/tests/pipelines/pag/test_pag_sdxl.py index dc412d9b341a..74969903a553 100644 --- a/tests/pipelines/pag/test_pag_sdxl.py +++ b/tests/pipelines/pag/test_pag_sdxl.py @@ -23,7 +23,6 @@ from diffusers import ( AutoencoderKL, - AutoPipelineForText2Image, EulerDiscreteScheduler, StableDiffusionXLPAGPipeline, StableDiffusionXLPipeline, @@ -320,6 +319,8 @@ def get_inputs(self, device, generator_device="cpu", seed=0, guidance_scale=7.0) return inputs def test_pag_cfg(self): + from diffusers import AutoPipelineForText2Image + pipeline = AutoPipelineForText2Image.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) @@ -342,6 +343,8 @@ def test_pag_cfg(self): ) def test_pag_uncond(self): + from diffusers import AutoPipelineForText2Image + pipeline = AutoPipelineForText2Image.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) diff --git a/tests/pipelines/pag/test_pag_sdxl_img2img.py b/tests/pipelines/pag/test_pag_sdxl_img2img.py index 3157e3a08e8d..b4a108cdab42 100644 --- a/tests/pipelines/pag/test_pag_sdxl_img2img.py +++ b/tests/pipelines/pag/test_pag_sdxl_img2img.py @@ -32,7 +32,6 @@ from diffusers import ( AutoencoderKL, - AutoPipelineForImage2Image, EulerDiscreteScheduler, StableDiffusionXLImg2ImgPipeline, StableDiffusionXLPAGImg2ImgPipeline, @@ -302,6 +301,8 @@ def get_inputs(self, device, generator_device="cpu", seed=0, guidance_scale=7.0) return inputs def test_pag_cfg(self): + from diffusers import AutoPipelineForImage2Image + pipeline = AutoPipelineForImage2Image.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) @@ -319,6 +320,8 @@ def test_pag_cfg(self): ) def test_pag_uncond(self): + from diffusers import AutoPipelineForImage2Image + pipeline = AutoPipelineForImage2Image.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) diff --git a/tests/pipelines/pag/test_pag_sdxl_inpaint.py b/tests/pipelines/pag/test_pag_sdxl_inpaint.py index 628da17c8dc3..3b6b23ed17db 100644 --- a/tests/pipelines/pag/test_pag_sdxl_inpaint.py +++ b/tests/pipelines/pag/test_pag_sdxl_inpaint.py @@ -33,7 +33,6 @@ from diffusers import ( AutoencoderKL, - AutoPipelineForInpainting, EulerDiscreteScheduler, StableDiffusionXLInpaintPipeline, StableDiffusionXLPAGInpaintPipeline, @@ -308,6 +307,8 @@ def get_inputs(self, device, generator_device="cpu", seed=0, guidance_scale=7.0) return inputs def test_pag_cfg(self): + from diffusers import AutoPipelineForInpainting + pipeline = AutoPipelineForInpainting.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) @@ -325,6 +326,8 @@ def test_pag_cfg(self): ) def test_pag_uncond(self): + from diffusers import AutoPipelineForInpainting + pipeline = AutoPipelineForInpainting.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) diff --git a/tests/pipelines/test_pipelines_common.py b/tests/pipelines/test_pipelines_common.py index 106ba55cf149..71c92a10311d 100644 --- a/tests/pipelines/test_pipelines_common.py +++ b/tests/pipelines/test_pipelines_common.py @@ -53,7 +53,7 @@ get_autoencoder_tiny_config, get_consistency_vae_config, ) -from ..models.transformers.test_models_transformer_flux import create_flux_ip_adapter_state_dict +from ..models.transformers.utils import create_flux_ip_adapter_state_dict from ..models.unets.test_models_unet_2d_condition import ( create_ip_adapter_faceid_state_dict, create_ip_adapter_state_dict, From 4242566fdee9b6ac21a139a652c4f369198bae7a Mon Sep 17 00:00:00 2001 From: DN6 Date: Wed, 26 Aug 2026 11:42:08 +0530 Subject: [PATCH 03/24] [CI] Give `single_file` model tests their own fetcher job; they need an HF token, not an accelerator --- utils/tests_fetcher.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index 8afa2994fa44..718e029319fb 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -74,10 +74,12 @@ "memory": ["memory", "cpu_offload", "group_offload"], "cache": ["cache"], "ip_adapter": ["ip_adapter"], + # Needs only an HF token (gated checkpoints), which the CPU job provides; skips on fork PRs. + "single_file": ["single_file"], } -# Gated on an accelerator, multi-GPU, or an HF token, so every test skips on the CPU runner. Excluded -# from `core` and given no job rather than spinning up a runner to skip everything. -CPU_SKIPPED_MARKERS = ["quantization", "compile", "single_file", "training", "context_parallel", "tensor_parallel"] +# Gated on an accelerator or multi-GPU, so every test skips on the CPU runner. Excluded from `core` and +# given no job rather than spinning up a runner to skip everything. +CPU_SKIPPED_MARKERS = ["quantization", "compile", "training", "context_parallel", "tensor_parallel"] FEATURE_MARKERS = [m for markers in FEATURE_GROUPS.values() for m in markers] CORE_MARKERS = "not (" + " or ".join(FEATURE_MARKERS + CPU_SKIPPED_MARKERS) + ")" # ============================================================ From bd8d8b8f4a6d7755fd9e26b3491192453c361a1e Mon Sep 17 00:00:00 2001 From: DN6 Date: Wed, 26 Aug 2026 14:57:37 +0530 Subject: [PATCH 04/24] [tests] Keep `create_flux_ip_adapter_state_dict` in `test_models_transformer_flux.py` instead of a separate `utils.py` --- .../test_models_transformer_flux.py | 51 +++++++++++++++- tests/models/transformers/utils.py | 58 ------------------- tests/pipelines/test_pipelines_common.py | 2 +- 3 files changed, 51 insertions(+), 60 deletions(-) delete mode 100644 tests/models/transformers/utils.py diff --git a/tests/models/transformers/test_models_transformer_flux.py b/tests/models/transformers/test_models_transformer_flux.py index 27a80c4b9a02..d60e34e2ea3f 100644 --- a/tests/models/transformers/test_models_transformer_flux.py +++ b/tests/models/transformers/test_models_transformer_flux.py @@ -23,6 +23,7 @@ import torch from diffusers import BitsAndBytesConfig, FluxTransformer2DModel, GGUFQuantizationConfig +from diffusers.models.embeddings import ImageProjection from diffusers.models.transformers.transformer_flux import FluxIPAdapterAttnProcessor from diffusers.utils.torch_utils import randn_tensor @@ -59,12 +60,60 @@ TorchCompileTesterMixin, TrainingTesterMixin, ) -from .utils import create_flux_ip_adapter_state_dict enable_full_determinism() +# TODO: This standalone function maintains backward compatibility with pipeline tests +# (tests/pipelines/test_pipelines_common.py) and will be refactored. +def create_flux_ip_adapter_state_dict(model) -> dict[str, dict[str, Any]]: + """Create a dummy IP Adapter state dict for Flux transformer testing.""" + ip_cross_attn_state_dict = {} + key_id = 0 + + for name in model.attn_processors.keys(): + if name.startswith("single_transformer_blocks"): + continue + + joint_attention_dim = model.config["joint_attention_dim"] + hidden_size = model.config["num_attention_heads"] * model.config["attention_head_dim"] + sd = FluxIPAdapterAttnProcessor( + hidden_size=hidden_size, cross_attention_dim=joint_attention_dim, scale=1.0 + ).state_dict() + ip_cross_attn_state_dict.update( + { + f"{key_id}.to_k_ip.weight": sd["to_k_ip.0.weight"], + f"{key_id}.to_v_ip.weight": sd["to_v_ip.0.weight"], + f"{key_id}.to_k_ip.bias": sd["to_k_ip.0.bias"], + f"{key_id}.to_v_ip.bias": sd["to_v_ip.0.bias"], + } + ) + key_id += 1 + + image_projection = ImageProjection( + cross_attention_dim=model.config["joint_attention_dim"], + image_embed_dim=( + model.config["pooled_projection_dim"] if "pooled_projection_dim" in model.config.keys() else 768 + ), + num_image_text_embeds=4, + ) + + ip_image_projection_state_dict = {} + sd = image_projection.state_dict() + ip_image_projection_state_dict.update( + { + "proj.weight": sd["image_embeds.weight"], + "proj.bias": sd["image_embeds.bias"], + "norm.weight": sd["norm.weight"], + "norm.bias": sd["norm.bias"], + } + ) + + del sd + return {"image_proj": ip_image_projection_state_dict, "ip_adapter": ip_cross_attn_state_dict} + + class FluxTransformerTesterConfig(BaseModelTesterConfig): @property def model_class(self): diff --git a/tests/models/transformers/utils.py b/tests/models/transformers/utils.py deleted file mode 100644 index 2759b0550c2b..000000000000 --- a/tests/models/transformers/utils.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Shared test helpers for transformer model tests. - -Diffusers imports inside helpers are deliberately deferred to function bodies so the test-fetcher's -module-level import graph doesn't propagate edits to specific transformer source files through this -file into every pipeline test that imports it. -""" - -from typing import Any - - -def create_flux_ip_adapter_state_dict(model) -> dict[str, dict[str, Any]]: - """Create a dummy IP Adapter state dict for Flux transformer testing.""" - from diffusers.models.embeddings import ImageProjection - from diffusers.models.transformers.transformer_flux import FluxIPAdapterAttnProcessor - - ip_cross_attn_state_dict = {} - key_id = 0 - - for name in model.attn_processors.keys(): - if name.startswith("single_transformer_blocks"): - continue - - joint_attention_dim = model.config["joint_attention_dim"] - hidden_size = model.config["num_attention_heads"] * model.config["attention_head_dim"] - sd = FluxIPAdapterAttnProcessor( - hidden_size=hidden_size, cross_attention_dim=joint_attention_dim, scale=1.0 - ).state_dict() - ip_cross_attn_state_dict.update( - { - f"{key_id}.to_k_ip.weight": sd["to_k_ip.0.weight"], - f"{key_id}.to_v_ip.weight": sd["to_v_ip.0.weight"], - f"{key_id}.to_k_ip.bias": sd["to_k_ip.0.bias"], - f"{key_id}.to_v_ip.bias": sd["to_v_ip.0.bias"], - } - ) - key_id += 1 - - image_projection = ImageProjection( - cross_attention_dim=model.config["joint_attention_dim"], - image_embed_dim=( - model.config["pooled_projection_dim"] if "pooled_projection_dim" in model.config.keys() else 768 - ), - num_image_text_embeds=4, - ) - - ip_image_projection_state_dict = {} - sd = image_projection.state_dict() - ip_image_projection_state_dict.update( - { - "proj.weight": sd["image_embeds.weight"], - "proj.bias": sd["image_embeds.bias"], - "norm.weight": sd["norm.weight"], - "norm.bias": sd["norm.bias"], - } - ) - - del sd - return {"image_proj": ip_image_projection_state_dict, "ip_adapter": ip_cross_attn_state_dict} diff --git a/tests/pipelines/test_pipelines_common.py b/tests/pipelines/test_pipelines_common.py index 71c92a10311d..106ba55cf149 100644 --- a/tests/pipelines/test_pipelines_common.py +++ b/tests/pipelines/test_pipelines_common.py @@ -53,7 +53,7 @@ get_autoencoder_tiny_config, get_consistency_vae_config, ) -from ..models.transformers.utils import create_flux_ip_adapter_state_dict +from ..models.transformers.test_models_transformer_flux import create_flux_ip_adapter_state_dict from ..models.unets.test_models_unet_2d_condition import ( create_ip_adapter_faceid_state_dict, create_ip_adapter_state_dict, From b19052cf3e8eb00ac16ff1738db67d49ab0eda38 Mon Sep 17 00:00:00 2001 From: DN6 Date: Wed, 30 Sep 2026 16:24:37 +0530 Subject: [PATCH 05/24] update --- .github/workflows/pr_test_fetcher.yml | 5 + utils/tests_fetcher.py | 164 ++++++++++++++++++++------ 2 files changed, 135 insertions(+), 34 deletions(-) diff --git a/.github/workflows/pr_test_fetcher.yml b/.github/workflows/pr_test_fetcher.yml index aef711f01cb2..f97be5df1b2c 100644 --- a/.github/workflows/pr_test_fetcher.yml +++ b/.github/workflows/pr_test_fetcher.yml @@ -43,6 +43,11 @@ jobs: - name: Fetch Tests run: | python utils/tests_fetcher.py | tee test_preparation.txt + - name: Report selection in job summary + run: | + if [ -f test_fetcher_summary.md ]; then + cat test_fetcher_summary.md >> $GITHUB_STEP_SUMMARY + fi - name: Report fetched tests uses: actions/upload-artifact@v6 with: diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index 718e029319fb..eb6ec15559c1 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -21,14 +21,14 @@ it), and select the impacted tests. There is no automatic full-suite trigger. If a change is in territory the import graph can't see -correctly (dynamic dispatch via auto-mappings, lazy `_import_structure`, etc.), apply the `run-all-tests` -PR label or pass `--force_full_suite` to bypass selection. +correctly (dynamic dispatch via auto-mappings, lazy `_import_structure`, etc.), pass `--force_full_suite` +to bypass selection. Pipeline-specific note: diffusers' `__init__.py` files use the `_import_structure = {...}` lazy-loading pattern paired with an `if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT:` block containing real -`from .submodule import Class` statements. AST extraction sees the TYPE_CHECKING imports (since -`ast.walk` descends into `If` / `Try` blocks regardless of runtime conditions), so the import graph -mirrors the actual public API. +`from .submodule import Class` statements. The import walker descends into `If` / `Try` blocks regardless +of runtime conditions, so it sees the TYPE_CHECKING imports and the import graph mirrors the actual public +API. Stage 1 — diff: list modified Python files (vs. merge-base with main, or the previous commit on main). Docstring/comment-only changes are filtered out by content comparison. @@ -38,6 +38,8 @@ Stage 4 — bucket: group tests by top-level `tests/` folder for the CI matrix. `tests/models` and `tests/pipelines` are split further into one job per feature mixin (via the pytest markers the mixins carry) so a large selection fans out instead of serialising in one job. +Stage 5 — report: print the import chain that selected each test (kept as a CI artifact) and write a + markdown job summary of how many tests each modified file triggers. Usage: @@ -149,7 +151,7 @@ def get_diff(repo: Repo, base_commit, commits) -> List[str]: code_diff.extend(paths) elif not diff_is_docstring_only(repo, commit, d.b_path): code_diff.append(d.b_path) - return code_diff + return list(dict.fromkeys(code_diff)) def get_modified_python_files(diff_with_last_commit: bool = False) -> List[str]: @@ -166,15 +168,6 @@ def get_modified_python_files(diff_with_last_commit: bool = False) -> List[str]: return get_diff(repo, repo.head.commit, commits) -def get_all_tests() -> List[str]: - """Top-level entries under `tests/` (folders + `tests/test_*.py`), used to expand a full-suite selection.""" - return sorted( - f"tests/{p.name}" - for p in PATH_TO_TESTS.iterdir() - if "__pycache__" not in p.name and (p.is_dir() or p.name.startswith("test_")) - ) - - # ============================================================ # AST-based import extraction # ============================================================ @@ -282,7 +275,7 @@ def get_module_dependencies(module_file: str, cache: Dict[str, List[Tuple[str, L dependencies: List[str] = [] queue: List[Tuple[str, List[str]]] = list(cache[module_file]) - seen_inits: set = set() + seen: set = set() while queue: target, symbols = queue.pop(0) @@ -291,17 +284,21 @@ def get_module_dependencies(module_file: str, cache: Dict[str, List[Tuple[str, L dependencies.append(target) continue - # Avoid cycles through inits importing each other. - if target in seen_inits: + # A symbol arriving at an init it already passed through means inits import each other in a cycle; + # keep the init as the dep instead of looping. Tracked per (init, symbol) so a second statement + # importing other symbols from the same package still resolves them to their defining modules. + unseen = [s for s in symbols if (target, s) not in seen] + if len(unseen) < len(symbols): dependencies.append(target) + if not unseen: continue - seen_inits.add(target) + seen.update((target, s) for s in unseen) if target not in cache: cache[target] = _extract_imports(target) init_imports = cache[target] - unresolved = list(symbols) + unresolved = unseen for sub_target, sub_names in init_imports: matched = [s for s in unresolved if s in sub_names] if matched: @@ -334,19 +331,26 @@ def _merged_nested_deps(m: str, direct_deps: Dict[str, List[str]]) -> bool: return merged -def create_reverse_dependency_map() -> Dict[str, List[str]]: - """Build the reverse dependency map: file → list of files that transitively depend on it. - - 1. Compute direct deps for every .py under `src/diffusers/` and `tests/`. - 2. Transitively close (skipping inits during recursion to avoid pulling in the universe via the root init). - 3. Invert. - """ +def create_direct_dependency_map() -> Dict[str, List[str]]: + """Direct (one-hop) deps of every .py under `src/diffusers/` and `tests/`: file → files it imports.""" cache: Dict[str, List[Tuple[str, List[str]]]] = {} all_modules = [ str(p.relative_to(PATH_TO_REPO)) for p in list(PATH_TO_DIFFUSERS.glob("**/*.py")) + list(PATH_TO_TESTS.glob("**/*.py")) ] - direct_deps: Dict[str, List[str]] = {m: get_module_dependencies(m, cache) for m in all_modules} + return {m: get_module_dependencies(m, cache) for m in all_modules} + + +def create_reverse_dependency_map(direct_deps: Dict[str, List[str]]) -> Dict[str, List[str]]: + """Build the reverse dependency map: file → list of files that transitively depend on it. + + 1. Transitively close the direct deps (skipping inits during recursion to avoid pulling in the universe + via the root init). + 2. Invert. + """ + all_modules = list(direct_deps) + one_hop_deps = direct_deps + direct_deps = {m: list(deps) for m, deps in direct_deps.items()} # Each pass propagates dependency info one level deeper. Loop until a full pass adds nothing. changed = True @@ -363,7 +367,7 @@ def create_reverse_dependency_map() -> Dict[str, List[str]]: # For inits, do the forward direction: editing an init impacts everything it re-exports. for init in [m for m in all_modules if m.endswith("__init__.py")]: - deps = get_module_dependencies(init, cache) + deps = one_hop_deps[init] impacted = set(deps) for d in deps: if not d.endswith("__init__.py"): @@ -373,6 +377,88 @@ def create_reverse_dependency_map() -> Dict[str, List[str]]: return dict(reverse_map) +def _import_chain(test_file: str, modified_files: List[str], direct_deps: Dict[str, List[str]]) -> Optional[List[str]]: + """Shortest import chain from a modified file to `test_file`, or None if there is none. + + Walks `test_file`'s imports outward with the same rules as the transitive closure: an `__init__.py` + reached as a dependency is not expanded further. A modified init is reached through the modules it + re-exports, mirroring the forward direction `create_reverse_dependency_map` applies to inits. + """ + targets = {f: f for f in modified_files} + for f in modified_files: + if f.endswith("__init__.py"): + for d in direct_deps.get(f, []): + if not d.endswith("__init__.py"): + targets.setdefault(d, f) + + parents: Dict[str, Optional[str]] = {test_file: None} + queue = collections.deque([test_file]) + while queue: + node = queue.popleft() + if node in targets: + chain = [] if targets[node] == node else [targets[node]] + while node is not None: + chain.append(node) + node = parents[node] + return chain + if node.endswith("__init__.py") and node != test_file: + continue + for d in direct_deps.get(node, []): + if d not in parents: + parents[d] = node + queue.append(d) + return None + + +def _print_selection_report(test_files: List[str], modified_files: List[str], direct_deps: Dict[str, List[str]]): + """Print, for every selected test, the import chain from the modified file that pulled it in.""" + print(f"\nSelected {len(test_files)} test files from {len(modified_files)} modified Python files.") + print("Each entry shows the import chain from a modified file to the test (modified file first):\n") + for test_file in test_files: + chain = _import_chain(test_file, modified_files, direct_deps) + if chain == [test_file]: + reason = "modified directly" + elif chain is None: + reason = "no direct import chain found; selected through the reverse map" + else: + reason = " -> ".join(chain[:-1]) + print(f"{test_file}\n {reason}") + + +def _tests_triggered_by(f: str, reverse_map: Dict[str, List[str]]) -> int: + """Number of test files a change to `f` selects, counting `f` itself when it is a test.""" + triggered = {t for t in reverse_map.get(f, []) if _is_test_file(t)} + if _is_test_file(f): + triggered.add(f) + return len(triggered) + + +def _write_summary( + summary_file: str, modified_files: List[str], test_files: List[str], reverse_map: Dict[str, List[str]] +): + """Write a short markdown report for the CI job summary: how many tests each modified file triggers. + + A file with outsized reach is the cue for a maintainer to open the per-test import chains in the artifact. + """ + lines = [ + "## Test fetcher", + "", + f"Selected {len(test_files)} test files from {len(modified_files)} modified Python files. " + "The import chain that selected each test is in the `test_fetched` artifact.", + "", + "| Modified file | Tests it triggers |", + "|---|---|", + ] + # A modified file that pulls in nothing beyond itself adds no information; only files with reach get a row. + triggered = {f: _tests_triggered_by(f, reverse_map) for f in modified_files} + for f in sorted(modified_files, key=lambda f: (-triggered[f], f)): + if triggered[f] > (1 if _is_test_file(f) else 0): + lines.append(f"| `{f}` | {triggered[f]} |") + lines.append(f"| **Distinct tests selected** | **{len(test_files)}** |") + + Path(summary_file).write_text("\n".join(lines) + "\n", encoding="utf-8") + + # ============================================================ # Test selection # ============================================================ @@ -462,7 +548,8 @@ def _feature_groups_for(paths: List[str], mixin_markers: Dict[str, List[str]]) - for group, markers in FEATURE_GROUPS.items(): if any(m & set(markers) for m in class_markers): groups.append(group) - return groups + # Function-style test files declare no class, so no marker: they are core tests. + return groups or ["core"] def _matrix_entries(test_map: Dict[str, List[str]]) -> List[Dict[str, str]]: @@ -490,10 +577,11 @@ def _is_test_file(path: str) -> bool: return path.startswith("tests/") and Path(path).name.startswith("test_") -def fetch_tests_to_run(json_output_file: str, diff_with_last_commit: bool): - """Determine the tests to run from the diff and write `test_map.json`.""" +def fetch_tests_to_run(json_output_file: str, summary_file: str, diff_with_last_commit: bool): + """Determine the tests to run from the diff, write `test_map.json` and the markdown job summary.""" modified_files = get_modified_python_files(diff_with_last_commit=diff_with_last_commit) - reverse_map = create_reverse_dependency_map() + direct_deps = create_direct_dependency_map() + reverse_map = create_reverse_dependency_map(direct_deps) # Each modified file contributes itself (if it's a test) plus tests transitively impacted by it. selected = set() @@ -503,6 +591,8 @@ def fetch_tests_to_run(json_output_file: str, diff_with_last_commit: bool): selected.update(t for t in reverse_map.get(f, []) if _is_test_file(t)) test_files_to_run = sorted(p for p in selected if (PATH_TO_REPO / p).exists()) + _print_selection_report(test_files_to_run, modified_files, direct_deps) + _write_summary(summary_file, modified_files, test_files_to_run, reverse_map) _write_matrix(json_output_file, test_files_to_run) @@ -531,6 +621,12 @@ def _write_full_suite(json_output_file: str): default="test_map.json", help="Where to store the list of matrix entries (name / paths / markers) consumed by CI.", ) + parser.add_argument( + "--summary_output_file", + type=str, + default="test_fetcher_summary.md", + help="Where to store the markdown report appended to the CI job summary.", + ) parser.add_argument( "--diff_with_last_commit", action="store_true", @@ -555,7 +651,7 @@ def _write_full_suite(json_output_file: str): diff_with_last_commit = True try: - fetch_tests_to_run(args.json_output_file, diff_with_last_commit) + fetch_tests_to_run(args.json_output_file, args.summary_output_file, diff_with_last_commit) except Exception as e: import traceback From 7e674e092e0bcb463429a67d4591cf281f1b6705 Mon Sep 17 00:00:00 2001 From: DN6 Date: Wed, 30 Sep 2026 16:27:32 +0530 Subject: [PATCH 06/24] [CI] Detect the `main` branch by name in `tests_fetcher.py`; a branch checkout has no local `main` ref --- utils/tests_fetcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index eb6ec15559c1..3a5841a45669 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -646,7 +646,7 @@ def _write_full_suite(json_output_file: str): repo = Repo(PATH_TO_REPO) diff_with_last_commit = args.diff_with_last_commit - if not diff_with_last_commit and not repo.head.is_detached and repo.head.ref == repo.refs.main: + if not diff_with_last_commit and not repo.head.is_detached and repo.head.ref.name == "main": print("main branch detected, fetching tests against last commit.") diff_with_last_commit = True From dbc8d8ac77213801c042204b4e9b1b685e898e57 Mon Sep 17 00:00:00 2001 From: DN6 Date: Wed, 30 Sep 2026 16:42:41 +0530 Subject: [PATCH 07/24] [CI] Mark the workspace as a git safe directory before running `tests_fetcher.py` in the container --- .github/workflows/pr_test_fetcher.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pr_test_fetcher.yml b/.github/workflows/pr_test_fetcher.yml index f97be5df1b2c..db8c974512ec 100644 --- a/.github/workflows/pr_test_fetcher.yml +++ b/.github/workflows/pr_test_fetcher.yml @@ -41,7 +41,10 @@ jobs: diffusers-cli env echo $(git --version) - name: Fetch Tests + # The checkout step only marks the workspace safe in a temporary git config, so git processes the + # fetcher spawns in the container would otherwise refuse the repo as dubiously owned. run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" python utils/tests_fetcher.py | tee test_preparation.txt - name: Report selection in job summary run: | From 51e07287bc3794512e3e47267c7f0c133c57ddd3 Mon Sep 17 00:00:00 2001 From: DN6 Date: Wed, 30 Sep 2026 17:24:02 +0530 Subject: [PATCH 08/24] [CI] Select core tests with a `core` marker derived from the `is_*` decorators `tests/conftest.py` derives `NON_CORE_MARKERS` from the markers the `is_*` decorators in `testing_utils.py` apply and marks every test carrying none of them as `core` at collection time. The fetcher imports that set instead of keeping its own list and emits `-m core` for the core jobs in place of the `not (...)` complement expression. Also drop `-v` from the pytest run, which no other workflow passes. --- .github/workflows/pr_test_fetcher.yml | 2 +- tests/conftest.py | 41 ++++++++++++++++++++++++ utils/tests_fetcher.py | 45 +++++++++++---------------- 3 files changed, 60 insertions(+), 28 deletions(-) diff --git a/.github/workflows/pr_test_fetcher.yml b/.github/workflows/pr_test_fetcher.yml index db8c974512ec..5f7621b68b1f 100644 --- a/.github/workflows/pr_test_fetcher.yml +++ b/.github/workflows/pr_test_fetcher.yml @@ -103,7 +103,7 @@ jobs: - name: Run all selected tests on CPU run: | - pytest -n 2 --dist=loadfile -v -m "${{ matrix.markers }}" --make-reports=${{ matrix.name }}_tests_cpu ${{ matrix.paths }} + pytest -n 2 --dist=loadfile -m "${{ matrix.markers }}" --make-reports=${{ matrix.name }}_tests_cpu ${{ matrix.paths }} - name: Failure short reports if: ${{ failure() }} diff --git a/tests/conftest.py b/tests/conftest.py index b6f8c18183c7..48546c9c1a1e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,10 +15,13 @@ # tests directory-specific settings - this file is run automatically # by pytest before any tests are run +import ast import sys import warnings from os.path import abspath, dirname, join +import pytest + # allow having multiple repository checkouts and not needing to remember to rerun # 'pip install -e .[dev]' when switching between checkouts and running tests. @@ -30,7 +33,37 @@ warnings.simplefilter(action="ignore", category=FutureWarning) +def marker_decorators(): + """Map each `is_*` decorator in testing_utils.py to the pytest marker it applies. + + Read from the source with `ast` rather than imported, so `utils/tests_fetcher.py` can call this without + importing torch. Matches `def is_x(test_case): return pytest.mark.(test_case)`. + """ + tree = ast.parse(open(join(dirname(__file__), "testing_utils.py"), encoding="utf-8").read()) + decorators = {} + for node in ast.walk(tree): + if not isinstance(node, ast.FunctionDef) or not node.name.startswith("is_"): + continue + for ret in ast.walk(node): + if isinstance(ret, ast.Return) and isinstance(ret.value, ast.Call): + func = ret.value.func + if ( + isinstance(func, ast.Attribute) + and isinstance(func.value, ast.Attribute) + and func.value.attr == "mark" + ): + decorators[node.name] = func.attr + return decorators + + +# Every marker an `is_*` decorator applies takes a test out of the `core` set. CI runs each feature in its +# own job (`-m lora`, ...) and everything else as `-m core`, so a test carrying none of these is marked +# `core` at collection time. `utils/tests_fetcher.py` uses the same set to decide which jobs to schedule. +NON_CORE_MARKERS = set(marker_decorators().values()) + + def pytest_configure(config): + config.addinivalue_line("markers", "core: marks tests carrying none of the feature markers (added automatically)") config.addinivalue_line("markers", "big_accelerator: marks tests as requiring big accelerator resources") config.addinivalue_line("markers", "lora: marks tests for LoRA/PEFT functionality") config.addinivalue_line("markers", "ip_adapter: marks tests for IP Adapter functionality") @@ -54,6 +87,14 @@ def pytest_configure(config): config.addinivalue_line("markers", "nightly: mark test as nightly") +@pytest.hookimpl(tryfirst=True) +def pytest_collection_modifyitems(items): + """Mark every test with no feature marker as `core`, before `-m` deselection runs.""" + for item in items: + if not any(m.name in NON_CORE_MARKERS for m in item.iter_markers()): + item.add_marker("core") + + def pytest_addoption(parser): from .testing_utils import pytest_addoption_shared diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index 3a5841a45669..04e986ba53b3 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -54,6 +54,7 @@ import ast import collections import json +import sys from contextlib import contextmanager from pathlib import Path from typing import Dict, List, Optional, Tuple @@ -62,13 +63,20 @@ PATH_TO_REPO = Path(__file__).parent.parent.resolve() +sys.path.insert(0, str(PATH_TO_REPO)) +from tests.conftest import NON_CORE_MARKERS, marker_decorators # noqa: E402 + + PATH_TO_DIFFUSERS = PATH_TO_REPO / "src/diffusers" PATH_TO_TESTS = PATH_TO_REPO / "tests" # `tests/models` and `tests/pipelines` compose one test class per feature mixin, and each mixin carries a # pytest marker (see the `is_*` decorators in tests/testing_utils.py). Each group below becomes its own # matrix job, selected with `pytest -m `, but only when a selected file composes a mixin carrying -# one of the group's markers; `core` is everything no group and no CPU-skipped marker claims. +# one of the group's markers. `core` is every test carrying none of `NON_CORE_MARKERS` (tests/conftest.py +# derives that set from the decorators and marks those tests at collection time). Non-core markers with no +# group here are gated on an accelerator or multi-GPU, so they get no CPU job rather than spinning up a +# runner to skip everything. SPLIT_BY_FEATURE = {"models", "pipelines"} FEATURE_GROUPS = { "lora": ["lora"], @@ -79,11 +87,13 @@ # Needs only an HF token (gated checkpoints), which the CPU job provides; skips on fork PRs. "single_file": ["single_file"], } -# Gated on an accelerator or multi-GPU, so every test skips on the CPU runner. Excluded from `core` and -# given no job rather than spinning up a runner to skip everything. -CPU_SKIPPED_MARKERS = ["quantization", "compile", "training", "context_parallel", "tensor_parallel"] FEATURE_MARKERS = [m for markers in FEATURE_GROUPS.values() for m in markers] -CORE_MARKERS = "not (" + " or ".join(FEATURE_MARKERS + CPU_SKIPPED_MARKERS) + ")" +_unknown = sorted(set(FEATURE_MARKERS) - NON_CORE_MARKERS) +if _unknown: + raise ValueError( + f"FEATURE_GROUPS markers {_unknown} have no `is_*` decorator in tests/testing_utils.py, so their tests " + "would also run in `core`. Add the decorator or drop the group." + ) # ============================================================ # Generic helpers # ============================================================ @@ -482,25 +492,6 @@ def _base_names(node: ast.ClassDef) -> List[str]: ] -def _marker_decorators() -> Dict[str, str]: - """Map decorator function name → pytest marker, from `def is_x(test_case): return pytest.mark.(test_case)`.""" - tree = ast.parse((PATH_TO_TESTS / "testing_utils.py").read_text(encoding="utf-8")) - decorators = {} - for node in ast.walk(tree): - if not isinstance(node, ast.FunctionDef) or not node.name.startswith("is_"): - continue - for ret in ast.walk(node): - if isinstance(ret, ast.Return) and isinstance(ret.value, ast.Call): - func = ret.value.func - if ( - isinstance(func, ast.Attribute) - and isinstance(func.value, ast.Attribute) - and func.value.attr == "mark" - ): - decorators[node.name] = func.attr - return decorators - - def _mixin_markers(bucket: str, decorators: Dict[str, str]) -> Dict[str, List[str]]: """Map mixin class name → markers it carries, from the `@is_*` decorators on classes in `tests//testing_utils/`. Markers are inherited through pytest, so a mixin also carries those of @@ -541,7 +532,7 @@ def _class_markers(test_file: str, mixin_markers: Dict[str, List[str]]) -> List[ def _feature_groups_for(paths: List[str], mixin_markers: Dict[str, List[str]]) -> List[str]: """Names of the feature groups (including `core`) that at least one test class in `paths` would land in.""" class_markers = [m for p in paths for m in _class_markers(p, mixin_markers)] - non_core = set(FEATURE_MARKERS + CPU_SKIPPED_MARKERS) + non_core = NON_CORE_MARKERS groups = [] if any(not m & non_core for m in class_markers): groups.append("core") @@ -554,7 +545,7 @@ def _feature_groups_for(paths: List[str], mixin_markers: Dict[str, List[str]]) - def _matrix_entries(test_map: Dict[str, List[str]]) -> List[Dict[str, str]]: """Expand buckets into CI matrix entries: `{"name", "paths", "markers"}`, one job each.""" - decorators = _marker_decorators() + decorators = marker_decorators() entries = [] for bucket, paths in sorted(test_map.items()): joined = " ".join(paths) @@ -562,7 +553,7 @@ def _matrix_entries(test_map: Dict[str, List[str]]) -> List[Dict[str, str]]: entries.append({"name": bucket, "paths": joined, "markers": ""}) continue for group in _feature_groups_for(paths, _mixin_markers(bucket, decorators)): - expr = CORE_MARKERS if group == "core" else " or ".join(FEATURE_GROUPS[group]) + expr = "core" if group == "core" else " or ".join(FEATURE_GROUPS[group]) entries.append({"name": f"{bucket}-{group}", "paths": joined, "markers": expr}) return entries From e2581306d08e8c7084cbe1d827a30e87bfb972b3 Mon Sep 17 00:00:00 2001 From: DN6 Date: Mon, 5 Oct 2026 14:07:03 +0530 Subject: [PATCH 09/24] [CI] Credit mixin markers only for bases imported from `testing_utils`; legacy mixins share their names The legacy cache and IP-Adapter mixins in `test_pipelines_common.py` are unmarked but have the same class names as the marked ones, so matching by bare name scheduled a `pipelines-cache` job whose `-m cache` run collected nothing and failed with exit code 5. --- utils/tests_fetcher.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index 04e986ba53b3..34fd7914b29e 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -519,19 +519,30 @@ def _mixin_markers(bucket: str, decorators: Dict[str, str]) -> Dict[str, List[st return {cls: sorted(m) for cls, m in markers.items() if m} -def _class_markers(test_file: str, mixin_markers: Dict[str, List[str]]) -> List[set]: - """Markers of every test class in `test_file`, derived from the marked mixins it composes.""" +def _class_markers(test_file: str, bucket: str, mixin_markers: Dict[str, List[str]]) -> List[set]: + """Markers of every test class in `test_file`, derived from the marked mixins it composes. + + A base counts only if the file imports it from `tests//testing_utils`: the legacy mixins in + `test_pipelines_common.py` share names with the marked ones but carry no marker, and crediting them + would schedule a feature job that collects nothing. + """ tree = ast.parse((PATH_TO_REPO / test_file).read_text(encoding="utf-8")) + importer_pkg = list(Path(test_file).parts[:-1]) + imported: Dict[str, str] = {} + for node in _iter_module_level_imports(tree): + parts = _resolve_import(node.module, node.level, importer_pkg) + if parts is not None and parts[:3] == ["tests", bucket, "testing_utils"]: + imported.update({alias.asname or alias.name: alias.name for alias in node.names}) return [ - {m for base in _base_names(node) for m in mixin_markers.get(base, [])} + {m for base in _base_names(node) if base in imported for m in mixin_markers.get(imported[base], [])} for node in ast.walk(tree) if isinstance(node, ast.ClassDef) ] -def _feature_groups_for(paths: List[str], mixin_markers: Dict[str, List[str]]) -> List[str]: +def _feature_groups_for(paths: List[str], bucket: str, mixin_markers: Dict[str, List[str]]) -> List[str]: """Names of the feature groups (including `core`) that at least one test class in `paths` would land in.""" - class_markers = [m for p in paths for m in _class_markers(p, mixin_markers)] + class_markers = [m for p in paths for m in _class_markers(p, bucket, mixin_markers)] non_core = NON_CORE_MARKERS groups = [] if any(not m & non_core for m in class_markers): @@ -552,7 +563,7 @@ def _matrix_entries(test_map: Dict[str, List[str]]) -> List[Dict[str, str]]: if bucket not in SPLIT_BY_FEATURE: entries.append({"name": bucket, "paths": joined, "markers": ""}) continue - for group in _feature_groups_for(paths, _mixin_markers(bucket, decorators)): + for group in _feature_groups_for(paths, bucket, _mixin_markers(bucket, decorators)): expr = "core" if group == "core" else " or ".join(FEATURE_GROUPS[group]) entries.append({"name": f"{bucket}-{group}", "paths": joined, "markers": expr}) return entries From a8b2c8e0968bce8d6fb185f27bc4f96d51c65396 Mon Sep 17 00:00:00 2001 From: "hf-security-analysis[bot]" <265538906+hf-security-analysis[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 10:37:41 +0200 Subject: [PATCH 10/24] fix(ci): harden GitHub Actions workflows (#14607) (#14945) fix(ci): harden workflow files flagged on #14607 Co-authored-by: hf-security-analysis[bot] <265538906+hf-security-analysis[bot]@users.noreply.github.com> --- .github/workflows/pr_test_fetcher.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr_test_fetcher.yml b/.github/workflows/pr_test_fetcher.yml index 5f7621b68b1f..087f2a2bee86 100644 --- a/.github/workflows/pr_test_fetcher.yml +++ b/.github/workflows/pr_test_fetcher.yml @@ -30,7 +30,7 @@ jobs: matrix: ${{ steps.set_matrix.outputs.matrix }} steps: - name: Checkout diffusers - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: fetch-depth: 0 - name: Install dependencies @@ -52,7 +52,7 @@ jobs: cat test_fetcher_summary.md >> $GITHUB_STEP_SUMMARY fi - name: Report fetched tests - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: test_fetched path: test_preparation.txt @@ -88,7 +88,7 @@ jobs: shell: bash steps: - name: Checkout diffusers - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: fetch-depth: 2 @@ -114,7 +114,7 @@ jobs: - name: Test suite reports artifacts if: ${{ always() }} - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: ${{ matrix.name }}_test_reports path: reports @@ -143,7 +143,7 @@ jobs: steps: - name: Checkout diffusers - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: fetch-depth: 2 @@ -169,7 +169,7 @@ jobs: - name: Test suite reports artifacts if: ${{ always() }} - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: pr_${{ matrix.config.report }}_test_reports path: reports From a5af804725e6246e9aa346c073277f5756383b40 Mon Sep 17 00:00:00 2001 From: DN6 Date: Mon, 5 Oct 2026 14:36:38 +0530 Subject: [PATCH 11/24] [CI] Count test files in the fetcher summary and add a row for modified test files outside the listed reach --- utils/tests_fetcher.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index 34fd7914b29e..42559377b768 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -435,12 +435,12 @@ def _print_selection_report(test_files: List[str], modified_files: List[str], di print(f"{test_file}\n {reason}") -def _tests_triggered_by(f: str, reverse_map: Dict[str, List[str]]) -> int: - """Number of test files a change to `f` selects, counting `f` itself when it is a test.""" +def _tests_triggered_by(f: str, reverse_map: Dict[str, List[str]]) -> set: + """Test files a change to `f` selects, including `f` itself when it is a test.""" triggered = {t for t in reverse_map.get(f, []) if _is_test_file(t)} if _is_test_file(f): triggered.add(f) - return len(triggered) + return triggered def _write_summary( @@ -453,18 +453,23 @@ def _write_summary( lines = [ "## Test fetcher", "", - f"Selected {len(test_files)} test files from {len(modified_files)} modified Python files. " - "The import chain that selected each test is in the `test_fetched` artifact.", + f"Selected {len(test_files)} test files from {len(modified_files)} modified Python files.", "", - "| Modified file | Tests it triggers |", + "| Modified file | Test files it triggers |", "|---|---|", ] - # A modified file that pulls in nothing beyond itself adds no information; only files with reach get a row. + # A modified file that pulls in nothing beyond itself gets no row of its own; whatever the listed rows do + # not reach is modified test files running only themselves, reported as one line. triggered = {f: _tests_triggered_by(f, reverse_map) for f in modified_files} - for f in sorted(modified_files, key=lambda f: (-triggered[f], f)): - if triggered[f] > (1 if _is_test_file(f) else 0): - lines.append(f"| `{f}` | {triggered[f]} |") - lines.append(f"| **Distinct tests selected** | **{len(test_files)}** |") + covered = set() + for f in sorted(modified_files, key=lambda f: (-len(triggered[f]), f)): + if len(triggered[f]) > (1 if _is_test_file(f) else 0): + lines.append(f"| `{f}` | {len(triggered[f])} |") + covered |= triggered[f] + remaining = set(test_files) - covered + if remaining: + lines.append(f"| Modified test files | {len(remaining)} |") + lines.append(f"| **Distinct test files selected** | **{len(test_files)}** |") Path(summary_file).write_text("\n".join(lines) + "\n", encoding="utf-8") From bcfeaa93161e5f99d4ea99326227478fcc218f09 Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 13 Oct 2026 17:07:15 +0530 Subject: [PATCH 12/24] [CI] Compare each modified file's test reach to the base commit in the fetcher summary --- utils/tests_fetcher.py | 108 +++++++++++++++++++++++++++++++---------- 1 file changed, 83 insertions(+), 25 deletions(-) diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index 42559377b768..80e9cf003d58 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -39,7 +39,8 @@ `tests/pipelines` are split further into one job per feature mixin (via the pytest markers the mixins carry) so a large selection fans out instead of serialising in one job. Stage 5 — report: print the import chain that selected each test (kept as a CI artifact) and write a - markdown job summary of how many tests each modified file triggers. + markdown job summary of how many tests each modified file triggers and how many source files trigger + each modified test, on the PR and on the base commit. Usage: @@ -164,8 +165,11 @@ def get_diff(repo: Repo, base_commit, commits) -> List[str]: return list(dict.fromkeys(code_diff)) -def get_modified_python_files(diff_with_last_commit: bool = False) -> List[str]: - """List Python files modified between HEAD and either main (default) or the previous commit.""" +def get_modified_python_files(diff_with_last_commit: bool = False) -> Tuple[List[str], str]: + """List Python files modified between HEAD and either main (default) or the previous commit. + + Also returns the base commit the diff was taken against, so the import graph can be rebuilt at that point. + """ repo = Repo(PATH_TO_REPO) if diff_with_last_commit: base_label = "previous commit" @@ -175,7 +179,7 @@ def get_modified_python_files(diff_with_last_commit: bool = False) -> List[str]: base_label = f"merge-base with main ({upstream_main.commit})" commits = repo.merge_base(upstream_main, repo.head) print(f"Diffing HEAD ({repo.head.commit}) against {base_label}: {[str(c) for c in commits]}") - return get_diff(repo, repo.head.commit, commits) + return get_diff(repo, repo.head.commit, commits), commits[0].hexsha # ============================================================ @@ -443,34 +447,88 @@ def _tests_triggered_by(f: str, reverse_map: Dict[str, List[str]]) -> set: return triggered +def _graph_at(commit: str) -> Tuple[Dict[str, List[str]], Dict[str, List[str]]]: + """Direct and reverse dependency maps with the repo checked out at `commit`.""" + with checkout_commit(Repo(PATH_TO_REPO), commit): + direct_deps = create_direct_dependency_map() + return direct_deps, create_reverse_dependency_map(direct_deps) + + +def _test_reach(reverse_map: Dict[str, List[str]]) -> Dict[str, int]: + """For every test file, the number of `src/` files whose change selects it.""" + reach: Dict[str, int] = collections.Counter() + for src, dependents in reverse_map.items(): + if src.startswith("src/"): + reach.update(t for t in dependents if _is_test_file(t)) + return reach + + def _write_summary( - summary_file: str, modified_files: List[str], test_files: List[str], reverse_map: Dict[str, List[str]] + summary_file: str, + modified_files: List[str], + test_files: List[str], + reverse_map: Dict[str, List[str]], + base_commit: str, ): - """Write a short markdown report for the CI job summary: how many tests each modified file triggers. - - A file with outsized reach is the cue for a maintainer to open the per-test import chains in the artifact. + """Write a short markdown report for the CI job summary, comparing the PR's import graph to the base's. + + Modified source files: how many test files each one triggers, on the PR and on the base. Modified and + added test files: the same, plus how many source files trigger the test (its reach). A top-level import + of a hub such as `auto_pipeline.py` roughly doubles a test's reach, and from then on the test runs on + every PR touching any pipeline; the PR adding the import is the only place to see it. A new file has no + base value, so its numbers stand alone. Test files that trigger only themselves and whose reach did not + change are counted on one line instead of listed. """ + base_direct, base_reverse = _graph_at(base_commit) + reach, base_reach = _test_reach(reverse_map), _test_reach(base_reverse) + triggered = {f: len(_tests_triggered_by(f, reverse_map)) for f in modified_files} + base_triggered = {f: len(_tests_triggered_by(f, base_reverse)) for f in modified_files} + on_base = lambda f, value: "new file" if f not in base_direct else str(value) # noqa: E731 + lines = [ "## Test fetcher", "", f"Selected {len(test_files)} test files from {len(modified_files)} modified Python files.", - "", - "| Modified file | Test files it triggers |", - "|---|---|", ] - # A modified file that pulls in nothing beyond itself gets no row of its own; whatever the listed rows do - # not reach is modified test files running only themselves, reported as one line. - triggered = {f: _tests_triggered_by(f, reverse_map) for f in modified_files} - covered = set() - for f in sorted(modified_files, key=lambda f: (-len(triggered[f]), f)): - if len(triggered[f]) > (1 if _is_test_file(f) else 0): - lines.append(f"| `{f}` | {len(triggered[f])} |") - covered |= triggered[f] - remaining = set(test_files) - covered - if remaining: - lines.append(f"| Modified test files | {len(remaining)} |") - lines.append(f"| **Distinct test files selected** | **{len(test_files)}** |") + by_reach = sorted(modified_files, key=lambda f: (-triggered[f], f)) + src_rows = [ + f"| `{f}` | {triggered[f]} | {on_base(f, base_triggered[f])} |" + for f in by_reach + if not _is_test_file(f) and (triggered[f] or base_triggered[f]) + ] + if src_rows: + lines += [ + "", + "**Modified source files**", + "", + "| File | Test files it triggers | On base |", + "|---|---|---|", + ] + src_rows + + test_rows, quiet = [], [] + for f in by_reach: + if f not in test_files: # a source file, or a test file the diff deleted + continue + if triggered[f] > 1 or f not in base_direct or reach[f] != base_reach[f]: + test_rows.append(f"| `{f}` | {triggered[f]} | {reach[f]} | {on_base(f, base_reach[f])} |") + else: + quiet.append(f) + if quiet: + noun = "file" if len(quiet) == 1 else "files" + test_rows.append( + f"| {len(quiet)} other modified test {noun}, each triggering only itself | {len(quiet)} | unchanged | |" + ) + if test_rows: + lines += [ + "", + "**Modified and added test files**", + "", + "| File | Test files it triggers | Source files that trigger it | On base |", + "|---|---|---|---|", + ] + test_rows + + lines += ["", f"**Distinct test files selected: {len(test_files)}**"] Path(summary_file).write_text("\n".join(lines) + "\n", encoding="utf-8") @@ -586,7 +644,7 @@ def _is_test_file(path: str) -> bool: def fetch_tests_to_run(json_output_file: str, summary_file: str, diff_with_last_commit: bool): """Determine the tests to run from the diff, write `test_map.json` and the markdown job summary.""" - modified_files = get_modified_python_files(diff_with_last_commit=diff_with_last_commit) + modified_files, base_commit = get_modified_python_files(diff_with_last_commit=diff_with_last_commit) direct_deps = create_direct_dependency_map() reverse_map = create_reverse_dependency_map(direct_deps) @@ -599,7 +657,7 @@ def fetch_tests_to_run(json_output_file: str, summary_file: str, diff_with_last_ test_files_to_run = sorted(p for p in selected if (PATH_TO_REPO / p).exists()) _print_selection_report(test_files_to_run, modified_files, direct_deps) - _write_summary(summary_file, modified_files, test_files_to_run, reverse_map) + _write_summary(summary_file, modified_files, test_files_to_run, reverse_map, base_commit) _write_matrix(json_output_file, test_files_to_run) From 7e67c09f048bedf70f8a2c81437404acf5af1808 Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 13 Oct 2026 17:57:23 +0530 Subject: [PATCH 13/24] [CI] Show each modified file's reach on branch and on main in the fetcher summary --- utils/tests_fetcher.py | 69 ++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 36 deletions(-) diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index 80e9cf003d58..ef6e64e26ad2 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -89,6 +89,8 @@ "single_file": ["single_file"], } FEATURE_MARKERS = [m for markers in FEATURE_GROUPS.values() for m in markers] +# Rows per section of the job summary; a wide migration PR touches hundreds of files and the rest are counted. +MAX_SUMMARY_ROWS = 10 _unknown = sorted(set(FEATURE_MARKERS) - NON_CORE_MARKERS) if _unknown: raise ValueError( @@ -483,7 +485,7 @@ def _write_summary( reach, base_reach = _test_reach(reverse_map), _test_reach(base_reverse) triggered = {f: len(_tests_triggered_by(f, reverse_map)) for f in modified_files} base_triggered = {f: len(_tests_triggered_by(f, base_reverse)) for f in modified_files} - on_base = lambda f, value: "new file" if f not in base_direct else str(value) # noqa: E731 + on_main = lambda f, value: "new file" if f not in base_direct else str(value) # noqa: E731 lines = [ "## Test fetcher", @@ -492,41 +494,36 @@ def _write_summary( ] by_reach = sorted(modified_files, key=lambda f: (-triggered[f], f)) - src_rows = [ - f"| `{f}` | {triggered[f]} | {on_base(f, base_triggered[f])} |" - for f in by_reach - if not _is_test_file(f) and (triggered[f] or base_triggered[f]) - ] - if src_rows: - lines += [ - "", - "**Modified source files**", - "", - "| File | Test files it triggers | On base |", - "|---|---|---|", - ] + src_rows - - test_rows, quiet = [], [] - for f in by_reach: - if f not in test_files: # a source file, or a test file the diff deleted - continue - if triggered[f] > 1 or f not in base_direct or reach[f] != base_reach[f]: - test_rows.append(f"| `{f}` | {triggered[f]} | {reach[f]} | {on_base(f, base_reach[f])} |") - else: - quiet.append(f) - if quiet: - noun = "file" if len(quiet) == 1 else "files" - test_rows.append( - f"| {len(quiet)} other modified test {noun}, each triggering only itself | {len(quiet)} | unchanged | |" - ) - if test_rows: - lines += [ - "", - "**Modified and added test files**", - "", - "| File | Test files it triggers | Source files that trigger it | On base |", - "|---|---|---|---|", - ] + test_rows + def section(title: str, rows: List[str], hidden: int): + if not rows: + return + lines.extend(["", f"**{title}**", "", "| File | On branch | On main |", "|---|---|---|"]) + lines.extend(rows) + if hidden: + lines.append(f"| {hidden} more not shown | | |") + + # Source files, and test helpers (anything under tests/ pytest does not collect): what they trigger. A + # zero is worth a row: the change runs no tests, either for lack of coverage or because the import graph + # cannot see how the module is reached. Deleted files are left out since they trigger nothing either way. + for title, keep in ( + ("Modified source files: test files each one triggers", lambda f: f.startswith("src/")), + ( + "Modified test helpers: test files each one triggers", + lambda f: f.startswith("tests/") and not _is_test_file(f), + ), + ): + rows = [ + f"| `{f}` | {triggered[f]} | {on_main(f, base_triggered[f])} |" + for f in by_reach + if keep(f) and (PATH_TO_REPO / f).exists() + ] + section(title, rows[:MAX_SUMMARY_ROWS], len(rows[MAX_SUMMARY_ROWS:])) + + # Test files: every one the diff touches, largest change in reach first. + present = [f for f in test_files if f in triggered] + present.sort(key=lambda f: (-abs(reach[f] - base_reach.get(f, 0)), f)) + rows = [f"| `{f}` | {reach[f]} | {on_main(f, base_reach[f])} |" for f in present[:MAX_SUMMARY_ROWS]] + section("Modified and added test files: source files that trigger each one", rows, len(present[MAX_SUMMARY_ROWS:])) lines += ["", f"**Distinct test files selected: {len(test_files)}**"] Path(summary_file).write_text("\n".join(lines) + "\n", encoding="utf-8") From 8b5edc135a6669f9d454d7acae3f24d748b73c84 Mon Sep 17 00:00:00 2001 From: DN6 Date: Mon, 9 Nov 2026 14:52:14 +0530 Subject: [PATCH 14/24] [tests] Use the concrete pipeline classes instead of `AutoPipeline*` in PAG, Kandinsky3, and flashpack tests --- tests/others/test_flashpack.py | 7 +++---- tests/pipelines/kandinsky3/test_kandinsky3.py | 9 +++------ .../pipelines/kandinsky3/test_kandinsky3_img2img.py | 4 +--- tests/pipelines/pag/test_pag_sd.py | 8 ++------ tests/pipelines/pag/test_pag_sd3_img2img.py | 12 ++++-------- tests/pipelines/pag/test_pag_sd_img2img.py | 8 ++------ tests/pipelines/pag/test_pag_sd_inpaint.py | 8 ++------ tests/pipelines/pag/test_pag_sdxl.py | 8 ++------ tests/pipelines/pag/test_pag_sdxl_img2img.py | 8 ++------ tests/pipelines/pag/test_pag_sdxl_inpaint.py | 8 ++------ 10 files changed, 23 insertions(+), 57 deletions(-) diff --git a/tests/others/test_flashpack.py b/tests/others/test_flashpack.py index e7eefd6835ff..644e3340721f 100644 --- a/tests/others/test_flashpack.py +++ b/tests/others/test_flashpack.py @@ -15,6 +15,7 @@ import pytest +from diffusers import FluxPipeline from diffusers.models.auto_model import AutoModel from ..testing_utils import is_torch_available, require_flashpack, require_torch_gpu @@ -39,13 +40,11 @@ def test_save_load_model(self, tmp_path): @require_flashpack def test_save_load_pipeline(self, tmp_path): - from diffusers import AutoPipelineForText2Image - - pipeline = AutoPipelineForText2Image.from_pretrained(self.model_id) + pipeline = FluxPipeline.from_pretrained(self.model_id) pipeline.save_pretrained(tmp_path, use_flashpack=True) assert (tmp_path / "transformer" / "model.flashpack").exists() assert (tmp_path / "vae" / "model.flashpack").exists() - pipeline = AutoPipelineForText2Image.from_pretrained(tmp_path, use_flashpack=True) + pipeline = FluxPipeline.from_pretrained(tmp_path, use_flashpack=True) @require_torch_gpu @require_flashpack diff --git a/tests/pipelines/kandinsky3/test_kandinsky3.py b/tests/pipelines/kandinsky3/test_kandinsky3.py index 3f556faf6cc8..d4f81927d32f 100644 --- a/tests/pipelines/kandinsky3/test_kandinsky3.py +++ b/tests/pipelines/kandinsky3/test_kandinsky3.py @@ -22,6 +22,7 @@ from transformers import AutoConfig, AutoTokenizer, T5EncoderModel from diffusers import ( + Kandinsky3Img2ImgPipeline, Kandinsky3Pipeline, Kandinsky3UNet, VQModel, @@ -175,9 +176,7 @@ def cleanup(self): backend_empty_cache(torch_device) def test_kandinskyV3(self): - from diffusers import AutoPipelineForText2Image - - pipe = AutoPipelineForText2Image.from_pretrained( + pipe = Kandinsky3Pipeline.from_pretrained( "kandinsky-community/kandinsky-3", variant="fp16", torch_dtype=torch.float16 ) pipe.enable_model_cpu_offload(device=torch_device) @@ -203,9 +202,7 @@ def test_kandinskyV3(self): assert np.allclose(image_np, expected_image_np, atol=5e-2) def test_kandinskyV3_img2img(self): - from diffusers import AutoPipelineForImage2Image - - pipe = AutoPipelineForImage2Image.from_pretrained( + pipe = Kandinsky3Img2ImgPipeline.from_pretrained( "kandinsky-community/kandinsky-3", variant="fp16", torch_dtype=torch.float16 ) pipe.enable_model_cpu_offload(device=torch_device) diff --git a/tests/pipelines/kandinsky3/test_kandinsky3_img2img.py b/tests/pipelines/kandinsky3/test_kandinsky3_img2img.py index 0dfccd146e89..071a2fc75b92 100644 --- a/tests/pipelines/kandinsky3/test_kandinsky3_img2img.py +++ b/tests/pipelines/kandinsky3/test_kandinsky3_img2img.py @@ -191,9 +191,7 @@ def cleanup(self): backend_empty_cache(torch_device) def test_kandinskyV3_img2img(self): - from diffusers import AutoPipelineForImage2Image - - pipe = AutoPipelineForImage2Image.from_pretrained( + pipe = Kandinsky3Img2ImgPipeline.from_pretrained( "kandinsky-community/kandinsky-3", variant="fp16", torch_dtype=torch.float16 ) pipe.enable_model_cpu_offload(device=torch_device) diff --git a/tests/pipelines/pag/test_pag_sd.py b/tests/pipelines/pag/test_pag_sd.py index 57fd5bc323c8..5957cb0a47aa 100644 --- a/tests/pipelines/pag/test_pag_sd.py +++ b/tests/pipelines/pag/test_pag_sd.py @@ -252,9 +252,7 @@ def get_inputs(self, device, generator_device="cpu", seed=1, guidance_scale=7.0) return inputs def test_pag_cfg(self): - from diffusers import AutoPipelineForText2Image - - pipeline = AutoPipelineForText2Image.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16) + pipeline = StableDiffusionPAGPipeline.from_pretrained(self.repo_id, torch_dtype=torch.float16) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) @@ -272,9 +270,7 @@ def test_pag_cfg(self): ) def test_pag_uncond(self): - from diffusers import AutoPipelineForText2Image - - pipeline = AutoPipelineForText2Image.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16) + pipeline = StableDiffusionPAGPipeline.from_pretrained(self.repo_id, torch_dtype=torch.float16) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) diff --git a/tests/pipelines/pag/test_pag_sd3_img2img.py b/tests/pipelines/pag/test_pag_sd3_img2img.py index 10eb1a7571eb..a58910048738 100644 --- a/tests/pipelines/pag/test_pag_sd3_img2img.py +++ b/tests/pipelines/pag/test_pag_sd3_img2img.py @@ -195,10 +195,8 @@ def get_inputs( return inputs def test_pag_cfg(self): - from diffusers import AutoPipelineForImage2Image - - pipeline = AutoPipelineForImage2Image.from_pretrained( - self.repo_id, enable_pag=True, torch_dtype=torch.float16, pag_applied_layers=["blocks.17"] + pipeline = StableDiffusion3PAGImg2ImgPipeline.from_pretrained( + self.repo_id, torch_dtype=torch.float16, pag_applied_layers=["blocks.17"] ) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) @@ -225,10 +223,8 @@ def test_pag_cfg(self): ) def test_pag_uncond(self): - from diffusers import AutoPipelineForImage2Image - - pipeline = AutoPipelineForImage2Image.from_pretrained( - self.repo_id, enable_pag=True, torch_dtype=torch.float16, pag_applied_layers=["blocks.(4|17)"] + pipeline = StableDiffusion3PAGImg2ImgPipeline.from_pretrained( + self.repo_id, torch_dtype=torch.float16, pag_applied_layers=["blocks.(4|17)"] ) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) diff --git a/tests/pipelines/pag/test_pag_sd_img2img.py b/tests/pipelines/pag/test_pag_sd_img2img.py index 86a0904d43d6..12280dafa947 100644 --- a/tests/pipelines/pag/test_pag_sd_img2img.py +++ b/tests/pipelines/pag/test_pag_sd_img2img.py @@ -196,9 +196,7 @@ def get_inputs(self, device, generator_device="cpu", dtype=torch.float32, seed=0 return inputs def test_pag_cfg(self): - from diffusers import AutoPipelineForImage2Image - - pipeline = AutoPipelineForImage2Image.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16) + pipeline = StableDiffusionPAGImg2ImgPipeline.from_pretrained(self.repo_id, torch_dtype=torch.float16) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) @@ -216,9 +214,7 @@ def test_pag_cfg(self): ) def test_pag_uncond(self): - from diffusers import AutoPipelineForImage2Image - - pipeline = AutoPipelineForImage2Image.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16) + pipeline = StableDiffusionPAGImg2ImgPipeline.from_pretrained(self.repo_id, torch_dtype=torch.float16) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) diff --git a/tests/pipelines/pag/test_pag_sd_inpaint.py b/tests/pipelines/pag/test_pag_sd_inpaint.py index cc589e787b03..b70515d12c2a 100644 --- a/tests/pipelines/pag/test_pag_sd_inpaint.py +++ b/tests/pipelines/pag/test_pag_sd_inpaint.py @@ -274,9 +274,7 @@ def get_inputs(self, device, generator_device="cpu", seed=0, guidance_scale=7.0) return inputs def test_pag_cfg(self): - from diffusers import AutoPipelineForInpainting - - pipeline = AutoPipelineForInpainting.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16) + pipeline = StableDiffusionPAGInpaintPipeline.from_pretrained(self.repo_id, torch_dtype=torch.float16) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) @@ -294,9 +292,7 @@ def test_pag_cfg(self): ) def test_pag_uncond(self): - from diffusers import AutoPipelineForInpainting - - pipeline = AutoPipelineForInpainting.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16) + pipeline = StableDiffusionPAGInpaintPipeline.from_pretrained(self.repo_id, torch_dtype=torch.float16) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) diff --git a/tests/pipelines/pag/test_pag_sdxl.py b/tests/pipelines/pag/test_pag_sdxl.py index 5dc2417fa531..fc6d2ce62486 100644 --- a/tests/pipelines/pag/test_pag_sdxl.py +++ b/tests/pipelines/pag/test_pag_sdxl.py @@ -259,9 +259,7 @@ def get_inputs(self, device, generator_device="cpu", seed=0, guidance_scale=7.0) return inputs def test_pag_cfg(self): - from diffusers import AutoPipelineForText2Image - - pipeline = AutoPipelineForText2Image.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16) + pipeline = StableDiffusionXLPAGPipeline.from_pretrained(self.repo_id, torch_dtype=torch.float16) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) @@ -283,9 +281,7 @@ def test_pag_cfg(self): ) def test_pag_uncond(self): - from diffusers import AutoPipelineForText2Image - - pipeline = AutoPipelineForText2Image.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16) + pipeline = StableDiffusionXLPAGPipeline.from_pretrained(self.repo_id, torch_dtype=torch.float16) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) diff --git a/tests/pipelines/pag/test_pag_sdxl_img2img.py b/tests/pipelines/pag/test_pag_sdxl_img2img.py index e22c349a4433..6a0eea41a655 100644 --- a/tests/pipelines/pag/test_pag_sdxl_img2img.py +++ b/tests/pipelines/pag/test_pag_sdxl_img2img.py @@ -259,9 +259,7 @@ def get_inputs(self, device, generator_device="cpu", seed=0, guidance_scale=7.0) return inputs def test_pag_cfg(self): - from diffusers import AutoPipelineForImage2Image - - pipeline = AutoPipelineForImage2Image.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16) + pipeline = StableDiffusionXLPAGImg2ImgPipeline.from_pretrained(self.repo_id, torch_dtype=torch.float16) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) @@ -278,9 +276,7 @@ def test_pag_cfg(self): ) def test_pag_uncond(self): - from diffusers import AutoPipelineForImage2Image - - pipeline = AutoPipelineForImage2Image.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16) + pipeline = StableDiffusionXLPAGImg2ImgPipeline.from_pretrained(self.repo_id, torch_dtype=torch.float16) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) diff --git a/tests/pipelines/pag/test_pag_sdxl_inpaint.py b/tests/pipelines/pag/test_pag_sdxl_inpaint.py index c80ed4b6f20e..174134d16cff 100644 --- a/tests/pipelines/pag/test_pag_sdxl_inpaint.py +++ b/tests/pipelines/pag/test_pag_sdxl_inpaint.py @@ -266,9 +266,7 @@ def get_inputs(self, device, generator_device="cpu", seed=0, guidance_scale=7.0) return inputs def test_pag_cfg(self): - from diffusers import AutoPipelineForInpainting - - pipeline = AutoPipelineForInpainting.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16) + pipeline = StableDiffusionXLPAGInpaintPipeline.from_pretrained(self.repo_id, torch_dtype=torch.float16) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) @@ -285,9 +283,7 @@ def test_pag_cfg(self): ) def test_pag_uncond(self): - from diffusers import AutoPipelineForInpainting - - pipeline = AutoPipelineForInpainting.from_pretrained(self.repo_id, enable_pag=True, torch_dtype=torch.float16) + pipeline = StableDiffusionXLPAGInpaintPipeline.from_pretrained(self.repo_id, torch_dtype=torch.float16) pipeline.enable_model_cpu_offload(device=torch_device) pipeline.set_progress_bar_config(disable=None) From bc0e8cb8fc0d8df37e9d5e84f2c0af6a306b4fb8 Mon Sep 17 00:00:00 2001 From: DN6 Date: Mon, 9 Nov 2026 16:22:42 +0530 Subject: [PATCH 15/24] [tests] Ban `AutoPipeline*` imports in tests and switch the SD and SDXL tests to the concrete classes --- pyproject.toml | 19 ++++++++++++++++++- .../stable_diffusion/test_stable_diffusion.py | 9 ++++----- .../test_stable_diffusion_xl.py | 5 ++--- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fdda8a6977be..7667ecc4e6b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,12 +7,29 @@ extend-exclude = [ [tool.ruff.lint] # Never enforce `E501` (line length violations). ignore = ["C901", "E501", "E721", "E741", "F402", "F823"] -select = ["C", "E", "F", "I", "W"] +select = ["C", "E", "F", "I", "W", "TID251"] + +# `utils/tests_fetcher.py` selects the tests a PR runs from the import graph. `auto_pipeline.py` imports every +# pipeline, so a test that imports an AutoPipeline is selected on every pipeline change. Tests use the concrete +# pipeline class instead; the auto pipelines' own test is the one exception. +[tool.ruff.lint.flake8-tidy-imports.banned-api] +"diffusers.AutoPipelineForText2Image".msg = "AutoPipeline imports are not allowed in tests because they interfere with the test fetcher's selection logic. Use the specific pipeline class instead (e.g. `FluxPipeline`)." +"diffusers.AutoPipelineForImage2Image".msg = "AutoPipeline imports are not allowed in tests because they interfere with the test fetcher's selection logic. Use the specific pipeline class instead (e.g. `FluxPipeline`)." +"diffusers.AutoPipelineForInpainting".msg = "AutoPipeline imports are not allowed in tests because they interfere with the test fetcher's selection logic. Use the specific pipeline class instead (e.g. `FluxPipeline`)." +"diffusers.AutoPipelineForText2Audio".msg = "AutoPipeline imports are not allowed in tests because they interfere with the test fetcher's selection logic. Use the specific pipeline class instead (e.g. `FluxPipeline`)." # Ignore import violations in all `__init__.py` files. [tool.ruff.lint.per-file-ignores] "__init__.py" = ["E402", "F401", "F403", "F811"] "src/diffusers/utils/dummy_*.py" = ["F401"] +# The AutoPipeline ban only concerns test selection; everything outside tests/ may use them. +"tests/pipelines/test_pipelines_auto.py" = ["TID251"] +"src/**" = ["TID251"] +"examples/**" = ["TID251"] +"benchmarks/**" = ["TID251"] +"scripts/**" = ["TID251"] +"utils/**" = ["TID251"] +"docs/**" = ["TID251"] [tool.ruff.lint.isort] lines-after-imports = 2 diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion.py b/tests/pipelines/stable_diffusion/test_stable_diffusion.py index e0142d612925..a7f519e8a42c 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion.py @@ -31,8 +31,6 @@ from diffusers import ( AutoencoderKL, - AutoPipelineForImage2Image, - AutoPipelineForText2Image, DDIMScheduler, DiffusionPipeline, DPMSolverMultistepScheduler, @@ -41,6 +39,7 @@ LCMScheduler, LMSDiscreteScheduler, PNDMScheduler, + StableDiffusionImg2ImgPipeline, StableDiffusionPipeline, UNet2DConditionModel, logging, @@ -1821,7 +1820,7 @@ def test_load_unload_load_kohya_lora(self): def test_not_empty_state_dict(self): # Makes sure https://github.com/huggingface/diffusers/issues/7054 does not happen again - pipe = AutoPipelineForText2Image.from_pretrained( + pipe = StableDiffusionPipeline.from_pretrained( "stable-diffusion-v1-5/stable-diffusion-v1-5", torch_dtype=torch.float16 ).to(torch_device) pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config) @@ -1835,7 +1834,7 @@ def test_not_empty_state_dict(self): def test_load_unload_load_state_dict(self): # Makes sure https://github.com/huggingface/diffusers/issues/7054 does not happen again - pipe = AutoPipelineForText2Image.from_pretrained( + pipe = StableDiffusionPipeline.from_pretrained( "stable-diffusion-v1-5/stable-diffusion-v1-5", torch_dtype=torch.float16 ).to(torch_device) pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config) @@ -1884,7 +1883,7 @@ def test_sdv1_5_lcm_lora(self): release_memory(pipe) def test_sdv1_5_lcm_lora_img2img(self): - pipe = AutoPipelineForImage2Image.from_pretrained( + pipe = StableDiffusionImg2ImgPipeline.from_pretrained( "stable-diffusion-v1-5/stable-diffusion-v1-5", torch_dtype=torch.float16 ) pipe.to(torch_device) diff --git a/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl.py b/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl.py index 8728dd2a3bdc..334a6874786f 100644 --- a/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl.py +++ b/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl.py @@ -27,7 +27,6 @@ from diffusers import ( AutoencoderKL, - AutoPipelineForText2Image, ControlNetModel, DDIMScheduler, DPMSolverMultistepScheduler, @@ -1520,7 +1519,7 @@ def cleanup(self): def test_sdxl_turbo_512(self): generator = torch.Generator("cpu").manual_seed(0) - pipe = AutoPipelineForText2Image.from_pretrained(self.ckpt_id, torch_dtype=torch.float16, variant="fp16") + pipe = StableDiffusionXLPipeline.from_pretrained(self.ckpt_id, torch_dtype=torch.float16, variant="fp16") pipe.to(torch_device) backend_synchronize(torch_device) pipe.set_progress_bar_config(disable=None) @@ -1547,7 +1546,7 @@ def test_sdxl_turbo_neuron_compile_256(self): device = torch.neuron.current_device() generator = torch.Generator("cpu").manual_seed(0) - pipe = AutoPipelineForText2Image.from_pretrained(self.ckpt_id, torch_dtype=torch.bfloat16, variant="fp16") + pipe = StableDiffusionXLPipeline.from_pretrained(self.ckpt_id, torch_dtype=torch.bfloat16, variant="fp16") pipe = pipe.to(device) backend_synchronize(torch_device) From fee0b3097a092e251406da2ea9ae296d7fc270c4 Mon Sep 17 00:00:00 2001 From: DN6 Date: Mon, 9 Nov 2026 16:22:42 +0530 Subject: [PATCH 16/24] [CI] Summarize the selection per changed file with a collapsible import tree, and drop the base-commit comparison --- utils/tests_fetcher.py | 208 ++++++++++++++++++++++++++--------------- 1 file changed, 135 insertions(+), 73 deletions(-) diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index ef6e64e26ad2..90f76cdea980 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -39,8 +39,7 @@ `tests/pipelines` are split further into one job per feature mixin (via the pytest markers the mixins carry) so a large selection fans out instead of serialising in one job. Stage 5 — report: print the import chain that selected each test (kept as a CI artifact) and write a - markdown job summary of how many tests each modified file triggers and how many source files trigger - each modified test, on the PR and on the base commit. + markdown job summary saying, for each changed file, which tests it selects and through which file. Usage: @@ -89,8 +88,11 @@ "single_file": ["single_file"], } FEATURE_MARKERS = [m for markers in FEATURE_GROUPS.values() for m in markers] -# Rows per section of the job summary; a wide migration PR touches hundreds of files and the rest are counted. +# Job summary limits: rows in the table, tests named before they are grouped by folder, folders named before +# the rest are counted. A wide migration PR touches hundreds of files and selects most of the suite. MAX_SUMMARY_ROWS = 10 +NAME_TESTS_UP_TO = 6 +NAME_FOLDERS_UP_TO = 5 _unknown = sorted(set(FEATURE_MARKERS) - NON_CORE_MARKERS) if _unknown: raise ValueError( @@ -167,11 +169,8 @@ def get_diff(repo: Repo, base_commit, commits) -> List[str]: return list(dict.fromkeys(code_diff)) -def get_modified_python_files(diff_with_last_commit: bool = False) -> Tuple[List[str], str]: - """List Python files modified between HEAD and either main (default) or the previous commit. - - Also returns the base commit the diff was taken against, so the import graph can be rebuilt at that point. - """ +def get_modified_python_files(diff_with_last_commit: bool = False) -> List[str]: + """List Python files modified between HEAD and either main (default) or the previous commit.""" repo = Repo(PATH_TO_REPO) if diff_with_last_commit: base_label = "previous commit" @@ -181,7 +180,7 @@ def get_modified_python_files(diff_with_last_commit: bool = False) -> Tuple[List base_label = f"merge-base with main ({upstream_main.commit})" commits = repo.merge_base(upstream_main, repo.head) print(f"Diffing HEAD ({repo.head.commit}) against {base_label}: {[str(c) for c in commits]}") - return get_diff(repo, repo.head.commit, commits), commits[0].hexsha + return get_diff(repo, repo.head.commit, commits) # ============================================================ @@ -327,7 +326,7 @@ def get_module_dependencies(module_file: str, cache: Dict[str, List[Tuple[str, L # correct: changes to the init will trigger this module). dependencies.append(target) - return list(set(dependencies)) + return sorted(set(dependencies)) def _merged_nested_deps(m: str, direct_deps: Dict[str, List[str]]) -> bool: @@ -449,83 +448,146 @@ def _tests_triggered_by(f: str, reverse_map: Dict[str, List[str]]) -> set: return triggered -def _graph_at(commit: str) -> Tuple[Dict[str, List[str]], Dict[str, List[str]]]: - """Direct and reverse dependency maps with the repo checked out at `commit`.""" - with checkout_commit(Repo(PATH_TO_REPO), commit): - direct_deps = create_direct_dependency_map() - return direct_deps, create_reverse_dependency_map(direct_deps) +def _selection_routes(changed: str, tests: set, direct_deps: Dict[str, List[str]]): + """Split the tests `changed` selects into those importing it directly and those reached through another file. - -def _test_reach(reverse_map: Dict[str, List[str]]) -> Dict[str, int]: - """For every test file, the number of `src/` files whose change selects it.""" - reach: Dict[str, int] = collections.Counter() - for src, dependents in reverse_map.items(): - if src.startswith("src/"): - reach.update(t for t in dependents if _is_test_file(t)) - return reach + Tests reached indirectly are grouped under the intermediate file that the most of them pass through + (nearest the tests on a tie): that is where a cascade is visible (`auto_pipeline.py`, a test file imported + by other tests, a helper that imports a model). Groups come back largest first. + """ + others = sorted(t for t in tests if t != changed) + chains = {t: (_import_chain(t, [changed], direct_deps) or [changed, t])[1:-1] for t in others} + traffic = collections.Counter(hop for hops in chains.values() for hop in hops) + direct, via = [], collections.defaultdict(list) + for t, hops in chains.items(): + if not hops: + direct.append(t) + else: + via[max(reversed(hops), key=lambda h: traffic[h])].append(t) + return direct, sorted(via.items(), key=lambda kv: (-len(kv[1]), kv[0])) + + +def _suite_lines(tests: List[str], suite_totals: Dict[str, int]) -> List[str]: + """Summarise many tests as suite coverage: `tests/models/: 80 of 104 test files`. A suite only brushed by + the group (fewer than ten tests and under a quarter of the suite) is folded into one `N test files + across …` line, since five against two hundred says nothing.""" + by_suite = collections.Counter(_suite_of(t) for t in tests) + significant = lambda suite, n: n >= 10 or n >= suite_totals[suite] / 4 # noqa: E731 + major = [(suite, n) for suite, n in by_suite.most_common() if significant(suite, n)] + minor = [(suite, n) for suite, n in by_suite.most_common() if not significant(suite, n)] + lines = [f"{suite}: {n} of {suite_totals[suite]} test files" for suite, n in major] + if minor: + lines.append(f"{sum(n for _, n in minor)} test files across " + ", ".join(suite for suite, _ in minor)) + return lines + + +def _suite_of(test_file: str) -> str: + parts = test_file.split("/") + return f"{parts[0]}/{parts[1]}/" if len(parts) > 2 else f"{parts[0]}/" + + +def _selection_tree( + changed: str, tests: set, direct_deps: Dict[str, List[str]], suite_totals: Dict[str, int] +) -> List[str]: + """Text tree of how `changed` reaches the tests it selects, in a code block. + + Tests importing `changed` directly hang off the root; tests reached through another file hang off a + `via ` branch. Up to `NAME_TESTS_UP_TO` tests are listed by name; more are summarised as suite + coverage. Routes beyond the first `NAME_FOLDERS_UP_TO` collapse into one branch describing the modules + they pass through and what those modules import from `changed`, which for a hub is the whole explanation. + """ + direct, via = _selection_routes(changed, tests, direct_deps) + + def leaves(ts: List[str]) -> List[str]: + return ts if len(ts) <= NAME_TESTS_UP_TO else _suite_lines(ts, suite_totals) + + branches: List[Tuple[str, List[str]]] = [(leaf, []) for leaf in leaves(direct)] + branches += [(f"via {hop}", leaves(ts)) for hop, ts in via[:NAME_FOLDERS_UP_TO]] + if len(via) > NAME_FOLDERS_UP_TO: + rest = via[NAME_FOLDERS_UP_TO:] + by_package: Dict[str, List] = collections.defaultdict(lambda: [0, 0, collections.Counter()]) + for hop, ts in rest: + entry = by_package["/".join(hop.split("/")[:-1][:3]) + "/"] + entry[0] += 1 + entry[1] += len(ts) + entry[2].update(name for target, names in _extract_imports(hop) if target == changed for name in names) + children = [] + for pkg, (n, t, names) in sorted(by_package.items(), key=lambda kv: -kv[1][1])[:NAME_FOLDERS_UP_TO]: + imported = ", ".join(name for name, _ in names.most_common(3)) + children.append( + f"{n} module{'s' if n != 1 else ''} in {pkg} → {t} test{'s' if t != 1 else ''}" + + (f", importing {imported}" if imported else "") + ) + total = sum(len(ts) for _, ts in rest) + branches.append((f"via {len(rest)} other modules → {total} tests", children)) + + lines = ["```", changed] + for i, (label, children) in enumerate(branches): + last = i == len(branches) - 1 + lines.append(("└── " if last else "├── ") + label) + for j, child in enumerate(children): + lines.append((" " if last else "│ ") + ("└── " if j == len(children) - 1 else "├── ") + child) + lines.append("```") + return lines def _write_summary( summary_file: str, modified_files: List[str], test_files: List[str], + direct_deps: Dict[str, List[str]], reverse_map: Dict[str, List[str]], - base_commit: str, ): - """Write a short markdown report for the CI job summary, comparing the PR's import graph to the base's. - - Modified source files: how many test files each one triggers, on the PR and on the base. Modified and - added test files: the same, plus how many source files trigger the test (its reach). A top-level import - of a hub such as `auto_pipeline.py` roughly doubles a test's reach, and from then on the test runs on - every PR touching any pipeline; the PR adding the import is the only place to see it. A new file has no - base value, so its numbers stand alone. Test files that trigger only themselves and whose reach did not - change are counted on one line instead of listed. + """Write the CI job summary: how many tests each changed file selects, and a collapsible tree per file + showing through which files it reaches them. + + Rows are changed files, largest selection first. A source file that selects nothing keeps its row, since + that change runs no tests; anything else that selects nothing (conftest, helpers, files outside the graph) + is left out. Changed test files that select only themselves are counted on one row. The import chain that + selected every individual test is in the stdout report, kept as a CI artifact. """ - base_direct, base_reverse = _graph_at(base_commit) - reach, base_reach = _test_reach(reverse_map), _test_reach(base_reverse) - triggered = {f: len(_tests_triggered_by(f, reverse_map)) for f in modified_files} - base_triggered = {f: len(_tests_triggered_by(f, base_reverse)) for f in modified_files} - on_main = lambda f, value: "new file" if f not in base_direct else str(value) # noqa: E731 + selected_by = {f: _tests_triggered_by(f, reverse_map) for f in modified_files if (PATH_TO_REPO / f).exists()} + selected_by = {f: t for f, t in selected_by.items() if t or f.startswith("src/")} + self_only = [f for f, tests in selected_by.items() if tests == {f}] + rows = sorted((f for f in selected_by if f not in self_only), key=lambda f: (-len(selected_by[f]), f)) + suite_totals = collections.Counter(_suite_of(t) for t in _all_test_files()) + files = "file" if len(modified_files) == 1 else "files" lines = [ "## Test fetcher", "", - f"Selected {len(test_files)} test files from {len(modified_files)} modified Python files.", + f"{len(modified_files)} Python {files} changed. {len(test_files)} test files will run.", + "", + "| Changed file | Test files selected |", + "|---|---|", ] - by_reach = sorted(modified_files, key=lambda f: (-triggered[f], f)) - - def section(title: str, rows: List[str], hidden: int): - if not rows: - return - lines.extend(["", f"**{title}**", "", "| File | On branch | On main |", "|---|---|---|"]) - lines.extend(rows) - if hidden: - lines.append(f"| {hidden} more not shown | | |") - - # Source files, and test helpers (anything under tests/ pytest does not collect): what they trigger. A - # zero is worth a row: the change runs no tests, either for lack of coverage or because the import graph - # cannot see how the module is reached. Deleted files are left out since they trigger nothing either way. - for title, keep in ( - ("Modified source files: test files each one triggers", lambda f: f.startswith("src/")), - ( - "Modified test helpers: test files each one triggers", - lambda f: f.startswith("tests/") and not _is_test_file(f), - ), - ): - rows = [ - f"| `{f}` | {triggered[f]} | {on_main(f, base_triggered[f])} |" - for f in by_reach - if keep(f) and (PATH_TO_REPO / f).exists() - ] - section(title, rows[:MAX_SUMMARY_ROWS], len(rows[MAX_SUMMARY_ROWS:])) - - # Test files: every one the diff touches, largest change in reach first. - present = [f for f in test_files if f in triggered] - present.sort(key=lambda f: (-abs(reach[f] - base_reach.get(f, 0)), f)) - rows = [f"| `{f}` | {reach[f]} | {on_main(f, base_reach[f])} |" for f in present[:MAX_SUMMARY_ROWS]] - section("Modified and added test files: source files that trigger each one", rows, len(present[MAX_SUMMARY_ROWS:])) + for f in rows[:MAX_SUMMARY_ROWS]: + lines.append(f"| `{f}` | {len(selected_by[f])} |") + if len(rows) > MAX_SUMMARY_ROWS: + lines.append(f"| {len(rows) - MAX_SUMMARY_ROWS} more changed files | |") + if self_only: + covered = set().union(*(selected_by[f] for f in rows)) if rows else set() + inside = sum(1 for f in self_only if f in covered) + if len(self_only) == 1: + lines.append(f"| `{self_only[0]}` (selects only itself) | 1 |") + else: + note = f", {inside} already counted above" if inside else "" + lines.append( + f"| {len(self_only)} changed test files, each selecting only itself{note} | {len(self_only)} |" + ) - lines += ["", f"**Distinct test files selected: {len(test_files)}**"] + for f in rows[:MAX_SUMMARY_ROWS]: + if not selected_by[f]: + continue + n = len(selected_by[f]) + lines += [ + "", + "
", + f"{f} → {n} test {'file' if n == 1 else 'files'}", + "", + ] + lines += _selection_tree(f, selected_by[f], direct_deps, suite_totals) + lines += ["", "
"] Path(summary_file).write_text("\n".join(lines) + "\n", encoding="utf-8") @@ -641,7 +703,7 @@ def _is_test_file(path: str) -> bool: def fetch_tests_to_run(json_output_file: str, summary_file: str, diff_with_last_commit: bool): """Determine the tests to run from the diff, write `test_map.json` and the markdown job summary.""" - modified_files, base_commit = get_modified_python_files(diff_with_last_commit=diff_with_last_commit) + modified_files = get_modified_python_files(diff_with_last_commit=diff_with_last_commit) direct_deps = create_direct_dependency_map() reverse_map = create_reverse_dependency_map(direct_deps) @@ -654,7 +716,7 @@ def fetch_tests_to_run(json_output_file: str, summary_file: str, diff_with_last_ test_files_to_run = sorted(p for p in selected if (PATH_TO_REPO / p).exists()) _print_selection_report(test_files_to_run, modified_files, direct_deps) - _write_summary(summary_file, modified_files, test_files_to_run, reverse_map, base_commit) + _write_summary(summary_file, modified_files, test_files_to_run, direct_deps, reverse_map) _write_matrix(json_output_file, test_files_to_run) From 8668474436ad86b9e746716c746d18d13d48df44 Mon Sep 17 00:00:00 2001 From: DN6 Date: Mon, 9 Nov 2026 21:59:07 +0530 Subject: [PATCH 17/24] [CI] Flag test files that import a module which imports much of the library in the fetcher summary --- utils/tests_fetcher.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index 90f76cdea980..ad5dd686f233 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -93,6 +93,9 @@ MAX_SUMMARY_ROWS = 10 NAME_TESTS_UP_TO = 6 NAME_FOLDERS_UP_TO = 5 +# A test that imports a module importing this many others depends on most of the library and will run on +# nearly every PR. `auto_pipeline.py` imports 119 modules; the widest ordinary pipeline imports 28. +WIDE_IMPORT_FANOUT = 50 _unknown = sorted(set(FEATURE_MARKERS) - NON_CORE_MARKERS) if _unknown: raise ValueError( @@ -494,7 +497,8 @@ def _selection_tree( Tests importing `changed` directly hang off the root; tests reached through another file hang off a `via ` branch. Up to `NAME_TESTS_UP_TO` tests are listed by name; more are summarised as suite coverage. Routes beyond the first `NAME_FOLDERS_UP_TO` collapse into one branch describing the modules - they pass through and what those modules import from `changed`, which for a hub is the whole explanation. + they pass through and what those modules import from `changed`, which for a widely imported module is the + whole explanation. """ direct, via = _selection_routes(changed, tests, direct_deps) @@ -576,6 +580,28 @@ def _write_summary( f"| {len(self_only)} changed test files, each selecting only itself{note} | {len(self_only)} |" ) + # The one thing worth saying about a changed test file on the PR that changes it: a new import of a module + # that itself imports much of the library. The test still selects only itself here, but from now on it runs + # whenever any of that module's imports change. Package inits and the tester helpers import widely by + # design and are excluded. + wide_imports = [ + (t, d, len(direct_deps[d])) + for t in test_files + if t in modified_files + for d in direct_deps.get(t, []) + if not d.endswith("__init__.py") + and "/testing_utils" not in d + and len(direct_deps.get(d, [])) >= WIDE_IMPORT_FANOUT + ] + if wide_imports: + lines += ["", "**Imports that make a test depend on most of the library**", ""] + for t, d, n in wide_imports: + lines.append( + f"- `{t}` imports `{d}`, which imports {n} other modules, so this test will run whenever any of " + f"them change. Import the specific class the test needs instead. If tests should never import " + f"`{d.rsplit('/', 1)[-1]}`, add it to the banned imports in `pyproject.toml`." + ) + for f in rows[:MAX_SUMMARY_ROWS]: if not selected_by[f]: continue From 966349c145f4ee41c97d8c0808341299d692f45b Mon Sep 17 00:00:00 2001 From: DN6 Date: Mon, 9 Nov 2026 22:29:37 +0530 Subject: [PATCH 18/24] [CI] Give the tail of a wide diff its own nested dropdowns in the fetcher summary --- utils/tests_fetcher.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index ad5dd686f233..1fb98af08ff6 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -562,13 +562,15 @@ def _write_summary( "", f"{len(modified_files)} Python {files} changed. {len(test_files)} test files will run.", "", - "| Changed file | Test files selected |", + "| Changed file | Test files it reaches |", "|---|---|", ] for f in rows[:MAX_SUMMARY_ROWS]: lines.append(f"| `{f}` | {len(selected_by[f])} |") - if len(rows) > MAX_SUMMARY_ROWS: - lines.append(f"| {len(rows) - MAX_SUMMARY_ROWS} more changed files | |") + overflow = rows[MAX_SUMMARY_ROWS:] + if overflow: + bound = len(selected_by[overflow[0]]) + lines.append(f"| {len(overflow)} more changed files, each reaching {bound} or fewer tests (listed below) | |") if self_only: covered = set().union(*(selected_by[f] for f in rows)) if rows else set() inside = sum(1 for f in self_only if f in covered) @@ -602,17 +604,24 @@ def _write_summary( f"`{d.rsplit('/', 1)[-1]}`, add it to the banned imports in `pyproject.toml`." ) - for f in rows[:MAX_SUMMARY_ROWS]: - if not selected_by[f]: - continue + def dropdown(f: str) -> List[str]: n = len(selected_by[f]) - lines += [ - "", - "
", - f"{f} → {n} test {'file' if n == 1 else 'files'}", - "", - ] - lines += _selection_tree(f, selected_by[f], direct_deps, suite_totals) + title = f"{f} → {n} test {'file' if n == 1 else 'files'}" + return ( + ["", "
", title, ""] + + _selection_tree(f, selected_by[f], direct_deps, suite_totals) + + ["", "
"] + ) + + for f in rows[:MAX_SUMMARY_ROWS]: + if selected_by[f]: + lines += dropdown(f) + # The tail of a wide diff: one collapsed section holding a dropdown per remaining file, so nothing is + # left unexplained while the top level stays at MAX_SUMMARY_ROWS entries. + if overflow: + lines += ["", "
", f"{len(overflow)} more changed files"] + for f in overflow: + lines += dropdown(f) if selected_by[f] else ["", f"`{f}` reaches no tests", ""] lines += ["", "
"] Path(summary_file).write_text("\n".join(lines) + "\n", encoding="utf-8") From 5835f5f1f06e26ecc2c0cc0558da64d06ef6defd Mon Sep 17 00:00:00 2001 From: DN6 Date: Mon, 9 Nov 2026 23:07:36 +0530 Subject: [PATCH 19/24] [CI] Call out changed files that run no tests and move the import trees into one Details section --- utils/tests_fetcher.py | 57 +++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index 1fb98af08ff6..9b17a1640b4a 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -520,7 +520,7 @@ def leaves(ts: List[str]) -> List[str]: imported = ", ".join(name for name, _ in names.most_common(3)) children.append( f"{n} module{'s' if n != 1 else ''} in {pkg} → {t} test{'s' if t != 1 else ''}" - + (f", importing {imported}" if imported else "") + + (f"; they import {imported} from {changed.rsplit('/', 1)[-1]}" if imported else "") ) total = sum(len(ts) for _, ts in rest) branches.append((f"via {len(rest)} other modules → {total} tests", children)) @@ -542,20 +542,23 @@ def _write_summary( direct_deps: Dict[str, List[str]], reverse_map: Dict[str, List[str]], ): - """Write the CI job summary: how many tests each changed file selects, and a collapsible tree per file - showing through which files it reaches them. + """Write the CI job summary: which tests each changed file selects and through which files. - Rows are changed files, largest selection first. A source file that selects nothing keeps its row, since - that change runs no tests; anything else that selects nothing (conftest, helpers, files outside the graph) - is left out. Changed test files that select only themselves are counted on one row. The import chain that - selected every individual test is in the stdout report, kept as a CI artifact. + The table lists changed files largest selection first, capped at `MAX_SUMMARY_ROWS` rows; changed test + files that select only themselves are counted on one row. Changed files that reach no tests are called + out, since CI will not exercise them. A single collapsed Details section then holds one dropdown per + changed file with the import tree that explains its selection. The import chain for every selected test + is in the stdout report, kept as a CI artifact. """ selected_by = {f: _tests_triggered_by(f, reverse_map) for f in modified_files if (PATH_TO_REPO / f).exists()} selected_by = {f: t for f, t in selected_by.items() if t or f.startswith("src/")} + suite_totals = collections.Counter(_suite_of(t) for t in _all_test_files()) self_only = [f for f, tests in selected_by.items() if tests == {f}] - rows = sorted((f for f in selected_by if f not in self_only), key=lambda f: (-len(selected_by[f]), f)) + no_tests = sorted(f for f, tests in selected_by.items() if not tests) + rows = sorted( + (f for f in selected_by if f not in self_only and f not in no_tests), key=lambda f: (-len(selected_by[f]), f) + ) - suite_totals = collections.Counter(_suite_of(t) for t in _all_test_files()) files = "file" if len(modified_files) == 1 else "files" lines = [ "## Test fetcher", @@ -570,7 +573,7 @@ def _write_summary( overflow = rows[MAX_SUMMARY_ROWS:] if overflow: bound = len(selected_by[overflow[0]]) - lines.append(f"| {len(overflow)} more changed files, each reaching {bound} or fewer tests (listed below) | |") + lines.append(f"| {len(overflow)} more changed files, each reaching {bound} or fewer tests (see Details) | |") if self_only: covered = set().union(*(selected_by[f] for f in rows)) if rows else set() inside = sum(1 for f in self_only if f in covered) @@ -582,6 +585,14 @@ def _write_summary( f"| {len(self_only)} changed test files, each selecting only itself{note} | {len(self_only)} |" ) + if no_tests: + listed = ", ".join(f"`{f}`" for f in no_tests) + lines += [ + "", + f"**Changed files that run no tests:** {listed}. No test imports them, so CI does not exercise these " + "changes.", + ] + # The one thing worth saying about a changed test file on the PR that changes it: a new import of a module # that itself imports much of the library. The test still selects only itself here, but from now on it runs # whenever any of that module's imports change. Package inits and the tester helpers import widely by @@ -604,24 +615,14 @@ def _write_summary( f"`{d.rsplit('/', 1)[-1]}`, add it to the banned imports in `pyproject.toml`." ) - def dropdown(f: str) -> List[str]: - n = len(selected_by[f]) - title = f"{f} → {n} test {'file' if n == 1 else 'files'}" - return ( - ["", "
", title, ""] - + _selection_tree(f, selected_by[f], direct_deps, suite_totals) - + ["", "
"] - ) - - for f in rows[:MAX_SUMMARY_ROWS]: - if selected_by[f]: - lines += dropdown(f) - # The tail of a wide diff: one collapsed section holding a dropdown per remaining file, so nothing is - # left unexplained while the top level stays at MAX_SUMMARY_ROWS entries. - if overflow: - lines += ["", "
", f"{len(overflow)} more changed files"] - for f in overflow: - lines += dropdown(f) if selected_by[f] else ["", f"`{f}` reaches no tests", ""] + if rows: + lines += ["", "
", "Details: how each changed file reaches its tests"] + for f in rows: + n = len(selected_by[f]) + title = f"{f} → {n} test {'file' if n == 1 else 'files'}" + lines += ["", "
", title, ""] + lines += _selection_tree(f, selected_by[f], direct_deps, suite_totals) + lines += ["", "
"] lines += ["", "
"] Path(summary_file).write_text("\n".join(lines) + "\n", encoding="utf-8") From 40470d3e2aec129be6d2c43ed1b6352a4ed67eef Mon Sep 17 00:00:00 2001 From: DN6 Date: Tue, 10 Nov 2026 14:08:10 +0530 Subject: [PATCH 20/24] [tests] Import the Flux and SD3 transformers at module level in `test_modeling_common.py`; the file is a leaf --- tests/models/test_modeling_common.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/models/test_modeling_common.py b/tests/models/test_modeling_common.py index fcc7f7593d95..655e49ab83f9 100644 --- a/tests/models/test_modeling_common.py +++ b/tests/models/test_modeling_common.py @@ -27,7 +27,7 @@ from huggingface_hub.utils import HfHubHTTPError, is_jinja_available from safetensors.torch import save_file -from diffusers.models import UNet2DConditionModel +from diffusers.models import FluxTransformer2DModel, SD3Transformer2DModel, UNet2DConditionModel from diffusers.models.model_loading_utils import _load_shard_file_tp from ..others.test_utils import TOKEN, USER, is_staging_test @@ -148,8 +148,6 @@ def test_cached_files_are_used_when_no_internet(self): assert False, "Parameters not the same!" def test_local_files_only_with_sharded_checkpoint(self): - from diffusers.models import FluxTransformer2DModel - repo_id = "hf-internal-testing/tiny-flux-sharded" error_response = mock.Mock(status_code=500, headers={}, json=mock.Mock(return_value={})) # `resolve_revision` inspects `error.response.status_code` to tell a Hub outage from a definitive answer, @@ -262,8 +260,6 @@ def test_keep_modules_in_fp32(self): A simple tests to check if the modules under `_keep_in_fp32_modules` are kept in fp32 when we load the model in fp16/bf16 Also ensures if inference works. """ - from diffusers.models import SD3Transformer2DModel - fp32_modules = SD3Transformer2DModel._keep_in_fp32_modules for torch_dtype in [torch.bfloat16, torch.float16]: From 45905d6bf1efb237cdd68e1334ae8c2d5ce88184 Mon Sep 17 00:00:00 2001 From: DN6 Date: Mon, 16 Nov 2026 12:42:54 +0530 Subject: [PATCH 21/24] [tests] Document `marker_decorators` with an example, name the file it reads, and explain the `core` collection hook --- tests/conftest.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9661340d45fc..36234fe2d870 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -34,10 +34,22 @@ def marker_decorators(): - """Map each `is_*` decorator in testing_utils.py to the pytest marker it applies. + """Map each `is_*` decorator in `tests/testing_utils.py` to the pytest marker it applies. - Read from the source with `ast` rather than imported, so `utils/tests_fetcher.py` can call this without - importing torch. Matches `def is_x(test_case): return pytest.mark.(test_case)`. + Only that file is read: it is where the marker decorators are defined. The `testing_utils/` packages + under `tests/models/` and `tests/pipelines/` hold the tester mixins that *use* these decorators, and the + per-pipeline `testing_utils.py` files hold test helpers; neither defines markers. Read from the source + with `ast` rather than imported, so `utils/tests_fetcher.py` can call this without importing torch. + Matches `def is_x(test_case): return pytest.mark.(test_case)`, so with + + def is_lora(test_case): + return pytest.mark.lora(test_case) + + def is_torch_compile(test_case): + return pytest.mark.compile(test_case) + + in `tests/testing_utils.py` the result is `{"is_lora": "lora", "is_torch_compile": "compile"}`. A new `is_*` + decorator is picked up automatically; nothing else needs registering. """ tree = ast.parse(open(join(dirname(__file__), "testing_utils.py"), encoding="utf-8").read()) decorators = {} @@ -89,7 +101,14 @@ def pytest_configure(config): @pytest.hookimpl(tryfirst=True) def pytest_collection_modifyitems(items): - """Mark every test with no feature marker as `core`, before `-m` deselection runs.""" + """Mark every test carrying none of `NON_CORE_MARKERS` as `core`. + + CI runs the feature jobs with `-m lora`, `-m memory`, ... and everything else with `-m core`. A positive + marker is the only way to say "everything else" to pytest: `-m "not (lora or memory or ...)"` would have + to list every feature marker and silently go stale. Adding the marker here, to every collected test + without a feature marker, keeps `core` exhaustive by construction. `tryfirst` runs this before pytest's + own `-m` deselection, which happens in the same hook. + """ for item in items: if not any(m.name in NON_CORE_MARKERS for m in item.iter_markers()): item.add_marker("core") From 14e737cebb83fd412630a89572f351cc642a45ae Mon Sep 17 00:00:00 2001 From: DN6 Date: Mon, 16 Nov 2026 13:32:30 +0530 Subject: [PATCH 22/24] [CI] Obtain the Hub token for single-file tests through OIDC in the fetcher workflow --- .github/workflows/pr_test_fetcher.yml | 7 +++++++ utils/tests_fetcher.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr_test_fetcher.yml b/.github/workflows/pr_test_fetcher.yml index 202c4614efca..684e2a9d6b0d 100644 --- a/.github/workflows/pr_test_fetcher.yml +++ b/.github/workflows/pr_test_fetcher.yml @@ -74,6 +74,7 @@ jobs: run_pr_tests: permissions: contents: read + id-token: write name: ${{ matrix.name }} needs: setup_pr_tests if: needs.setup_pr_tests.outputs.matrix != '[]' @@ -106,6 +107,12 @@ jobs: diffusers-cli env - name: Run all selected tests on CPU + # The gated single-file checkpoints need a Hub token, obtained through OIDC like the nightly and release + # workflows. Only runs that can mint an id token get the resource: a fork PR cannot, and huggingface_hub + # raises rather than falling back when `HF_OIDC_RESOURCE` is set with no id token, so there it is left + # empty and the `require_hf_token` tests skip instead. + env: + HF_OIDC_RESOURCE: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'Diffusers-CI-Bot' || '' }} run: | pytest -n 2 --dist=loadfile -m "${{ matrix.markers }}" --make-reports=${{ matrix.name }}_tests_cpu ${{ matrix.paths }} diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index 9b17a1640b4a..89f3c9e2e578 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -84,7 +84,7 @@ "memory": ["memory", "cpu_offload", "group_offload"], "cache": ["cache"], "ip_adapter": ["ip_adapter"], - # Needs only an HF token (gated checkpoints), which the CPU job provides; skips on fork PRs. + # Needs only a Hub token for gated checkpoints, which the CPU job obtains through OIDC; skips on fork PRs. "single_file": ["single_file"], } FEATURE_MARKERS = [m for markers in FEATURE_GROUPS.values() for m in markers] From e409a9fc75f7bc51d53434c5a606554ed1548568 Mon Sep 17 00:00:00 2001 From: DN6 Date: Mon, 16 Nov 2026 14:40:46 +0530 Subject: [PATCH 23/24] [CI] Simplify `tests_fetcher.py` and use dashed CLI flags --- utils/tests_fetcher.py | 97 ++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 50 deletions(-) diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index 89f3c9e2e578..630d3f72f40c 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -21,7 +21,7 @@ it), and select the impacted tests. There is no automatic full-suite trigger. If a change is in territory the import graph can't see -correctly (dynamic dispatch via auto-mappings, lazy `_import_structure`, etc.), pass `--force_full_suite` +correctly (dynamic dispatch via auto-mappings, lazy `_import_structure`, etc.), pass `--force-full-suite` to bypass selection. Pipeline-specific note: diffusers' `__init__.py` files use the `_import_structure = {...}` lazy-loading @@ -45,8 +45,8 @@ ```bash python utils/tests_fetcher.py # PR mode: diff against main -python utils/tests_fetcher.py --diff_with_last_commit # main mode: diff against last commit -python utils/tests_fetcher.py --force_full_suite # bypass selection, run everything +python utils/tests_fetcher.py --diff-with-last-commit # main mode: diff against last commit +python utils/tests_fetcher.py --force-full-suite # bypass selection, run everything ``` """ @@ -366,26 +366,24 @@ def create_reverse_dependency_map(direct_deps: Dict[str, List[str]]) -> Dict[str via the root init). 2. Invert. """ - all_modules = list(direct_deps) - one_hop_deps = direct_deps - direct_deps = {m: list(deps) for m, deps in direct_deps.items()} + closed = {m: list(deps) for m, deps in direct_deps.items()} # Each pass propagates dependency info one level deeper. Loop until a full pass adds nothing. changed = True while changed: changed = False - for m in all_modules: - if _merged_nested_deps(m, direct_deps): + for m in closed: + if _merged_nested_deps(m, closed): changed = True reverse_map: Dict[str, List[str]] = collections.defaultdict(list) - for m in all_modules: - for d in direct_deps[m]: + for m, deps in closed.items(): + for d in deps: reverse_map[d].append(m) # For inits, do the forward direction: editing an init impacts everything it re-exports. - for init in [m for m in all_modules if m.endswith("__init__.py")]: - deps = one_hop_deps[init] + for init in [m for m in direct_deps if m.endswith("__init__.py")]: + deps = direct_deps[init] impacted = set(deps) for d in deps: if not d.endswith("__init__.py"): @@ -470,6 +468,11 @@ def _selection_routes(changed: str, tests: set, direct_deps: Dict[str, List[str] return direct, sorted(via.items(), key=lambda kv: (-len(kv[1]), kv[0])) +def _suite_of(test_file: str) -> str: + parts = test_file.split("/") + return f"{parts[0]}/{parts[1]}/" if len(parts) > 2 else f"{parts[0]}/" + + def _suite_lines(tests: List[str], suite_totals: Dict[str, int]) -> List[str]: """Summarise many tests as suite coverage: `tests/models/: 80 of 104 test files`. A suite only brushed by the group (fewer than ten tests and under a quarter of the suite) is folded into one `N test files @@ -484,11 +487,6 @@ def _suite_lines(tests: List[str], suite_totals: Dict[str, int]) -> List[str]: return lines -def _suite_of(test_file: str) -> str: - parts = test_file.split("/") - return f"{parts[0]}/{parts[1]}/" if len(parts) > 2 else f"{parts[0]}/" - - def _selection_tree( changed: str, tests: set, direct_deps: Dict[str, List[str]], suite_totals: Dict[str, int] ) -> List[str]: @@ -509,14 +507,21 @@ def leaves(ts: List[str]) -> List[str]: branches += [(f"via {hop}", leaves(ts)) for hop, ts in via[:NAME_FOLDERS_UP_TO]] if len(via) > NAME_FOLDERS_UP_TO: rest = via[NAME_FOLDERS_UP_TO:] - by_package: Dict[str, List] = collections.defaultdict(lambda: [0, 0, collections.Counter()]) + by_package: Dict[str, List[Tuple[str, List[str]]]] = collections.defaultdict(list) for hop, ts in rest: - entry = by_package["/".join(hop.split("/")[:-1][:3]) + "/"] - entry[0] += 1 - entry[1] += len(ts) - entry[2].update(name for target, names in _extract_imports(hop) if target == changed for name in names) + by_package["/".join(hop.split("/")[:-1][:3]) + "/"].append((hop, ts)) children = [] - for pkg, (n, t, names) in sorted(by_package.items(), key=lambda kv: -kv[1][1])[:NAME_FOLDERS_UP_TO]: + for pkg, routes in sorted(by_package.items(), key=lambda kv: -sum(len(ts) for _, ts in kv[1]))[ + :NAME_FOLDERS_UP_TO + ]: + n, t = len(routes), sum(len(ts) for _, ts in routes) + names = collections.Counter( + name + for hop, _ in routes + for target, syms in _extract_imports(hop) + if target == changed + for name in syms + ) imported = ", ".join(name for name, _ in names.most_common(3)) children.append( f"{n} module{'s' if n != 1 else ''} in {pkg} → {t} test{'s' if t != 1 else ''}" @@ -550,8 +555,14 @@ def _write_summary( changed file with the import tree that explains its selection. The import chain for every selected test is in the stdout report, kept as a CI artifact. """ - selected_by = {f: _tests_triggered_by(f, reverse_map) for f in modified_files if (PATH_TO_REPO / f).exists()} - selected_by = {f: t for f, t in selected_by.items() if t or f.startswith("src/")} + # Deleted files select nothing; files outside src/ that select nothing (conftest, helpers) have nothing to say. + selected_by = { + f: tests + for f in modified_files + if (PATH_TO_REPO / f).exists() + for tests in [_tests_triggered_by(f, reverse_map)] + if tests or f.startswith("src/") + } suite_totals = collections.Counter(_suite_of(t) for t in _all_test_files()) self_only = [f for f, tests in selected_by.items() if tests == {f}] no_tests = sorted(f for f, tests in selected_by.items() if not tests) @@ -601,10 +612,8 @@ def _write_summary( (t, d, len(direct_deps[d])) for t in test_files if t in modified_files - for d in direct_deps.get(t, []) - if not d.endswith("__init__.py") - and "/testing_utils" not in d - and len(direct_deps.get(d, [])) >= WIDE_IMPORT_FANOUT + for d in direct_deps[t] + if not d.endswith("__init__.py") and "/testing_utils" not in d and len(direct_deps[d]) >= WIDE_IMPORT_FANOUT ] if wide_imports: lines += ["", "**Imports that make a test depend on most of the library**", ""] @@ -701,9 +710,8 @@ def _class_markers(test_file: str, bucket: str, mixin_markers: Dict[str, List[st def _feature_groups_for(paths: List[str], bucket: str, mixin_markers: Dict[str, List[str]]) -> List[str]: """Names of the feature groups (including `core`) that at least one test class in `paths` would land in.""" class_markers = [m for p in paths for m in _class_markers(p, bucket, mixin_markers)] - non_core = NON_CORE_MARKERS groups = [] - if any(not m & non_core for m in class_markers): + if any(not m & NON_CORE_MARKERS for m in class_markers): groups.append("core") for group, markers in FEATURE_GROUPS.items(): if any(m & set(markers) for m in class_markers): @@ -743,13 +751,7 @@ def fetch_tests_to_run(json_output_file: str, summary_file: str, diff_with_last_ direct_deps = create_direct_dependency_map() reverse_map = create_reverse_dependency_map(direct_deps) - # Each modified file contributes itself (if it's a test) plus tests transitively impacted by it. - selected = set() - for f in modified_files: - if _is_test_file(f): - selected.add(f) - selected.update(t for t in reverse_map.get(f, []) if _is_test_file(t)) - + selected = set().union(*(_tests_triggered_by(f, reverse_map) for f in modified_files)) test_files_to_run = sorted(p for p in selected if (PATH_TO_REPO / p).exists()) _print_selection_report(test_files_to_run, modified_files, direct_deps) _write_summary(summary_file, modified_files, test_files_to_run, direct_deps, reverse_map) @@ -757,17 +759,12 @@ def fetch_tests_to_run(json_output_file: str, summary_file: str, diff_with_last_ def _all_test_files() -> List[str]: - """Enumerate every `tests/.../test_*.py` (used for the full-suite path).""" + """Every `tests/.../test_*.py`: the full suite, and the denominator for suite coverage in the summary.""" return sorted( str(p.relative_to(PATH_TO_REPO)) for p in PATH_TO_TESTS.glob("**/test_*.py") if "__pycache__" not in p.parts ) -def _write_full_suite(json_output_file: str): - """Schedule the entire test suite. Used by `--force_full_suite` and as exception fallback.""" - _write_matrix(json_output_file, _all_test_files()) - - # ============================================================ # CLI # ============================================================ @@ -776,24 +773,24 @@ def _write_full_suite(json_output_file: str): if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( - "--json_output_file", + "--json-output-file", type=str, default="test_map.json", help="Where to store the list of matrix entries (name / paths / markers) consumed by CI.", ) parser.add_argument( - "--summary_output_file", + "--summary-output-file", type=str, default="test_fetcher_summary.md", help="Where to store the markdown report appended to the CI job summary.", ) parser.add_argument( - "--diff_with_last_commit", + "--diff-with-last-commit", action="store_true", help="Diff against the previous commit instead of main (use on main branch jobs)", ) parser.add_argument( - "--force_full_suite", + "--force-full-suite", action="store_true", help="Bypass selection and write outputs that schedule the entire test suite.", ) @@ -801,7 +798,7 @@ def _write_full_suite(json_output_file: str): if args.force_full_suite: print("Forcing full test suite.") - _write_full_suite(args.json_output_file) + _write_matrix(args.json_output_file, _all_test_files()) raise SystemExit(0) repo = Repo(PATH_TO_REPO) @@ -818,4 +815,4 @@ def _write_full_suite(json_output_file: str): print(f"\nError when trying to grab the relevant tests: {e}\n") traceback.print_exc() print("\nRunning all tests.") - _write_full_suite(args.json_output_file) + _write_matrix(args.json_output_file, _all_test_files()) From 2bd7b43829229455f2c81621e1834e05576bdb5d Mon Sep 17 00:00:00 2001 From: DN6 Date: Mon, 16 Nov 2026 16:45:50 +0530 Subject: [PATCH 24/24] [CI] Write the matrix before the summary so a report failure cannot schedule the full suite --- utils/tests_fetcher.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index 630d3f72f40c..7c21628171cf 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -753,9 +753,11 @@ def fetch_tests_to_run(json_output_file: str, summary_file: str, diff_with_last_ selected = set().union(*(_tests_triggered_by(f, reverse_map) for f in modified_files)) test_files_to_run = sorted(p for p in selected if (PATH_TO_REPO / p).exists()) + # The matrix is what CI runs; write it before the reports so a rendering bug in them cannot fall into the + # full-suite fallback in `__main__`. + _write_matrix(json_output_file, test_files_to_run) _print_selection_report(test_files_to_run, modified_files, direct_deps) _write_summary(summary_file, modified_files, test_files_to_run, direct_deps, reverse_map) - _write_matrix(json_output_file, test_files_to_run) def _all_test_files() -> List[str]: