diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 5e2a43d828..94156c8d9e 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -13,3 +13,4 @@ ## Checklist - [ ] I have read and confirmed this PR follows the [contribution guidelines](https://github.com/leejet/stable-diffusion.cpp/blob/master/CONTRIBUTING.md). +- [ ] If this PR adds or changes model support, I have added or updated its usage guide and relevant model index/README entries following the [model documentation requirements](https://github.com/leejet/stable-diffusion.cpp/blob/master/CONTRIBUTING.md#model-documentation). diff --git a/.gitignore b/.gitignore index 1a4646a5e9..13a77540c6 100644 --- a/.gitignore +++ b/.gitignore @@ -16,7 +16,7 @@ logs/ *.exe *.gguf output*.png -models* +/models* *.log preview.png .claude/ diff --git a/AGENTS.md b/AGENTS.md index a48040869d..ea93c92a7a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -91,7 +91,11 @@ documentation. ### Documentation and Tooling * `docs/` - documentation for supported models, build options, behavior, and - workflows. + workflows; start from `docs/README.md`. +* `docs/models/` - model and component usage guides, including LoRAs, adapters, + auxiliary models, and model-specific examples. +* `docs/guides/` - shared usage workflows, configuration, and performance guides. +* `docs/development/` - internal implementation and model integration conventions. * `scripts/` - development, model processing, build automation, formatting, and tooling scripts. * `cmake/` - CMake support modules. @@ -115,9 +119,13 @@ documentation. 1. Identify the relevant modules under `src/`. 2. Check whether the change touches the public API in `include/`. 3. Consult relevant `docs/` and examples before changing user-facing behavior. -4. Follow existing local patterns before adding new abstractions. -5. Keep edits scoped to the requested behavior. -6. Run the narrowest useful build, test, or inspection command available. +4. For model or component integrations, read and follow + [the model documentation requirements](CONTRIBUTING.md#model-documentation). + Add or update the usage guide and relevant index/README entries in the same + change. Treat the integration as incomplete until this documentation is included. +5. Follow existing local patterns before adding new abstractions. +6. Keep edits scoped to the requested behavior. +7. Run the narrowest useful build, test, or inspection command available. Follow `CONTRIBUTING.md` for formatting, naming, PR expectations, dependency update policy, and security rules. For tokenizer additions, follow its embedded-data diff --git a/CMakeLists.txt b/CMakeLists.txt index 14901f8f05..59107d4f40 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -240,6 +240,8 @@ file(GLOB SD_LIB_SOURCES CONFIGURE_DEPENDS "src/*.h" "src/*.cpp" "src/*.hpp" + "src/api/*.h" + "src/api/*.cpp" "src/conditioning/*.h" "src/conditioning/*.cpp" "src/conditioning/*.hpp" @@ -368,8 +370,15 @@ if (SD_CUDA) set_property(SOURCE src/core/ggml_extend_backend.cpp APPEND PROPERTY COMPILE_DEFINITIONS SD_USE_CUDA) if (NOT SD_USE_UPSTREAM_GGML) include(cmake/cutlass.cmake) + if (TARGET ggml-cuda) + # ggml resolves CUDA architectures in a child directory scope. + get_target_property(CMAKE_CUDA_ARCHITECTURES ggml-cuda CUDA_ARCHITECTURES) + endif() enable_language(CUDA) - set_target_properties(${SD_LIB} PROPERTIES CUDA_STANDARD 17 CUDA_STANDARD_REQUIRED ON) + set_target_properties(${SD_LIB} PROPERTIES + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + CUDA_ARCHITECTURES "${CMAKE_CUDA_ARCHITECTURES}") if (SD_CUDA_CUTLASS_PATH) target_sources(${SD_LIB} PRIVATE src/core/cuda/cute_attention.cu src/core/cuda/i8_projection.cu src/core/cuda/i8_gated_residual.cu src/core/cuda/i8_gemm.cu src/core/cuda/bf16_gemm.cu) set_source_files_properties(src/core/cuda/cute_attention.cu src/core/cuda/i8_projection.cu src/core/cuda/i8_gated_residual.cu src/core/cuda/i8_gemm.cu src/core/cuda/bf16_gemm.cu PROPERTIES @@ -424,6 +433,9 @@ target_include_directories(${SD_LIB} PUBLIC include) target_include_directories(${SD_LIB} PRIVATE src src/core) target_include_directories(${SD_LIB} PUBLIC thirdparty) target_compile_features(${SD_LIB} PUBLIC c_std_11 cxx_std_17) +set_target_properties(${SD_LIB} PROPERTIES + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON) if (SD_BUILD_EXAMPLES) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 15045dcc7e..1b5cc654d3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,19 +31,75 @@ PRs should include: ## Documentation -Write model usage guides in `docs/` for users. Focus on supported weights, -setup, runnable commands, parameter meanings, output behavior, and practical -limits such as memory requirements. Include technical details only when they -help users choose settings or troubleshoot a problem. +Write model and component usage guides in `docs/models/` for users. Focus on +supported weights, setup, runnable commands, parameter meanings, output behavior, +and practical limits such as memory requirements. Include technical details only +when they help users choose settings or troubleshoot a problem. + +Put shared usage workflows and configuration in `docs/guides/`, and internal +implementation details and development conventions in `docs/development/`. +Keep build, Docker, and troubleshooting guides at the root of `docs/`. +Link new documentation from [the documentation index](docs/README.md) or +[the model index](docs/models/README.md), as appropriate. Keep supported model +entries and their guide links in the project README. + +### Model Documentation + +Every new model or component integration must include a user guide in +`docs/models/` in the same PR. For a variant or new task of an existing family, +update that family's guide instead of creating a duplicate page. Add new guides +to the model index, and update the supported-model entries and guide links in +the project README when adding a model family or capability. + +Model guides should use the model name as their single top-level heading and +start with a short description of supported tasks and compatibility requirements. +`Model files` and `Usage` are required. Use the following sections in order, +omitting other sections when they have no content: + +- `Supported variants` when variants need a separate comparison. +- `Model files`, with component, weights/downloads, CLI option, and requirement + columns. Include tokenizers, projectors, connectors, downloads, conversion, and + required preprocessing. +- `Usage`, with `Web UI` and `CLI` subsections where applicable. Link to the + server's shared Web UI guide and list compatible recipes/components. Group + complete CLI examples by task or variant within this section; do not add a + separate `Quick start` or `Examples` section. Keep API usage in its own + subsection when needed. +- `Options` for model-specific settings. +- `Notes and limitations` for input/output constraints, memory requirements, + and unsupported functionality. + +Follow the [model example conventions](docs/models/README.md#example-conventions) +for working directories and shell syntax. Provide Windows PowerShell and +Linux/macOS forms of executable examples, using `.\bin\Release\sd-cli.exe` and +`./bin/sd-cli` respectively from the build directory. Use the corresponding +server paths for Web UI startup. Keep backend-specific examples clearly labeled. +Use multiline commands with mode and +model components first, followed by inputs/prompts, sampling settings, dimensions, +other model options, memory/backend options, and output/seed settings. Preserve +checkpoint-specific values when reorganizing existing examples. Keep example +images next to their commands, and update links when section anchors change. +Keep shared Web UI operating instructions in `examples/server/README.md` and +link to them from model guides. Document shared settings by linking to +`docs/guides/`. Only describe Web UI +recipes and capabilities that already exist; documentation cleanup alone is not +a reason to add or change a built-in recipe. + +Before submitting, check download links, relative links, command paths, CLI +options, and any recipe/component mappings against the implementation. Report +which examples were actually run and which were only inspected in the PR's +verification notes. Keep tensor layouts, internal class names, kernel implementation details, algorithm derivations, and development history out of usage guides. Keep task-specific profiling logs, numerical comparison procedures, and test artifacts local, and summarize relevant verification results in the PR description. -## README UI Listings +## Application and UI Listings -The [UIs list in the README](README.md#uis) only includes open-source projects that use `stable-diffusion.cpp` as a backend. Do not submit PRs to add closed-source UIs. +The [applications and UIs list](docs/ecosystem.md#applications-and-uis) only +includes open-source projects that use `stable-diffusion.cpp` as a backend. +Do not submit PRs to add closed-source applications or UIs. ## Server Recipes @@ -76,7 +132,8 @@ Naming conventions: Some older code in the project may not fully follow the current conventions. Please do not submit PRs that only rewrite existing code to match style rules. -When adding or modifying model implementations, follow the model config and weight detection conventions in [docs/model_config.md](docs/model_config.md). +When adding or modifying model implementations, follow the +[model config and weight detection conventions](docs/development/model_config.md). ## Tokenizer Data diff --git a/README.md b/README.md index 92103a357d..6d62d57539 100644 --- a/README.md +++ b/README.md @@ -39,68 +39,69 @@ API and command-line option may change frequently.*** - Plain C/C++ implementation based on [ggml](https://github.com/ggml-org/ggml), working in the same way as [llama.cpp](https://github.com/ggml-org/llama.cpp) - Super lightweight and without external dependencies +- Built-in [Web UI](./examples/server/README.md) for image and video generation - Supported models - Image Models - - [SD1.x, SD2.x, SD-Turbo](./docs/sd.md) - - [SDXL, SDXL-Turbo](./docs/sd.md) - - [Some SD1.x and SDXL distilled models](./docs/distilled_sd.md) - - [SD3/SD3.5](./docs/sd3.md) - - [FLUX.1-dev/FLUX.1-schnell](./docs/flux.md) - - [FLUX.2-dev/FLUX.2-klein](./docs/flux2.md) - - [FLUX.3 Image Klein / Klein Base](./docs/flux3.md) - - [MOSS-Image-Base / MOSS-Image-SFT](./docs/moss_image.md) - - [Fibo 2.0 Dev](./docs/fibo2.md) - - [Lens](./docs/lens.md) - - [Chroma](./docs/chroma.md) - - [Chroma1-Radiance](./docs/chroma_radiance.md) - - [Qwen Image](./docs/qwen_image.md) - - [Qwen Image 2.1](./docs/qwen_image_2.1.md) - - [PiD](./docs/pid.md) - - [LongCat Image](./docs/longcat_image.md) - - [Z-Image](./docs/z_image.md) - - [Z-Image L2P](./docs/z_image_l2p.md) - - [MiniT2I](./docs/minit2i.md) - - [P-H3-Image](./docs/p_h3_image.md) - - [MiniMax-H3-Image](./docs/minimax_h3_image.md) - - [SenseNova U1.5](./docs/sensenova_u1.md) - - [Ovis-Image](./docs/ovis_image.md) - - [Anima](./docs/anima.md) - - [ERNIE-Image](./docs/ernie_image.md) - - [Boogu Image](./docs/boogu_image.md) - - [Krea2](./docs/krea2.md) - - [Mage-Flow](./docs/mage_flow.md) - - [SeFi-Image](./docs/sefi_image.md) - - [HiDream-O1-Image](./docs/hidream_o1_image.md) - - [Ideogram4](./docs/ideogram4.md) - - [LLaDA-Image](./docs/llada_image.md) - - [Ming-Image Design](./docs/ming_image.md) - - [PixArt](./docs/pixart.md) - - [Iris-3B](./docs/iris.md) - - [Image Edit Models](./docs/edit.md) - - [FLUX.1-Kontext-dev](./docs/kontext.md) - - [Qwen Image Edit series](./docs/qwen_image_edit.md) - - [LongCat Image Edit](./docs/longcat_image.md) - - [Boogu Image Edit](./docs/boogu_image.md) - - [Mage-Flow-Edit](./docs/mage_flow.md#image-editing) - - [LLaDA-Image Edit](./docs/llada_image.md#image-editing) - - [Fibo 2.0 Dev / Edit Turbo](./docs/fibo2.md#editing-and-turbo) + - [SD1.x, SD2.x, SD-Turbo](./docs/models/sd.md) + - [SDXL, SDXL-Turbo](./docs/models/sd.md) + - [Some SD1.x and SDXL distilled models](./docs/models/distilled_sd.md) + - [SD3/SD3.5](./docs/models/sd3.md) + - [FLUX.1-dev/FLUX.1-schnell](./docs/models/flux.md) + - [FLUX.2-dev/FLUX.2-klein](./docs/models/flux2.md) + - [FLUX.3 Image Klein / Klein Base](./docs/models/flux3.md) + - [MOSS-Image-Base / MOSS-Image-SFT](./docs/models/moss_image.md) + - [Fibo 2.0 Dev](./docs/models/fibo2.md) + - [Lens](./docs/models/lens.md) + - [Chroma](./docs/models/chroma.md) + - [Chroma1-Radiance](./docs/models/chroma_radiance.md) + - [Qwen Image](./docs/models/qwen_image.md) + - [Qwen Image 2.1](./docs/models/qwen_image_2.1.md) + - [PiD](./docs/models/pid.md) + - [LongCat Image](./docs/models/longcat_image.md) + - [Z-Image](./docs/models/z_image.md) + - [Z-Image L2P](./docs/models/z_image_l2p.md) + - [MiniT2I](./docs/models/minit2i.md) + - [P-H3-Image](./docs/models/p_h3_image.md) + - [MiniMax-H3-Image](./docs/models/minimax_h3_image.md) + - [SenseNova U1.5](./docs/models/sensenova_u1.md) + - [Ovis-Image](./docs/models/ovis_image.md) + - [Anima](./docs/models/anima.md) + - [ERNIE-Image](./docs/models/ernie_image.md) + - [Boogu Image](./docs/models/boogu_image.md) + - [Krea2](./docs/models/krea2.md) + - [Mage-Flow](./docs/models/mage_flow.md) + - [SeFi-Image](./docs/models/sefi_image.md) + - [HiDream-O1-Image](./docs/models/hidream_o1_image.md) + - [Ideogram4](./docs/models/ideogram4.md) + - [LLaDA-Image](./docs/models/llada_image.md) + - [Ming-Image Design](./docs/models/ming_image.md) + - [PixArt](./docs/models/pixart.md) + - [Iris-3B](./docs/models/iris.md) + - [Image Edit Models](./docs/guides/edit.md) + - [FLUX.1-Kontext-dev](./docs/models/kontext.md) + - [Qwen Image Edit series](./docs/models/qwen_image_edit.md) + - [LongCat Image Edit](./docs/models/longcat_image.md) + - [Boogu Image Edit](./docs/models/boogu_image.md) + - [Mage-Flow-Edit](./docs/models/mage_flow.md#image-editing) + - [LLaDA-Image Edit](./docs/models/llada_image.md#image-editing) + - [Fibo 2.0 Dev / Edit Turbo](./docs/models/fibo2.md#fibo-20-edit-turbo) - Video Models - - [Wan2.1/Wan2.2](./docs/wan.md) - - [MiniMax-H3](./docs/minimax_h3.md) - - [LTX-2.3/LTX-2.5](./docs/ltx2.md) - - [HunyuanVideo 1.5](./docs/hunyuan_video.md) - - [LingBot-Video](./docs/lingbot_video.md) + - [Wan2.1/Wan2.2](./docs/models/wan.md) + - [MiniMax-H3](./docs/models/minimax_h3.md) + - [LTX-2.3/LTX-2.5](./docs/models/ltx2.md) + - [HunyuanVideo 1.5](./docs/models/hunyuan_video.md) + - [LingBot-Video](./docs/models/lingbot_video.md) - Video Upscale Models - - [RelayVSR](./docs/relay_vsr.md) - - [SeedVR2](./docs/seedvr2.md) - - [PhotoMaker](./docs/photo_maker.md) support. - - [IP-Adapter](./docs/ip_adapter.md) support (SD 1.5 and SDXL, including Plus) + - [RelayVSR](./docs/models/relay_vsr.md) + - [SeedVR2](./docs/models/seedvr2.md) + - [PhotoMaker](./docs/models/photo_maker.md) support. + - [IP-Adapter](./docs/models/ip_adapter.md) support (SD 1.5 and SDXL, including Plus) - Control Net support with SD 1.5 - - [ADetailer](./docs/adetailer.md) + - [ADetailer](./docs/models/adetailer.md) - LoRA support, same as [stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Features#lora) - Latent Consistency Models support (LCM/LCM-LoRA) - - Faster and memory efficient latent decoding with [TAESD](./docs/taesd.md) - - Upscale images generated with [ESRGAN](./docs/esrgan.md) + - Faster and memory efficient latent decoding with [TAESD](./docs/models/taesd.md) + - Upscale images generated with [ESRGAN](./docs/models/esrgan.md) - Supported backends - CPU (AVX, AVX2 and AVX512 support for x86 architectures) - CUDA @@ -122,16 +123,7 @@ API and command-line option may change frequently.*** - Negative prompt - [stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui) style tokenizer (not all the features, only token weighting for now) - VAE tiling processing for reduce memory usage -- Sampling method - - `Euler A` - - `Euler` - - `Heun` - - `DPM2` - - `DPM++ 2M` - - [`DPM++ 2M v2`](https://github.com/AUTOMATIC1111/stable-diffusion-webui/discussions/8457) - - `DPM++ 2S a` - - `ER-SDE` - - [`LCM`](https://github.com/AUTOMATIC1111/stable-diffusion-webui/issues/13952) +- Support for a wide range of [sampling methods and schedulers](./docs/guides/sampling.md) - Cross-platform reproducibility - `--rng cuda`, default, consistent with the `stable-diffusion-webui GPU RNG` - `--rng cpu`, consistent with the `comfyui RNG` @@ -139,73 +131,92 @@ API and command-line option may change frequently.*** ## Quick Start -### Get the sd executable +### Get the executables -- Download pre-built binaries from the [releases page](https://github.com/leejet/stable-diffusion.cpp/releases) +- Download and extract pre-built binaries from the [releases page](https://github.com/leejet/stable-diffusion.cpp/releases) - Or build from source by following the [build guide](./docs/build.md) +The executable examples below run from the build directory (`cd build` from +the repository root). Windows examples use the Visual Studio Release path; +adjust the executable path for other build configurations or extracted archives. +Model files stay under the repository's `models/` directory. + ### Download model weights -- download weights(.ckpt or .safetensors or .gguf). For example - - Stable Diffusion v1.5 from https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5 +Choose a model from the [model guides](./docs/models/README.md) and download its +weights and required components. For a simple example, save Stable Diffusion 1.5 +under `models/checkpoints/`: + +**Windows (PowerShell)** + +```powershell +New-Item -ItemType Directory -Force ..\models\checkpoints | Out-Null +curl.exe -L -o ..\models\checkpoints\v1-5-pruned-emaonly.safetensors https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.safetensors +``` + +**Linux/macOS** + +```sh +mkdir -p ../models/checkpoints +curl -L -o ../models/checkpoints/v1-5-pruned-emaonly.safetensors https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.safetensors +``` + +### Option 1: Web UI - ```sh - curl -L -O https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5/resolve/main/v1-5-pruned-emaonly.safetensors - ``` +Start the server: + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-server.exe --model-dir ..\models +``` -### Generate an image with just one command +**Linux/macOS** ```sh -./bin/sd-cli -m ../models/v1-5-pruned-emaonly.safetensors -p "a lovely cat" +./bin/sd-server --model-dir ../models ``` -***For detailed command-line arguments, check out [cli doc](./examples/cli/README.md).*** +Open , choose **SD 1.x** in **Recipe**, select the +checkpoint, and click **Load models**. Enter a prompt and click **Generate**. +See the [Web UI guide](./examples/server/README.md#web-ui) for the full workflow. + +### Option 2: CLI + +Generate an image with one command: + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe -m ..\models\checkpoints\v1-5-pruned-emaonly.safetensors -p "a lovely cat" +``` + +**Linux/macOS** + +```sh +./bin/sd-cli -m ../models/checkpoints/v1-5-pruned-emaonly.safetensors -p "a lovely cat" +``` + +Run `.\bin\Release\sd-cli.exe --help` on Windows or `./bin/sd-cli --help` on +Linux/macOS for all command-line options. See the +[CLI guide](./examples/cli/README.md) for additional usage notes. ## Performance -If you want to improve performance or reduce VRAM/RAM usage, please refer to [performance guide](./docs/performance.md). -For runtime and parameter backend placement, see the [backend selection guide](./docs/backend.md). +If you want to improve performance or reduce VRAM/RAM usage, please refer to [performance guide](./docs/guides/performance.md). +For runtime and parameter backend placement, see the [backend selection guide](./docs/guides/backend.md). + +## Documentation -## More Guides +Browse the [documentation index](./docs/README.md) or jump to a topic: +- [Getting started: build, Docker, CLI, and server](./docs/README.md#get-started) +- [Models and components](./docs/models/README.md) +- [Generation and inputs](./docs/README.md#generation-and-inputs) +- [Devices and performance](./docs/README.md#devices-and-performance) +- [Weights and quantization](./docs/README.md#weights-and-quantization) - [Troubleshooting](./docs/troubleshooting.md) -- [Backend selection](./docs/backend.md) -- [RPC](./docs/rpc.md) -- [LoRA](./docs/lora.md) -- [LCM/LCM-LoRA](./docs/lcm.md) -- [Docker](./docs/docker.md) -- [Quantization and GGUF](./docs/quantization_and_gguf.md) -- [INT8 convrot safetensors](./docs/int8_convrot.md) -- [Inference acceleration via caching](./docs/caching.md) - -## Bindings - -These projects wrap `stable-diffusion.cpp` for easier use in other languages/frameworks. - -* Golang (non-cgo): [seasonjs/stable-diffusion](https://github.com/seasonjs/stable-diffusion) -* Golang (cgo): [Binozo/GoStableDiffusion](https://github.com/Binozo/GoStableDiffusion) -* Golang (non-cgo): [l8bloom/gosd](https://github.com/l8bloom/gosd) -* C#: [DarthAffe/StableDiffusion.NET](https://github.com/DarthAffe/StableDiffusion.NET) -* Python: [william-murray1204/stable-diffusion-cpp-python](https://github.com/william-murray1204/stable-diffusion-cpp-python) -* Rust: [newfla/diffusion-rs](https://github.com/newfla/diffusion-rs) -* Flutter/Dart: [rmatif/Local-Diffusion](https://github.com/rmatif/Local-Diffusion) - -## UIs - -These projects use `stable-diffusion.cpp` as a backend for their image generation. - -- [GIMP Plugins](https://github.com/themanyone/gimp-plugins) -- [Stable Diffusion GUI](https://github.com/fszontagh/sd.cpp.gui.wx) -- [Stable Diffusion CLI-GUI](https://github.com/piallai/stable-diffusion.cpp) -- [Local Diffusion](https://github.com/rmatif/Local-Diffusion) -- [sd.cpp-webui](https://github.com/daniandtheweb/sd.cpp-webui) -- [LocalAI](https://github.com/mudler/LocalAI) -- [Neural-Pixel](https://github.com/Luiz-Alcantara/Neural-Pixel) -- [KoboldCpp](https://github.com/LostRuins/koboldcpp) -- [Kokon](https://github.com/GabrieleLozupone/kokon) -- [NodeTool](https://github.com/nodetool-ai/nodetool) -- [Quo](https://github.com/quo-ai/quo) -- [Prisme](https://github.com/d3Ita/prisme) +- [Development and contributing](./docs/README.md#development) ## Contributors diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000000..5490146b20 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,49 @@ +# Documentation + +For the project overview and supported models, see the [project README](../README.md). + +## Get started + +- [Model and component guides](models/README.md): download weights and run model-specific examples. +- [Build from source](build.md) +- [Docker](docker.md) +- [CLI usage guide](../examples/cli/README.md) +- [Server setup](../examples/server/README.md) +- [Server API reference](../examples/server/api.md) +- [Community bindings, applications, and UIs](ecosystem.md) +- [Troubleshooting](troubleshooting.md) + +## Generation and inputs + +- [C API and migration](guides/c_api.md) +- [Sampling methods and schedulers](guides/sampling.md) +- [Image editing](guides/edit.md) +- [Images in prompts](guides/prompt_images.md) +- [Image preprocessing](guides/image_preprocessing.md) +- [Generation results and metadata](guides/generation_metadata.md) +- [JSON tokenizers](guides/tokenizers.md) + +## Devices and performance + +- [Backend selection and device placement](guides/backend.md) +- [Performance and memory usage](guides/performance.md) +- [Inference caching](guides/caching.md) +- [RPC](guides/rpc.md) +- [SageAttention](guides/sage_attention.md) + +## Weights and quantization + +- [Quantization and GGUF conversion](guides/quantization_and_gguf.md) +- [Importance matrix quantization](guides/imatrix.md) +- [INT8 and INT4 convrot weights](guides/int8_convrot.md) + +## Development + +- [C API internals](development/c_api.md) +- [Model configuration and integration conventions](development/model_config.md) +- [ggml operator extensions](development/ggml_extensions.md) +- [Backend performance probes](development/backend_probe.md) +- [Position embedding cache](development/pe_cache.md) +- [PuLID implementation and validation](development/pulid.md) +- [Qwen Image 2.1 prefix cache internals](development/qwen_image_prefix_cache.md) +- [Contribution guidelines](../CONTRIBUTING.md) diff --git a/docs/anima.md b/docs/anima.md deleted file mode 100644 index 18b94952b6..0000000000 --- a/docs/anima.md +++ /dev/null @@ -1,21 +0,0 @@ -# How to Use - -## Download weights - -- Download Anima - - safetensors: https://huggingface.co/circlestone-labs/Anima/tree/main/split_files/diffusion_models - - gguf: https://huggingface.co/Bedovyy/Anima-GGUF/tree/main - - gguf Anima2: https://huggingface.co/JusteLeo/Anima2-GGUF/tree/main -- Download vae - - safetensors: https://huggingface.co/circlestone-labs/Anima/tree/main/split_files/vae -- Download Qwen3-0.6B-Base - - safetensors: https://huggingface.co/circlestone-labs/Anima/tree/main/split_files/text_encoders - - gguf: https://huggingface.co/mradermacher/Qwen3-0.6B-Base-GGUF/tree/main - -## Examples - -```sh -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\anima-preview.safetensors --vae ..\models\vae\qwen_image_vae.safetensors --llm ..\models\text_encoders\qwen_3_06b_base.safetensors -p "a lovely cat holding a sign says 'anima.cpp'" --cfg-scale 6.0 --sampling-method euler -v --offload-to-cpu -``` - -anima image example diff --git a/docs/animatediff.md b/docs/animatediff.md deleted file mode 100644 index 80c46ae64f..0000000000 --- a/docs/animatediff.md +++ /dev/null @@ -1,171 +0,0 @@ -# AnimateDiff (SD 1.5) - -AnimateDiff adds motion to a frozen Stable Diffusion 1.5 checkpoint by -injecting a temporal-attention module at 20 UNet slots. The base SD 1.5 -model, VAE, and text encoder are unchanged; only the motion module produces -the temporal residual that turns a batch of independent frames into a -coherent animation. Reference: Guo et al., "AnimateDiff: Animate Your -Personalized Text-to-Image Diffusion Models without Specific Tuning" -(https://arxiv.org/abs/2307.04725). - -## Download weights - -- Motion module (v3, recommended) - - fp16 safetensors: https://huggingface.co/conrevo/AnimateDiff-A1111/resolve/main/motion_module/mm_sd15_v3.safetensors - - original checkpoint: https://huggingface.co/guoyww/animatediff/resolve/main/v3_sd15_mm.ckpt -- SD 1.5 base model - - any SD 1.5 checkpoint works. `realisticVisionV60B1` and `toonyou_beta3` - are the ones used in guoyww's reference configs. -- Domain Adapter LoRA (optional, v3 only, sharpens the base UNet's output - toward the motion module's trained distribution) - - ckpt: https://huggingface.co/guoyww/animatediff/resolve/main/v3_sd15_adapter.ckpt - - place under your `--lora-model-dir` and reference in the prompt as - ``. - -The motion module is `~836 MB` and loads alongside the SD 1.5 UNet via -`--motion-module`. - -## Motion module versions - -Per [animatediff.net/models](https://animatediff.net/models): - -| Module | Base | Native res | Character | -|---------------------|------|------------|-----------| -| `mm_sd_v14.ckpt` | 1.5 | 256x256 | earliest, more jittery | -| `mm_sd_v15.ckpt` | 1.5 | 256x256 | improved stability over v1.4 | -| `mm_sd_v15_v2.ckpt` | 1.5 | 384x384 | significantly better motion dynamics | -| `v3_sd15_mm.ckpt` | 1.5 | 512x512 | smoothest, highest quality; pairs with a Domain Adapter LoRA | -| `mm_sdxl_v10_beta` | SDXL | 512x512 | experimental, not yet supported here | - -Match your `-H -W` to the module's native resolution for best results. v3 is -trained at 512x512 - going smaller (e.g. 384x384) still works but the motion -character is closer to v2. - -## Examples - -Generate an 8-frame animation at 512x512, seed 42, 20 steps. The sampler / -scheduler / CFG values below match what mm_sd15_v3 was trained with; using -SD 1.5 defaults (euler_a, low CFG) produces noise-like output. - -``` -.\bin\Release\sd-cli.exe -M vid_gen \ - --model ..\models\checkpoints\realisticVisionV60B1.safetensors \ - --motion-module ..\models\animatediff\mm_sd15_v3.safetensors \ - --offload-to-cpu \ - -p "a red apple on a wooden table" \ - --cfg-scale 8.0 --sampling-method euler --scheduler discrete \ - -H 512 -W 512 --video-frames 8 --fps 8 --steps 20 -s 42 \ - -o out.avi -``` - -Generate at the motion module's native 16-frame context (recommended for -best temporal quality). Needs more VRAM at 512x512, so drop to 384x384 or -use layer streaming: - -``` -.\bin\Release\sd-cli.exe -M vid_gen \ - --model ..\models\checkpoints\realisticVisionV60B1.safetensors \ - --motion-module ..\models\animatediff\mm_sd15_v3.safetensors \ - --offload-to-cpu \ - -p "photo of coastline, rocks, storm weather, wind, waves, lightning" \ - --cfg-scale 8.0 --sampling-method euler --scheduler discrete \ - -H 384 -W 384 --video-frames 16 --fps 8 --steps 20 -s 42 \ - -o out.avi -``` - -Low-VRAM streaming (verified with a 2 GiB cap on RTX 3060): - -``` -.\bin\Release\sd-cli.exe -M vid_gen \ - --model ..\models\checkpoints\realisticVisionV60B1.safetensors \ - --motion-module ..\models\animatediff\mm_sd15_v3.safetensors \ - --max-vram 2.0 \ - -p "photo of coastline, rocks, storm weather, wind, waves, lightning" \ - --cfg-scale 8.0 --sampling-method euler --scheduler discrete \ - -H 384 -W 384 --video-frames 8 --fps 8 --steps 20 -s 42 \ - -o out.avi -``` - -## Reference-quality reproduction - -Using guoyww's official reference configs on this impl (RealisticVision v6.0 -base + `mm_sd15_v3` or `mm_sd_v15_v2` + native resolution + 16 frames + euler -+ 25 steps + CFG 8 + linear beta schedule) reproduces the reference -AnimateDiff output style. - -### v3 (512x512, `mm_sd15_v3`) - -| Prompt | Sample | -|---------------------------------------|--------| -| B&W man on stormy coastline | | -| Close-up rabbit macro shot | | -| Coastline, storm, waves, lightning | | -| Old house, storm, forest, night | | - -### v2 (384x384, `mm_sd_v15_v2.ckpt`) - -| Prompt | Sample | -|---------------------------------------|--------| -| B&W man on stormy coastline | | -| Close-up rabbit macro shot | | -| Coastline, storm, waves, lightning | | -| Old house, storm, forest, night | | - -Motion is strong for scenes with motion cues in the prompt (storm/waves/wind) -and subtle for static subjects (close-up macro), matching reference behavior. -v2 has an additional motion module at the UNet middle block that v3 dropped; -this impl auto-detects the topology from the checkpoint. - -### v3 + Domain Adapter LoRA - -Attaching the `v3_sd15_adapter` LoRA sharpens the base UNet output toward -the training distribution the motion module was fine-tuned against. Same -prompt, seed, config as above: - - - -Individual fur strands, glowing inner-ear, and richer forest detail become -visible compared to the no-LoRA rendering. - -``` -sd-cli -M vid_gen --model realisticVisionV60B1.safetensors \ - --motion-module mm_sd15_v3.safetensors \ - --lora-model-dir ./loras \ - -p "close up photo of a rabbit ..." ... -``` - -## img2video - -Pass a pre-rendered image via `-i / --init-img` to animate FROM it. All N output frames start from the encoded init latent, then per-frame noise is added at `--strength`. Character identity, composition, and quality are anchored by the init image; the motion module adds subtle motion on top. - -Left: init image rendered with `-M img_gen`. Right: 8-frame vid_gen output. - - - -``` -sd-cli -M img_gen ... -o init.png # any high-quality still -sd-cli -M vid_gen --motion-module mm_sd15_v3.safetensors \ - -i init.png --strength 0.75 \ - --cfg-scale 7.0 --sampling-method euler --scheduler karras \ - -H 512 -W 512 --video-frames 8 --steps 25 -s 42 \ - -p "..." -o out.avi -``` - -`--strength` controls how far the motion module is allowed to deviate from the init image (higher = more motion, lower = more static). - -## Notes - -- The motion module was trained at `video_length=16`. Running with - `--video-frames 16` gives the best coherence; F=8 works but shows a shorter - motion arc. Frame counts up to 32 are supported by the positional encoding - but exceed the trained regime and produce more static output. -- At `--video-frames 1` the motion module is skipped entirely and the output - is bit-identical to `-M img_gen`. This avoids the single-token - temporal-attention degeneracy that would otherwise emit an untrained-magnitude - residual on a single-frame sample. -- The base UNet is frozen, so character identity and style follow the SD 1.5 - checkpoint you pass to `--model`. LoRAs and prompt weighting attach to the - base model in the usual way. -- No mid_block motion module in v3. `mm_sdxl_v10_beta` (SDXL variant) is not - supported yet. -- Output is written as MJPEG AVI. Use `--fps` to set playback speed. diff --git a/docs/boogu_image.md b/docs/boogu_image.md deleted file mode 100644 index c5ad97dd85..0000000000 --- a/docs/boogu_image.md +++ /dev/null @@ -1,31 +0,0 @@ -# How to Use - -Boogu Image uses a Boogu diffusion transformer, the FLUX VAE, and Qwen3-VL as the LLM text and vision encoder. - -## Download weights - -- Download Boogu Image - - safetensors: https://huggingface.co/Comfy-Org/Boogu-Image/tree/main/diffusion_models -- Download vae - - safetensors: https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/ae.safetensors -- Download Qwen3-VL 8B - - gguf: https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/tree/main - - For image editing with GGUF text encoders, also download the matching mmproj file and pass it with `--llm_vision`. - -## Examples - -### Boogu Image Base - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\boogu_image_base_bf16.safetensors --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf --vae ..\models\vae\ae.sft -p "a lovely cat" -v --offload-to-cpu -``` - -Boogu Image Base example - -### Boogu Image Edit - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\boogu_image_edit_bf16.safetensors --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf --llm_vision ..\models\text_encoders\mmproj-Qwen3VL-8B-Instruct-F16.gguf --vae ..\models\vae\ae.sft -v --offload-to-cpu -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'boogu.cpp'" -``` - -Boogu Image Edit example \ No newline at end of file diff --git a/docs/build.md b/docs/build.md index 575436a218..1533cfb2bc 100644 --- a/docs/build.md +++ b/docs/build.md @@ -21,25 +21,6 @@ git pull origin master git submodule update --init --recursive ``` -## MSVC compilation parallelism - -MSVC builds limit parallel C/C++ compilation within each project by default. -For `n` logical processors, the limit is `n` when `n <= 2`, `n - 1` when -`3 <= n <= 8`, and `n - 2` when `n > 8`. The default is calculated when the -build directory is first configured with this setting. Override it with -`-DSD_MSVC_MP_COUNT=4`, or pass `-USD_MSVC_MP_COUNT` to recalculate it. - -For Visual Studio builds, keep project-level parallelism at one to avoid -multiplying the per-project limit: - -```shell -cmake --build build --config Release --parallel 1 -``` - -This setting controls MSVC's `/MP` option, not CUDA compiler parallelism or a -hard CPU usage limit. Other build tools, such as Ninja, schedule compilation -jobs separately; use `cmake --build build --parallel N` to limit those jobs. - ## Selecting a GGML source tree By default, sd.cpp builds the patched GGML submodule in `ggml/`. To build with @@ -120,7 +101,7 @@ cmake --build . --config Release The sd.cpp SageAttention extension is included when using CUDA with patched GGML (`SD_USE_UPSTREAM_GGML=OFF`). -See [SageAttention](sage_attention.md) for GPU requirements and `--sage-attn` usage. +See [SageAttention](guides/sage_attention.md) for GPU requirements and `--sage-attn` usage. This provides GPU acceleration using NVIDIA GPU. Make sure to have the CUDA toolkit installed. You can download it from your Linux distro's package manager (e.g. `apt install nvidia-cuda-toolkit`) or from here: [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads). Recommended to have at least 4 GB of VRAM. @@ -130,6 +111,11 @@ cmake .. -DSD_CUDA=ON cmake --build . --config Release ``` +When building GGML from source, sd.cpp's CUDA extensions use the same target +architectures as GGML, including native GPU detection with `GGML_NATIVE=ON`. +To select target GPUs explicitly, pass `-DCMAKE_CUDA_ARCHITECTURES=89` (Ada), +for example, or a quoted list such as `-DCMAKE_CUDA_ARCHITECTURES="80;89"`. + CUDA extensions use sd.cpp's `thirdparty/cutlass` submodule by default for eligible INT8 GEMM and CuTe attention operations. If its headers are missing, CMake stops with an initialization hint; it does not download them automatically. @@ -152,21 +138,83 @@ the old bundled path becomes `thirdparty/cutlass`, while custom paths and an empty disabled value are preserved. An explicitly set `SD_CUDA_CUTLASS_PATH` takes precedence. Non-CUDA builds do not require CUTLASS. -## Build with HipBLAS +## Build with HIPBLAS + +Use `-DSD_HIPBLAS=ON` for AMD GPU acceleration through HIP and hipBLAS. +Install a compatible AMD GPU driver and the ROCm SDK, including the HIP compiler, +hipBLAS, rocBLAS, and their development files. See +[AMD's ROCm installation guide](https://rocm.docs.amd.com/en/latest/install/rocm.html) +for supported GPUs, operating systems, and installation methods. + +When the hipBLASLt development package (version 1.0 or later) is available, +CMake also enables INT8 GEMM tuning. The first run of an eligible matrix shape +benchmarks the no-workspace candidates returned by hipBLASLt for the current +device, including CDNA devices, and caches the result in the process. It +selects a hipBLASLt candidate that runs successfully and beats hipBLAS, or a +working candidate if hipBLAS does not support the operation. +Set `SD_HIP_DISABLE_I8_LT=1` to use the hipBLAS GEMM path for comparison. + +With a compiled wave32 WMMA kernel, the HIP flash attention path supports +head dimensions 40, 52, 64, 72, 80, 120, 128, 160, 256, and 320, including +grouped-query attention. It requires unmasked attention with at least 512 +query tokens and 512 key tokens. Set `SD_HIP_DISABLE_WMMA_ATTN=1` to use +ggml's native attention path for comparison. -This provides GPU acceleration using AMD GPU. Make sure to have the ROCm toolkit installed. -To build for another GPU architecture than installed in your system, set `$GFX_NAME` manually to the desired architecture (replace first command). This is also necessary if your GPU is not officially supported by ROCm, for example you have to set `$GFX_NAME` manually to `gfx1030` for consumer RDNA2 cards. +**Linux** -Windows User Refer to [docs/hipBLAS_on_Windows.md](docs%2FhipBLAS_on_Windows.md) for a comprehensive guide. +With ROCm installed, use the default CMake generator and host compilers: ```shell mkdir build && cd build -if command -v rocminfo; then export GFX_NAME=$(rocminfo | awk '/ *Name: +gfx[1-9]/ {print $2; exit}'); else echo "rocminfo missing!"; fi -if [ -z "${GFX_NAME}" ]; then echo "Error: Couldn't detect GPU!"; else echo "Building for GPU: ${GFX_NAME}"; fi -cmake .. -G "Ninja" -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DSD_HIPBLAS=ON -DCMAKE_BUILD_TYPE=Release -DGPU_TARGETS=$GFX_NAME -DAMDGPU_TARGETS=$GFX_NAME -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -DCMAKE_POSITION_INDEPENDENT_CODE=ON +cmake .. -DSD_HIPBLAS=ON cmake --build . --config Release ``` +For an SDK outside the default installation path, set `ROCM_PATH` to its root +and add `$ROCM_PATH/bin` to `PATH` before configuring. If its shared libraries +are not on the system library search path, add `$ROCM_PATH/lib` and +`$ROCM_PATH/lib64` to `LD_LIBRARY_PATH` when running the executable. + +**Windows** + +Follow the Windows environment setup in [Ninja](#ninja). The Windows HIP build +requires ROCm Clang with Ninja; the Visual Studio generator is not supported. + +Set `HIP_PATH` to the SDK root. The HIP SDK installer normally sets this variable; +for [ROCm wheels](https://github.com/ROCm/TheRock/blob/main/RELEASES.md), run +`rocm-sdk init` and use `rocm-sdk path --root`. The build commands below cover +both the installer and wheel SDK compiler layouts. + +For the wheel SDK, also set the device library path before building: + +```powershell +$env:HIP_DEVICE_LIB_PATH = "$env:HIP_PATH\lib\llvm\amdgcn\bitcode" +``` + +Then configure and build: + +```powershell +$env:ROCM_PATH = $env:HIP_PATH +$env:PATH = "$env:HIP_PATH\bin;$env:HIP_PATH\lib\llvm\bin;$env:PATH" +cmake -S . -B build-hip -G Ninja -DSD_HIPBLAS=ON -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Release +cmake --build build-hip --config Release +``` + +Keep the SDK's `bin` directory on `PATH` when running `build-hip\bin\sd-cli.exe` +so its runtime DLLs can be found. + +**GPU targets and verification** + +CMake normally detects the installed GPU. To select a target explicitly or build +on a machine without an AMD GPU, add `-DGPU_TARGETS=gfx1100` to the configure +command (for example, for an RX 7900 XTX, XT, or GRE). For multiple targets, use a +quoted list such as `-DGPU_TARGETS="gfx1100;gfx1200"`. Check AMD's +[GPU specifications](https://rocm.docs.amd.com/en/latest/reference/gpu-specs.html) +for the correct target; the installed SDK libraries must also support it. + +Use a fresh build directory when switching SDKs or compilers. Run the resulting +`sd-cli --version` to check that the executable starts without loading a model. + ## Build with MUSA This provides GPU acceleration using Moore Threads GPUs. The configuration below follows upstream validation with MUSA SDK 5.2.0 and MTT S5000 (PH1, `mp_31`). Install that toolkit before building. Older SDKs such as 4.2 may lack the CUB headers required by the current backend. @@ -210,10 +258,8 @@ Currently, it supports only Adreno GPUs and is primarily optimized for Q4_0 type To build for Windows ARM please refers to [Windows 11 Arm64](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/OPENCL.md#windows-11-arm64) -Building for Android: - - Android NDK: - Download and install the Android NDK from the [official Android developer site](https://developer.android.com/ndk/downloads). +For Android, prepare [Ninja](#ninja) and install the Android NDK from the +[official Android developer site](https://developer.android.com/ndk/downloads). Setup OpenCL Dependencies for NDK: @@ -245,7 +291,7 @@ You need to provide OpenCL headers and the ICD loader library to your NDK sysroo -DANDROID_PLATFORM=24 \ -DANDROID_STL=c++_shared - ninja + cmake --build . # Replace # e.g., cp libOpenCL.so /path/to/android-ndk-r26c/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/aarch64-linux-android sudo cp libOpenCL.so /toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/aarch64-linux-android @@ -266,7 +312,7 @@ cmake .. -G Ninja \ -DGGML_OPENMP=OFF \ -DSD_OPENCL=ON -ninja +cmake --build . ``` *(Note: Don't forget to include `LD_LIBRARY_PATH=/vendor/lib64` in your command line before running the binary)* @@ -278,6 +324,9 @@ Using SYCL makes the computation run on the Intel GPU. Please make sure you have # Export relevant ENV variables source /opt/intel/oneapi/setvars.sh +# Make and enter build dir +mkdir build && cd build + # Option 1: Use FP32 (recommended for better performance in most cases) cmake .. -DSD_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx @@ -286,3 +335,89 @@ cmake .. -DSD_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SY cmake --build . --config Release ``` + +## Build tools and compiler cache + +The settings in this section can be combined with the backend options in +this guide. + +### Ninja + +Install Ninja and make sure `ninja --version` works in your build terminal. +For native Windows builds with MSVC or ROCm Clang, install Visual Studio +2022 C++ build tools with the Windows SDK, CMake, and Ninja, then open an +x64 **Developer PowerShell for VS 2022**. + +From the repository root, configure and build a Release version with: + +```shell +cmake -S . -B build-ninja -G Ninja -DCMAKE_BUILD_TYPE=Release +cmake --build build-ninja --parallel 8 +``` + +This builds the CPU backend by default. Add backend options such as +`-DSD_CUDA=ON` or `-DSD_VULKAN=ON` to the configure command as needed. +With MSVC, you can explicitly select the compiler by adding +`-DCMAKE_C_COMPILER=cl.exe -DCMAKE_CXX_COMPILER=cl.exe`. +Adjust `--parallel 8` for your available CPU and memory resources. + +Use a separate build directory when switching from another CMake generator, +such as Visual Studio. With this Ninja generator, select the build type at +configure time using `CMAKE_BUILD_TYPE`. + +### ccache (optional) + +Install [ccache](https://ccache.dev/download.html) and add it to `PATH`. +Check that it is available and optionally set a cache size limit: + +```shell +ccache --version +ccache --max-size=20G +``` + +To cache C and C++ compilation, configure with compiler launchers: + +```shell +cmake -S . -B build-ninja -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache +cmake --build build-ninja --parallel 8 +ccache --show-stats +``` + +For CUDA builds, also add `-DSD_CUDA=ON` and +`-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache` to the configure command so CUDA +compilation is cached as well. Keep any other backend or compiler options +used for your build. + +These [compiler launcher settings](https://cmake.org/cmake/help/latest/prop_tgt/LANG_COMPILER_LAUNCHER.html) +work with Ninja and Makefile generators. The Visual Studio generator does +not use them. + +The first compilation populates the cache. After a successful build, you can +check reuse by rebuilding the same configuration with the cache intact: + +```shell +ccache --zero-stats +cmake --build build-ninja --clean-first --parallel 8 +ccache --show-stats +``` + +Look for cache hits in the statistics. A normal incremental build with no +changed files skips compilation and does not produce new cache hits. + +### MSVC compilation parallelism + +MSVC builds limit parallel C/C++ compilation within each project by default. +For `n` logical processors, the limit is `n` when `n <= 2`, `n - 1` when +`3 <= n <= 8`, and `n - 2` when `n > 8`. The default is calculated when the +build directory is first configured with this setting. Override it with +`-DSD_MSVC_MP_COUNT=4`, or pass `-USD_MSVC_MP_COUNT` to recalculate it. + +For Visual Studio builds, keep project-level parallelism at one to avoid +multiplying the per-project limit: + +```shell +cmake --build build --config Release --parallel 1 +``` + +This setting controls MSVC's `/MP` option, not CUDA compiler parallelism or a +hard CPU usage limit. For Ninja job limits, see [Ninja](#ninja). diff --git a/docs/chroma.md b/docs/chroma.md deleted file mode 100644 index 606a85fe5e..0000000000 --- a/docs/chroma.md +++ /dev/null @@ -1,32 +0,0 @@ -# How to Use - -You can run Chroma using stable-diffusion.cpp with a GPU that has 6GB or even 4GB of VRAM, without needing to offload to RAM. - -## Download weights - -- Download Chroma - - If you don't want to do the conversion yourself, download the preconverted gguf model from [silveroxides/Chroma-GGUF](https://huggingface.co/silveroxides/Chroma-GGUF) - - Otherwise, download chroma's safetensors from [lodestones/Chroma](https://huggingface.co/lodestones/Chroma) -- Download vae from https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/ae.safetensors -- Download t5xxl from https://huggingface.co/comfyanonymous/flux_text_encoders/blob/main/t5xxl_fp16.safetensors - -## Convert Chroma weights - -You can download the preconverted gguf weights from [silveroxides/Chroma-GGUF](https://huggingface.co/silveroxides/Chroma-GGUF), this way you don't have to do the conversion yourself. - -``` -.\bin\Release\sd-cli.exe -M convert -m ..\models\unet\chroma-unlocked-v40.safetensors -o ..\models\chroma-unlocked-v40-q8_0.gguf -v --type q8_0 -``` - -## Run - -### Example -For example: - -``` - .\bin\Release\sd-cli.exe --diffusion-model ..\models\chroma-unlocked-v40-q8_0.gguf --vae ..\models\ae.sft --t5xxl ..\models\t5xxl_fp16.safetensors -p "a lovely cat holding a sign says 'chroma.cpp'" --cfg-scale 4.0 --sampling-method euler -v --model-args chroma_use_dit_mask=false --clip-on-cpu -``` - -![](../assets/flux/chroma_v40.png) - - diff --git a/docs/chroma_radiance.md b/docs/chroma_radiance.md deleted file mode 100644 index fdd6d0ade4..0000000000 --- a/docs/chroma_radiance.md +++ /dev/null @@ -1,21 +0,0 @@ -# How to Use - -## Download weights - -- Download Chroma1-Radiance - - safetensors: https://huggingface.co/lodestones/Chroma1-Radiance/tree/main - - gguf: https://huggingface.co/silveroxides/Chroma1-Radiance-GGUF/tree/main - -- Download t5xxl - - safetensors: https://huggingface.co/comfyanonymous/flux_text_encoders/blob/main/t5xxl_fp16.safetensors - -## Examples - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\Chroma1-Radiance-v0.4-Q8_0.gguf --t5xxl ..\models\clip\t5xxl_fp16.safetensors -p "a lovely cat holding a sign says 'chroma radiance cpp'" --cfg-scale 4.0 --sampling-method euler -v -``` - -Chroma1-Radiance - - - diff --git a/docs/backend_probe.md b/docs/development/backend_probe.md similarity index 100% rename from docs/backend_probe.md rename to docs/development/backend_probe.md diff --git a/docs/development/c_api.md b/docs/development/c_api.md new file mode 100644 index 0000000000..58587a6fac --- /dev/null +++ b/docs/development/c_api.md @@ -0,0 +1,63 @@ +# C API Internals + +The public declarations live in [stable-diffusion.h](../../include/stable-diffusion.h). +See the [C API guide](../guides/c_api.md) for keys, value types and ownership rules. + +## Internal modules + +| Module | Responsibility | +| --- | --- | +| `src/api/options.h/.cpp` | Opaque options, key registration, setter validation and conversion to typed configuration. | +| `src/config.h/.cpp` | Typed configuration and the library defaults for each configuration group. | +| `src/api/tensor.h/.cpp` | Opaque tensor handles backed by `sd::Tensor`, reference counting and conversion to private media views. | +| `src/core/media.h` | Private raw media views used by inference and codecs. | +| `src/api/result.h/.cpp` | Internal result storage, opaque result accessors, metadata attachment and media ownership. | +| `src/api/api.h/.cpp` | Context, generation, callback and query entry points, plus thread-local diagnostics and exception-to-status handling. | +| `src/generation_info.h/.cpp` | Construction of generation metadata JSON. | +| `src/pipeline/generation.h` | Internal generation entry declarations. | +| `src/stable-diffusion.cpp` | Internal typed image/video generation and model-specific validation. | +| `src/core/option_types.h/.cpp` | Internal mode enums, name lookup and parsing. | + +Options are validated and converted once at the API boundary. The runtime reads +typed configuration fields. Internal callers such as ADetailer pass typed +configuration directly to the internal generation entry; they do not rebuild +options or call public setters. + +The option map alone records whether a key was supplied. Zero, false, empty +strings and explicit automatic choices remain entries until `unset` removes +them. Each conversion starts with a fresh `OptConfig`, then overlays supplied +values and validates combinations. Reusing the output configuration cannot +retain a previous request's overrides. ADetailer applies its operation-specific +strength default before the overlay. + +Fixed defaults live in the configuration constructors. Internal automatic +policies (such as model-selected sampler/scheduler and automatic eta/flow shift) +are resolved by the pipeline after refreshing the model sources. Their internal +sentinel values describe an automatic policy, not the presence of an input key. +Do not infer presence by comparing a typed value against its default. + +CLI/server inference inputs use `std::optional`; their option builders emit +only engaged fields. Recipe defaults fill missing startup inputs; request +fields then override the configured recipe/startup values. Explicit empty +numeric arrays replace inherited arrays, and JSON `null` clears an optional +input. Frontend-only settings such as output formats and listen ports keep +local defaults. Frontend checks cover request syntax, file inputs and server +resource policy. Parameter ranges are checked by the public setters used by the +option builders; combinations and model applicability are checked when executing. +Examples include only public library headers and do not query library defaults. +Capability defaults describe configured overrides; null means that the library +selects the value, and result metadata reports effective values. Post-generation +ADetailer clears inherited dimensions and strength before applying pass overrides, +so its operation defaults also come from the library. + +Result storage owns tensors and metadata. Tensor handles contain a dtype-specific +`sd::Tensor` and an atomic reference count; options retain handles rather than +copying tensor payloads. Raw pipeline output buffers are converted to tensors at +the boundary and released there. Internal producers with an `sd::Tensor` can +move it directly into a handle. Video results keep per-frame tensors and assemble +a contiguous video tensor only when requested. Cached assembly is synchronized. +Preview tensors are independent snapshots and may be retained by callbacks. +Metadata construction is separate from its attachment to a result. + +These headers are private implementation details and are not installed as part +of the public C API. diff --git a/docs/ggml_extensions.md b/docs/development/ggml_extensions.md similarity index 100% rename from docs/ggml_extensions.md rename to docs/development/ggml_extensions.md diff --git a/docs/model_config.md b/docs/development/model_config.md similarity index 98% rename from docs/model_config.md rename to docs/development/model_config.md index d42c3226ce..398211682f 100644 --- a/docs/model_config.md +++ b/docs/development/model_config.md @@ -139,6 +139,9 @@ To add a model: 5. Add version detection signatures and any required weight-name mappings in `src/model_loader.cpp` and `src/name_conversion.cpp` respectively. Update shared conditioning or runtime code only when existing behavior cannot serve the model. +6. Add or update the model's usage guide and relevant index/README entries in the + same change, following the + [model documentation requirements](../../CONTRIBUTING.md#model-documentation). Use the existing family integration as the starting point for a variant. For a new family, `src/model/integrations/moss_image.cpp` is a compact example of a diff --git a/docs/pe_cache.md b/docs/development/pe_cache.md similarity index 100% rename from docs/pe_cache.md rename to docs/development/pe_cache.md diff --git a/docs/development/pulid.md b/docs/development/pulid.md new file mode 100644 index 0000000000..b9ed66bee4 --- /dev/null +++ b/docs/development/pulid.md @@ -0,0 +1,46 @@ +# PuLID implementation and validation + +For model files, identity preparation, and generation commands, see the +[PuLID usage guide](../models/pulid.md). + +## Architecture summary + +The PuLID-Flux contribution to the Flux denoise loop is a stack of 20 +small cross-attention modules (`PerceiverAttentionCA`) inserted between +the Flux transformer blocks: + +- After every 2nd of the 19 double-stream blocks (10 hook points) +- After every 4th of the 38 single-stream blocks (10 hook points) + +Each cross-attention layer takes the current image tokens as query, the +32-token / 2048-dim identity embedding as key+value, and adds its output +(scaled by `id_weight`, typically 1.0) back to the image tokens. + +## Format (gguf) + +The embedding is a standard **gguf** container holding a single tensor: + +``` +tensor name : "pulid_id" +shape : [token_dim, num_tokens] (ggml order; typically [2048, 32]) +type : F16 (also accepts F32 / BF16) +metadata : general.architecture = "pulid", pulid.version = 1 +``` + +stable-diffusion.cpp loads it with the normal gguf reader +(`gguf_init_from_file`) and converts to fp32 at load time -- no bespoke +parser. Total file size for the typical (32, 2048, fp16) case is ~131 KB. + +## Verification + +A three-way SHA-256 check is the recommended sanity test when bringing up +a new combination of model + backend + hardware: + +| Run | Expected hash relation | +|----------------------------------------------|------------------------------------| +| A: no `--pulid-*` flags | baseline | +| B: PuLID flags, `--pulid-id-weight 0.0` | **byte-identical to A** | +| C: PuLID flags, `--pulid-id-weight 1.0` | **different from A,B**, preserves source identity | + +If A and C differ but A and B differ too, the injection is allocating +or computing something even at zero weight -- likely a bug. diff --git a/docs/development/qwen_image_prefix_cache.md b/docs/development/qwen_image_prefix_cache.md new file mode 100644 index 0000000000..bb5d70c687 --- /dev/null +++ b/docs/development/qwen_image_prefix_cache.md @@ -0,0 +1,32 @@ +# Qwen Image 2.1 prefix cache + +For cache options and memory estimates, see the +[Qwen Image 2.1 usage guide](../models/qwen_image_2.1.md#prefix-cache). + +## Cache behavior + +By default, the first denoising call for each fixed condition saves the text and reference-image keys and values from every transformer layer. Later calls only compute the target-image tokens. Positive and negative conditions use separate caches, which are released when sampling ends. + +Set `qwen_image_2_1_prefix_cache_type` in `--model-args` to `auto` or a type name using the same parser and case-sensitive names as `--type`: + +- `auto` (default): use FP16 only when Flash Attention is enabled, Sage Attention is disabled, the attention scale is unchanged, and every attention operation in the cache-writing or cache-reading graph selects Flash Attention after backend support checks. If an operation falls back, rebuild the prefix in FP32 before executing and keep FP32 for the rest of that sampling run. +- `f32`: always store FP32 keys and values. +- `f16`: always store FP16 keys and values, including with ordinary attention or custom attention scaling. This saves cache memory but can introduce additional rounding error. +- Other types, such as `bf16`, `q4_1`, `q5_0`, `q5_1`, `q8_0`, `q4_K`, `q6_K`, `iq4_nl`, and `iq4_xs`: use the requested storage type if the ggml build provides runtime conversion to and from FP32. Quantization is lossy and must be selected explicitly; `auto` never selects a quantized type. + +Cache data is packed into contiguous rows of `hidden_size` elements before conversion, so 256-element quantization blocks work with the model's 128-element attention heads without padding. The type's block size must divide `hidden_size`. Unknown types, types lacking runtime conversion (for example `q8_1` and several IQ formats), and incompatible block sizes are ignored with a warning, leaving the previous setting or the default `auto` unchanged. + +For example, use `--model-args qwen_image_2_1_prefix_cache_type=q8_0` to enable 8-bit cache storage. Cached keys and values are converted back to the attention input type before concatenating with the current target tokens. This reduces persistent cache memory; attention working buffers still use floating-point values, and conversion adds work on each step. Backends without the required conversion operations use the existing CPU fallback. + +For the default 32-layer model, a prefix of 4096 tokens takes approximately the following memory per condition, excluding weights, working buffers, and allocation overhead: + +| Cache type | Memory | +| --- | ---: | +| `f32` | 4 GiB | +| `f16` | 2 GiB | +| `q8_0` | 1.0625 GiB | +| `q4_0` | 0.5625 GiB | + +The runner accounts for the cache when checking the memory budget. If a cached execution runs out of memory, it releases the prefix caches, disables caching for the rest of that sampling run, and retries the full sequence once. Per-step conditioning extensions currently use the full-sequence path. + +Disable this optimization with `--model-args qwen_image_2_1_prefix_cache=false`. It reuses step-independent activations; numerical results can still differ slightly because the matrix sizes change. diff --git a/docs/distilled_sd.md b/docs/distilled_sd.md deleted file mode 100644 index 7aa8fbedea..0000000000 --- a/docs/distilled_sd.md +++ /dev/null @@ -1,118 +0,0 @@ -# Running distilled models: SSD1B, Vega and SDx.x with tiny U-Nets - -## Preface - -These models feature a reduced U-Net architecture. Unlike standard SDXL models, the SSD-1B and Vega U-Net contains only one middle block and fewer attention layers in its up- and down-blocks, resulting in significantly smaller file sizes. Using these models can reduce inference time by more than 33%. For more details, refer to Segmind's paper: https://arxiv.org/abs/2401.02677v1. -Similarly, SD1.x- and SD2.x-style models with a tiny U-Net consist of only 6 U-Net blocks, leading to very small files and time savings of up to 50%. For more information, see the paper: https://arxiv.org/pdf/2305.15798.pdf. - -## SSD1B - -Note that not all of these models follow the standard parameter naming conventions. However, several useful SSD-1B models are available online, such as: - - * https://huggingface.co/segmind/SSD-1B/resolve/main/SSD-1B-A1111.safetensors - * https://huggingface.co/hassenhamdi/SSD-1B-fp8_e4m3fn/resolve/main/SSD-1B_fp8_e4m3fn.safetensors - -Useful LoRAs are also available: - - * https://huggingface.co/seungminh/lora-swarovski-SSD-1B/resolve/main/pytorch_lora_weights.safetensors - * https://huggingface.co/kylielee505/mylcmlorassd/resolve/main/pytorch_lora_weights.safetensors - -## Vega - -Segmind's Vega model is available online here: - - * https://huggingface.co/segmind/Segmind-Vega/resolve/main/segmind-vega.safetensors - -VegaRT is an example for an LCM-LoRA: - - * https://huggingface.co/segmind/Segmind-VegaRT/resolve/main/pytorch_lora_weights.safetensors - -Both files can be used out-of-the-box, unlike the models described in next sections. - - -## SD1.x, SD2.x with tiny U-Nets - -These models require conversion before use. You will need a Python script provided by the diffusers team, available on GitHub: - - * https://raw.githubusercontent.com/huggingface/diffusers/refs/heads/main/scripts/convert_diffusers_to_original_stable_diffusion.py - -### SD2.x - -NotaAI provides the following model online: - -* https://huggingface.co/nota-ai/bk-sdm-v2-tiny - -Creating a .safetensors file involves two steps. First, run this short Python script to download the model from Hugging Face: - -```python -from diffusers import StableDiffusionPipeline -pipe = StableDiffusionPipeline.from_pretrained("nota-ai/bk-sdm-v2-tiny",cache_dir="./") -``` - -Second, create the .safetensors file by running: - -```bash -python convert_diffusers_to_original_stable_diffusion.py \ - --model_path models--nota-ai--bk-sdm-v2-tiny/snapshots/68277af553777858cd47e133f92e4db47321bc74 \ - --checkpoint_path bk-sdm-v2-tiny.safetensors --half --use_safetensors -``` - -This will generate the **file bk-sdm-v2-tiny.safetensors**, which is now ready for use with sd.cpp. - -### SD1.x - -Several Tiny SD 1.x models are available online, such as: - - * https://huggingface.co/segmind/tiny-sd - * https://huggingface.co/segmind/portrait-finetuned - * https://huggingface.co/nota-ai/bk-sdm-tiny - -These models also require conversion, partly because some tensors are stored in a non-contiguous manner. To create a usable checkpoint file, follow these simple steps: -Download and prepare the model using Python: - -##### Download the model using Python on your computer, for example this way: - -```python -import torch -from diffusers import StableDiffusionPipeline -pipe = StableDiffusionPipeline.from_pretrained("segmind/tiny-sd") -unet=pipe.unet -for param in unet.parameters(): - param.data = param.data.contiguous() # <- important here -pipe.save_pretrained("segmindtiny-sd", safe_serialization=True) -``` - -##### Run the conversion script: - -```bash -python convert_diffusers_to_original_stable_diffusion.py \ - --model_path ./segmindtiny-sd \ - --checkpoint_path ./segmind_tiny-sd.safetensors --half --use_safetensors -``` - -The file segmind_tiny-sd.safetensors will be generated and is now ready for use with sd.cpp. You can follow a similar process for the other models mentioned above. - - -### SDXS-512-DreamShaper - -Another very tiny and **incredibly fast** model is SDXS by IDKiro et al. The authors refer to it as *"Real-Time One-Step Latent Diffusion Models with Image Conditions"*. For details read the paper: https://arxiv.org/pdf/2403.16627 . Once again the authors removed some more blocks of U-Net part and unlike other SD1 models they use an adjusted _AutoEncoderTiny_ instead of default _AutoEncoderKL_ for the VAE part. -##### Some ready-to-run SDXS-512 model files are available online, such as: - -* https://huggingface.co/akleine/sdxs-512 -* https://huggingface.co/concedo/sdxs-512-tinySDdistilled-GGUF - -##### Run the model as follows: -```bash -~/stable-diffusion.cpp/build/bin/sd-cli -m sdxs.safetensors -p "portrait of a lovely cat" \ - --cfg-scale 1 --steps 1 -``` -Both options: ``` --cfg-scale 1 ``` and ``` --steps 1 ``` are mandatory here. - -### SDXS-512-0.9 - -Even though the name "SDXS-512-0.9" is similar to "SDXS-512-DreamShaper", it is *completely different* but also **incredibly fast**. Sometimes it is preferred, so try it yourself. -##### Download a ready-to-run file from here: - -* https://huggingface.co/akleine/sdxs-09 - -For the use of this model, both options ``` --cfg-scale 1 ``` and ``` --steps 1 ``` are again absolutely necessary. diff --git a/docs/ecosystem.md b/docs/ecosystem.md new file mode 100644 index 0000000000..dc150c2e0d --- /dev/null +++ b/docs/ecosystem.md @@ -0,0 +1,32 @@ +# Bindings, Applications, and UIs + +## Bindings + +These projects wrap `stable-diffusion.cpp` for easier use in other languages/frameworks. + +* Golang (non-cgo): [seasonjs/stable-diffusion](https://github.com/seasonjs/stable-diffusion) +* Golang (cgo): [Binozo/GoStableDiffusion](https://github.com/Binozo/GoStableDiffusion) +* Golang (non-cgo): [l8bloom/gosd](https://github.com/l8bloom/gosd) +* C#: [DarthAffe/StableDiffusion.NET](https://github.com/DarthAffe/StableDiffusion.NET) +* C#: [TensorStack.StableDiffusionCpp](https://github.com/saddam213/TensorStack/tree/master/TensorStack.StableDiffusionCpp) +* Python: [william-murray1204/stable-diffusion-cpp-python](https://github.com/william-murray1204/stable-diffusion-cpp-python) +* Rust: [newfla/diffusion-rs](https://github.com/newfla/diffusion-rs) +* Flutter/Dart: [rmatif/Local-Diffusion](https://github.com/rmatif/Local-Diffusion) + +## Applications and UIs + +These projects include dedicated interfaces for `stable-diffusion.cpp` and +applications that integrate it as an image generation backend. + +- [GIMP Plugins](https://github.com/themanyone/gimp-plugins) +- [Stable Diffusion GUI](https://github.com/fszontagh/sd.cpp.gui.wx) +- [Stable Diffusion CLI-GUI](https://github.com/piallai/stable-diffusion.cpp) +- [Local Diffusion](https://github.com/rmatif/Local-Diffusion) +- [sd.cpp-webui](https://github.com/daniandtheweb/sd.cpp-webui) +- [LocalAI](https://github.com/mudler/LocalAI) +- [Neural-Pixel](https://github.com/Luiz-Alcantara/Neural-Pixel) +- [KoboldCpp](https://github.com/LostRuins/koboldcpp) +- [Kokon](https://github.com/GabrieleLozupone/kokon) +- [NodeTool](https://github.com/nodetool-ai/nodetool) +- [Quo](https://github.com/quo-ai/quo) +- [Prisme](https://github.com/d3Ita/prisme) diff --git a/docs/ernie_image.md b/docs/ernie_image.md deleted file mode 100644 index 3b1fc124cb..0000000000 --- a/docs/ernie_image.md +++ /dev/null @@ -1,35 +0,0 @@ -# How to Use - -You can run ERNIE-Image with stable-diffusion.cpp on GPUs with 4GB of VRAM — or even less. - -## Download weights - -- Download ERNIE-Image-Turbo - - safetensors: https://huggingface.co/Comfy-Org/ERNIE-Image/tree/main/diffusion_models - - gguf: https://huggingface.co/unsloth/ERNIE-Image-Turbo-GGUF/tree/main -- Download ERNIE-Image - - safetensors: https://huggingface.co/Comfy-Org/ERNIE-Image/tree/main/diffusion_models - - gguf: https://huggingface.co/unsloth/ERNIE-Image-GGUF/tree/main -- Download vae - - safetensors: https://huggingface.co/Comfy-Org/ERNIE-Image/tree/main/vae -- Download ministral 3b - - safetensors: https://huggingface.co/Comfy-Org/ERNIE-Image/tree/main/text_encoders - - gguf: https://huggingface.co/unsloth/Ministral-3-3B-Instruct-2512-GGUF/tree/main - -## Examples - -### ERNIE-Image-Turbo - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\ernie-image-turbo.safetensors --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\ministral-3-3b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 8 -v --offload-to-cpu -``` - -ERNIE-Image Turbo example - -### ERNIE-Image - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\ernie-image-UD-Q4_K_M.gguf --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\ministral-3-3b.safetensors -p "a lovely cat" --cfg-scale 5.0 -v --offload-to-cpu -``` - -ERNIE-Image example diff --git a/docs/esrgan.md b/docs/esrgan.md deleted file mode 100644 index 4068ef11c7..0000000000 --- a/docs/esrgan.md +++ /dev/null @@ -1,13 +0,0 @@ -## Using ESRGAN to upscale results - -You can use ESRGAN—such as the model [RealESRGAN_x4plus_anime_6B.pth](https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.2.4/RealESRGAN_x4plus_anime_6B.pth)—to upscale the generated images and improve their overall resolution and clarity. - -MoSR models with the PixelShuffle upsampler, such as [4xNomos2_hq_mosr.pth](https://github.com/Phhofm/models/releases/download/4xNomos2_hq_mosr/4xNomos2_hq_mosr.pth), are also supported. The architecture is detected from the weights, so they are used the same way as ESRGAN models. - -RGBA images, including Qwen Image 2.1 output, keep their alpha channel during model upscaling and hires fix. The upscaler processes the RGB channels; the alpha channel is resized with bilinear interpolation and recombined with the upscaled image. - -- Specify the model path using the `--upscale-model PATH` parameter. example: - -```bash -sd-cli -m ../models/v1-5-pruned-emaonly.safetensors -p "a lovely cat" --upscale-model ../models/RealESRGAN_x4plus_anime_6B.pth -``` diff --git a/docs/fibo2.md b/docs/fibo2.md deleted file mode 100644 index 064d800892..0000000000 --- a/docs/fibo2.md +++ /dev/null @@ -1,53 +0,0 @@ -# How to Use - -Fibo 2.0 uses a Fibo diffusion transformer, the standard FLUX.2 VAE, and Qwen3-VL 4B as the LLM text encoder. The Qwen tokenizer is built in. - -## Download weights - -- Download Fibo 2.0 Dev - - safetensors: https://huggingface.co/Comfy-Org/Fibo-2.0/tree/main/diffusion_models -- Download Fibo 2.0 Edit Turbo - - safetensors: https://huggingface.co/Comfy-Org/Fibo-2.0/tree/main/diffusion_models -- Download vae - - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main -- Download Qwen3-VL 4B - - safetensors: https://huggingface.co/Comfy-Org/Fibo-2.0/tree/main/text_encoders - - gguf: https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct-GGUF/tree/main - -The original Diffusers weights from [briaai/Fibo-2.0-dev](https://huggingface.co/briaai/Fibo-2.0-dev) and [briaai/Fibo-2.0-edit-turbo](https://huggingface.co/briaai/Fibo-2.0-edit-turbo) are also supported. Pass the transformer's `diffusion_pytorch_model.safetensors.index.json` to `--diffusion-model` and keep its shards alongside it. - -INT8 convrot weights load directly; see [INT8 convrot](./int8_convrot.md) for backend requirements. - -## Examples - -### Fibo 2.0 Dev - -Dev supports both text-to-image generation and reference-image editing. - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\fibo2_dev_int8_convrot.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --llm_vision ..\models\text_encoders\Qwen3-VL-4B-Instruct-mmproj-BF16.gguf --vae ..\models\vae\flux2_ae.safetensors -p "a lovely cat holding a sign says 'fibo2.cpp'" --cfg-scale 5.0 --sampling-method euler -v --offload-to-cpu -``` - - - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\fibo2_dev_int8_convrot.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --llm_vision ..\models\text_encoders\Qwen3-VL-4B-Instruct-mmproj-BF16.gguf --vae ..\models\vae\flux2_ae.safetensors -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'fibo2.cpp'" --cfg-scale 5.0 --sampling-method euler -v --offload-to-cpu -``` - - - -### Fibo 2.0 Edit Turbo - -Edit Turbo is distilled for reference-image editing in 4 steps with CFG 1. - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\fibo2_edit_turbo_int8_convrot.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --llm_vision ..\models\text_encoders\Qwen3-VL-4B-Instruct-mmproj-BF16.gguf --vae ..\models\vae\flux2_ae.safetensors -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'fibo2.cpp'" --cfg-scale 1 --steps 4 --sampling-method euler -v --offload-to-cpu -``` - -The BF16 files accept the same options. Use dimensions divisible by 16. Repeat `-r` to use multiple reference images; instructions can refer to ``, ``, etc. See [Image Editing](./edit.md) for reference-image preprocessing options. Separate GGUF text encoders need their vision projector passed with `--llm_vision` for editing. - -The automatic `fibo2` reference preset combines `resize_single_ref_to_target=true`, `vae_size_buckets=true`, `vae_center_crop=true`, and `vlm_use_vae_input=true`. A single reference is scaled to cover the output dimensions and center-cropped. Multiple references use dimensions divisible by 16, chosen within ±10% of one megapixel to minimize cropping for each image's aspect ratio; `vae_input_max_pixels` overrides that target area. The VAE and vision encoder receive the same crops, with smaller copies used by the vision encoder. These are generic reference-image options and can be overridden individually: for example, `--ref-image-args vae_center_crop=false` uses direct resizing, while `resize_before_vae=false` skips this automatic crop and resize step entirely. - -The vision encoder defaults to `vlm_resize_mode=longest_side,vlm_max_size=768`. Override these with `--ref-image-args`, for example `vlm_max_size=512`, or use `vlm_resize_mode=none` to skip size-based scaling. Vision inputs are still aligned to the encoder's patch grid. - -Plain prompts and structured JSON captions are accepted. This integration uses the caption-only text-rendering path (`render_text_images=False`); it does not generate auxiliary glyph images or expand prompts with Bria VGL. Native Fibo mask preprocessing is not connected to `--mask`. diff --git a/docs/flux.md b/docs/flux.md deleted file mode 100644 index 6c61600c57..0000000000 --- a/docs/flux.md +++ /dev/null @@ -1,66 +0,0 @@ -# How to Use - -You can run Flux using stable-diffusion.cpp with a GPU that has 6GB or even 4GB of VRAM, without needing to offload to RAM. - -## Download weights - -- Download flux - - If you don't want to do the conversion yourself, download the preconverted gguf model from [FLUX.1-dev-gguf](https://huggingface.co/leejet/FLUX.1-dev-gguf) or [FLUX.1-schnell](https://huggingface.co/leejet/FLUX.1-schnell-gguf) - - Otherwise, download flux-dev from https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/flux1-dev.safetensors or flux-schnell from https://huggingface.co/black-forest-labs/FLUX.1-schnell/blob/main/flux1-schnell.safetensors -- Download vae from https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/ae.safetensors -- Download clip_l from https://huggingface.co/comfyanonymous/flux_text_encoders/blob/main/clip_l.safetensors -- Download t5xxl from https://huggingface.co/comfyanonymous/flux_text_encoders/blob/main/t5xxl_fp16.safetensors - -## Convert flux weights - -You can download the preconverted gguf weights from [FLUX.1-dev-gguf](https://huggingface.co/leejet/FLUX.1-dev-gguf) or [FLUX.1-schnell](https://huggingface.co/leejet/FLUX.1-schnell-gguf), this way you don't have to do the conversion yourself. - -For example: -``` -.\bin\Release\sd-cli.exe -M convert -m ..\models\unet\flux1-dev.sft -o ..\models\flux1-dev-q8_0.gguf -v --type q8_0 -``` - -## Run - -- `--cfg-scale` is recommended to be set to 1. - -### Flux-dev -For example: - -``` - .\bin\Release\sd-cli.exe --diffusion-model ..\models\flux1-dev-q8_0.gguf --vae ..\models\ae.sft --clip_l ..\models\clip_l.safetensors --t5xxl ..\models\t5xxl_fp16.safetensors -p "a lovely cat holding a sign says 'flux.cpp'" --cfg-scale 1.0 --sampling-method euler -v --clip-on-cpu -``` - -Using formats of different precisions will yield results of varying quality. - -| Type | q8_0 | q4_0 | q4_k | q3_k | q2_k | -|---- | ---- |---- |---- |---- |---- | -| **Memory** | 12068.09 MB | 6394.53 MB | 6395.17 MB | 4888.16 MB | 3735.73 MB | -| **Result** | ![](../assets/flux/flux1-dev-q8_0.png) |![](../assets/flux/flux1-dev-q4_0.png) |![](../assets/flux/flux1-dev-q4_k.png) |![](../assets/flux/flux1-dev-q3_k.png) |![](../assets/flux/flux1-dev-q2_k.png)| - - - -### Flux-schnell - - -``` - .\bin\Release\sd-cli.exe --diffusion-model ..\models\flux1-schnell-q8_0.gguf --vae ..\models\ae.sft --clip_l ..\models\clip_l.safetensors --t5xxl ..\models\t5xxl_fp16.safetensors -p "a lovely cat holding a sign says 'flux.cpp'" --cfg-scale 1.0 --sampling-method euler -v --steps 4 --clip-on-cpu -``` - -| q8_0 | -| ---- | -|![](../assets/flux/flux1-schnell-q8_0.png) | - -## Run with LoRA - -Since many flux LoRA training libraries have used various LoRA naming formats, it is possible that not all flux LoRA naming formats are supported. It is recommended to use LoRA with naming formats compatible with ComfyUI. - -### Flux-dev q8_0 with LoRA - -- LoRA model from https://huggingface.co/XLabs-AI/flux-lora-collection/tree/main (using comfy converted version!!!) - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\flux1-dev-q8_0.gguf --vae ...\models\ae.sft --clip_l ..\models\clip_l.safetensors --t5xxl ..\models\t5xxl_fp16.safetensors -p "a lovely cat holding a sign says 'flux.cpp'" --cfg-scale 1.0 --sampling-method euler -v --lora-model-dir ../models --clip-on-cpu -``` - -![output](../assets/flux/flux1-dev-q8_0%20with%20lora.png) diff --git a/docs/flux2.md b/docs/flux2.md deleted file mode 100644 index 53fc8be5cf..0000000000 --- a/docs/flux2.md +++ /dev/null @@ -1,96 +0,0 @@ -# How to Use - -## Flux.2-dev - -### Download weights - -- Download FLUX.2-dev - - gguf: https://huggingface.co/city96/FLUX.2-dev-gguf/tree/main -- Download vae - - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main -- Download FLUX.2-small-decoder (full_encoder_small_decoder.safetensors) as an alternative VAE option - - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-small-decoder/tree/main -- Download Mistral-Small-3.2-24B-Instruct-2506-GGUF - - gguf: https://huggingface.co/unsloth/Mistral-Small-3.2-24B-Instruct-2506-GGUF/tree/main - -### Examples - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux2-dev-Q4_K_S.gguf --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\Mistral-Small-3.2-24B-Instruct-2506-Q4_K_M.gguf -r .\kontext_input.png -p "change 'flux.cpp' to 'flux2-dev.cpp'" --cfg-scale 1.0 --sampling-method euler -v --offload-to-cpu -``` - -flux2 example - -## Flux.2 klein 4B / Flux.2 klein base 4B - -### Download weights - -- Download FLUX.2-klein-4B - - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-klein-4B - - gguf: https://huggingface.co/leejet/FLUX.2-klein-4B-GGUF/tree/main -- Download FLUX.2-klein-base-4B - - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-klein-base-4B - - gguf: https://huggingface.co/leejet/FLUX.2-klein-base-4B-GGUF/tree/main -- Download vae - - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main -- Download FLUX.2-small-decoder (full_encoder_small_decoder.safetensors) as an alternative VAE option - - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-small-decoder/tree/main -- Download Qwen3 4b - - safetensors: https://huggingface.co/Comfy-Org/flux2-klein-4B/tree/main/split_files/text_encoders - - gguf: https://huggingface.co/unsloth/Qwen3-4B-GGUF/tree/main - -### Examples - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux-2-klein-4b.safetensors --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\qwen_3_4b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 4 -v --offload-to-cpu -``` - -flux2-klein-4b - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux-2-klein-4b.safetensors --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\qwen_3_4b.safetensors -r .\kontext_input.png -p "change 'flux.cpp' to 'klein.cpp'" --cfg-scale 1.0 --sampling-method euler -v --offload-to-cpu --steps 4 -``` - -flux2-klein-4b-edit - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux-2-klein-base-4b.safetensors --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\qwen_3_4b.safetensors -p "a lovely cat" --cfg-scale 4.0 --steps 20 -v --offload-to-cpu -``` - -flux2-klein-base-4b - -## Flux.2 klein 9B / Flux.2 klein base 9B - -### Download weights - -- Download FLUX.2-klein-9B - - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-klein-9B - - gguf: https://huggingface.co/leejet/FLUX.2-klein-9B-GGUF/tree/main -- Download FLUX.2-klein-base-9B - - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-klein-base-9B - - gguf: https://huggingface.co/leejet/FLUX.2-klein-base-9B-GGUF/tree/main -- Download vae - - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main -- Download Qwen3 8B - - safetensors: https://huggingface.co/Comfy-Org/flux2-klein-9B/tree/main/split_files/text_encoders - - gguf: https://huggingface.co/unsloth/Qwen3-8B-GGUF/tree/main - -### Examples - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux-2-klein-9b.safetensors --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\qwen_3_8b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 4 -v --offload-to-cpu -``` - -flux2-klein-9b - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux-2-klein-9b.safetensors --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\qwen_3_8b.safetensors -r .\kontext_input.png -p "change 'flux.cpp' to 'klein.cpp'" --cfg-scale 1.0 --sampling-method euler -v --offload-to-cpu --steps 4 -``` - -flux2-klein-9b-edit - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux-2-klein-base-9b.safetensors --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\qwen_3_8b.safetensors -p "a lovely cat" --cfg-scale 4.0 --steps 20 -v --offload-to-cpu -``` - -flux2-klein-base-9b \ No newline at end of file diff --git a/docs/flux3.md b/docs/flux3.md deleted file mode 100644 index 8bc0b2ec29..0000000000 --- a/docs/flux3.md +++ /dev/null @@ -1,50 +0,0 @@ -# How to Use - -FLUX.3 Image Klein uses a FLUX.3 diffusion transformer, the FLUX.2 VAE, and Qwen3-VL 4B as the LLM text encoder. - -## Download weights - -- Download FLUX.3 Image Klein - - safetensors: https://huggingface.co/black-forest-labs/flux-3-image-klein/tree/main - - gguf: https://huggingface.co/leejet/flux-3-image-klein-GGUF/tree/main -- Download FLUX.3 Image Klein Base - - safetensors: https://huggingface.co/black-forest-labs/flux-3-image-klein-base/tree/main - - gguf: https://huggingface.co/leejet/flux-3-image-klein-base-GGUF/tree/main -- Download vae - - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main - - The `ae.safetensors` included with FLUX.3 can also be used. -- Download Qwen3-VL 4B - - safetensors: https://huggingface.co/Comfy-Org/Krea-2/tree/main/text_encoders - - gguf: https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct-GGUF/tree/main - -Use the root `model.safetensors` for the diffusion model. - -## Examples - -### FLUX.3 Image Klein - -The distilled model uses 16 steps and CFG 1.0. - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux-3-image-klein.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --vae ..\models\vae\flux2_ae.safetensors -p "a lovely cat holding a sign says 'flux3.cpp'" --cfg-scale 1.0 --steps 16 --sampling-method euler -v --offload-to-cpu -``` - - - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux-3-image-klein.safetensors --vae ..\models\vae\flux2_ae.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'flux3-image-klein.cpp'" --cfg-scale 1.0 --sampling-method euler -v --offload-to-cpu --steps 16 -``` - - - -Repeat `-r` to use multiple reference images, up to 10. See [Image Editing](./edit.md) for reference-image preprocessing options. - -### FLUX.3 Image Klein Base - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux-3-image-klein-base.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --vae ..\models\vae\flux2_ae.safetensors -p "a lovely cat holding a sign says 'flux3.cpp'" --cfg-scale 3.5 --steps 20 --sampling-method euler -v --offload-to-cpu -``` - - - -The default discrete scheduler uses the upstream time shift of 6.93 for both models. Use `--flow-shift` to override it. diff --git a/docs/backend.md b/docs/guides/backend.md similarity index 98% rename from docs/backend.md rename to docs/guides/backend.md index cc6545da92..f7edddf592 100644 --- a/docs/backend.md +++ b/docs/guides/backend.md @@ -273,8 +273,8 @@ flags are deprecated and emit a warning, but remain supported when `--conv-direct` is omitted. Use `--conv-direct auto,diffusion=on,controlnet=on` and `--conv-direct auto,vae=on`, respectively. An explicit `--conv-direct` takes precedence over both legacy flags regardless of order. -Library callers can set the appended `sd_ctx_params_t::conv_direct_mode` field -to the same string; null or empty selects `auto` while honoring legacy true +Library callers can set the `conv_direct_mode` context-option key with +`sd_opt_set_string()` to the same string; unset or empty selects `auto` while honoring legacy true flags. The context copies the string. ## Flash attention (`--flash-attn on|off|auto`) @@ -339,9 +339,10 @@ row-split configurations, which can fall back to layer split during initializati Use `--flash-attn on` and `--flash-attn off,diffusion=on`, respectively. Both aliases write the same mode string as `--flash-attn`; the last occurrence of any of these options wins. Library callers -can set `sd_ctx_params_t::flash_attn_mode` to the same string, including +can set the `flash_attn_mode` context-option key with `sd_opt_set_string()` +to the same string, including `"on"`, `"off"`, `"auto"`, or `"auto,te=off,vae=on"`. -A null or empty string uses automatic selection unless a +An unset or empty string uses automatic selection unless a legacy `flash_attn` or `diffusion_flash_attn` boolean is set. The context copies the string; no new enum is required. diff --git a/docs/guides/c_api.md b/docs/guides/c_api.md new file mode 100644 index 0000000000..055056d57e --- /dev/null +++ b/docs/guides/c_api.md @@ -0,0 +1,507 @@ +# C API + +Use purpose-specific opaque options for model loading, generation and +postprocessing, and +opaque `sd_result_t` objects to read generated outputs. The library allocates and +frees these objects; applications must not allocate them by size or access fields. + +## Loading and generation + +This C example loads a checkpoint and generates an image. Check every setter: +an unsuccessful setter preserves the previous value, including its unset state. + +```c +#include +#include "stable-diffusion.h" + +int generate(const char* model_path) { + int rc = 1; + sd_ctx_t* ctx = NULL; + sd_result_t* result = NULL; + sd_ctx_opt_t* ctx_opt = sd_ctx_opt_create(); + sd_img_gen_opt_t* image_opt = NULL; + if (!ctx_opt) goto error; + if (sd_opt_set_string(ctx_opt, "model_path", model_path) != SD_STATUS_OK || + sd_opt_set_int(ctx_opt, "n_threads", 8) != SD_STATUS_OK || + sd_ctx_create(ctx_opt, &ctx) != SD_STATUS_OK) + goto error; + sd_opt_free(ctx_opt); + ctx_opt = NULL; + + image_opt = sd_img_gen_opt_create(); + if (!image_opt) goto error; + if (sd_opt_set_string(image_opt, "prompt", "a small house") != SD_STATUS_OK || + sd_opt_set_int(image_opt, "width", 512) != SD_STATUS_OK || + sd_opt_set_int(image_opt, "height", 512) != SD_STATUS_OK || + sd_opt_set_int(image_opt, "seed", 42) != SD_STATUS_OK || + sd_opt_set_int(image_opt, "sample.steps", 20) != SD_STATUS_OK || + sd_opt_set_string(image_opt, "sample.method", "euler") != SD_STATUS_OK || + sd_generate_image(ctx, image_opt, &result) != SD_STATUS_OK) + goto error; + + for (size_t i = 0; i < sd_result_get_image_count(result); ++i) { + const sd_tensor_t* image = sd_result_get_image(result, i); + const char* info = sd_result_get_generation_info(result, i); + /* Read pixels with sd_tensor_data(image), and consume the UTF-8 JSON info. */ + (void) image; + (void) info; + } + rc = 0; + goto cleanup; +error: + fprintf(stderr, "%s\n", sd_get_last_error()); +cleanup: + sd_result_free(result); + sd_ctx_free(ctx); + sd_opt_free(ctx_opt); + sd_opt_free(image_opt); + return rc; +} +``` + +Use `sd_generate_video()` with options from `sd_vid_gen_opt_create()` to generate video. Its result +exposes frames through the same image accessors, optional audio through +`sd_result_get_audio()`, and effective frame rate through `sd_result_get_fps()`. +`sd_result_get_kind()` distinguishes image batches from videos. Video generation +metadata is sequence-wide and is returned for every valid frame index. + +All option names are aliases of the same opaque `sd_opt_t` type and share +all `sd_opt_set_*()`, `sd_opt_unset()` and `sd_opt_free()` functions. The constructor +permanently binds the object's purpose. Setters reject keys for another purpose +immediately, and operation entry points reject an option object of the wrong +purpose even if empty. For example, `prompt` cannot be set on context options, +and image options cannot be passed to `sd_ctx_create()`. These checks occur at +runtime; the C typedef aliases provide descriptive names rather than distinct +compile-time types. + +## Operations + +Each operation has its own constructor and uses the shared setters and +`sd_opt_free()`. Creation and execution return `sd_status_t`; output pointers +are required and set to `NULL` on failure. All outputs use `sd_result_t`. + +| Option constructor | Operation | +| --- | --- | +| `sd_ctx_opt_create()` | `sd_ctx_create()` | +| `sd_img_gen_opt_create()` | `sd_generate_image()` | +| `sd_vid_gen_opt_create()` | `sd_generate_video()` | +| `sd_upscaler_opt_create()` | `sd_upscaler_create()` | +| `sd_upscale_opt_create()` | `sd_upscale()` | +| `sd_vid_upscaler_opt_create()` | `sd_vid_upscaler_create()` | +| `sd_vid_upscale_opt_create()` | `sd_upscale_video()` | +| `sd_adetailer_opt_create()` | `sd_adetailer_create()` | +| `sd_adetail_opt_create()` | `sd_adetail()` | + +Image upscaling loads `model_path` with optional `n_threads`, `backend`, +`params_backend`, `conv_direct` and `tile_size` (default 128). Execution takes +an RGB `image`. Its scale is fixed by the loaded model and can be queried with +`sd_upscaler_get_scale()`; metadata records that actual scale. The former +image-upscale factor argument was unused and has been removed. + +```c +sd_upscaler_opt_t* load = sd_upscaler_opt_create(); +sd_upscaler_t* upscaler = NULL; +sd_upscale_opt_t* opt = NULL; +sd_result_t* result = NULL; +sd_status_t status = load ? sd_opt_set_string(load, "model_path", model_path) + : SD_STATUS_OUT_OF_MEMORY; +if (status == SD_STATUS_OK) status = sd_upscaler_create(load, &upscaler); +if (status == SD_STATUS_OK) { + opt = sd_upscale_opt_create(); + status = sd_opt_set_tensor(opt, "image", image); + if (status == SD_STATUS_OK) status = sd_upscale(upscaler, opt, &result); +} +/* Consume result on success. */ +sd_result_free(result); +sd_opt_free(opt); +sd_upscaler_free(upscaler); +sd_opt_free(load); +``` + +Video-upscaler loading requires `diffusion_model_path`, `vae_path` and +`upscale_factor`; backend/thread/attention settings are optional. The scale must +match both checkpoints (RelayVSR uses 2 or 4). Execution takes RGB `frames` with +equal dimensions. Defaults are FPS 24, seed 42, RelayVSR `window_size` 16, and +SeedVR2 `temporal_chunk_size` 5. Free the model with `sd_vid_upscaler_free()`. + +ADetailer loading takes the detector `model_path` and optional backend/thread +settings. Call `sd_adetail(adetailer, diffusion_ctx, opt, &result)` with an +`sd_adetail_opt_t`. Set `image` for the source image, and reuse the applicable +image-generation keys for inpainting: `prompt`, `negative_prompt`, dimensions, +strength, seed, sampling, LoRAs, cache, VAE tiling and circular axes. Inpainting +defaults to 512×512 and strength 0.4. `adetail.prompt` and +`adetail.negative_prompt` supply region prompt overrides; `adetail.args` accepts +the existing detector/mask/inpaint argument string documented by the CLI +`--extra-ad-args` option. Reference/control images, hires and batching are not +ADetailer options. Free the detector with `sd_adetailer_free()`. + +## String-valued modes + +Sampler, scheduler, prediction, RNG, cache, control, hires and VAE option enums +are internal. Set their documented string values through `sd_opt_set_string()`. +Leave `sample.method` and `sample.scheduler` unset, or set them to `"auto"`, +to select defaults for the loaded model and sampler. + +The bundled CLI and server retain these modes as strings and pass them directly +to setters. Enum parsers, enum name helpers, parameter configuration layouts and +name tables are internal and are not exported by the shared library. + +Preview registration also uses strings: + +```c +sd_status_t status = sd_set_preview_callback(callback, "proj", 5, + true, false, user_data); +/* Disable registration when the context is idle. */ +sd_set_preview_callback(NULL, "none", 1, false, false, NULL); +``` + +Preview modes are `none`, `proj`, `tae` and `vae`. A null or unknown mode returns +`SD_STATUS_INVALID_ARGUMENT` and preserves the previous registration. See the +public header for interval, noisy/denoised and callback lifetime semantics. + +## Tensor data + +All media payloads use opaque `sd_tensor_t` handles, backed by the library's +`sd::Tensor`. The declarations are in `stable-diffusion.h`; no additional +public header is required. Dimensions are ordered from most contiguous to least +contiguous, matching `sd::Tensor`: + +| Media | Element type | Shape | +| --- | --- | --- | +| Image | `SD_TENSOR_U8` | `[channels, width, height]` | +| Audio samples | `SD_TENSOR_F32` | `[channels, samples]` | +| Video frames | `SD_TENSOR_U8` | `[channels, width, height, frames]` | + +Channels are interleaved. Audio duration is `samples / sample_rate`; sample count +and channel count come from the tensor dimensions. `sd_audio_t` adds `sample_rate` +to a sample tensor. `sd_video_t` adds FPS and an optional audio descriptor to a +video tensor. Image dimensions and channel count are entirely in the tensor. + +`sd_tensor_create(type, rank, shape, data, data_size)` copies the supplied bytes. +The byte count must exactly match the element type and dimensions. General +tensors support U8, I32, I64, F32 and F64, rank up to 8, rank-zero scalars and +zero-sized dimensions. Media keys validate their required type, rank and positive +dimensions. + +```c +int64_t shape[] = {3, width, height}; +sd_tensor_t* image = sd_tensor_create(SD_TENSOR_U8, 3, shape, + pixels, (size_t)width * height * 3); +if (image) { + sd_status_t status = sd_opt_set_tensor(opt, "init_image", image); + sd_tensor_free(image); /* opt retains the tensor after a successful setter. */ + (void)status; +} +``` + +Use `sd_opt_set_tensors()` for image lists such as `ref_images`; individual images +may have different dimensions. Use one 4D tensor for `control_frames` and video +upscaler `frames`. Use `sd_opt_set_audios()` and `sd_opt_set_videos()` for lists +whose items also carry sample-rate/FPS metadata. These setters retain all nested +tensors. Freeing the caller's tensor handles or descriptor arrays does not +invalidate configured options. + +`sd_result_get_image()` returns a U8 image/frame tensor. `sd_result_get_audio()` +returns sample-rate metadata and a F32 sample tensor. `sd_result_get_video()` +returns FPS, optional audio and a contiguous 4D tensor, assembled and cached on +first access. Reading individual video frames avoids that assembly and copy. + +Tensor data is read-only. `sd_tensor_retain()` extends the lifetime of any tensor, +including result and preview tensors, independently of its original owner; +release each owned reference with `sd_tensor_free()`. Retain/free accept NULL and +preserve the last diagnostic. Preview callbacks receive an array of borrowed +image tensors. Prompt-image callbacks return a borrowed image tensor that stays +valid until the next callback. `sd_preprocess_canny()` returns a new image tensor +and leaves its input unchanged. + +## Values, validation and ownership + +- Unset keys retain existing model-loading and generation defaults. + `sd_opt_unset()` restores this state and succeeds for an applicable known absent key. + Zero, false, empty strings and empty arrays remain explicit values, subject to + that key's validation. `NULL` strings are invalid; use `unset` for defaults. +- Keys and enum values are case-sensitive. Unknown keys return + `SD_STATUS_UNKNOWN_KEY`; incorrect setter types return `SD_STATUS_TYPE_MISMATCH`. + Numeric overflow/range errors and invalid enum names are rejected by setters. + No automatic conversion occurs between integer, floating-point and boolean values. +- Setters and unset reject keys for another purpose with `SD_STATUS_UNSUPPORTED`, + even when set to a default-looking value. Loading/generation checks the option + object purpose and parameter combinations. Model-dependent features are checked at generation: + for example UCache requires UNet and EasyCache/CacheDIT require DiT. Known + high-noise sampling settings and `moe_boundary` are ignored without a high-noise + stage; `vace_strength` is ignored by non-VACE models. Actual control-frame inputs + still require VACE. Existing size/frame alignment and model-dependent automatic + defaults are retained. +- Scalar strings, numeric arrays, LoRA paths and embedding names/paths are copied. + Tensor arrays and audio/video metadata are copied, and tensor references are + retained. Empty arrays accept a `NULL` pointer and count 0. +- Successful context creation owns the configuration it retains. Free or reuse + the loading params immediately afterwards. Generation params may be freed or + changed after the generation call returns. Do not mutate/free an option object + while another operation is using it. +- Results own their media and metadata independently of the context and params. + Read-only views remain valid until `sd_result_free()`. Do not free or modify + buffers through a view. Image results have no audio and return FPS 0. Invalid + indices return `NULL` with a diagnostic. All new free functions accept `NULL`. +- Output pointers are required and set to `NULL` on failure. Partial batch + cancellation can return successful completed outputs; full cancellation returns + `SD_STATUS_CANCELLED` without a result. Other failures distinguish invalid + arguments, unsupported features, memory exhaustion, loading and generation. +- `sd_get_last_error()` returns a thread-local diagnostic. Status/accessor calls + clear it on success; free functions preserve it. Read it before another API call. + +Option and tensor objects have no public field-layout contract. Future settings +normally add a key while retaining existing key names, types and meanings. To +keep an output after releasing the result, retain its tensor first; release the +retained handle with `sd_tensor_free()` when finished. Result metadata and media +metadata descriptors remain borrowed from the result and should be copied when +needed independently. + +## Keys + +Types below correspond to `sd_opt_set_()`. `context`, `image` and `video` +refer to model loading and image/video generation. Other operation labels match +the corresponding option constructor, such as `vid_upscale` for +`sd_vid_upscale_opt_create()`. +Array lengths are supplied to setters; there are no separately writable count keys. + +Top-level keys retain their existing parameter meanings. Nested names map as follows: + +| Key prefix | Configuration | +| --- | --- | +| `sample.*` | Base sampling settings; `method`, `steps`, `sigmas`, `args` select the sampler, step count, custom schedule and extra arguments. | +| `high_noise_sample.*` | High-noise sampling settings for two-stage video models. | +| `sample.guidance.*` | Text/image CFG, distilled guidance and SLG settings. | +| `cache.*` | [Inference cache](caching.md) settings. | +| `hires.*` | High-resolution second pass and upscaling settings. | +| `vae_tiling.*` | Spatial/temporal VAE tiling settings. | +| `photomaker.*` | PhotoMaker identity images/embedding and style strength. | +| `pulid.*` | PuLID identity embedding path and weight. | +| `image_preprocess.rules` | [Image preprocessing](image_preprocessing.md) rule string. | + +The library supplies inference defaults. CLI/server inputs that are omitted do +not override them. Model recipes may supply presets; explicit startup options +and request fields take priority over recipe values. A supplied value equal to +a default is still explicit and undergoes model applicability checks. + +Generation defaults include 512×512, a random seed (`-1`), 20 sampling steps, +model-selected sampler/scheduler, text CFG 7, distilled guidance 3.5, strength +0.75, and disabled caching/hires/tiling. Image batches default to 1; videos default +to 6 requested frames at 16 FPS, subject to model alignment. Omitted eta, flow shift +and image CFG retain automatic defaults; do not pass infinity to request defaults. +Use `unset`. High-noise steps default to `-1` (automatic allocation). The high-noise pass +shares the base scheduler, sigmas, flow shift and timestep shifting; set those +through `sample.*`. Separate high-noise keys for these settings are not supported. + +Sampler/scheduler values follow the [sampling guide](sampling.md); `"auto"` +requests automatic selection. RNG values are `std_default`, `cuda`, `cpu`, `auto`. +Prediction modes are `auto`, `eps`, `v`, `edm_v`, `sd3_flow`, `flux_flow`, +`sefi_flow`, `minit2i_flow`, `sensenova_u1_flow`, `flux3_flow`, `h3_av_flow`, +`ming_image_flow`. Weight types use the names returned by `sd_type_name()`, +plus `auto`. VAE formats are `auto`, `flux`, `sd3`, +`flux2`, `wan`. LoRA apply modes are `auto`, `immediately`, `at_runtime`. +Cache modes are `disabled`, `easycache`, `ucache`, `dbcache`, `taylorseer`, +`cache_dit`, `spectrum`. Control types are `none`, `openpose`, `depth`, `scribble`, `canny`, `normal`, +`segment`, `tile`, `repaint`, with the existing aliases such as `hed` and `lineart`. Hires upscalers are `none`, `latent`, +`latent_nearest`, `latent_nearest_exact`, `latent_antialiased`, `latent_bicubic`, +`latent_bicubic_antialiased`, `lanczos`, `nearest`, `model`. + +Integers must fit the underlying field (32 bits except `seed`, which is signed +64-bit). Dimensions, batch size, video frames, FPS and base steps are positive. +Floating-point settings must be finite and fit float32; strength, denoising +strength, percent/range settings and MOE boundary lie in [0, 1]. Custom sigma +arrays are empty or strictly decreasing nonnegative finite values ending in zero, +with at least two entries. SLG layer indices are nonnegative integers. + +| Key | Setter type | Operation | +| --- | --- | --- | +| `adetail.args` | `string` | adetail | +| `adetail.negative_prompt` | `string` | adetail | +| `adetail.prompt` | `string` | adetail | +| `attn_scale` | `float` | context | +| `audio_encoder_path` | `string` | context | +| `audio_vae_path` | `string` | context | +| `auto_fit` | `bool` | context | +| `backend` | `string` | context, upscaler, vid_upscaler, adetailer | +| `batch_count` | `int` | image | +| `batched_cfg` | `bool` | context | +| `cache.bn_compute_blocks` | `int` | image, video, adetail | +| `cache.end_percent` | `float` | image, video, adetail | +| `cache.error_decay_rate` | `float` | image, video, adetail | +| `cache.fn_compute_blocks` | `int` | image, video, adetail | +| `cache.max_cached_steps` | `int` | image, video, adetail | +| `cache.max_continuous_cached_steps` | `int` | image, video, adetail | +| `cache.max_warmup_steps` | `int` | image, video, adetail | +| `cache.mode` | `string` | image, video, adetail | +| `cache.reset_error_on_compute` | `bool` | image, video, adetail | +| `cache.residual_diff_threshold` | `float` | image, video, adetail | +| `cache.reuse_threshold` | `float` | image, video, adetail | +| `cache.scm_mask` | `string` | image, video, adetail | +| `cache.scm_policy_dynamic` | `bool` | image, video, adetail | +| `cache.spectrum_flex_window` | `float` | image, video, adetail | +| `cache.spectrum_lam` | `float` | image, video, adetail | +| `cache.spectrum_m` | `int` | image, video, adetail | +| `cache.spectrum_stop_percent` | `float` | image, video, adetail | +| `cache.spectrum_w` | `float` | image, video, adetail | +| `cache.spectrum_warmup_steps` | `int` | image, video, adetail | +| `cache.spectrum_window_size` | `int` | image, video, adetail | +| `cache.start_percent` | `float` | image, video, adetail | +| `cache.taylorseer_n_derivatives` | `int` | image, video, adetail | +| `cache.taylorseer_skip_interval` | `int` | image, video, adetail | +| `cache.use_relative_threshold` | `bool` | image, video, adetail | +| `circular_x` | `bool` | image, video, adetail | +| `circular_y` | `bool` | image, video, adetail | +| `clip_g_path` | `string` | context | +| `clip_l_path` | `string` | context | +| `clip_skip` | `int` | image, video, adetail | +| `clip_vision_path` | `string` | context | +| `conditioning_cache_size` | `int` | context | +| `control_frames` | `tensor` | video | +| `control_image` | `tensor` | image | +| `control_net_path` | `string` | context | +| `control_strength` | `float` | image | +| `control_type` | `string` | image | +| `conv_direct` | `bool` | upscaler | +| `conv_direct_mode` | `string` | context, vid_upscaler | +| `diffusion_conv_direct` | `bool` | context | +| `diffusion_flash_attn` | `bool` | context | +| `diffusion_model_path` | `string` | context, vid_upscaler | +| `disable_prefetch` | `bool` | context | +| `disable_segmented_compute` | `bool` | context | +| `eager_load` | `bool` | context | +| `embeddings` | `embeddings` | context | +| `embeddings_connectors_path` | `string` | context | +| `enable_mmap` | `bool` | context | +| `end_image` | `tensor` | video | +| `flash_attn` | `bool` | context | +| `flash_attn_mode` | `string` | context, vid_upscaler | +| `force_sdxl_vae_conv_scale` | `bool` | context | +| `fps` | `int` | video, vid_upscale | +| `frames` | `tensor` | vid_upscale | +| `height` | `int` | image, video, adetail | +| `high_noise_diffusion_model_path` | `string` | context | +| `high_noise_sample.args` | `string` | video | +| `high_noise_sample.eta` | `float` | video | +| `high_noise_sample.guidance.distilled_guidance` | `float` | video | +| `high_noise_sample.guidance.img_cfg` | `float` | video | +| `high_noise_sample.guidance.slg.layer_end` | `float` | video | +| `high_noise_sample.guidance.slg.layer_start` | `float` | video | +| `high_noise_sample.guidance.slg.layers` | `ints` | video | +| `high_noise_sample.guidance.slg.scale` | `float` | video | +| `high_noise_sample.guidance.txt_cfg` | `float` | video | +| `high_noise_sample.method` | `string` | video | +| `high_noise_sample.steps` | `int` | video | +| `hires.denoising_strength` | `float` | image, video | +| `hires.enabled` | `bool` | image, video | +| `hires.model_path` | `string` | image, video | +| `hires.scale` | `float` | image, video | +| `hires.sigmas` | `floats` | image, video | +| `hires.steps` | `int` | image, video | +| `hires.target_height` | `int` | image, video | +| `hires.target_width` | `int` | image, video | +| `hires.upscale_tile_size` | `int` | image, video | +| `hires.upscaler` | `string` | image, video | +| `image` | `tensor` | upscale, adetail | +| `image_preprocess.rules` | `string` | image, video | +| `init_image` | `tensor` | image, video | +| `ip_adapter_image` | `tensor` | image | +| `ip_adapter_path` | `string` | context | +| `ip_adapter_strength` | `float` | image | +| `linear_scale` | `float` | context | +| `llm_path` | `string` | context | +| `llm_vision_path` | `string` | context | +| `lora_apply_mode` | `string` | context | +| `loras` | `loras` | image, video, adetail | +| `mask_image` | `tensor` | image | +| `max_vram` | `string` | context | +| `model_args` | `string` | context | +| `model_path` | `string` | context, upscaler, adetailer | +| `moe_boundary` | `float` | video | +| `motion_module_path` | `string` | context | +| `n_threads` | `int` | context, upscaler, vid_upscaler, adetailer | +| `negative_prompt` | `string` | image, video, adetail | +| `params_backend` | `string` | context, upscaler, vid_upscaler, adetailer | +| `photo_maker_path` | `string` | context | +| `photomaker.id_embed_path` | `string` | image | +| `photomaker.id_images` | `tensors` | image | +| `photomaker.style_strength` | `float` | image | +| `prediction` | `string` | context | +| `prompt` | `string` | image, video, adetail | +| `pulid.id_embedding_path` | `string` | image | +| `pulid.id_weight` | `float` | image | +| `pulid_weights_path` | `string` | context | +| `qwen_image_layers` | `int` | image | +| `ref_audios` | `audios` | video | +| `ref_image_args` | `string` | image, video | +| `ref_images` | `tensors` | image, video | +| `ref_videos` | `videos` | video | +| `rng_type` | `string` | context | +| `rpc_servers` | `string` | context | +| `sage_attn` | `bool` | context | +| `sample.args` | `string` | image, video, adetail | +| `sample.eta` | `float` | image, video, adetail | +| `sample.flow_shift` | `float` | image, video, adetail | +| `sample.guidance.distilled_guidance` | `float` | image, video, adetail | +| `sample.guidance.img_cfg` | `float` | image, video, adetail | +| `sample.guidance.slg.layer_end` | `float` | image, video, adetail | +| `sample.guidance.slg.layer_start` | `float` | image, video, adetail | +| `sample.guidance.slg.layers` | `ints` | image, video, adetail | +| `sample.guidance.slg.scale` | `float` | image, video, adetail | +| `sample.guidance.txt_cfg` | `float` | image, video, adetail | +| `sample.method` | `string` | image, video, adetail | +| `sample.scheduler` | `string` | image, video, adetail | +| `sample.shifted_timestep` | `int` | image, video, adetail | +| `sample.sigmas` | `floats` | image, video, adetail | +| `sample.steps` | `int` | image, video, adetail | +| `sampler_rng_type` | `string` | context | +| `seed` | `int` | image, video, vid_upscale, adetail | +| `split_mode` | `string` | context | +| `strength` | `float` | image, video, adetail | +| `t5xxl_path` | `string` | context | +| `tae_preview_only` | `bool` | context | +| `taesd_path` | `string` | context | +| `temporal_chunk_size` | `int` | vid_upscale | +| `tensor_type_rules` | `string` | context | +| `tile_size` | `int` | upscaler | +| `tokenizer` | `string` | context | +| `uncond_diffusion_model_path` | `string` | context | +| `upscale_factor` | `int` | vid_upscaler | +| `vace_strength` | `float` | video | +| `vae_conv_direct` | `bool` | context | +| `vae_format` | `string` | context | +| `vae_path` | `string` | context, vid_upscaler | +| `vae_tiling.enabled` | `bool` | image, video, adetail | +| `vae_tiling.extra_tiling_args` | `string` | image, video, adetail | +| `vae_tiling.rel_size_h` | `float` | image, video, adetail | +| `vae_tiling.rel_size_w` | `float` | image, video, adetail | +| `vae_tiling.target_overlap` | `float` | image, video, adetail | +| `vae_tiling.temporal_tiling` | `bool` | image, video, adetail | +| `vae_tiling.tile_size_h` | `int` | image, video, adetail | +| `vae_tiling.tile_size_w` | `int` | image, video, adetail | +| `video_frames` | `int` | video | +| `width` | `int` | image, video, adetail | +| `window_size` | `int` | vid_upscale | +| `wtype` | `string` | context | + +## Migration + +This is a breaking C API change. The former public generation parameter layouts, +their initialization/formatting helpers, `new_sd_ctx()`, `free_sd_ctx()`, +`generate_image()`, `generate_video()` and public result containers/destructors +have been removed. Applications and bindings must migrate and rebuild; no +compatibility layer is provided. + +Create context/image/video options with the corresponding constructor, populate +them through the shared typed setters, and use `sd_ctx_create()`, +`sd_generate_image()` and `sd_generate_video()`. Read or retain outputs through +`sd_result_get_*()` and `sd_tensor_retain()` and release the result with +`sd_result_free()`. ADetailer and image/video upscalers use purpose-specific opt constructors and +return the same opaque result type. Their former public parameter structures, +long positional signatures and bool-returning entry points have been removed. +Device selection and cancellation controls retain their interfaces. Preview +registration now uses a mode string and returns a status. + +Native generation JSON retains its existing schema and names, including +`sample_params.sample_method`; translate these to the corresponding option keys +(`sample.method`, etc.) when replaying metadata. See +[generation results and metadata](generation_metadata.md). diff --git a/docs/caching.md b/docs/guides/caching.md similarity index 93% rename from docs/caching.md rename to docs/guides/caching.md index f69cfbeca8..66dfc27a00 100644 --- a/docs/caching.md +++ b/docs/guides/caching.md @@ -5,11 +5,11 @@ Caching methods accelerate diffusion inference by reusing intermediate computati ### Conditioning Cache Conditioning results are cached per model context using an LRU cache. The default -capacity is **0 (disabled) for `sd-cli`** and **4 entries for `sd-server` and the C -API**. Set `--conditioning-cache-size N` to change the limit; `0` disables caching. -For example, `sd-cli -m model.safetensors -p "a cat" --conditioning-cache-size 4` -enables the cache in the CLI. The C API option is -`sd_ctx_params_t::conditioning_cache_size`, initialized by `sd_ctx_params_init()`. +capacity is **4 entries for `sd-cli`, `sd-server` and the C API**. +Set `--conditioning-cache-size N` to change the limit; `0` disables caching. +For example, `sd-cli -m model.safetensors -p "a cat" --conditioning-cache-size 8` +increases the CLI cache capacity to 8 entries. The C API option is +`conditioning_cache_size`, set with `sd_opt_set_int()` on context options. This cache is independent of the diffusion-step `--cache-mode` options below. ### Cache Modes diff --git a/docs/edit.md b/docs/guides/edit.md similarity index 90% rename from docs/edit.md rename to docs/guides/edit.md index 1312854b71..942c395f3d 100644 --- a/docs/edit.md +++ b/docs/guides/edit.md @@ -9,17 +9,17 @@ Depending on the architecture, different models handle reference images differen | Model | Default Preset | | :--- | :--- | -| [**FLUX.1-Kontext-dev**](./kontext.md) | `flux_kontext` | -| [**LongCat Image Edit**](./longcat_image.md) | `longcat` | -| [**Qwen Image Edit**](./qwen_image_edit.md) | `qwen` | +| [**FLUX.1-Kontext-dev**](../models/kontext.md) | `flux_kontext` | +| [**LongCat Image Edit**](../models/longcat_image.md) | `longcat` | +| [**Qwen Image Edit**](../models/qwen_image_edit.md) | `qwen` | | **Qwen Image LAYERED** | `qwen_layered` | -| [**Flux.2 [Dev] / Flux.2 [Klein]**](./flux2.md) | `flux2` | -| [**FLUX.3 Image Klein / Klein Base**](./flux3.md) | `flux2` | -| [**Boogu Image Edit**](./boogu_image.md) | `z_image_omni` | +| [**Flux.2 [Dev] / Flux.2 [Klein]**](../models/flux2.md) | `flux2` | +| [**FLUX.3 Image Klein / Klein Base**](../models/flux3.md) | `flux2` | +| [**Boogu Image Edit**](../models/boogu_image.md) | `z_image_omni` | | **Krea2 (Community Edit LoRAs)** | `krea2_ostris_edit` | -| [**Mage-Flow-Edit**](./mage_flow.md#image-editing) | `mage_flow` | -| [**LLaDA-Image**](./llada_image.md#image-editing) | `llada_image` | -| [**Fibo 2.0 Dev / Edit Turbo**](./fibo2.md#examples) | `fibo2` | +| [**Mage-Flow-Edit**](../models/mage_flow.md#image-editing) | `mage_flow` | +| [**LLaDA-Image**](../models/llada_image.md#image-editing) | `llada_image` | +| [**Fibo 2.0 Dev / Edit Turbo**](../models/fibo2.md#fibo-20-edit-turbo) | `fibo2` | | **Anima (Community Edit LoRAs)** | `cosmos_reference` | Stable-diffusion.spp also supports basic Unet-based editing models like instruct-pix2pix or CosXL-Edit. This document is not about those. @@ -98,7 +98,7 @@ For a technical overview of how each preset is configured, see the table below. | `flux_kontext` | No | `fixed` | `none` | | | `longcat` | Yes | `fixed` | `area` | | | `flux2` | No | `increase` | `none` | | -| `fibo2` | Yes | `increase` | `longest` | Shared reference crops; clean reference timesteps; see [Fibo 2.0](./fibo2.md) | +| `fibo2` | Yes | `increase` | `longest` | Shared reference crops; clean reference timesteps; see [Fibo 2.0](../models/fibo2.md) | | `qwen` | Yes | `increase` | `area` | | | `qwen_layered` | Yes | `decrease` | `area` | | | `mage_flow` | Yes | `increase` | `longest` | `vlm_max_size = 384`, VAE input resized to target | diff --git a/docs/generation_metadata.md b/docs/guides/generation_metadata.md similarity index 81% rename from docs/generation_metadata.md rename to docs/guides/generation_metadata.md index 1f7fc3338c..f24bda25fb 100644 --- a/docs/generation_metadata.md +++ b/docs/guides/generation_metadata.md @@ -1,30 +1,31 @@ # Generation results and metadata -`generate_image()`, `generate_video()`, `adetail_image()` and `upscale()` return -owned results containing the media and its generation record. This is a breaking -C API change: applications and bindings must update their calls and rebuild. +`sd_generate_image()`, `sd_generate_video()`, `sd_adetail()`, `sd_upscale()` and +`sd_upscale_video()` return opaque `sd_result_t` objects containing generated media +and generation metadata. See the [C API guide](c_api.md) for option creation and +migration from the removed parameter/result structures. ```c -sd_img_gen_params_t params; -sd_img_gen_params_init(¶ms); -params.prompt = "a small house"; - -sd_image_generation_result_t* result = NULL; -if (generate_image(ctx, ¶ms, &result)) { - for (int i = 0; i < result->count; ++i) { - const sd_image_result_t* item = &result->items[i]; - /* Consume item->image and item->generation_info together. */ +sd_img_gen_opt_t* opt = sd_img_gen_opt_create(); +sd_result_t* result = NULL; +if (opt && sd_opt_set_string(opt, "prompt", "a small house") == SD_STATUS_OK && + sd_generate_image(ctx, opt, &result) == SD_STATUS_OK) { + for (size_t i = 0; i < sd_result_get_image_count(result); ++i) { + const sd_tensor_t* image = sd_result_get_image(result, i); + const char* info = sd_result_get_generation_info(result, i); + /* Consume image and info together. */ } - free_sd_image_generation_result(result); } +sd_result_free(result); +sd_opt_free(opt); ``` -`result_out` is required and is set to `NULL` on failure. A successful result -owns its pixels and UTF-8 JSON strings. The video result additionally owns its -audio and contains the effective `fps` and `frame_count`. Results remain valid -after freeing the generation context. Use the matching library result destructor; -do not separately free fields unless transferring ownership and clearing them. -Both result destructors accept `NULL`. +`result_out` is required and set to `NULL` on failure. Successful results own +their pixels, audio and UTF-8 JSON independently of the generation context. +Video results expose effective FPS and frame count through accessors; each valid +frame index returns the shared video generation record. Result views are borrowed +until `sd_result_free()`. Use `sd_tensor_retain()` to keep media independently, +and release the retained handle with `sd_tensor_free()`. Free functions accept `NULL`. ## Reproduction parameters @@ -33,7 +34,7 @@ same loaded context and the same input images, masks, reference media and other assets. It is created after `GenerationRequest` resolves the request, before preprocessing or sampling. A failed call returns no result or metadata. -Fields follow the C generation parameter structs: `prompt`, `negative_prompt`, +The JSON schema retains its established generation field names: `prompt`, `negative_prompt`, `width`, `height`, `seed`, `sample_params`, `hires`, `cache`, `vae_tiling_params`, `image_preprocess`, and the image/video-specific options. `kind` distinguishes image and video parameters. There is no inferred `task`. @@ -77,10 +78,10 @@ and shares them across a batch. With a stochastic VAE encoder, replaying a later image alone using its sampling seed can change those input latents. Per-image seeds therefore do not guarantee independent reproduction for these paths. -To replay a record, initialize the matching C parameter struct, populate its -fields from the JSON, restore the same input buffers, and call generation with -the same context. Keep custom-sigma/SLG arrays and string storage alive for the -call. Paths supplied as generation parameters (LoRAs, hires models and identity +To replay a record, create image/video options, translate JSON fields to the documented C API +keys, restore the same input buffers, and call generation with the same context. +Setters copy strings and numeric arrays and retain media tensors; keep options alive +for the call. Paths supplied as generation parameters (LoRAs, hires models and identity embeddings) are retained. `models` identifies successfully loaded context model files by component (`model`, `diffusion_model`, `clip_l`, `vae`, etc.), using filenames with extensions and omitting directories. Optional components disabled diff --git a/docs/image_preprocessing.md b/docs/guides/image_preprocessing.md similarity index 91% rename from docs/image_preprocessing.md rename to docs/guides/image_preprocessing.md index c097e5a901..dc0aca8bc3 100644 --- a/docs/image_preprocessing.md +++ b/docs/guides/image_preprocessing.md @@ -156,23 +156,23 @@ user rules for replay with the same input images; image encodings and channel co ## C API -Set `image_preprocess` on the existing image/video generation parameters. -Generation returns media and actual parameters together; see [generation metadata](generation_metadata.md): +Set the `image_preprocess.rules` key on image/video options. Generation returns +media and actual parameters together; see [generation metadata](generation_metadata.md): ```c -sd_img_gen_params_t params; -sd_img_gen_params_init(¶ms); +sd_img_gen_opt_t* opt = sd_img_gen_opt_create(); +sd_result_t* result = NULL; /* Set prompt, original-resolution input images, and generation options. */ -params.image_preprocess.rules = "target=init,mode=crop-resize,filter=lanczos;" - "target=mask,filter=nearest-exact"; -sd_image_generation_result_t* result = NULL; -bool ok = generate_image(ctx, ¶ms, &result); -/* Consume result->items and their generation_info on success. */ -free_sd_image_generation_result(result); +sd_status_t status = sd_opt_set_string(opt, "image_preprocess.rules", + "target=init,mode=crop-resize,filter=lanczos;target=mask,filter=nearest-exact"); +if (status == SD_STATUS_OK) + status = sd_generate_image(ctx, opt, &result); +/* Read outputs through sd_result_get_*() on success. */ +sd_result_free(result); +sd_opt_free(opt); ``` -Both generation parameter initializers set `image_preprocess.rules` to `NULL`, -selecting input presets. Rule strings are borrowed for the synchronous call. +Leaving the key unset selects input presets. Rule strings are copied by setters. The library owns temporary transformed pixels; caller images and arrays are not modified. Add `canny=true` to the desired target's rule in `image_preprocess.rules` to enable Canny. diff --git a/docs/imatrix.md b/docs/guides/imatrix.md similarity index 100% rename from docs/imatrix.md rename to docs/guides/imatrix.md diff --git a/docs/int8_convrot.md b/docs/guides/int8_convrot.md similarity index 99% rename from docs/int8_convrot.md rename to docs/guides/int8_convrot.md index cf6219edd4..c105f15bb1 100644 --- a/docs/int8_convrot.md +++ b/docs/guides/int8_convrot.md @@ -5,7 +5,7 @@ sd.cpp can load and execute ComfyUI `int8_tensorwise` safetensors with `convrot` This uses sd.cpp's INT8 tensorwise/convrot operators through the patched GGML extension API. Schemas and graph builders live in sd.cpp, alongside the backend kernels and fusion rules; GGML supplies generic dispatch and workspace -services. See [operator extensions](ggml_extensions.md) for the implementation +services. See [operator extensions](../development/ggml_extensions.md) for the implementation and the preserved INT8 API entry points. Builds with `SD_USE_UPSTREAM_GGML=ON` reject these files during loading. diff --git a/docs/performance.md b/docs/guides/performance.md similarity index 99% rename from docs/performance.md rename to docs/guides/performance.md index 1f5771c849..76fb3da9eb 100644 --- a/docs/performance.md +++ b/docs/guides/performance.md @@ -95,7 +95,7 @@ Execution failures are not retried, and encoding has no automatic OOM retry. `--temporal-tiling` remains independent of spatial tiling. The original MiniMax-H3 video VAE uses spatial tiling (256x256 pixels and 25% overlap by default) and its own temporal windows. With `--vae-tiling`, its overlap follows `--vae-tile-overlap` and -explicit spatial sizes are honored. [Sol-H3-VAE](minimax_h3.md#sol-h3-vae-decoding) +explicit spatial sizes are honored. [Sol-H3-VAE](../models/minimax_h3.md#sol-h3-vae-decoding) decodes the whole clip by default and supports opt-in spatial and block-aligned temporal tiling. Tiling may affect output quality. diff --git a/docs/guides/prompt_images.md b/docs/guides/prompt_images.md new file mode 100644 index 0000000000..e6c301f0c4 --- /dev/null +++ b/docs/guides/prompt_images.md @@ -0,0 +1,112 @@ +# Images in prompts + +Use `` to place an image between text fragments. Multiple images +are read in prompt order, including adjacent or repeated references: + +```text +Use the subject from with the colors from . +``` + +Inline images are available to conditioners using Qwen2.5-VL, Qwen3-VL, or +Qwen3.5 with compatible vision weights. Supply the vision encoder with +`--llm_vision` or use a combined text/vision encoder. Each model keeps its own +prompt template, text weighting, padding, and output-layer selection. Missing +vision weights or a text-only LLM produce an error. See [MOSS-Image setup](../models/moss_image.md) +for a Qwen3.5 example. + +## CLI + +Pass image markers in the usual `-p` or `-n` argument. Relative image paths are +resolved against the current working directory. Spaces, colons, and parentheses +are allowed inside paths, including Windows paths: + +```text +-p "Draw in the style of " +``` + +A path must be nonempty and cannot contain `>`. Empty or unclosed markers are +errors. Write `\` to include a marker as literal text. A marker +is escaped when preceded by an odd number of backslashes; one backslash is +removed. Backslashes inside image paths are preserved. + +Images contribute visual information at their specified positions. They do not +automatically become edit source images or determine the output dimensions. +Continue using `--ref-image` and `ref_images` for existing editing workflows. +Positive and negative prompts resolve their own image markers independently. + +The context limit includes visual tokens and the model's prompt template. An +image-containing prompt that exceeds the limit fails; images are never partially +truncated. + +Each prompt supports at most 32 unique image sources, checked before loading any +of them. Repeated sources share their resolved pixels. Qwen2.5-VL also limits +each resized image to 8192 vision patches (2048 visual tokens), keeping its +window mask within 256 MiB. Larger grids fail before vision encoding; use a +smaller source image if this limit is exceeded. + +## C API + +Register a context-local image resolver with `sd_set_prompt_image_callback()`, +then call `sd_generate_image()` or `sd_generate_video()`. The +library passes each marker's `source` string unchanged to the callback, allowing +paths, application resource IDs, or in-memory images. The library does not open +files or fetch URLs itself. + +```c +bool load_prompt_image(const char* source, void* user_data, const sd_tensor_t** image_out); + +sd_img_gen_opt_t* opt = sd_img_gen_opt_create(); +sd_status_t status = sd_opt_set_string(opt, "prompt", + "Draw the subject from in a watercolor landscape."); +sd_result_t* result = NULL; +sd_set_prompt_image_callback(ctx, load_prompt_image, application_images); +if (status == SD_STATUS_OK) + status = sd_generate_image(ctx, opt, &result); +sd_set_prompt_image_callback(ctx, NULL, NULL); +sd_result_free(result); +sd_opt_free(opt); +``` + +The loader returns `true` with a borrowed U8 tensor of shape +`[channels,width,height]`, containing interleaved pixels and one to four channels. +The caller owns the tensor and keeps it valid until the next callback. The +library copies its pixels before calling the loader again. Repeated sources +within one prompt are loaded once and appear at every specified position. +Resolved image pixels participate in conditioning-cache comparisons. + +Register or clear the callback while the context is idle. Different contexts +have independent callbacks. Generation parameter structures and generation +entry points are unchanged. + +## Server + +Native `/sdcpp/v1` image and video requests accept a `prompt_images` object +mapping IDs to base64 images or image data URIs: + +```json +{ + "prompt": "Use with the colors from .", + "prompt_images": { + "subject": "data:image/png;base64,...", + "palette": "data:image/png;base64,..." + } +} +``` + +For OpenAI-compatible `/v1/images/generations`, `/v1/images/edits`, and +`/sdapi/v1` image requests, pass `prompt_images` only through the existing +`sd_cpp_extra_args` block embedded in `prompt`: + +```json +{ + "prompt": "Use . {\"prompt_images\":{\"subject\":\"data:image/png;base64,...\"}}" +} +``` + +For multipart `/v1/images/edits` requests, put the same block in the `prompt` +form field. Standard `image` and `image[]` uploads continue to be edit references. +Compatibility APIs do not read separate `prompt_images` fields or file uploads. + +The server resolves IDs only against the images owned by that request. An ID +never opens a server file or fetches a URL, and queued requests retain their own +image data until execution finishes. diff --git a/docs/quantization_and_gguf.md b/docs/guides/quantization_and_gguf.md similarity index 100% rename from docs/quantization_and_gguf.md rename to docs/guides/quantization_and_gguf.md diff --git a/docs/rpc.md b/docs/guides/rpc.md similarity index 98% rename from docs/rpc.md rename to docs/guides/rpc.md index cb28e256f0..6edd61da8f 100644 --- a/docs/rpc.md +++ b/docs/guides/rpc.md @@ -17,7 +17,7 @@ cmake .. \ cmake --build . --config Release -j $(nproc) ``` -> **Note:** Ensure you add the other flags you would normally use (e.g., `-DSD_VULKAN=ON`, `-DSD_CUDA=ON`, `-DSD_HIPBLAS=ON`, or `-DGGML_METAL=ON`), for more information about building `stable-diffusion.cpp` from source, please refer to the [build.md](build.md) documentation. +> **Note:** Ensure you add the other flags you would normally use (e.g., `-DSD_VULKAN=ON`, `-DSD_CUDA=ON`, `-DSD_HIPBLAS=ON`, or `-DGGML_METAL=ON`), for more information about building `stable-diffusion.cpp` from source, please refer to the [build.md](../build.md) documentation. ## 2. Ensure `llama.cpp` is at the correct commit diff --git a/docs/sage_attention.md b/docs/guides/sage_attention.md similarity index 94% rename from docs/sage_attention.md rename to docs/guides/sage_attention.md index 2c5d9602d3..2f3a0d09ca 100644 --- a/docs/sage_attention.md +++ b/docs/guides/sage_attention.md @@ -61,7 +61,7 @@ speed, and report model loading and first-step initialization separately. Quantization, smoothing, and format conversion costs are included in generation time, so short sequences may not benefit. -Library callers set `sd_ctx_params_t.sage_attn = true` before `new_sd_ctx()`, -like `diffusion_flash_attn`. Context creation fails if the requested feature is -unavailable. Initialize the parameter structure with `sd_ctx_params_init()`. +Library callers set `sage_attn` with `sd_opt_set_bool(ctx_opt, "sage_attn", true)` +before `sd_ctx_create()`. Context creation fails if the requested feature is +unavailable. Create options with `sd_ctx_opt_create()`. Rebuild library callers against the updated public header. diff --git a/docs/guides/sampling.md b/docs/guides/sampling.md new file mode 100644 index 0000000000..7955ce1f35 --- /dev/null +++ b/docs/guides/sampling.md @@ -0,0 +1,143 @@ +# Sampling methods and schedulers + +The sampling method controls how each denoising step updates the image or video +latent. The scheduler chooses the noise levels (sigmas) used across those steps. +Choose both according to the [model guide](../models/README.md) for your checkpoint. + +## Usage + +For example, using the Stable Diffusion 1.5 checkpoint from the +[quick start](../../README.md#quick-start): + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -m ..\models\checkpoints\v1-5-pruned-emaonly.safetensors ` + --sampling-method euler_a ` + --scheduler discrete ` + --steps 20 ` + -p "a lovely cat" +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -m ../models/checkpoints/v1-5-pruned-emaonly.safetensors \ + --sampling-method euler_a \ + --scheduler discrete \ + --steps 20 \ + -p "a lovely cat" +``` + +Run the example from the build directory. See the +[shared command conventions](../models/README.md#example-conventions) for build +layouts and platform-specific syntax. + +| CLI option | Purpose | +| --- | --- | +| `--sampling-method NAME` | Select the sampling method. | +| `--scheduler NAME` | Select the sigma scheduler. | +| `--steps N` | Set the number of sampling steps. | +| `--high-noise-sampling-method NAME` | Select the high-noise sampler for two-stage video models. | +| `--sigmas VALUES_OR_PRESET` | Supply a custom schedule instead of generating one with the scheduler. | +| `--hires-sigmas VALUES_OR_PRESET` | Supply a custom schedule for the highres fix second pass. | + +The names below are the exact, case-sensitive CLI values. Run `.\bin\Release\sd-cli.exe --help` on Windows or `./bin/sd-cli --help` on +Linux/macOS to check the options supported by your installed build. + +For native server requests, use `sample_params.sample_method` and +`sample_params.scheduler`. See the [server API reference](../../examples/server/api.md) +for request fields and capability discovery. + +## Sampling methods + +| Method | CLI value | +| --- | --- | +| Euler | `euler` | +| Euler ancestral | `euler_a` | +| Heun | `heun` | +| DPM2 | `dpm2` | +| DPM++ 2S ancestral | `dpm++2s_a` | +| DPM++ 2M | `dpm++2m` | +| DPM++ 2M v2 (modified) | `dpm++2mv2` | +| iPNDM | `ipndm` | +| iPNDM V | `ipndm_v` | +| LCM | `lcm` | +| DDIM trailing | `ddim_trailing` | +| TCD | `tcd` | +| RES Multistep | `res_multistep` | +| RES 2S | `res_2s` | +| ER-SDE | `er_sde` | +| Euler CFG++ | `euler_cfg_pp` | +| Euler ancestral CFG++ | `euler_a_cfg_pp` | +| Euler GE (gradient estimation) | `euler_ge` | +| DPM++ 2M SDE | `dpm++2m_sde` | +| DPM++ 2M SDE with Brownian tree noise | `dpm++2m_sde_bt` | +| LMS | `lms` | + +## Schedulers + +| Scheduler | CLI value | +| --- | --- | +| Discrete | `discrete` | +| Karras | `karras` | +| Exponential | `exponential` | +| AYS | `ays` | +| GITS | `gits` | +| SGM Uniform | `sgm_uniform` | +| Simple | `simple` | +| SmoothStep | `smoothstep` | +| KL Optimal | `kl_optimal` | +| LCM | `lcm` | +| Bong Tangent | `bong_tangent` | +| LTX-2 | `ltx2` | +| Logit Normal | `logit_normal` | +| FLUX.2 | `flux2` | +| FLUX | `flux` | +| Beta | `beta` | +| LLaDA-Image | `llada_image` | + +`normal` is accepted as an alias for `discrete`. + +## Defaults and model compatibility + +Omitting `--sampling-method` and `--scheduler` uses defaults selected for the +model and sampler. A Web UI recipe can supply its own defaults. + +- For EDM v-prediction checkpoints, the automatic scheduler is `exponential`. + Otherwise, the `lcm` and `tcd` samplers default to the `lcm` scheduler, and + `ddim_trailing` defaults to `simple`. Other samplers use the model's default + scheduler. +- AYS has reference schedules for SD 1.x, SDXL, and SVD. It is not designed for + SD 2.x or other model families. +- GITS requires at least two steps. +- Model-specific schedules should follow the matching guide, such as + [FLUX.2](../models/flux2.md), [LTX-2](../models/ltx2.md), + [Ideogram4](../models/ideogram4.md), or [LLaDA-Image](../models/llada_image.md). +- Some models use specialized sampling paths or schedules, such as + [SeFi-Image](../models/sefi_image.md). Changing a generic sampler or scheduler + option may not change that model's behavior. +- For few-step checkpoints or adapters, keep the sampler, step count, guidance, + and schedule specified in the model guide. See [LCM/LCM-LoRA](../models/lcm.md) + for an example. + +## Custom sigma schedules + +`--sigmas` accepts comma-separated noise levels or one of the presets below. +Supply descending values ending in zero when specifying your own schedule, for +example `--sigmas "14.61,7.8,3.5,0.0"`. The values replace the generated schedule; +the number of sampling steps becomes the number of values minus one. + +| Preset | Steps | Model guide | +| --- | --- | --- | +| `qwen-image-2.1-turbo` | 8 | [Qwen Image 2.1 Turbo](../models/qwen_image_2.1.md#turbo) | +| `p-h3-image` | 8 | [P-H3-Image](../models/p_h3_image.md) | +| `minimax-h3-image` | 6 | [MiniMax-H3-Image](../models/minimax_h3_image.md) | +| `hyperflow-h3-flash-4` | 4 | [HyperFlow H3 Flash](../models/minimax_h3.md#hyperflow-h3-flash) | +| `hyperflow-h3-flash-6` | 6 | [HyperFlow H3 Flash](../models/minimax_h3.md#hyperflow-h3-flash) | +| `hyperflow-h3-flash-8` | 8 | [HyperFlow H3 Flash](../models/minimax_h3.md#hyperflow-h3-flash) | + +`--hires-sigmas` accepts the same format and presets for the highres fix second +pass. Use each preset with the checkpoint and settings in its model guide. diff --git a/docs/tokenizers.md b/docs/guides/tokenizers.md similarity index 94% rename from docs/tokenizers.md rename to docs/guides/tokenizers.md index 272df9d438..fb93550a9d 100644 --- a/docs/tokenizers.md +++ b/docs/guides/tokenizers.md @@ -52,15 +52,17 @@ sd-cli --diffusion-model sd3.gguf --clip_l clip_l.safetensors \ -p "a cat" ``` -The C API accepts the same string in `sd_ctx_params_t::tokenizer`. A null or +The C API accepts the same string through the `tokenizer` context-option key. An unset or empty value keeps an embedded tokenizer where available; PiD and Lens require a nonempty main tokenizer path. The CLI passes the string through; `TokenizerConfig` parses and validates it when text encoders are initialized. ```c -sd_ctx_params_t params; -sd_ctx_params_init(¶ms); -params.tokenizer = "clip-l=tokenizer_clip.json,clip-g=tokenizer_clip.json"; +sd_ctx_opt_t* opt = sd_ctx_opt_create(); +sd_status_t status = sd_opt_set_string(opt, "tokenizer", + "clip-l=tokenizer_clip.json,clip-g=tokenizer_clip.json"); +/* Set model paths, then call sd_ctx_create(opt, &ctx). */ +sd_opt_free(opt); ``` Rebuild applications against the updated public header when using the updated diff --git a/docs/hidream_o1_image.md b/docs/hidream_o1_image.md deleted file mode 100644 index 529164285a..0000000000 --- a/docs/hidream_o1_image.md +++ /dev/null @@ -1,20 +0,0 @@ -# How to Use - -## Download weights - -- Download HiDream-O1-Image-Dev - - safetensors: https://huggingface.co/Comfy-Org/HiDream-O1-Image/tree/main/checkpoints -- Download HiDream-O1-Image - - safetensors: https://huggingface.co/Comfy-Org/HiDream-O1-Image/tree/main/checkpoints - -## Examples - -### HiDream-O1-Image-Dev - -``` -.\bin\Release\sd-cli.exe -m ..\models\diffusion_models\hidream_o1_image_dev_bf16.safetensors -p "a lovely cat holding a sign says -'hidream o1 cpp'" --cfg-scale 1.0 -v -H 1024 -W 1024 -``` - -HiDream-O1-Image-Dev example - diff --git a/docs/hipBLAS_on_Windows.md b/docs/hipBLAS_on_Windows.md deleted file mode 100644 index cd8465721b..0000000000 --- a/docs/hipBLAS_on_Windows.md +++ /dev/null @@ -1,88 +0,0 @@ -# Using hipBLAS on Windows - -To get hipBLAS in `stable-diffusion.cpp` working on Windows, go through this guide section by section. - -## Build Tools for Visual Studio 2022 - -Skip this step if you already have Build Tools installed. - -To install Build Tools, go to [Visual Studio Downloads](https://visualstudio.microsoft.com/vs/), download `Visual Studio 2022 and other Products` and run the installer. - -## CMake - -Skip this step if you already have CMake installed: running `cmake --version` should output `cmake version x.y.z`. - -Download latest `Windows x64 Installer` from [Download | CMake](https://cmake.org/download/) and run it. - -## ROCm - -Skip this step if you already have Build Tools installed. - -The [validation tools](https://rocm.docs.amd.com/en/latest/reference/validation_tools.html) not support on Windows. So you should confirm the Version of `ROCM` by yourself. - -Fortunately, `AMD` provides complete help documentation, you can use the help documentation to install [ROCM](https://rocm.docs.amd.com/en/latest/deploy/windows/quick_start.html) - ->**If you encounter an error, if it is [AMD ROCm Windows Installation Error 215](https://github.com/RadeonOpenCompute/ROCm/issues/2363), don't worry about this error. ROCM has been installed correctly, but the vs studio plugin installation failed, we can ignore it.** - -Then we must set `ROCM` as environment variables before running cmake. - -Usually if you install according to the official tutorial and do not modify the ROCM path, then there is a high probability that it is here `C:\Program Files\AMD\ROCm\7.1.1\bin` - -This is what I use to set the clang: -```Commandline -set CC=C:\Program Files\AMD\ROCm\7.1.1\bin\clang.exe -set CXX=C:\Program Files\AMD\ROCm\7.1.1\bin\clang++.exe -``` - -## Ninja - -Skip this step if you already have Ninja installed: running `ninja --version` should output `1.11.1`. - -Download latest `ninja-win.zip` from [GitHub Releases Page](https://github.com/ninja-build/ninja/releases/tag/v1.11.1) and unzip. Then set as environment variables. I unzipped it in `C:\Program Files\ninja`, so I set it like this: - -```Commandline -set ninja=C:\Program Files\ninja\ninja.exe -``` -## Building stable-diffusion.cpp - -The thing different from the regular CPU build is `-DSD_HIPBLAS=ON` , -`-G "Ninja"`, `-DCMAKE_C_COMPILER=clang`, `-DCMAKE_CXX_COMPILER=clang++`, `-DAMDGPU_TARGETS=gfx1150;gfx1151;gfx1200;gfx1201;gfx1100;gfx1101;gfx1102;gfx1030;gfx1031;gfx1032` - ->**Notice**: check the `clang` and `clang++` information: -```Commandline -clang --version -clang++ --version -``` - -If you see like this, we can continue: -``` -clang version 17.0.0 (git@github.amd.com:Compute-Mirrors/llvm-project e3201662d21c48894f2156d302276eb1cf47c7be) -Target: x86_64-pc-windows-msvc -Thread model: posix -InstalledDir: C:\Program Files\AMD\ROCm\7.1.1\bin -``` - -``` -clang version 17.0.0 (git@github.amd.com:Compute-Mirrors/llvm-project e3201662d21c48894f2156d302276eb1cf47c7be) -Target: x86_64-pc-windows-msvc -Thread model: posix -InstalledDir: C:\Program Files\AMD\ROCm\7.1.1\bin -``` - ->**Notice** that the GPU targets are now compatible with multiple GPU architectures (ROCm 7.1.1 targets). You can change them to match your GPU architecture. Click here to see your architecture [LLVM Target](https://rocm.docs.amd.com/en/latest/release/windows_support.html#windows-supported-gpus) - -Examples: -- AMD Radeon™ RX 7900 XTX Graphics: `gfx1100` -- AMD Radeon™ RX 7900 XT Graphics: `gfx1101` -- AMD Radeon™ RX 7900 GRE Graphics: `gfx1102` - -option: - -```commandline -mkdir build -cd build -cmake .. -G "Ninja" -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DSD_HIPBLAS=ON -DCMAKE_BUILD_TYPE=Release -DAMDGPU_TARGETS="gfx1150;gfx1151;gfx1200;gfx1201;gfx1100;gfx1101;gfx1102;gfx1030;gfx1031;gfx1032" -cmake --build . --config Release -``` - -If everything went OK, `build\bin\sd-cli.exe` file should appear. diff --git a/docs/hunyuan_video.md b/docs/hunyuan_video.md deleted file mode 100644 index fadb884ef4..0000000000 --- a/docs/hunyuan_video.md +++ /dev/null @@ -1,24 +0,0 @@ -# HunyuanVideo 1.5 - -HunyuanVideo 1.5 uses a HunyuanVideo diffusion transformer, a causal video VAE, Qwen2.5-VL 7B for the main text conditioning, -and ByT5 Small GlyphXL for glyph-aware text conditioning. - -## Download weights - -- Download HunyuanVideo 1.5 - - safetensors: https://huggingface.co/Comfy-Org/HunyuanVideo_1.5_repackaged/tree/main/split_files/diffusion_models -- Download vae - - safetensors: https://huggingface.co/Comfy-Org/HunyuanVideo_1.5_repackaged/tree/main/split_files/vae -- Download qwen_2.5_vl 7b - - safetensors: https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/tree/main/split_files/text_encoders - - gguf: https://huggingface.co/mradermacher/Qwen2.5-VL-7B-Instruct-GGUF/tree/main -- Download byt5 small glyphxl - - safetensros: https://huggingface.co/Comfy-Org/HunyuanVideo_1.5_repackaged/tree/main/split_files/text_encoders - -## Text-to-video example - -```shell -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\hunyuanvideo1.5_720p_t2v_fp16.safetensors --vae ..\models\vae\hunyuanvideo15_vae_fp16.safetensors --llm ..\models\text_encoders\qwen_2.5_vl_7b.safetensors --t5xxl ..\models\text_encoders\byt5_small_glyphxl_fp16.safetensors -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -W 1280 -H 720 --offload-to-cpu --video-frames 33 --vae-tiling -``` - - \ No newline at end of file diff --git a/docs/ideogram4.md b/docs/ideogram4.md deleted file mode 100644 index f0f1dd2500..0000000000 --- a/docs/ideogram4.md +++ /dev/null @@ -1,55 +0,0 @@ -# How to Use - -## Download weights - -- Download Ideogram4 - - safetensors: https://huggingface.co/ideogram-ai/ideogram-4-fp8/tree/main/transformer -- Download Ideogram4 uncond - - safetensors: https://huggingface.co/ideogram-ai/ideogram-4-fp8/tree/main/unconditional_transformer -- Download vae - - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main -- Download Qwen3-VL-8B-Instruct - - gguf: https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/tree/main - -## Use original FP8 weights - -The original Ideogram4 FP8 safetensors can be loaded directly. FP8 tensors stay -at one byte per element in RAM and VRAM. Backends that cannot multiply FP8 -weights directly cast only the active layer to a temporary BF16 tensor during -execution; the loader does not expand the entire checkpoint to BF16. - -With `SD_USE_UPSTREAM_GGML=ON`, FP8 tensors are converted to F16 at load time -instead (two bytes per element in RAM and VRAM). - -Use `ideogram4_fp8.safetensors` and `ideogram4_uncond_fp8.safetensors` directly -with `--diffusion-model` and `--uncond-diffusion-model`, respectively. - -## Optional conversion for quantization - -The following conversion is only needed when creating a quantized GGUF model. - -fp8 scale -> bf16 - -``` -python .\convert_fp8_scale_to_bf16.py --input .\ideogram4_fp8.safetensors --output ideogram4_bf16.safetensors -python .\convert_fp8_scale_to_bf16.py --input .\ideogram4_uncond_fp8.safetensors --output ideogram4_uncond_bf16.safetensors -``` - -bf16 -> q8 - -``` -.\bin\Release\sd-cli.exe -M convert -m ideogram4_bf16.safetensors -o ideogram4-Q8_0.gguf --tensor-type-rules "^layers.*adaln_modulation.*weight=q8_0,layers.*attention.o.*weight=q8_0,layers.*attention.qkv.*weight=q8_0,layers.*feed_forward.*weight=q8_0" -v - -.\bin\Release\sd-cli.exe -M convert -m ideogram4_uncond_bf16.safetensors -o ideogram4_uncond-Q8_0.gguf --tensor-type-rules "^layers.*adaln_modulation.*weight=q8_0,layers.*attention.o.*weight=q8_0,layers.*attention.qkv.*weight=q8_0,layers.*feed_forward.*weight=q8_0" -v -``` - -If you want lower VRAM usage, you can change the quantization from q8_0 to a lower-level quantization, such as q4_0. - - -## Examples - -```sh -.\bin\Release\sd-cli.exe --diffusion-model ideogram4-Q8_0.gguf --uncond-diffusion-model ideogram4_uncond-Q8_0.gguf --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf --vae ..\models\vae\flux2_ae.safetensors -p '{"high_level_description":"A square 1024 x 1024 luxury fashion magazine cover featuring exactly one short chubby fluffy cat as the main model. The cat sits on a soft ivory studio floor, facing the viewer with a stylish calm expression, wearing tiny black sunglasses, a red silk scarf, and a small gold collar charm. In front of the cat on the floor is a wide horizontal luxury nameplate that clearly reads ideogram4.cpp. The whole design feels premium, fashionable, clean, and editorial.","style_description":{"aesthetics":"luxury fashion magazine cover, high-end pet couture campaign, minimalist editorial design, elegant studio photography, soft paper texture, refined typography, fashionable and polished","lighting":"Soft diffused studio lighting, gentle spotlight on the cat, subtle floor shadow, warm ivory highlights, clean separation between subject and background","photo":"high-resolution fashion editorial photography look, front-facing cat portrait, crisp fur details, glossy sunglasses, clear readable nameplate text, shallow depth of field","medium":"mixed media fashion photography and premium editorial graphic design","color_palette":["#F4EFE7","#111111","#D8B56D","#B73A3A","#FFFFFF","#8A7A6A"]},"compositional_deconstruction":{"canvas":"Square 1024 x 1024 canvas with a normal upright orientation. Do not rotate the poster or any text. Use a clean fashion magazine cover layout.","background":"Warm ivory studio backdrop with subtle paper grain, a soft spotlight gradient, faint floor shadow, and a few minimal gold editorial lines. The background is spacious, premium, and uncluttered.","layout":"Top center has a small elegant headline. Center area features one cat as the main fashion model. Lower foreground has a wide horizontal luxury nameplate placed on the floor in front of the cat. Bottom center has a small footer. All text is horizontal, upright, and readable left to right.","elements":[{"type":"text","desc":"Top center headline reading LOOK WHAT I FOUND in a refined high-fashion serif font. The headline is horizontal, centered, elegant, and secondary to the nameplate text."},{"type":"obj","desc":"Exactly one short chubby fluffy cat sitting in the center like a luxury fashion model. The cat has a large round head, compact body, short legs, soft detailed fur, expressive eyes, and a calm confident pose. The cat is cute and rounded, not tall, not stretched, not duplicated."},{"type":"obj","desc":"Tiny glossy black sunglasses worn naturally by the cat, slightly oversized but still showing the cat face clearly. The sunglasses add a chic fashion-editorial attitude."},{"type":"obj","desc":"A red silk scarf tied neatly around the cat neck, with soft folds and a couture feeling. The scarf must not cover the cat face or the nameplate."},{"type":"obj","desc":"A small gold collar charm or fashion accessory under the scarf, subtle and premium, adding a luxury campaign detail."},{"type":"obj","desc":"In the lower foreground, place a wide horizontal luxury nameplate on the floor in front of the cat. The nameplate is low, flat, landscape-oriented, much wider than tall, like a fashion show seat card or premium display plaque. It is centered, front-facing, level, and fully visible. It must not become vertical, tall, standing, rotated, or side-facing."},{"type":"text","desc":"Print the exact text ideogram4.cpp only on the wide horizontal nameplate. Use clean bold black lettering, perfectly spelled, lowercase, with the number 4 and .cpp extension. The text must fit completely inside the nameplate, stay horizontal, and be readable from left to right."},{"type":"obj","desc":"Add sparse premium editorial accents around the edges: thin gold lines, small code brackets, tiny cursor marks, subtle dots, and minimal geometric details. No extra cats, no stickers, no animal faces, no busy decorations."},{"type":"text","desc":"Bottom center footer reading tiny paws, big compile energy in a small refined monospace or editorial font. The footer is horizontal, centered, understated, and much smaller than the nameplate text."}]}}' -v --offload-to-cpu -H 1024 -W 1024 -``` - -ideogram4 image example diff --git a/docs/ip_adapter.md b/docs/ip_adapter.md deleted file mode 100644 index 27f9ae6896..0000000000 --- a/docs/ip_adapter.md +++ /dev/null @@ -1,87 +0,0 @@ -# IP-Adapter - -stable-diffusion.cpp supports [IP-Adapter](https://github.com/tencent-ailab/IP-Adapter) -image-prompt conditioning for SD 1.5 and SDXL. Given a reference image, -IP-Adapter transfers the subject and appearance of that image into the -generation, alongside the text prompt. - -IP-Adapter encodes the reference image with a CLIP-Vision (ViT-H/14) -encoder, projects the embedding into a few image tokens, and injects them -through a decoupled cross-attention added to every attn2 layer of the -UNet. It composes with Control Net, so a reference image (appearance) and -an OpenPose hint (pose) can be combined in a single generation. - -Both the classic adapters and the higher-fidelity **Plus** adapters are -supported; see [Plus variants](#plus-variants) below. The variant is -detected from the weight file, so the same options work for both. - -## Required weights - -1. A base SD 1.5 or SDXL model. -2. A CLIP-Vision (ViT-H/14) image encoder, passed with `--clip_vision` - (for example `clip_vision_h.safetensors`). -3. An IP-Adapter weight file, passed with `--ip-adapter`. The `vit-h` - variants reuse the same ViT-H encoder as above. From - [h94/IP-Adapter](https://huggingface.co/h94/IP-Adapter): - - SD 1.5: `models/ip-adapter_sd15.safetensors` - - SDXL: `sdxl_models/ip-adapter_sdxl_vit-h.safetensors` - - SD 1.5 Plus: `models/ip-adapter-plus_sd15.safetensors` - - SDXL Plus: `sdxl_models/ip-adapter-plus_sdxl_vit-h.safetensors` - - The Plus files (`ip-adapter-plus_*`) are used exactly like the classic - ones; see [Plus variants](#plus-variants). - -## Options - -- `--ip-adapter ` path to the IP-Adapter weight file. -- `--ip-adapter-image ` path to the reference image. -- `--ip-adapter-strength ` strength of the IP-Adapter injection - (default 1.0). Lower values let the text prompt dominate; 0.6 to 0.8 is - a good starting range. - -## Example (SD 1.5) - -``` -sd-cli -m ..\models\sd_v1.5.safetensors --clip_vision ..\models\clip_vision_h.safetensors --ip-adapter ..\models\ip-adapter_sd15.safetensors --ip-adapter-image ..\assets\reference.png --ip-adapter-strength 0.8 -p "a woman, best quality" -n "lowres, bad anatomy" --cfg-scale 7 --steps 30 --sampling-method dpm++2m --scheduler karras -W 512 -H 512 -``` - -## Example (SDXL) - -``` -sd-cli -m ..\models\sdxl.safetensors --clip_vision ..\models\clip_vision_h.safetensors --ip-adapter ..\models\ip-adapter_sdxl_vit-h.safetensors --ip-adapter-image ..\assets\reference.png --ip-adapter-strength 0.8 -p "a woman, best quality" -n "lowres, bad anatomy" --cfg-scale 6 --steps 25 --sampling-method dpm++2m --scheduler karras -W 1024 -H 1024 --vae-tiling -``` - -The SDXL VAE decode at 1024x1024 is memory heavy; add `--vae-tiling` (and -`--offload-to-cpu`) on GPUs with limited VRAM. - -## Plus variants - -The Plus adapters (`ip-adapter-plus_sd15`, `ip-adapter-plus_sdxl_vit-h`) -replace the small linear image projection with a Resampler (a -Perceiver-style module with learned latent queries). Instead of pooling the -CLIP-Vision output into one vector, the Resampler attends over the full grid -of penultimate CLIP-Vision hidden states and emits more image tokens (16 -instead of 4). The result transfers finer detail and layout from the -reference, at a small extra cost in the image-projection step. - -No extra flags are needed. The variant is detected from the weight file (the -Resampler's `image_proj.latents` tensor), and every Resampler dimension is -read from the tensor shapes, so the same `--ip-adapter`, -`--ip-adapter-image`, and `--ip-adapter-strength` options apply. Plus -composes with Control Net in the same way as the classic adapters. - -``` -sd-cli -m ..\models\sd_v1.5.safetensors --clip_vision ..\models\clip_vision_h.safetensors --ip-adapter ..\models\ip-adapter-plus_sd15.safetensors --ip-adapter-image ..\assets\reference.png --ip-adapter-strength 0.8 -p "a woman, best quality" -n "lowres, bad anatomy" --cfg-scale 7 --steps 30 --sampling-method dpm++2m --scheduler karras -W 512 -H 512 -``` - -The startup log line `IP-Adapter: 16 image tokens` (versus `4` for the -classic adapters) confirms a Plus file was loaded. - -## Combining with Control Net - -Add the usual Control Net options to keep the reference appearance while -controlling the pose: - -``` -sd-cli -m ..\models\sdxl.safetensors --clip_vision ..\models\clip_vision_h.safetensors --ip-adapter ..\models\ip-adapter_sdxl_vit-h.safetensors --ip-adapter-image ..\assets\character.png --ip-adapter-strength 0.9 --control-net ..\models\OpenPoseXL2.safetensors --control-image ..\assets\pose.png --control-strength 0.8 -p "a character, side view" --cfg-scale 6 --steps 25 -W 1024 -H 1024 --vae-tiling -``` diff --git a/docs/iris.md b/docs/iris.md deleted file mode 100644 index 322a3da1a4..0000000000 --- a/docs/iris.md +++ /dev/null @@ -1,24 +0,0 @@ -# How to Use - -Iris-3B generates images directly in pixel space and uses Qwen3-VL-4B-Instruct as the text encoder. No VAE is required. - -## Download weights - -- Download Iris-3B - - safetensors: https://huggingface.co/speridlabs/iris-3b/tree/main (`model.safetensors` in the root directory) -- Download Qwen3-VL-4B-Instruct - - safetensors: https://huggingface.co/Comfy-Org/Krea-2/tree/main/text_encoders - - gguf: https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct-GGUF/tree/main - -## Examples - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\iris-3b.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf -p "a lovely cat" --cfg-scale 3 -H 1024 -W 1024 -``` - -iris-3b example - -## Notes - -- Width and height must be multiples of 16. Do not pass `--vae`. -- Captions are limited to 300 tokens including the assistant-turn suffix. Positive and negative prompts use the same template. diff --git a/docs/kontext.md b/docs/kontext.md deleted file mode 100644 index 46da3c3994..0000000000 --- a/docs/kontext.md +++ /dev/null @@ -1,39 +0,0 @@ -# How to Use - -You can run Kontext using stable-diffusion.cpp with a GPU that has 6GB or even 4GB of VRAM, without needing to offload to RAM. - -## Download weights - -- Download Kontext - - If you don't want to do the conversion yourself, download the preconverted gguf model from [FLUX.1-Kontext-dev-GGUF](https://huggingface.co/QuantStack/FLUX.1-Kontext-dev-GGUF) - - Otherwise, download FLUX.1-Kontext-dev from https://huggingface.co/black-forest-labs/FLUX.1-Kontext-dev/blob/main/flux1-kontext-dev.safetensors -- Download vae from https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/ae.safetensors -- Download clip_l from https://huggingface.co/comfyanonymous/flux_text_encoders/blob/main/clip_l.safetensors -- Download t5xxl from https://huggingface.co/comfyanonymous/flux_text_encoders/blob/main/t5xxl_fp16.safetensors - -## Convert Kontext weights - -You can download the preconverted gguf weights from [FLUX.1-Kontext-dev-GGUF](https://huggingface.co/QuantStack/FLUX.1-Kontext-dev-GGUF), this way you don't have to do the conversion yourself. - -``` -.\bin\Release\sd-cli.exe -M convert -m ..\models\unet\flux1-kontext-dev.safetensors -o ..\models\flux1-kontext-dev-q8_0.gguf -v --type q8_0 -``` - -## Run - -- `--cfg-scale` is recommended to be set to 1. - -### Example -For example: - -``` - .\bin\Release\sd-cli.exe -r .\flux1-dev-q8_0.png --diffusion-model ..\models\flux1-kontext-dev-q8_0.gguf --vae ..\models\ae.sft --clip_l ..\models\clip_l.safetensors --t5xxl ..\models\t5xxl_fp16.safetensors -p "change 'flux.cpp' to 'kontext.cpp'" --cfg-scale 1.0 --sampling-method euler -v --clip-on-cpu -``` - - -| ref_image | prompt | output | -| ---- | ---- |---- | -| ![](../assets/flux/flux1-dev-q8_0.png) | change 'flux.cpp' to 'kontext.cpp' |![](../assets/flux/kontext1_dev_output.png) | - - - diff --git a/docs/krea2.md b/docs/krea2.md deleted file mode 100644 index 67fad023ed..0000000000 --- a/docs/krea2.md +++ /dev/null @@ -1,27 +0,0 @@ -# How to Use - -Krea2 uses a Krea2 diffusion transformer, the Wan2.1 VAE, and Qwen3-VL 4B as the LLM text encoder. - -## Download weights - -- Download Krea2 Raw - - safetensors: https://huggingface.co/krea/Krea-2-Raw/tree/main - - gguf: https://huggingface.co/realrebelai/KREA-2_GGUFs/tree/main/BASE -- Download Krea2 Turbo - - safetensors: https://huggingface.co/krea/Krea-2-Turbo/tree/main - - gguf: https://huggingface.co/realrebelai/KREA-2_GGUFs/tree/main/TURBO -- Download vae - - safetensors: https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/blob/main/split_files/vae/wan_2.1_vae.safetensors -- Download Qwen3-VL 4B - - safetensors: https://huggingface.co/Comfy-Org/Krea-2/tree/main/text_encoders - - gguf: https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct-GGUF/tree/main - -## Examples - -### Krea2 - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\Krea-2-Raw-Q8_0.gguf --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --vae ..\models\vae\wan_2.1_vae.safetensors -p "a lovely cat holding a sign says 'krea2.cpp'" -v --offload-to-cpu -``` - -Krea2 Raw example diff --git a/docs/lcm.md b/docs/lcm.md deleted file mode 100644 index 3a11938aad..0000000000 --- a/docs/lcm.md +++ /dev/null @@ -1,15 +0,0 @@ -## LCM/LCM-LoRA - -- Download LCM-LoRA form https://huggingface.co/latent-consistency/lcm-lora-sdv1-5 -- Specify LCM-LoRA by adding `` to prompt -- It's advisable to set `--cfg-scale` to `1.0` instead of the default `7.0`. For `--steps`, a range of `2-8` steps is recommended. For `--sampling-method`, `lcm`/`euler_a` is recommended. - -Here's a simple example: - -``` -./bin/sd-cli -m ../models/v1-5-pruned-emaonly.safetensors -p "a lovely cat" --steps 4 --lora-model-dir ../models -v --cfg-scale 1 -``` - -| without LCM-LoRA (--cfg-scale 7) | with LCM-LoRA (--cfg-scale 1) | -| ---- |---- | -| ![](../assets/without_lcm.png) |![](../assets/with_lcm.png) | \ No newline at end of file diff --git a/docs/lens.md b/docs/lens.md deleted file mode 100644 index 09e9575ac9..0000000000 --- a/docs/lens.md +++ /dev/null @@ -1,36 +0,0 @@ -# How to Use - -Lens uses a Lens diffusion transformer, the FLUX.2 VAE, and GPT-OSS-20B as the LLM text encoder. - -## Download weights - -- Download Lens - - safetensors: https://huggingface.co/Comfy-Org/Lens/tree/main/diffusion_models -- Download Lens Turbo - - safetensors: https://huggingface.co/Comfy-Org/Lens/tree/main/diffusion_models -- Download vae - - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main -- Download GPT-OSS-20B - - gguf: https://huggingface.co/unsloth/gpt-oss-20b-GGUF/tree/main -- Download GPT-OSS-20B tokenizer.json - - https://huggingface.co/openai/gpt-oss-20b/tree/main - -Lens and Lens Turbo require an external GPT-OSS `tokenizer.json` matching the text encoder checkpoint. Save it as `tokenizer_gpt_oss.json` and pass it with `--tokenizer`; the tokenizer is not embedded in sd.cpp. See [JSON tokenizers](tokenizers.md) for CLI and C API usage. - -## Examples - -### Lens - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\lens_bf16.safetensors --llm "..\models\text_encoders\gpt-oss-20b-UD-Q8_K_XL.gguf" --tokenizer ..\models\tokenizers\tokenizer_gpt_oss.json --vae ..\models\vae\flux2_ae.safetensors --cfg-scale 5.0 -p "A crystal dragon soaring through an aurora borealis sky, its entire body made of transparent faceted crystal refracting the green and purple aurora light into rainbow spectra, ice particles trailing from its wings, high fantasy digital art" -v -``` - -Lens example - -### Lens Turbo - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\lens_turbo_bf16.safetensors --llm "..\models\text_encoders\gpt-oss-20b-UD-Q8_K_XL.gguf" --tokenizer ..\models\tokenizers\tokenizer_gpt_oss.json --vae ..\models\vae\flux2_ae.safetensors --cfg-scale 1.0 -p "A crystal dragon soaring through an aurora borealis sky, its entire body made of transparent faceted crystal refracting the green and purple aurora light into rainbow spectra, ice particles trailing from its wings, high fantasy digital art" -v --steps 4 -``` - -Lens Turbo example diff --git a/docs/lingbot_video.md b/docs/lingbot_video.md deleted file mode 100644 index b92165ab5f..0000000000 --- a/docs/lingbot_video.md +++ /dev/null @@ -1,32 +0,0 @@ -# How to Use - -Lingbot Video uses a Lingbot diffusion transformer, the Wan2.1 VAE, and Qwen3-VL 4B as the LLM text encoder. - -## Download weights - -- Download lingbot-video-dense-1.3b - - safetensors: https://huggingface.co/robbyant/lingbot-video-dense-1.3b/tree/main/transformer -- Download lingbot-video-moe-30b-a3b - - safetensors: https://huggingface.co/robbyant/lingbot-video-moe-30b-a3b/tree/main/transformer -- Download vae - - safetensors: https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/blob/main/split_files/vae/wan_2.1_vae.safetensors -- Download Qwen3-VL 4B - - safetensors: https://huggingface.co/Comfy-Org/Krea-2/tree/main/text_encoders - - gguf: https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct-GGUF/tree/main - -## Notes - -- Use `-M vid_gen`. -- T2V uses the text prompt only. -- I2V uses `-i` as the first video frame. The same image is also passed to the - Qwen3-VL prompt enhancer when vision weights are available. -- Video frames are aligned to Wan-style temporal compression, so use frame counts - like 33, 49, or 81. - -## Examples - -### LingBot-Video T2V - -```sh -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\lingbot-video-dens-1.3b.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --vae ..\models\vae\wan_2.1_vae.safetensors -p '{"caption":{"comprehensive_description":"A lovely cat sits comfortably on a soft cushion near a sunlit window, looking calm, gentle, and adorable. The cat has soft fluffy fur, bright expressive eyes, small rounded ears, delicate whiskers, and a relaxed posture. Warm daylight falls across the cat from one side, creating soft highlights on the fur and a cozy glow around the scene. The background is softly blurred, showing hints of a peaceful indoor home environment with warm tones and gentle natural light. The overall atmosphere is cute, tender, serene, and photorealistic, emphasizing the cat''s charming appearance, soft texture, and affectionate presence.","camera_info":{"color":"Warm","frame_size":"Close Up","shot_type_angle":"Eye level","lens_size":"Medium Lens","composition":"Centered balanced","lighting":"Soft light","lighting_type":"Daylight"},"world_knowledge":[],"prominent_elements":[{"name":"lovely cat","description":"A cute and gentle domestic cat with soft fluffy fur, expressive eyes, and a calm relaxed presence.","location":"center of the frame","relative_size":"large","shape_and_color":"Small animal body with rounded face, triangular ears, bright eyes, and soft fur in warm natural tones","texture":"soft, fluffy, silky","appearance_details":"The cat has clean well-groomed fur, delicate whiskers, small ears, a cute nose, and bright attentive eyes. Its expression appears calm, affectionate, and slightly curious.","relationship":"Acts as the main subject and emotional focal point of the scene.","orientation":"facing the camera","pose":"sitting comfortably with a relaxed posture","expression":"gentle, adorable, calm, slightly curious","clothing":"","gender":"","skin_tone_and_texture":""},{"name":"cat eyes","description":"Bright expressive eyes that give the cat a sweet and affectionate appearance.","location":"upper center of the cat face","relative_size":"small","shape_and_color":"Round almond-like eyes with glossy reflections","texture":"clear, glossy, reflective","appearance_details":"The eyes catch the soft daylight, creating small natural highlights that make the cat look vivid and alive.","relationship":"Enhance the emotional charm and cuteness of the cat.","orientation":"looking toward the camera","pose":"","expression":"soft and attentive","clothing":"","gender":"","skin_tone_and_texture":""},{"name":"soft cushion","description":"A comfortable cushion or blanket where the cat is resting.","location":"bottom portion of the frame","relative_size":"medium","shape_and_color":"Soft rounded fabric surface in light warm neutral tones","texture":"plush, fabric, cozy","appearance_details":"The cushion gently supports the cat and adds a comfortable home-like feeling to the scene.","relationship":"Provides a cozy resting place for the cat.","orientation":"horizontal","pose":"","expression":"","clothing":"","gender":"","skin_tone_and_texture":""},{"name":"sunlit indoor background","description":"A softly blurred indoor background with warm daylight and peaceful home atmosphere.","location":"behind the cat, filling the upper and side areas of the frame","relative_size":"large","shape_and_color":"Soft abstract shapes in warm beige, cream, and pale golden tones","texture":"soft, blurry, bokeh-like","appearance_details":"The background is intentionally out of focus, keeping attention on the cat while creating a cozy and serene mood.","relationship":"Provides a warm and gentle environment that supports the cute domestic scene.","orientation":"upright","pose":"","expression":"","clothing":"","gender":"","skin_tone_and_texture":"","is_cluster":true,"number_of_objects":"numerous"}]}}' -n '{"universal_negative":{"visual_quality":["low quality","worst quality","blurry","pixelated","jpeg artifacts","low resolution","unstable color","color flicker","underexposed","overexposed","invisible subject","subject hidden in darkness"],"artistic_style":["painting","illustration","drawing","cartoon","3d render","cgi","sketch","digital art"],"composition_and_content":["text","watermark","signature","logo","subtitles","pillarboxed","side bars","portrait image in landscape frame"],"temporal_and_motion_stability":["flickering","jittery","motion blur","temporal inconsistency","warping","morphing","incoherent motion","unnatural movement","static object with sudden jump","frame-to-frame inconsistency"],"material_and_structure":["plastic-like glass","unrealistic texture","deformed bottle","liquid freezing improperly","distorted reflections"]}}' --offload-to-cpu --cfg-scale 3 --video-frames 33 -v -``` diff --git a/docs/llada_image.md b/docs/llada_image.md deleted file mode 100644 index d939e78cea..0000000000 --- a/docs/llada_image.md +++ /dev/null @@ -1,155 +0,0 @@ -# How to Use - -LLaDA-Image is a 6B text-to-image and instruction-guided editing model. The denoiser is a -Lumina2/Z-Image-style NextDiT conditioned by a LLaDA2-MoE diffusion-LLM text encoder, and it -reuses the Flux.2 VAE. Two checkpoints are published: a 50-step base model and -LLaDA-Image-Turbo, a 4-step distilled model. - -## Download weights - -Four components are required: a transformer, a text encoder, a VAE, and a connectors file -holding the QueryFormer, the text projection and, for editing, the SigVQ image encoder. - -The two published checkpoints are **not** interchangeable. LLaDA-Image-Turbo and LLaDA-Image -ship different transformers, text encoders, QueryFormers and text projections; only the VAE, -the SigVQ encoder and the tokenizer are shared. Mixing the two produces degraded output rather -than a clean error, so keep each checkpoint's files together. - -Both need an external LLaDA2 `tokenizer.json`, which is not embedded in sd.cpp and is the same -file for either checkpoint. Take `tokenizer/tokenizer.json` from either repository and pass it -with `--tokenizer`. See [JSON tokenizers](tokenizers.md) for CLI and C API usage. - -### LLaDA-Image-Turbo (4 steps) - -Converted transformer, text encoder and pre-merged connectors are at -https://huggingface.co/fszontagh/LLaDA-Image-Turbo-GGUF: - -- `llada-image-turbo-f16.gguf` -- `llada-image-turbo-text_encoder-q8_0.gguf` -- `llada-image-turbo-connectors.safetensors` for text to image, or - `llada-image-turbo-connectors-edit.safetensors`, which also carries the SigVQ encoder that - editing needs. - -Other quantizations of the transformer and the text encoder are in the same repository. - -The VAE comes from the original repository, -https://huggingface.co/inclusionAI/LLaDA-Image-Turbo: `vae/diffusion_pytorch_model.safetensors`, -referred to below as `llada_vae.safetensors`. - -### LLaDA-Image (50 steps) - -Converted transformer, text encoder and pre-merged connectors are at -https://huggingface.co/fszontagh/LLaDA-Image-GGUF: - -- `llada-image-f16.gguf` -- `llada-image-text_encoder-q8_0.gguf` -- `llada-image-connectors.safetensors` for text to image, or - `llada-image-connectors-edit.safetensors`, which also carries the SigVQ encoder that editing - needs. - -Other quantizations of the transformer and the text encoder are in the same repository. - -The VAE comes from the original repository, -https://huggingface.co/inclusionAI/LLaDA-Image, and is the same file as the Turbo one. - -### Converting the weights yourself - -The transformer has to go in through `--diffusion-model` so that its tensor names keep the -prefix the loader expects, while the text encoder goes in through `-m`: - -```bash -./bin/sd-cli -M convert --diffusion-model transformer/diffusion_pytorch_model.safetensors.index.json \ - -o llada-image-f16.gguf --type f16 -./bin/sd-cli -M convert -m text_encoder/model.safetensors.index.json \ - -o llada-image-text_encoder-q8_0.gguf --type q8_0 -``` - -### Building the connector file yourself - -`--embeddings-connectors` takes one file, so the QueryFormer, the text projection and -(for editing) the SigVQ encoder have to be combined into a single Safetensors file, each -tensor name prefixed with its component name. Leaving `sigvq` out skips loading the 2.6 GB -encoder: - -```python -from safetensors.torch import load_file, save_file - -merged = {} -for prefix, path in [ - ("queryformer", "queryformer/diffusion_pytorch_model.safetensors"), - ("text_projection", "text_projection/diffusion_pytorch_model.safetensors"), - ("sigvq", "sigvq/diffusion_pytorch_model.safetensors"), -]: - for name, tensor in load_file(path).items(): - merged[f"{prefix}.{name}"] = tensor -save_file(merged, "llada_connectors.safetensors") -``` - -## Examples - -### Text to image - -```bash -./bin/sd-cli \ - --diffusion-model /path/to/llada-image-turbo-f16.gguf \ - --llm /path/to/llada-image-turbo-text_encoder-q8_0.gguf \ - --tokenizer /path/to/tokenizer.json \ - --vae /path/to/llada_vae.safetensors \ - --embeddings-connectors /path/to/llada-image-turbo-connectors.safetensors \ - --prompt "a lovely cat holding a sign says 'llada.cpp'" \ - --width 1024 \ - --height 1024 \ - --steps 4 \ - --cfg-scale 1.0 \ - --seed 42 \ - --output output.png -``` - -LLaDA-Image example - -### Image editing - -```bash -./bin/sd-cli \ - --diffusion-model /path/to/llada-image-turbo-f16.gguf \ - --llm /path/to/llada-image-turbo-text_encoder-q8_0.gguf \ - --tokenizer /path/to/tokenizer.json \ - --vae /path/to/llada_vae.safetensors \ - --embeddings-connectors /path/to/llada-image-turbo-connectors-edit.safetensors \ - --ref-image /path/to/input.png \ - --prompt "change the sign text to 'sd.cpp'" \ - --width 1024 \ - --height 1024 \ - --steps 4 \ - --cfg-scale 1.0 \ - --output output.png -``` - -LLaDA-Image edit example - -See [edit.md](./edit.md) for the shared reference-image options. LLaDA-Image uses the -`llada_image` preset by default, resizing the reference image to the output width and height -before VAE encoding. SigVQ uses bilinear resizing to half the output resolution and inputs -normalized to `[-1, 1]`. CFG keeps the source latent in both branches and uses SigVQ features -only in the positive branch. Editing requires connectors that include the SigVQ weights. - -## Notes - -- Use 4 steps and `--cfg-scale 1.0` for LLaDA-Image-Turbo; the guidance is distilled away, so - a higher CFG degrades output and doubles the text encoder cost. The 50-step base model uses - `--steps 50 --cfg-scale 5`. -- Width and height are rounded up to a multiple of 16. For editing the reference pipeline - requires them to be divisible by 32. -- Edit the 50-step base model at 1024x1024. At 512x512 it returns the reference image almost - unchanged instead of applying the instruction; LLaDA-Image-Turbo edits correctly at both. -- Editing runs the reference and the target in one sequence, so it needs roughly twice the - tokens of text to image at the same size. On 12 GB, editing at 1024x1024 needs - flash attention; without it the diffusion graph does not fit. -- The weights total about 16 GB, but segmented execution streams them, so a much smaller - budget works. At 512x512, `--max-vram 6` costs almost nothing over unconstrained execution, - and `--max-vram 3` still produces byte-identical output at roughly 2.5x the time. -- `--scheduler` defaults to `llada_image`, which reproduces the reference Kumaraswamy sigma - grid. `--extra-sample-args uniform=1` selects the uniform grid instead. -- Prompt templating is handled automatically; pass a plain description. -- VQ-conditioned generation (`generation_mode="vq"`, where the text encoder decodes image - tokens before diffusion) is not implemented. diff --git a/docs/longcat_image.md b/docs/longcat_image.md deleted file mode 100644 index ae730b4727..0000000000 --- a/docs/longcat_image.md +++ /dev/null @@ -1,30 +0,0 @@ -# How to Use - -LongCat-Image uses a LongCat diffusion transformer, the FLUX VAE, and Qwen2.5-VL as the LLM text encoder. - -## Download weights - -- Download LongCat Image - - safetensors: https://huggingface.co/Comfy-Org/LongCat-Image/tree/main/split_files/diffusion_models - - gguf: https://huggingface.co/vantagewithai/LongCat-Image-GGUF/tree/main/comfy -- Download LongCat Image Edit - - LongCat Image Edit Turbo: https://huggingface.co/meituan-longcat/LongCat-Image-Edit-Turbo - - gguf: https://huggingface.co/vantagewithai/LongCat-Image-Edit-GGUF/tree/main -- Download vae - - safetensors: https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/ae.safetensors -- Download qwen_2.5_vl 7b - - safetensors: https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/tree/main/split_files/text_encoders - - gguf: https://huggingface.co/mradermacher/Qwen2.5-VL-7B-Instruct-GGUF/tree/main - - For image editing with GGUF text encoders, also download the matching mmproj file and pass it with `--llm_vision`. - -## Run - -LongCat uses quoted text for character-level text rendering. Put target text inside single quotes, double quotes, or Chinese quotes. - -### LongCat Image - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\LongCat-Image-Q4_K_M.gguf --vae ..\models\vae\ae.sft --llm ..\models\text_encoders\Qwen2.5-VL-7B-Instruct-Q8_0.gguf -p "a lovely cat holding a sign says 'longcat.cpp'" --cfg-scale 5.0 --sampling-method euler --flow-shift 3 -v --offload-to-cpu -``` - -longcat example diff --git a/docs/lora.md b/docs/lora.md deleted file mode 100644 index d6dc718991..0000000000 --- a/docs/lora.md +++ /dev/null @@ -1,26 +0,0 @@ -## LoRA - -- You can specify the directory where the lora weights are stored via `--lora-model-dir`. If not specified, the default is the current working directory. - -- LoRA is specified via prompt, just like [stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Features#lora). - -Here's a simple example: - -``` -./bin/sd-cli -m ../models/v1-5-pruned-emaonly.safetensors -p "a lovely cat" --lora-model-dir ../models -``` - -`../models/marblesh.safetensors` or `../models/marblesh.ckpt` will be applied to the model - -# Lora Apply Mode - -There are two ways to apply LoRA: **immediately** and **at_runtime**. You can specify it using the `--lora-apply-mode` parameter. - -By default, the mode is selected automatically: - -* If the model weights contain any quantized parameters, the **at_runtime** mode is used; -* Otherwise, the **immediately** mode is used. - -The **immediately** mode may have precision and compatibility issues with quantized parameters, but it usually offers faster inference speed and, in some cases, lower memory usage. -In contrast, the **at_runtime** mode provides better compatibility and higher precision, but inference may be slower and memory usage may be higher in some cases. - diff --git a/docs/ltx2.md b/docs/ltx2.md deleted file mode 100644 index e42698ffb3..0000000000 --- a/docs/ltx2.md +++ /dev/null @@ -1,139 +0,0 @@ -# How to Use - -Both LTX-2.3 and LTX-2.5 are supported. The two share a transformer, video VAE and audio -VAE architecture; LTX-2.5 drops the video FFN biases, adds a learned keyframe -absolute-position embedding, and pairs with a Gemma 4 text encoder instead of Gemma 3. -Everything is detected from the weights, so the command lines differ only in which files -you pass. - -# LTX-2.3 - -## Download weights - -### LTX-2.3 - -- Download LTX-2.3 - - safetensors: https://huggingface.co/Kijai/LTX2.3_comfy/tree/main/diffusion_models - - gguf: https://huggingface.co/unsloth/LTX-2.3-GGUF/tree/main -- Download gemma-3-12b-it - - gguf: https://huggingface.co/unsloth/gemma-3-12b-it-GGUF/tree/main -- Download embeddings connectors - - safetensors: https://huggingface.co/unsloth/LTX-2.3-GGUF/tree/main/text_encoders -- Download vae - - safetensors: https://huggingface.co/unsloth/LTX-2.3-GGUF/tree/main/vae -- Download audio vae - - safetensors: https://huggingface.co/unsloth/LTX-2.3-GGUF/tree/main/vae -- Download LTX spatial latent upscaler - - safetensors: https://huggingface.co/Lightricks/LTX-2.3/resolve/main/ltx-2.3-spatial-upscaler-x2-1.1.safetensors - -### LTX-2.5 - -- Download LTX-2.5 - - safetensors: https://huggingface.co/Lightricks/LTX-2.5/tree/main/diffusion_models - - gguf: https://huggingface.co/vantagewithai/LTX-2.5-GGUF/tree/main -- Download the text encoder. This is a Gemma 4 12B fine-tuned for LTX with the text - projection bundled in, so no separate `--embeddings-connectors` file is needed. Google's - stock Gemma 4 is not a substitute. - - safetensors: https://huggingface.co/Lightricks/LTX-2.5/blob/main/text_encoders/gemma4-12b-with-proj-ltx-2.5-bf16.safetensors -- Download the video vae. Use the **conv** variant: `ltx-2.5-video-vae-conv-bf16.safetensors`. - The default `ltx-2.5-video-vae-bf16.safetensors` is a diffusion decoder, which is not - implemented here. - - safetensors: https://huggingface.co/Lightricks/LTX-2.5/blob/main/vae/ltx-2.5-video-vae-conv-bf16.safetensors -- Download the audio vae - - safetensors: https://huggingface.co/Lightricks/LTX-2.5/blob/main/vae/ltx-2.5-audio-vae-bf16.safetensors -- Download the LTX spatial latent upscaler - - safetensors: https://huggingface.co/Lightricks/LTX-2.5/blob/main/latent_upscale_models/ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0.safetensors - -To run the text encoder quantized, convert it once with sd-cli: - -``` -.\bin\Release\sd-cli.exe -M convert -m ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-bf16.safetensors --type q8_0 -o ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf -``` - -## Examples - -### LTX-2.3 dev T2V - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\ltx-2.3-22b-dev-UD-Q4_K_M.gguf --vae ..\models\vae\ltx-2.3-22b-dev_video_vae.safetensors --audio-vae ..\models\vae\ltx-2.3-22b-dev_audio_vae.safetensors --llm ..\models\text_encoders\gemma-3-12b-it-qat-UD-Q4_K_XL.gguf --embeddings-connectors ..\models\text_encoders\ltx-2.3-22b-dev_embeddings_connectors.safetensors -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "worst quality, low quality, blurry, distorted, artifacts" -W 1280 -H 720 --offload-to-cpu --video-frames 33 --fps 24 -o t2v.webm -``` - - - -### LTX-2.3 dev I2V - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\ltx-2.3-22b-dev-UD-Q4_K_M.gguf --vae ..\models\vae\ltx-2.3-22b-dev_video_vae.safetensors --audio-vae ..\models\vae\ltx-2.3-22b-dev_audio_vae.safetensors --llm ..\models\text_encoders\gemma-3-12b-it-qat-UD-Q4_K_XL.gguf --embeddings-connectors ..\models\text_encoders\ltx-2.3-22b-dev_embeddings_connectors.safetensors -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -W 1280 -H 720 --offload-to-cpu --video-frames 33 -i ..\assets\ernie_image\turbo_example.png -o i2v.webm -``` - - - -### LTX-2.3 dev FLF2V - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\ltx-2.3-22b-dev-UD-Q4_K_M.gguf --vae ..\models\vae\ltx-2.3-22b-dev_video_vae.safetensors --audio-vae ..\models\vae\ltx-2.3-22b-dev_audio_vae.safetensors --llm ..\models\text_encoders\gemma-3-12b-it-qat-UD-Q4_K_XL.gguf --embeddings-connectors ..\models\text_encoders\ltx-2.3-22b-dev_embeddings_connectors.safetensors -p "glass flower blossom" --cfg-scale 6.0 --sampling-method euler -v -W 1280 -H 720 --offload-to-cpu --video-frames 33 --init-img ..\..\ComfyUI\input\start_image.png --end-img ..\..\ComfyUI\input\end_image.png -o flf2v.webm -``` - - - -### LTX-2.3 spatial latent upscale - -LTX spatial latent upscale runs a model-backed x2 latent upsampler between the low-resolution video pass and the high-resolution refine pass. `-W` and `-H` are the pre-upscale generation size; the spatial upsampler produces x2 latent dimensions. - -Put `ltx-2.3-spatial-upscaler-x2-1.1.safetensors` under the directory passed to `--hires-upscalers-dir`, then use the model name without path or extension in `--hires-upscaler`. - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\ltx-2.3-22b-dev-UD-Q4_K_M.gguf --vae ..\models\vae\ltx-2.3-22b-dev_video_vae.safetensors --audio-vae ..\models\vae\ltx-2.3-22b-dev_audio_vae.safetensors --llm ..\models\text_encoders\gemma-3-12b-it-qat-UD-Q4_K_XL.gguf --embeddings-connectors ..\models\text_encoders\ltx-2.3-22b-dev_embeddings_connectors.safetensors --hires-upscalers-dir ..\models\latent_upscale_models --hires-upscaler ltx-2.3-spatial-upscaler-x2-1.1 --hires --hires-steps 4 -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -W 640 -H 360 --offload-to-cpu --video-frames 33 -i ..\assets\ernie_image\turbo_example.png -o hires_i2v.webm -``` - -By default, the hires refine pass uses the main sampler and scheduler, then trims the second-pass sigma schedule by `--hires-denoising-strength` (`0.7` by default). To reproduce a ComfyUI-style explicit refine schedule, pass custom hires sigmas: - -``` ---hires-sigmas "0.85,0.725,0.421875,0.0" -``` - - - -### LTX-2.5 dev T2V - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\ltx-2.5-22b-dev-transformer-Q8_0.gguf --vae ..\models\vae\ltx-2.5-video-vae-conv-bf16.safetensors --audio-vae ..\models\vae\ltx-2.5-audio-vae-bf16.safetensors --llm ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf -p "A wide aerial shot of a red vintage convertible driving along a coastal cliff road at sunset, waves crashing below" --cfg-scale 3.0 --sampling-method euler -v -n "worst quality, low quality, blurry, distorted, artifacts" -W 1280 -H 720 --offload-to-cpu --video-frames 121 --fps 24 -o t2v.webm -``` - -### LTX-2.5 dev I2V - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\ltx-2.5-22b-dev-transformer-Q8_0.gguf --vae ..\models\vae\ltx-2.5-video-vae-conv-bf16.safetensors --audio-vae ..\models\vae\ltx-2.5-audio-vae-bf16.safetensors --llm ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf -p "a lovely cat blinking slowly, gentle camera push in" --cfg-scale 3.0 --sampling-method euler -v -W 1280 -H 720 --offload-to-cpu --video-frames 121 -i ..\assets\ernie_image\turbo_example.png -o i2v.webm -``` - -### LTX-2.5 spatial latent upscale - -Works exactly like the LTX-2.3 upscaler described below; put -`ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0.safetensors` under `--hires-upscalers-dir` and -pass its name without path or extension to `--hires-upscaler`. - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\ltx-2.5-22b-dev-transformer-Q8_0.gguf --vae ..\models\vae\ltx-2.5-video-vae-conv-bf16.safetensors --audio-vae ..\models\vae\ltx-2.5-audio-vae-bf16.safetensors --llm ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf --hires-upscalers-dir ..\models\latent_upscale_models --hires-upscaler ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0 --hires --hires-steps 6 -p "a lovely cat" --cfg-scale 3.0 --sampling-method euler -v -W 640 -H 360 --offload-to-cpu --video-frames 121 -o hires_t2v.webm -``` - -## Not implemented - -- The diffusion video decoder (`ltx-2.5-video-vae-bf16.safetensors`). Use the conv VAE. -- The temporal latent upscaler and the duration head (`--auto-duration`); pass - `--video-frames` explicitly. \ No newline at end of file diff --git a/docs/mage_flow.md b/docs/mage_flow.md deleted file mode 100644 index 0adef4afb6..0000000000 --- a/docs/mage_flow.md +++ /dev/null @@ -1,33 +0,0 @@ -# Mage-Flow - -[Mage-Flow](https://github.com/microsoft/Mage) uses a 4B native-resolution multimodal diffusion transformer, Qwen3-VL for text and image conditioning, and the 128-channel Mage-VAE. Both text-to-image and instruction-based image editing checkpoints are supported. - -## Download weights - -- Download Mage-Flow diffusion from https://huggingface.co/Comfy-Org/Mage-Flow/tree/main/diffusion_models -- Download Mage-Flow vae from https://huggingface.co/Comfy-Org/Mage-Flow/tree/main/vae -- Download Qwen3-VL 4B - - safetensors: https://huggingface.co/Comfy-Org/Krea-2/tree/main/text_encoders - - gguf: https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct-GGUF/tree/main - -## Text-to-image - -Use 30 steps for Base models and 4 steps with `--cfg-scale 1` for Turbo models. Image dimensions must be multiples of 16; the official checkpoints are trained for native resolutions from 512 to 2048 pixels. - -```bash -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\Mage-Flow-Turbo.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --vae ..\models\vae\mage_vae.safetensors -p "a lovely cat holding a sign says 'mage.cpp'" --cfg-scale 1.0 --steps 4 -v --offload-to-cpu -``` - -Mage-Flow example - -## Image editing - -Mage-Flow-Edit accepts one or more reference images. The default `mage_flow` reference preset sends each image to both Qwen3-VL and the diffusion transformer, caps the VLM copy's longest edge at 384 pixels, and keeps the VAE copy at the requested output resolution. - -For the Turbo edit checkpoint, use 4 steps and `--cfg-scale 1`. - -```bash -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\Mage-Flow-Edit.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --llm_vision ..\models\text_encoders\Qwen3-VL-4B-Instruct-mmproj-BF16.gguf --vae ..\models\vae\mage_vae.safetensors -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'mage.cpp'" --cfg-scale 4.0 --sampling-method euler -v --offload-to-cpu -``` - -Mage-Flow-Edit example diff --git a/docs/ming_image.md b/docs/ming_image.md deleted file mode 100644 index c242559212..0000000000 --- a/docs/ming_image.md +++ /dev/null @@ -1,24 +0,0 @@ -# Ming-Image - -[Ming-Image](https://github.com/inclusionAI/Ming-Image) 0.1 Design uses a 6B diffusion transformer (DiT), Ling-mini-2.0 for text conditioning, and the Ming-Image VAE. Text-to-image generation with RGBA output is supported. - -## Download weights - -- Download Ming-Image 0.1 Design DiT - - safetensors: https://huggingface.co/Comfy-Org/Ming-Image/tree/main/diffusion_models -- Download Ling-mini-2.0 BF16 - - safetensors: https://huggingface.co/Comfy-Org/Ming-Image/tree/main/text_encoders -- Download Ming-Image VAE - - safetensors: https://huggingface.co/Comfy-Org/Ming-Image/tree/main/vae -- Download Ling tokenizer - - tokenizer.json: https://huggingface.co/inclusionAI/Ming-Image-0.1-Design/blob/main/mllm/tokenizer.json - -The example below uses `ming_image_0.1_design_bf16.safetensors` for the DiT. You can also use `ming_image_0.1_design_int8_convrot.safetensors` with [INT8 convrot support](int8_convrot.md). Use the BF16 text encoder. - -## Text-to-image - -Pass the Ling `tokenizer.json` with `--tokenizer` and use the matching Ming-Image VAE. Image dimensions must be multiples of 16. Save the output as PNG to preserve the alpha channel. - -```bash -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\ming_image_0.1_design_bf16.safetensors --llm ..\models\text_encoders\ming_image_0.1_ling_mini_2.0_bf16.safetensors --vae ..\models\vae\ming_image_vae_bf16.safetensors --tokenizer ..\models\text_encoders\tokenizer.json -p "A cheerful orange cat sticker, transparent background" --width 1024 --height 1024 --steps 12 --cfg-scale 1 --sampling-method euler -v --offload-to-cpu -o ming_image.png -``` diff --git a/docs/minimax_h3.md b/docs/minimax_h3.md deleted file mode 100644 index 6b672ba99f..0000000000 --- a/docs/minimax_h3.md +++ /dev/null @@ -1,204 +0,0 @@ -# MiniMax-H3 - -MiniMax-H3 jointly generates video and stereo audio with a packed diffusion -transformer. The implementation supports text-to-audio-video (T2VA), optional -first-frame conditioning (I2VA), first/last-frame conditioning (FL2VA), and -image/video/audio reference conditioning (Ref2VA). - -## Model files - -Pass the four MiniMax-H3 components separately: - -- `--diffusion-model`: MiniMax-H3 diffusion transformer -- `--vae`: MiniMax-H3 video VAE -- `--audio-vae`: MiniMax-H3 audio VAE -- `--llm`: the MiniMax-H3 Qwen3-VL-32B text encoder checkpoint - -The text encoder must be the MiniMax-H3 variant: Qwen3-VL-32B truncated to 50 -language layers and exported without the final language-model normalization. -Its Qwen3-VL vision tower, including the three DeepStack mergers, must also be -present. If the vision tower is stored separately, pass it with `--llm_vision`. - -Both the original time-embedder DiT and the smaller AdaLN curve-table variant -are detected from their weights. - -### Download weights - -- Download minimax_h3_fl2va/minimax_h3_ref2va - - safetensors: https://huggingface.co/Comfy-Org/MiniMax-H3/tree/main/diffusion_models - - gguf: https://huggingface.co/leejet/MiniMax-H3-GGUF/tree/main -- Download qwen3vl_32b_minimax_h3 - - safetensors: https://huggingface.co/Comfy-Org/MiniMax-H3/tree/main/text_encoders - - gguf: https://huggingface.co/leejet/MiniMax-H3-GGUF/tree/main -- Download vae - - safetensors: https://huggingface.co/Comfy-Org/MiniMax-H3/tree/main/vae -- Download audio vae - - safetensors: https://huggingface.co/Comfy-Org/MiniMax-H3/tree/main/vae - -## Text-to-audio-video - -```sh -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\minimax_h3_fl2va-Q4_K_M.gguf --vae ..\models\vae\minimax_h3_video_vae_fp16.safetensors --audio-vae ..\models\vae\minimax_h3_audio_vae_fp32.safetensors --llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf -p "A cute American Shorthair silver tabby kitten surfs on a tropical ocean wave, riding a white surfboard with the clear text 'sd.cpp' on it. Cinematic tracking shot, realistic water, bright sunlight, smooth motion, and consistent character appearance. Add upbeat tropical surf-rock background music with cheerful drums and guitar, synchronized with the kitten’s energetic surfing." --cfg-scale 1.0 -v -W 864 -H 480 --offload-to-cpu --rng cpu --fps 24 --video-frames 56 -``` - - - -Omitting `--audio-vae` still runs the joint diffusion model but produces video without a -decoded audio track. - -## First/last-frame conditioning - -Add `--init-img` for I2VA, or both `--init-img` and `--end-img` for FL2VA: - -```sh -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\minimax_h3_fl2va-Q4_K_M.gguf --vae ..\models\vae\minimax_h3_video_vae_fp16.safetensors --audio-vae ..\models\vae\minimax_h3_audio_vae_fp32.safetensors --llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf -p "a lovely cat" -i ..\assets\ernie_image\turbo_example.png --cfg-scale 1.0 -v -W 864 -H 480 --offload-to-cpu --rng cpu --fps 24 --video-frames 56 -``` - - - -## Reference-to-audio-video conditioning - -Ref2VA accepts any combination of reference images, reference videos, paired -video soundtracks, and standalone audio references: - -```sh -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\minimax_h3_ref2va_pruned-Q4_K_M.gguf --vae ..\models\vae\minimax_h3_video_vae_fp16.safetensors --audio-vae ..\models\vae\minimax_h3_audio_vae_fp32.safetensors --llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf -p "Use the cat from as the main character. Keep the cat’s appearance, fur color, facial features, and identity consistent with the reference image. Create a 2-second cinematic video: start with an extreme close-up shot of the cat’s face, focusing on its cute expression and detailed fur texture. The camera slowly rotates around the cat’s head, creating a dynamic reveal. Then smoothly pull back and zoom out to reveal the full scene: the cat is standing confidently on a surfboard, riding ocean waves. Water splashes around the board, sea breeze gently moves the cat’s fur, and the cat maintains a cute and fearless expression while surfing. Smooth camera movement, cinematic orbit shot, seamless zoom-out transition, low-angle wide shot, realistic ocean environment, golden sunlight, dynamic waves, high-quality realistic style, natural motion, no distortion, keep the cat’s identity unchanged." -r ..\assets\ernie_image\turbo_example.png --cfg-scale 1.0 -v -W 864 -H 480 --offload-to-cpu --rng cpu --fps 24 --video-frames 56 -``` - - - -`--ref-image`, `--ref-video`, and `--ref-audio` can each be repeated. A -reference video is a directory of image frames sorted lexicographically and is -treated as 24 fps. Repeated `--ref-video-audio` WAV files are paired by index -with repeated `--ref-video` inputs. WAV PCM (8/16/24/32-bit) and 32/64-bit -floating-point samples are accepted; audio is converted to stereo 32 kHz by the -pipeline. - -Reference inputs are presented to Qwen3-VL in image, video, then audio order. -Videos are sampled at 2 fps for the Qwen presentation while their full 24 fps -latents condition the diffusion transformer. Paired video and audio references -share the same timeline. Ref2VA cannot be combined with `--init-img` or -`--end-img` in one request. - -Reference images keep their aspect ratio and are only downscaled when they -exceed the configured limit; neither mode scales a reference up. The limit is -selected by `ref_image_size`. The default, `match`, uses the pixel area of the -requested generation canvas. Pass `--ref-image-args "ref_image_size=max"` (or -`ref_image_size` in `sd_vid_gen_params_t.ref_image_args`) to use a 2048 px short -edge instead, which preserves more identity detail at a higher sampling cost: -reference latents are attended at every sampling step. - -Whichever limit applies, each axis is then snapped to the nearest 32-pixel -multiple. An axis of at least 16 pixels can therefore grow by up to 16 pixels -past the scaled size (a 48-pixel axis becomes 64), and references are never -narrower than 32 pixels. -`resize_before_vae=false` is not supported here; MiniMax-H3 keeps references on -the 32-pixel grid and reports a warning. - -The C API exposes the same inputs through `ref_images`, `ref_videos`, and -`ref_audios` in `sd_vid_gen_params_t`. Each `sd_ref_video_t` supplies its own -frame rate and optional soundtrack; non-24-fps inputs are resampled internally. - -## Sol-H3-VAE decoding - -Pass a [Sol-H3-VAE checkpoint](https://huggingface.co/Comfy-Org/MiniMax-H3/tree/main/vae) -with `--vae` to use the distilled video decoder. The ComfyUI FP16 and INT8 -ConvRot checkpoints include the MiniMax-H3 encoder for image and video inputs. -Supply `--audio-vae` as usual to decode the soundtrack. - -Sol decodes the whole clip by default. Use `--vae-tiling` for spatial tiling or -`--temporal-tiling` for temporal tiling. Either can reduce memory use but may -affect output quality because the decoder has less context at tile boundaries. - -Temporal tiles default to 10 latent frames with 5 latent frames of overlap. -Set `temporal_tile_frames` and `temporal_tile_overlap` in -`--extra-tiling-args` to override them. Both must be multiples of 5 latent -frames, and overlap must be smaller than the tile. The last tile may be shorter. -Circular decoding requires spatial tiling on each circular axis, with a tile -size smaller than the input on that axis. - -## Latent hires second pass (two-stage sampling) - -MiniMax-H3 supports `--hires` with two interchangeable second-stage upscalers: - -- an interpolating latent upscaler (`Latent`, `Latent (nearest)`, - `Latent (bicubic)`, ...): after the first pass finishes, the packed AV latent - is resampled in latent space to `--hires-width` x `--hires-height` and - refined by a second sampling pass; -- a model-backed upscaler (`--hires-upscaler ` + `--hires-upscalers-dir`): - a H3-shaped learned latent upscaler checkpoint runs between the passes, - like the LTX spatial upsampler. This is the sd.cpp equivalent of the - two-sampler workflows used for H3 in node-based frontends (e.g. the - community 3D-conv latent upscaler from - [LBH-123-AI/Minimax_h3_latent_Upscaler](https://huggingface.co/LBH-123-AI/Minimax_h3_latent_Upscaler)). - -Draft at a low megapixel count (e.g. 0.4 MP) and refine at a higher one -(e.g. 0.8 MP) instead of paying for the full resolution in every step: - -```sh -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\minimax_h3_fl2va-Q4_K_M.gguf --vae ..\models\vae\minimax_h3_video_vae_fp16.safetensors --audio-vae ..\models\vae\minimax_h3_audio_vae_fp32.safetensors --llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf -p "a lovely cat" --cfg-scale 1.0 -v -W 864 -H 480 --diffusion-fa --offload-to-cpu --rng cpu --video-frames 56 --hires --hires-upscaler "Latent (bicubic)" --hires-width 1216 --hires-height 672 --hires-steps 4 --hires-denoising-strength 0.4 -``` - -or with the learned upscaler: - -```sh -.\bin\Release\sd-cli.exe -M vid_gen ... --hires --hires-upscalers-dir ..\models\latent_upscale_models --hires-upscaler h3-latent-upscaler-3d --hires-width 1216 --hires-height 672 --hires-steps 4 --hires-denoising-strength 0.4 -``` - -Notes for this mode: - -- The learned upscaler expects a second normalization of the diffusion latent, - matching its training convention. The runner applies and reverses this extra - transform internally; callers should pass the usual H3 diffusion latents. -- The frame count is never changed by the second stage, so the packed audio - latent length stays valid and the generated soundtrack is preserved. -- Width and height still have to be multiples of 32; the hires target is - aligned like any other resolution. The learned upscaler additionally wants - an even latent size on both axes, which 32-pixel targets provide. -- `--hires-sigmas` can replace `--hires-denoising-strength` when the second - stage should resume from an explicit sigma schedule, which is how most - two-stage H3 workflows split the sampler. -- Keyframe conditioning (`--init-img` / `--end-img`) is not available together - with hires yet: those latents are encoded at the first stage resolution and - would have to be re-encoded for the hires canvas. Use T2VA or Ref2VA. -- The second stage runs at the larger latent, so it is the memory-hungry half; - combine it with quantization, `--vae-tiling` and `--offload-to-cpu` on - small cards. -- The learned upscaler is scale-conditional: the target-to-source ratio is - fed into the network as an embedding, 1.2-2.0x is the trained domain, and - downscaling is rejected. - -## Shape and runtime notes - -- Width and height are aligned upward to a multiple of 32. -- Frame count is aligned upward to the `17k + 5` grid, with a minimum of 5. -- MiniMax-H3 runs at 24 fps; another requested value is overridden. -- Attention head grouping is controlled by - `--model-args minimax_h3_attn_chunk_budget=2`, in GiB (default: `2`). - The budget estimates the size of a single FP32 Q/K tensor per group: - `head_dim * tokens * batch * 4 bytes * heads_per_group`. The group size adapts - to the sequence length, with a minimum of one head; inputs that fit use all - heads together. Set `0` to disable grouping, or a smaller budget such as - `0.25` to reduce peak activation memory for long sequences at the cost of - extra launches and concatenation. Each head still attends to the full sequence. - This is a grouping target, not a cap on total attention memory or VRAM usage; - other intermediates, backend workspace, and weights require additional memory. - `--max-vram` and `--offload-to-cpu` control weight residency, but the active - block's working memory must also fit within the VRAM budget. -- The default video flow shift is 12. The audio stream is mapped internally to - its shift of 3, so the regular samplers can operate on the packed AV latent. - -## HyperFlow H3 Flash - -[HyperFlow H3 Flash](https://huggingface.co/VideoRebirthAI/hyperflow-h3-flash) -uses the shared components and input options described in this guide. Replace -`--diffusion-model` with the matching HyperFlow FL2VA or Ref2VA checkpoint and -use these sampling options: - -```sh ---sampling-method euler --cfg-scale 1 --sigmas hyperflow-h3-flash-6 -``` - -Use `hyperflow-h3-flash-4` or `hyperflow-h3-flash-8` for 4 or 8 steps. These -presets also work with `--hires-sigmas`; their length determines the step count. -They already include video flow shift 12, so keep the default flow shift. diff --git a/docs/minimax_h3_image.md b/docs/minimax_h3_image.md deleted file mode 100644 index 0355663d26..0000000000 --- a/docs/minimax_h3_image.md +++ /dev/null @@ -1,65 +0,0 @@ -# MiniMax-H3-Image - -MiniMax-H3-Image supports text-to-image generation and reference-image editing -with the same checkpoint. It reuses the MiniMax-H3 text encoder and uses a -dedicated image VAE. - -## Model files - -Pass the three MiniMax-H3-Image components separately: - -- `--diffusion-model`: MiniMax-H3-Image diffusion transformer -- `--vae`: MiniMax-H3-Image VAE -- `--llm`: the MiniMax-H3 Qwen3-VL-32B text encoder checkpoint - -Use the MiniMax-H3 text encoder, including its Qwen3-VL vision tower and -DeepStack mergers. An existing MiniMax-H3 text encoder can be reused. If the -vision tower is stored separately, pass it with `--llm_vision`. - -Use a single-file diffusion checkpoint with the six-step PDD output heads -included, such as `minimax_h3_image_pruned_bf16.safetensors` or its INT8 ConvRot -version. Use the MiniMax-H3-Image VAE; the MiniMax-H3 video VAE and P-H3-Image -VAE are incompatible. No audio VAE or P-H3 text connector is needed. - -### Download weights - -- Download minimax_h3_image_pruned - - safetensors: https://huggingface.co/Comfy-Org/MiniMax-H3-Image/tree/main/diffusion_models -- Download qwen3vl_32b_minimax_h3 - - safetensors: https://huggingface.co/Comfy-Org/MiniMax-H3-Image/tree/main/text_encoders - - gguf: https://huggingface.co/leejet/MiniMax-H3-GGUF/tree/main -- Download vae - - safetensors: https://huggingface.co/Comfy-Org/MiniMax-H3-Image/tree/main/vae - -## Text-to-image - -```sh -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\minimax_h3_image_pruned_bf16.safetensors --vae ..\models\vae\minimax_h3_image_vae_fp16.safetensors --llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf -p "A silver tabby kitten holding a sign reading 'sd.cpp', natural daylight" --cfg-scale 1.0 --steps 6 --sampling-method euler --sigmas minimax-h3-image -v -W 1024 -H 1024 --offload-to-cpu --rng cpu --diffusion-fa --vae-tiling -``` - -Use Euler, CFG 1, and `--sigmas minimax-h3-image` for the model's six-step -schedule. The default discrete scheduler also produces this schedule with -`--steps 6`. Prompt expansion is not included. - -## Reference-image editing - -Add `--ref-image` (or `-r`) for reference-image editing: - -```sh -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\minimax_h3_image_pruned_bf16.safetensors --vae ..\models\vae\minimax_h3_image_vae_fp16.safetensors --llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf -p "Edit . Put a red scarf on the cat, preserving its face and the background." -r ..\assets\ernie_image\turbo_example.png --cfg-scale 1.0 --steps 6 --sampling-method euler --sigmas minimax-h3-image -v -W 1024 -H 1024 --offload-to-cpu --rng cpu --diffusion-fa --vae-tiling -``` - -`--ref-image` can be repeated. Reference images are numbered in argument order -as ``, ``, and so on, with up to 16 references supported -by the model. Prompts are passed through without an automatic editing prefix. - -Reference images keep their aspect ratio. The text encoder sees a 1024-pixel -short edge, while VAE inputs use a short edge between 1024 and 2048 pixels. -Small references are enlarged and dimensions are rounded to multiples of 32. - -## Shape and runtime notes - -- Width and height are aligned upward to a multiple of 32. -- Start at 1024 × 1024. Larger canvases and more references increase memory use. -- Use quantized weights, `--offload-to-cpu`, and `--vae-tiling` to reduce GPU - memory requirements. diff --git a/docs/minit2i.md b/docs/minit2i.md deleted file mode 100644 index 9f813a962f..0000000000 --- a/docs/minit2i.md +++ /dev/null @@ -1,47 +0,0 @@ -# How to Use - -MiniT2I uses a MiniT2I diffusion transformer and `google/flan-t5-large` as the text encoder. - -## Download weights - -- Download MiniT2I diffusion model - - safetensors: https://huggingface.co/MiniT2I/MiniT2I/tree/main/minit2i-b-16/transformer (`diffusion_pytorch_model.safetensors`) -- Download flan-t5-large text encoder - - safetensors: https://huggingface.co/google/flan-t5-large/tree/main (`model.safetensors`) - -## Examples - -### Mac Metal - -``` -./bin/sd-cli \ - --backend metal \ - --diffusion-model ../models/minit2i/diffusion_pytorch_model.safetensors \ - --t5xxl ../models/flan-t5-large/model.safetensors \ - --prompt "a cat" \ - --steps 100 \ - --cfg-scale 6 \ - --width 512 \ - --height 512 \ - --seed 42 \ - --sampling-method euler \ - --rng cpu \ - --output minit2i_metal.png \ - --threads 8 -``` - -### CUDA with diffusion flash attention - -``` -./bin/sd-cli \ - --diffusion-model ../models/minit2i/diffusion_pytorch_model.safetensors \ - --t5xxl ../models/flan-t5-large/model.safetensors \ - --prompt "a cat" \ - --steps 100 \ - --cfg-scale 6 \ - --width 512 \ - --height 512 \ - --seed 42 \ - --sampling-method euler \ - --output minit2i_cuda.png -``` diff --git a/docs/models/README.md b/docs/models/README.md new file mode 100644 index 0000000000..24da05cee8 --- /dev/null +++ b/docs/models/README.md @@ -0,0 +1,120 @@ +# Model and component guides + +Each guide covers weights, setup, and runnable examples for a model or component. +For shared generation options, device configuration, and troubleshooting, see the +[documentation index](../README.md). + +When adding model support, follow the +[model documentation requirements](../../CONTRIBUTING.md#model-documentation) +for guide structure, examples, and index updates. + +## Example conventions + +Run the executable examples from the CMake build directory, for example after +`cd build` from the repository root. Each guide gives both command forms: + +| Platform | CLI path from the build directory | Server path | +| --- | --- | --- | +| Windows, Visual Studio Release | `.\bin\Release\sd-cli.exe` | `.\bin\Release\sd-server.exe` | +| Linux/macOS | `./bin/sd-cli` | `./bin/sd-server` | + +Windows examples use PowerShell backticks for line continuation. Linux/macOS +examples use POSIX shell backslashes. Backend-specific examples identify their +requirements; Metal examples apply to macOS only. A different CMake generator, +build configuration, or extracted release archive can use another executable +location; substitute that path while keeping the arguments. + +Keep weights under the repository's `models/` directory, organized into the +categories described in the server guide. From the build directory these paths +start with `../models/` on Linux/macOS and `..\models\` on Windows. + +Provide input images, masks, audio, or frame directories at the example paths, +or change the corresponding arguments. Paths under `../inputs/` refer to local +inputs you supply. Filenames in commands identify the example weights; adjust +paths to match your downloads. Conversion and preprocessing scripts identify +when they must be run from the source repository root instead. + +The [server Web UI guide](../../examples/server/README.md#web-ui) covers file +placement, server startup, model loading, inputs, and generation. Each model +guide lists its compatible recipes and components. Choose files from the same +variant; a VAE or TAE selection must match that model. + +## Image generation + +- [Anima](anima.md) +- [Boogu Image](boogu_image.md) +- [Chroma](chroma.md) +- [Chroma1-Radiance](chroma_radiance.md) +- [Distilled SD and SDXL models](distilled_sd.md) +- [ERNIE-Image](ernie_image.md) +- [Fibo 2.0](fibo2.md) +- [FLUX.1](flux.md) +- [FLUX.2](flux2.md) +- [FLUX.3 Image](flux3.md) +- [HiDream-O1-Image](hidream_o1_image.md) +- [Ideogram4](ideogram4.md) +- [Iris-3B](iris.md) +- [Krea2](krea2.md) +- [Lens](lens.md) +- [LLaDA-Image](llada_image.md) +- [LongCat Image](longcat_image.md) +- [Mage-Flow](mage_flow.md) +- [Ming-Image Design](ming_image.md) +- [MiniMax-H3-Image](minimax_h3_image.md) +- [MiniT2I](minit2i.md) +- [MOSS-Image](moss_image.md) +- [Ovis-Image](ovis_image.md) +- [P-H3-Image](p_h3_image.md) +- [PiD](pid.md) +- [PixArt](pixart.md) +- [Qwen Image](qwen_image.md) +- [Qwen Image 2.1](qwen_image_2.1.md) +- [SeFi-Image](sefi_image.md) +- [SenseNova U1.5](sensenova_u1.md) +- [Stable Diffusion 1.x, 2.x, and SDXL, including Turbo variants](sd.md) +- [Stable Diffusion 3 / 3.5](sd3.md) +- [Z-Image](z_image.md) +- [Z-Image L2P](z_image_l2p.md) + +## Image editing + +See the [image editing guide](../guides/edit.md) for shared reference-image options +and model presets. Models that support both generation and editing use the same +model guide. + +- [Boogu Image Edit](boogu_image.md#boogu-image-edit) +- [Fibo 2.0 Dev / Edit Turbo](fibo2.md#fibo-20-edit-turbo) +- [FLUX.1-Kontext-dev](kontext.md) +- [FLUX.2](flux2.md) +- [FLUX.3 Image](flux3.md) +- [LLaDA-Image editing](llada_image.md#image-editing) +- [LongCat Image Edit](longcat_image.md) +- [Mage-Flow-Edit](mage_flow.md#image-editing) +- [MiniMax-H3-Image reference-image editing](minimax_h3_image.md#reference-image-editing) +- [Qwen Image 2.1 editing](qwen_image_2.1.md#image-editing) +- [Qwen Image Edit series](qwen_image_edit.md) + +## Video generation + +- [HunyuanVideo 1.5](hunyuan_video.md) +- [LingBot-Video](lingbot_video.md) +- [LTX-2.3 / LTX-2.5](ltx2.md) +- [MiniMax-H3](minimax_h3.md) +- [Wan2.1 / Wan2.2](wan.md) + +## Image and video super-resolution + +- [ESRGAN and MoSR](esrgan.md) +- [RelayVSR](relay_vsr.md) +- [SeedVR2](seedvr2.md) + +## LoRAs, adapters, and auxiliary models + +- [ADetailer](adetailer.md) +- [AnimateDiff](animatediff.md) +- [IP-Adapter](ip_adapter.md) +- [LCM / LCM-LoRA](lcm.md) +- [LoRA](lora.md) +- [PhotoMaker](photo_maker.md) +- [PuLID-Flux](pulid.md) +- [TAESD and related tiny autoencoders](taesd.md) diff --git a/docs/adetailer.md b/docs/models/adetailer.md similarity index 75% rename from docs/adetailer.md rename to docs/models/adetailer.md index f42831823c..2072644beb 100644 --- a/docs/adetailer.md +++ b/docs/models/adetailer.md @@ -6,13 +6,29 @@ inpaint pass for every detected object. The implementation supports YOLOv8 detection checkpoints. YOLOv8 segmentation and MediaPipe models are not supported yet. -## Convert a detector +## Model files -Ultralytics checkpoints must be converted before use. The converter fuses -BatchNorm into convolution layers and writes a safetensors file with the weight -names expected by the native GGML implementation. +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Base image model | [Compatible image-generation model](README.md) | `-m` / model component options | Required for the inpainting pass | +| YOLOv8 detector | Converted Safetensors checkpoint prepared below | `--ad-model` | Required; detection checkpoints only | -```bash +### Prepare the detector + +Run the conversion command from the source repository root, then place the +converted file at the path used by `--ad-model`. + +Ultralytics checkpoints must be converted before use. The converter writes the Safetensors detector used by sd.cpp. + +**Windows (PowerShell)** + +```powershell +python scripts\convert_yolov8_to_safetensors.py face_yolov8n.pt face_yolov8n.safetensors +``` + +**Linux/macOS** + +```sh python scripts/convert_yolov8_to_safetensors.py face_yolov8n.pt face_yolov8n.safetensors ``` @@ -22,17 +38,44 @@ Only YOLOv8 detection checkpoints are accepted. PyTorch checkpoints use pickle internally, so only convert `.pt` files from a trusted source. -## Repair an existing image +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### CLI + +#### Repair an existing image Use the dedicated `adetailer` mode to detect and repair objects in an existing image: -```bash +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M adetailer ` + -m ..\models\checkpoints\model.safetensors ` + --ad-model ..\models\detectors\face_yolov8n.safetensors ` + -i input.png ` + -p "detailed portrait photo" ` + --negative-prompt "deformed face" ` + --steps 24 ` + --cfg-scale 6 ` + --strength 0.4 ` + --sampling-method dpm++2m ` + --scheduler karras ` + --extra-ad-args "confidence=0.3,inpaint_padding=32,mask_blur=4" ` + -o repaired.png +``` + +**Linux/macOS** + +```sh ./bin/sd-cli \ -M adetailer \ - -m model.safetensors \ + -m ../models/checkpoints/model.safetensors \ + --ad-model ../models/detectors/face_yolov8n.safetensors \ -i input.png \ - -o repaired.png \ -p "detailed portrait photo" \ --negative-prompt "deformed face" \ --steps 24 \ @@ -40,8 +83,8 @@ image: --strength 0.4 \ --sampling-method dpm++2m \ --scheduler karras \ - --ad-model face_yolov8n.safetensors \ - --extra-ad-args "confidence=0.3,inpaint_padding=32,mask_blur=4" + --extra-ad-args confidence=0.3,inpaint_padding=32,mask_blur=4 \ + -o repaired.png ``` This mode reuses the normal image-generation options for the detail pass: @@ -55,18 +98,32 @@ This mode reuses the normal image-generation options for the detail pass: Values provided in `--extra-ad-args`, such as `steps`, `cfg_scale`, `denoising_strength`, or `inpaint_width`, take precedence over inherited values. -## Repair generated images +#### Repair generated images ADetailer can also run automatically after normal image generation: -```bash +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -m ..\models\checkpoints\model.safetensors ` + --ad-model ..\models\detectors\face_yolov8n.safetensors ` + -p "portrait photo" ` + --ad-prompt "[PROMPT], detailed face" ` + --ad-negative-prompt "" ` + --extra-ad-args "confidence=0.3,denoising_strength=0.4,inpaint_width=512,inpaint_height=512" +``` + +**Linux/macOS** + +```sh ./bin/sd-cli \ - -m model.safetensors \ + -m ../models/checkpoints/model.safetensors \ + --ad-model ../models/detectors/face_yolov8n.safetensors \ -p "portrait photo" \ - --ad-model face_yolov8n.safetensors \ --ad-prompt "[PROMPT], detailed face" \ --ad-negative-prompt "" \ - --extra-ad-args "confidence=0.3,denoising_strength=0.4,inpaint_width=512,inpaint_height=512" + --extra-ad-args confidence=0.3,denoising_strength=0.4,inpaint_width=512,inpaint_height=512 ``` The same options can be supplied when starting `sd-server`. When `--ad-model` is set, its detector runs after OpenAI-compatible and native asynchronous image generation. SDAPI requests enable ADetailer through `alwayson_scripts`, as described below. If no object is detected, the original image is returned unchanged. RGBA output keeps its original alpha channel, including when detected regions are repaired. Detector initialization or detail-pass failures are reported as request failures. @@ -99,6 +156,8 @@ Supported settings include prompts, confidence, mask filtering by area or confid Model/VAE switching, non-default noise multipliers, face restoration, ControlNet, and skipping the initial img2img pass are not supported; active requests for these features return an error. Unknown settings and invalid active values also return errors. During detailing, `/sdapi/v1/progress` remains active with an `adetailer` job; its percentage and preview stay fixed until detailing finishes. +## Options + An empty ADetailer prompt inherits the main prompt. `[PROMPT]` inserts the main prompt, `[SEP]` assigns different prompts to consecutive masks, and `[SKIP]` skips the corresponding mask. @@ -138,6 +197,19 @@ for the next mask, and the seed is incremented by the mask index. Use The detector uses the `detector` backend module. For example, keep detection on the CPU while diffusion runs on CUDA: -```bash +**Windows (PowerShell) arguments** + +```text --backend "diffusion=cuda0,detector=cpu" ``` + +**Linux/macOS arguments** + +```text +--backend diffusion=cuda0,detector=cpu +``` + +## Notes and limitations + +YOLOv8 segmentation and MediaPipe detectors are not supported. See the +[server API reference](../../examples/server/api.md) for request schemas. diff --git a/docs/models/anima.md b/docs/models/anima.md new file mode 100644 index 0000000000..38f19f612a --- /dev/null +++ b/docs/models/anima.md @@ -0,0 +1,62 @@ +# Anima + +Anima generates images from text using a Qwen3 text encoder and a compatible Qwen Image VAE. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Anima | [Safetensors](https://huggingface.co/circlestone-labs/Anima/tree/main/split_files/diffusion_models)
[GGUF](https://huggingface.co/Bedovyy/Anima-GGUF/tree/main)
gguf Anima2: [GGUF](https://huggingface.co/JusteLeo/Anima2-GGUF/tree/main) | `--diffusion-model` | Required | +| vae | [Safetensors](https://huggingface.co/circlestone-labs/Anima/tree/main/split_files/vae) | `--vae` | Use matching VAE | +| Qwen3-0.6B-Base | [Safetensors](https://huggingface.co/circlestone-labs/Anima/tree/main/split_files/text_encoders)
[GGUF](https://huggingface.co/mradermacher/Qwen3-0.6B-Base-GGUF/tree/main) | `--llm` | Use matching text encoder | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| Anima (`anima`) | `diffusion_model`, `llm`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\anima-preview.safetensors ` + --vae ..\models\vae\qwen_image_vae.safetensors ` + --llm ..\models\text_encoders\qwen_3_06b_base.safetensors ` + -p "a lovely cat holding a sign says 'anima.cpp'" ` + --cfg-scale 6.0 ` + --sampling-method euler ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/anima-preview.safetensors \ + --vae ../models/vae/qwen_image_vae.safetensors \ + --llm ../models/text_encoders/qwen_3_06b_base.safetensors \ + -p "a lovely cat holding a sign says 'anima.cpp'" \ + --cfg-scale 6.0 \ + --sampling-method euler \ + --offload-to-cpu \ + -v +``` + +anima image example diff --git a/docs/models/animatediff.md b/docs/models/animatediff.md new file mode 100644 index 0000000000..ef110febcc --- /dev/null +++ b/docs/models/animatediff.md @@ -0,0 +1,341 @@ +# AnimateDiff + +AnimateDiff adds video generation to an SD 1.5 checkpoint using a compatible motion module. + +Use the SD 1.5 checkpoint for the desired characters and style, and choose the +motion module and resolution below. See the +[AnimateDiff paper](https://arxiv.org/abs/2307.04725) for the model reference. + +## Supported variants + +Per [animatediff.net/models](https://animatediff.net/models): + +| Module | Base | Native res | Character | +|---------------------|------|------------|-----------| +| `mm_sd_v14.ckpt` | 1.5 | 256x256 | earliest, more jittery | +| `mm_sd_v15.ckpt` | 1.5 | 256x256 | improved stability over v1.4 | +| `mm_sd_v15_v2.ckpt` | 1.5 | 384x384 | significantly better motion dynamics | +| `v3_sd15_mm.ckpt` | 1.5 | 512x512 | smoothest, highest quality; pairs with a Domain Adapter LoRA | +| `mm_sdxl_v10_beta` | SDXL | 512x512 | experimental, not yet supported here | + +Match your `-H -W` to the module's native resolution for best results. v3 is +trained at 512x512 - going smaller (e.g. 384x384) still works but the motion +character is closer to v2. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| SD 1.5 checkpoint | [Compatible SD 1.5 checkpoint](sd.md) | `-m` / `--model` | Required; SDXL motion modules are not supported | +| Motion module | v1/v2/v3 motion weights listed below | `--motion-module` | Required; match the resolution and version | +| Domain adapter LoRA | v3 adapter listed below | `--lora-model-dir` and prompt | Optional; v3 only | + +- Motion module (v3, recommended) + - fp16 safetensors: https://huggingface.co/conrevo/AnimateDiff-A1111/resolve/main/motion_module/mm_sd15_v3.safetensors + - original checkpoint: https://huggingface.co/guoyww/animatediff/resolve/main/v3_sd15_mm.ckpt +- SD 1.5 base model + - any SD 1.5 checkpoint works. `realisticVisionV60B1` and `toonyou_beta3` + are the ones used in guoyww's reference configs. +- Domain Adapter LoRA (optional, v3 only, sharpens the base UNet's output + toward the motion module's trained distribution) + - ckpt: https://huggingface.co/guoyww/animatediff/resolve/main/v3_sd15_adapter.ckpt + - place under your `--lora-model-dir` and reference in the prompt as + ``. + +The motion module is `~836 MB` and loads alongside the SD 1.5 UNet via +`--motion-module`. + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| AnimateDiff SD 1.5 (`animatediff-sd15`) | `model`, `motion_module`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### Text-to-video + +Generate an 8-frame animation at 512x512, seed 42, 20 steps. The sampler / +scheduler / CFG values below match what mm_sd15_v3 was trained with; using +SD 1.5 defaults (euler_a, low CFG) produces noise-like output. + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --model ..\models\checkpoints\realisticVisionV60B1.safetensors ` + --motion-module ..\models\motion_modules\animatediff\mm_sd15_v3.safetensors ` + -p "a red apple on a wooden table" ` + --cfg-scale 8.0 ` + --sampling-method euler ` + --scheduler discrete ` + --steps 20 ` + -H 512 ` + -W 512 ` + --video-frames 8 ` + --fps 8 ` + --offload-to-cpu ` + -s 42 ` + -o out.avi +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --model ../models/checkpoints/realisticVisionV60B1.safetensors \ + --motion-module ../models/motion_modules/animatediff/mm_sd15_v3.safetensors \ + -p "a red apple on a wooden table" \ + --cfg-scale 8.0 \ + --sampling-method euler \ + --scheduler discrete \ + --steps 20 \ + -H 512 \ + -W 512 \ + --video-frames 8 \ + --fps 8 \ + --offload-to-cpu \ + -s 42 \ + -o out.avi +``` + +Generate at the motion module's native 16-frame context (recommended for +best temporal quality). Needs more VRAM at 512x512, so drop to 384x384 or +use layer streaming: + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --model ..\models\checkpoints\realisticVisionV60B1.safetensors ` + --motion-module ..\models\motion_modules\animatediff\mm_sd15_v3.safetensors ` + -p "photo of coastline, rocks, storm weather, wind, waves, lightning" ` + --cfg-scale 8.0 ` + --sampling-method euler ` + --scheduler discrete ` + --steps 20 ` + -H 384 ` + -W 384 ` + --video-frames 16 ` + --fps 8 ` + --offload-to-cpu ` + -s 42 ` + -o out.avi +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --model ../models/checkpoints/realisticVisionV60B1.safetensors \ + --motion-module ../models/motion_modules/animatediff/mm_sd15_v3.safetensors \ + -p "photo of coastline, rocks, storm weather, wind, waves, lightning" \ + --cfg-scale 8.0 \ + --sampling-method euler \ + --scheduler discrete \ + --steps 20 \ + -H 384 \ + -W 384 \ + --video-frames 16 \ + --fps 8 \ + --offload-to-cpu \ + -s 42 \ + -o out.avi +``` + +Low-VRAM streaming (verified with a 2 GiB cap on RTX 3060): + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --model ..\models\checkpoints\realisticVisionV60B1.safetensors ` + --motion-module ..\models\motion_modules\animatediff\mm_sd15_v3.safetensors ` + -p "photo of coastline, rocks, storm weather, wind, waves, lightning" ` + --cfg-scale 8.0 ` + --sampling-method euler ` + --scheduler discrete ` + --steps 20 ` + -H 384 ` + -W 384 ` + --video-frames 8 ` + --fps 8 ` + --max-vram 2.0 ` + -s 42 ` + -o out.avi +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --model ../models/checkpoints/realisticVisionV60B1.safetensors \ + --motion-module ../models/motion_modules/animatediff/mm_sd15_v3.safetensors \ + -p "photo of coastline, rocks, storm weather, wind, waves, lightning" \ + --cfg-scale 8.0 \ + --sampling-method euler \ + --scheduler discrete \ + --steps 20 \ + -H 384 \ + -W 384 \ + --video-frames 8 \ + --fps 8 \ + --max-vram 2.0 \ + -s 42 \ + -o out.avi +``` + +#### Reference-quality reproduction + +Using guoyww's official reference configs on this impl (RealisticVision v6.0 +base + `mm_sd15_v3` or `mm_sd_v15_v2` + native resolution + 16 frames + euler ++ 25 steps + CFG 8 + linear beta schedule) reproduces the reference +AnimateDiff output style. + +##### v3 (512x512, `mm_sd15_v3`) + +| Prompt | Sample | +|---------------------------------------|--------| +| B&W man on stormy coastline | | +| Close-up rabbit macro shot | | +| Coastline, storm, waves, lightning | | +| Old house, storm, forest, night | | + +##### v2 (384x384, `mm_sd_v15_v2.ckpt`) + +| Prompt | Sample | +|---------------------------------------|--------| +| B&W man on stormy coastline | | +| Close-up rabbit macro shot | | +| Coastline, storm, waves, lightning | | +| Old house, storm, forest, night | | + +Motion is strong for scenes with motion cues in the prompt (storm/waves/wind) +and subtle for static subjects (close-up macro), matching reference behavior. +v2 has an additional motion module at the UNet middle block that v3 dropped; +this impl auto-detects the topology from the checkpoint. + +##### v3 + Domain Adapter LoRA + +Attaching the `v3_sd15_adapter` LoRA sharpens the base UNet output toward +the training distribution the motion module was fine-tuned against. Same +prompt, seed, config as above: + + + +Individual fur strands, glowing inner-ear, and richer forest detail become +visible compared to the no-LoRA rendering. + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --model ..\models\checkpoints\realisticVisionV60B1.safetensors ` + --motion-module ..\models\motion_modules\mm_sd15_v3.safetensors ` + --lora-model-dir .\loras ` + -p "close up photo of a rabbit ..." ... +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --model ../models/checkpoints/realisticVisionV60B1.safetensors \ + --motion-module ../models/motion_modules/mm_sd15_v3.safetensors \ + --lora-model-dir ./loras \ + -p "close up photo of a rabbit ..." ... +``` + +#### img2video + +Pass a pre-rendered image via `-i / --init-img` to animate FROM it. All N output frames start from the encoded init latent, then per-frame noise is added at `--strength`. Character identity, composition, and quality are anchored by the init image; the motion module adds subtle motion on top. + +Left: init image rendered with `-M img_gen`. Right: 8-frame vid_gen output. + + + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M img_gen ... ` + -o init.png "#" any high-quality still +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --motion-module ..\models\motion_modules\mm_sd15_v3.safetensors ` + -i init.png ` + -p ... ` + --strength 0.75 ` + --cfg-scale 7.0 ` + --sampling-method euler ` + --scheduler karras ` + --steps 25 ` + -H 512 ` + -W 512 ` + --video-frames 8 ` + -s 42 ` + -o out.avi +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M img_gen ... \ + -o init.png "#" any high-quality still +./bin/sd-cli \ + -M vid_gen \ + --motion-module ../models/motion_modules/mm_sd15_v3.safetensors \ + -i init.png \ + -p ... \ + --strength 0.75 \ + --cfg-scale 7.0 \ + --sampling-method euler \ + --scheduler karras \ + --steps 25 \ + -H 512 \ + -W 512 \ + --video-frames 8 \ + -s 42 \ + -o out.avi +``` + +`--strength` controls how far the motion module is allowed to deviate from the init image (higher = more motion, lower = more static). + +## Notes and limitations + +- The motion module was trained at `video_length=16`. Running with + `--video-frames 16` gives the best coherence; F=8 works but shows a shorter + motion arc. Frame counts up to 32 are supported by the positional encoding + but exceed the trained regime and produce more static output. +- At `--video-frames 1` the motion module is skipped entirely and the output + is bit-identical to `-M img_gen`. This avoids the single-token + temporal-attention degeneracy that would otherwise emit an untrained-magnitude + residual on a single-frame sample. +- The base UNet is frozen, so character identity and style follow the SD 1.5 + checkpoint you pass to `--model`. LoRAs and prompt weighting attach to the + base model in the usual way. +- No mid_block motion module in v3. `mm_sdxl_v10_beta` (SDXL variant) is not + supported yet. +- Output is written as MJPEG AVI. Use `--fps` to set playback speed. diff --git a/docs/models/boogu_image.md b/docs/models/boogu_image.md new file mode 100644 index 0000000000..d7d133cd3f --- /dev/null +++ b/docs/models/boogu_image.md @@ -0,0 +1,93 @@ +# Boogu Image + +Boogu Image uses a Boogu diffusion transformer, the FLUX VAE, and Qwen3-VL as the LLM text and vision encoder. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Boogu Image | [Safetensors](https://huggingface.co/Comfy-Org/Boogu-Image/tree/main/diffusion_models) | `--diffusion-model` | Required | +| vae | [Safetensors](https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/ae.safetensors) | `--vae` | Use matching VAE | +| Qwen3-VL 8B | [GGUF](https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/tree/main)
For image editing with GGUF text encoders, also download the matching mmproj file and pass it with `--llm_vision`. | `--llm` | Use matching text encoder | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| Boogu Image Base (`boogu-image`) | `diffusion_model`, `llm`, `vae`, `tae` | +| Boogu Image Edit (`boogu-image-edit`) | `diffusion_model`, `llm`, `llm_vision`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### Boogu Image Base + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\boogu_image_base_bf16.safetensors ` + --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf ` + --vae ..\models\vae\ae.sft ` + -p "a lovely cat" ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/boogu_image_base_bf16.safetensors \ + --llm ../models/text_encoders/Qwen3VL-8B-Instruct-Q4_K_M.gguf \ + --vae ../models/vae/ae.sft \ + -p "a lovely cat" \ + --offload-to-cpu \ + -v +``` + +Boogu Image Base example + +#### Boogu Image Edit + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\boogu_image_edit_bf16.safetensors ` + --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf ` + --llm_vision ..\models\text_encoders\mmproj-Qwen3VL-8B-Instruct-F16.gguf ` + --vae ..\models\vae\ae.sft ` + -r ..\inputs\flux\flux1-dev-q8_0.png ` + -p "change 'flux.cpp' to 'boogu.cpp'" ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/boogu_image_edit_bf16.safetensors \ + --llm ../models/text_encoders/Qwen3VL-8B-Instruct-Q4_K_M.gguf \ + --llm_vision ../models/text_encoders/mmproj-Qwen3VL-8B-Instruct-F16.gguf \ + --vae ../models/vae/ae.sft \ + -r ../inputs/flux/flux1-dev-q8_0.png \ + -p "change 'flux.cpp' to 'boogu.cpp'" \ + --offload-to-cpu \ + -v +``` + +Boogu Image Edit example diff --git a/docs/models/chroma.md b/docs/models/chroma.md new file mode 100644 index 0000000000..51c9685981 --- /dev/null +++ b/docs/models/chroma.md @@ -0,0 +1,101 @@ +# Chroma + +Chroma generates images from text using T5-XXL and the FLUX.1 VAE. + +You can run Chroma using stable-diffusion.cpp with a GPU that has 6GB or even 4GB of VRAM, without needing to offload to RAM. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Chroma | If you don't want to do the conversion yourself, download the preconverted gguf model from [silveroxides/Chroma-GGUF](https://huggingface.co/silveroxides/Chroma-GGUF)
Otherwise, download chroma's safetensors from [lodestones/Chroma](https://huggingface.co/lodestones/Chroma) | `--diffusion-model` | Required | +| vae | vae from [Download](https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/ae.safetensors) | `--vae` | Use matching VAE | +| t5xxl | t5xxl from [Download](https://huggingface.co/comfyanonymous/flux_text_encoders/blob/main/t5xxl_fp16.safetensors) | `--t5xxl` | Required | + +### Optional conversion + +Run executable conversion examples from the build directory using the +[shared path and shell conventions](README.md#example-conventions). + +#### Convert Chroma weights + +You can download the preconverted gguf weights from [silveroxides/Chroma-GGUF](https://huggingface.co/silveroxides/Chroma-GGUF), this way you don't have to do the conversion yourself. + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M convert ` + -m ..\models\unet\chroma-unlocked-v40.safetensors ` + --type q8_0 ` + -o ..\models\diffusion_models\chroma-unlocked-v40-q8_0.gguf ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M convert \ + -m ../models/unet/chroma-unlocked-v40.safetensors \ + --type q8_0 \ + -o ../models/diffusion_models/chroma-unlocked-v40-q8_0.gguf \ + -v +``` + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| Chroma (`chroma`) | `diffusion_model`, `t5xxl`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### Example + +For example: + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\chroma-unlocked-v40-q8_0.gguf ` + --vae ..\models\vae\ae.sft ` + --t5xxl ..\models\text_encoders\t5xxl_fp16.safetensors ` + --model-args chroma_use_dit_mask=false ` + -p "a lovely cat holding a sign says 'chroma.cpp'" ` + --cfg-scale 4.0 ` + --sampling-method euler ` + --clip-on-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/chroma-unlocked-v40-q8_0.gguf \ + --vae ../models/vae/ae.sft \ + --t5xxl ../models/text_encoders/t5xxl_fp16.safetensors \ + --model-args chroma_use_dit_mask=false \ + -p "a lovely cat holding a sign says 'chroma.cpp'" \ + --cfg-scale 4.0 \ + --sampling-method euler \ + --clip-on-cpu \ + -v +``` + +![](../../assets/flux/chroma_v40.png) diff --git a/docs/models/chroma_radiance.md b/docs/models/chroma_radiance.md new file mode 100644 index 0000000000..dbb8ad5744 --- /dev/null +++ b/docs/models/chroma_radiance.md @@ -0,0 +1,57 @@ +# Chroma1-Radiance + +Chroma1-Radiance generates images directly in pixel space using T5-XXL. No VAE is required. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Chroma1-Radiance | [Safetensors](https://huggingface.co/lodestones/Chroma1-Radiance/tree/main)
[GGUF](https://huggingface.co/silveroxides/Chroma1-Radiance-GGUF/tree/main) | `--diffusion-model` | Required | +| t5xxl | [Safetensors](https://huggingface.co/comfyanonymous/flux_text_encoders/blob/main/t5xxl_fp16.safetensors) | `--t5xxl` | Required | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| Chroma Radiance (`chroma-radiance`) | `diffusion_model`, `t5xxl` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\Chroma1-Radiance-v0.4-Q8_0.gguf ` + --t5xxl ..\models\text_encoders\clip\t5xxl_fp16.safetensors ` + -p "a lovely cat holding a sign says 'chroma radiance cpp'" ` + --cfg-scale 4.0 ` + --sampling-method euler ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/Chroma1-Radiance-v0.4-Q8_0.gguf \ + --t5xxl ../models/text_encoders/clip/t5xxl_fp16.safetensors \ + -p "a lovely cat holding a sign says 'chroma radiance cpp'" \ + --cfg-scale 4.0 \ + --sampling-method euler \ + -v +``` + +Chroma1-Radiance diff --git a/docs/models/distilled_sd.md b/docs/models/distilled_sd.md new file mode 100644 index 0000000000..30e75aa4bc --- /dev/null +++ b/docs/models/distilled_sd.md @@ -0,0 +1,202 @@ +# Distilled Stable Diffusion models + +This guide covers SSD-1B, Vega, Tiny SD/SD2, and SDXS checkpoints. Choose the +base-model family and preparation steps matching the checkpoint. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| SSD-1B | [SSD-1B-A1111.safetensors](https://huggingface.co/segmind/SSD-1B/resolve/main/SSD-1B-A1111.safetensors) | `-m` / `--model` | Single-file SDXL-family checkpoint | +| Vega | [Segmind Vega](https://huggingface.co/segmind/Segmind-Vega) | `-m` / `--model` | Single-file SDXL-family checkpoint | +| Tiny SD / SD2 | [Repositories and conversion steps](#tiny-sd-and-tiny-sd2) | `-m` / `--model` | Convert Diffusers directories first | +| SDXS | [Ready-to-run SDXS weights](#sdxs-512-dreamshaper-weights) | `-m` / `--model` | Use the compatible autoencoder and one-step settings | + +### SSD1B + +Note that not all of these models follow the standard parameter naming conventions. However, several useful SSD-1B models are available online, such as: + + * https://huggingface.co/segmind/SSD-1B/resolve/main/SSD-1B-A1111.safetensors + * https://huggingface.co/hassenhamdi/SSD-1B-fp8_e4m3fn/resolve/main/SSD-1B_fp8_e4m3fn.safetensors + +Useful LoRAs are also available: + + * https://huggingface.co/seungminh/lora-swarovski-SSD-1B/resolve/main/pytorch_lora_weights.safetensors + * https://huggingface.co/kylielee505/mylcmlorassd/resolve/main/pytorch_lora_weights.safetensors + +### Vega + +Segmind's Vega model is available online here: + + * https://huggingface.co/segmind/Segmind-Vega/resolve/main/segmind-vega.safetensors + +VegaRT is an example for an LCM-LoRA: + + * https://huggingface.co/segmind/Segmind-VegaRT/resolve/main/pytorch_lora_weights.safetensors + +Both files can be used out-of-the-box, unlike the models described in next sections. + +### Tiny SD and Tiny SD2 + +These models require conversion before use. You will need a Python script provided by the diffusers team, available on GitHub: + + * https://raw.githubusercontent.com/huggingface/diffusers/refs/heads/main/scripts/convert_diffusers_to_original_stable_diffusion.py + +#### SD2.x + +NotaAI provides the following model online: + +* https://huggingface.co/nota-ai/bk-sdm-v2-tiny + +Creating a .safetensors file involves two steps. First, run this short Python script to download the model from Hugging Face: + +```python +from diffusers import StableDiffusionPipeline +pipe = StableDiffusionPipeline.from_pretrained("nota-ai/bk-sdm-v2-tiny",cache_dir="./") +``` + +Second, create the .safetensors file by running: + +**Windows (PowerShell)** + +```powershell +python convert_diffusers_to_original_stable_diffusion.py --model_path models--nota-ai--bk-sdm-v2-tiny/snapshots/68277af553777858cd47e133f92e4db47321bc74 --checkpoint_path bk-sdm-v2-tiny.safetensors --half --use_safetensors +``` + +**Linux/macOS** + +```sh +python convert_diffusers_to_original_stable_diffusion.py --model_path models--nota-ai--bk-sdm-v2-tiny/snapshots/68277af553777858cd47e133f92e4db47321bc74 --checkpoint_path bk-sdm-v2-tiny.safetensors --half --use_safetensors +``` + +This will generate the **file bk-sdm-v2-tiny.safetensors**, which is now ready for use with sd.cpp. + +#### SD1.x + +Several Tiny SD 1.x models are available online, such as: + + * https://huggingface.co/segmind/tiny-sd + * https://huggingface.co/segmind/portrait-finetuned + * https://huggingface.co/nota-ai/bk-sdm-tiny + +These models also require conversion, partly because some tensors are stored in a non-contiguous manner. To create a usable checkpoint file, follow these simple steps: +Download and prepare the model using Python: + +Download the model using Python on your computer, for example this way: + +```python +import torch +from diffusers import StableDiffusionPipeline +pipe = StableDiffusionPipeline.from_pretrained("segmind/tiny-sd") +unet=pipe.unet +for param in unet.parameters(): + param.data = param.data.contiguous() # <- important here +pipe.save_pretrained("segmindtiny-sd", safe_serialization=True) +``` + +Run the conversion script: + +**Windows (PowerShell)** + +```powershell +python convert_diffusers_to_original_stable_diffusion.py --model_path .\segmindtiny-sd --checkpoint_path .\segmind_tiny-sd.safetensors --half --use_safetensors +``` + +**Linux/macOS** + +```sh +python convert_diffusers_to_original_stable_diffusion.py --model_path ./segmindtiny-sd --checkpoint_path ./segmind_tiny-sd.safetensors --half --use_safetensors +``` + +The file segmind_tiny-sd.safetensors will be generated and is now ready for use with sd.cpp. You can follow a similar process for the other models mentioned above. + +### SDXS-512-DreamShaper weights + +Another very tiny and **incredibly fast** model is SDXS by IDKiro et al. The authors refer to it as *"Real-Time One-Step Latent Diffusion Models with Image Conditions"*. For details read the paper: https://arxiv.org/pdf/2403.16627 . Once again the authors removed some more blocks of U-Net part and unlike other SD1 models they use an adjusted _AutoEncoderTiny_ instead of default _AutoEncoderKL_ for the VAE part. + +Some ready-to-run SDXS-512 model files are available online, such as: + +* https://huggingface.co/akleine/sdxs-512 +* https://huggingface.co/concedo/sdxs-512-tinySDdistilled-GGUF + +### SDXS-512-0.9 weights + +Even though the name "SDXS-512-0.9" is similar to "SDXS-512-DreamShaper", it is *completely different* but also **incredibly fast**. Sometimes it is preferred, so try it yourself. + +Download a ready-to-run file from here: + +* https://huggingface.co/akleine/sdxs-09 + +For the use of this model, both options `--cfg-scale 1` and `--steps 1` are again absolutely necessary. + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| SDXL (`sdxl-checkpoint`) | `model`, `vae`, `tae` | +| SD 1.x (`sd1-checkpoint`) | `model`, `vae`, `tae` | +| SD 2.1 512 (`sd2-512-checkpoint`) | `model`, `vae`, `tae` | +| SDXS (`sdxs`) | `model`, `vae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### SSD1B + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe -m ..\models\checkpoints\SSD-1B-A1111.safetensors -p "a lovely cat" +``` + +**Linux/macOS** + +```sh +./bin/sd-cli -m ../models/checkpoints/SSD-1B-A1111.safetensors -p "a lovely cat" +``` + +#### SDXS-512-DreamShaper + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -m ..\models\checkpoints\sdxs.safetensors ` + -p "portrait of a lovely cat" ` + --cfg-scale 1 ` + --steps 1 +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -m ../models/checkpoints/sdxs.safetensors \ + -p "portrait of a lovely cat" \ + --cfg-scale 1 \ + --steps 1 +``` + +Both options: `--cfg-scale 1` and `--steps 1` are mandatory here. + +## Notes and limitations + +SSD-1B, Vega, and Tiny SD variants reduce the base model size. Details are in the +[SSD-1B paper](https://arxiv.org/abs/2401.02677v1) and +[Tiny SD paper](https://arxiv.org/pdf/2305.15798.pdf). Follow the [SDXS generation settings](#sdxs-512-dreamshaper); CFG 1 and one step are required. + +Run the Python conversion commands in [Model files](#model-files) from the source repository root, +with the listed conversion script and Python dependencies installed. Return to +the build directory for the platform-specific CLI commands. diff --git a/docs/models/ernie_image.md b/docs/models/ernie_image.md new file mode 100644 index 0000000000..e4e30698af --- /dev/null +++ b/docs/models/ernie_image.md @@ -0,0 +1,99 @@ +# ERNIE-Image + +ERNIE-Image and ERNIE-Image-Turbo generate images from text using the FLUX.2 VAE +and a Ministral 3 3B text encoder. + +You can run ERNIE-Image with stable-diffusion.cpp on GPUs with 4GB of VRAM — or even less. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| ERNIE-Image-Turbo | [Safetensors](https://huggingface.co/Comfy-Org/ERNIE-Image/tree/main/diffusion_models)
[GGUF](https://huggingface.co/unsloth/ERNIE-Image-Turbo-GGUF/tree/main) | `--diffusion-model` | Choose Turbo or Base | +| ERNIE-Image | [Safetensors](https://huggingface.co/Comfy-Org/ERNIE-Image/tree/main/diffusion_models)
[GGUF](https://huggingface.co/unsloth/ERNIE-Image-GGUF/tree/main) | `--diffusion-model` | Choose Turbo or Base | +| FLUX.2 VAE | [Safetensors](https://huggingface.co/Comfy-Org/ERNIE-Image/tree/main/vae) | `--vae` | Required for these examples | +| Ministral 3 3B text encoder | [Safetensors](https://huggingface.co/Comfy-Org/ERNIE-Image/tree/main/text_encoders)
[GGUF](https://huggingface.co/unsloth/Ministral-3-3B-Instruct-2512-GGUF/tree/main) | `--llm` | Required | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| ERNIE Image Turbo (`ernie-image-turbo`) | `diffusion_model`, `llm`, `vae`, `tae` | +| ERNIE Image (`ernie-image`) | `diffusion_model`, `llm`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### ERNIE-Image-Turbo + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\ernie-image-turbo.safetensors ` + --vae ..\models\vae\flux2_ae.safetensors ` + --llm ..\models\text_encoders\ministral-3-3b.safetensors ` + -p "a lovely cat" ` + --cfg-scale 1.0 ` + --steps 8 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/ernie-image-turbo.safetensors \ + --vae ../models/vae/flux2_ae.safetensors \ + --llm ../models/text_encoders/ministral-3-3b.safetensors \ + -p "a lovely cat" \ + --cfg-scale 1.0 \ + --steps 8 \ + --offload-to-cpu \ + -v +``` + +ERNIE-Image Turbo example + +#### ERNIE-Image + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\ernie-image-UD-Q4_K_M.gguf ` + --vae ..\models\vae\flux2_ae.safetensors ` + --llm ..\models\text_encoders\ministral-3-3b.safetensors ` + -p "a lovely cat" ` + --cfg-scale 5.0 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/ernie-image-UD-Q4_K_M.gguf \ + --vae ../models/vae/flux2_ae.safetensors \ + --llm ../models/text_encoders/ministral-3-3b.safetensors \ + -p "a lovely cat" \ + --cfg-scale 5.0 \ + --offload-to-cpu \ + -v +``` + +ERNIE-Image example diff --git a/docs/models/esrgan.md b/docs/models/esrgan.md new file mode 100644 index 0000000000..3d1a589e28 --- /dev/null +++ b/docs/models/esrgan.md @@ -0,0 +1,42 @@ +# ESRGAN and MoSR + +ESRGAN and compatible MoSR weights upscale generated images. Pass the chosen +upscaler with `--upscale-model`. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Base model | [Stable Diffusion checkpoint](sd.md) | `-m` / `--model` | Required for generation below | +| ESRGAN upscaler | [RealESRGAN_x4plus_anime_6B.pth](https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.2.4/RealESRGAN_x4plus_anime_6B.pth) | `--upscale-model` | Choose a compatible upscaler | +| MoSR upscaler | [4xNomos2_hq_mosr.pth](https://github.com/Phhofm/models/releases/download/4xNomos2_hq_mosr/4xNomos2_hq_mosr.pth) | `--upscale-model` | Alternative; requires the PixelShuffle upsampler | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### CLI + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -m ..\models\checkpoints\v1-5-pruned-emaonly.safetensors ` + --upscale-model ..\models\upscale_models\RealESRGAN_x4plus_anime_6B.pth ` + -p "a lovely cat" +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -m ../models/checkpoints/v1-5-pruned-emaonly.safetensors \ + --upscale-model ../models/upscale_models/RealESRGAN_x4plus_anime_6B.pth \ + -p "a lovely cat" +``` + +## Notes and limitations + +RGBA output retains its alpha channel during model upscaling and hires fix. +The upscaler processes RGB; alpha is resized with bilinear interpolation and +recombined with the result. The model architecture is detected from the weights. diff --git a/docs/models/fibo2.md b/docs/models/fibo2.md new file mode 100644 index 0000000000..4919bda126 --- /dev/null +++ b/docs/models/fibo2.md @@ -0,0 +1,154 @@ +# Fibo 2.0 + +Fibo 2.0 uses a Fibo diffusion transformer, the standard FLUX.2 VAE, and Qwen3-VL 4B as the LLM text encoder. The Qwen tokenizer is built in. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Fibo 2.0 Dev | [Safetensors](https://huggingface.co/Comfy-Org/Fibo-2.0/tree/main/diffusion_models) | `--diffusion-model` | Choose matching variant | +| Fibo 2.0 Edit Turbo | [Safetensors](https://huggingface.co/Comfy-Org/Fibo-2.0/tree/main/diffusion_models) | `--diffusion-model` | Choose matching variant | +| vae | [Safetensors](https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main) | `--vae` | Use matching VAE | +| Qwen3-VL 4B | [Safetensors](https://huggingface.co/Comfy-Org/Fibo-2.0/tree/main/text_encoders)
[GGUF](https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct-GGUF/tree/main) | `--llm` | Use matching text encoder | + +The original Diffusers weights from [briaai/Fibo-2.0-dev](https://huggingface.co/briaai/Fibo-2.0-dev) and [briaai/Fibo-2.0-edit-turbo](https://huggingface.co/briaai/Fibo-2.0-edit-turbo) are also supported. Pass the transformer's `diffusion_pytorch_model.safetensors.index.json` to `--diffusion-model` and keep its shards alongside it. + +INT8 convrot weights load directly; see [INT8 convrot](../guides/int8_convrot.md) for backend requirements. + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| Fibo 2.0 Dev (`fibo2-dev`) | `diffusion_model`, `llm`, `llm_vision`, `vae` | +| Fibo 2.0 Edit Turbo (`fibo2-edit-turbo`) | `diffusion_model`, `llm`, `llm_vision`, `vae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### Fibo 2.0 Dev + +Dev supports both text-to-image generation and reference-image editing. + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\fibo2_dev_int8_convrot.safetensors ` + --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf ` + --llm_vision ..\models\text_encoders\Qwen3-VL-4B-Instruct-mmproj-BF16.gguf ` + --vae ..\models\vae\flux2_ae.safetensors ` + -p "a lovely cat holding a sign says 'fibo2.cpp'" ` + --cfg-scale 5.0 ` + --sampling-method euler ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/fibo2_dev_int8_convrot.safetensors \ + --llm ../models/text_encoders/Qwen3-VL-4B-Instruct-Q4_K_M.gguf \ + --llm_vision ../models/text_encoders/Qwen3-VL-4B-Instruct-mmproj-BF16.gguf \ + --vae ../models/vae/flux2_ae.safetensors \ + -p "a lovely cat holding a sign says 'fibo2.cpp'" \ + --cfg-scale 5.0 \ + --sampling-method euler \ + --offload-to-cpu \ + -v +``` + + + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\fibo2_dev_int8_convrot.safetensors ` + --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf ` + --llm_vision ..\models\text_encoders\Qwen3-VL-4B-Instruct-mmproj-BF16.gguf ` + --vae ..\models\vae\flux2_ae.safetensors ` + -r ..\inputs\flux\flux1-dev-q8_0.png ` + -p "change 'flux.cpp' to 'fibo2.cpp'" ` + --cfg-scale 5.0 ` + --sampling-method euler ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/fibo2_dev_int8_convrot.safetensors \ + --llm ../models/text_encoders/Qwen3-VL-4B-Instruct-Q4_K_M.gguf \ + --llm_vision ../models/text_encoders/Qwen3-VL-4B-Instruct-mmproj-BF16.gguf \ + --vae ../models/vae/flux2_ae.safetensors \ + -r ../inputs/flux/flux1-dev-q8_0.png \ + -p "change 'flux.cpp' to 'fibo2.cpp'" \ + --cfg-scale 5.0 \ + --sampling-method euler \ + --offload-to-cpu \ + -v +``` + + + +#### Fibo 2.0 Edit Turbo + +Edit Turbo is distilled for reference-image editing in 4 steps with CFG 1. + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\fibo2_edit_turbo_int8_convrot.safetensors ` + --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf ` + --llm_vision ..\models\text_encoders\Qwen3-VL-4B-Instruct-mmproj-BF16.gguf ` + --vae ..\models\vae\flux2_ae.safetensors ` + -r ..\inputs\flux\flux1-dev-q8_0.png ` + -p "change 'flux.cpp' to 'fibo2.cpp'" ` + --cfg-scale 1 ` + --steps 4 ` + --sampling-method euler ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/fibo2_edit_turbo_int8_convrot.safetensors \ + --llm ../models/text_encoders/Qwen3-VL-4B-Instruct-Q4_K_M.gguf \ + --llm_vision ../models/text_encoders/Qwen3-VL-4B-Instruct-mmproj-BF16.gguf \ + --vae ../models/vae/flux2_ae.safetensors \ + -r ../inputs/flux/flux1-dev-q8_0.png \ + -p "change 'flux.cpp' to 'fibo2.cpp'" \ + --cfg-scale 1 \ + --steps 4 \ + --sampling-method euler \ + --offload-to-cpu \ + -v +``` + +The BF16 files accept the same options. Use dimensions divisible by 16. Repeat `-r` to use multiple reference images; instructions can refer to ``, ``, etc. See [Image Editing](../guides/edit.md) for reference-image preprocessing options. Separate GGUF text encoders need their vision projector passed with `--llm_vision` for editing. + +The automatic `fibo2` reference preset combines `resize_single_ref_to_target=true`, `vae_size_buckets=true`, `vae_center_crop=true`, and `vlm_use_vae_input=true`. A single reference is scaled to cover the output dimensions and center-cropped. Multiple references use dimensions divisible by 16, chosen within ±10% of one megapixel to minimize cropping for each image's aspect ratio; `vae_input_max_pixels` overrides that target area. The VAE and vision encoder receive the same crops, with smaller copies used by the vision encoder. These are generic reference-image options and can be overridden individually: for example, `--ref-image-args vae_center_crop=false` uses direct resizing, while `resize_before_vae=false` skips this automatic crop and resize step entirely. + +The vision encoder defaults to `vlm_resize_mode=longest_side,vlm_max_size=768`. Override these with `--ref-image-args`, for example `vlm_max_size=512`, or use `vlm_resize_mode=none` to skip size-based scaling. Vision inputs are still aligned to the encoder's patch grid. + +Plain prompts and structured JSON captions are accepted. This integration uses the caption-only text-rendering path (`render_text_images=False`); it does not generate auxiliary glyph images or expand prompts with Bria VGL. Native Fibo mask preprocessing is not connected to `--mask`. diff --git a/docs/models/flux.md b/docs/models/flux.md new file mode 100644 index 0000000000..dd67bbd67a --- /dev/null +++ b/docs/models/flux.md @@ -0,0 +1,191 @@ +# FLUX.1 + +FLUX.1 dev and schnell generate images from text using CLIP-L, T5-XXL, and the FLUX.1 VAE. + +You can run Flux using stable-diffusion.cpp with a GPU that has 6GB or even 4GB of VRAM, without needing to offload to RAM. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| flux | If you don't want to do the conversion yourself, download the preconverted gguf model from [FLUX.1-dev-gguf](https://huggingface.co/leejet/FLUX.1-dev-gguf) or [FLUX.1-schnell](https://huggingface.co/leejet/FLUX.1-schnell-gguf)
Otherwise, download flux-dev from [Download](https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/flux1-dev.safetensors) or flux-schnell from [Safetensors](https://huggingface.co/black-forest-labs/FLUX.1-schnell/blob/main/flux1-schnell.safetensors) | `--diffusion-model` | Required | +| vae | vae from [Download](https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/ae.safetensors) | `--vae` | Use matching VAE | +| clip_l | clip_l from [Download](https://huggingface.co/comfyanonymous/flux_text_encoders/blob/main/clip_l.safetensors) | `--clip_l` | Required | +| t5xxl | t5xxl from [Download](https://huggingface.co/comfyanonymous/flux_text_encoders/blob/main/t5xxl_fp16.safetensors) | `--t5xxl` | Required | + +### Optional conversion + +Run executable conversion examples from the build directory using the +[shared path and shell conventions](README.md#example-conventions). + +#### Convert flux weights + +You can download the preconverted gguf weights from [FLUX.1-dev-gguf](https://huggingface.co/leejet/FLUX.1-dev-gguf) or [FLUX.1-schnell](https://huggingface.co/leejet/FLUX.1-schnell-gguf), this way you don't have to do the conversion yourself. + +For example: +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M convert ` + -m ..\models\unet\flux1-dev.sft ` + --type q8_0 ` + -o ..\models\diffusion_models\flux1-dev-q8_0.gguf ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M convert \ + -m ../models/unet/flux1-dev.sft \ + --type q8_0 \ + -o ../models/diffusion_models/flux1-dev-q8_0.gguf \ + -v +``` + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| FLUX.1 dev (`flux1-dev`) | `diffusion_model`, `clip_l`, `t5xxl`, `vae`, `tae` | +| FLUX.1 schnell (`flux1-schnell`) | `diffusion_model`, `clip_l`, `t5xxl`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +- `--cfg-scale` is recommended to be set to 1. + +#### Flux-dev + +For example: + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\flux1-dev-q8_0.gguf ` + --vae ..\models\vae\ae.sft ` + --clip_l ..\models\text_encoders\clip_l.safetensors ` + --t5xxl ..\models\text_encoders\t5xxl_fp16.safetensors ` + -p "a lovely cat holding a sign says 'flux.cpp'" ` + --cfg-scale 1.0 ` + --sampling-method euler ` + --clip-on-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/flux1-dev-q8_0.gguf \ + --vae ../models/vae/ae.sft \ + --clip_l ../models/text_encoders/clip_l.safetensors \ + --t5xxl ../models/text_encoders/t5xxl_fp16.safetensors \ + -p "a lovely cat holding a sign says 'flux.cpp'" \ + --cfg-scale 1.0 \ + --sampling-method euler \ + --clip-on-cpu \ + -v +``` + +Using formats of different precisions will yield results of varying quality. + +| Type | q8_0 | q4_0 | q4_k | q3_k | q2_k | +|---- | ---- |---- |---- |---- |---- | +| **Memory** | 12068.09 MB | 6394.53 MB | 6395.17 MB | 4888.16 MB | 3735.73 MB | +| **Result** | ![](../../assets/flux/flux1-dev-q8_0.png) |![](../../assets/flux/flux1-dev-q4_0.png) |![](../../assets/flux/flux1-dev-q4_k.png) |![](../../assets/flux/flux1-dev-q3_k.png) |![](../../assets/flux/flux1-dev-q2_k.png)| + +#### Flux-schnell + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\flux1-schnell-q8_0.gguf ` + --vae ..\models\vae\ae.sft ` + --clip_l ..\models\text_encoders\clip_l.safetensors ` + --t5xxl ..\models\text_encoders\t5xxl_fp16.safetensors ` + -p "a lovely cat holding a sign says 'flux.cpp'" ` + --cfg-scale 1.0 ` + --sampling-method euler ` + --steps 4 ` + --clip-on-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/flux1-schnell-q8_0.gguf \ + --vae ../models/vae/ae.sft \ + --clip_l ../models/text_encoders/clip_l.safetensors \ + --t5xxl ../models/text_encoders/t5xxl_fp16.safetensors \ + -p "a lovely cat holding a sign says 'flux.cpp'" \ + --cfg-scale 1.0 \ + --sampling-method euler \ + --steps 4 \ + --clip-on-cpu \ + -v +``` + +| q8_0 | +| ---- | +|![](../../assets/flux/flux1-schnell-q8_0.png) | + +#### Run with LoRA + +Since many flux LoRA training libraries have used various LoRA naming formats, it is possible that not all flux LoRA naming formats are supported. It is recommended to use LoRA with naming formats compatible with ComfyUI. + +##### Flux-dev q8_0 with LoRA + +- LoRA model from https://huggingface.co/XLabs-AI/flux-lora-collection/tree/main (using comfy converted version!!!) + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\flux1-dev-q8_0.gguf ` + --vae ..\models\vae\ae.sft ` + --clip_l ..\models\text_encoders\clip_l.safetensors ` + --t5xxl ..\models\text_encoders\t5xxl_fp16.safetensors ` + --lora-model-dir ..\models\loras ` + -p "a lovely cat holding a sign says 'flux.cpp'" ` + --cfg-scale 1.0 ` + --sampling-method euler ` + --clip-on-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/flux1-dev-q8_0.gguf \ + --vae ../models/vae/ae.sft \ + --clip_l ../models/text_encoders/clip_l.safetensors \ + --t5xxl ../models/text_encoders/t5xxl_fp16.safetensors \ + --lora-model-dir ../models/loras \ + -p "a lovely cat holding a sign says 'flux.cpp'" \ + --cfg-scale 1.0 \ + --sampling-method euler \ + --clip-on-cpu \ + -v +``` + +![output](../../assets/flux/flux1-dev-q8_0%20with%20lora.png) diff --git a/docs/models/flux2.md b/docs/models/flux2.md new file mode 100644 index 0000000000..fdca1b2456 --- /dev/null +++ b/docs/models/flux2.md @@ -0,0 +1,285 @@ +# FLUX.2 + +FLUX.2 supports text-to-image generation and reference-image editing. Choose +the text encoder matching the transformer variant. + +## Model files + +### Flux.2-dev + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| FLUX.2 dev | [GGUF](https://huggingface.co/city96/FLUX.2-dev-gguf/tree/main) | `--diffusion-model` | Required for dev | +| FLUX.2 VAE | [Safetensors](https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main) | `--vae` | Choose one compatible VAE | +| FLUX.2 small decoder | [Safetensors](https://huggingface.co/black-forest-labs/FLUX.2-small-decoder/tree/main) | `--vae` | Alternative VAE | +| Mistral Small 3.2 24B | [GGUF](https://huggingface.co/unsloth/Mistral-Small-3.2-24B-Instruct-2506-GGUF/tree/main) | `--llm` | Required for dev | + +### Flux.2 klein 4B / Flux.2 klein base 4B + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| FLUX.2 Klein 4B | [Safetensors](https://huggingface.co/black-forest-labs/FLUX.2-klein-4B)
[GGUF](https://huggingface.co/leejet/FLUX.2-klein-4B-GGUF/tree/main) | `--diffusion-model` | Choose Klein or Klein Base | +| FLUX.2 Klein Base 4B | [Safetensors](https://huggingface.co/black-forest-labs/FLUX.2-klein-base-4B)
[GGUF](https://huggingface.co/leejet/FLUX.2-klein-base-4B-GGUF/tree/main) | `--diffusion-model` | Choose Klein or Klein Base | +| FLUX.2 VAE | [Safetensors](https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main) | `--vae` | Choose one compatible VAE | +| FLUX.2 small decoder | [Safetensors](https://huggingface.co/black-forest-labs/FLUX.2-small-decoder/tree/main) | `--vae` | Alternative VAE | +| Qwen3 4B | [Safetensors](https://huggingface.co/Comfy-Org/flux2-klein-4B/tree/main/split_files/text_encoders)
[GGUF](https://huggingface.co/unsloth/Qwen3-4B-GGUF/tree/main) | `--llm` | Required for 4B variants | + +### Flux.2 klein 9B / Flux.2 klein base 9B + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| FLUX.2 Klein 9B | [Safetensors](https://huggingface.co/black-forest-labs/FLUX.2-klein-9B)
[GGUF](https://huggingface.co/leejet/FLUX.2-klein-9B-GGUF/tree/main) | `--diffusion-model` | Choose Klein or Klein Base | +| FLUX.2 Klein Base 9B | [Safetensors](https://huggingface.co/black-forest-labs/FLUX.2-klein-base-9B)
[GGUF](https://huggingface.co/leejet/FLUX.2-klein-base-9B-GGUF/tree/main) | `--diffusion-model` | Choose Klein or Klein Base | +| FLUX.2 VAE | [Safetensors](https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main) | `--vae` | Required for these examples | +| Qwen3 8B | [Safetensors](https://huggingface.co/Comfy-Org/flux2-klein-9B/tree/main/split_files/text_encoders)
[GGUF](https://huggingface.co/unsloth/Qwen3-8B-GGUF/tree/main) | `--llm` | Required for 9B variants | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| FLUX.2 dev (`flux2-dev`) | `diffusion_model`, `llm`, `vae`, `tae` | +| FLUX.2 Klein 4B (`flux2-klein-4b`) | `diffusion_model`, `llm`, `vae`, `tae` | +| FLUX.2 Klein Base 4B (`flux2-klein-base-4b`) | `diffusion_model`, `llm`, `vae`, `tae` | +| FLUX.2 Klein 9B (`flux2-klein-9b`) | `diffusion_model`, `llm`, `vae`, `tae` | +| FLUX.2 Klein Base 9B (`flux2-klein-base-9b`) | `diffusion_model`, `llm`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### Flux.2-dev + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\flux2-dev-Q4_K_S.gguf ` + --vae ..\models\vae\flux2_ae.safetensors ` + --llm ..\models\text_encoders\Mistral-Small-3.2-24B-Instruct-2506-Q4_K_M.gguf ` + -r .\kontext_input.png ` + -p "change 'flux.cpp' to 'flux2-dev.cpp'" ` + --cfg-scale 1.0 ` + --sampling-method euler ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/flux2-dev-Q4_K_S.gguf \ + --vae ../models/vae/flux2_ae.safetensors \ + --llm ../models/text_encoders/Mistral-Small-3.2-24B-Instruct-2506-Q4_K_M.gguf \ + -r ./kontext_input.png \ + -p "change 'flux.cpp' to 'flux2-dev.cpp'" \ + --cfg-scale 1.0 \ + --sampling-method euler \ + --offload-to-cpu \ + -v +``` + +flux2 example + +#### Flux.2 klein 4B / Flux.2 klein base 4B + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\flux-2-klein-4b.safetensors ` + --vae ..\models\vae\flux2_ae.safetensors ` + --llm ..\models\text_encoders\qwen_3_4b.safetensors ` + -p "a lovely cat" ` + --cfg-scale 1.0 ` + --steps 4 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/flux-2-klein-4b.safetensors \ + --vae ../models/vae/flux2_ae.safetensors \ + --llm ../models/text_encoders/qwen_3_4b.safetensors \ + -p "a lovely cat" \ + --cfg-scale 1.0 \ + --steps 4 \ + --offload-to-cpu \ + -v +``` + +flux2-klein-4b + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\flux-2-klein-4b.safetensors ` + --vae ..\models\vae\flux2_ae.safetensors ` + --llm ..\models\text_encoders\qwen_3_4b.safetensors ` + -r .\kontext_input.png ` + -p "change 'flux.cpp' to 'klein.cpp'" ` + --cfg-scale 1.0 ` + --sampling-method euler ` + --steps 4 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/flux-2-klein-4b.safetensors \ + --vae ../models/vae/flux2_ae.safetensors \ + --llm ../models/text_encoders/qwen_3_4b.safetensors \ + -r ./kontext_input.png \ + -p "change 'flux.cpp' to 'klein.cpp'" \ + --cfg-scale 1.0 \ + --sampling-method euler \ + --steps 4 \ + --offload-to-cpu \ + -v +``` + +flux2-klein-4b-edit + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\flux-2-klein-base-4b.safetensors ` + --vae ..\models\vae\flux2_ae.safetensors ` + --llm ..\models\text_encoders\qwen_3_4b.safetensors ` + -p "a lovely cat" ` + --cfg-scale 4.0 ` + --steps 20 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/flux-2-klein-base-4b.safetensors \ + --vae ../models/vae/flux2_ae.safetensors \ + --llm ../models/text_encoders/qwen_3_4b.safetensors \ + -p "a lovely cat" \ + --cfg-scale 4.0 \ + --steps 20 \ + --offload-to-cpu \ + -v +``` + +flux2-klein-base-4b + +#### Flux.2 klein 9B / Flux.2 klein base 9B + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\flux-2-klein-9b.safetensors ` + --vae ..\models\vae\flux2_ae.safetensors ` + --llm ..\models\text_encoders\qwen_3_8b.safetensors ` + -p "a lovely cat" ` + --cfg-scale 1.0 ` + --steps 4 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/flux-2-klein-9b.safetensors \ + --vae ../models/vae/flux2_ae.safetensors \ + --llm ../models/text_encoders/qwen_3_8b.safetensors \ + -p "a lovely cat" \ + --cfg-scale 1.0 \ + --steps 4 \ + --offload-to-cpu \ + -v +``` + +flux2-klein-9b + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\flux-2-klein-9b.safetensors ` + --vae ..\models\vae\flux2_ae.safetensors ` + --llm ..\models\text_encoders\qwen_3_8b.safetensors ` + -r .\kontext_input.png ` + -p "change 'flux.cpp' to 'klein.cpp'" ` + --cfg-scale 1.0 ` + --sampling-method euler ` + --steps 4 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/flux-2-klein-9b.safetensors \ + --vae ../models/vae/flux2_ae.safetensors \ + --llm ../models/text_encoders/qwen_3_8b.safetensors \ + -r ./kontext_input.png \ + -p "change 'flux.cpp' to 'klein.cpp'" \ + --cfg-scale 1.0 \ + --sampling-method euler \ + --steps 4 \ + --offload-to-cpu \ + -v +``` + +flux2-klein-9b-edit + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\flux-2-klein-base-9b.safetensors ` + --vae ..\models\vae\flux2_ae.safetensors ` + --llm ..\models\text_encoders\qwen_3_8b.safetensors ` + -p "a lovely cat" ` + --cfg-scale 4.0 ` + --steps 20 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/flux-2-klein-base-9b.safetensors \ + --vae ../models/vae/flux2_ae.safetensors \ + --llm ../models/text_encoders/qwen_3_8b.safetensors \ + -p "a lovely cat" \ + --cfg-scale 4.0 \ + --steps 20 \ + --offload-to-cpu \ + -v +``` + +flux2-klein-base-9b diff --git a/docs/models/flux3.md b/docs/models/flux3.md new file mode 100644 index 0000000000..7591d7cd13 --- /dev/null +++ b/docs/models/flux3.md @@ -0,0 +1,144 @@ +# FLUX.3 Image + +FLUX.3 Image Klein uses a FLUX.3 diffusion transformer, the FLUX.2 VAE, and Qwen3-VL 4B as the LLM text encoder. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| FLUX.3 Image Klein | [Safetensors](https://huggingface.co/black-forest-labs/flux-3-image-klein/tree/main)
[GGUF](https://huggingface.co/leejet/flux-3-image-klein-GGUF/tree/main) | `--diffusion-model` | Choose matching variant | +| FLUX.3 Image Klein Base | [Safetensors](https://huggingface.co/black-forest-labs/flux-3-image-klein-base/tree/main)
[GGUF](https://huggingface.co/leejet/flux-3-image-klein-base-GGUF/tree/main) | `--diffusion-model` | Choose matching variant | +| vae | [Safetensors](https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main)
The `ae.safetensors` included with FLUX.3 can also be used. | `--vae` | Use matching VAE | +| Qwen3-VL 4B | [Safetensors](https://huggingface.co/Comfy-Org/Krea-2/tree/main/text_encoders)
[GGUF](https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct-GGUF/tree/main) | `--llm` | Use matching text encoder | + +Use the root `model.safetensors` for the diffusion model. + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| FLUX.3 Image Klein (`flux3-klein`) | `diffusion_model`, `llm`, `vae`, `tae` | +| FLUX.3 Image Klein Base (`flux3-klein-base`) | `diffusion_model`, `llm`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### FLUX.3 Image Klein + +The distilled model uses 16 steps and CFG 1.0. + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\flux-3-image-klein.safetensors ` + --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf ` + --vae ..\models\vae\flux2_ae.safetensors ` + -p "a lovely cat holding a sign says 'flux3.cpp'" ` + --cfg-scale 1.0 ` + --steps 16 ` + --sampling-method euler ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/flux-3-image-klein.safetensors \ + --llm ../models/text_encoders/Qwen3-VL-4B-Instruct-Q4_K_M.gguf \ + --vae ../models/vae/flux2_ae.safetensors \ + -p "a lovely cat holding a sign says 'flux3.cpp'" \ + --cfg-scale 1.0 \ + --steps 16 \ + --sampling-method euler \ + --offload-to-cpu \ + -v +``` + + + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\flux-3-image-klein.safetensors ` + --vae ..\models\vae\flux2_ae.safetensors ` + --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf ` + -r ..\inputs\flux\flux1-dev-q8_0.png ` + -p "change 'flux.cpp' to 'flux3-image-klein.cpp'" ` + --cfg-scale 1.0 ` + --sampling-method euler ` + --steps 16 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/flux-3-image-klein.safetensors \ + --vae ../models/vae/flux2_ae.safetensors \ + --llm ../models/text_encoders/Qwen3-VL-4B-Instruct-Q4_K_M.gguf \ + -r ../inputs/flux/flux1-dev-q8_0.png \ + -p "change 'flux.cpp' to 'flux3-image-klein.cpp'" \ + --cfg-scale 1.0 \ + --sampling-method euler \ + --steps 16 \ + --offload-to-cpu \ + -v +``` + + + +Repeat `-r` to use multiple reference images, up to 10. See [Image Editing](../guides/edit.md) for reference-image preprocessing options. + +#### FLUX.3 Image Klein Base + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\flux-3-image-klein-base.safetensors ` + --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf ` + --vae ..\models\vae\flux2_ae.safetensors ` + -p "a lovely cat holding a sign says 'flux3.cpp'" ` + --cfg-scale 3.5 ` + --steps 20 ` + --sampling-method euler ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/flux-3-image-klein-base.safetensors \ + --llm ../models/text_encoders/Qwen3-VL-4B-Instruct-Q4_K_M.gguf \ + --vae ../models/vae/flux2_ae.safetensors \ + -p "a lovely cat holding a sign says 'flux3.cpp'" \ + --cfg-scale 3.5 \ + --steps 20 \ + --sampling-method euler \ + --offload-to-cpu \ + -v +``` + + + +The default discrete scheduler uses the upstream time shift of 6.93 for both models. Use `--flow-shift` to override it. diff --git a/docs/models/hidream_o1_image.md b/docs/models/hidream_o1_image.md new file mode 100644 index 0000000000..09f7ac725f --- /dev/null +++ b/docs/models/hidream_o1_image.md @@ -0,0 +1,59 @@ +# HiDream-O1-Image + +HiDream-O1-Image uses a bundled model checkpoint. The example below uses the Dev checkpoint. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| HiDream-O1-Image-Dev | [Safetensors](https://huggingface.co/Comfy-Org/HiDream-O1-Image/tree/main/checkpoints) | `--model` | Choose matching variant | +| HiDream-O1-Image | [Safetensors](https://huggingface.co/Comfy-Org/HiDream-O1-Image/tree/main/checkpoints) | `--model` | Choose matching variant | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| HiDream O1 Image dev (`hidream-o1-dev`) | `model` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### HiDream-O1-Image-Dev + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -m ..\models\checkpoints\hidream_o1_image_dev_bf16.safetensors ` + -p "a lovely cat holding a sign says 'hidream o1 cpp'" ` + --cfg-scale 1.0 ` + -H 1024 ` + -W 1024 ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -m ../models/checkpoints/hidream_o1_image_dev_bf16.safetensors \ + -p "a lovely cat holding a sign says 'hidream o1 cpp'" \ + --cfg-scale 1.0 \ + -H 1024 \ + -W 1024 \ + -v +``` + +HiDream-O1-Image-Dev example diff --git a/docs/models/hunyuan_video.md b/docs/models/hunyuan_video.md new file mode 100644 index 0000000000..f31f8cfbd2 --- /dev/null +++ b/docs/models/hunyuan_video.md @@ -0,0 +1,82 @@ +# HunyuanVideo 1.5 + +HunyuanVideo 1.5 uses a HunyuanVideo diffusion transformer, a causal video VAE, Qwen2.5-VL 7B for the main text conditioning, +and ByT5 Small GlyphXL for glyph-aware text conditioning. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| HunyuanVideo 1.5 | [Safetensors](https://huggingface.co/Comfy-Org/HunyuanVideo_1.5_repackaged/tree/main/split_files/diffusion_models) | `--diffusion-model` | Required | +| vae | [Safetensors](https://huggingface.co/Comfy-Org/HunyuanVideo_1.5_repackaged/tree/main/split_files/vae) | `--vae` | Use matching VAE | +| qwen_2.5_vl 7b | [Safetensors](https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/tree/main/split_files/text_encoders)
[GGUF](https://huggingface.co/mradermacher/Qwen2.5-VL-7B-Instruct-GGUF/tree/main) | `--llm` | Use matching text encoder | +| byt5 small glyphxl | safetensros: [Download](https://huggingface.co/Comfy-Org/HunyuanVideo_1.5_repackaged/tree/main/split_files/text_encoders) | `--t5xxl` | Required | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| HunyuanVideo 1.5 T2V (`hunyuan-video-15-t2v`) | `diffusion_model`, `llm`, `t5xxl`, `vae`, `tae` | +| HunyuanVideo 1.5 I2V (`hunyuan-video-15-i2v`) | `diffusion_model`, `llm`, `t5xxl`, `vae`, `tae` | + +Use the T2V checkpoint with the T2V recipe. For I2V, select the I2V checkpoint +and recipe, then supply the starting image in Video Generation. + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### Text-to-video example + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\hunyuanvideo1.5_720p_t2v_fp16.safetensors ` + --vae ..\models\vae\hunyuanvideo15_vae_fp16.safetensors ` + --llm ..\models\text_encoders\qwen_2.5_vl_7b.safetensors ` + --t5xxl ..\models\text_encoders\byt5_small_glyphxl_fp16.safetensors ` + -p "a lovely cat" ` + --cfg-scale 6.0 ` + --sampling-method euler ` + -W 1280 ` + -H 720 ` + --video-frames 33 ` + --offload-to-cpu ` + --vae-tiling ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/hunyuanvideo1.5_720p_t2v_fp16.safetensors \ + --vae ../models/vae/hunyuanvideo15_vae_fp16.safetensors \ + --llm ../models/text_encoders/qwen_2.5_vl_7b.safetensors \ + --t5xxl ../models/text_encoders/byt5_small_glyphxl_fp16.safetensors \ + -p "a lovely cat" \ + --cfg-scale 6.0 \ + --sampling-method euler \ + -W 1280 \ + -H 720 \ + --video-frames 33 \ + --offload-to-cpu \ + --vae-tiling \ + -v +``` + + diff --git a/docs/models/ideogram4.md b/docs/models/ideogram4.md new file mode 100644 index 0000000000..64e8121631 --- /dev/null +++ b/docs/models/ideogram4.md @@ -0,0 +1,143 @@ +# Ideogram4 + +Ideogram4 generates images from text. Classifier-free guidance uses a separate matching unconditional transformer. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Ideogram4 | [Safetensors](https://huggingface.co/ideogram-ai/ideogram-4-fp8/tree/main/transformer) | `--diffusion-model` | Required | +| Ideogram4 uncond | [Safetensors](https://huggingface.co/ideogram-ai/ideogram-4-fp8/tree/main/unconditional_transformer) | `--uncond-diffusion-model` | Required | +| vae | [Safetensors](https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main) | `--vae` | Use matching VAE | +| Qwen3-VL-8B-Instruct | [GGUF](https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/tree/main) | `--llm` | Use matching text encoder | + +### Use original FP8 weights + +The original Ideogram4 FP8 safetensors can be loaded directly. FP8 tensors stay +at one byte per element in RAM and VRAM. Backends that cannot multiply FP8 +weights directly cast only the active layer to a temporary BF16 tensor during +execution; the loader does not expand the entire checkpoint to BF16. + +With `SD_USE_UPSTREAM_GGML=ON`, FP8 tensors are converted to F16 at load time +instead (two bytes per element in RAM and VRAM). + +Use `ideogram4_fp8.safetensors` and `ideogram4_uncond_fp8.safetensors` directly +with `--diffusion-model` and `--uncond-diffusion-model`, respectively. + +### Optional conversion + +Run executable conversion examples from the build directory using the +[shared path and shell conventions](README.md#example-conventions). + +#### Optional conversion for quantization + +The following conversion is only needed when creating a quantized GGUF model. + +fp8 scale -> bf16 + +**Windows (PowerShell)** + +```powershell +python .convert_fp8_scale_to_bf16.py --input .ideogram4_fp8.safetensors --output ideogram4_bf16.safetensors +python .convert_fp8_scale_to_bf16.py --input .ideogram4_uncond_fp8.safetensors --output ideogram4_uncond_bf16.safetensors +``` + +**Linux/macOS** + +```sh +python .convert_fp8_scale_to_bf16.py --input .ideogram4_fp8.safetensors --output ideogram4_bf16.safetensors +python .convert_fp8_scale_to_bf16.py --input .ideogram4_uncond_fp8.safetensors --output ideogram4_uncond_bf16.safetensors +``` + +bf16 -> q8 + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M convert ` + -m ..\models\ideogram4_bf16.safetensors ` + --tensor-type-rules "^layers.*adaln_modulation.*weight=q8_0,layers.*attention.o.*weight=q8_0,layers.*attention.qkv.*weight=q8_0,layers.*feed_forward.*weight=q8_0" ` + -o ideogram4-Q8_0.gguf ` + -v + +.\bin\Release\sd-cli.exe ` + -M convert ` + -m ..\models\ideogram4_uncond_bf16.safetensors ` + --tensor-type-rules "^layers.*adaln_modulation.*weight=q8_0,layers.*attention.o.*weight=q8_0,layers.*attention.qkv.*weight=q8_0,layers.*feed_forward.*weight=q8_0" ` + -o ideogram4_uncond-Q8_0.gguf ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M convert \ + -m ../models/ideogram4_bf16.safetensors \ + --tensor-type-rules "^layers.*adaln_modulation.*weight=q8_0,layers.*attention.o.*weight=q8_0,layers.*attention.qkv.*weight=q8_0,layers.*feed_forward.*weight=q8_0" \ + -o ideogram4-Q8_0.gguf \ + -v + +./bin/sd-cli \ + -M convert \ + -m ../models/ideogram4_uncond_bf16.safetensors \ + --tensor-type-rules "^layers.*adaln_modulation.*weight=q8_0,layers.*attention.o.*weight=q8_0,layers.*attention.qkv.*weight=q8_0,layers.*feed_forward.*weight=q8_0" \ + -o ideogram4_uncond-Q8_0.gguf \ + -v +``` + +If you want lower VRAM usage, you can change the quantization from q8_0 to a lower-level quantization, such as q4_0. + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| Ideogram 4 (`ideogram4`) | `diffusion_model`, `uncond_diffusion_model`, `llm`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\ideogram4-Q8_0.gguf ` + --uncond-diffusion-model ..\models\diffusion_models\ideogram4_uncond-Q8_0.gguf ` + --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf ` + --vae ..\models\vae\flux2_ae.safetensors ` + -p "{`"high_level_description`":`"A square 1024 x 1024 luxury fashion magazine cover featuring exactly one short chubby fluffy cat as the main model. The cat sits on a soft ivory studio floor, facing the viewer with a stylish calm expression, wearing tiny black sunglasses, a red silk scarf, and a small gold collar charm. In front of the cat on the floor is a wide horizontal luxury nameplate that clearly reads ideogram4.cpp. The whole design feels premium, fashionable, clean, and editorial.`",`"style_description`":{`"aesthetics`":`"luxury fashion magazine cover, high-end pet couture campaign, minimalist editorial design, elegant studio photography, soft paper texture, refined typography, fashionable and polished`",`"lighting`":`"Soft diffused studio lighting, gentle spotlight on the cat, subtle floor shadow, warm ivory highlights, clean separation between subject and background`",`"photo`":`"high-resolution fashion editorial photography look, front-facing cat portrait, crisp fur details, glossy sunglasses, clear readable nameplate text, shallow depth of field`",`"medium`":`"mixed media fashion photography and premium editorial graphic design`",`"color_palette`":[`"#F4EFE7`",`"#111111`",`"#D8B56D`",`"#B73A3A`",`"#FFFFFF`",`"#8A7A6A`"]},`"compositional_deconstruction`":{`"canvas`":`"Square 1024 x 1024 canvas with a normal upright orientation. Do not rotate the poster or any text. Use a clean fashion magazine cover layout.`",`"background`":`"Warm ivory studio backdrop with subtle paper grain, a soft spotlight gradient, faint floor shadow, and a few minimal gold editorial lines. The background is spacious, premium, and uncluttered.`",`"layout`":`"Top center has a small elegant headline. Center area features one cat as the main fashion model. Lower foreground has a wide horizontal luxury nameplate placed on the floor in front of the cat. Bottom center has a small footer. All text is horizontal, upright, and readable left to right.`",`"elements`":[{`"type`":`"text`",`"desc`":`"Top center headline reading LOOK WHAT I FOUND in a refined high-fashion serif font. The headline is horizontal, centered, elegant, and secondary to the nameplate text.`"},{`"type`":`"obj`",`"desc`":`"Exactly one short chubby fluffy cat sitting in the center like a luxury fashion model. The cat has a large round head, compact body, short legs, soft detailed fur, expressive eyes, and a calm confident pose. The cat is cute and rounded, not tall, not stretched, not duplicated.`"},{`"type`":`"obj`",`"desc`":`"Tiny glossy black sunglasses worn naturally by the cat, slightly oversized but still showing the cat face clearly. The sunglasses add a chic fashion-editorial attitude.`"},{`"type`":`"obj`",`"desc`":`"A red silk scarf tied neatly around the cat neck, with soft folds and a couture feeling. The scarf must not cover the cat face or the nameplate.`"},{`"type`":`"obj`",`"desc`":`"A small gold collar charm or fashion accessory under the scarf, subtle and premium, adding a luxury campaign detail.`"},{`"type`":`"obj`",`"desc`":`"In the lower foreground, place a wide horizontal luxury nameplate on the floor in front of the cat. The nameplate is low, flat, landscape-oriented, much wider than tall, like a fashion show seat card or premium display plaque. It is centered, front-facing, level, and fully visible. It must not become vertical, tall, standing, rotated, or side-facing.`"},{`"type`":`"text`",`"desc`":`"Print the exact text ideogram4.cpp only on the wide horizontal nameplate. Use clean bold black lettering, perfectly spelled, lowercase, with the number 4 and .cpp extension. The text must fit completely inside the nameplate, stay horizontal, and be readable from left to right.`"},{`"type`":`"obj`",`"desc`":`"Add sparse premium editorial accents around the edges: thin gold lines, small code brackets, tiny cursor marks, subtle dots, and minimal geometric details. No extra cats, no stickers, no animal faces, no busy decorations.`"},{`"type`":`"text`",`"desc`":`"Bottom center footer reading tiny paws, big compile energy in a small refined monospace or editorial font. The footer is horizontal, centered, understated, and much smaller than the nameplate text.`"}]}}" ` + -H 1024 ` + -W 1024 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/ideogram4-Q8_0.gguf \ + --uncond-diffusion-model ../models/diffusion_models/ideogram4_uncond-Q8_0.gguf \ + --llm ../models/text_encoders/Qwen3VL-8B-Instruct-Q4_K_M.gguf \ + --vae ../models/vae/flux2_ae.safetensors \ + -p "{\"high_level_description\":\"A square 1024 x 1024 luxury fashion magazine cover featuring exactly one short chubby fluffy cat as the main model. The cat sits on a soft ivory studio floor, facing the viewer with a stylish calm expression, wearing tiny black sunglasses, a red silk scarf, and a small gold collar charm. In front of the cat on the floor is a wide horizontal luxury nameplate that clearly reads ideogram4.cpp. The whole design feels premium, fashionable, clean, and editorial.\",\"style_description\":{\"aesthetics\":\"luxury fashion magazine cover, high-end pet couture campaign, minimalist editorial design, elegant studio photography, soft paper texture, refined typography, fashionable and polished\",\"lighting\":\"Soft diffused studio lighting, gentle spotlight on the cat, subtle floor shadow, warm ivory highlights, clean separation between subject and background\",\"photo\":\"high-resolution fashion editorial photography look, front-facing cat portrait, crisp fur details, glossy sunglasses, clear readable nameplate text, shallow depth of field\",\"medium\":\"mixed media fashion photography and premium editorial graphic design\",\"color_palette\":[\"#F4EFE7\",\"#111111\",\"#D8B56D\",\"#B73A3A\",\"#FFFFFF\",\"#8A7A6A\"]},\"compositional_deconstruction\":{\"canvas\":\"Square 1024 x 1024 canvas with a normal upright orientation. Do not rotate the poster or any text. Use a clean fashion magazine cover layout.\",\"background\":\"Warm ivory studio backdrop with subtle paper grain, a soft spotlight gradient, faint floor shadow, and a few minimal gold editorial lines. The background is spacious, premium, and uncluttered.\",\"layout\":\"Top center has a small elegant headline. Center area features one cat as the main fashion model. Lower foreground has a wide horizontal luxury nameplate placed on the floor in front of the cat. Bottom center has a small footer. All text is horizontal, upright, and readable left to right.\",\"elements\":[{\"type\":\"text\",\"desc\":\"Top center headline reading LOOK WHAT I FOUND in a refined high-fashion serif font. The headline is horizontal, centered, elegant, and secondary to the nameplate text.\"},{\"type\":\"obj\",\"desc\":\"Exactly one short chubby fluffy cat sitting in the center like a luxury fashion model. The cat has a large round head, compact body, short legs, soft detailed fur, expressive eyes, and a calm confident pose. The cat is cute and rounded, not tall, not stretched, not duplicated.\"},{\"type\":\"obj\",\"desc\":\"Tiny glossy black sunglasses worn naturally by the cat, slightly oversized but still showing the cat face clearly. The sunglasses add a chic fashion-editorial attitude.\"},{\"type\":\"obj\",\"desc\":\"A red silk scarf tied neatly around the cat neck, with soft folds and a couture feeling. The scarf must not cover the cat face or the nameplate.\"},{\"type\":\"obj\",\"desc\":\"A small gold collar charm or fashion accessory under the scarf, subtle and premium, adding a luxury campaign detail.\"},{\"type\":\"obj\",\"desc\":\"In the lower foreground, place a wide horizontal luxury nameplate on the floor in front of the cat. The nameplate is low, flat, landscape-oriented, much wider than tall, like a fashion show seat card or premium display plaque. It is centered, front-facing, level, and fully visible. It must not become vertical, tall, standing, rotated, or side-facing.\"},{\"type\":\"text\",\"desc\":\"Print the exact text ideogram4.cpp only on the wide horizontal nameplate. Use clean bold black lettering, perfectly spelled, lowercase, with the number 4 and .cpp extension. The text must fit completely inside the nameplate, stay horizontal, and be readable from left to right.\"},{\"type\":\"obj\",\"desc\":\"Add sparse premium editorial accents around the edges: thin gold lines, small code brackets, tiny cursor marks, subtle dots, and minimal geometric details. No extra cats, no stickers, no animal faces, no busy decorations.\"},{\"type\":\"text\",\"desc\":\"Bottom center footer reading tiny paws, big compile energy in a small refined monospace or editorial font. The footer is horizontal, centered, understated, and much smaller than the nameplate text.\"}]}}" \ + -H 1024 \ + -W 1024 \ + --offload-to-cpu \ + -v +``` + +ideogram4 image example diff --git a/docs/models/ip_adapter.md b/docs/models/ip_adapter.md new file mode 100644 index 0000000000..854ed88d78 --- /dev/null +++ b/docs/models/ip_adapter.md @@ -0,0 +1,223 @@ +# IP-Adapter + +[IP-Adapter](https://github.com/tencent-ailab/IP-Adapter) uses a reference image +to guide SD 1.5 or SDXL generation alongside +the text prompt. Classic and Plus adapters are supported, including combination +with Control Net. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Base checkpoint | [SD 1.5 or SDXL](sd.md) | `-m` / `--model` | Required; match adapter family | +| CLIP Vision H | `clip_vision_h.safetensors` from [IP-Adapter](https://huggingface.co/h94/IP-Adapter) | `--clip_vision` | Required; ViT-H/14 | +| SD 1.5 adapter | [`models/ip-adapter_sd15.safetensors`](https://huggingface.co/h94/IP-Adapter) | `--ip-adapter` | For SD 1.5 | +| SDXL adapter | [`sdxl_models/ip-adapter_sdxl_vit-h.safetensors`](https://huggingface.co/h94/IP-Adapter) | `--ip-adapter` | For SDXL | +| SD 1.5 Plus | [`models/ip-adapter-plus_sd15.safetensors`](https://huggingface.co/h94/IP-Adapter) | `--ip-adapter` | Alternative for SD 1.5 | +| SDXL Plus | [`sdxl_models/ip-adapter-plus_sdxl_vit-h.safetensors`](https://huggingface.co/h94/IP-Adapter) | `--ip-adapter` | Alternative for SDXL | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| SD 1.5 IP-Adapter / Plus (`sd1-ip-adapter`) | `model`, `clip_vision`, `ip_adapter`, `vae`, `tae` | +| SDXL IP-Adapter / Plus (`sdxl-ip-adapter`) | `model`, `clip_vision`, `ip_adapter`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### Example (SD 1.5) + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -m ..\models\checkpoints\sd_v1.5.safetensors ` + --clip_vision ..\models\clip_vision\clip_vision_h.safetensors ` + --ip-adapter ..\models\ipadapter\ip-adapter_sd15.safetensors ` + --ip-adapter-image ..\inputs\reference.png ` + -p "a woman, best quality" ` + -n "lowres, bad anatomy" ` + --cfg-scale 7 ` + --steps 30 ` + --sampling-method dpm++2m ` + --scheduler karras ` + -W 512 ` + -H 512 ` + --ip-adapter-strength 0.8 +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -m ../models/checkpoints/sd_v1.5.safetensors \ + --clip_vision ../models/clip_vision/clip_vision_h.safetensors \ + --ip-adapter ../models/ipadapter/ip-adapter_sd15.safetensors \ + --ip-adapter-image ../inputs/reference.png \ + -p "a woman, best quality" \ + -n "lowres, bad anatomy" \ + --cfg-scale 7 \ + --steps 30 \ + --sampling-method dpm++2m \ + --scheduler karras \ + -W 512 \ + -H 512 \ + --ip-adapter-strength 0.8 +``` + +#### Example (SDXL) + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -m ..\models\checkpoints\sdxl.safetensors ` + --clip_vision ..\models\clip_vision\clip_vision_h.safetensors ` + --ip-adapter ..\models\ipadapter\ip-adapter_sdxl_vit-h.safetensors ` + --ip-adapter-image ..\inputs\reference.png ` + -p "a woman, best quality" ` + -n "lowres, bad anatomy" ` + --cfg-scale 6 ` + --steps 25 ` + --sampling-method dpm++2m ` + --scheduler karras ` + -W 1024 ` + -H 1024 ` + --ip-adapter-strength 0.8 ` + --vae-tiling +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -m ../models/checkpoints/sdxl.safetensors \ + --clip_vision ../models/clip_vision/clip_vision_h.safetensors \ + --ip-adapter ../models/ipadapter/ip-adapter_sdxl_vit-h.safetensors \ + --ip-adapter-image ../inputs/reference.png \ + -p "a woman, best quality" \ + -n "lowres, bad anatomy" \ + --cfg-scale 6 \ + --steps 25 \ + --sampling-method dpm++2m \ + --scheduler karras \ + -W 1024 \ + -H 1024 \ + --ip-adapter-strength 0.8 \ + --vae-tiling +``` + +The SDXL VAE decode at 1024x1024 is memory heavy; add `--vae-tiling` (and +`--offload-to-cpu`) on GPUs with limited VRAM. + +#### Plus variants + +Use the Plus weights matching your base model. The same IP-Adapter options +apply; no extra flag is needed. Plus can also be combined with Control Net. + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -m ..\models\checkpoints\sd_v1.5.safetensors ` + --clip_vision ..\models\clip_vision\clip_vision_h.safetensors ` + --ip-adapter ..\models\ipadapter\ip-adapter-plus_sd15.safetensors ` + --ip-adapter-image ..\inputs\reference.png ` + -p "a woman, best quality" ` + -n "lowres, bad anatomy" ` + --cfg-scale 7 ` + --steps 30 ` + --sampling-method dpm++2m ` + --scheduler karras ` + -W 512 ` + -H 512 ` + --ip-adapter-strength 0.8 +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -m ../models/checkpoints/sd_v1.5.safetensors \ + --clip_vision ../models/clip_vision/clip_vision_h.safetensors \ + --ip-adapter ../models/ipadapter/ip-adapter-plus_sd15.safetensors \ + --ip-adapter-image ../inputs/reference.png \ + -p "a woman, best quality" \ + -n "lowres, bad anatomy" \ + --cfg-scale 7 \ + --steps 30 \ + --sampling-method dpm++2m \ + --scheduler karras \ + -W 512 \ + -H 512 \ + --ip-adapter-strength 0.8 +``` + +The startup log line `IP-Adapter: 16 image tokens` (versus `4` for the +classic adapters) confirms a Plus file was loaded. + +#### Combining with Control Net + +Add the usual Control Net options to keep the reference appearance while +controlling the pose: + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -m ..\models\checkpoints\sdxl.safetensors ` + --clip_vision ..\models\clip_vision\clip_vision_h.safetensors ` + --ip-adapter ..\models\ipadapter\ip-adapter_sdxl_vit-h.safetensors ` + --control-net ..\models\controlnet\OpenPoseXL2.safetensors ` + --ip-adapter-image ..\inputs\character.png ` + --control-image ..\inputs\pose.png ` + -p "a character, side view" ` + --cfg-scale 6 ` + --steps 25 ` + -W 1024 ` + -H 1024 ` + --ip-adapter-strength 0.9 ` + --control-strength 0.8 ` + --vae-tiling +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -m ../models/checkpoints/sdxl.safetensors \ + --clip_vision ../models/clip_vision/clip_vision_h.safetensors \ + --ip-adapter ../models/ipadapter/ip-adapter_sdxl_vit-h.safetensors \ + --control-net ../models/controlnet/OpenPoseXL2.safetensors \ + --ip-adapter-image ../inputs/character.png \ + --control-image ../inputs/pose.png \ + -p "a character, side view" \ + --cfg-scale 6 \ + --steps 25 \ + -W 1024 \ + -H 1024 \ + --ip-adapter-strength 0.9 \ + --control-strength 0.8 \ + --vae-tiling +``` + +## Options + +- `--ip-adapter ` path to the IP-Adapter weight file. +- `--ip-adapter-image ` path to the reference image. +- `--ip-adapter-strength ` strength of the IP-Adapter injection + (default 1.0). Lower values let the text prompt dominate; 0.6 to 0.8 is + a good starting range. diff --git a/docs/models/iris.md b/docs/models/iris.md new file mode 100644 index 0000000000..643b0d2fb4 --- /dev/null +++ b/docs/models/iris.md @@ -0,0 +1,62 @@ +# Iris-3B + +Iris-3B generates images directly in pixel space and uses Qwen3-VL-4B-Instruct as the text encoder. No VAE is required. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Iris-3B | [Safetensors](https://huggingface.co/speridlabs/iris-3b/tree/main) (`model.safetensors` in the root directory) | `--diffusion-model` | Required | +| Qwen3-VL-4B-Instruct | [Safetensors](https://huggingface.co/Comfy-Org/Krea-2/tree/main/text_encoders)
[GGUF](https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct-GGUF/tree/main) | `--llm` | Use matching text encoder | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| Iris 3B (`iris-3b`) | `diffusion_model`, `llm` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\iris-3b.safetensors ` + --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf ` + -p "a lovely cat" ` + --cfg-scale 3 ` + -H 1024 ` + -W 1024 +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/iris-3b.safetensors \ + --llm ../models/text_encoders/Qwen3-VL-4B-Instruct-Q4_K_M.gguf \ + -p "a lovely cat" \ + --cfg-scale 3 \ + -H 1024 \ + -W 1024 +``` + +iris-3b example + +## Notes and limitations + +- Width and height must be multiples of 16. Do not pass `--vae`. +- Captions are limited to 300 tokens including the assistant-turn suffix. Positive and negative prompts use the same template. diff --git a/docs/models/kontext.md b/docs/models/kontext.md new file mode 100644 index 0000000000..c711f3227d --- /dev/null +++ b/docs/models/kontext.md @@ -0,0 +1,108 @@ +# FLUX.1 Kontext + +FLUX.1 Kontext supports reference-image editing using the FLUX.1 VAE, CLIP-L, and T5-XXL. + +You can run Kontext using stable-diffusion.cpp with a GPU that has 6GB or even 4GB of VRAM, without needing to offload to RAM. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Kontext | If you don't want to do the conversion yourself, download the preconverted gguf model from [FLUX.1-Kontext-dev-GGUF](https://huggingface.co/QuantStack/FLUX.1-Kontext-dev-GGUF)
Otherwise, download FLUX.1-Kontext-dev from [Download](https://huggingface.co/black-forest-labs/FLUX.1-Kontext-dev/blob/main/flux1-kontext-dev.safetensors) | `--diffusion-model` | Required | +| vae | vae from [Download](https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/ae.safetensors) | `--vae` | Use matching VAE | +| clip_l | clip_l from [Download](https://huggingface.co/comfyanonymous/flux_text_encoders/blob/main/clip_l.safetensors) | `--clip_l` | Required | +| t5xxl | t5xxl from [Download](https://huggingface.co/comfyanonymous/flux_text_encoders/blob/main/t5xxl_fp16.safetensors) | `--t5xxl` | Required | + +### Optional conversion + +Run executable conversion examples from the build directory using the +[shared path and shell conventions](README.md#example-conventions). + +#### Convert Kontext weights + +You can download the preconverted gguf weights from [FLUX.1-Kontext-dev-GGUF](https://huggingface.co/QuantStack/FLUX.1-Kontext-dev-GGUF), this way you don't have to do the conversion yourself. + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M convert ` + -m ..\models\unet\flux1-kontext-dev.safetensors ` + --type q8_0 ` + -o ..\models\diffusion_models\flux1-kontext-dev-q8_0.gguf ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M convert \ + -m ../models/unet/flux1-kontext-dev.safetensors \ + --type q8_0 \ + -o ../models/diffusion_models/flux1-kontext-dev-q8_0.gguf \ + -v +``` + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| FLUX.1 Kontext dev (`flux1-kontext`) | `diffusion_model`, `clip_l`, `t5xxl`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +- `--cfg-scale` is recommended to be set to 1. + +#### Example + +For example: + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\flux1-kontext-dev-q8_0.gguf ` + --vae ..\models\vae\ae.sft ` + --clip_l ..\models\text_encoders\clip_l.safetensors ` + --t5xxl ..\models\text_encoders\t5xxl_fp16.safetensors ` + -r .\flux1-dev-q8_0.png ` + -p "change 'flux.cpp' to 'kontext.cpp'" ` + --cfg-scale 1.0 ` + --sampling-method euler ` + --clip-on-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/flux1-kontext-dev-q8_0.gguf \ + --vae ../models/vae/ae.sft \ + --clip_l ../models/text_encoders/clip_l.safetensors \ + --t5xxl ../models/text_encoders/t5xxl_fp16.safetensors \ + -r ./flux1-dev-q8_0.png \ + -p "change 'flux.cpp' to 'kontext.cpp'" \ + --cfg-scale 1.0 \ + --sampling-method euler \ + --clip-on-cpu \ + -v +``` + +| ref_image | prompt | output | +| ---- | ---- |---- | +| ![](../../assets/flux/flux1-dev-q8_0.png) | change 'flux.cpp' to 'kontext.cpp' |![](../../assets/flux/kontext1_dev_output.png) | diff --git a/docs/models/krea2.md b/docs/models/krea2.md new file mode 100644 index 0000000000..c6367111ed --- /dev/null +++ b/docs/models/krea2.md @@ -0,0 +1,62 @@ +# Krea2 + +Krea2 uses a Krea2 diffusion transformer, the Wan2.1 VAE, and Qwen3-VL 4B as the LLM text encoder. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Krea2 Raw | [Safetensors](https://huggingface.co/krea/Krea-2-Raw/tree/main)
[GGUF](https://huggingface.co/realrebelai/KREA-2_GGUFs/tree/main/BASE) | `--diffusion-model` | Choose matching variant | +| Krea2 Turbo | [Safetensors](https://huggingface.co/krea/Krea-2-Turbo/tree/main)
[GGUF](https://huggingface.co/realrebelai/KREA-2_GGUFs/tree/main/TURBO) | `--diffusion-model` | Choose matching variant | +| vae | [Safetensors](https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/blob/main/split_files/vae/wan_2.1_vae.safetensors) | `--vae` | Use matching VAE | +| Qwen3-VL 4B | [Safetensors](https://huggingface.co/Comfy-Org/Krea-2/tree/main/text_encoders)
[GGUF](https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct-GGUF/tree/main) | `--llm` | Use matching text encoder | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| Krea 2 Raw (`krea2-raw`) | `diffusion_model`, `llm`, `vae`, `tae` | +| Krea 2 Turbo (`krea2-turbo`) | `diffusion_model`, `llm`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### Krea2 + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\Krea-2-Raw-Q8_0.gguf ` + --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf ` + --vae ..\models\vae\wan_2.1_vae.safetensors ` + -p "a lovely cat holding a sign says 'krea2.cpp'" ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/Krea-2-Raw-Q8_0.gguf \ + --llm ../models/text_encoders/Qwen3-VL-4B-Instruct-Q4_K_M.gguf \ + --vae ../models/vae/wan_2.1_vae.safetensors \ + -p "a lovely cat holding a sign says 'krea2.cpp'" \ + --offload-to-cpu \ + -v +``` + +Krea2 Raw example diff --git a/docs/models/lcm.md b/docs/models/lcm.md new file mode 100644 index 0000000000..90330d2a6e --- /dev/null +++ b/docs/models/lcm.md @@ -0,0 +1,76 @@ +# LCM and LCM-LoRA + +LCM checkpoints and LCM-LoRAs support generation with a small number of steps. +The example below applies an LCM-LoRA to SD 1.5. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Base model | [Stable Diffusion 1.5 checkpoint](sd.md) | `-m` / `--model` | Required for the SD 1.5 LoRA example | +| LCM-LoRA | [LCM-LoRA SD 1.5](https://huggingface.co/latent-consistency/lcm-lora-sdv1-5) | `--lora-model-dir` and prompt | Match the LoRA to the base model | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| SD 1.x (`sd1-checkpoint`) | `model`, `vae`, `tae` | +| SDXL (`sdxl-checkpoint`) | `model`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +Select the matching LCM checkpoint or apply a compatible LCM-LoRA, then set the +sampling method, CFG, and steps for that model. + +### CLI + +#### LCM-LoRA with SD 1.5 + +Add `` to the prompt: + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -m ..\models\checkpoints\v1-5-pruned-emaonly.safetensors ` + --lora-model-dir ..\models\loras ` + -p "a lovely cat" ` + --steps 4 ` + --cfg-scale 1 ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -m ../models/checkpoints/v1-5-pruned-emaonly.safetensors \ + --lora-model-dir ../models/loras \ + -p "a lovely cat" \ + --steps 4 \ + --cfg-scale 1 \ + -v +``` + +#### LCM-LoRA comparison + +| without LCM-LoRA (--cfg-scale 7) | with LCM-LoRA (--cfg-scale 1) | +| ---- |---- | +| ![](../../assets/without_lcm.png) |![](../../assets/with_lcm.png) | + +## Options + +Use `--cfg-scale 1.0` and 2-8 steps for the LCM-LoRA workflow. The original +example uses `euler_a`; `lcm` sampling is also supported. See +[sampling methods and schedulers](../guides/sampling.md) for their defaults. diff --git a/docs/models/lens.md b/docs/models/lens.md new file mode 100644 index 0000000000..42708eb85a --- /dev/null +++ b/docs/models/lens.md @@ -0,0 +1,99 @@ +# Lens + +Lens uses a Lens diffusion transformer, the FLUX.2 VAE, and GPT-OSS-20B as the LLM text encoder. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Lens | [Safetensors](https://huggingface.co/Comfy-Org/Lens/tree/main/diffusion_models) | `--diffusion-model` | Choose matching variant | +| Lens Turbo | [Safetensors](https://huggingface.co/Comfy-Org/Lens/tree/main/diffusion_models) | `--diffusion-model` | Choose matching variant | +| vae | [Safetensors](https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main) | `--vae` | Use matching VAE | +| GPT-OSS-20B | [GGUF](https://huggingface.co/unsloth/gpt-oss-20b-GGUF/tree/main) | `--llm` | Use matching text encoder | +| GPT-OSS-20B tokenizer.json | [Download](https://huggingface.co/openai/gpt-oss-20b/tree/main) | `--tokenizer` | Required; match text encoder | + +Lens and Lens Turbo require an external GPT-OSS `tokenizer.json` matching the text encoder checkpoint. Save it as `tokenizer_gpt_oss.json` and pass it with `--tokenizer`; the tokenizer is not embedded in sd.cpp. See [JSON tokenizers](../guides/tokenizers.md) for CLI and C API usage. + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| LENS (`lens`) | `diffusion_model`, `llm`, `tokenizer`, `vae`, `tae` | +| LENS Turbo (`lens-turbo`) | `diffusion_model`, `llm`, `tokenizer`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### Lens + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\lens_bf16.safetensors ` + --llm ..\models\text_encoders\gpt-oss-20b-UD-Q8_K_XL.gguf ` + --tokenizer ..\models\tokenizers\tokenizer_gpt_oss.json ` + --vae ..\models\vae\flux2_ae.safetensors ` + -p "A crystal dragon soaring through an aurora borealis sky, its entire body made of transparent faceted crystal refracting the green and purple aurora light into rainbow spectra, ice particles trailing from its wings, high fantasy digital art" ` + --cfg-scale 5.0 ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/lens_bf16.safetensors \ + --llm ../models/text_encoders/gpt-oss-20b-UD-Q8_K_XL.gguf \ + --tokenizer ../models/tokenizers/tokenizer_gpt_oss.json \ + --vae ../models/vae/flux2_ae.safetensors \ + -p "A crystal dragon soaring through an aurora borealis sky, its entire body made of transparent faceted crystal refracting the green and purple aurora light into rainbow spectra, ice particles trailing from its wings, high fantasy digital art" \ + --cfg-scale 5.0 \ + -v +``` + +Lens example + +#### Lens Turbo + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\lens_turbo_bf16.safetensors ` + --llm ..\models\text_encoders\gpt-oss-20b-UD-Q8_K_XL.gguf ` + --tokenizer ..\models\tokenizers\tokenizer_gpt_oss.json ` + --vae ..\models\vae\flux2_ae.safetensors ` + -p "A crystal dragon soaring through an aurora borealis sky, its entire body made of transparent faceted crystal refracting the green and purple aurora light into rainbow spectra, ice particles trailing from its wings, high fantasy digital art" ` + --cfg-scale 1.0 ` + --steps 4 ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/lens_turbo_bf16.safetensors \ + --llm ../models/text_encoders/gpt-oss-20b-UD-Q8_K_XL.gguf \ + --tokenizer ../models/tokenizers/tokenizer_gpt_oss.json \ + --vae ../models/vae/flux2_ae.safetensors \ + -p "A crystal dragon soaring through an aurora borealis sky, its entire body made of transparent faceted crystal refracting the green and purple aurora light into rainbow spectra, ice particles trailing from its wings, high fantasy digital art" \ + --cfg-scale 1.0 \ + --steps 4 \ + -v +``` + +Lens Turbo example diff --git a/docs/models/lingbot_video.md b/docs/models/lingbot_video.md new file mode 100644 index 0000000000..a085c891b3 --- /dev/null +++ b/docs/models/lingbot_video.md @@ -0,0 +1,77 @@ +# LingBot-Video + +Lingbot Video uses a Lingbot diffusion transformer, the Wan2.1 VAE, and Qwen3-VL 4B as the LLM text encoder. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| lingbot-video-dense-1.3b | [Safetensors](https://huggingface.co/robbyant/lingbot-video-dense-1.3b/tree/main/transformer) | `--diffusion-model` | Choose matching variant | +| lingbot-video-moe-30b-a3b | [Safetensors](https://huggingface.co/robbyant/lingbot-video-moe-30b-a3b/tree/main/transformer) | `--diffusion-model` | Choose matching variant | +| vae | [Safetensors](https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/blob/main/split_files/vae/wan_2.1_vae.safetensors) | `--vae` | Use matching VAE | +| Qwen3-VL 4B | [Safetensors](https://huggingface.co/Comfy-Org/Krea-2/tree/main/text_encoders)
[GGUF](https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct-GGUF/tree/main) | `--llm` | Use matching text encoder | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| LingBot Video Dense 1.3B (`lingbot-video-dense`) | `diffusion_model`, `llm`, `vae`, `tae` | +| LingBot Video MoE 30B A3B (`lingbot-video-moe`) | `diffusion_model`, `llm`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### LingBot-Video T2V + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\lingbot-video-dens-1.3b.safetensors ` + --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf ` + --vae ..\models\vae\wan_2.1_vae.safetensors ` + -p "{`"caption`":{`"comprehensive_description`":`"A lovely cat sits comfortably on a soft cushion near a sunlit window, looking calm, gentle, and adorable. The cat has soft fluffy fur, bright expressive eyes, small rounded ears, delicate whiskers, and a relaxed posture. Warm daylight falls across the cat from one side, creating soft highlights on the fur and a cozy glow around the scene. The background is softly blurred, showing hints of a peaceful indoor home environment with warm tones and gentle natural light. The overall atmosphere is cute, tender, serene, and photorealistic, emphasizing the cats charming appearance, soft texture, and affectionate presence.`",`"camera_info`":{`"color`":`"Warm`",`"frame_size`":`"Close Up`",`"shot_type_angle`":`"Eye level`",`"lens_size`":`"Medium Lens`",`"composition`":`"Centered balanced`",`"lighting`":`"Soft light`",`"lighting_type`":`"Daylight`"},`"world_knowledge`":[],`"prominent_elements`":[{`"name`":`"lovely cat`",`"description`":`"A cute and gentle domestic cat with soft fluffy fur, expressive eyes, and a calm relaxed presence.`",`"location`":`"center of the frame`",`"relative_size`":`"large`",`"shape_and_color`":`"Small animal body with rounded face, triangular ears, bright eyes, and soft fur in warm natural tones`",`"texture`":`"soft, fluffy, silky`",`"appearance_details`":`"The cat has clean well-groomed fur, delicate whiskers, small ears, a cute nose, and bright attentive eyes. Its expression appears calm, affectionate, and slightly curious.`",`"relationship`":`"Acts as the main subject and emotional focal point of the scene.`",`"orientation`":`"facing the camera`",`"pose`":`"sitting comfortably with a relaxed posture`",`"expression`":`"gentle, adorable, calm, slightly curious`",`"clothing`":`"`",`"gender`":`"`",`"skin_tone_and_texture`":`"`"},{`"name`":`"cat eyes`",`"description`":`"Bright expressive eyes that give the cat a sweet and affectionate appearance.`",`"location`":`"upper center of the cat face`",`"relative_size`":`"small`",`"shape_and_color`":`"Round almond-like eyes with glossy reflections`",`"texture`":`"clear, glossy, reflective`",`"appearance_details`":`"The eyes catch the soft daylight, creating small natural highlights that make the cat look vivid and alive.`",`"relationship`":`"Enhance the emotional charm and cuteness of the cat.`",`"orientation`":`"looking toward the camera`",`"pose`":`"`",`"expression`":`"soft and attentive`",`"clothing`":`"`",`"gender`":`"`",`"skin_tone_and_texture`":`"`"},{`"name`":`"soft cushion`",`"description`":`"A comfortable cushion or blanket where the cat is resting.`",`"location`":`"bottom portion of the frame`",`"relative_size`":`"medium`",`"shape_and_color`":`"Soft rounded fabric surface in light warm neutral tones`",`"texture`":`"plush, fabric, cozy`",`"appearance_details`":`"The cushion gently supports the cat and adds a comfortable home-like feeling to the scene.`",`"relationship`":`"Provides a cozy resting place for the cat.`",`"orientation`":`"horizontal`",`"pose`":`"`",`"expression`":`"`",`"clothing`":`"`",`"gender`":`"`",`"skin_tone_and_texture`":`"`"},{`"name`":`"sunlit indoor background`",`"description`":`"A softly blurred indoor background with warm daylight and peaceful home atmosphere.`",`"location`":`"behind the cat, filling the upper and side areas of the frame`",`"relative_size`":`"large`",`"shape_and_color`":`"Soft abstract shapes in warm beige, cream, and pale golden tones`",`"texture`":`"soft, blurry, bokeh-like`",`"appearance_details`":`"The background is intentionally out of focus, keeping attention on the cat while creating a cozy and serene mood.`",`"relationship`":`"Provides a warm and gentle environment that supports the cute domestic scene.`",`"orientation`":`"upright`",`"pose`":`"`",`"expression`":`"`",`"clothing`":`"`",`"gender`":`"`",`"skin_tone_and_texture`":`"`",`"is_cluster`":true,`"number_of_objects`":`"numerous`"}]}}" ` + -n "{`"universal_negative`":{`"visual_quality`":[`"low quality`",`"worst quality`",`"blurry`",`"pixelated`",`"jpeg artifacts`",`"low resolution`",`"unstable color`",`"color flicker`",`"underexposed`",`"overexposed`",`"invisible subject`",`"subject hidden in darkness`"],`"artistic_style`":[`"painting`",`"illustration`",`"drawing`",`"cartoon`",`"3d render`",`"cgi`",`"sketch`",`"digital art`"],`"composition_and_content`":[`"text`",`"watermark`",`"signature`",`"logo`",`"subtitles`",`"pillarboxed`",`"side bars`",`"portrait image in landscape frame`"],`"temporal_and_motion_stability`":[`"flickering`",`"jittery`",`"motion blur`",`"temporal inconsistency`",`"warping`",`"morphing`",`"incoherent motion`",`"unnatural movement`",`"static object with sudden jump`",`"frame-to-frame inconsistency`"],`"material_and_structure`":[`"plastic-like glass`",`"unrealistic texture`",`"deformed bottle`",`"liquid freezing improperly`",`"distorted reflections`"]}}" ` + --cfg-scale 3 ` + --video-frames 33 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/lingbot-video-dens-1.3b.safetensors \ + --llm ../models/text_encoders/Qwen3-VL-4B-Instruct-Q4_K_M.gguf \ + --vae ../models/vae/wan_2.1_vae.safetensors \ + -p "{\"caption\":{\"comprehensive_description\":\"A lovely cat sits comfortably on a soft cushion near a sunlit window, looking calm, gentle, and adorable. The cat has soft fluffy fur, bright expressive eyes, small rounded ears, delicate whiskers, and a relaxed posture. Warm daylight falls across the cat from one side, creating soft highlights on the fur and a cozy glow around the scene. The background is softly blurred, showing hints of a peaceful indoor home environment with warm tones and gentle natural light. The overall atmosphere is cute, tender, serene, and photorealistic, emphasizing the cats charming appearance, soft texture, and affectionate presence.\",\"camera_info\":{\"color\":\"Warm\",\"frame_size\":\"Close Up\",\"shot_type_angle\":\"Eye level\",\"lens_size\":\"Medium Lens\",\"composition\":\"Centered balanced\",\"lighting\":\"Soft light\",\"lighting_type\":\"Daylight\"},\"world_knowledge\":[],\"prominent_elements\":[{\"name\":\"lovely cat\",\"description\":\"A cute and gentle domestic cat with soft fluffy fur, expressive eyes, and a calm relaxed presence.\",\"location\":\"center of the frame\",\"relative_size\":\"large\",\"shape_and_color\":\"Small animal body with rounded face, triangular ears, bright eyes, and soft fur in warm natural tones\",\"texture\":\"soft, fluffy, silky\",\"appearance_details\":\"The cat has clean well-groomed fur, delicate whiskers, small ears, a cute nose, and bright attentive eyes. Its expression appears calm, affectionate, and slightly curious.\",\"relationship\":\"Acts as the main subject and emotional focal point of the scene.\",\"orientation\":\"facing the camera\",\"pose\":\"sitting comfortably with a relaxed posture\",\"expression\":\"gentle, adorable, calm, slightly curious\",\"clothing\":\"\",\"gender\":\"\",\"skin_tone_and_texture\":\"\"},{\"name\":\"cat eyes\",\"description\":\"Bright expressive eyes that give the cat a sweet and affectionate appearance.\",\"location\":\"upper center of the cat face\",\"relative_size\":\"small\",\"shape_and_color\":\"Round almond-like eyes with glossy reflections\",\"texture\":\"clear, glossy, reflective\",\"appearance_details\":\"The eyes catch the soft daylight, creating small natural highlights that make the cat look vivid and alive.\",\"relationship\":\"Enhance the emotional charm and cuteness of the cat.\",\"orientation\":\"looking toward the camera\",\"pose\":\"\",\"expression\":\"soft and attentive\",\"clothing\":\"\",\"gender\":\"\",\"skin_tone_and_texture\":\"\"},{\"name\":\"soft cushion\",\"description\":\"A comfortable cushion or blanket where the cat is resting.\",\"location\":\"bottom portion of the frame\",\"relative_size\":\"medium\",\"shape_and_color\":\"Soft rounded fabric surface in light warm neutral tones\",\"texture\":\"plush, fabric, cozy\",\"appearance_details\":\"The cushion gently supports the cat and adds a comfortable home-like feeling to the scene.\",\"relationship\":\"Provides a cozy resting place for the cat.\",\"orientation\":\"horizontal\",\"pose\":\"\",\"expression\":\"\",\"clothing\":\"\",\"gender\":\"\",\"skin_tone_and_texture\":\"\"},{\"name\":\"sunlit indoor background\",\"description\":\"A softly blurred indoor background with warm daylight and peaceful home atmosphere.\",\"location\":\"behind the cat, filling the upper and side areas of the frame\",\"relative_size\":\"large\",\"shape_and_color\":\"Soft abstract shapes in warm beige, cream, and pale golden tones\",\"texture\":\"soft, blurry, bokeh-like\",\"appearance_details\":\"The background is intentionally out of focus, keeping attention on the cat while creating a cozy and serene mood.\",\"relationship\":\"Provides a warm and gentle environment that supports the cute domestic scene.\",\"orientation\":\"upright\",\"pose\":\"\",\"expression\":\"\",\"clothing\":\"\",\"gender\":\"\",\"skin_tone_and_texture\":\"\",\"is_cluster\":true,\"number_of_objects\":\"numerous\"}]}}" \ + -n "{\"universal_negative\":{\"visual_quality\":[\"low quality\",\"worst quality\",\"blurry\",\"pixelated\",\"jpeg artifacts\",\"low resolution\",\"unstable color\",\"color flicker\",\"underexposed\",\"overexposed\",\"invisible subject\",\"subject hidden in darkness\"],\"artistic_style\":[\"painting\",\"illustration\",\"drawing\",\"cartoon\",\"3d render\",\"cgi\",\"sketch\",\"digital art\"],\"composition_and_content\":[\"text\",\"watermark\",\"signature\",\"logo\",\"subtitles\",\"pillarboxed\",\"side bars\",\"portrait image in landscape frame\"],\"temporal_and_motion_stability\":[\"flickering\",\"jittery\",\"motion blur\",\"temporal inconsistency\",\"warping\",\"morphing\",\"incoherent motion\",\"unnatural movement\",\"static object with sudden jump\",\"frame-to-frame inconsistency\"],\"material_and_structure\":[\"plastic-like glass\",\"unrealistic texture\",\"deformed bottle\",\"liquid freezing improperly\",\"distorted reflections\"]}}" \ + --cfg-scale 3 \ + --video-frames 33 \ + --offload-to-cpu \ + -v +``` + +## Notes and limitations + +- Use `-M vid_gen`. +- T2V uses the text prompt only. +- I2V uses `-i` as the first video frame. The same image is also passed to the + Qwen3-VL prompt enhancer when vision weights are available. +- Video frames are aligned to Wan-style temporal compression, so use frame counts + like 33, 49, or 81. diff --git a/docs/models/llada_image.md b/docs/models/llada_image.md new file mode 100644 index 0000000000..d627b7bd1b --- /dev/null +++ b/docs/models/llada_image.md @@ -0,0 +1,251 @@ +# LLaDA-Image + +LLaDA-Image is a 6B text-to-image and instruction-guided editing model using a +LLaDA2 text encoder and the FLUX.2 VAE. Choose the 50-step Base checkpoint or the +4-step Turbo checkpoint, keeping each variant's text encoder and connectors +together. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Transformer | Choose the matching Base or Turbo weights below | `--diffusion-model` | Required | +| Text encoder | Use the text encoder from the same Base or Turbo checkpoint | `--llm` | Required; variants are not interchangeable | +| Tokenizer | `tokenizer/tokenizer.json` from either original repository below | `--tokenizer` | Required; shared by Base and Turbo | +| FLUX.2 VAE | `llada_vae.safetensors`, downloaded from either original repository below | `--vae` | Required; shared by Base and Turbo | +| Connectors | Matching `*-connectors.safetensors` or `*-connectors-edit.safetensors` below | `--embeddings-connectors` | Editing requires the file including SigVQ | + +Four components are required: a transformer, a text encoder, a VAE, and a connectors file +holding the QueryFormer, the text projection and, for editing, the SigVQ image encoder. + +The two published checkpoints are **not** interchangeable. LLaDA-Image-Turbo and LLaDA-Image +ship different transformers, text encoders, QueryFormers and text projections; only the VAE, +the SigVQ encoder and the tokenizer are shared. Mixing the two produces degraded output rather +than a clean error, so keep each checkpoint's files together. + +Both need an external LLaDA2 `tokenizer.json`, which is not embedded in sd.cpp and is the same +file for either checkpoint. Take `tokenizer/tokenizer.json` from either repository and pass it +with `--tokenizer`. See [JSON tokenizers](../guides/tokenizers.md) for CLI and C API usage. + +### LLaDA-Image-Turbo (4 steps) + +Converted transformer, text encoder and pre-merged connectors are at +https://huggingface.co/fszontagh/LLaDA-Image-Turbo-GGUF: + +- `llada-image-turbo-f16.gguf` +- `llada-image-turbo-text_encoder-q8_0.gguf` +- `llada-image-turbo-connectors.safetensors` for text to image, or + `llada-image-turbo-connectors-edit.safetensors`, which also carries the SigVQ encoder that + editing needs. + +Other quantizations of the transformer and the text encoder are in the same repository. + +The VAE comes from the original repository, +https://huggingface.co/inclusionAI/LLaDA-Image-Turbo: `vae/diffusion_pytorch_model.safetensors`, +referred to below as `llada_vae.safetensors`. + +### LLaDA-Image (50 steps) + +Converted transformer, text encoder and pre-merged connectors are at +https://huggingface.co/fszontagh/LLaDA-Image-GGUF: + +- `llada-image-f16.gguf` +- `llada-image-text_encoder-q8_0.gguf` +- `llada-image-connectors.safetensors` for text to image, or + `llada-image-connectors-edit.safetensors`, which also carries the SigVQ encoder that editing + needs. + +Other quantizations of the transformer and the text encoder are in the same repository. + +The VAE comes from the original repository, +https://huggingface.co/inclusionAI/LLaDA-Image, and is the same file as the Turbo one. + +### Optional conversion + +Run executable conversion examples from the build directory using the +[shared path and shell conventions](README.md#example-conventions). + +#### Converting the weights yourself + +The transformer has to go in through `--diffusion-model` so that its tensor names keep the +prefix the loader expects, while the text encoder goes in through `-m`: + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M convert ` + --diffusion-model ..\models\transformer\diffusion_pytorch_model.safetensors.index.json ` + --type f16 ` + -o llada-image-f16.gguf +.\bin\Release\sd-cli.exe ` + -M convert ` + -m ..\models\text_encoder\model.safetensors.index.json ` + --type q8_0 ` + -o llada-image-text_encoder-q8_0.gguf +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M convert \ + --diffusion-model ../models/transformer/diffusion_pytorch_model.safetensors.index.json \ + --type f16 \ + -o llada-image-f16.gguf +./bin/sd-cli \ + -M convert \ + -m ../models/text_encoder/model.safetensors.index.json \ + --type q8_0 \ + -o llada-image-text_encoder-q8_0.gguf +``` + +#### Building the connector file yourself + +`--embeddings-connectors` takes one file, so the QueryFormer, the text projection and +(for editing) the SigVQ encoder have to be combined into a single Safetensors file, each +tensor name prefixed with its component name. Leaving `sigvq` out skips loading the 2.6 GB +encoder: + +```python +from safetensors.torch import load_file, save_file + +merged = {} +for prefix, path in [ + ("queryformer", "queryformer/diffusion_pytorch_model.safetensors"), + ("text_projection", "text_projection/diffusion_pytorch_model.safetensors"), + ("sigvq", "sigvq/diffusion_pytorch_model.safetensors"), +]: + for name, tensor in load_file(path).items(): + merged[f"{prefix}.{name}"] = tensor +save_file(merged, "llada_connectors.safetensors") +``` + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| LLaDA Image Turbo (`llada-image-turbo`) | `diffusion_model`, `llm`, `tokenizer`, `embeddings_connectors`, `vae`, `tae` | +| LLaDA Image (base) (`llada-image`) | `diffusion_model`, `llm`, `tokenizer`, `embeddings_connectors`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### Text to image + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\llada-image-turbo-f16.gguf ` + --llm ..\models\text_encoders\llada-image-turbo-text_encoder-q8_0.gguf ` + --tokenizer ..\models\tokenizers\tokenizer.json ` + --vae ..\models\vae\llada_vae.safetensors ` + --embeddings-connectors ..\models\text_encoders\llada-image-turbo-connectors.safetensors ` + --prompt "a lovely cat holding a sign says 'llada.cpp'" ` + --steps 4 ` + --cfg-scale 1.0 ` + --width 1024 ` + --height 1024 ` + --seed 42 ` + --output output.png +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/llada-image-turbo-f16.gguf \ + --llm ../models/text_encoders/llada-image-turbo-text_encoder-q8_0.gguf \ + --tokenizer ../models/tokenizers/tokenizer.json \ + --vae ../models/vae/llada_vae.safetensors \ + --embeddings-connectors ../models/text_encoders/llada-image-turbo-connectors.safetensors \ + --prompt "a lovely cat holding a sign says 'llada.cpp'" \ + --steps 4 \ + --cfg-scale 1.0 \ + --width 1024 \ + --height 1024 \ + --seed 42 \ + --output output.png +``` + +LLaDA-Image example + +#### Image editing + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\llada-image-turbo-f16.gguf ` + --llm ..\models\text_encoders\llada-image-turbo-text_encoder-q8_0.gguf ` + --tokenizer ..\models\tokenizers\tokenizer.json ` + --vae ..\models\vae\llada_vae.safetensors ` + --embeddings-connectors ..\models\text_encoders\llada-image-turbo-connectors-edit.safetensors ` + --ref-image ..\inputs\input.png ` + --prompt "change the sign text to 'sd.cpp'" ` + --steps 4 ` + --cfg-scale 1.0 ` + --width 1024 ` + --height 1024 ` + --output output.png +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/llada-image-turbo-f16.gguf \ + --llm ../models/text_encoders/llada-image-turbo-text_encoder-q8_0.gguf \ + --tokenizer ../models/tokenizers/tokenizer.json \ + --vae ../models/vae/llada_vae.safetensors \ + --embeddings-connectors ../models/text_encoders/llada-image-turbo-connectors-edit.safetensors \ + --ref-image ../inputs/input.png \ + --prompt "change the sign text to 'sd.cpp'" \ + --steps 4 \ + --cfg-scale 1.0 \ + --width 1024 \ + --height 1024 \ + --output output.png +``` + +LLaDA-Image edit example + +See [edit.md](../guides/edit.md) for the shared reference-image options. LLaDA-Image uses the +`llada_image` preset by default, resizing the reference image to the output width and height +before VAE encoding. SigVQ uses bilinear resizing to half the output resolution and inputs +normalized to `[-1, 1]`. CFG keeps the source latent in both branches and uses SigVQ features +only in the positive branch. Editing requires connectors that include the SigVQ weights. + +## Notes and limitations + +- Use 4 steps and `--cfg-scale 1.0` for LLaDA-Image-Turbo; the guidance is distilled away, so + a higher CFG degrades output and doubles the text encoder cost. The 50-step base model uses + `--steps 50 --cfg-scale 5`. +- Width and height are rounded up to a multiple of 16. For editing the reference pipeline + requires them to be divisible by 32. +- Edit the 50-step base model at 1024x1024. At 512x512 it returns the reference image almost + unchanged instead of applying the instruction; LLaDA-Image-Turbo edits correctly at both. +- Editing runs the reference and the target in one sequence, so it needs roughly twice the + tokens of text to image at the same size. On 12 GB, editing at 1024x1024 needs + flash attention; without it the diffusion graph does not fit. +- The weights total about 16 GB, but segmented execution streams them, so a much smaller + budget works. At 512x512, `--max-vram 6` costs almost nothing over unconstrained execution, + and `--max-vram 3` still produces byte-identical output at roughly 2.5x the time. +- `--scheduler` defaults to `llada_image`, which reproduces the reference Kumaraswamy sigma + grid. `--extra-sample-args uniform=1` selects the uniform grid instead. +- Prompt templating is handled automatically; pass a plain description. +- VQ-conditioned generation (`generation_mode="vq"`, where the text encoder decodes image + tokens before diffusion) is not implemented. diff --git a/docs/models/longcat_image.md b/docs/models/longcat_image.md new file mode 100644 index 0000000000..ec2fae131c --- /dev/null +++ b/docs/models/longcat_image.md @@ -0,0 +1,70 @@ +# LongCat Image + +LongCat-Image uses a LongCat diffusion transformer, the FLUX VAE, and Qwen2.5-VL as the LLM text encoder. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| LongCat Image | [Safetensors](https://huggingface.co/Comfy-Org/LongCat-Image/tree/main/split_files/diffusion_models)
[GGUF](https://huggingface.co/vantagewithai/LongCat-Image-GGUF/tree/main/comfy) | `--diffusion-model` | Choose matching variant | +| LongCat Image Edit | LongCat Image Edit Turbo: [Download](https://huggingface.co/meituan-longcat/LongCat-Image-Edit-Turbo)
[GGUF](https://huggingface.co/vantagewithai/LongCat-Image-Edit-GGUF/tree/main) | `--diffusion-model` | Choose matching variant | +| vae | [Safetensors](https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/ae.safetensors) | `--vae` | Use matching VAE | +| qwen_2.5_vl 7b | [Safetensors](https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/tree/main/split_files/text_encoders)
[GGUF](https://huggingface.co/mradermacher/Qwen2.5-VL-7B-Instruct-GGUF/tree/main)
For image editing with GGUF text encoders, also download the matching mmproj file and pass it with `--llm_vision`. | `--llm` | Use matching text encoder | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| LongCat Image (`longcat-image`) | `diffusion_model`, `llm`, `vae`, `tae` | +| LongCat Image Edit (`longcat-image-edit`) | `diffusion_model`, `llm`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +LongCat uses quoted text for character-level text rendering. Put target text inside single quotes, double quotes, or Chinese quotes. + +#### LongCat Image + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\LongCat-Image-Q4_K_M.gguf ` + --vae ..\models\vae\ae.sft ` + --llm ..\models\text_encoders\Qwen2.5-VL-7B-Instruct-Q8_0.gguf ` + -p "a lovely cat holding a sign says 'longcat.cpp'" ` + --cfg-scale 5.0 ` + --sampling-method euler ` + --flow-shift 3 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/LongCat-Image-Q4_K_M.gguf \ + --vae ../models/vae/ae.sft \ + --llm ../models/text_encoders/Qwen2.5-VL-7B-Instruct-Q8_0.gguf \ + -p "a lovely cat holding a sign says 'longcat.cpp'" \ + --cfg-scale 5.0 \ + --sampling-method euler \ + --flow-shift 3 \ + --offload-to-cpu \ + -v +``` + +longcat example diff --git a/docs/models/lora.md b/docs/models/lora.md new file mode 100644 index 0000000000..56cb60b7d3 --- /dev/null +++ b/docs/models/lora.md @@ -0,0 +1,62 @@ +# LoRA + +LoRA weights adapt a compatible base model. Select a LoRA by name and strength +using `` in the prompt. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Base model | [Compatible model checkpoint](README.md) | `-m` / model component options | Required | +| LoRA weights | `marblesh.safetensors` or `marblesh.ckpt` for the example | `--lora-model-dir` and `` in the prompt | Match the base model family | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Load a compatible base model with its recipe, then select the LoRA for generation. +See [Web UI guide](../../examples/server/README.md#web-ui) for the shared workflow. + +### CLI + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -m ..\models\checkpoints\v1-5-pruned-emaonly.safetensors ` + --lora-model-dir ..\models\loras ` + -p "a lovely cat" +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -m ../models/checkpoints/v1-5-pruned-emaonly.safetensors \ + --lora-model-dir ../models/loras \ + -p "a lovely cat" +``` + +The example loads `../models/loras/marblesh.safetensors` or +`../models/loras/marblesh.ckpt` (use backslashes on Windows). + +## Options + +### LoRA apply mode + +There are two ways to apply LoRA: **immediately** and **at_runtime**. You can specify it using the `--lora-apply-mode` parameter. + +By default, the mode is selected automatically: + +* If the model weights contain any quantized parameters, the **at_runtime** mode is used; +* Otherwise, the **immediately** mode is used. + +The **immediately** mode may have precision and compatibility issues with quantized parameters, but it usually offers faster inference speed and, in some cases, lower memory usage. +In contrast, the **at_runtime** mode provides better compatibility and higher precision, but inference may be slower and memory usage may be higher in some cases. + +## Notes and limitations + +Choose LoRA weights trained for the base model. See the +[WebUI-style LoRA syntax](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Features#lora). diff --git a/docs/models/ltx2.md b/docs/models/ltx2.md new file mode 100644 index 0000000000..1785487224 --- /dev/null +++ b/docs/models/ltx2.md @@ -0,0 +1,470 @@ +# LTX-2 + +LTX-2.3 and LTX-2.5 support text-to-video and image-to-video generation. +LTX-2.3 uses Gemma 3 with separate embedding connectors. LTX-2.5 requires its +LTX-specific Gemma 4 encoder with the text projection included and the conv video VAE. + +## Model files + +### LTX-2.3 + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| LTX-2.3 transformer | [Safetensors](https://huggingface.co/Kijai/LTX2.3_comfy/tree/main/diffusion_models)
[GGUF](https://huggingface.co/unsloth/LTX-2.3-GGUF/tree/main) | `--diffusion-model` | Required | +| Gemma 3 12B text encoder | [GGUF](https://huggingface.co/unsloth/gemma-3-12b-it-GGUF/tree/main) | `--llm` | Required for 2.3 | +| LTX-2.3 connectors | [Safetensors](https://huggingface.co/unsloth/LTX-2.3-GGUF/tree/main/text_encoders) | `--embeddings-connectors` | Required for 2.3 | +| LTX-2.3 video VAE | [Safetensors](https://huggingface.co/unsloth/LTX-2.3-GGUF/tree/main/vae) | `--vae` | Required for these examples | +| LTX-2.3 audio VAE | [Safetensors](https://huggingface.co/unsloth/LTX-2.3-GGUF/tree/main/vae) | `--audio-vae` | For audio output | +| LTX-2.3 spatial upscaler | [Safetensors](https://huggingface.co/Lightricks/LTX-2.3/resolve/main/ltx-2.3-spatial-upscaler-x2-1.1.safetensors) | `--hires-upscalers-dir` | For latent spatial upscaling | + +### LTX-2.5 + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| LTX-2.5 transformer | [Safetensors](https://huggingface.co/Lightricks/LTX-2.5/tree/main/diffusion_models)
[GGUF](https://huggingface.co/vantagewithai/LTX-2.5-GGUF/tree/main) | `--diffusion-model` | Required | +| LTX Gemma 4 12B with projection | projection bundled in, so no separate `--embeddings-connectors` file is needed. Google's
stock Gemma 4 is not a substitute.
[Safetensors](https://huggingface.co/Lightricks/LTX-2.5/blob/main/text_encoders/gemma4-12b-with-proj-ltx-2.5-bf16.safetensors) | `--llm` | Required; stock Gemma 4 is incompatible | +| LTX-2.5 conv video VAE | The default `ltx-2.5-video-vae-bf16.safetensors` is a diffusion decoder, which is not
implemented here.
[Safetensors](https://huggingface.co/Lightricks/LTX-2.5/blob/main/vae/ltx-2.5-video-vae-conv-bf16.safetensors) | `--vae` | Use the conv variant | +| LTX-2.5 audio VAE | [Safetensors](https://huggingface.co/Lightricks/LTX-2.5/blob/main/vae/ltx-2.5-audio-vae-bf16.safetensors) | `--audio-vae` | For audio output | +| LTX-2.5 spatial upscaler | [Safetensors](https://huggingface.co/Lightricks/LTX-2.5/blob/main/latent_upscale_models/ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0.safetensors) | `--hires-upscalers-dir` | For latent spatial upscaling | + +### Optional conversion + +Run executable conversion examples from the build directory using the +[shared path and shell conventions](README.md#example-conventions). + +#### Quantize the LTX-2.5 text encoder + +To run the text encoder quantized, convert it once with sd-cli: + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M convert ` + -m ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-bf16.safetensors ` + --type q8_0 ` + -o ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M convert \ + -m ../models/text_encoders/gemma4-12b-with-proj-ltx-2.5-bf16.safetensors \ + --type q8_0 \ + -o ../models/text_encoders/gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf +``` + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| LTX 2.3 dev (`ltx23-dev`) | `diffusion_model`, `llm`, `embeddings_connectors`, `vae`, `tae`, `audio_vae` | +| LTX 2.3 Distilled 1.1 (`ltx23-distilled`) | `diffusion_model`, `llm`, `embeddings_connectors`, `vae`, `tae`, `audio_vae` | +| LTX 2.5 dev (`ltx25-dev`) | `diffusion_model`, `llm`, `vae`, `tae`, `audio_vae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### LTX-2.3 dev T2V + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\ltx-2.3-22b-dev-UD-Q4_K_M.gguf ` + --vae ..\models\vae\ltx-2.3-22b-dev_video_vae.safetensors ` + --audio-vae ..\models\vae\ltx-2.3-22b-dev_audio_vae.safetensors ` + --llm ..\models\text_encoders\gemma-3-12b-it-qat-UD-Q4_K_XL.gguf ` + --embeddings-connectors ..\models\text_encoders\ltx-2.3-22b-dev_embeddings_connectors.safetensors ` + -p "a lovely cat" ` + -n "worst quality, low quality, blurry, distorted, artifacts" ` + --cfg-scale 6.0 ` + --sampling-method euler ` + -W 1280 ` + -H 720 ` + --video-frames 33 ` + --fps 24 ` + --offload-to-cpu ` + -v ` + -o t2v.webm +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/ltx-2.3-22b-dev-UD-Q4_K_M.gguf \ + --vae ../models/vae/ltx-2.3-22b-dev_video_vae.safetensors \ + --audio-vae ../models/vae/ltx-2.3-22b-dev_audio_vae.safetensors \ + --llm ../models/text_encoders/gemma-3-12b-it-qat-UD-Q4_K_XL.gguf \ + --embeddings-connectors ../models/text_encoders/ltx-2.3-22b-dev_embeddings_connectors.safetensors \ + -p "a lovely cat" \ + -n "worst quality, low quality, blurry, distorted, artifacts" \ + --cfg-scale 6.0 \ + --sampling-method euler \ + -W 1280 \ + -H 720 \ + --video-frames 33 \ + --fps 24 \ + --offload-to-cpu \ + -v \ + -o t2v.webm +``` + + + +#### LTX-2.3 dev I2V + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\ltx-2.3-22b-dev-UD-Q4_K_M.gguf ` + --vae ..\models\vae\ltx-2.3-22b-dev_video_vae.safetensors ` + --audio-vae ..\models\vae\ltx-2.3-22b-dev_audio_vae.safetensors ` + --llm ..\models\text_encoders\gemma-3-12b-it-qat-UD-Q4_K_XL.gguf ` + --embeddings-connectors ..\models\text_encoders\ltx-2.3-22b-dev_embeddings_connectors.safetensors ` + -p "a lovely cat" ` + -i ..\inputs\ernie_image\turbo_example.png ` + --cfg-scale 6.0 ` + --sampling-method euler ` + -W 1280 ` + -H 720 ` + --video-frames 33 ` + --offload-to-cpu ` + -v ` + -o i2v.webm +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/ltx-2.3-22b-dev-UD-Q4_K_M.gguf \ + --vae ../models/vae/ltx-2.3-22b-dev_video_vae.safetensors \ + --audio-vae ../models/vae/ltx-2.3-22b-dev_audio_vae.safetensors \ + --llm ../models/text_encoders/gemma-3-12b-it-qat-UD-Q4_K_XL.gguf \ + --embeddings-connectors ../models/text_encoders/ltx-2.3-22b-dev_embeddings_connectors.safetensors \ + -p "a lovely cat" \ + -i ../inputs/ernie_image/turbo_example.png \ + --cfg-scale 6.0 \ + --sampling-method euler \ + -W 1280 \ + -H 720 \ + --video-frames 33 \ + --offload-to-cpu \ + -v \ + -o i2v.webm +``` + + + +#### LTX-2.3 dev FLF2V + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\ltx-2.3-22b-dev-UD-Q4_K_M.gguf ` + --vae ..\models\vae\ltx-2.3-22b-dev_video_vae.safetensors ` + --audio-vae ..\models\vae\ltx-2.3-22b-dev_audio_vae.safetensors ` + --llm ..\models\text_encoders\gemma-3-12b-it-qat-UD-Q4_K_XL.gguf ` + --embeddings-connectors ..\models\text_encoders\ltx-2.3-22b-dev_embeddings_connectors.safetensors ` + -p "glass flower blossom" ` + --init-img ..\inputs\start_image.png ` + --end-img ..\inputs\end_image.png ` + --cfg-scale 6.0 ` + --sampling-method euler ` + -W 1280 ` + -H 720 ` + --video-frames 33 ` + --offload-to-cpu ` + -v ` + -o flf2v.webm +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/ltx-2.3-22b-dev-UD-Q4_K_M.gguf \ + --vae ../models/vae/ltx-2.3-22b-dev_video_vae.safetensors \ + --audio-vae ../models/vae/ltx-2.3-22b-dev_audio_vae.safetensors \ + --llm ../models/text_encoders/gemma-3-12b-it-qat-UD-Q4_K_XL.gguf \ + --embeddings-connectors ../models/text_encoders/ltx-2.3-22b-dev_embeddings_connectors.safetensors \ + -p "glass flower blossom" \ + --init-img ../inputs/start_image.png \ + --end-img ../inputs/end_image.png \ + --cfg-scale 6.0 \ + --sampling-method euler \ + -W 1280 \ + -H 720 \ + --video-frames 33 \ + --offload-to-cpu \ + -v \ + -o flf2v.webm +``` + + + +#### LTX-2.3 spatial latent upscale + +LTX spatial latent upscale runs a model-backed x2 latent upsampler between the low-resolution video pass and the high-resolution refine pass. `-W` and `-H` are the pre-upscale generation size; the spatial upsampler produces x2 latent dimensions. + +Put `ltx-2.3-spatial-upscaler-x2-1.1.safetensors` under the directory passed to `--hires-upscalers-dir`, then use the model name without path or extension in `--hires-upscaler`. + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\ltx-2.3-22b-dev-UD-Q4_K_M.gguf ` + --vae ..\models\vae\ltx-2.3-22b-dev_video_vae.safetensors ` + --audio-vae ..\models\vae\ltx-2.3-22b-dev_audio_vae.safetensors ` + --llm ..\models\text_encoders\gemma-3-12b-it-qat-UD-Q4_K_XL.gguf ` + --embeddings-connectors ..\models\text_encoders\ltx-2.3-22b-dev_embeddings_connectors.safetensors ` + --hires-upscalers-dir ..\models\latent_upscale_models ` + -p "a lovely cat" ` + -i ..\inputs\ernie_image\turbo_example.png ` + --cfg-scale 6.0 ` + --sampling-method euler ` + -W 640 ` + -H 360 ` + --video-frames 33 ` + --hires-upscaler ltx-2.3-spatial-upscaler-x2-1.1 ` + --hires ` + --hires-steps 4 ` + --offload-to-cpu ` + -v ` + -o hires_i2v.webm +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/ltx-2.3-22b-dev-UD-Q4_K_M.gguf \ + --vae ../models/vae/ltx-2.3-22b-dev_video_vae.safetensors \ + --audio-vae ../models/vae/ltx-2.3-22b-dev_audio_vae.safetensors \ + --llm ../models/text_encoders/gemma-3-12b-it-qat-UD-Q4_K_XL.gguf \ + --embeddings-connectors ../models/text_encoders/ltx-2.3-22b-dev_embeddings_connectors.safetensors \ + --hires-upscalers-dir ../models/latent_upscale_models \ + -p "a lovely cat" \ + -i ../inputs/ernie_image/turbo_example.png \ + --cfg-scale 6.0 \ + --sampling-method euler \ + -W 640 \ + -H 360 \ + --video-frames 33 \ + --hires-upscaler ltx-2.3-spatial-upscaler-x2-1.1 \ + --hires \ + --hires-steps 4 \ + --offload-to-cpu \ + -v \ + -o hires_i2v.webm +``` + +By default, the hires refine pass uses the main sampler and scheduler, then trims the second-pass sigma schedule by `--hires-denoising-strength` (`0.7` by default). To reproduce a ComfyUI-style explicit refine schedule, pass custom hires sigmas: + +**Windows (PowerShell) arguments** + +```text +--hires-sigmas "0.85,0.725,0.421875,0.0" +``` + +**Linux/macOS arguments** + +```text +--hires-sigmas 0.85,0.725,0.421875,0.0 +``` + + + +#### LTX-2.5 dev T2V + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\ltx-2.5-22b-dev-transformer-Q8_0.gguf ` + --vae ..\models\vae\ltx-2.5-video-vae-conv-bf16.safetensors ` + --audio-vae ..\models\vae\ltx-2.5-audio-vae-bf16.safetensors ` + --llm ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf ` + -p "A wide aerial shot of a red vintage convertible driving along a coastal cliff road at sunset, waves crashing below" ` + -n "worst quality, low quality, blurry, distorted, artifacts" ` + --cfg-scale 3.0 ` + --sampling-method euler ` + -W 1280 ` + -H 720 ` + --video-frames 121 ` + --fps 24 ` + --offload-to-cpu ` + -v ` + -o t2v.webm +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/ltx-2.5-22b-dev-transformer-Q8_0.gguf \ + --vae ../models/vae/ltx-2.5-video-vae-conv-bf16.safetensors \ + --audio-vae ../models/vae/ltx-2.5-audio-vae-bf16.safetensors \ + --llm ../models/text_encoders/gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf \ + -p "A wide aerial shot of a red vintage convertible driving along a coastal cliff road at sunset, waves crashing below" \ + -n "worst quality, low quality, blurry, distorted, artifacts" \ + --cfg-scale 3.0 \ + --sampling-method euler \ + -W 1280 \ + -H 720 \ + --video-frames 121 \ + --fps 24 \ + --offload-to-cpu \ + -v \ + -o t2v.webm +``` + +#### LTX-2.5 dev I2V + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\ltx-2.5-22b-dev-transformer-Q8_0.gguf ` + --vae ..\models\vae\ltx-2.5-video-vae-conv-bf16.safetensors ` + --audio-vae ..\models\vae\ltx-2.5-audio-vae-bf16.safetensors ` + --llm ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf ` + -p "a lovely cat blinking slowly, gentle camera push in" ` + -i ..\inputs\ernie_image\turbo_example.png ` + --cfg-scale 3.0 ` + --sampling-method euler ` + -W 1280 ` + -H 720 ` + --video-frames 121 ` + --offload-to-cpu ` + -v ` + -o i2v.webm +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/ltx-2.5-22b-dev-transformer-Q8_0.gguf \ + --vae ../models/vae/ltx-2.5-video-vae-conv-bf16.safetensors \ + --audio-vae ../models/vae/ltx-2.5-audio-vae-bf16.safetensors \ + --llm ../models/text_encoders/gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf \ + -p "a lovely cat blinking slowly, gentle camera push in" \ + -i ../inputs/ernie_image/turbo_example.png \ + --cfg-scale 3.0 \ + --sampling-method euler \ + -W 1280 \ + -H 720 \ + --video-frames 121 \ + --offload-to-cpu \ + -v \ + -o i2v.webm +``` + +#### LTX-2.5 spatial latent upscale + +Works like the LTX-2.3 upscaler described above; put +`ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0.safetensors` under `--hires-upscalers-dir` and +pass its name without path or extension to `--hires-upscaler`. + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\ltx-2.5-22b-dev-transformer-Q8_0.gguf ` + --vae ..\models\vae\ltx-2.5-video-vae-conv-bf16.safetensors ` + --audio-vae ..\models\vae\ltx-2.5-audio-vae-bf16.safetensors ` + --llm ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf ` + --hires-upscalers-dir ..\models\latent_upscale_models ` + -p "a lovely cat" ` + --cfg-scale 3.0 ` + --sampling-method euler ` + -W 640 ` + -H 360 ` + --video-frames 121 ` + --hires-upscaler ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0 ` + --hires ` + --hires-steps 6 ` + --offload-to-cpu ` + -v ` + -o hires_t2v.webm +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/ltx-2.5-22b-dev-transformer-Q8_0.gguf \ + --vae ../models/vae/ltx-2.5-video-vae-conv-bf16.safetensors \ + --audio-vae ../models/vae/ltx-2.5-audio-vae-bf16.safetensors \ + --llm ../models/text_encoders/gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf \ + --hires-upscalers-dir ../models/latent_upscale_models \ + -p "a lovely cat" \ + --cfg-scale 3.0 \ + --sampling-method euler \ + -W 640 \ + -H 360 \ + --video-frames 121 \ + --hires-upscaler ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0 \ + --hires \ + --hires-steps 6 \ + --offload-to-cpu \ + -v \ + -o hires_t2v.webm +``` + +## Notes and limitations + +Not implemented: + +- The diffusion video decoder (`ltx-2.5-video-vae-bf16.safetensors`). Use the conv VAE. +- The temporal latent upscaler and the duration head (`--auto-duration`); pass + `--video-frames` explicitly. diff --git a/docs/models/mage_flow.md b/docs/models/mage_flow.md new file mode 100644 index 0000000000..eaccdcf80f --- /dev/null +++ b/docs/models/mage_flow.md @@ -0,0 +1,108 @@ +# Mage-Flow + +[Mage-Flow](https://github.com/microsoft/Mage) uses a 4B native-resolution multimodal diffusion transformer, Qwen3-VL for text and image conditioning, and the 128-channel Mage-VAE. Both text-to-image and instruction-based image editing checkpoints are supported. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Mage-Flow diffusion | Mage-Flow diffusion from [Download](https://huggingface.co/Comfy-Org/Mage-Flow/tree/main/diffusion_models) | `--diffusion-model` | Required | +| Mage-Flow vae | Mage-Flow vae from [Download](https://huggingface.co/Comfy-Org/Mage-Flow/tree/main/vae) | `--vae` | Use matching VAE | +| Qwen3-VL 4B | [Safetensors](https://huggingface.co/Comfy-Org/Krea-2/tree/main/text_encoders)
[GGUF](https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct-GGUF/tree/main) | `--llm` | Use matching text encoder | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| Mage Flow (`mage-flow`) | `diffusion_model`, `llm`, `vae` | +| Mage Flow Turbo (`mage-flow-turbo`) | `diffusion_model`, `llm`, `vae` | +| Mage Flow Edit (`mage-flow-edit`) | `diffusion_model`, `llm`, `vae`, `llm_vision` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### Text-to-image + +Use 30 steps for Base models and 4 steps with `--cfg-scale 1` for Turbo models. Image dimensions must be multiples of 16; the official checkpoints are trained for native resolutions from 512 to 2048 pixels. + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\Mage-Flow-Turbo.safetensors ` + --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf ` + --vae ..\models\vae\mage_vae.safetensors ` + -p "a lovely cat holding a sign says 'mage.cpp'" ` + --cfg-scale 1.0 ` + --steps 4 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/Mage-Flow-Turbo.safetensors \ + --llm ../models/text_encoders/Qwen3-VL-4B-Instruct-Q4_K_M.gguf \ + --vae ../models/vae/mage_vae.safetensors \ + -p "a lovely cat holding a sign says 'mage.cpp'" \ + --cfg-scale 1.0 \ + --steps 4 \ + --offload-to-cpu \ + -v +``` + +Mage-Flow example + +#### Image editing + +Mage-Flow-Edit accepts one or more reference images. The default `mage_flow` reference preset sends each image to both Qwen3-VL and the diffusion transformer, caps the VLM copy's longest edge at 384 pixels, and keeps the VAE copy at the requested output resolution. + +For the Turbo edit checkpoint, use 4 steps and `--cfg-scale 1`. + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\Mage-Flow-Edit.safetensors ` + --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf ` + --llm_vision ..\models\text_encoders\Qwen3-VL-4B-Instruct-mmproj-BF16.gguf ` + --vae ..\models\vae\mage_vae.safetensors ` + -r ..\inputs\flux\flux1-dev-q8_0.png ` + -p "change 'flux.cpp' to 'mage.cpp'" ` + --cfg-scale 4.0 ` + --sampling-method euler ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/Mage-Flow-Edit.safetensors \ + --llm ../models/text_encoders/Qwen3-VL-4B-Instruct-Q4_K_M.gguf \ + --llm_vision ../models/text_encoders/Qwen3-VL-4B-Instruct-mmproj-BF16.gguf \ + --vae ../models/vae/mage_vae.safetensors \ + -r ../inputs/flux/flux1-dev-q8_0.png \ + -p "change 'flux.cpp' to 'mage.cpp'" \ + --cfg-scale 4.0 \ + --sampling-method euler \ + --offload-to-cpu \ + -v +``` + +Mage-Flow-Edit example diff --git a/docs/models/ming_image.md b/docs/models/ming_image.md new file mode 100644 index 0000000000..53dc7a96e6 --- /dev/null +++ b/docs/models/ming_image.md @@ -0,0 +1,77 @@ +# Ming-Image + +[Ming-Image](https://github.com/inclusionAI/Ming-Image) 0.1 Design uses a 6B diffusion transformer (DiT), Ling-mini-2.0 for text conditioning, and the Ming-Image VAE. Text-to-image generation with RGBA output is supported. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Ming-Image 0.1 Design DiT | [Safetensors](https://huggingface.co/Comfy-Org/Ming-Image/tree/main/diffusion_models) | `--diffusion-model` | Required | +| Ling-mini-2.0 BF16 | [Safetensors](https://huggingface.co/Comfy-Org/Ming-Image/tree/main/text_encoders) | `--llm` | Use matching text encoder | +| Ming-Image VAE | [Safetensors](https://huggingface.co/Comfy-Org/Ming-Image/tree/main/vae) | `--vae` | Use matching VAE | +| Ling tokenizer | tokenizer.json: [Download](https://huggingface.co/inclusionAI/Ming-Image-0.1-Design/blob/main/mllm/tokenizer.json) | `--tokenizer` | Required; match text encoder | + +The example below uses `ming_image_0.1_design_bf16.safetensors` for the DiT. You can also use `ming_image_0.1_design_int8_convrot.safetensors` with [INT8 convrot support](../guides/int8_convrot.md). Use the BF16 text encoder. + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| Ming Image 0.1 (`ming-image`) | `diffusion_model`, `llm`, `tokenizer`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### Text-to-image + +Pass the Ling `tokenizer.json` with `--tokenizer` and use the matching Ming-Image VAE. Image dimensions must be multiples of 16. Save the output as PNG to preserve the alpha channel. + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\ming_image_0.1_design_bf16.safetensors ` + --llm ..\models\text_encoders\ming_image_0.1_ling_mini_2.0_bf16.safetensors ` + --vae ..\models\vae\ming_image_vae_bf16.safetensors ` + --tokenizer ..\models\tokenizers\tokenizer.json ` + -p "A cheerful orange cat sticker, transparent background" ` + --steps 12 ` + --cfg-scale 1 ` + --sampling-method euler ` + --width 1024 ` + --height 1024 ` + --offload-to-cpu ` + -v ` + -o ming_image.png +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/ming_image_0.1_design_bf16.safetensors \ + --llm ../models/text_encoders/ming_image_0.1_ling_mini_2.0_bf16.safetensors \ + --vae ../models/vae/ming_image_vae_bf16.safetensors \ + --tokenizer ../models/tokenizers/tokenizer.json \ + -p "A cheerful orange cat sticker, transparent background" \ + --steps 12 \ + --cfg-scale 1 \ + --sampling-method euler \ + --width 1024 \ + --height 1024 \ + --offload-to-cpu \ + -v \ + -o ming_image.png +``` diff --git a/docs/models/minimax_h3.md b/docs/models/minimax_h3.md new file mode 100644 index 0000000000..862b04d3e9 --- /dev/null +++ b/docs/models/minimax_h3.md @@ -0,0 +1,458 @@ +# MiniMax-H3 + +MiniMax-H3 jointly generates video and stereo audio with a packed diffusion +transformer. The implementation supports text-to-audio-video (T2VA), optional +first-frame conditioning (I2VA), first/last-frame conditioning (FL2VA), and +image/video/audio reference conditioning (Ref2VA). + +## Model files + +Pass the four MiniMax-H3 components separately: + +- `--diffusion-model`: MiniMax-H3 diffusion transformer +- `--vae`: MiniMax-H3 video VAE +- `--audio-vae`: MiniMax-H3 audio VAE +- `--llm`: the MiniMax-H3 Qwen3-VL-32B text encoder, or Qwen3-VL-4B with the P-H3 text connector + +For the 32B option, use the MiniMax-H3 variant: Qwen3-VL-32B truncated to 50 +language layers and exported without the final language-model normalization. + +For the smaller 4B option, pass the P-H3-Image text encoder, which bundles the +connector, with `--llm`. Alternatively, pass Qwen3-VL-4B with `--llm` and the +[separate P-H3 text connector](https://huggingface.co/PrunaAI/P-H3-Image/blob/main/text_connector/diffusion_pytorch_model.safetensors) +with `--embeddings-connectors`. For example, replace the `--llm` argument in the +commands below with either: + +**Windows (PowerShell) arguments** + +```text +--llm ..\models\text_encoders\p-h3-image_text_encoder_int8_convrot.safetensors +``` + +**Linux/macOS arguments** + +```text +--llm ../models/text_encoders/p-h3-image_text_encoder_int8_convrot.safetensors +``` + +or: + +**Windows (PowerShell) arguments** + +```text +--llm ..\models\text_encoders\qwen3vl_4b.gguf ` + --embeddings-connectors ..\models\text_encoders\p-h3-image_text_connector.safetensors +``` + +**Linux/macOS arguments** + +```text +--llm ../models/text_encoders/qwen3vl_4b.gguf \ + --embeddings-connectors ../models/text_encoders/p-h3-image_text_connector.safetensors +``` + +Reference-image and reference-video conditioning needs the matching encoder's +Qwen3-VL vision tower, including the three DeepStack mergers. If stored +separately, pass it with `--llm_vision`. Keep using the MiniMax-H3 video and audio +VAEs with either text encoder. + +Both the original time-embedder DiT and the smaller AdaLN curve-table variant +are detected from their weights. + +### Download weights + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| minimax_h3_fl2va/minimax_h3_ref2va | [Safetensors](https://huggingface.co/Comfy-Org/MiniMax-H3/tree/main/diffusion_models)
[GGUF](https://huggingface.co/leejet/MiniMax-H3-GGUF/tree/main) | `--diffusion-model` | Required | +| qwen3vl_32b_minimax_h3 | [Safetensors](https://huggingface.co/Comfy-Org/MiniMax-H3/tree/main/text_encoders)
[GGUF](https://huggingface.co/leejet/MiniMax-H3-GGUF/tree/main) | `--llm` | Use matching text encoder | +| P-H3-Image text encoder (Qwen3-VL-4B with the text connector) | [Safetensors](https://huggingface.co/PrunaAI/P-H3-Image-Comfy/tree/main/text_encoders) | `--llm` | Use matching text encoder | +| vae | [Safetensors](https://huggingface.co/Comfy-Org/MiniMax-H3/tree/main/vae) | `--vae` | Use matching VAE | +| audio vae | [Safetensors](https://huggingface.co/Comfy-Org/MiniMax-H3/tree/main/vae) | `--audio-vae` | Required | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| MiniMax H3 FL2VA (`minimax-h3-fl2va`) | `diffusion_model`, `llm`, `llm_vision`, `embeddings_connectors`, `vae`, `tae`, `audio_vae` | +| MiniMax H3 Ref2VA (`minimax-h3-ref2va`) | `diffusion_model`, `llm`, `llm_vision`, `embeddings_connectors`, `vae`, `tae`, `audio_vae` | +| HyperFlow H3 Flash FL2VA (`hyperflow-h3-flash-fl2va`) | `diffusion_model`, `llm`, `llm_vision`, `embeddings_connectors`, `vae`, `tae`, `audio_vae` | +| HyperFlow H3 Flash Ref2VA (`hyperflow-h3-flash-ref2va`) | `diffusion_model`, `llm`, `llm_vision`, `embeddings_connectors`, `vae`, `tae`, `audio_vae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### Text-to-audio-video + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\minimax_h3_fl2va-Q4_K_M.gguf ` + --vae ..\models\vae\minimax_h3_video_vae_fp16.safetensors ` + --audio-vae ..\models\vae\minimax_h3_audio_vae_fp32.safetensors ` + --llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf ` + -p "A cute American Shorthair silver tabby kitten surfs on a tropical ocean wave, riding a white surfboard with the clear text 'sd.cpp' on it. Cinematic tracking shot, realistic water, bright sunlight, smooth motion, and consistent character appearance. Add upbeat tropical surf-rock background music with cheerful drums and guitar, synchronized with the kitten`’s energetic surfing." ` + --cfg-scale 1.0 ` + -W 864 ` + -H 480 ` + --fps 24 ` + --video-frames 56 ` + --rng cpu ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/minimax_h3_fl2va-Q4_K_M.gguf \ + --vae ../models/vae/minimax_h3_video_vae_fp16.safetensors \ + --audio-vae ../models/vae/minimax_h3_audio_vae_fp32.safetensors \ + --llm ../models/text_encoders/qwen3vl_32b_minimax_h3-Q4_K_M.gguf \ + -p "A cute American Shorthair silver tabby kitten surfs on a tropical ocean wave, riding a white surfboard with the clear text 'sd.cpp' on it. Cinematic tracking shot, realistic water, bright sunlight, smooth motion, and consistent character appearance. Add upbeat tropical surf-rock background music with cheerful drums and guitar, synchronized with the kitten’s energetic surfing." \ + --cfg-scale 1.0 \ + -W 864 \ + -H 480 \ + --fps 24 \ + --video-frames 56 \ + --rng cpu \ + --offload-to-cpu \ + -v +``` + + + +Omitting `--audio-vae` still runs the joint diffusion model but produces video without a +decoded audio track. + +#### First/last-frame conditioning + +Add `--init-img` for I2VA, or both `--init-img` and `--end-img` for FL2VA: + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\minimax_h3_fl2va-Q4_K_M.gguf ` + --vae ..\models\vae\minimax_h3_video_vae_fp16.safetensors ` + --audio-vae ..\models\vae\minimax_h3_audio_vae_fp32.safetensors ` + --llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf ` + -p "a lovely cat" ` + -i ..\inputs\ernie_image\turbo_example.png ` + --cfg-scale 1.0 ` + -W 864 ` + -H 480 ` + --fps 24 ` + --video-frames 56 ` + --rng cpu ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/minimax_h3_fl2va-Q4_K_M.gguf \ + --vae ../models/vae/minimax_h3_video_vae_fp16.safetensors \ + --audio-vae ../models/vae/minimax_h3_audio_vae_fp32.safetensors \ + --llm ../models/text_encoders/qwen3vl_32b_minimax_h3-Q4_K_M.gguf \ + -p "a lovely cat" \ + -i ../inputs/ernie_image/turbo_example.png \ + --cfg-scale 1.0 \ + -W 864 \ + -H 480 \ + --fps 24 \ + --video-frames 56 \ + --rng cpu \ + --offload-to-cpu \ + -v +``` + + + +#### Reference-to-audio-video conditioning + +Ref2VA accepts any combination of reference images, reference videos, paired +video soundtracks, and standalone audio references: + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\minimax_h3_ref2va_pruned-Q4_K_M.gguf ` + --vae ..\models\vae\minimax_h3_video_vae_fp16.safetensors ` + --audio-vae ..\models\vae\minimax_h3_audio_vae_fp32.safetensors ` + --llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf ` + -p "Use the cat from as the main character. Keep the cat`’s appearance, fur color, facial features, and identity consistent with the reference image. Create a 2-second cinematic video: start with an extreme close-up shot of the cat`’s face, focusing on its cute expression and detailed fur texture. The camera slowly rotates around the cat`’s head, creating a dynamic reveal. Then smoothly pull back and zoom out to reveal the full scene: the cat is standing confidently on a surfboard, riding ocean waves. Water splashes around the board, sea breeze gently moves the cat`’s fur, and the cat maintains a cute and fearless expression while surfing. Smooth camera movement, cinematic orbit shot, seamless zoom-out transition, low-angle wide shot, realistic ocean environment, golden sunlight, dynamic waves, high-quality realistic style, natural motion, no distortion, keep the cat`’s identity unchanged." ` + -r ..\inputs\ernie_image\turbo_example.png ` + --cfg-scale 1.0 ` + -W 864 ` + -H 480 ` + --fps 24 ` + --video-frames 56 ` + --rng cpu ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/minimax_h3_ref2va_pruned-Q4_K_M.gguf \ + --vae ../models/vae/minimax_h3_video_vae_fp16.safetensors \ + --audio-vae ../models/vae/minimax_h3_audio_vae_fp32.safetensors \ + --llm ../models/text_encoders/qwen3vl_32b_minimax_h3-Q4_K_M.gguf \ + -p "Use the cat from as the main character. Keep the cat’s appearance, fur color, facial features, and identity consistent with the reference image. Create a 2-second cinematic video: start with an extreme close-up shot of the cat’s face, focusing on its cute expression and detailed fur texture. The camera slowly rotates around the cat’s head, creating a dynamic reveal. Then smoothly pull back and zoom out to reveal the full scene: the cat is standing confidently on a surfboard, riding ocean waves. Water splashes around the board, sea breeze gently moves the cat’s fur, and the cat maintains a cute and fearless expression while surfing. Smooth camera movement, cinematic orbit shot, seamless zoom-out transition, low-angle wide shot, realistic ocean environment, golden sunlight, dynamic waves, high-quality realistic style, natural motion, no distortion, keep the cat’s identity unchanged." \ + -r ../inputs/ernie_image/turbo_example.png \ + --cfg-scale 1.0 \ + -W 864 \ + -H 480 \ + --fps 24 \ + --video-frames 56 \ + --rng cpu \ + --offload-to-cpu \ + -v +``` + + + +`--ref-image`, `--ref-video`, and `--ref-audio` can each be repeated. A +reference video is a directory of image frames sorted lexicographically and is +treated as 24 fps. Repeated `--ref-video-audio` WAV files are paired by index +with repeated `--ref-video` inputs. WAV PCM (8/16/24/32-bit) and 32/64-bit +floating-point samples are accepted; audio is converted to stereo 32 kHz by the +pipeline. + +Reference inputs are presented to Qwen3-VL in image, video, then audio order. +Videos are sampled at 2 fps for the Qwen presentation while their full 24 fps +latents condition the diffusion transformer. Paired video and audio references +share the same timeline. Ref2VA cannot be combined with `--init-img` or +`--end-img` in one request. + +Reference images keep their aspect ratio and are only downscaled when they +exceed the configured limit; neither mode scales a reference up. The limit is +selected by `ref_image_size`. The default, `match`, uses the pixel area of the +requested generation canvas. Pass `--ref-image-args "ref_image_size=max"` (or +`ref_image_size` in the `ref_image_args` video-option string) to use a 2048 px short +edge instead, which preserves more identity detail at a higher sampling cost: +reference latents are attended at every sampling step. + +Whichever limit applies, each axis is then snapped to the nearest 32-pixel +multiple. An axis of at least 16 pixels can therefore grow by up to 16 pixels +past the scaled size (a 48-pixel axis becomes 64), and references are never +narrower than 32 pixels. +`resize_before_vae=false` is not supported here; MiniMax-H3 keeps references on +the 32-pixel grid and reports a warning. + +The C API exposes the same inputs through `ref_images`, `ref_videos`, and +the `ref_audios` video-option key. Each `sd_video_t` supplies its own +frame rate and optional soundtrack; non-24-fps inputs are resampled internally. + +#### Sol-H3-VAE decoding + +Pass a [Sol-H3-VAE checkpoint](https://huggingface.co/Comfy-Org/MiniMax-H3/tree/main/vae) +with `--vae` to use the distilled video decoder. The ComfyUI FP16 and INT8 +ConvRot checkpoints include the MiniMax-H3 encoder for image and video inputs. +Supply `--audio-vae` as usual to decode the soundtrack. + +Sol decodes the whole clip by default. Use `--vae-tiling` for spatial tiling or +`--temporal-tiling` for temporal tiling. Either can reduce memory use but may +affect output quality because the decoder has less context at tile boundaries. + +Temporal tiles default to 10 latent frames with 5 latent frames of overlap. +Set `temporal_tile_frames` and `temporal_tile_overlap` in +`--extra-tiling-args` to override them. Both must be multiples of 5 latent +frames, and overlap must be smaller than the tile. The last tile may be shorter. +Circular decoding requires spatial tiling on each circular axis, with a tile +size smaller than the input on that axis. + +#### Latent hires second pass (two-stage sampling) + +MiniMax-H3 supports `--hires` with two interchangeable second-stage upscalers: + +- an interpolating latent upscaler (`Latent`, `Latent (nearest)`, + `Latent (bicubic)`, ...): after the first pass finishes, the packed AV latent + is resampled in latent space to `--hires-width` x `--hires-height` and + refined by a second sampling pass; +- a model-backed upscaler (`--hires-upscaler ` + `--hires-upscalers-dir`): + a H3-shaped learned latent upscaler checkpoint runs between the passes, + like the LTX spatial upsampler. This is the sd.cpp equivalent of the + two-sampler workflows used for H3 in node-based frontends (e.g. the + community 3D-conv latent upscaler from + [LBH-123-AI/Minimax_h3_latent_Upscaler](https://huggingface.co/LBH-123-AI/Minimax_h3_latent_Upscaler)). + +Draft at a low megapixel count (e.g. 0.4 MP) and refine at a higher one +(e.g. 0.8 MP) instead of paying for the full resolution in every step: + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\minimax_h3_fl2va-Q4_K_M.gguf ` + --vae ..\models\vae\minimax_h3_video_vae_fp16.safetensors ` + --audio-vae ..\models\vae\minimax_h3_audio_vae_fp32.safetensors ` + --llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf ` + -p "a lovely cat" ` + --cfg-scale 1.0 ` + -W 864 ` + -H 480 ` + --video-frames 56 ` + --rng cpu ` + --hires ` + --hires-upscaler "Latent (bicubic)" ` + --hires-width 1216 ` + --hires-height 672 ` + --hires-steps 4 ` + --hires-denoising-strength 0.4 ` + --diffusion-fa ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/minimax_h3_fl2va-Q4_K_M.gguf \ + --vae ../models/vae/minimax_h3_video_vae_fp16.safetensors \ + --audio-vae ../models/vae/minimax_h3_audio_vae_fp32.safetensors \ + --llm ../models/text_encoders/qwen3vl_32b_minimax_h3-Q4_K_M.gguf \ + -p "a lovely cat" \ + --cfg-scale 1.0 \ + -W 864 \ + -H 480 \ + --video-frames 56 \ + --rng cpu \ + --hires \ + --hires-upscaler "Latent (bicubic)" \ + --hires-width 1216 \ + --hires-height 672 \ + --hires-steps 4 \ + --hires-denoising-strength 0.4 \ + --diffusion-fa \ + --offload-to-cpu \ + -v +``` + +or with the learned upscaler: + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ... ` + --hires-upscalers-dir ..\models\latent_upscale_models ` + --hires ` + --hires-upscaler h3-latent-upscaler-3d ` + --hires-width 1216 ` + --hires-height 672 ` + --hires-steps 4 ` + --hires-denoising-strength 0.4 +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen ... \ + --hires-upscalers-dir ../models/latent_upscale_models \ + --hires \ + --hires-upscaler h3-latent-upscaler-3d \ + --hires-width 1216 \ + --hires-height 672 \ + --hires-steps 4 \ + --hires-denoising-strength 0.4 +``` + +Notes for this mode: + +- The learned upscaler expects a second normalization of the diffusion latent, + matching its training convention. The runner applies and reverses this extra + transform internally; callers should pass the usual H3 diffusion latents. +- The frame count is never changed by the second stage, so the packed audio + latent length stays valid and the generated soundtrack is preserved. +- Width and height still have to be multiples of 32; the hires target is + aligned like any other resolution. The learned upscaler additionally wants + an even latent size on both axes, which 32-pixel targets provide. +- `--hires-sigmas` can replace `--hires-denoising-strength` when the second + stage should resume from an explicit sigma schedule, which is how most + two-stage H3 workflows split the sampler. +- Keyframe conditioning (`--init-img` / `--end-img`) is not available together + with hires yet: those latents are encoded at the first stage resolution and + would have to be re-encoded for the hires canvas. Use T2VA or Ref2VA. +- The second stage runs at the larger latent, so it is the memory-hungry half; + combine it with quantization, `--vae-tiling` and `--offload-to-cpu` on + small cards. +- The learned upscaler is scale-conditional: the target-to-source ratio is + fed into the network as an embedding, 1.2-2.0x is the trained domain, and + downscaling is rejected. + +#### HyperFlow H3 Flash + +[HyperFlow H3 Flash](https://huggingface.co/VideoRebirthAI/hyperflow-h3-flash) +uses the shared components and input options described in this guide. Replace +`--diffusion-model` with the matching HyperFlow FL2VA or Ref2VA checkpoint and +use these sampling options: + +**Windows (PowerShell) arguments** + +```text +--sampling-method euler ` + --cfg-scale 1 ` + --sigmas hyperflow-h3-flash-6 +``` + +**Linux/macOS arguments** + +```text +--sampling-method euler \ + --cfg-scale 1 \ + --sigmas hyperflow-h3-flash-6 +``` + +Use `hyperflow-h3-flash-4` or `hyperflow-h3-flash-8` for 4 or 8 steps. These +presets also work with `--hires-sigmas`; their length determines the step count. +They already include video flow shift 12, so keep the default flow shift. + +## Notes and limitations + +- Width and height are aligned upward to a multiple of 32. +- Frame count is aligned upward to the `17k + 5` grid, with a minimum of 5. +- MiniMax-H3 runs at 24 fps; another requested value is overridden. +- Attention head grouping is controlled by + `--model-args minimax_h3_attn_chunk_budget=2`, in GiB (default: `2`). + The budget estimates the size of a single FP32 Q/K tensor per group: + `head_dim * tokens * batch * 4 bytes * heads_per_group`. The group size adapts + to the sequence length, with a minimum of one head; inputs that fit use all + heads together. Set `0` to disable grouping, or a smaller budget such as + `0.25` to reduce peak activation memory for long sequences at the cost of + extra launches and concatenation. Each head still attends to the full sequence. + This is a grouping target, not a cap on total attention memory or VRAM usage; + other intermediates, backend workspace, and weights require additional memory. + `--max-vram` and `--offload-to-cpu` control weight residency, but the active + block's working memory must also fit within the VRAM budget. +- The default video flow shift is 12. The audio stream is mapped internally to + its shift of 3, so the regular samplers can operate on the packed AV latent. diff --git a/docs/models/minimax_h3_image.md b/docs/models/minimax_h3_image.md new file mode 100644 index 0000000000..6a06236d2e --- /dev/null +++ b/docs/models/minimax_h3_image.md @@ -0,0 +1,186 @@ +# MiniMax-H3-Image + +MiniMax-H3-Image supports text-to-image generation and reference-image editing +with the same checkpoint. It reuses the MiniMax-H3 text encoder and uses a +dedicated image VAE. + +## Model files + +Pass the three MiniMax-H3-Image components separately: + +- `--diffusion-model`: MiniMax-H3-Image diffusion transformer +- `--vae`: MiniMax-H3-Image VAE +- `--llm`: the MiniMax-H3 Qwen3-VL-32B text encoder checkpoint + +Use the MiniMax-H3 text encoder, including its Qwen3-VL vision tower and +DeepStack mergers. An existing MiniMax-H3 text encoder can be reused. If the +vision tower is stored separately, pass it with `--llm_vision`. + +The P-H3-Image text encoder can be used instead: Qwen3-VL-4B with the P-H3 +text connector. It needs much less memory (about 5 GB for the INT8 ConvRot +file, versus 18 GB for the 32B encoder in Q4_K_M) and works for both +text-to-image and reference-image editing. Pass the P-H3-Image text encoder, +which includes the connector, with `--llm`. Alternatively, pass Qwen3-VL-4B +with `--llm` and the [separate text connector](https://huggingface.co/PrunaAI/P-H3-Image/blob/main/text_connector/diffusion_pytorch_model.safetensors) +with `--embeddings-connectors`. Reference-image editing needs the Qwen3-VL-4B +vision tower in the text encoder or in `--llm_vision`. + +Use a single-file diffusion checkpoint with the six-step PDD output heads +included, such as `minimax_h3_image_pruned_bf16.safetensors` or its INT8 ConvRot +version. Use the MiniMax-H3-Image VAE; the MiniMax-H3 video VAE and P-H3-Image +VAE are incompatible. No audio VAE is needed. + +### Download weights + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| minimax_h3_image_pruned | [Safetensors](https://huggingface.co/Comfy-Org/MiniMax-H3-Image/tree/main/diffusion_models) | `--diffusion-model` | Required | +| qwen3vl_32b_minimax_h3 | [Safetensors](https://huggingface.co/Comfy-Org/MiniMax-H3-Image/tree/main/text_encoders)
[GGUF](https://huggingface.co/leejet/MiniMax-H3-GGUF/tree/main) | `--llm` | Use matching text encoder | +| P-H3-Image text encoder (Qwen3-VL-4B with the text connector) | [Safetensors](https://huggingface.co/PrunaAI/P-H3-Image-Comfy/tree/main/text_encoders) | `--llm` | Use matching text encoder | +| vae | [Safetensors](https://huggingface.co/Comfy-Org/MiniMax-H3-Image/tree/main/vae) | `--vae` | Use matching VAE | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| MiniMax H3 Image (`minimax-h3-image`) | `diffusion_model`, `llm`, `llm_vision`, `embeddings_connectors`, `vae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### Text-to-image + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\minimax_h3_image_pruned_bf16.safetensors ` + --vae ..\models\vae\minimax_h3_image_vae_fp16.safetensors ` + --llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf ` + -p "A silver tabby kitten holding a sign reading 'sd.cpp', natural daylight" ` + --cfg-scale 1.0 ` + --steps 6 ` + --sampling-method euler ` + --sigmas minimax-h3-image ` + -W 1024 ` + -H 1024 ` + --rng cpu ` + --offload-to-cpu ` + --diffusion-fa ` + --vae-tiling ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/minimax_h3_image_pruned_bf16.safetensors \ + --vae ../models/vae/minimax_h3_image_vae_fp16.safetensors \ + --llm ../models/text_encoders/qwen3vl_32b_minimax_h3-Q4_K_M.gguf \ + -p "A silver tabby kitten holding a sign reading 'sd.cpp', natural daylight" \ + --cfg-scale 1.0 \ + --steps 6 \ + --sampling-method euler \ + --sigmas minimax-h3-image \ + -W 1024 \ + -H 1024 \ + --rng cpu \ + --offload-to-cpu \ + --diffusion-fa \ + --vae-tiling \ + -v +``` + +With the P-H3-Image text encoder, replace the `--llm` argument: + +**Windows (PowerShell) arguments** + +```text +--llm ..\models\text_encoders\p-h3-image_text_encoder_int8_convrot.safetensors +``` + +**Linux/macOS arguments** + +```text +--llm ../models/text_encoders/p-h3-image_text_encoder_int8_convrot.safetensors +``` + +Use Euler, CFG 1, and `--sigmas minimax-h3-image` for the model's six-step +schedule. The default discrete scheduler also produces this schedule with +`--steps 6`. Prompt expansion is not included. + +#### Reference-image editing + +Add `--ref-image` (or `-r`) for reference-image editing: + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\minimax_h3_image_pruned_bf16.safetensors ` + --vae ..\models\vae\minimax_h3_image_vae_fp16.safetensors ` + --llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf ` + -p "Edit . Put a red scarf on the cat, preserving its face and the background." ` + -r ..\inputs\ernie_image\turbo_example.png ` + --cfg-scale 1.0 ` + --steps 6 ` + --sampling-method euler ` + --sigmas minimax-h3-image ` + -W 1024 ` + -H 1024 ` + --rng cpu ` + --offload-to-cpu ` + --diffusion-fa ` + --vae-tiling ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/minimax_h3_image_pruned_bf16.safetensors \ + --vae ../models/vae/minimax_h3_image_vae_fp16.safetensors \ + --llm ../models/text_encoders/qwen3vl_32b_minimax_h3-Q4_K_M.gguf \ + -p "Edit . Put a red scarf on the cat, preserving its face and the background." \ + -r ../inputs/ernie_image/turbo_example.png \ + --cfg-scale 1.0 \ + --steps 6 \ + --sampling-method euler \ + --sigmas minimax-h3-image \ + -W 1024 \ + -H 1024 \ + --rng cpu \ + --offload-to-cpu \ + --diffusion-fa \ + --vae-tiling \ + -v +``` + +`--ref-image` can be repeated. Reference images are numbered in argument order +as ``, ``, and so on, with up to 16 references supported +by the model. Prompts are passed through without an automatic editing prefix. + +Reference images keep their aspect ratio. The text encoder sees a 1024-pixel +short edge, while VAE inputs use a short edge between 1024 and 2048 pixels. +Small references are enlarged and dimensions are rounded to multiples of 32. + +## Notes and limitations + +- Width and height are aligned upward to a multiple of 32. +- Start at 1024 × 1024. Larger canvases and more references increase memory use. +- Use quantized weights, `--offload-to-cpu`, and `--vae-tiling` to reduce GPU + memory requirements. diff --git a/docs/models/minit2i.md b/docs/models/minit2i.md new file mode 100644 index 0000000000..3e09b759c6 --- /dev/null +++ b/docs/models/minit2i.md @@ -0,0 +1,86 @@ +# MiniT2I + +MiniT2I uses a MiniT2I diffusion transformer and `google/flan-t5-large` as the text encoder. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| MiniT2I diffusion model | [Safetensors](https://huggingface.co/MiniT2I/MiniT2I/tree/main/minit2i-b-16/transformer) (`diffusion_pytorch_model.safetensors`) | `--diffusion-model` | Required | +| flan-t5-large text encoder | [Safetensors](https://huggingface.co/google/flan-t5-large/tree/main) (`model.safetensors`) | `--t5xxl` | Required | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| MiniT2I (`minit2i`) | `diffusion_model`, `t5xxl` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### CUDA + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\minit2i\diffusion_pytorch_model.safetensors ` + --t5xxl ..\models\text_encoders\flan-t5-large\model.safetensors ` + --prompt "a cat" ` + --steps 100 ` + --cfg-scale 6 ` + --sampling-method euler ` + --width 512 ` + --height 512 ` + --seed 42 ` + --output minit2i_cuda.png +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/minit2i/diffusion_pytorch_model.safetensors \ + --t5xxl ../models/text_encoders/flan-t5-large/model.safetensors \ + --prompt "a cat" \ + --steps 100 \ + --cfg-scale 6 \ + --sampling-method euler \ + --width 512 \ + --height 512 \ + --seed 42 \ + --output minit2i_cuda.png +``` + +#### macOS with Metal + +**macOS (Metal)** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/minit2i/diffusion_pytorch_model.safetensors \ + --t5xxl ../models/text_encoders/flan-t5-large/model.safetensors \ + --prompt "a cat" \ + --steps 100 \ + --cfg-scale 6 \ + --sampling-method euler \ + --width 512 \ + --height 512 \ + --rng cpu \ + --threads 8 \ + --backend metal \ + --seed 42 \ + --output minit2i_metal.png +``` diff --git a/docs/models/moss_image.md b/docs/models/moss_image.md new file mode 100644 index 0000000000..398b9193e6 --- /dev/null +++ b/docs/models/moss_image.md @@ -0,0 +1,108 @@ +# MOSS-Image + +MOSS-Image uses a 5B diffusion transformer, the FLUX.2 VAE, and Qwen3.5 9B as the LLM text encoder. Both Base and SFT are supported. Prompt enhancement is not included. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| MOSS-Image Base or SFT | [Safetensors](https://huggingface.co/Comfy-Org/MOSS-Image/tree/main/diffusion_models)
gguf:
Base: [Download](https://huggingface.co/leejet/MOSS-Image-Base-GGUF/tree/main)
SFT: [Download](https://huggingface.co/leejet/MOSS-Image-SFT-GGUF/tree/main) | `--diffusion-model` | Required | +| vae | [Safetensors](https://huggingface.co/Comfy-Org/MOSS-Image/tree/main/vae) | `--vae` | Use matching VAE | +| Qwen3.5 9B | [GGUF](https://huggingface.co/unsloth/Qwen3.5-9B-GGUF/tree/main) | `--llm` | Use matching text encoder | +| `tokenizer.json` | [Download](https://huggingface.co/OpenMOSS-Team/MOSS-Image-Base/tree/main/tokenizer) | `--tokenizer` | Required; match text encoder | + +Pass the external `tokenizer.json` with `--tokenizer`; Qwen3.5 uses a different vocabulary from the embedded Qwen tokenizer. + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| MOSS Image Base (`moss-image-base`) | `diffusion_model`, `llm`, `tokenizer`, `vae`, `tae` | +| MOSS Image SFT (`moss-image-sft`) | `diffusion_model`, `llm`, `tokenizer`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### MOSS-Image Base + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\moss_image_base_int8_convrot_attn.safetensors ` + --llm ..\models\text_encoders\Qwen3.5-9B-Q4_K_M.gguf ` + --vae ..\models\vae\flux2_ae.safetensors ` + --tokenizer ..\models\tokenizers\tokenizer_qwen3.5.json ` + -p "a lovely cat holding a sign says 'moss-image.cpp'" ` + --cfg-scale 3.0 ` + -H 1024 ` + -W 1024 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/moss_image_base_int8_convrot_attn.safetensors \ + --llm ../models/text_encoders/Qwen3.5-9B-Q4_K_M.gguf \ + --vae ../models/vae/flux2_ae.safetensors \ + --tokenizer ../models/tokenizers/tokenizer_qwen3.5.json \ + -p "a lovely cat holding a sign says 'moss-image.cpp'" \ + --cfg-scale 3.0 \ + -H 1024 \ + -W 1024 \ + --offload-to-cpu \ + -v +``` + +#### MOSS-Image SFT + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\moss_image_sft_int8_convrot_attn.safetensors ` + --llm ..\models\text_encoders\Qwen3.5-9B-Q4_K_M.gguf ` + --vae ..\models\vae\flux2_ae.safetensors ` + --tokenizer ..\models\tokenizers\tokenizer_qwen3.5.json ` + -p "a lovely cat holding a sign says 'moss-image.cpp'" ` + --cfg-scale 3.0 ` + -H 1024 ` + -W 1024 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/moss_image_sft_int8_convrot_attn.safetensors \ + --llm ../models/text_encoders/Qwen3.5-9B-Q4_K_M.gguf \ + --vae ../models/vae/flux2_ae.safetensors \ + --tokenizer ../models/tokenizers/tokenizer_qwen3.5.json \ + -p "a lovely cat holding a sign says 'moss-image.cpp'" \ + --cfg-scale 3.0 \ + -H 1024 \ + -W 1024 \ + --offload-to-cpu \ + -v +``` + + + +Use dimensions divisible by 16. The default discrete scheduler uses the upstream time shift of 5. Use `--flow-shift` to override it. The negative CFG branch skips DiT layer 22 (zero-based), matching the upstream pipeline. diff --git a/docs/models/ovis_image.md b/docs/models/ovis_image.md new file mode 100644 index 0000000000..ddf6c21579 --- /dev/null +++ b/docs/models/ovis_image.md @@ -0,0 +1,60 @@ +# Ovis-Image + +Ovis-Image generates images from text using an Ovis 2.5 encoder and the FLUX VAE. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Ovis-Image-7B | [Safetensors](https://huggingface.co/Comfy-Org/Ovis-Image/tree/main/split_files/diffusion_models)
[GGUF](https://huggingface.co/leejet/Ovis-Image-7B-GGUF) | `--diffusion-model` | Required | +| vae | [Safetensors](https://huggingface.co/black-forest-labs/FLUX.1-schnell/tree/main) | `--vae` | Use matching VAE | +| Ovis 2.5 | [Safetensors](https://huggingface.co/Comfy-Org/Ovis-Image/tree/main/split_files/text_encoders) | `--llm` | Use matching text encoder | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| Ovis Image (`ovis-image`) | `diffusion_model`, `llm`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\ovis_image-Q4_0.gguf ` + --vae ..\models\vae\ae.sft ` + --llm ..\models\text_encoders\ovis_2.5.safetensors ` + -p "a lovely cat" ` + --cfg-scale 5.0 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/ovis_image-Q4_0.gguf \ + --vae ../models/vae/ae.sft \ + --llm ../models/text_encoders/ovis_2.5.safetensors \ + -p "a lovely cat" \ + --cfg-scale 5.0 \ + --offload-to-cpu \ + -v +``` + +ovis image example diff --git a/docs/models/p_h3_image.md b/docs/models/p_h3_image.md new file mode 100644 index 0000000000..eed578416a --- /dev/null +++ b/docs/models/p_h3_image.md @@ -0,0 +1,147 @@ +# P-H3-Image + +P-H3-Image supports text-to-image and reference-image editing. It uses a MiniMax-H3 diffusion transformer, a single-frame VAE, and either Qwen3-VL-4B with a text connector or the MiniMax-H3 Qwen3-VL-32B text encoder. Prompt enhancement is not included. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| P-H3-Image | [Safetensors](https://huggingface.co/PrunaAI/P-H3-Image-Comfy/tree/main/diffusion_models) | `--diffusion-model` | Required | +| P-H3-Image VAE | [Safetensors](https://huggingface.co/PrunaAI/P-H3-Image-Comfy/tree/main/vae) | `--vae` | Use matching VAE | +| Qwen3-VL-4B with the text connector | [Download](https://huggingface.co/PrunaAI/P-H3-Image-Comfy/tree/main/text_encoders) | `--llm` | Use matching text encoder | +| MiniMax-H3 Qwen3-VL-32B text encoder | [Safetensors](https://huggingface.co/Comfy-Org/MiniMax-H3/tree/main/text_encoders)
[GGUF](https://huggingface.co/leejet/MiniMax-H3-GGUF/tree/main) | `--llm` | Use matching text encoder | + +Alternatively, reuse Qwen3-VL-4B, including its vision tower, with the [separate text connector](https://huggingface.co/PrunaAI/P-H3-Image/blob/main/text_connector/diffusion_pytorch_model.safetensors). Save the connector as `p-h3-image_text_connector.safetensors` and pass it with `--embeddings-connectors`. + +The MiniMax-H3 Qwen3-VL-32B encoder does not need a text connector. To use it in +either example below, omit `--embeddings-connectors` and replace `--llm` with: + +**Windows (PowerShell) arguments** + +```text +--llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf +``` + +**Linux/macOS arguments** + +```text +--llm ../models/text_encoders/qwen3vl_32b_minimax_h3-Q4_K_M.gguf +``` + +Reference-image editing needs the matching encoder's vision tower. If stored +separately, pass it with `--llm_vision`. + +Use the P-H3 image VAE; the MiniMax-H3 video VAE is not compatible. No audio VAE is needed. + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| P-H3-Image (`p-h3-image`) | `diffusion_model`, `llm`, `llm_vision`, `embeddings_connectors`, `vae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### Text-to-image + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\p-h3-image_int8_convrot.safetensors ` + --llm ..\models\text_encoders\qwen3vl_4b_bf16.safetensors ` + --embeddings-connectors ..\models\text_encoders\p-h3-image_text_connector.safetensors ` + --vae ..\models\vae\p-h3-image_vae_bf16.safetensors ` + -p "a lovely cat holding a sign says 'p-h3-image.cpp'" ` + --cfg-scale 1.0 ` + --steps 8 ` + --sampling-method euler ` + --sigmas p-h3-image ` + -H 2912 ` + -W 2912 ` + --rng cpu ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/p-h3-image_int8_convrot.safetensors \ + --llm ../models/text_encoders/qwen3vl_4b_bf16.safetensors \ + --embeddings-connectors ../models/text_encoders/p-h3-image_text_connector.safetensors \ + --vae ../models/vae/p-h3-image_vae_bf16.safetensors \ + -p "a lovely cat holding a sign says 'p-h3-image.cpp'" \ + --cfg-scale 1.0 \ + --steps 8 \ + --sampling-method euler \ + --sigmas p-h3-image \ + -H 2912 \ + -W 2912 \ + --rng cpu \ + --offload-to-cpu \ + -v +``` + +#### Image editing + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\p-h3-image_int8_convrot.safetensors ` + --llm ..\models\text_encoders\qwen3vl_4b_bf16.safetensors ` + --embeddings-connectors ..\models\text_encoders\p-h3-image_text_connector.safetensors ` + --vae ..\models\vae\p-h3-image_vae_bf16.safetensors ` + -r source.png ` + -r reference.png ` + -p "Edit . Give the cat the scarf from , preserving its face and the background." ` + --cfg-scale 1.0 ` + --steps 8 ` + --sampling-method euler ` + --sigmas p-h3-image ` + -H 2912 ` + -W 2912 ` + --rng cpu ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/p-h3-image_int8_convrot.safetensors \ + --llm ../models/text_encoders/qwen3vl_4b_bf16.safetensors \ + --embeddings-connectors ../models/text_encoders/p-h3-image_text_connector.safetensors \ + --vae ../models/vae/p-h3-image_vae_bf16.safetensors \ + -r source.png \ + -r reference.png \ + -p "Edit . Give the cat the scarf from , preserving its face and the background." \ + --cfg-scale 1.0 \ + --steps 8 \ + --sampling-method euler \ + --sigmas p-h3-image \ + -H 2912 \ + -W 2912 \ + --rng cpu \ + --offload-to-cpu \ + -v +``` + +Reference images are numbered in argument order as ``, ``, and so on. A plain edit instruction gets `Edit . ` prepended automatically. Prompts starting with `Edit ` or `subject_definitions:` are used directly. + +Use dimensions divisible by 32. The upstream default is 2912 × 2912; smaller canvases reduce memory use. Use CFG 1 and `--sigmas p-h3-image` for the upstream 8-step schedule. Keep the default VAE tiling size of 256 pixels. diff --git a/docs/models/photo_maker.md b/docs/models/photo_maker.md new file mode 100644 index 0000000000..1bc8ab3a2c --- /dev/null +++ b/docs/models/photo_maker.md @@ -0,0 +1,157 @@ +# PhotoMaker + +[PhotoMaker](https://github.com/TencentARC/PhotoMaker) personalizes image +generation from identity images. It requires an SDXL base model. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| SDXL checkpoint | [SDXL guide](sd.md) | `-m` / `--model` | Required; PhotoMaker is SDXL-only | +| SDXL VAE | VAE compatible with the selected SDXL checkpoint | `--vae` | Use an override when required | +| PhotoMaker v1 | [Safetensors weights](https://huggingface.co/bssrdf/PhotoMaker) | `--photo-maker` | For v1; the original .bin release is unsupported | +| PhotoMaker v2 | [Safetensors weights](https://huggingface.co/bssrdf/PhotoMakerV2) | `--photo-maker` | Alternative to v1; requires precomputed face embeddings | +| Identity images | A directory of portraits of the same person | `--pm-id-images-dir` | Required | +| PhotoMaker v2 embeddings | `id_embeds.bin` from the preparation below | `--pm-id-embed-path` | Required for v2 | + +### Prepare PhotoMaker v2 embeddings + +[PhotoMaker v2](https://github.com/TencentARC/PhotoMaker/blob/main/README_pmv2.md) +adds a face-embedding preparation step. From the source repository root, run +[`scripts/face_detect.py`](../../scripts/face_detect.py) on the identity image directory: + +**Windows (PowerShell)** + +```powershell +python scripts\face_detect.py .\inputs\photomaker_examples\scarletthead_woman +``` + +**Linux/macOS** + +```sh +python scripts/face_detect.py ./inputs/photomaker_examples/scarletthead_woman +``` + +The script requires its Python dependencies, including PyTorch, NumPy, Diffusers, +and InsightFace. It writes `id_embeds.bin` inside the input directory. Compute it +once and reuse it for later generations. The image directory and `id_embeds.bin` stay under the repository's `inputs/` +directory. + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| SDXL PhotoMaker (`sdxl-photomaker`) | `model`, `photo_maker`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +Select the appropriate PhotoMaker weights and supply the identity images. +PhotoMaker v2 also needs the prepared identity embedding. + +### CLI + +#### PhotoMaker Version 1 + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -m ..\models\checkpoints\sdxlUnstableDiffusers_v11.safetensors ` + --vae ..\models\vae\sdxl_vae.safetensors ` + --photo-maker ..\models\photomaker\photomaker-v1.safetensors ` + --pm-id-images-dir ..\inputs\photomaker_examples\scarletthead_woman ` + -p "a girl img, retro futurism, retro game art style but extremely beautiful, intricate details, masterpiece, best quality, space-themed, cosmic, celestial, stars, galaxies, nebulas, planets, science fiction, highly detailed" ` + -n "realistic, photo-realistic, worst quality, greyscale, bad anatomy, bad hands, error, text" ` + --cfg-scale 5.0 ` + --sampling-method euler ` + --steps 50 ` + -H 1024 ` + -W 1024 ` + --pm-style-strength 10 ` + --vae-on-cpu +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -m ../models/checkpoints/sdxlUnstableDiffusers_v11.safetensors \ + --vae ../models/vae/sdxl_vae.safetensors \ + --photo-maker ../models/photomaker/photomaker-v1.safetensors \ + --pm-id-images-dir ../inputs/photomaker_examples/scarletthead_woman \ + -p "a girl img, retro futurism, retro game art style but extremely beautiful, intricate details, masterpiece, best quality, space-themed, cosmic, celestial, stars, galaxies, nebulas, planets, science fiction, highly detailed" \ + -n "realistic, photo-realistic, worst quality, greyscale, bad anatomy, bad hands, error, text" \ + --cfg-scale 5.0 \ + --sampling-method euler \ + --steps 50 \ + -H 1024 \ + -W 1024 \ + --pm-style-strength 10 \ + --vae-on-cpu +``` + +#### PhotoMaker Version 2 + +From the build directory, use the prepared identity embedding: + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -m ..\models\checkpoints\sdxlUnstableDiffusers_v11.safetensors ` + --vae ..\models\vae\sdxl_vae.safetensors ` + --photo-maker ..\models\photomaker\photomaker-v2.safetensors ` + --pm-id-images-dir ..\inputs\photomaker_examples\scarletthead_woman ` + --pm-id-embed-path ..\inputs\photomaker_examples\scarletthead_woman\id_embeds.bin ` + -p "a girl img, retro futurism, retro game art style but extremely beautiful, intricate details, masterpiece, best quality, space-themed, cosmic, celestial, stars, galaxies, nebulas, planets, science fiction, highly detailed" ` + -n "realistic, photo-realistic, worst quality, greyscale, bad anatomy, bad hands, error, text" ` + --cfg-scale 5.0 ` + --sampling-method euler ` + --steps 50 ` + -H 1024 ` + -W 1024 ` + --pm-style-strength 10 ` + --vae-on-cpu +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -m ../models/checkpoints/sdxlUnstableDiffusers_v11.safetensors \ + --vae ../models/vae/sdxl_vae.safetensors \ + --photo-maker ../models/photomaker/photomaker-v2.safetensors \ + --pm-id-images-dir ../inputs/photomaker_examples/scarletthead_woman \ + --pm-id-embed-path ../inputs/photomaker_examples/scarletthead_woman/id_embeds.bin \ + -p "a girl img, retro futurism, retro game art style but extremely beautiful, intricate details, masterpiece, best quality, space-themed, cosmic, celestial, stars, galaxies, nebulas, planets, science fiction, highly detailed" \ + -n "realistic, photo-realistic, worst quality, greyscale, bad anatomy, bad hands, error, text" \ + --cfg-scale 5.0 \ + --sampling-method euler \ + --steps 50 \ + -H 1024 \ + -W 1024 \ + --pm-style-strength 10 \ + --vae-on-cpu +``` + +## Options + +The prompt must include a class word followed by the trigger word `img`, for +example `a girl img`. Class words include `man`, `woman`, `girl`, and `boy`. +For Asian identity images, the original workflow places `Asian` before the class word. + +`--pm-style-strength` accepts 0-100 and defaults to 20. The original examples use +10-20; lower values follow the source identity more closely. Use CFG 5 and +1024 x 1024 as in the example. On GPUs with 8 GB or less, `--vae-on-cpu` can help +with VAE memory usage. diff --git a/docs/models/pid.md b/docs/models/pid.md new file mode 100644 index 0000000000..7f89f1b0b3 --- /dev/null +++ b/docs/models/pid.md @@ -0,0 +1,98 @@ +# PiD + +PiD is NVIDIA's Pixel Diffusion Decoder. It replaces the usual VAE decode or decode-then-upscale path with a pixel-space diffusion decoder conditioned on a +source latent and text prompt. Both the original PiD checkpoints and PiD 1.5 are supported. + +In stable-diffusion.cpp, PiD currently runs as an image edit pipeline: provide a reference image with `-r`/`--ref-image`, encode that image with a matching VAE, then let the PiD diffusion model decode/upscale directly to RGB. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| PiD | [Safetensors](https://huggingface.co/Comfy-Org/PixelDiT/tree/main/diffusion_models) | `--diffusion-model` | Required | +| Gemma 2 2B | [Safetensors](https://huggingface.co/Comfy-Org/PixelDiT/tree/main/text_encoders) | `--llm` | Use matching text encoder | +| Gemma 2 2B tokenizer.json | [Download](https://huggingface.co/google/gemma-2-2b/tree/main) | `--tokenizer` | Required; match text encoder | +| VAE that matches the PiD checkpoint backbone | [Safetensors](https://huggingface.co/nvidia/PiD/tree/main/checkpoints)
Flux / Z-Image PiD: use the Flux VAE and pass `--vae-format flux`
SD3 PiD: use the SD3 VAE and pass `--vae-format sd3`
Flux.2 PiD: use the Flux.2 VAE and pass `--vae-format flux2`
Qwen-Image PiD: use the Qwen-Image 2D VAE and pass `--vae-format wan` | `--vae` | Use matching VAE | + +The official PiD model card should be checked before use. At the time of the initial PiD release, the official weights are under the NSCLv1 non-commercial license. + +PiD and PiD 1.5 require an external Gemma 2 `tokenizer.json` matching the text encoder checkpoint. Save it as `tokenizer_gemma2.json` and pass it with `--tokenizer`; the tokenizer is not embedded in sd.cpp. See [JSON tokenizers](../guides/tokenizers.md) for CLI and C API usage. + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| PiD (Flux / Z-Image VAE) (`pid`) | `diffusion_model`, `llm`, `tokenizer`, `vae` | +| PiD 1.5 (Flux / Z-Image VAE) (`pid-15`) | `diffusion_model`, `llm`, `tokenizer`, `vae` | +| PiD (SD3 VAE) (`pid-sd3`) | `diffusion_model`, `llm`, `tokenizer`, `vae` | +| PiD 1.5 (SD3 VAE) (`pid-15-sd3`) | `diffusion_model`, `llm`, `tokenizer`, `vae` | +| PiD (Flux.2 VAE) (`pid-flux2`) | `diffusion_model`, `llm`, `tokenizer`, `vae` | +| PiD 1.5 (Flux.2 VAE) (`pid-15-flux2`) | `diffusion_model`, `llm`, `tokenizer`, `vae` | +| PiD (Qwen Image VAE) (`pid-qwen-image`) | `diffusion_model`, `llm`, `tokenizer`, `vae` | +| PiD 1.5 (Qwen Image VAE) (`pid-15-qwen-image`) | `diffusion_model`, `llm`, `tokenizer`, `vae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\pid_flux1_512_to_2048_4step_bf16.safetensors ` + --llm ..\models\text_encoders\gemma_2_2b_it_elm_bf16.safetensors ` + --tokenizer ..\models\tokenizers\tokenizer_gemma2.json ` + --vae ..\models\vae\ae.sft ` + --vae-format flux ` + -p "a lovely cat" ` + -r ..\inputs\ernie_image\turbo_example.png ` + --cfg-scale 1.0 ` + --steps 4 ` + -H 2048 ` + -W 2048 ` + --rng cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/pid_flux1_512_to_2048_4step_bf16.safetensors \ + --llm ../models/text_encoders/gemma_2_2b_it_elm_bf16.safetensors \ + --tokenizer ../models/tokenizers/tokenizer_gemma2.json \ + --vae ../models/vae/ae.sft \ + --vae-format flux \ + -p "a lovely cat" \ + -r ../inputs/ernie_image/turbo_example.png \ + --cfg-scale 1.0 \ + --steps 4 \ + -H 2048 \ + -W 2048 \ + --rng cpu \ + -v +``` + +Before: + +ERNIE-Image Turbo example + +After: +PiD example + +## Notes and limitations + +- `-r`/`--ref-image` is required. PiD uses the first reference image as the source latent condition. +- `--vae-format` should match the VAE latent layout used by the PiD checkpoint. This is important when using standalone VAE files because the PiD diffusion + checkpoint alone does not identify the VAE format. diff --git a/docs/models/pixart.md b/docs/models/pixart.md new file mode 100644 index 0000000000..336e1ee272 --- /dev/null +++ b/docs/models/pixart.md @@ -0,0 +1,89 @@ +# PixArt + +You can run PixArt-α / PixArt-Σ with stable-diffusion.cpp. + +PixArt is a DiT-based text-to-image model family conditioned by a T5-XXL text +encoder and a 4-channel VAE: SDXL-style for PixArt-Σ and SD1.x-style for PixArt-α. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| transformer (diffusion model) | PixArt-Σ XL-2 1024-MS: [Download](https://huggingface.co/PixArt-alpha/PixArt-Sigma-XL-2-1024-MS/tree/main/transformer)
PixArt-α XL-2 1024-MS: [Download](https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS/tree/main/transformer) | `--diffusion-model` | Required | +| T5-XXL text encoder | [Safetensors](https://huggingface.co/PixArt-alpha/PixArt-Sigma-XL-2-1024-MS/tree/main/text_encoder) | `--t5xxl` | Required | +| VAE | PixArt-Σ: [Download](https://huggingface.co/PixArt-alpha/PixArt-Sigma-XL-2-1024-MS/tree/main/vae)
PixArt-α: [Download](https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS/tree/main/vae)
Use the VAE matching the checkpoint's latent space. For TAE decoding or
preview, use TAESDXL for PixArt-Σ and TAESD for PixArt-α. | `--vae` | Use matching VAE | + +- Tokenizer: the T5 vocabulary is embedded; no extra tokenizer file is needed. + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| PixArt Alpha XL 512 (`pixart-alpha-512`) | `diffusion_model`, `t5xxl`, `vae`, `tae` | +| PixArt Alpha XL 1024 (`pixart-alpha-1024`) | `diffusion_model`, `t5xxl`, `vae`, `tae` | +| PixArt Sigma XL 1024 (`pixart-sigma`) | `diffusion_model`, `t5xxl`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\pixart_sigma_xl2_1024_ms.safetensors ` + --t5xxl ..\models\text_encoders\t5xxl.safetensors ` + --vae ..\models\vae\pixart_vae.safetensors ` + -p "a lovely cat" ` + --cfg-scale 4.5 ` + --steps 20 ` + -W 1024 ` + -H 1024 ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/pixart_sigma_xl2_1024_ms.safetensors \ + --t5xxl ../models/text_encoders/t5xxl.safetensors \ + --vae ../models/vae/pixart_vae.safetensors \ + -p "a lovely cat" \ + --cfg-scale 4.5 \ + --steps 20 \ + -W 1024 \ + -H 1024 \ + -v +``` + +## Notes and limitations + +- The VAE scaling factor defaults to `0.13025` for PixArt-Σ. PixArt-α + checkpoints with resolution micro-condition weights use `0.18215`. + PixArt-α 512 has the same tensor layout as PixArt-Σ, so it requires an + explicit override: `--model-args "pixart_vae_scale_factor=0.18215"`. + This argument can also override the scale for other compatible checkpoints. +- PixArt-Σ checkpoints compute 2D sincos positional embeddings at runtime; + the trained grid is 64x64 patches with an interpolation scale of 2. + For checkpoints trained at a different resolution, the positional embedding + parameters can be adjusted via model args: + `--model-args "pixart_pos_embed_base_size=,pixart_interpolation_scale="` + (e.g. `pixart_pos_embed_base_size=32,pixart_interpolation_scale=1,pixart_vae_scale_factor=0.18215` for + PixArt-α XL-2 512). +- Checkpoints carrying resolution/aspect-ratio micro-condition weights are + detected but those conditions are not applied yet; a warning is logged and + generation proceeds with the timestep embedding only. +- The transformer predicts 8 channels (noise + learned variance); only the + noise half is used for sampling, matching the reference implementation. diff --git a/docs/models/pulid.md b/docs/models/pulid.md new file mode 100644 index 0000000000..188fbb8477 --- /dev/null +++ b/docs/models/pulid.md @@ -0,0 +1,199 @@ +# PuLID-Flux + +[PuLID-Flux](https://github.com/ToTheBeginning/PuLID) preserves a source portrait +identity while generating new images with FLUX.1 dev or schnell. Compute the +identity embedding once with the external Python tool, then reuse it. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| FLUX.1 base components | [FLUX.1 guide](flux.md) | `--diffusion-model`, `--vae`, `--clip_l`, `--t5xxl` | Required; dev or schnell | +| PuLID v0.9.x weights | [guozinan/PuLID](https://huggingface.co/guozinan/PuLID) | `--pulid-weights` | v0.9.1 is the tested version; v1.1 is unsupported | +| Identity embedding | `source.pulidembd` from preparation below | `--pulid-id-embedding` | Required for identity preservation | + +### Prepare the identity embedding + +The precompute tool extracts an identity embedding from a single portrait +and writes it to a `.pulidembd` file. Run it once per source +person; the same file is reused for any number of generations. + +Use +[`scripts/pulid_extract_id.py`](../../scripts/pulid_extract_id.py). It +requires: +- A working CUDA / CPU PyTorch stack +- `insightface`, `facexlib`, `eva-clip`, `torchvision`, `opencv-python`, + `huggingface_hub`, `gguf` +- The PuLID weights file (same one stable-diffusion.cpp will load below) +- The ToTheBeginning/PuLID repo's `pulid/` package (including + `pulid/pipeline_flux.py`) and `eva_clip/` package on `PYTHONPATH`; `flux/` + is not needed for embedding extraction + +From the source repository root, run: + +**Windows (PowerShell)** + +```powershell +python scripts\pulid_extract_id.py --portrait .\inputs\source-photo.jpg --pulid-weights .\models\pulid\pulid_flux_v0.9.1.safetensors --out .\inputs\source.pulidembd +``` + +**Linux/macOS** + +```sh +python scripts/pulid_extract_id.py --portrait ./inputs/source-photo.jpg --pulid-weights ./models/pulid/pulid_flux_v0.9.1.safetensors --out ./inputs/source.pulidembd +``` + +The script writes the embedding to the repository's `inputs/` directory. +The examples below read the same file from the build directory. + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| FLUX.1 dev PuLID (`flux1-pulid`) | `diffusion_model`, `clip_l`, `t5xxl`, `pulid_weights`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\flux1-schnell-Q4_K_S.gguf ` + --vae ..\models\vae\ae.safetensors ` + --clip_l ..\models\text_encoders\clip_l.safetensors ` + --t5xxl ..\models\text_encoders\t5xxl_fp16.safetensors ` + --pulid-weights ..\models\pulid\pulid_flux_v0.9.1.safetensors ` + --pulid-id-embedding ..\inputs\source.pulidembd ` + -p "candid photograph of a young woman on a beach at sunset" ` + --cfg-scale 1.0 ` + --sampling-method euler ` + --steps 4 ` + -W 512 ` + -H 512 ` + --pulid-id-weight 1.0 ` + --clip-on-cpu ` + --seed 42 ` + -o out.png +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/flux1-schnell-Q4_K_S.gguf \ + --vae ../models/vae/ae.safetensors \ + --clip_l ../models/text_encoders/clip_l.safetensors \ + --t5xxl ../models/text_encoders/t5xxl_fp16.safetensors \ + --pulid-weights ../models/pulid/pulid_flux_v0.9.1.safetensors \ + --pulid-id-embedding ../inputs/source.pulidembd \ + -p "candid photograph of a young woman on a beach at sunset" \ + --cfg-scale 1.0 \ + --sampling-method euler \ + --steps 4 \ + -W 512 \ + -H 512 \ + --pulid-id-weight 1.0 \ + --clip-on-cpu \ + --seed 42 \ + -o out.png +``` + +For Flux Dev (instead of Schnell), add `--guidance 3.5` and `--steps 20`. + +## Options + +| Flag | Purpose | +|----------------------------|-------------------------------------------------------------------| +| `--pulid-weights ` | Path to `pulid_flux_v0.9.x.safetensors`. Loaded with the model. | +| `--pulid-id-embedding

` | Path to a `.pulidembd` binary produced by the precompute tool. | +| `--pulid-id-weight ` | Identity-injection strength. Typical 0.7-1.2; default 1.0. | + +Supply both the PuLID weights and identity embedding to activate identity +conditioning. `--pulid-id-weight` defaults to 1.0; set it to 0 to disable identity +injection. Loading `--pulid-weights` without an embedding leaves identity +conditioning inactive. + +### Backend selection + +The standard `--backend` flag works as documented. Common patterns: + +**Windows (PowerShell) arguments** + +```text +# AMD Vulkan +--backend "diffusion=vulkan0,vae=cpu" + +# NVIDIA Vulkan +--backend "diffusion=vulkan1,vae=cpu" + +# CUDA +--backend "diffusion=cuda0,vae=cpu" +``` + +**Linux/macOS arguments** + +```text +# AMD Vulkan +--backend diffusion=vulkan0,vae=cpu + +# NVIDIA Vulkan +--backend diffusion=vulkan1,vae=cpu + +# CUDA +--backend diffusion=cuda0,vae=cpu +``` + +The PuLID cross-attention layers run on the same backend as the main +diffusion model. They have not yet been independently profiled on every +backend; only Vulkan and CPU have been tested by the original contributor. + +## Notes and limitations + +PuLID v1.1 weights are unsupported. Use PuLID and skip-layer guidance +(`--skip-layers`) separately. One precomputed identity embedding is accepted per +request; prepare it using the external tool. Identity conditioning is applied +to the positive CFG branch. + +At 512x512, 4 steps (Schnell), the 20 cross-attention layers add roughly +10% to denoise time and almost nothing to peak VRAM. Tested on a 12 GB +consumer card alongside Flux Schnell Q4 GGUF + CPU-offloaded clip_l and +t5xxl + GPU-resident VAE. + +At 1024x1024 with Flux Dev Q4 + 20 steps + PuLID, the VAE decode compute +buffer doesn't fit on a 12 GB card even with `--vae-on-cpu`. Workaround: +explicitly route VAE to the CPU backend instead of the offload flag: + +**Windows (PowerShell) arguments** + +```text +--backend "diffusion=vulkan0,vae=cpu" +``` + +**Linux/macOS arguments** + +```text +--backend diffusion=vulkan0,vae=cpu +``` + +The `--vae-on-cpu` flag offloads VAE weights but leaves the compute graph +on the default backend; this is existing stable-diffusion.cpp behavior, +not a PuLID-specific issue. Documented here because anyone running PuLID +at 1024 will hit it. + +The original contributor tested Vulkan and CPU. Other backends were not +validated in those tests. See [backend selection](../guides/backend.md) for +device placement. Internal format details and validation procedures are in the +[PuLID developer notes](../development/pulid.md). diff --git a/docs/models/qwen_image.md b/docs/models/qwen_image.md new file mode 100644 index 0000000000..030f2f772f --- /dev/null +++ b/docs/models/qwen_image.md @@ -0,0 +1,68 @@ +# Qwen Image + +Qwen Image generates images from text using Qwen2.5-VL and the Qwen Image VAE. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Qwen Image | [Safetensors](https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/tree/main/split_files/diffusion_models)
[GGUF](https://huggingface.co/QuantStack/Qwen-Image-GGUF/tree/main) | `--diffusion-model` | Required | +| vae | [Safetensors](https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/tree/main/split_files/vae) | `--vae` | Use matching VAE | +| qwen_2.5_vl 7b | [Safetensors](https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/tree/main/split_files/text_encoders)
[GGUF](https://huggingface.co/mradermacher/Qwen2.5-VL-7B-Instruct-GGUF/tree/main) | `--llm` | Use matching text encoder | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| Qwen Image (`qwen-image`) | `diffusion_model`, `llm`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\qwen-image-Q8_0.gguf ` + --vae ..\models\vae\qwen_image_vae.safetensors ` + --llm ..\models\text_encoders\Qwen2.5-VL-7B-Instruct-Q8_0.gguf ` + -p "一个穿着`"QWEN`"标志的T恤的中国美女正拿着黑色的马克笔面相镜头微笑。她身后的玻璃板上手写体写着 `“一、Qwen-Image的技术路线: 探索视觉生成基础模型的极限,开创理解与生成一体化的未来。二、Qwen-Image的模型特色:1、复杂文字渲染。支持中英渲染、自动布局; 2、精准图像编辑。支持文字编辑、物体增减、风格变换。三、Qwen-Image的未来愿景:赋能专业内容创作、助力生成式AI发展。`”" ` + --cfg-scale 2.5 ` + --sampling-method euler ` + --flow-shift 3 ` + -H 1024 ` + -W 1024 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/qwen-image-Q8_0.gguf \ + --vae ../models/vae/qwen_image_vae.safetensors \ + --llm ../models/text_encoders/Qwen2.5-VL-7B-Instruct-Q8_0.gguf \ + -p "一个穿着\"QWEN\"标志的T恤的中国美女正拿着黑色的马克笔面相镜头微笑。她身后的玻璃板上手写体写着 “一、Qwen-Image的技术路线: 探索视觉生成基础模型的极限,开创理解与生成一体化的未来。二、Qwen-Image的模型特色:1、复杂文字渲染。支持中英渲染、自动布局; 2、精准图像编辑。支持文字编辑、物体增减、风格变换。三、Qwen-Image的未来愿景:赋能专业内容创作、助力生成式AI发展。”" \ + --cfg-scale 2.5 \ + --sampling-method euler \ + --flow-shift 3 \ + -H 1024 \ + -W 1024 \ + --offload-to-cpu \ + -v +``` + +qwen example diff --git a/docs/models/qwen_image_2.1.md b/docs/models/qwen_image_2.1.md new file mode 100644 index 0000000000..7ddfda5a2d --- /dev/null +++ b/docs/models/qwen_image_2.1.md @@ -0,0 +1,201 @@ +# Qwen Image 2.1 + +Qwen Image 2.1 supports text-to-image generation and image editing, using Qwen3-VL-8B as the text encoder and its own VAE. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Qwen Image 2.1 | [Safetensors](https://huggingface.co/Comfy-Org/Qwen-Image-2.1/tree/main/diffusion_models)
[GGUF](https://huggingface.co/leejet/Qwen-Image-2.1-GGUF/tree/main) | `--diffusion-model` | Required | +| vae | [Safetensors](https://huggingface.co/Comfy-Org/Qwen-Image-2.1/tree/main/vae) | `--vae` | Use matching VAE | +| Qwen3-VL-8B-Instruct | safetensors (BF16 or INT8 convrot): [Safetensors](https://huggingface.co/Comfy-Org/Qwen-Image-2.1/tree/main/text_encoders)
[GGUF](https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct-GGUF/tree/main)
For image editing with a GGUF text encoder, also download `mmproj-Qwen3VL-8B-Instruct-F16.gguf` from the same repository and pass it with `--llm_vision`. | `--llm` | Use matching text encoder | + +Use `qwen_image_2.1_vae_bf16.safetensors` with this model. The earlier Qwen Image and Wan 2.2 VAE weights are not interchangeable with the Qwen Image 2.1 VAE weights. + +For faster encoding, decoding, or previews, see [TAEQI2.1](taesd.md#qwen-image-21-taeqi21). + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| Qwen Image 2.1 (`qwen-image-21`) | `diffusion_model`, `llm`, `llm_vision`, `vae`, `tae` | +| Qwen Image 2.1 Turbo (`qwen-image-21-turbo`) | `diffusion_model`, `llm`, `llm_vision`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +Use image dimensions divisible by 32. The resolution-dependent flow schedule is selected automatically. + +#### Text to image + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\qwen_image_2.1_int8_convrot.safetensors ` + --vae ..\models\vae\qwen_image_2.1_vae_bf16.safetensors ` + --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf ` + -p "a lovely cat holding a sign says 'qwen2.1.cpp'" ` + --cfg-scale 6.0 ` + --sampling-method euler ` + --offload-to-cpu ` + -v ` + -o qwen_image_2.1.png +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/qwen_image_2.1_int8_convrot.safetensors \ + --vae ../models/vae/qwen_image_2.1_vae_bf16.safetensors \ + --llm ../models/text_encoders/Qwen3VL-8B-Instruct-Q4_K_M.gguf \ + -p "a lovely cat holding a sign says 'qwen2.1.cpp'" \ + --cfg-scale 6.0 \ + --sampling-method euler \ + --offload-to-cpu \ + -v \ + -o qwen_image_2.1.png +``` + +Qwen Image 2.1 example + +To use GGUF diffusion weights, set `--diffusion-model` to the path of a file such as `qwen_image_2.1-Q4_K.gguf`. + +#### Turbo + +For [Qwen Image 2.1 Turbo](https://huggingface.co/Qwen/Qwen-Image-2.1-Turbo), use the Turbo diffusion weights and select its fixed 8-step sigma schedule with `--sigmas qwen-image-2.1-turbo`. For example, add these sampling options to the command above: + +**Windows (PowerShell) arguments** + +```text +--sigmas qwen-image-2.1-turbo ` + --steps 8 ` + --cfg-scale 1.0 ` + --sampling-method euler +``` + +**Linux/macOS arguments** + +```text +--sigmas qwen-image-2.1-turbo \ + --steps 8 \ + --cfg-scale 1.0 \ + --sampling-method euler +``` + +The preset is equivalent to `--sigmas 1.0,0.978453,0.95418,0.926626,0.89508,0.845148,0.704534,0.414568,0.0`. It only supplies the sigma values; CFG scale and sampling method remain independently configurable. As with an explicit sigma list, the sigma count determines the actual number of sampling steps. The same preset name is also accepted by `--hires-sigmas` for the highres fix second pass. + +#### Image editing + +Pass the reference image with `-r` and describe the edit in `-p`. Vision weights are required; the example below loads them separately with `--llm_vision`. + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\qwen_image_2.1_int8_convrot.safetensors ` + --vae ..\models\vae\qwen_image_2.1_vae_bf16.safetensors ` + --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf ` + --llm_vision ..\models\text_encoders\Qwen3VL-8B-Instruct-mmproj-BF16.gguf ` + -r ..\inputs\qwen\qwen_image_2.1.png ` + -p "change 'qwen2.1.cpp' to 'sd.cpp'" ` + --cfg-scale 6.0 ` + --sampling-method euler ` + --offload-to-cpu ` + -v ` + -o qwen_image_2.1_edit.png +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/qwen_image_2.1_int8_convrot.safetensors \ + --vae ../models/vae/qwen_image_2.1_vae_bf16.safetensors \ + --llm ../models/text_encoders/Qwen3VL-8B-Instruct-Q4_K_M.gguf \ + --llm_vision ../models/text_encoders/Qwen3VL-8B-Instruct-mmproj-BF16.gguf \ + -r ../inputs/qwen/qwen_image_2.1.png \ + -p "change 'qwen2.1.cpp' to 'sd.cpp'" \ + --cfg-scale 6.0 \ + --sampling-method euler \ + --offload-to-cpu \ + -v \ + -o qwen_image_2.1_edit.png +``` + +For multiple reference images, repeat `-r` in the desired order, for example `-r first.png -r second.png`. + +#### Alpha channel + +This model supports alpha channel output. As the model determines whether to output a regular image or with transparency through the prompt, according to [official recommendation](https://github.com/QwenLM/Qwen-Image-2.1#transparent-image-generation-rgba), use the following prompt format for better results: + +> `This is an RGBA image with transparency. . The image has alpha channel and the background is transparent.` + +Since transparency is decided by the prompt rather than by the input or an explicit switch, the same format applies equally to editing, whether or not the reference image itself has an alpha channel. Note that alpha is kept only in `.png` and `.webp` outputs; saving as `.jpg` drops the transparency. + +Here are some examples ran with Q6_K quantization: +| Input | Prompt | Output | +| --- | --- | --- | +| ![Qwen Image 2.1 alpha input example 1](../../assets/qwen/qwen-image-2.1-alpha-in1.png) | This is an RGBA image with transparency. Replace the text "BLOOM" with "Qwen Image 2.1", keeping the same font of the original text. The image has alpha channel and the background is transparent. | ![Qwen Image 2.1 alpha output example 1](../../assets/qwen/qwen-image-2.1-alpha-out1.png) | +| ![Qwen Image 2.1 alpha input example 2](../../assets/logo.png) | This is an RGBA image with transparency. Remove the background of the image, keeping only the text and cat. The image has alpha channel and the background is transparent. | ![Qwen Image 2.1 alpha output example 2](../../assets/qwen/qwen-image-2.1-alpha-out2.png) | + +#### Other features + +Other features of the model could be found on the [model card from QwenLM/Qwen-Image-2.1 repo](https://github.com/QwenLM/Qwen-Image-2.1), including 2 finetuned prompt rewriting Qwen3.5-9B model. + +## Options + +### Prefix cache + +Qwen Image 2.1 caches the text and reference-image condition between denoising +steps. Positive and negative conditions use separate caches, released when +sampling ends. + +Set `qwen_image_2_1_prefix_cache_type` through `--model-args`: + +- `auto` (default): selects FP16 when the Flash Attention path supports it, + otherwise FP32. Sage Attention and custom attention scaling use FP32 in this + automatic mode. +- `f32`: always uses FP32 cache storage. +- `f16`: always uses FP16 cache storage, including with ordinary attention. +- `bf16`, `q4_1`, `q5_0`, `q5_1`, `q8_0`, `q4_K`, `q6_K`, `iq4_nl`, and `iq4_xs`: + explicitly select another supported storage type. Quantized storage is lossy; + `auto` never selects it. Unsupported or incompatible types are ignored with a + warning, keeping the previous setting or the default. + +For example, add `--model-args qwen_image_2_1_prefix_cache_type=q8_0` for 8-bit +cache storage. Smaller storage types reduce persistent cache memory but can +change numerical results and add conversion work. Attention working buffers +still require floating-point storage. + +For the default 32-layer model and 4096 condition tokens, approximate memory per +condition is as follows, excluding weights, working buffers, and overhead: + +| Cache type | Memory | +| --- | ---: | +| `f32` | 4 GiB | +| `f16` | 2 GiB | +| `q8_0` | 1.0625 GiB | +| `q4_0` | 0.5625 GiB | + +If cached execution runs out of memory, the runtime releases the caches and +retries the full sequence without caching for the rest of that sampling run. +Some conditioning extensions also use the full-sequence path. + +Disable caching with `--model-args qwen_image_2_1_prefix_cache=false`. Numerical +results can differ slightly because caching changes computation sizes. See the +[prefix cache developer notes](../development/qwen_image_prefix_cache.md) for +storage compatibility and implementation details. diff --git a/docs/models/qwen_image_edit.md b/docs/models/qwen_image_edit.md new file mode 100644 index 0000000000..adec6fff3d --- /dev/null +++ b/docs/models/qwen_image_edit.md @@ -0,0 +1,150 @@ +# Qwen Image Edit + +Qwen Image Edit supports reference-image editing with the original, 2509, and 2511 checkpoints. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Qwen Image | Qwen Image Edit
[Safetensors](https://huggingface.co/Comfy-Org/Qwen-Image-Edit_ComfyUI/tree/main/split_files/diffusion_models)
[GGUF](https://huggingface.co/QuantStack/Qwen-Image-Edit-GGUF/tree/main)
Qwen Image Edit 2509
[Safetensors](https://huggingface.co/Comfy-Org/Qwen-Image-Edit_ComfyUI/tree/main/split_files/diffusion_models)
[GGUF](https://huggingface.co/QuantStack/Qwen-Image-Edit-2509-GGUF/tree/main)
Qwen Image Edit 2511
[Safetensors](https://huggingface.co/Comfy-Org/Qwen-Image-Edit_ComfyUI/tree/main/split_files/diffusion_models)
[GGUF](https://huggingface.co/unsloth/Qwen-Image-Edit-2511-GGUF/tree/main) | `--diffusion-model` | Required | +| vae | [Safetensors](https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/tree/main/split_files/vae) | `--vae` | Use matching VAE | +| qwen_2.5_vl 7b | [Safetensors](https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/tree/main/split_files/text_encoders)
[GGUF](https://huggingface.co/mradermacher/Qwen2.5-VL-7B-Instruct-GGUF/tree/main) | `--llm` | Use matching text encoder | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| Qwen Image Edit (`qwen-image-edit`) | `diffusion_model`, `llm`, `llm_vision`, `vae`, `tae` | +| Qwen Image Edit 2509 (`qwen-image-edit-2509`) | `diffusion_model`, `llm`, `llm_vision`, `vae`, `tae` | +| Qwen Image Edit 2511 (`qwen-image-edit-2511`) | `diffusion_model`, `llm`, `llm_vision`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### Qwen Image Edit + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\Qwen_Image_Edit-Q8_0.gguf ` + --vae ..\models\vae\qwen_image_vae.safetensors ` + --llm ..\models\text_encoders\qwen_2.5_vl_7b.safetensors ` + -r ..\inputs\flux\flux1-dev-q8_0.png ` + -p "change 'flux.cpp' to 'edit.cpp'" ` + --cfg-scale 2.5 ` + --sampling-method euler ` + --flow-shift 3 ` + --offload-to-cpu ` + -v ` + --seed 1118877715456453 +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/Qwen_Image_Edit-Q8_0.gguf \ + --vae ../models/vae/qwen_image_vae.safetensors \ + --llm ../models/text_encoders/qwen_2.5_vl_7b.safetensors \ + -r ../inputs/flux/flux1-dev-q8_0.png \ + -p "change 'flux.cpp' to 'edit.cpp'" \ + --cfg-scale 2.5 \ + --sampling-method euler \ + --flow-shift 3 \ + --offload-to-cpu \ + -v \ + --seed 1118877715456453 +``` + +qwen_image_edit + +#### Qwen Image Edit 2509 + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\Qwen-Image-Edit-2509-Q4_K_S.gguf ` + --vae ..\models\vae\qwen_image_vae.safetensors ` + --llm ..\models\text_encoders\Qwen2.5-VL-7B-Instruct-Q8_0.gguf ` + --llm_vision ..\models\text_encoders\Qwen2.5-VL-7B-Instruct.mmproj-Q8_0.gguf ` + -r ..\inputs\flux\flux1-dev-q8_0.png ` + -p "change 'flux.cpp' to 'Qwen Image Edit 2509'" ` + --cfg-scale 2.5 ` + --sampling-method euler ` + --flow-shift 3 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/Qwen-Image-Edit-2509-Q4_K_S.gguf \ + --vae ../models/vae/qwen_image_vae.safetensors \ + --llm ../models/text_encoders/Qwen2.5-VL-7B-Instruct-Q8_0.gguf \ + --llm_vision ../models/text_encoders/Qwen2.5-VL-7B-Instruct.mmproj-Q8_0.gguf \ + -r ../inputs/flux/flux1-dev-q8_0.png \ + -p "change 'flux.cpp' to 'Qwen Image Edit 2509'" \ + --cfg-scale 2.5 \ + --sampling-method euler \ + --flow-shift 3 \ + --offload-to-cpu \ + -v +``` + +qwen_image_edit_2509 + +#### Qwen Image Edit 2511 + +To use the new Qwen Image Edit 2511 mode, `--model-args qwen_image_zero_cond_t=true` must be enabled; otherwise, image editing quality will degrade significantly. + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\qwen-image-edit-2511-Q4_K_M.gguf ` + --vae ..\models\vae\qwen_image_vae.safetensors ` + --llm ..\models\text_encoders\qwen_2.5_vl_7b.safetensors ` + --model-args qwen_image_zero_cond_t=true ` + -r ..\inputs\flux\flux1-dev-q8_0.png ` + -p "change 'flux.cpp' to 'edit.cpp'" ` + --cfg-scale 2.5 ` + --sampling-method euler ` + --flow-shift 3 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/qwen-image-edit-2511-Q4_K_M.gguf \ + --vae ../models/vae/qwen_image_vae.safetensors \ + --llm ../models/text_encoders/qwen_2.5_vl_7b.safetensors \ + --model-args qwen_image_zero_cond_t=true \ + -r ../inputs/flux/flux1-dev-q8_0.png \ + -p "change 'flux.cpp' to 'edit.cpp'" \ + --cfg-scale 2.5 \ + --sampling-method euler \ + --flow-shift 3 \ + --offload-to-cpu \ + -v +``` + +qwen_image_edit_2509 diff --git a/docs/relay_vsr.md b/docs/models/relay_vsr.md similarity index 57% rename from docs/relay_vsr.md rename to docs/models/relay_vsr.md index e35f5fdb7f..4900fa4fb8 100644 --- a/docs/relay_vsr.md +++ b/docs/models/relay_vsr.md @@ -1,9 +1,14 @@ -# RelayVSR image and video super-resolution +# RelayVSR [RelayVSR](https://github.com/kopperx/RelayVSR) upscales images and videos by 2x or 4x with `-M vid_upscale`. -## Weights +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| RelayVSR diffusion model | [ComfyUI RelayVSR](https://huggingface.co/Comfy-Org/RelayVSR) | `--diffusion-model` | Match the requested 2x or 4x scale | +| RelayVSR VAE | Matching scale-specific VAE from the same repository | `--vae` | Required; ordinary Wan VAE is incompatible | [ComfyUI safetensors weights](https://huggingface.co/Comfy-Org/RelayVSR) load directly. Select the matching pair for the requested scale: @@ -17,14 +22,40 @@ No separate text encoder or ordinary Wan VAE is needed. ## Usage +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### CLI + For a video, extract numbered frames into a directory, then run: +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_upscale ` + --diffusion-model ..\models\diffusion_models\relayvsr_x2_lightx2v_int8_convrot.safetensors ` + --vae ..\models\vae\relayvsr_flash_vae_x2_fp32.safetensors ` + --input-frames input_frames ` + --upscale-factor 2 ` + --fps 24 ` + --vsr-window-size 16 ` + -s 42 ` + -o output.avi +``` + +**Linux/macOS** + ```sh -sd-cli -M vid_upscale \ - --diffusion-model models/diffusion_models/relayvsr_x2_lightx2v_int8_convrot.safetensors \ - --vae models/vae/relayvsr_flash_vae_x2_fp32.safetensors \ - --input-frames input_frames --upscale-factor 2 --fps 24 \ - --vsr-window-size 16 -s 42 -o output.avi +./bin/sd-cli \ + -M vid_upscale \ + --diffusion-model ../models/diffusion_models/relayvsr_x2_lightx2v_int8_convrot.safetensors \ + --vae ../models/vae/relayvsr_flash_vae_x2_fp32.safetensors \ + --input-frames input_frames \ + --upscale-factor 2 \ + --fps 24 \ + --vsr-window-size 16 \ + -s 42 \ + -o output.avi ``` Frames are read in filename order. Set `--fps` to the source video's frame rate. @@ -35,7 +66,7 @@ For a single image, replace `--input-frames input_frames` with `-i input.png` and use `-o output.png` (or JPEG/WebP). No FPS setting is needed for an image. `-i` / `--init-img` and `--input-frames` are mutually exclusive. -## Settings and limits +## Notes and limitations - `--upscale-factor` is required. Use 2 or 4 with the corresponding weights. Output width and height are exactly the input dimensions times the scale. @@ -43,7 +74,7 @@ and use `-o output.png` (or JPEG/WebP). No FPS setting is needed for an image. The default 16 selects reference frames 0, 15, 30, and so on, and always includes the final frame. Every input frame appears once in the output. - `--flash-attn auto` and `--conv-direct auto` are enabled by default. See - [backend selection](backend.md) to choose a device or weight placement. + [backend selection](../guides/backend.md) to choose a device or weight placement. - Sampler, scheduler, and step settings do not configure RelayVSR. `--vsr-temporal-chunk-size` is a SeedVR2 option. - Long videos require enough system memory to hold the input and output frames. diff --git a/docs/models/sd.md b/docs/models/sd.md new file mode 100644 index 0000000000..2d1ab53e36 --- /dev/null +++ b/docs/models/sd.md @@ -0,0 +1,138 @@ +# Stable Diffusion 1.x, 2.x, and SDXL + +Stable Diffusion 1.x, 2.x, and SDXL support text-to-image and image-to-image +generation from single-file checkpoints. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| SD 1.4 checkpoint | [CompVis SD 1.4](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original) | `-m` / `--model` | Choose a matching SD checkpoint | +| SD 1.5 checkpoint | [Stable Diffusion 1.5](https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5) | `-m` / `--model` | Choose a matching SD checkpoint | +| SD 2.1 checkpoint | [Stable Diffusion 2.1](https://huggingface.co/Manojb/stable-diffusion-2-1-base) | `-m` / `--model` | Choose a matching SD checkpoint | +| SDXL checkpoint | `sd_xl_base_1.0.safetensors` from [SDXL](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) | `-m` / `--model` | For SDXL examples | +| SDXL VAE | [`sdxl_vae-fp16-fix.safetensors`](https://huggingface.co/madebyollin/sdxl-vae-fp16-fix) | `--vae` | Use an override when required by the checkpoint | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| SD 1.x (`sd1-checkpoint`) | `model`, `vae`, `tae` | +| SD 2.1 512 (`sd2-512-checkpoint`) | `model`, `vae`, `tae` | +| SD 2.1 768 (`sd2-768-checkpoint`) | `model`, `vae`, `tae` | +| SDXL (`sdxl-checkpoint`) | `model`, `vae`, `tae` | +| SD Turbo (`sd-turbo`) | `model`, `vae`, `tae` | +| SDXL Turbo (`sdxl-turbo`) | `model`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### SD 1.4 + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe -m ..\models\checkpoints\sd-v1-4.ckpt -p "a lovely cat" +``` + +**Linux/macOS** + +```sh +./bin/sd-cli -m ../models/checkpoints/sd-v1-4.ckpt -p "a lovely cat" +``` + +#### SD 1.5 + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe -m ..\models\checkpoints\v1-5-pruned-emaonly.safetensors -p "a lovely cat" +``` + +**Linux/macOS** + +```sh +./bin/sd-cli -m ../models/checkpoints/v1-5-pruned-emaonly.safetensors -p "a lovely cat" +``` + +#### SDXL + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -m ..\models\checkpoints\sd_xl_base_1.0.safetensors ` + --vae ..\models\vae\sdxl_vae-fp16-fix.safetensors ` + -p "a lovely cat" ` + -H 1024 ` + -W 1024 ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -m ../models/checkpoints/sd_xl_base_1.0.safetensors \ + --vae ../models/vae/sdxl_vae-fp16-fix.safetensors \ + -p "a lovely cat" \ + -H 1024 \ + -W 1024 \ + -v +``` + +#### img2img example + +Use the image generated above as `./output.png`. + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -m ..\models\checkpoints\sd-v1-4.ckpt ` + -p "cat with blue eyes" ` + -i .\output.png ` + --strength 0.4 ` + -o .\img2img_output.png +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -m ../models/checkpoints/sd-v1-4.ckpt \ + -p "cat with blue eyes" \ + -i ./output.png \ + --strength 0.4 \ + -o ./img2img_output.png +``` + +

+ +

+ +#### Precision comparison + +Using formats of different precisions will yield results of varying quality. + +| f32 | f16 |q8_0 |q5_0 |q5_1 |q4_0 |q4_1 | +| ---- |---- |---- |---- |---- |---- |---- | +| ![](../../assets/f32.png) |![](../../assets/f16.png) |![](../../assets/q8_0.png) |![](../../assets/q5_0.png) |![](../../assets/q5_1.png) |![](../../assets/q4_0.png) |![](../../assets/q4_1.png) | + +## Notes and limitations + +See [SD3 / SD3.5](sd3.md) for Stable Diffusion 3, and [FLUX.1](flux.md) for FLUX. +The [SD3 model repository](https://huggingface.co/stabilityai/stable-diffusion-3-medium) uses a different component setup. +For distilled and tiny variants, see [distilled SD models](distilled_sd.md). diff --git a/docs/models/sd3.md b/docs/models/sd3.md new file mode 100644 index 0000000000..12cab00039 --- /dev/null +++ b/docs/models/sd3.md @@ -0,0 +1,75 @@ +# Stable Diffusion 3 / 3.5 + +Stable Diffusion 3 and 3.5 generate images from text. The example below uses SD3.5 Large with separate text encoders. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| sd3.5_large | sd3.5_large from [Download](https://huggingface.co/stabilityai/stable-diffusion-3.5-large/blob/main/sd3.5_large.safetensors) | `--model` | Required | +| clip_g | clip_g from [Download](https://huggingface.co/Comfy-Org/stable-diffusion-3.5-fp8/blob/main/text_encoders/clip_g.safetensors) | `--clip_g` | Required | +| clip_l | clip_l from [Download](https://huggingface.co/Comfy-Org/stable-diffusion-3.5-fp8/blob/main/text_encoders/clip_l.safetensors) | `--clip_l` | Required | +| t5xxl | t5xxl from [Download](https://huggingface.co/Comfy-Org/stable-diffusion-3.5-fp8/blob/main/text_encoders/t5xxl_fp16.safetensors) | `--t5xxl` | Required | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| SD 3 Medium (`sd3-medium`) | `model`, `vae`, `tae` | +| SD 3.5 Medium (`sd35-medium`) | `model`, `clip_l`, `clip_g`, `t5xxl`, `vae`, `tae` | +| SD 3.5 Large (`sd35-large`) | `model`, `clip_l`, `clip_g`, `t5xxl`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### SD3.5 Large + +For example: + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -m ..\models\checkpoints\sd3.5_large.safetensors ` + --clip_l ..\models\text_encoders\clip_l.safetensors ` + --clip_g ..\models\text_encoders\clip_g.safetensors ` + --t5xxl ..\models\text_encoders\t5xxl_fp16.safetensors ` + -p "a lovely cat holding a sign says \`"Stable diffusion 3.5 Large\`"" ` + --cfg-scale 4.5 ` + --sampling-method euler ` + -H 1024 ` + -W 1024 ` + --clip-on-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -m ../models/checkpoints/sd3.5_large.safetensors \ + --clip_l ../models/text_encoders/clip_l.safetensors \ + --clip_g ../models/text_encoders/clip_g.safetensors \ + --t5xxl ../models/text_encoders/t5xxl_fp16.safetensors \ + -p "a lovely cat holding a sign says \\\"Stable diffusion 3.5 Large\\\"" \ + --cfg-scale 4.5 \ + --sampling-method euler \ + -H 1024 \ + -W 1024 \ + --clip-on-cpu \ + -v +``` + +![](../../assets/sd3.5_large.png) diff --git a/docs/seedvr2.md b/docs/models/seedvr2.md similarity index 54% rename from docs/seedvr2.md rename to docs/models/seedvr2.md index 24f55032d6..0a48914bf2 100644 --- a/docs/seedvr2.md +++ b/docs/models/seedvr2.md @@ -1,8 +1,13 @@ -# SeedVR2 image and video super-resolution +# SeedVR2 SeedVR2 upscales images and videos with `-M vid_upscale`. Support is experimental. -## Weights +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Diffusion model | [ComfyUI SeedVR2](https://huggingface.co/Comfy-Org/SeedVR2) | `--diffusion-model` | Choose 3B or 7B Sharp below | +| SeedVR2 VAE | `seedvr2_ema_vae_fp16.safetensors` from the same repository | `--vae` | Required; shared by both variants | These [ComfyUI safetensors weights](https://huggingface.co/Comfy-Org/SeedVR2) load directly, without conversion or a separate text encoder: @@ -14,30 +19,79 @@ load directly, without conversion or a separate text encoder: Use the matching SeedVR2 VAE for either model. The 7B model requires more memory. -## Upscale an image +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### CLI + +#### Upscale an image + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_upscale ` + --diffusion-model ..\models\diffusion_models\seedvr2_3b_int8_convrot.safetensors ` + --vae ..\models\vae\seedvr2_ema_vae_fp16.safetensors ` + -i input.png ` + --upscale-factor 2 ` + -s 42 ` + -o restored.png +``` + +**Linux/macOS** ```sh -sd-cli -M vid_upscale \ - --diffusion-model models/diffusion_models/seedvr2_3b_int8_convrot.safetensors \ - --vae models/vae/seedvr2_ema_vae_fp16.safetensors \ - -i input.png --upscale-factor 2 -s 42 -o restored.png +./bin/sd-cli \ + -M vid_upscale \ + --diffusion-model ../models/diffusion_models/seedvr2_3b_int8_convrot.safetensors \ + --vae ../models/vae/seedvr2_ema_vae_fp16.safetensors \ + -i input.png \ + --upscale-factor 2 \ + -s 42 \ + -o restored.png ``` `-i` / `--init-img` accepts a single image. Outputs can be PNG, JPEG, or WebP. No FPS or temporal chunk setting is needed for an image. -## Upscale a video +#### Upscale a video Extract the video into numbered frames, then pass the frame directory: +**Windows (PowerShell)** + +```powershell +mkdir frames +ffmpeg -i input.mp4 -fps_mode passthrough frames\%06d.png +.\bin\Release\sd-cli.exe ` + -M vid_upscale ` + --diffusion-model ..\models\diffusion_models\seedvr2_3b_int8_convrot.safetensors ` + --vae ..\models\vae\seedvr2_ema_vae_fp16.safetensors ` + --input-frames frames ` + --upscale-factor 2 ` + --fps 24 ` + --vsr-temporal-chunk-size 1 ` + -s 42 ` + -o restored.avi +``` + +**Linux/macOS** + ```sh mkdir frames ffmpeg -i input.mp4 -fps_mode passthrough frames/%06d.png -sd-cli -M vid_upscale \ - --diffusion-model models/diffusion_models/seedvr2_3b_int8_convrot.safetensors \ - --vae models/vae/seedvr2_ema_vae_fp16.safetensors \ - --input-frames frames --upscale-factor 2 --fps 24 \ - --vsr-temporal-chunk-size 1 -s 42 -o restored.avi +./bin/sd-cli \ + -M vid_upscale \ + --diffusion-model ../models/diffusion_models/seedvr2_3b_int8_convrot.safetensors \ + --vae ../models/vae/seedvr2_ema_vae_fp16.safetensors \ + --input-frames frames \ + --upscale-factor 2 \ + --fps 24 \ + --vsr-temporal-chunk-size 1 \ + -s 42 \ + -o restored.avi ``` Set `--fps` to the source video's frame rate; 24 is only the example above. @@ -46,7 +100,7 @@ Frames are read in filename order. Use either `-i` or `--input-frames`, not both Video outputs support AVI, WebM, and animated WebP. To save individual frames, use an output pattern such as `-o restored/%06d.png`. -## Settings and limits +## Notes and limitations - `--upscale-factor` is required and must be a positive integer. Output width and height are exactly the input dimensions multiplied by this value. @@ -59,7 +113,7 @@ use an output pattern such as `-o restored/%06d.png`. - If you run out of GPU memory, try `--vsr-temporal-chunk-size 1`. A five-frame chunk at 1728x960 output resolution exceeded 24 GB of GPU memory in testing. - `--flash-attn auto` and `--conv-direct auto` are enabled by default. See - [backend selection](backend.md) to choose a device or weight placement. + [backend selection](../guides/backend.md) to choose a device or weight placement. - SeedVR2 does not use sampler, scheduler, or step settings. `--vsr-window-size` is a RelayVSR option and does not affect SeedVR2. diff --git a/docs/models/sefi_image.md b/docs/models/sefi_image.md new file mode 100644 index 0000000000..1d6ffbdb76 --- /dev/null +++ b/docs/models/sefi_image.md @@ -0,0 +1,178 @@ +# SeFi-Image + +SeFi-Image uses a Flux2-style dual-time transformer (semantic + texture streams), the standard Flux2 VAE, and Qwen3-VL as the LLM text encoder. Tech report: [arXiv:2606.22568](https://arxiv.org/abs/2606.22568). + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| SeFi transformer | [SeFi-Image repositories](https://huggingface.co/SeFi-Image) | `--diffusion-model` | Choose matching scale and family; prepare as described below | +| Qwen3-VL text encoder | 2B for SeFi 1B/2B; 4B for SeFi 5B | `--llm` | Required; convert as described below | +| FLUX.2 VAE | [FLUX.2 dev](https://huggingface.co/black-forest-labs/FLUX.2-dev) | `--vae` | Required; `flux2_ae.safetensors` | + +The SeFi-Image family ships in three scales (1B / 2B / 5B) and three families (Base / RL / turbo), all gated on Hugging Face under https://huggingface.co/SeFi-Image. + +- 1B and 2B variants pair with Qwen3-VL-2B-Instruct. +- 5B variants pair with Qwen3-VL-4B-Instruct. +- All variants use the standard Flux2 VAE (`flux2_ae.safetensors` from https://huggingface.co/black-forest-labs/FLUX.2-dev). + +Prepare the transformer and text encoder from the source repository root, using the conversion scripts: + +**Windows (PowerShell)** + +```powershell +python scripts\convert_sefi.py "" "/sefi__.safetensors" +python scripts\convert_qwen3_vl.py "/Qwen3-VL-XB-Instruct" "/qwen3_vl_b.safetensors" +``` + +**Linux/macOS** + +```sh +python3 scripts/convert_sefi.py "" "/sefi__.safetensors" +python3 scripts/convert_qwen3_vl.py "/Qwen3-VL-XB-Instruct" "/qwen3_vl_b.safetensors" +``` + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| SeFi Image 5B RL (`sefi-image`) | `diffusion_model`, `llm`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### 1B / 2B turbo + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\sefi_1b_turbo.safetensors ` + --vae ..\models\vae\flux2_ae.safetensors ` + --llm ..\models\text_encoders\qwen3_vl_2b.safetensors ` + -p "a photograph of an orange tabby cat sitting on a couch" ` + --cfg-scale 1.0 ` + --steps 4 ` + -W 1024 ` + -H 1024 ` + --offload-to-cpu ` + -s 42 ` + -o out.png +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/sefi_1b_turbo.safetensors \ + --vae ../models/vae/flux2_ae.safetensors \ + --llm ../models/text_encoders/qwen3_vl_2b.safetensors \ + -p "a photograph of an orange tabby cat sitting on a couch" \ + --cfg-scale 1.0 \ + --steps 4 \ + -W 1024 \ + -H 1024 \ + --offload-to-cpu \ + -s 42 \ + -o out.png +``` + +#### 1B / 2B base + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\sefi_1b_base.safetensors ` + --vae ..\models\vae\flux2_ae.safetensors ` + --llm ..\models\text_encoders\qwen3_vl_2b.safetensors ` + -p "a photograph of an orange tabby cat sitting on a couch" ` + --cfg-scale 4.0 ` + --steps 50 ` + -W 1024 ` + -H 1024 ` + --offload-to-cpu ` + -s 42 ` + -o out.png +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/sefi_1b_base.safetensors \ + --vae ../models/vae/flux2_ae.safetensors \ + --llm ../models/text_encoders/qwen3_vl_2b.safetensors \ + -p "a photograph of an orange tabby cat sitting on a couch" \ + --cfg-scale 4.0 \ + --steps 50 \ + -W 1024 \ + -H 1024 \ + --offload-to-cpu \ + -s 42 \ + -o out.png +``` + +#### 5B (needs streaming on 12 GiB VRAM) + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\sefi_5b_turbo.safetensors ` + --vae ..\models\vae\flux2_ae.safetensors ` + --llm ..\models\text_encoders\qwen3_vl_4b.safetensors ` + -p "a photograph of an orange tabby cat sitting on a couch" ` + --cfg-scale 1.0 ` + --steps 4 ` + -W 1024 ` + -H 1024 ` + --max-vram 8 ` + --offload-to-cpu ` + -s 42 ` + -o out.png +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/sefi_5b_turbo.safetensors \ + --vae ../models/vae/flux2_ae.safetensors \ + --llm ../models/text_encoders/qwen3_vl_4b.safetensors \ + -p "a photograph of an orange tabby cat sitting on a couch" \ + --cfg-scale 1.0 \ + --steps 4 \ + -W 1024 \ + -H 1024 \ + --max-vram 8 \ + --offload-to-cpu \ + -s 42 \ + -o out.png +``` + +SeFi-Image 5B turbo example + +## Options + +### Variant defaults + +| Family | timestep_shift_alpha | steps | cfg-scale | +|---|---|---|---| +| Base | 0.3 | 50 | 4.0 | +| RL | 0.3 | 50 | 4.0 | +| turbo | 1.0 | 4 | 1.0 | + +The dispatcher picks `alpha` from the filename (`turbo` substring => 1.0, otherwise 0.3). Override via `--extra-sample-args sefi_alpha=` or `sefi_delta_t=`. diff --git a/docs/sensenova_u1.md b/docs/models/sensenova_u1.md similarity index 50% rename from docs/sensenova_u1.md rename to docs/models/sensenova_u1.md index 57b22a4ef1..e77ab39e6a 100644 --- a/docs/sensenova_u1.md +++ b/docs/models/sensenova_u1.md @@ -1,37 +1,69 @@ -# How to Use +# SenseNova U1.5 SenseNova U1.5 is an 8B MoT model that performs diffusion directly in RGB pixel space. It does not require a separate text encoder or VAE. -## Download weights +## Model files -- Download SenseNova U1.5 8B MoT - - safetensors: https://huggingface.co/sensenova/SenseNova-U1.5-8B-MoT +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| SenseNova U1.5 8B MoT | [Safetensors](https://huggingface.co/sensenova/SenseNova-U1.5-8B-MoT) | `--model` | Required | Pass the complete downloaded repository directory to `--model`. The directory must contain `model.safetensors.index.json`, every referenced Safetensors shard, and the tokenizer files. -## Examples +## Usage -### CUDA +CLI examples use the [shared path and shell conventions](README.md#example-conventions). -```bash +### Web UI + +Follow the server instructions for +[sharded checkpoints](../../examples/server/README.md#sharded-checkpoints). +Pass the complete repository directory with `--model` when starting the server, +using the same model path as the CLI examples below. The browser then uses +**Command-line model (all features)** for that loaded checkpoint. + +### CLI + +#### CUDA + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --model ..\models\checkpoints\SenseNova-U1.5-8B-MoT ` + --prompt "a red cube on a white background" ` + --steps 50 ` + --cfg-scale 4 ` + --flow-shift 3 ` + --sampling-method euler ` + --width 2048 ` + --height 2048 ` + --rng cuda ` + --seed 42 ` + --output output.png +``` + +**Linux/macOS** + +```sh ./bin/sd-cli \ - --model /path/to/SenseNova-U1.5-8B-MoT \ + --model ../models/checkpoints/SenseNova-U1.5-8B-MoT \ --prompt "a red cube on a white background" \ - --width 2048 \ - --height 2048 \ --steps 50 \ --cfg-scale 4 \ --flow-shift 3 \ - --seed 42 \ --sampling-method euler \ + --width 2048 \ + --height 2048 \ --rng cuda \ + --seed 42 \ --output output.png ``` -## Notes +## Notes and limitations - To match the official non-thinking text-to-image pipeline, use 50 Euler steps, CFG 4, flow shift 3, seed 42, CUDA RNG, and an empty negative prompt. diff --git a/docs/models/taesd.md b/docs/models/taesd.md new file mode 100644 index 0000000000..607c0b419c --- /dev/null +++ b/docs/models/taesd.md @@ -0,0 +1,130 @@ +# TAESD and related tiny autoencoders + +Tiny autoencoders provide an alternative to full VAE encoding, decoding, or +previews. Choose the file matching the base model. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| TAESD | [Safetensors weights](https://huggingface.co/madebyollin/taesd/blob/main/diffusion_pytorch_model.safetensors) | `--taesd` / `--tae` | For compatible SD 1.x models | +| TAEHV for Wan 2.1 | [taew2_1.safetensors](https://github.com/madebyollin/taehv/blob/main/safetensors/taew2_1.safetensors) | `--taesd` / `--tae` | Qwen Image, Wan 2.1, and Wan 2.2 A14B | +| TAEHV for Wan 2.2 5B | [taew2_2.safetensors](https://github.com/madebyollin/taehv/blob/main/safetensors/taew2_2.safetensors) | `--taesd` / `--tae` | Wan 2.2 TI2V 5B only | +| TAEQI2.1 | [taeqi2_1.safetensors](https://huggingface.co/madebyollin/taeqi2_1/blob/main/taeqi2_1.safetensors) | `--taesd` / `--tae` | Qwen Image 2.1, including RGBA | + +### Qwen-Image and wan (TAEHV) + +[TAEHV](https://github.com/madebyollin/taehv) provides tiny autoencoders for the +model families in the table. Replace `--vae PATH` with `--taesd PATH` using the +matching file. A tiny autoencoder can reduce VAE memory requirements, with a +possible quality tradeoff. + +### Qwen Image 2.1 (TAEQI2.1) + +[TAEQI2.1](https://github.com/madebyollin/taesd) supports Qwen Image 2.1 RGBA +images. Download the official file directly; conversion is unnecessary. Replace +`--vae PATH` with `--taesd taeqi2_1.safetensors` in the +[Qwen Image 2.1 examples](qwen_image_2.1.md). + +### Download commands + +Save the files under the repository's `models/vae/` directory. From the build +directory, create it if needed: + +**Windows (PowerShell)** + +```powershell +New-Item -ItemType Directory -Force ..\models\vae | Out-Null +``` + +**Linux/macOS** + +```sh +mkdir -p ../models/vae +``` + +**Windows (PowerShell)** + +```powershell +curl.exe -L -o ..\models\vae\diffusion_pytorch_model.safetensors https://huggingface.co/madebyollin/taesd/resolve/main/diffusion_pytorch_model.safetensors +``` + +**Linux/macOS** + +```sh +curl -L -o ../models/vae/diffusion_pytorch_model.safetensors https://huggingface.co/madebyollin/taesd/resolve/main/diffusion_pytorch_model.safetensors +``` + +**Windows (PowerShell)** + +```powershell +curl.exe -L -o ..\models\vae\taew2_1.safetensors https://github.com/madebyollin/taehv/raw/refs/heads/main/safetensors/taew2_1.safetensors +``` + +**Linux/macOS** + +```sh +curl -L -o ../models/vae/taew2_1.safetensors https://github.com/madebyollin/taehv/raw/refs/heads/main/safetensors/taew2_1.safetensors +``` + +**Windows (PowerShell)** + +```powershell +curl.exe -L -o ..\models\vae\taew2_2.safetensors https://github.com/madebyollin/taehv/raw/refs/heads/main/safetensors/taew2_2.safetensors +``` + +**Linux/macOS** + +```sh +curl -L -o ../models/vae/taew2_2.safetensors https://github.com/madebyollin/taehv/raw/refs/heads/main/safetensors/taew2_2.safetensors +``` + +**Windows (PowerShell)** + +```powershell +curl.exe -L -o ..\models\vae\taeqi2_1.safetensors https://huggingface.co/madebyollin/taeqi2_1/resolve/main/taeqi2_1.safetensors +``` + +**Linux/macOS** + +```sh +curl -L -o ../models/vae/taeqi2_1.safetensors https://huggingface.co/madebyollin/taeqi2_1/resolve/main/taeqi2_1.safetensors +``` + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Use a compatible base model recipe that offers the `tae` slot, and select the +tiny autoencoder matching that model. See [Web UI guide](../../examples/server/README.md#web-ui). + +### CLI + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -m ..\models\checkpoints\v1-5-pruned-emaonly.safetensors ` + --taesd ..\models\vae\diffusion_pytorch_model.safetensors ` + -p "a lovely cat" +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -m ../models/checkpoints/v1-5-pruned-emaonly.safetensors \ + --taesd ../models/vae/diffusion_pytorch_model.safetensors \ + -p "a lovely cat" +``` + +## Options + +For previews only, keep the full `--vae PATH` and add +`--taesd PATH --taesd-preview-only --preview tae`. + +`sd-server` accepts `--taesd` and `--taesd-preview-only` at startup; choose the +preview method in each request. For device placement and convolution settings, +see [backend selection](../guides/backend.md). diff --git a/docs/models/wan.md b/docs/models/wan.md new file mode 100644 index 0000000000..1b4695536e --- /dev/null +++ b/docs/models/wan.md @@ -0,0 +1,1063 @@ +# Wan + +Wan 2.1 and 2.2 support video generation with task-specific checkpoints for text, +images, audio, and VACE conditioning. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Wan2.1 T2V 1.3B | [safetensors](https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/tree/main/split_files/diffusion_models) | `--diffusion-model` | Match the task and variant | +| Wan2.1 T2V 14B | [safetensors](https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/tree/main/split_files/diffusion_models)
[gguf](https://huggingface.co/city96/Wan2.1-T2V-14B-gguf/tree/main) | `--diffusion-model` | Match the task and variant | +| Wan2.1 I2V 14B 480P | [safetensors](https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/tree/main/split_files/diffusion_models)
[gguf](https://huggingface.co/city96/Wan2.1-I2V-14B-480P-gguf/tree/main) | `--diffusion-model` | Match the task and variant | +| Wan2.1 I2V 14B 720P | [safetensors](https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/tree/main/split_files/diffusion_models)
[gguf](https://huggingface.co/city96/Wan2.1-I2V-14B-720P-gguf/tree/main) | `--diffusion-model` | Match the task and variant | +| Wan2.1 FLF2V 14B 720P | [safetensors](https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/tree/main/split_files/diffusion_models)
[gguf](https://huggingface.co/city96/Wan2.1-FLF2V-14B-720P-gguf/tree/main) | `--diffusion-model` | Match the task and variant | +| Wan2.1 VACE 1.3B | [safetensors](https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/tree/main/split_files/diffusion_models)
[gguf](https://huggingface.co/calcuis/wan-1.3b-gguf/tree/main) | `--diffusion-model` | Match the task and variant | +| Wan2.1 VACE 14B | [safetensors](https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/tree/main/split_files/diffusion_models)
[gguf](https://huggingface.co/QuantStack/Wan2.1_14B_VACE-GGUF/tree/main) | `--diffusion-model` | Match the task and variant | +| Wan2.2 TI2V 5B | [safetensors](https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/tree/main/split_files/diffusion_models)
[gguf](https://huggingface.co/QuantStack/Wan2.2-TI2V-5B-GGUF/tree/main) | `--diffusion-model` | Match the task and variant | +| Wan2.2 T2V A14B | [safetensors](https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/tree/main/split_files/diffusion_models)
[gguf](https://huggingface.co/QuantStack/Wan2.2-T2V-A14B-GGUF/tree/main) | `--diffusion-model` | Match the task and variant | +| Wan2.2 I2V A14B | [safetensors](https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/tree/main/split_files/diffusion_models)
[gguf](https://huggingface.co/QuantStack/Wan2.2-I2V-A14B-GGUF/tree/main) | `--diffusion-model` | Match the task and variant | +| Wan2.2 S2V 14B | [safetensors](https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/tree/main/split_files/diffusion_models)
[gguf](https://huggingface.co/QuantStack/Wan2.2-S2V-14B-GGUF/tree/main)
[int8_convrot safetensors](https://huggingface.co/noctrex/Wan2.2-S2V-14B-int8_convrot) | `--diffusion-model` | Match the task and variant | +| Wan2.2 VACE-Fun A14B | [safetensors](https://huggingface.co/alibaba-pai/Wan2.2-VACE-Fun-A14B)
[gguf](https://huggingface.co/QuantStack/Wan2.2-VACE-Fun-A14B-GGUF/tree/main) | `--diffusion-model` | Match the task and variant | +| wan_2.1_vae (for all the wan model except Wan2.2 TI2V 5B) | [safetensors](https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/blob/main/split_files/vae/wan_2.1_vae.safetensors) | `--vae` | Match the model family | +| wan_2.2_vae (for Wan2.2 TI2V 5B only) | [safetensors](https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/blob/main/split_files/vae/wan2.2_vae.safetensors) | `--vae` | Match the model family | +| umt5_xxl | [safetensors](https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/blob/main/split_files/text_encoders/umt5_xxl_fp16.safetensors)
[gguf](https://huggingface.co/city96/umt5-xxl-encoder-gguf/tree/main) | `--t5xxl` | Required for the indicated task | +| clip_vison_h (for Wan2.1 I2V/FLF2V only) | [safetensors](https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/blob/main/split_files/clip_vision/clip_vision_h.safetensors) | `--clip_vision` | Required for the indicated task | +| audio_encoder (for Wan2.2 S2V only) | [safetensors](https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/blob/main/split_files/audio_encoders/wav2vec2_large_english_fp16.safetensors) | `--audio-encoder` | Required for the indicated task | + +Use the Wan 2.2 VAE only for TI2V 5B; other variants use the matching Wan 2.1 +VAE shown above. Wan 2.2 A14B T2V/I2V use separate high-noise and low-noise +transformers; load both, using `--high-noise-diffusion-model` for the high-noise +file. For lower VAE memory use, see [tiny autoencoders](taesd.md). + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| Wan 2.1 T2V (`wan21-t2v`) | `diffusion_model`, `t5xxl`, `vae`, `tae` | +| Wan 2.1 I2V (`wan21-i2v`) | `diffusion_model`, `t5xxl`, `vae`, `tae`, `clip_vision` | +| Wan 2.1 FLF2V 14B (`wan21-flf2v`) | `diffusion_model`, `t5xxl`, `clip_vision`, `vae`, `tae` | +| Wan 2.1 VACE 1.3B / 14B (`wan21-vace`) | `diffusion_model`, `t5xxl`, `vae`, `tae` | +| Wan 2.2 T2V A14B (`wan22-t2v`) | `diffusion_model`, `high_noise_diffusion_model`, `t5xxl`, `vae`, `tae` | +| Wan 2.2 I2V A14B (`wan22-i2v`) | `diffusion_model`, `high_noise_diffusion_model`, `t5xxl`, `vae`, `tae` | +| Wan 2.2 S2V 14B (`wan22-s2v`) | `diffusion_model`, `t5xxl`, `audio_encoder`, `vae`, `tae` | +| Wan 2.2 TI2V 5B (`wan22-ti2v-5b`) | `diffusion_model`, `t5xxl`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### Wan2.1 T2V 1.3B + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\wan2.1_t2v_1.3B_fp16.safetensors ` + --vae ..\models\vae\wan_2.1_vae.safetensors ` + --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf ` + -p "a lovely cat" ` + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" ` + --cfg-scale 6.0 ` + --sampling-method euler ` + --flow-shift 3.0 ` + -W 832 ` + -H 480 ` + --video-frames 33 ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/wan2.1_t2v_1.3B_fp16.safetensors \ + --vae ../models/vae/wan_2.1_vae.safetensors \ + --t5xxl ../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf \ + -p "a lovely cat" \ + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" \ + --cfg-scale 6.0 \ + --sampling-method euler \ + --flow-shift 3.0 \ + -W 832 \ + -H 480 \ + --video-frames 33 \ + -v +``` + + + +#### Wan2.1 T2V 14B + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\wan2.1-t2v-14b-Q8_0.gguf ` + --vae ..\models\vae\wan_2.1_vae.safetensors ` + --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf ` + -p "a lovely cat" ` + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" ` + --cfg-scale 6.0 ` + --sampling-method euler ` + --flow-shift 3.0 ` + -W 832 ` + -H 480 ` + --video-frames 33 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/wan2.1-t2v-14b-Q8_0.gguf \ + --vae ../models/vae/wan_2.1_vae.safetensors \ + --t5xxl ../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf \ + -p "a lovely cat" \ + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" \ + --cfg-scale 6.0 \ + --sampling-method euler \ + --flow-shift 3.0 \ + -W 832 \ + -H 480 \ + --video-frames 33 \ + --offload-to-cpu \ + -v +``` + + + +#### Wan2.1 I2V 14B + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\wan2.1-i2v-14b-480p-Q8_0.gguf ` + --vae ..\models\vae\wan_2.1_vae.safetensors ` + --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf ` + --clip_vision ..\models\clip_vision\clip_vision_h.safetensors ` + -p "a lovely cat" ` + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" ` + -i ..\inputs\cat_with_sd_cpp_42.png ` + --cfg-scale 6.0 ` + --sampling-method euler ` + --flow-shift 3.0 ` + -W 480 ` + -H 832 ` + --video-frames 33 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/wan2.1-i2v-14b-480p-Q8_0.gguf \ + --vae ../models/vae/wan_2.1_vae.safetensors \ + --t5xxl ../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf \ + --clip_vision ../models/clip_vision/clip_vision_h.safetensors \ + -p "a lovely cat" \ + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" \ + -i ../inputs/cat_with_sd_cpp_42.png \ + --cfg-scale 6.0 \ + --sampling-method euler \ + --flow-shift 3.0 \ + -W 480 \ + -H 832 \ + --video-frames 33 \ + --offload-to-cpu \ + -v +``` + + + +#### Wan2.2 T2V A14B + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf ` + --high-noise-diffusion-model ..\models\diffusion_models\Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf ` + --vae ..\models\vae\wan_2.1_vae.safetensors ` + --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf ` + -p "a lovely cat" ` + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" ` + --cfg-scale 3.5 ` + --sampling-method euler ` + --steps 10 ` + --high-noise-cfg-scale 3.5 ` + --high-noise-sampling-method euler ` + --high-noise-steps 8 ` + --flow-shift 3.0 ` + -W 832 ` + -H 480 ` + --video-frames 33 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf \ + --high-noise-diffusion-model ../models/diffusion_models/Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf \ + --vae ../models/vae/wan_2.1_vae.safetensors \ + --t5xxl ../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf \ + -p "a lovely cat" \ + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" \ + --cfg-scale 3.5 \ + --sampling-method euler \ + --steps 10 \ + --high-noise-cfg-scale 3.5 \ + --high-noise-sampling-method euler \ + --high-noise-steps 8 \ + --flow-shift 3.0 \ + -W 832 \ + -H 480 \ + --video-frames 33 \ + --offload-to-cpu \ + -v +``` + + + +#### Wan2.2 I2V A14B + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\Wan2.2-I2V-A14B-LowNoise-Q8_0.gguf ` + --high-noise-diffusion-model ..\models\diffusion_models\Wan2.2-I2V-A14B-HighNoise-Q8_0.gguf ` + --vae ..\models\vae\wan_2.1_vae.safetensors ` + --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf ` + -p "a lovely cat" ` + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" ` + -i ..\inputs\cat_with_sd_cpp_42.png ` + --cfg-scale 3.5 ` + --sampling-method euler ` + --steps 10 ` + --high-noise-cfg-scale 3.5 ` + --high-noise-sampling-method euler ` + --high-noise-steps 8 ` + --flow-shift 3.0 ` + -W 832 ` + -H 480 ` + --video-frames 33 ` + --offload-to-cpu ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/Wan2.2-I2V-A14B-LowNoise-Q8_0.gguf \ + --high-noise-diffusion-model ../models/diffusion_models/Wan2.2-I2V-A14B-HighNoise-Q8_0.gguf \ + --vae ../models/vae/wan_2.1_vae.safetensors \ + --t5xxl ../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf \ + -p "a lovely cat" \ + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" \ + -i ../inputs/cat_with_sd_cpp_42.png \ + --cfg-scale 3.5 \ + --sampling-method euler \ + --steps 10 \ + --high-noise-cfg-scale 3.5 \ + --high-noise-sampling-method euler \ + --high-noise-steps 8 \ + --flow-shift 3.0 \ + -W 832 \ + -H 480 \ + --video-frames 33 \ + --offload-to-cpu \ + --offload-to-cpu \ + -v +``` + + + +#### Wan2.2 S2V 14B + +Audio-driven video (speech-to-video). The reference image (`-i`) is the speaker +portrait, `--audio` is the driving audio track and `--audio-encoder` is the +wav2vec2 audio encoder. Wan2.2 S2V requires the wan_2.1 vae (16 channel), not +the wan2.2 vae. + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\wan2.2_s2v-14B-Q8_0.gguf ` + --audio-encoder ..\models\text_encoders\audio_encoders\wav2vec2_large_english_fp16.safetensors ` + --vae ..\models\vae\wan_2.1_vae.safetensors ` + --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf ` + -p "a person is talking" ` + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" ` + -i ..\inputs\cat_with_sd_cpp_42.png ` + --audio .\input\speech.wav ` + --cfg-scale 6.0 ` + --steps 20 ` + --sampling-method euler ` + --flow-shift 3.0 ` + -W 832 ` + -H 480 ` + --video-frames 81 ` + --offload-to-cpu ` + --vae-tiling ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/wan2.2_s2v-14B-Q8_0.gguf \ + --audio-encoder ../models/text_encoders/audio_encoders/wav2vec2_large_english_fp16.safetensors \ + --vae ../models/vae/wan_2.1_vae.safetensors \ + --t5xxl ../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf \ + -p "a person is talking" \ + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" \ + -i ../inputs/cat_with_sd_cpp_42.png \ + --audio ./input/speech.wav \ + --cfg-scale 6.0 \ + --steps 20 \ + --sampling-method euler \ + --flow-shift 3.0 \ + -W 832 \ + -H 480 \ + --video-frames 81 \ + --offload-to-cpu \ + --vae-tiling \ + -v +``` + +Notes: + +- Recommended settings: `--sampling-method euler --steps 20 --cfg-scale 6.0`. + `dpm++2m` produces heavy artifacts on S2V. 4 steps with the lightning LoRA + (below) is the fast option. +- Resolutions: width and height must be multiples of 16; the examples use + multiples of 64. 832x480 is a fast starting point; generation cost scales + with pixel area. +- `--audio` accepts a WAV file; it is downmixed to mono and resampled to 16 kHz + internally. Audio longer than the video is truncated, video longer than the + audio is padded with silence. Pick `--video-frames` to match the audio: + roughly `audio_seconds * 16` frames, capped at one chunk (77-81 frames, + ~5 s at the model's 16 fps). 33, 77 and 81 map to clean latent frame counts. +- S2V always uses 16 fps. Other requested frame rates are automatically + changed to 16 with a warning, including the CLI and server video output. + `generate_video()` returns the actual frame rate through `fps_out`; C API + callers should use that value when encoding the output video. +- One generation covers the first S2V chunk window (`--video-frames` frames). + Long-video chunked extend mode is not implemented yet. +- Speed: the lightx2v lightning LoRA works with S2V at 4 steps and + `--cfg-scale 1.0`. Use the **low_noise** variant; + the high_noise variant produces artifacts on S2V: + +**Windows (PowerShell) arguments** + +```text +--lora-model-dir ..\models\loras ` + -p "..." ` + --cfg-scale 1.0 ` + --steps 4 +``` + +**Linux/macOS arguments** + +```text +--lora-model-dir ../models/loras \ + -p "..." \ + --cfg-scale 1.0 \ + --steps 4 +``` + + Expect some quality/dynamics loss compared to the full 20-step run. + +#### Wan2.2 T2V A14B T2I + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf ` + --high-noise-diffusion-model ..\models\diffusion_models\Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf ` + --vae ..\models\vae\wan_2.1_vae.safetensors ` + --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf ` + -p "a lovely cat" ` + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" ` + --cfg-scale 3.5 ` + --sampling-method euler ` + --steps 10 ` + --high-noise-cfg-scale 3.5 ` + --high-noise-sampling-method euler ` + --high-noise-steps 8 ` + --flow-shift 3.0 ` + -W 832 ` + -H 480 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf \ + --high-noise-diffusion-model ../models/diffusion_models/Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf \ + --vae ../models/vae/wan_2.1_vae.safetensors \ + --t5xxl ../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf \ + -p "a lovely cat" \ + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" \ + --cfg-scale 3.5 \ + --sampling-method euler \ + --steps 10 \ + --high-noise-cfg-scale 3.5 \ + --high-noise-sampling-method euler \ + --high-noise-steps 8 \ + --flow-shift 3.0 \ + -W 832 \ + -H 480 \ + --offload-to-cpu \ + -v +``` + +Wan2 2_14B_t2i + +#### Wan2.2 T2V 14B with Lora + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf ` + --high-noise-diffusion-model ..\models\diffusion_models\Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf ` + --vae ..\models\vae\wan_2.1_vae.safetensors ` + --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf ` + --lora-model-dir ..\models\loras ` + -p "a lovely cat" ` + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" ` + --cfg-scale 3.5 ` + --sampling-method euler ` + --steps 4 ` + --high-noise-cfg-scale 3.5 ` + --high-noise-sampling-method euler ` + --high-noise-steps 4 ` + --flow-shift 3.0 ` + -W 832 ` + -H 480 ` + --video-frames 33 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf \ + --high-noise-diffusion-model ../models/diffusion_models/Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf \ + --vae ../models/vae/wan_2.1_vae.safetensors \ + --t5xxl ../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf \ + --lora-model-dir ../models/loras \ + -p "a lovely cat" \ + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" \ + --cfg-scale 3.5 \ + --sampling-method euler \ + --steps 4 \ + --high-noise-cfg-scale 3.5 \ + --high-noise-sampling-method euler \ + --high-noise-steps 4 \ + --flow-shift 3.0 \ + -W 832 \ + -H 480 \ + --video-frames 33 \ + --offload-to-cpu \ + -v +``` + + + +#### Wan2.2 TI2V 5B + +##### T2V + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\wan2.2_ti2v_5B_fp16.safetensors ` + --vae ..\models\vae\wan2.2_vae.safetensors ` + --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf ` + -p "a lovely cat" ` + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" ` + --cfg-scale 6.0 ` + --sampling-method euler ` + --flow-shift 3.0 ` + -W 480 ` + -H 832 ` + --video-frames 33 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/wan2.2_ti2v_5B_fp16.safetensors \ + --vae ../models/vae/wan2.2_vae.safetensors \ + --t5xxl ../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf \ + -p "a lovely cat" \ + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" \ + --cfg-scale 6.0 \ + --sampling-method euler \ + --flow-shift 3.0 \ + -W 480 \ + -H 832 \ + --video-frames 33 \ + --offload-to-cpu \ + -v +``` + + + +##### I2V + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\wan2.2_ti2v_5B_fp16.safetensors ` + --vae ..\models\vae\wan2.2_vae.safetensors ` + --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf ` + -p "a lovely cat" ` + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" ` + -i ..\inputs\cat_with_sd_cpp_42.png ` + --cfg-scale 6.0 ` + --sampling-method euler ` + --flow-shift 3.0 ` + -W 480 ` + -H 832 ` + --video-frames 33 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/wan2.2_ti2v_5B_fp16.safetensors \ + --vae ../models/vae/wan2.2_vae.safetensors \ + --t5xxl ../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf \ + -p "a lovely cat" \ + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" \ + -i ../inputs/cat_with_sd_cpp_42.png \ + --cfg-scale 6.0 \ + --sampling-method euler \ + --flow-shift 3.0 \ + -W 480 \ + -H 832 \ + --video-frames 33 \ + --offload-to-cpu \ + -v +``` + + + +TI2V 5B can decode multiple latent frames together when temporal tiling is enabled. For example, `--temporal-tiling --vae-tiling --vae-tile-size 256x256 --extra-tiling-args "temporal_tile_frames=4,temporal_decode_frames=2"` keeps up to four latent frames in each temporal tile and decodes up to two at once, with spatial tiles of 256×256 pixels. `temporal_decode_frames` defaults to `1`, preserving frame-by-frame decoding; other model versions ignore it. Increasing it requires more memory. Adjust the spatial tile size to fit the available memory. + +#### Wan2.1 FLF2V 14B + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\wan2.1-flf2v-14b-720p-Q8_0.gguf ` + --vae ..\models\vae\wan_2.1_vae.safetensors ` + --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf ` + --clip_vision ..\models\clip_vision\clip_vision_h.safetensors ` + -p "glass flower blossom" ` + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" ` + --init-img ..\inputs\start_image.png ` + --end-img ..\inputs\end_image.png ` + --cfg-scale 6.0 ` + --sampling-method euler ` + --flow-shift 3.0 ` + -W 480 ` + -H 832 ` + --video-frames 33 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/wan2.1-flf2v-14b-720p-Q8_0.gguf \ + --vae ../models/vae/wan_2.1_vae.safetensors \ + --t5xxl ../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf \ + --clip_vision ../models/clip_vision/clip_vision_h.safetensors \ + -p "glass flower blossom" \ + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" \ + --init-img ../inputs/start_image.png \ + --end-img ../inputs/end_image.png \ + --cfg-scale 6.0 \ + --sampling-method euler \ + --flow-shift 3.0 \ + -W 480 \ + -H 832 \ + --video-frames 33 \ + --offload-to-cpu \ + -v +``` + + + +#### Wan2.2 FLF2V 14B + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\Wan2.2-I2V-A14B-LowNoise-Q8_0.gguf ` + --high-noise-diffusion-model ..\models\diffusion_models\Wan2.2-I2V-A14B-HighNoise-Q8_0.gguf ` + --vae ..\models\vae\wan_2.1_vae.safetensors ` + --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf ` + -p "glass flower blossom" ` + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" ` + --init-img ..\inputs\start_image.png ` + --end-img ..\inputs\end_image.png ` + --cfg-scale 3.5 ` + --sampling-method euler ` + --steps 10 ` + --high-noise-cfg-scale 3.5 ` + --high-noise-sampling-method euler ` + --high-noise-steps 8 ` + --flow-shift 3.0 ` + -W 480 ` + -H 832 ` + --video-frames 33 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/Wan2.2-I2V-A14B-LowNoise-Q8_0.gguf \ + --high-noise-diffusion-model ../models/diffusion_models/Wan2.2-I2V-A14B-HighNoise-Q8_0.gguf \ + --vae ../models/vae/wan_2.1_vae.safetensors \ + --t5xxl ../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf \ + -p "glass flower blossom" \ + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" \ + --init-img ../inputs/start_image.png \ + --end-img ../inputs/end_image.png \ + --cfg-scale 3.5 \ + --sampling-method euler \ + --steps 10 \ + --high-noise-cfg-scale 3.5 \ + --high-noise-sampling-method euler \ + --high-noise-steps 8 \ + --flow-shift 3.0 \ + -W 480 \ + -H 832 \ + --video-frames 33 \ + --offload-to-cpu \ + -v +``` + + + +#### Wan2.1 VACE 1.3B + +##### T2V + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\wan2.1-vace-1.3b-q8_0.gguf ` + --vae ..\models\vae\wan_2.1_vae.safetensors ` + --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf ` + -p "a lovely cat" ` + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" ` + --cfg-scale 6.0 ` + --sampling-method euler ` + -W 832 ` + -H 480 ` + --video-frames 1 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/wan2.1-vace-1.3b-q8_0.gguf \ + --vae ../models/vae/wan_2.1_vae.safetensors \ + --t5xxl ../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf \ + -p "a lovely cat" \ + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" \ + --cfg-scale 6.0 \ + --sampling-method euler \ + -W 832 \ + -H 480 \ + --video-frames 1 \ + --offload-to-cpu \ + -v +``` + + + +##### R2V + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\wan2.1-vace-1.3b-q8_0.gguf ` + --vae ..\models\vae\wan_2.1_vae.safetensors ` + --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf ` + -p "a lovely cat" ` + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" ` + -i ..\inputs\cat_with_sd_cpp_42.png ` + --cfg-scale 6.0 ` + --sampling-method euler ` + -W 832 ` + -H 480 ` + --video-frames 33 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/wan2.1-vace-1.3b-q8_0.gguf \ + --vae ../models/vae/wan_2.1_vae.safetensors \ + --t5xxl ../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf \ + -p "a lovely cat" \ + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" \ + -i ../inputs/cat_with_sd_cpp_42.png \ + --cfg-scale 6.0 \ + --sampling-method euler \ + -W 832 \ + -H 480 \ + --video-frames 33 \ + --offload-to-cpu \ + -v +``` + + + +##### V2V + +**Windows (PowerShell)** + +```powershell +mkdir post+depth +ffmpeg -i ....ComfyUIinputpost+depth.mp4 -qscale:v 1 -vf fps=8 post+depthframe_%04d.jpg +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\wan2.1-vace-1.3b-q8_0.gguf ` + --vae ..\models\vae\wan_2.1_vae.safetensors ` + --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf ` + -p "The girl is dancing in a sea of flowers, slowly moving her hands. There is a close - up shot of her upper body. The character is surrounded by other transparent glass flowers in the style of Nicoletta Ceccoli, creating a beautiful, surreal, and emotionally expressive movie scene with a white. transparent feel and a dreamyl atmosphere." ` + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" ` + -i ..\inputs\dance_girl.jpg ` + --cfg-scale 6.0 ` + --sampling-method euler ` + -W 480 ` + -H 832 ` + --video-frames 33 ` + --control-video ./post+depth ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +mkdir post+depth +ffmpeg -i ....ComfyUIinputpost+depth.mp4 -qscale:v 1 -vf fps=8 post+depthframe_%04d.jpg +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/wan2.1-vace-1.3b-q8_0.gguf \ + --vae ../models/vae/wan_2.1_vae.safetensors \ + --t5xxl ../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf \ + -p "The girl is dancing in a sea of flowers, slowly moving her hands. There is a close - up shot of her upper body. The character is surrounded by other transparent glass flowers in the style of Nicoletta Ceccoli, creating a beautiful, surreal, and emotionally expressive movie scene with a white. transparent feel and a dreamyl atmosphere." \ + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" \ + -i ../inputs/dance_girl.jpg \ + --cfg-scale 6.0 \ + --sampling-method euler \ + -W 480 \ + -H 832 \ + --video-frames 33 \ + --control-video ./post+depth \ + --offload-to-cpu \ + -v +``` + + + +#### Wan2.1 VACE 14B + +##### T2V + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\Wan2.1_14B_VACE-Q8_0.gguf ` + --vae ..\models\vae\wan_2.1_vae.safetensors ` + --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf ` + -p "a lovely cat" ` + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" ` + --cfg-scale 6.0 ` + --sampling-method euler ` + -W 832 ` + -H 480 ` + --video-frames 33 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/Wan2.1_14B_VACE-Q8_0.gguf \ + --vae ../models/vae/wan_2.1_vae.safetensors \ + --t5xxl ../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf \ + -p "a lovely cat" \ + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" \ + --cfg-scale 6.0 \ + --sampling-method euler \ + -W 832 \ + -H 480 \ + --video-frames 33 \ + --offload-to-cpu \ + -v +``` + + + +##### R2V + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\Wan2.1_14B_VACE-Q8_0.gguf ` + --vae ..\models\vae\wan_2.1_vae.safetensors ` + --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf ` + -p "a lovely cat" ` + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" ` + -i ..\inputs\cat_with_sd_cpp_42.png ` + --cfg-scale 6.0 ` + --sampling-method euler ` + -W 832 ` + -H 480 ` + --video-frames 33 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/Wan2.1_14B_VACE-Q8_0.gguf \ + --vae ../models/vae/wan_2.1_vae.safetensors \ + --t5xxl ../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf \ + -p "a lovely cat" \ + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" \ + -i ../inputs/cat_with_sd_cpp_42.png \ + --cfg-scale 6.0 \ + --sampling-method euler \ + -W 832 \ + -H 480 \ + --video-frames 33 \ + --offload-to-cpu \ + -v +``` + + + +##### V2V + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\Wan2.1_14B_VACE-Q8_0.gguf ` + --vae ..\models\vae\wan_2.1_vae.safetensors ` + --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf ` + -p "The girl is dancing in a sea of flowers, slowly moving her hands. There is a close - up shot of her upper body. The character is surrounded by other transparent glass flowers in the style of Nicoletta Ceccoli, creating a beautiful, surreal, and emotionally expressive movie scene with a white. transparent feel and a dreamyl atmosphere." ` + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" ` + -i ..\inputs\dance_girl.jpg ` + --cfg-scale 6.0 ` + --sampling-method euler ` + -W 480 ` + -H 832 ` + --video-frames 33 ` + --control-video ./post+depth ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/Wan2.1_14B_VACE-Q8_0.gguf \ + --vae ../models/vae/wan_2.1_vae.safetensors \ + --t5xxl ../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf \ + -p "The girl is dancing in a sea of flowers, slowly moving her hands. There is a close - up shot of her upper body. The character is surrounded by other transparent glass flowers in the style of Nicoletta Ceccoli, creating a beautiful, surreal, and emotionally expressive movie scene with a white. transparent feel and a dreamyl atmosphere." \ + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" \ + -i ../inputs/dance_girl.jpg \ + --cfg-scale 6.0 \ + --sampling-method euler \ + -W 480 \ + -H 832 \ + --video-frames 33 \ + --control-video ./post+depth \ + --offload-to-cpu \ + -v +``` + + + +#### Wan2.2 VACE-Fun A14B + +VACE-Fun runs as a MoE pair: `--diffusion-model` takes the low-noise expert and +`--high-noise-diffusion-model` the high-noise one. Reference-to-video uses `-i` +for the reference image, same as Wan2.1 VACE. + +##### R2V + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + -M vid_gen ` + --diffusion-model ..\models\diffusion_models\Wan2.2-VACE-Fun-A14B-low-noise-Q8_0.gguf ` + --high-noise-diffusion-model ..\models\diffusion_models\Wan2.2-VACE-Fun-A14B-high-noise-Q8_0.gguf ` + --vae ..\models\vae\wan_2.1_vae.safetensors ` + --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf ` + -p "a lovely cat" ` + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" ` + -i ..\inputs\cat_with_sd_cpp_42.png ` + --cfg-scale 3.5 ` + --sampling-method euler ` + --steps 10 ` + --high-noise-cfg-scale 3.5 ` + --high-noise-sampling-method euler ` + --high-noise-steps 8 ` + -W 832 ` + -H 480 ` + --video-frames 33 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + -M vid_gen \ + --diffusion-model ../models/diffusion_models/Wan2.2-VACE-Fun-A14B-low-noise-Q8_0.gguf \ + --high-noise-diffusion-model ../models/diffusion_models/Wan2.2-VACE-Fun-A14B-high-noise-Q8_0.gguf \ + --vae ../models/vae/wan_2.1_vae.safetensors \ + --t5xxl ../models/text_encoders/umt5-xxl-encoder-Q8_0.gguf \ + -p "a lovely cat" \ + -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" \ + -i ../inputs/cat_with_sd_cpp_42.png \ + --cfg-scale 3.5 \ + --sampling-method euler \ + --steps 10 \ + --high-noise-cfg-scale 3.5 \ + --high-noise-sampling-method euler \ + --high-noise-steps 8 \ + -W 832 \ + -H 480 \ + --video-frames 33 \ + --offload-to-cpu \ + -v +``` + + + +##### T2V + +Same command without `-i` (VACE context is synthesized from an empty control +video, like Wan2.1 VACE t2v). + +> On GPUs with ~12 GB VRAM, VACE also needs `--vae-tiling` — the control-video +> encode can exceed the budget otherwise. + +## Notes and limitations + +Wan models require `-M vid_gen`, including single-frame generation. `--video-frames` alone does not select video mode. Library callers must use `generate_video()` instead of `generate_image()`. + +Use the task-specific input and sampling settings in each example. VAE +encoding and decoding can require substantial memory; see +[performance and memory usage](../guides/performance.md). diff --git a/docs/models/z_image.md b/docs/models/z_image.md new file mode 100644 index 0000000000..9b39e3d67d --- /dev/null +++ b/docs/models/z_image.md @@ -0,0 +1,112 @@ +# Z-Image + +Z-Image and Z-Image-Turbo generate images from text using Qwen3 4B and the FLUX VAE. + +You can run Z-Image with stable-diffusion.cpp on GPUs with 4GB of VRAM — or even less. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| Z-Image-Turbo | [Safetensors](https://huggingface.co/Comfy-Org/z_image_turbo/tree/main/split_files/diffusion_models)
[GGUF](https://huggingface.co/leejet/Z-Image-Turbo-GGUF/tree/main) | `--diffusion-model` | Choose matching variant | +| Z-Image | [Safetensors](https://huggingface.co/Comfy-Org/z_image/tree/main/split_files/diffusion_models)
[GGUF](https://huggingface.co/unsloth/Z-Image-GGUF/tree/main) | `--diffusion-model` | Choose matching variant | +| vae | [Safetensors](https://huggingface.co/black-forest-labs/FLUX.1-schnell/tree/main) | `--vae` | Use matching VAE | +| Qwen3 4b | [Safetensors](https://huggingface.co/Comfy-Org/z_image_turbo/tree/main/split_files/text_encoders)
[GGUF](https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/tree/main) | `--llm` | Use matching text encoder | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| Z-Image (`z-image`) | `diffusion_model`, `llm`, `vae`, `tae` | +| Z-Image Turbo (`z-image-turbo`) | `diffusion_model`, `llm`, `vae`, `tae` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### Z-Image-Turbo + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\z_image_turbo-Q3_K.gguf ` + --vae ..\models\vae\ae.sft ` + --llm ..\models\text_encoders\Qwen3-4B-Instruct-2507-Q4_K_M.gguf ` + -p "A cinematic, melancholic photograph of a solitary hooded figure walking through a sprawling, rain-slicked metropolis at night. The city lights are a chaotic blur of neon orange and cool blue, reflecting on the wet asphalt. The scene evokes a sense of being a single component in a vast machine. Superimposed over the image in a sleek, modern, slightly glitched font is the philosophical quote: 'THE CITY IS A CIRCUIT BOARD, AND I AM A BROKEN TRANSISTOR.' -- moody, atmospheric, profound, dark academic" ` + --cfg-scale 1.0 ` + --steps 8 ` + -H 1024 ` + -W 512 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/z_image_turbo-Q3_K.gguf \ + --vae ../models/vae/ae.sft \ + --llm ../models/text_encoders/Qwen3-4B-Instruct-2507-Q4_K_M.gguf \ + -p "A cinematic, melancholic photograph of a solitary hooded figure walking through a sprawling, rain-slicked metropolis at night. The city lights are a chaotic blur of neon orange and cool blue, reflecting on the wet asphalt. The scene evokes a sense of being a single component in a vast machine. Superimposed over the image in a sleek, modern, slightly glitched font is the philosophical quote: 'THE CITY IS A CIRCUIT BOARD, AND I AM A BROKEN TRANSISTOR.' -- moody, atmospheric, profound, dark academic" \ + --cfg-scale 1.0 \ + --steps 8 \ + -H 1024 \ + -W 512 \ + --offload-to-cpu \ + -v +``` + +z-image example + +#### Z-Image-Base + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\z_image_bf16.safetensors ` + --vae ..\models\vae\ae.sft ` + --llm ..\models\text_encoders\qwen_3_4b.safetensors ` + -p "A cinematic, melancholic photograph of a solitary hooded figure walking through a sprawling, rain-slicked metropolis at night. The city lights are a chaotic blur of neon orange and cool blue, reflecting on the wet asphalt. The scene evokes a sense of being a single component in a vast machine. Superimposed over the image in a sleek, modern, slightly glitched font is the philosophical quote: 'THE CITY IS A CIRCUIT BOARD, AND I AM A BROKEN TRANSISTOR.' -- moody, atmospheric, profound, dark academic" ` + --cfg-scale 5.0 ` + -H 1024 ` + -W 512 ` + --offload-to-cpu ` + -v +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/z_image_bf16.safetensors \ + --vae ../models/vae/ae.sft \ + --llm ../models/text_encoders/qwen_3_4b.safetensors \ + -p "A cinematic, melancholic photograph of a solitary hooded figure walking through a sprawling, rain-slicked metropolis at night. The city lights are a chaotic blur of neon orange and cool blue, reflecting on the wet asphalt. The scene evokes a sense of being a single component in a vast machine. Superimposed over the image in a sleek, modern, slightly glitched font is the philosophical quote: 'THE CITY IS A CIRCUIT BOARD, AND I AM A BROKEN TRANSISTOR.' -- moody, atmospheric, profound, dark academic" \ + --cfg-scale 5.0 \ + -H 1024 \ + -W 512 \ + --offload-to-cpu \ + -v +``` + +z-image example + +#### Comparison of Different Quantization Types + +| bf16 | q8_0 | q6_K | q5_0 | q4_K | q4_0 | q3_K | q2_K| +|---|---|---|---|---|---|---|---| +| bf16 | q8_0 | q6_K | q5_0 | q4_K | q4_0 | q3_K | q2_K | diff --git a/docs/models/z_image_l2p.md b/docs/models/z_image_l2p.md new file mode 100644 index 0000000000..c572af8ca2 --- /dev/null +++ b/docs/models/z_image_l2p.md @@ -0,0 +1,72 @@ +# Z-Image L2P + +[L2P](https://nju-pcalab.github.io/projects/L2P/) (Latent-to-Pixel) generates images +directly in pixel space using Z-Image-Turbo and Qwen3 4B. No VAE is required. + +## Model files + +| Component | Weights and downloads | CLI option | Requirement | +| --- | --- | --- | --- | +| L2P (1K) | [Safetensors](https://huggingface.co/zhen-nan/L2P/tree/main) | `--diffusion-model` | Required | +| Qwen3 4b | [Safetensors](https://huggingface.co/Comfy-Org/z_image_turbo/tree/main/split_files/text_encoders)
[GGUF](https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/tree/main) | `--llm` | Use matching text encoder | + +## Usage + +CLI examples use the [shared path and shell conventions](README.md#example-conventions). + +### Web UI + +Follow the [Web UI guide](../../examples/server/README.md#web-ui) to start the +server, then select the recipe matching your weights. Fill the component slots +below and click **Load models**. +Use the files and compatibility requirements in [Model files](#model-files). + +| Recipe | Component slots | +| --- | --- | +| Z-Image L2P (`z-image-l2p`) | `diffusion_model`, `llm` | + +Slot IDs correspond to CLI component options: `diffusion_model` uses +`--diffusion-model`, `llm` uses `--llm`, and `model` uses `-m` / `--model`. +Select optional components when required by your checkpoint or task. + +### CLI + +#### Text-to-image + +Do not pass `--vae`. Image dimensions must be multiples of 16. The reference pipeline uses 30 steps with a CFG scale of 2.0. + +The released checkpoint keeps part of the transformer in F32. Pass `--type bf16` to load it in BF16, as the reference pipeline runs it; this reduces the diffusion model from about 18.6 GB to 11.8 GB. + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe ` + --diffusion-model ..\models\diffusion_models\model-1k-merge.safetensors ` + --llm ..\models\text_encoders\qwen_3_4b.safetensors ` + --type bf16 ` + -p "an origami pig on fire in the middle of a dark room with a pentagram on the floor" ` + --cfg-scale 2.0 ` + --steps 30 ` + --sampling-method euler ` + -W 1024 ` + -H 1024 ` + -v ` + -o z_image_l2p.png +``` + +**Linux/macOS** + +```sh +./bin/sd-cli \ + --diffusion-model ../models/diffusion_models/model-1k-merge.safetensors \ + --llm ../models/text_encoders/qwen_3_4b.safetensors \ + --type bf16 \ + -p "an origami pig on fire in the middle of a dark room with a pentagram on the floor" \ + --cfg-scale 2.0 \ + --steps 30 \ + --sampling-method euler \ + -W 1024 \ + -H 1024 \ + -v \ + -o z_image_l2p.png +``` diff --git a/docs/moss_image.md b/docs/moss_image.md deleted file mode 100644 index 27ab26f804..0000000000 --- a/docs/moss_image.md +++ /dev/null @@ -1,37 +0,0 @@ -# How to Use - -MOSS-Image uses a 5B diffusion transformer, the FLUX.2 VAE, and Qwen3.5 9B as the LLM text encoder. Both Base and SFT are supported. Prompt enhancement is not included. - -## Download weights - -- Download MOSS-Image Base or SFT - - safetensors: https://huggingface.co/Comfy-Org/MOSS-Image/tree/main/diffusion_models - - gguf: - - Base: https://huggingface.co/leejet/MOSS-Image-Base-GGUF/tree/main - - SFT: https://huggingface.co/leejet/MOSS-Image-SFT-GGUF/tree/main -- Download vae - - safetensors: https://huggingface.co/Comfy-Org/MOSS-Image/tree/main/vae -- Download Qwen3.5 9B - - gguf: https://huggingface.co/unsloth/Qwen3.5-9B-GGUF/tree/main -- Download `tokenizer.json` - - https://huggingface.co/OpenMOSS-Team/MOSS-Image-Base/tree/main/tokenizer - -Pass the external `tokenizer.json` with `--tokenizer`; Qwen3.5 uses a different vocabulary from the embedded Qwen tokenizer. - -## Examples - -### MOSS-Image Base - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\moss_image_base_int8_convrot_attn.safetensors --llm ..\models\text_encoders\Qwen3.5-9B-Q4_K_M.gguf --vae ..\models\vae\flux2_ae.safetensors --tokenizer ..\models\tokenizers\tokenizer_qwen3.5.json --cfg-scale 3.0 -p "a lovely cat holding a sign says 'moss-image.cpp'" -v --offload-to-cpu -H 1024 -W 1024 -``` - -### MOSS-Image SFT - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\moss_image_sft_int8_convrot_attn.safetensors --llm ..\models\text_encoders\Qwen3.5-9B-Q4_K_M.gguf --vae ..\models\vae\flux2_ae.safetensors --tokenizer ..\models\tokenizers\tokenizer_qwen3.5.json --cfg-scale 3.0 -p "a lovely cat holding a sign says 'moss-image.cpp'" -v --offload-to-cpu -H 1024 -W 1024 -``` - - - -Use dimensions divisible by 16. The default discrete scheduler uses the upstream time shift of 5. Use `--flow-shift` to override it. The negative CFG branch skips DiT layer 22 (zero-based), matching the upstream pipeline. diff --git a/docs/ovis_image.md b/docs/ovis_image.md deleted file mode 100644 index c561ff54b0..0000000000 --- a/docs/ovis_image.md +++ /dev/null @@ -1,19 +0,0 @@ -# How to Use - -## Download weights - -- Download Ovis-Image-7B - - safetensors: https://huggingface.co/Comfy-Org/Ovis-Image/tree/main/split_files/diffusion_models - - gguf: https://huggingface.co/leejet/Ovis-Image-7B-GGUF -- Download vae - - safetensors: https://huggingface.co/black-forest-labs/FLUX.1-schnell/tree/main -- Download Ovis 2.5 - - safetensors: https://huggingface.co/Comfy-Org/Ovis-Image/tree/main/split_files/text_encoders - -## Examples - -``` -.\bin\Release\sd-cli.exe --diffusion-model ovis_image-Q4_0.gguf --vae ..\models\vae\ae.sft --llm ..\models\text_encoders\ovis_2.5.safetensors -p "a lovely cat" --cfg-scale 5.0 -v --offload-to-cpu -``` - -ovis image example \ No newline at end of file diff --git a/docs/p_h3_image.md b/docs/p_h3_image.md deleted file mode 100644 index 20de6fb202..0000000000 --- a/docs/p_h3_image.md +++ /dev/null @@ -1,34 +0,0 @@ -# How to Use - -P-H3-Image supports text-to-image and reference-image editing. It uses a MiniMax-H3 diffusion transformer, a single-frame VAE, and Qwen3-VL-4B with a text connector. Prompt enhancement is not included. - -## Download weights - -- Download P-H3-Image - - safetensors: https://huggingface.co/PrunaAI/P-H3-Image-Comfy/tree/main/diffusion_models -- Download P-H3-Image VAE - - safetensors: https://huggingface.co/PrunaAI/P-H3-Image-Comfy/tree/main/vae -- Download Qwen3-VL-4B with the text connector - - https://huggingface.co/PrunaAI/P-H3-Image-Comfy/tree/main/text_encoders - -Alternatively, reuse Qwen3-VL-4B, including its vision tower, with the [separate text connector](https://huggingface.co/PrunaAI/P-H3-Image/blob/main/text_connector/diffusion_pytorch_model.safetensors). Save the connector as `p-h3-image_text_connector.safetensors` and pass it with `--embeddings-connectors`. - -Use the P-H3 image VAE; the MiniMax-H3 video VAE is not compatible. No audio VAE is needed. - -## Examples - -### Text-to-image - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\p-h3-image_int8_convrot.safetensors --llm ..\models\text_encoders\qwen3vl_4b_bf16.safetensors --embeddings-connectors ..\models\text_encoders\p-h3-image_text_connector.safetensors --vae ..\models\vae\p-h3-image_vae_bf16.safetensors --cfg-scale 1.0 --steps 8 --sampling-method euler --sigmas p-h3-image -p "a lovely cat holding a sign says 'p-h3-image.cpp'" -v --offload-to-cpu --rng cpu -H 2912 -W 2912 -``` - -### Image editing - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\p-h3-image_int8_convrot.safetensors --llm ..\models\text_encoders\qwen3vl_4b_bf16.safetensors --embeddings-connectors ..\models\text_encoders\p-h3-image_text_connector.safetensors --vae ..\models\vae\p-h3-image_vae_bf16.safetensors --cfg-scale 1.0 --steps 8 --sampling-method euler --sigmas p-h3-image -r source.png -r reference.png -p "Edit . Give the cat the scarf from , preserving its face and the background." -v --offload-to-cpu --rng cpu -H 2912 -W 2912 -``` - -Reference images are numbered in argument order as ``, ``, and so on. A plain edit instruction gets `Edit . ` prepended automatically. Prompts starting with `Edit ` or `subject_definitions:` are used directly. - -Use dimensions divisible by 32. The upstream default is 2912 × 2912; smaller canvases reduce memory use. Use CFG 1 and `--sigmas p-h3-image` for the upstream 8-step schedule. Keep the default VAE tiling size of 256 pixels. diff --git a/docs/photo_maker.md b/docs/photo_maker.md deleted file mode 100644 index 1a3463ef4b..0000000000 --- a/docs/photo_maker.md +++ /dev/null @@ -1,53 +0,0 @@ -## Using PhotoMaker to personalize image generation - -You can use [PhotoMaker](https://github.com/TencentARC/PhotoMaker) to personalize generated images with your own ID. - -**NOTE**, currently PhotoMaker **ONLY** works with **SDXL** (any SDXL model files will work). - -Download PhotoMaker model file (in safetensor format) [here](https://huggingface.co/bssrdf/PhotoMaker). The official release of the model file (in .bin format) does not work with ```stablediffusion.cpp```. - -- Specify the PhotoMaker model path using the `--photo-maker PATH` parameter. -- Specify the input images path using the `--pm-id-images-dir PATH` parameter. - -In prompt, make sure you have a class word followed by the trigger word ```"img"``` (hard-coded for now). The class word could be one of ```"man, woman, girl, boy"```. If input ID images contain asian faces, add ```Asian``` before the class -word. - -Another PhotoMaker specific parameter: - -- ```--pm-style-strength (0-100)%```: default is 20 and 10-20 typically gets good results. Lower ratio means more faithfully following input ID (not necessarily better quality). - -Other parameters recommended for running Photomaker: - -- ```--cfg-scale 5.0``` -- ```-H 1024``` -- ```-W 1024``` - -If on low memory GPUs (<= 8GB), recommend running with ```--vae-on-cpu``` option to get artifact free images. - -Example: - -```bash -bin/sd-cli -m ../models/sdxlUnstableDiffusers_v11.safetensors --vae ../models/sdxl_vae.safetensors --photo-maker ../models/photomaker-v1.safetensors --pm-id-images-dir ../assets/photomaker_examples/scarletthead_woman -p "a girl img, retro futurism, retro game art style but extremely beautiful, intricate details, masterpiece, best quality, space-themed, cosmic, celestial, stars, galaxies, nebulas, planets, science fiction, highly detailed" -n "realistic, photo-realistic, worst quality, greyscale, bad anatomy, bad hands, error, text" --cfg-scale 5.0 --sampling-method euler -H 1024 -W 1024 --pm-style-strength 10 --vae-on-cpu --steps 50 -``` - -## PhotoMaker Version 2 - -[PhotoMaker Version 2 (PMV2)](https://github.com/TencentARC/PhotoMaker/blob/main/README_pmv2.md) has some key improvements. Unfortunately it has a very heavy dependency which makes running it a bit involved in ```SD.cpp```. - -Running PMV2 is now a two-step process: - -- Run a python script ```face_detect.py``` to obtain **id_embeds** for the given input images -``` -python face_detect.py input_image_dir -``` -An ```id_embeds.bin``` file will be generated in ```input_images_dir``` - -**Note: this step is only needed to run once; the same ```id_embeds``` can be reused** - -- Run the same command as in version 1 but replacing ```photomaker-v1.safetensors``` with ```photomaker-v2.safetensors```. - - You can download ```photomaker-v2.safetensors``` from [here](https://huggingface.co/bssrdf/PhotoMakerV2) - -- All the command line parameters from Version 1 remain the same for Version 2 plus one extra pointing to a valid ```id_embeds``` file: --pm-id-embed-path [path_to__id_embeds.bin] - - diff --git a/docs/pid.md b/docs/pid.md deleted file mode 100644 index 3737f32ffe..0000000000 --- a/docs/pid.md +++ /dev/null @@ -1,44 +0,0 @@ -# How to Use - -PiD is NVIDIA's Pixel Diffusion Decoder. It replaces the usual VAE decode or decode-then-upscale path with a pixel-space diffusion decoder conditioned on a -source latent and text prompt. Both the original PiD checkpoints and PiD 1.5 are supported. - -In stable-diffusion.cpp, PiD currently runs as an image edit pipeline: provide a reference image with `-r`/`--ref-image`, encode that image with a matching VAE, then let the PiD diffusion model decode/upscale directly to RGB. - -## Download weights - -- Download PiD - - safetensors: https://huggingface.co/Comfy-Org/PixelDiT/tree/main/diffusion_models -- Download Gemma 2 2B - - safetensors: https://huggingface.co/Comfy-Org/PixelDiT/tree/main/text_encoders -- Download Gemma 2 2B tokenizer.json - - https://huggingface.co/google/gemma-2-2b/tree/main -- Download the VAE that matches the PiD checkpoint backbone - - safetensors: https://huggingface.co/nvidia/PiD/tree/main/checkpoints - - Flux / Z-Image PiD: use the Flux VAE and pass `--vae-format flux` - - SD3 PiD: use the SD3 VAE and pass `--vae-format sd3` - - Flux.2 PiD: use the Flux.2 VAE and pass `--vae-format flux2` - - Qwen-Image PiD: use the Qwen-Image 2D VAE and pass `--vae-format wan` - -The official PiD model card should be checked before use. At the time of the initial PiD release, the official weights are under the NSCLv1 non-commercial license. - -PiD and PiD 1.5 require an external Gemma 2 `tokenizer.json` matching the text encoder checkpoint. Save it as `tokenizer_gemma2.json` and pass it with `--tokenizer`; the tokenizer is not embedded in sd.cpp. See [JSON tokenizers](tokenizers.md) for CLI and C API usage. - -## Examples - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\pid_flux1_512_to_2048_4step_bf16.safetensors --llm "..\models\text_encoders\gemma_2_2b_it_elm_bf16.safetensors" --tokenizer ..\models\tokenizers\tokenizer_gemma2.json --vae ..\models\vae\ae.sft --vae-format flux --cfg-scale 1.0 -p "a lovely cat" -r ..\assets\ernie_image\turbo_example.png -v --steps 4 -H 2048 -W 2048 --rng cpu -``` - -Before: - -ERNIE-Image Turbo example - -After: -PiD example - -## Notes - -- `-r`/`--ref-image` is required. PiD uses the first reference image as the source latent condition. -- `--vae-format` should match the VAE latent layout used by the PiD checkpoint. This is important when using standalone VAE files because the PiD diffusion - checkpoint alone does not identify the VAE format. diff --git a/docs/pixart.md b/docs/pixart.md deleted file mode 100644 index f9386abf16..0000000000 --- a/docs/pixart.md +++ /dev/null @@ -1,46 +0,0 @@ -# How to Use - -You can run PixArt-α / PixArt-Σ with stable-diffusion.cpp. - -PixArt is a DiT-based text-to-image model family conditioned by a T5-XXL text -encoder and a 4-channel VAE: SDXL-style for PixArt-Σ and SD1.x-style for PixArt-α. - -## Download weights - -- Download the transformer (diffusion model) - - PixArt-Σ XL-2 1024-MS: https://huggingface.co/PixArt-alpha/PixArt-Sigma-XL-2-1024-MS/tree/main/transformer - - PixArt-α XL-2 1024-MS: https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS/tree/main/transformer -- Download the T5-XXL text encoder - - safetensors: https://huggingface.co/PixArt-alpha/PixArt-Sigma-XL-2-1024-MS/tree/main/text_encoder -- Download the VAE - - PixArt-Σ: https://huggingface.co/PixArt-alpha/PixArt-Sigma-XL-2-1024-MS/tree/main/vae - - PixArt-α: https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS/tree/main/vae - - Use the VAE matching the checkpoint's latent space. For TAE decoding or - preview, use TAESDXL for PixArt-Σ and TAESD for PixArt-α. -- Tokenizer: the T5 vocabulary is embedded; no extra tokenizer file is needed. - -## Examples - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\pixart_sigma_xl2_1024_ms.safetensors --t5xxl ..\models\text_encoders\t5xxl.safetensors --vae ..\models\vae\pixart_vae.safetensors -p "a lovely cat" --cfg-scale 4.5 -W 1024 -H 1024 --steps 20 -v -``` - -## Notes - -- The VAE scaling factor defaults to `0.13025` for PixArt-Σ. PixArt-α - checkpoints with resolution micro-condition weights use `0.18215`. - PixArt-α 512 has the same tensor layout as PixArt-Σ, so it requires an - explicit override: `--model-args "pixart_vae_scale_factor=0.18215"`. - This argument can also override the scale for other compatible checkpoints. -- PixArt-Σ checkpoints compute 2D sincos positional embeddings at runtime; - the trained grid is 64x64 patches with an interpolation scale of 2. - For checkpoints trained at a different resolution, the positional embedding - parameters can be adjusted via model args: - `--model-args "pixart_pos_embed_base_size=,pixart_interpolation_scale="` - (e.g. `pixart_pos_embed_base_size=32,pixart_interpolation_scale=1,pixart_vae_scale_factor=0.18215` for - PixArt-α XL-2 512). -- Checkpoints carrying resolution/aspect-ratio micro-condition weights are - detected but those conditions are not applied yet; a warning is logged and - generation proceeds with the timestep embedding only. -- The transformer predicts 8 channels (noise + learned variance); only the - noise half is used for sampling, matching the reference implementation. diff --git a/docs/pulid.md b/docs/pulid.md deleted file mode 100644 index 429c7fc71e..0000000000 --- a/docs/pulid.md +++ /dev/null @@ -1,196 +0,0 @@ -# PuLID-Flux face-identity preservation - -stable-diffusion.cpp supports the [PuLID-Flux](https://github.com/ToTheBeginning/PuLID) -identity-injection technique on top of Flux.1 (schnell or dev) models. -Given a single source portrait, PuLID-Flux produces new generations that -preserve the source person's face across arbitrary scenes, poses, and -prompts. - -Unlike PhotoMaker (which extracts the identity inside the inference -process from a directory of images), PuLID-Flux's identity extractor is -a heavy stack (insightface ArcFace + EVA-CLIP-L + IDFormer encoder) that -is impractical to port to C++/ggml. To keep this implementation small and -cross-vendor, **stable-diffusion.cpp consumes a precomputed identity -embedding** produced by an external Python tool that runs once per source -portrait. Everything downstream of that one-shot extraction is C++ and -runs on any backend (Vulkan, CUDA, Metal, ROCm, CPU). - -## Architecture summary - -The PuLID-Flux contribution to the Flux denoise loop is a stack of 20 -small cross-attention modules (`PerceiverAttentionCA`) inserted between -the Flux transformer blocks: - -- After every 2nd of the 19 double-stream blocks (10 hook points) -- After every 4th of the 38 single-stream blocks (10 hook points) - -Each cross-attention layer takes the current image tokens as query, the -32-token / 2048-dim identity embedding as key+value, and adds its output -(scaled by `id_weight`, typically 1.0) back to the image tokens. - -## Required weights - -Three files in addition to the standard Flux weight set: - -1. **Flux base** (transformer + VAE + clip_l + t5xxl) -- exactly as - [docs/flux.md](flux.md) describes. -2. **PuLID weights** -- download from - [guozinan/PuLID](https://huggingface.co/guozinan/PuLID): - - `pulid_flux_v0.9.0.safetensors` or `pulid_flux_v0.9.1.safetensors` - (recommended; this implementation is verified against v0.9.1) - - **v1.1 (`pulid_v1.1.safetensors`) is NOT yet supported** -- it uses - renamed keys (`id_adapter_attn_layers.*` instead of `pulid_ca.*`) - and possibly different module structure. Future PR. -3. **Identity embedding (.pulidembd)** -- produced by the precompute - tool below. - -## Precompute the identity embedding - -The precompute tool runs the PyTorch identity-extraction stack on a -single portrait image and writes the resulting `(32, 2048)` embedding -to a `.pulidembd` binary file (about 131 KB). Run it once per source -person; the same file is reused for any number of generations. - -A reference Python script is provided alongside this docs file at -[`scripts/pulid_extract_id.py`](../scripts/pulid_extract_id.py). It -requires: -- A working CUDA / CPU PyTorch stack -- `insightface`, `facexlib`, `eva-clip`, `torchvision`, `opencv-python`, - `huggingface_hub`, `gguf` -- The PuLID weights file (same one stable-diffusion.cpp will load below) -- The ToTheBeginning/PuLID repo's `pulid/` package (including - `pulid/pipeline_flux.py`) and `eva_clip/` package on `PYTHONPATH`; `flux/` - is not needed for embedding extraction - -Run it as: - -``` -python pulid_extract_id.py \ - --portrait /path/to/source-photo.jpg \ - --pulid-weights /path/to/pulid_flux_v0.9.1.safetensors \ - --out /path/to/source.pulidembd -``` - -## Format (gguf) - -The embedding is a standard **gguf** container holding a single tensor: - -``` -tensor name : "pulid_id" -shape : [token_dim, num_tokens] (ggml order; typically [2048, 32]) -type : F16 (also accepts F32 / BF16) -metadata : general.architecture = "pulid", pulid.version = 1 -``` - -stable-diffusion.cpp loads it with the normal gguf reader -(`gguf_init_from_file`) and converts to fp32 at load time -- no bespoke -parser. Total file size for the typical (32, 2048, fp16) case is ~131 KB. - -## Command-line usage - -``` -.\bin\Release\sd-cli.exe \ - --diffusion-model models\flux1-schnell-Q4_K_S.gguf \ - --vae models\ae.safetensors \ - --clip_l models\clip_l.safetensors \ - --t5xxl models\t5xxl_fp16.safetensors \ - --pulid-weights models\pulid_flux_v0.9.1.safetensors \ - --pulid-id-embedding source.pulidembd \ - --pulid-id-weight 1.0 \ - -p "candid photograph of a young woman on a beach at sunset" \ - --cfg-scale 1.0 --sampling-method euler --steps 4 -W 512 -H 512 \ - --seed 42 --clip-on-cpu \ - -o out.png -``` - -For Flux Dev (instead of Schnell), add `--guidance 3.5` and `--steps 20`. - -## Flags - -| Flag | Purpose | -|----------------------------|-------------------------------------------------------------------| -| `--pulid-weights ` | Path to `pulid_flux_v0.9.x.safetensors`. Loaded with the model. | -| `--pulid-id-embedding

` | Path to a `.pulidembd` binary produced by the precompute tool. | -| `--pulid-id-weight ` | Identity-injection strength. Typical 0.7-1.2; default 1.0. | - -All three flags must be set together to activate PuLID. Setting only -`--pulid-weights` (no embedding) loads the weights but disables injection -at runtime. Setting `--pulid-id-weight 0` zeros out the contribution -(useful for falsification testing: outputs should be byte-identical to -a no-PuLID run with the same seed). - -## Memory budget - -At 512x512, 4 steps (Schnell), the 20 cross-attention layers add roughly -10% to denoise time and almost nothing to peak VRAM. Tested on a 12 GB -consumer card alongside Flux Schnell Q4 GGUF + CPU-offloaded clip_l and -t5xxl + GPU-resident VAE. - -At 1024x1024 with Flux Dev Q4 + 20 steps + PuLID, the VAE decode compute -buffer doesn't fit on a 12 GB card even with `--vae-on-cpu`. Workaround: -explicitly route VAE to the CPU backend instead of the offload flag: - -``` ---backend "diffusion=vulkan0,vae=cpu" -``` - -The `--vae-on-cpu` flag offloads VAE weights but leaves the compute graph -on the default backend; this is existing stable-diffusion.cpp behavior, -not a PuLID-specific issue. Documented here because anyone running PuLID -at 1024 will hit it. - -## Backend selection - -The standard `--backend` flag works as documented. Common patterns: - -``` -# AMD Vulkan ---backend "diffusion=vulkan0,vae=cpu" - -# NVIDIA Vulkan ---backend "diffusion=vulkan1,vae=cpu" - -# CUDA ---backend "diffusion=cuda0,vae=cpu" -``` - -The PuLID cross-attention layers run on the same backend as the main -diffusion model. They have not yet been independently profiled on every -backend; only Vulkan and CPU have been tested by the original contributor. - -## Verification - -A three-way SHA-256 check is the recommended sanity test when bringing up -a new combination of model + backend + hardware: - -| Run | Expected hash relation | -|----------------------------------------------|------------------------------------| -| A: no `--pulid-*` flags | baseline | -| B: PuLID flags, `--pulid-id-weight 0.0` | **byte-identical to A** | -| C: PuLID flags, `--pulid-id-weight 1.0` | **different from A,B**, preserves source identity | - -If A and C differ but A and B differ too, the injection is allocating -or computing something even at zero weight -- likely a bug. - -## Limitations / not yet supported - -- **`--skip-layers` (skip-layer-guidance / SLG) combined with PuLID** is not - supported. The `pulid_ca` index advances per non-skipped block, so a - skipped block silently misaligns the cross-attention weight assignment - vs. the trained intervals. The reference PyTorch implementation does - not have SLG either, so there is no well-defined behavior to emulate. - Use either feature alone. -- **PuLID v1.1 weights** (`pulid_v1.1.safetensors`, renamed key layout). -- **Multiple ID images.** The reference PyTorch implementation can fuse - several portraits into one embedding for stronger identity. This - implementation accepts a single embedding produced from one or more - images by the external precompute tool. -- **Negative-prompt branch of CFG.** PuLID only injects on the positive - conditioning path in the published reference, and the implementation - here follows that. Flux's distilled guidance doesn't run a separate - uncond branch in normal use, so this matters only for `--true-cfg` - workflows that aren't standard for Flux. -- **Backends other than Vulkan and CPU** are untested by the original - contributor. The implementation is pure-ggml and should work on CUDA, - ROCm, and Metal, but verification by users on those backends is - welcomed. diff --git a/docs/qwen_image.md b/docs/qwen_image.md deleted file mode 100644 index 00ee50e24b..0000000000 --- a/docs/qwen_image.md +++ /dev/null @@ -1,23 +0,0 @@ -# How to Use - -## Download weights - -- Download Qwen Image - - safetensors: https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/tree/main/split_files/diffusion_models - - gguf: https://huggingface.co/QuantStack/Qwen-Image-GGUF/tree/main -- Download vae - - safetensors: https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/tree/main/split_files/vae -- Download qwen_2.5_vl 7b - - safetensors: https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/tree/main/split_files/text_encoders - - gguf: https://huggingface.co/mradermacher/Qwen2.5-VL-7B-Instruct-GGUF/tree/main - -## Examples - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\qwen-image-Q8_0.gguf --vae ..\models\vae\qwen_image_vae.safetensors --llm ..\models\text_encoders\Qwen2.5-VL-7B-Instruct-Q8_0.gguf -p '一个穿着"QWEN"标志的T恤的中国美女正拿着黑色的马克笔面相镜头微笑。她身后的玻璃板上手写体写着 “一、Qwen-Image的技术路线: 探索视觉生成基础模型的极限,开创理解与生成一体化的未来。二、Qwen-Image的模型特色:1、复杂文字渲染。支持中英渲染、自动布局; 2、精准图像编辑。支持文字编辑、物体增减、风格变换。三、Qwen-Image的未来愿景:赋能专业内容创作、助力生成式AI发展。”' --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu -H 1024 -W 1024 --flow-shift 3 -``` - -qwen example - - - diff --git a/docs/qwen_image_2.1.md b/docs/qwen_image_2.1.md deleted file mode 100644 index 8663731465..0000000000 --- a/docs/qwen_image_2.1.md +++ /dev/null @@ -1,99 +0,0 @@ -# How to Use - -Qwen Image 2.1 supports text-to-image generation and image editing, using Qwen3-VL-8B as the text encoder and its own VAE. - -## Download weights - -- Download Qwen Image 2.1 - - safetensors: https://huggingface.co/Comfy-Org/Qwen-Image-2.1/tree/main/diffusion_models - - gguf: https://huggingface.co/leejet/Qwen-Image-2.1-GGUF/tree/main -- Download vae - - safetensors: https://huggingface.co/Comfy-Org/Qwen-Image-2.1/tree/main/vae -- Download Qwen3-VL-8B-Instruct - - safetensors (BF16 or INT8 convrot): https://huggingface.co/Comfy-Org/Qwen-Image-2.1/tree/main/text_encoders - - gguf: https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct-GGUF/tree/main - - For image editing with a GGUF text encoder, also download `mmproj-Qwen3VL-8B-Instruct-F16.gguf` from the same repository and pass it with `--llm_vision`. - -Use `qwen_image_2.1_vae_bf16.safetensors` with this model. The earlier Qwen Image and Wan 2.2 VAE weights are not interchangeable with the Qwen Image 2.1 VAE weights. - -For faster encoding, decoding, or previews, see [TAEQI2.1](taesd.md#qwen-image-21-taeqi21). - -## Examples - -Run the following commands from the build directory. Use image dimensions divisible by 32. The resolution-dependent flow schedule is selected automatically. - -### Text to image - -```powershell -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\qwen_image_2.1_int8_convrot.safetensors --vae ..\models\vae\qwen_image_2.1_vae_bf16.safetensors --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf -p "a lovely cat holding a sign says 'qwen2.1.cpp'" --cfg-scale 6.0 --sampling-method euler -v --offload-to-cpu -o qwen_image_2.1.png -``` - -Qwen Image 2.1 example - -To use GGUF diffusion weights, set `--diffusion-model` to the path of a file such as `qwen_image_2.1-Q4_K.gguf`. - -### Turbo - -For [Qwen Image 2.1 Turbo](https://huggingface.co/Qwen/Qwen-Image-2.1-Turbo), use the Turbo diffusion weights and select its fixed 8-step sigma schedule with `--sigmas qwen-image-2.1-turbo`. For example, add these sampling options to the command above: - -```text ---sigmas qwen-image-2.1-turbo --steps 8 --cfg-scale 1.0 --sampling-method euler -``` - -The preset is equivalent to `--sigmas 1.0,0.978453,0.95418,0.926626,0.89508,0.845148,0.704534,0.414568,0.0`. It only supplies the sigma values; CFG scale and sampling method remain independently configurable. As with an explicit sigma list, the sigma count determines the actual number of sampling steps. The same preset name is also accepted by `--hires-sigmas` for the highres fix second pass. - -### Image editing - -Pass the reference image with `-r` and describe the edit in `-p`. Vision weights are required; the example below loads them separately with `--llm_vision`. - -```powershell -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\qwen_image_2.1_int8_convrot.safetensors --vae ..\models\vae\qwen_image_2.1_vae_bf16.safetensors --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf --llm_vision ..\models\text_encoders\Qwen3VL-8B-Instruct-mmproj-BF16.gguf -r ..\assets\qwen\qwen_image_2.1.png -p "change 'qwen2.1.cpp' to 'sd.cpp'" --cfg-scale 6.0 --sampling-method euler -v --offload-to-cpu -o qwen_image_2.1_edit.png -``` - -For multiple reference images, repeat `-r` in the desired order, for example `-r first.png -r second.png`. - -### Prefix cache - -By default, the first denoising call for each fixed condition saves the text and reference-image keys and values from every transformer layer. Later calls only compute the target-image tokens. Positive and negative conditions use separate caches, which are released when sampling ends. - -Set `qwen_image_2_1_prefix_cache_type` in `--model-args` to `auto` or a type name using the same parser and case-sensitive names as `--type`: - -- `auto` (default): use FP16 only when Flash Attention is enabled, Sage Attention is disabled, the attention scale is unchanged, and every attention operation in the cache-writing or cache-reading graph selects Flash Attention after backend support checks. If an operation falls back, rebuild the prefix in FP32 before executing and keep FP32 for the rest of that sampling run. -- `f32`: always store FP32 keys and values. -- `f16`: always store FP16 keys and values, including with ordinary attention or custom attention scaling. This saves cache memory but can introduce additional rounding error. -- Other types, such as `bf16`, `q4_1`, `q5_0`, `q5_1`, `q8_0`, `q4_K`, `q6_K`, `iq4_nl`, and `iq4_xs`: use the requested storage type if the ggml build provides runtime conversion to and from FP32. Quantization is lossy and must be selected explicitly; `auto` never selects a quantized type. - -Cache data is packed into contiguous rows of `hidden_size` elements before conversion, so 256-element quantization blocks work with the model's 128-element attention heads without padding. The type's block size must divide `hidden_size`. Unknown types, types lacking runtime conversion (for example `q8_1` and several IQ formats), and incompatible block sizes are ignored with a warning, leaving the previous setting or the default `auto` unchanged. - -For example, use `--model-args qwen_image_2_1_prefix_cache_type=q8_0` to enable 8-bit cache storage. Cached keys and values are converted back to the attention input type before concatenating with the current target tokens. This reduces persistent cache memory; attention working buffers still use floating-point values, and conversion adds work on each step. Backends without the required conversion operations use the existing CPU fallback. - -For the default 32-layer model, a prefix of 4096 tokens takes approximately the following memory per condition, excluding weights, working buffers, and allocation overhead: - -| Cache type | Memory | -| --- | ---: | -| `f32` | 4 GiB | -| `f16` | 2 GiB | -| `q8_0` | 1.0625 GiB | -| `q4_0` | 0.5625 GiB | - -The runner accounts for the cache when checking the memory budget. If a cached execution runs out of memory, it releases the prefix caches, disables caching for the rest of that sampling run, and retries the full sequence once. Per-step conditioning extensions currently use the full-sequence path. - -Disable this optimization with `--model-args qwen_image_2_1_prefix_cache=false`. It reuses step-independent activations; numerical results can still differ slightly because the matrix sizes change. - -### Alpha channel - -This model supports alpha channel output. As the model determines whether to output a regular image or with transparency through the prompt, according to [official recommendation](https://github.com/QwenLM/Qwen-Image-2.1#transparent-image-generation-rgba), use the following prompt format for better results: - -> `This is an RGBA image with transparency. . The image has alpha channel and the background is transparent.` - -Since transparency is decided by the prompt rather than by the input or an explicit switch, the same format applies equally to editing, whether or not the reference image itself has an alpha channel. Note that alpha is kept only in `.png` and `.webp` outputs; saving as `.jpg` drops the transparency. - -Here are some examples ran with Q6_K quantization: -| Input | Prompt | Output | -| --- | --- | --- | -| ![Qwen Image 2.1 alpha input example 1](../assets/qwen/qwen-image-2.1-alpha-in1.png) | This is an RGBA image with transparency. Replace the text "BLOOM" with "Qwen Image 2.1", keeping the same font of the original text. The image has alpha channel and the background is transparent. | ![Qwen Image 2.1 alpha output example 1](../assets/qwen/qwen-image-2.1-alpha-out1.png) | -| ![Qwen Image 2.1 alpha input example 2](../assets/logo.png) | This is an RGBA image with transparency. Remove the background of the image, keeping only the text and cat. The image has alpha channel and the background is transparent. | ![Qwen Image 2.1 alpha output example 2](../assets/qwen/qwen-image-2.1-alpha-out2.png) | - -### Other features - -Other features of the model could be found on the [model card from QwenLM/Qwen-Image-2.1 repo](https://github.com/QwenLM/Qwen-Image-2.1), including 2 finetuned prompt rewriting Qwen3.5-9B model. diff --git a/docs/qwen_image_edit.md b/docs/qwen_image_edit.md deleted file mode 100644 index 2d53d83354..0000000000 --- a/docs/qwen_image_edit.md +++ /dev/null @@ -1,48 +0,0 @@ -# How to Use - -## Download weights - -- Download Qwen Image - - Qwen Image Edit - - safetensors: https://huggingface.co/Comfy-Org/Qwen-Image-Edit_ComfyUI/tree/main/split_files/diffusion_models - - gguf: https://huggingface.co/QuantStack/Qwen-Image-Edit-GGUF/tree/main - - Qwen Image Edit 2509 - - safetensors: https://huggingface.co/Comfy-Org/Qwen-Image-Edit_ComfyUI/tree/main/split_files/diffusion_models - - gguf: https://huggingface.co/QuantStack/Qwen-Image-Edit-2509-GGUF/tree/main - - Qwen Image Edit 2511 - - safetensors: https://huggingface.co/Comfy-Org/Qwen-Image-Edit_ComfyUI/tree/main/split_files/diffusion_models - - gguf: https://huggingface.co/unsloth/Qwen-Image-Edit-2511-GGUF/tree/main -- Download vae - - safetensors: https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/tree/main/split_files/vae -- Download qwen_2.5_vl 7b - - safetensors: https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/tree/main/split_files/text_encoders - - gguf: https://huggingface.co/mradermacher/Qwen2.5-VL-7B-Instruct-GGUF/tree/main - -## Examples - -### Qwen Image Edit - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\Qwen_Image_Edit-Q8_0.gguf --vae ..\models\vae\qwen_image_vae.safetensors --llm ..\models\text_encoders\qwen_2.5_vl_7b.safetensors --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu --flow-shift 3 -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'edit.cpp'" --seed 1118877715456453 -``` - -qwen_image_edit - - -### Qwen Image Edit 2509 - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\Qwen-Image-Edit-2509-Q4_K_S.gguf --vae ..\models\vae\qwen_image_vae.safetensors --llm ..\models\text_encoders\Qwen2.5-VL-7B-Instruct-Q8_0.gguf --llm_vision ..\models\text_encoders\Qwen2.5-VL-7B-Instruct.mmproj-Q8_0.gguf --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu --flow-shift 3 -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'Qwen Image Edit 2509'" -``` - -qwen_image_edit_2509 - -### Qwen Image Edit 2511 - -To use the new Qwen Image Edit 2511 mode, `--model-args qwen_image_zero_cond_t=true` must be enabled; otherwise, image editing quality will degrade significantly. - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\qwen-image-edit-2511-Q4_K_M.gguf --vae ..\models\vae\qwen_image_vae.safetensors --llm ..\models\text_encoders\qwen_2.5_vl_7b.safetensors --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu --flow-shift 3 -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'edit.cpp'" --model-args qwen_image_zero_cond_t=true -``` - -qwen_image_edit_2509 diff --git a/docs/sd.md b/docs/sd.md deleted file mode 100644 index 44a4c1f99d..0000000000 --- a/docs/sd.md +++ /dev/null @@ -1,37 +0,0 @@ -## Download weights - -- download original weights(.ckpt or .safetensors). For example - - Stable Diffusion v1.4 from https://huggingface.co/CompVis/stable-diffusion-v-1-4-original - - Stable Diffusion v1.5 from https://huggingface.co/stable-diffusion-v1-5/stable-diffusion-v1-5 - - Stable Diffusion v2.1 from https://huggingface.co/Manojb/stable-diffusion-2-1-base - - Stable Diffusion 3 2B from https://huggingface.co/stabilityai/stable-diffusion-3-medium - -### txt2img example - -```sh -./bin/sd-cli -m ../models/sd-v1-4.ckpt -p "a lovely cat" -# ./bin/sd-cli -m ../models/v1-5-pruned-emaonly.safetensors -p "a lovely cat" -# ./bin/sd-cli -m ../models/sd_xl_base_1.0.safetensors --vae ../models/sdxl_vae-fp16-fix.safetensors -H 1024 -W 1024 -p "a lovely cat" -v -# ./bin/sd-cli -m ../models/sd3_medium_incl_clips_t5xxlfp16.safetensors -H 1024 -W 1024 -p 'a lovely cat holding a sign says \"Stable Diffusion CPP\"' --cfg-scale 4.5 --sampling-method euler -v --clip-on-cpu -# ./bin/sd-cli --diffusion-model ../models/flux1-dev-q3_k.gguf --vae ../models/ae.sft --clip_l ../models/clip_l.safetensors --t5xxl ../models/t5xxl_fp16.safetensors -p "a lovely cat holding a sign says 'flux.cpp'" --cfg-scale 1.0 --sampling-method euler -v --clip-on-cpu -# ./bin/sd-cli -m ..\models\sd3.5_large.safetensors --clip_l ..\models\clip_l.safetensors --clip_g ..\models\clip_g.safetensors --t5xxl ..\models\t5xxl_fp16.safetensors -H 1024 -W 1024 -p 'a lovely cat holding a sign says \"Stable diffusion 3.5 Large\"' --cfg-scale 4.5 --sampling-method euler -v --clip-on-cpu -``` - -Using formats of different precisions will yield results of varying quality. - -| f32 | f16 |q8_0 |q5_0 |q5_1 |q4_0 |q4_1 | -| ---- |---- |---- |---- |---- |---- |---- | -| ![](../assets/f32.png) |![](../assets/f16.png) |![](../assets/q8_0.png) |![](../assets/q5_0.png) |![](../assets/q5_1.png) |![](../assets/q4_0.png) |![](../assets/q4_1.png) | - -### img2img example - -- `./output.png` is the image generated from the above txt2img pipeline - - -``` -./bin/sd-cli -m ../models/sd-v1-4.ckpt -p "cat with blue eyes" -i ./output.png -o ./img2img_output.png --strength 0.4 -``` - -

- -

diff --git a/docs/sd3.md b/docs/sd3.md deleted file mode 100644 index 3893659f9f..0000000000 --- a/docs/sd3.md +++ /dev/null @@ -1,20 +0,0 @@ -# How to Use - -## Download weights - -- Download sd3.5_large from https://huggingface.co/stabilityai/stable-diffusion-3.5-large/blob/main/sd3.5_large.safetensors -- Download clip_g from https://huggingface.co/Comfy-Org/stable-diffusion-3.5-fp8/blob/main/text_encoders/clip_g.safetensors -- Download clip_l from https://huggingface.co/Comfy-Org/stable-diffusion-3.5-fp8/blob/main/text_encoders/clip_l.safetensors -- Download t5xxl from https://huggingface.co/Comfy-Org/stable-diffusion-3.5-fp8/blob/main/text_encoders/t5xxl_fp16.safetensors - - -## Run - -### SD3.5 Large -For example: - -``` -.\bin\Release\sd-cli.exe -m ..\models\sd3.5_large.safetensors --clip_l ..\models\clip_l.safetensors --clip_g ..\models\clip_g.safetensors --t5xxl ..\models\t5xxl_fp16.safetensors -H 1024 -W 1024 -p 'a lovely cat holding a sign says \"Stable diffusion 3.5 Large\"' --cfg-scale 4.5 --sampling-method euler -v --clip-on-cpu -``` - -![](../assets/sd3.5_large.png) \ No newline at end of file diff --git a/docs/sefi_image.md b/docs/sefi_image.md deleted file mode 100644 index 9989e4f8bd..0000000000 --- a/docs/sefi_image.md +++ /dev/null @@ -1,50 +0,0 @@ -# How to Use - -SeFi-Image uses a Flux2-style dual-time transformer (semantic + texture streams), the standard Flux2 VAE, and Qwen3-VL as the LLM text encoder. Tech report: [arXiv:2606.22568](https://arxiv.org/abs/2606.22568). - -## Download weights - -The SeFi-Image family ships in three scales (1B / 2B / 5B) and three families (Base / RL / turbo), all gated on Hugging Face under https://huggingface.co/SeFi-Image. - -- 1B and 2B variants pair with Qwen3-VL-2B-Instruct. -- 5B variants pair with Qwen3-VL-4B-Instruct. -- All variants use the standard Flux2 VAE (`flux2_ae.safetensors` from https://huggingface.co/black-forest-labs/FLUX.2-dev). - -Convert the transformer and text encoder to sd.cpp safetensors: - -```bash -python3 scripts/convert_sefi.py /sefi__.safetensors -python3 scripts/convert_qwen3_vl.py /Qwen3-VL-XB-Instruct /qwen3_vl_b.safetensors -``` - -## Variant defaults - -| Family | timestep_shift_alpha | steps | cfg-scale | -|---|---|---|---| -| Base | 0.3 | 50 | 4.0 | -| RL | 0.3 | 50 | 4.0 | -| turbo | 1.0 | 4 | 1.0 | - -The dispatcher picks `alpha` from the filename (`turbo` substring => 1.0, otherwise 0.3). Override via `--extra-sample-args sefi_alpha=` or `sefi_delta_t=`. - -## Examples - -### 1B / 2B turbo - -``` -./build/bin/sd-cli --diffusion-model /path/to/sefi_1b_turbo.safetensors --vae /path/to/flux2_ae.safetensors --llm /path/to/qwen3_vl_2b.safetensors -p "a photograph of an orange tabby cat sitting on a couch" --cfg-scale 1.0 --steps 4 -W 1024 -H 1024 -s 42 --offload-to-cpu -o out.png -``` - -### 1B / 2B base - -``` -./build/bin/sd-cli --diffusion-model /path/to/sefi_1b_base.safetensors --vae /path/to/flux2_ae.safetensors --llm /path/to/qwen3_vl_2b.safetensors -p "a photograph of an orange tabby cat sitting on a couch" --cfg-scale 4.0 --steps 50 -W 1024 -H 1024 -s 42 --offload-to-cpu -o out.png -``` - -### 5B (needs streaming on 12 GiB VRAM) - -``` -./build/bin/sd-cli --diffusion-model /path/to/sefi_5b_turbo.safetensors --vae /path/to/flux2_ae.safetensors --llm /path/to/qwen3_vl_4b.safetensors -p "a photograph of an orange tabby cat sitting on a couch" --cfg-scale 1.0 --steps 4 -W 1024 -H 1024 -s 42 --max-vram 8 --offload-to-cpu -o out.png -``` - -SeFi-Image 5B turbo example diff --git a/docs/taesd.md b/docs/taesd.md deleted file mode 100644 index 8331f441bf..0000000000 --- a/docs/taesd.md +++ /dev/null @@ -1,53 +0,0 @@ -## Using TAESD to faster decoding - -You can use TAESD to accelerate the decoding of latent images by following these steps: - -- Download the model [weights](https://huggingface.co/madebyollin/taesd/blob/main/diffusion_pytorch_model.safetensors). - -Or curl - -```bash -curl -L -O https://huggingface.co/madebyollin/taesd/resolve/main/diffusion_pytorch_model.safetensors -``` - -- Specify the model path using the `--taesd PATH` parameter. example: - -```bash -sd-cli -m ../models/v1-5-pruned-emaonly.safetensors -p "a lovely cat" --taesd ../models/diffusion_pytorch_model.safetensors -``` - -### Qwen-Image and wan (TAEHV) - -sd.cpp also supports [TAEHV](https://github.com/madebyollin/taehv) (#937), which can be used for Qwen-Image and wan. - -- For **Qwen-Image and wan2.1 and wan2.2-A14B**, download the wan2.1 tae [safetensors weights](https://github.com/madebyollin/taehv/blob/main/safetensors/taew2_1.safetensors) - - Or curl - - ```bash - curl -L -O https://github.com/madebyollin/taehv/raw/refs/heads/main/safetensors/taew2_1.safetensors - ``` - -- For **wan2.2-TI2V-5B**, use the wan2.2 tae [safetensors weights](https://github.com/madebyollin/taehv/blob/main/safetensors/taew2_2.safetensors) - - Or curl - - ```bash - curl -L -O https://github.com/madebyollin/taehv/raw/refs/heads/main/safetensors/taew2_2.safetensors - ``` - -Then simply replace the `--vae xxx.safetensors` with `--tae xxx.safetensors` in the commands. If it still out of VRAM, add `--vae-conv-direct` to your command though might be slower. - -### Qwen Image 2.1 (TAEQI2.1) - -For Qwen Image 2.1, use [taeqi2_1](https://github.com/madebyollin/taesd), which supports 64-channel latents, 16x spatial scaling, and RGBA images. - -Download the official [safetensors weights](https://huggingface.co/madebyollin/taeqi2_1/blob/main/taeqi2_1.safetensors) directly; no conversion is needed: - -```bash -curl -L -o taeqi2_1.safetensors https://huggingface.co/madebyollin/taeqi2_1/resolve/main/taeqi2_1.safetensors -``` - -Replace `--vae PATH` with `--taesd taeqi2_1.safetensors` in the [Qwen Image 2.1 examples](qwen_image_2.1.md) to use it for encoding and decoding. For previews only, keep `--vae PATH` and add `--taesd taeqi2_1.safetensors --taesd-preview-only --preview tae`. - -`sd-server` accepts the same `--taesd` and `--taesd-preview-only` options; the preview method is selected per request instead of via `--preview`. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 27731e1cce..c667cd4b89 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -45,7 +45,7 @@ the intended output scale: The two values can be set independently and apply across model components. The default `0` preserves each model's built-in settings; `1` explicitly disables the corresponding scaling. Overrides must be finite positive values. C API users can -set `linear_scale` and `attn_scale` in `sd_ctx_params_t`. +set the `linear_scale` and `attn_scale` context-option keys with `sd_opt_set_float()`. If the problem persists after trying the relevant steps above, [submit a bug report](https://github.com/leejet/stable-diffusion.cpp/issues/new?template=bug_report.yml). diff --git a/docs/wan.md b/docs/wan.md deleted file mode 100644 index 2031de7df0..0000000000 --- a/docs/wan.md +++ /dev/null @@ -1,285 +0,0 @@ -# How to Use - -Wan models require `-M vid_gen`, including single-frame generation. `--video-frames` alone does not select video mode. Library callers must use `generate_video()` instead of `generate_image()`. - -## Download weights - -- Download Wan - - Wan2.1 - - Wan2.1 T2V 1.3B - - safetensors: https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/tree/main/split_files/diffusion_models - - Wan2.1 T2V 14B - - safetensors: https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/tree/main/split_files/diffusion_models - - gguf: https://huggingface.co/city96/Wan2.1-T2V-14B-gguf/tree/main - - Wan2.1 I2V 14B 480P - - safetensors: https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/tree/main/split_files/diffusion_models - - gguf: https://huggingface.co/city96/Wan2.1-I2V-14B-480P-gguf/tree/main - - Wan2.1 I2V 14B 720P - - safetensors: https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/tree/main/split_files/diffusion_models - - gguf: https://huggingface.co/city96/Wan2.1-I2V-14B-720P-gguf/tree/main - - Wan2.1 FLF2V 14B 720P - - safetensors: https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/tree/main/split_files/diffusion_models - - gguf: https://huggingface.co/city96/Wan2.1-FLF2V-14B-720P-gguf/tree/main - - Wan2.1 VACE 1.3B - - safetensors: https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/tree/main/split_files/diffusion_models - - gguf: https://huggingface.co/calcuis/wan-1.3b-gguf/tree/main - - Wan2.1 VACE 14B - - safetensors: https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/tree/main/split_files/diffusion_models - - gguf: https://huggingface.co/QuantStack/Wan2.1_14B_VACE-GGUF/tree/main - - Wan2.2 - - Wan2.2 TI2V 5B - - safetensors: https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/tree/main/split_files/diffusion_models - - gguf: https://huggingface.co/QuantStack/Wan2.2-TI2V-5B-GGUF/tree/main - - Wan2.2 T2V A14B - - safetensors: https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/tree/main/split_files/diffusion_models - - gguf: https://huggingface.co/QuantStack/Wan2.2-T2V-A14B-GGUF/tree/main - - Wan2.2 I2V A14B - - safetensors: https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/tree/main/split_files/diffusion_models - - gguf: https://huggingface.co/QuantStack/Wan2.2-I2V-A14B-GGUF/tree/main - - Wan2.2 S2V 14B - - safetensors: https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/tree/main/split_files/diffusion_models - - gguf: https://huggingface.co/QuantStack/Wan2.2-S2V-14B-GGUF/tree/main - - int8_convrot safetensors: https://huggingface.co/noctrex/Wan2.2-S2V-14B-int8_convrot - - Wan2.2 VACE-Fun A14B - - safetensors: https://huggingface.co/alibaba-pai/Wan2.2-VACE-Fun-A14B - - gguf: https://huggingface.co/QuantStack/Wan2.2-VACE-Fun-A14B-GGUF/tree/main -- Download vae - - wan_2.1_vae (for all the wan model except Wan2.2 TI2V 5B) - - safetensors: https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/blob/main/split_files/vae/wan_2.1_vae.safetensors - - wan_2.2_vae (for Wan2.2 TI2V 5B only) - - safetensors: https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/blob/main/split_files/vae/wan2.2_vae.safetensors - - > Wan models vae requires really much VRAM! If you do not have enough VRAM, please try tae instead, though the results may be poorer. For tae usage, please refer to [taesd](taesd.md) - -- Download umt5_xxl - - safetensors: https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/blob/main/split_files/text_encoders/umt5_xxl_fp16.safetensors - - gguf: https://huggingface.co/city96/umt5-xxl-encoder-gguf/tree/main - -- Download clip_vison_h (for Wan2.1 I2V/FLF2V only) - - safetensors: https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/blob/main/split_files/clip_vision/clip_vision_h.safetensors - -- Download audio_encoder (for Wan2.2 S2V only) - - safetensors: https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/blob/main/split_files/audio_encoders/wav2vec2_large_english_fp16.safetensors - - -## Examples - -### Wan2.1 T2V 1.3B - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\wan2.1_t2v_1.3B_fp16.safetensors --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" -W 832 -H 480 --video-frames 33 --flow-shift 3.0 -``` - - - -### Wan2.1 T2V 14B - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\wan2.1-t2v-14b-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" -W 832 -H 480 --offload-to-cpu --video-frames 33 --flow-shift 3.0 -``` - - - - - -### Wan2.1 I2V 14B - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\wan2.1-i2v-14b-480p-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf --clip_vision ..\models\clip_vision\clip_vision_h.safetensors -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" -W 480 -H 832 --video-frames 33 --offload-to-cpu -i ..\assets\cat_with_sd_cpp_42.png --flow-shift 3.0 -``` - - - -### Wan2.2 T2V A14B - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model ..\models\diffusion_models\Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf -p "a lovely cat" --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" -W 832 -H 480 --offload-to-cpu --video-frames 33 --flow-shift 3.0 -``` - - - -### Wan2.2 I2V A14B - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\Wan2.2-I2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model ..\models\diffusion_models\Wan2.2-I2V-A14B-HighNoise-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf -p "a lovely cat" --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" -W 832 -H 480 --offload-to-cpu --video-frames 33 --offload-to-cpu -i ..\assets\cat_with_sd_cpp_42.png --flow-shift 3.0 -``` - - - -### Wan2.2 S2V 14B - -Audio-driven video (speech-to-video). The reference image (`-i`) is the speaker -portrait, `--audio` is the driving audio track and `--audio-encoder` is the -wav2vec2 audio encoder. Wan2.2 S2V requires the wan_2.1 vae (16 channel), not -the wan2.2 vae. - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\wan2.2_s2v-14B-Q8_0.gguf --audio-encoder ..\models\audio_encoders\wav2vec2_large_english_fp16.safetensors --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf -p "a person is talking" --cfg-scale 6.0 --steps 20 --sampling-method euler -v -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" -W 832 -H 480 --offload-to-cpu --vae-tiling --video-frames 81 -i ..\assets\cat_with_sd_cpp_42.png --audio .\input\speech.wav --flow-shift 3.0 -``` - -Notes: - -- Recommended settings: `--sampling-method euler --steps 20 --cfg-scale 6.0`. - `dpm++2m` produces heavy artifacts on S2V. 4 steps with the lightning LoRA - (below) is the fast option. -- Resolutions: width and height must be multiples of 16; the examples use - multiples of 64. 832x480 is a fast starting point; generation cost scales - with pixel area. -- `--audio` accepts a WAV file; it is downmixed to mono and resampled to 16 kHz - internally. Audio longer than the video is truncated, video longer than the - audio is padded with silence. Pick `--video-frames` to match the audio: - roughly `audio_seconds * 16` frames, capped at one chunk (77-81 frames, - ~5 s at the model's 16 fps). 33, 77 and 81 map to clean latent frame counts. -- S2V always uses 16 fps. Other requested frame rates are automatically - changed to 16 with a warning, including the CLI and server video output. - `generate_video()` returns the actual frame rate through `fps_out`; C API - callers should use that value when encoding the output video. -- One generation covers the first S2V chunk window (`--video-frames` frames). - Long-video chunked extend mode is not implemented yet. -- Speed: the lightx2v lightning LoRA works with S2V at 4 steps and - `--cfg-scale 1.0`. Use the **low_noise** variant; - the high_noise variant produces artifacts on S2V: - - ``` - --lora-model-dir ..\models\loras - -p "..." - --cfg-scale 1.0 --steps 4 - ``` - - Expect some quality/dynamics loss compared to the full 20-step run. - -### Wan2.2 T2V A14B T2I - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model ..\models\diffusion_models\Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf -p "a lovely cat" --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" -W 832 -H 480 --offload-to-cpu --flow-shift 3.0 -``` - -Wan2 2_14B_t2i - -### Wan2.2 T2V 14B with Lora - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model ..\models\diffusion_models\Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf -p "a lovely cat" --cfg-scale 3.5 --sampling-method euler --steps 4 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 4 -v -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" -W 832 -H 480 --offload-to-cpu --lora-model-dir ..\models\loras --video-frames 33 --flow-shift 3.0 -``` - - - - - -### Wan2.2 TI2V 5B - -#### T2V - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\wan2.2_ti2v_5B_fp16.safetensors --vae ..\models\vae\wan2.2_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" -W 480 -H 832 --offload-to-cpu --video-frames 33 --flow-shift 3.0 -``` - - - -#### I2V - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\wan2.2_ti2v_5B_fp16.safetensors --vae ..\models\vae\wan2.2_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" -W 480 -H 832 --offload-to-cpu --video-frames 33 -i ..\assets\cat_with_sd_cpp_42.png --flow-shift 3.0 -``` - - - -TI2V 5B can decode multiple latent frames together when temporal tiling is enabled. For example, `--temporal-tiling --vae-tiling --vae-tile-size 256x256 --extra-tiling-args "temporal_tile_frames=4,temporal_decode_frames=2"` keeps up to four latent frames in each temporal tile and decodes up to two at once, with spatial tiles of 256×256 pixels. `temporal_decode_frames` defaults to `1`, preserving frame-by-frame decoding; other model versions ignore it. Increasing it requires more memory. Adjust the spatial tile size to fit the available memory. - -### Wan2.1 FLF2V 14B - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\wan2.1-flf2v-14b-720p-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf --clip_vision ..\models\clip_vision\clip_vision_h.safetensors -p "glass flower blossom" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" -W 480 -H 832 --video-frames 33 --offload-to-cpu --init-img ..\..\ComfyUI\input\start_image.png --end-img ..\..\ComfyUI\input\end_image.png --flow-shift 3.0 -``` - - - - -### Wan2.2 FLF2V 14B - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\Wan2.2-I2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model ..\models\diffusion_models\Wan2.2-I2V-A14B-HighNoise-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -p "glass flower blossom" -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部,畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" -W 480 -H 832 --video-frames 33 --offload-to-cpu --init-img ..\..\ComfyUI\input\start_image.png --end-img ..\..\ComfyUI\input\end_image.png --flow-shift 3.0 -``` - - - -### Wan2.1 VACE 1.3B - -#### T2V - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\wan2.1-vace-1.3b-q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" -W 832 -H 480 --video-frames 1 --offload-to-cpu -``` - - - - -#### R2V - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\wan2.1-vace-1.3b-q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" -W 832 -H 480 -i ..\assets\cat_with_sd_cpp_42.png --video-frames 33 --offload-to-cpu -``` - - - - -#### V2V - -``` -mkdir post+depth -ffmpeg -i ..\..\ComfyUI\input\post+depth.mp4 -qscale:v 1 -vf fps=8 post+depth\frame_%04d.jpg -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\wan2.1-vace-1.3b-q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf -p "The girl is dancing in a sea of flowers, slowly moving her hands. There is a close - up shot of her upper body. The character is surrounded by other transparent glass flowers in the style of Nicoletta Ceccoli, creating a beautiful, surreal, and emotionally expressive movie scene with a white. transparent feel and a dreamyl atmosphere." --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" -W 480 -H 832 -i ..\..\ComfyUI\input\dance_girl.jpg --control-video ./post+depth --video-frames 33 --offload-to-cpu -``` - - - -### Wan2.1 VACE 14B - -#### T2V - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\Wan2.1_14B_VACE-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" -W 832 -H 480 --video-frames 33 --offload-to-cpu -``` - - - - -#### R2V - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\Wan2.1_14B_VACE-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" -W 832 -H 480 -i ..\assets\cat_with_sd_cpp_42.png --video-frames 33 --offload-to-cpu -``` - - - - - -#### V2V - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\Wan2.1_14B_VACE-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf -p "The girl is dancing in a sea of flowers, slowly moving her hands. There is a close - up shot of her upper body. The character is surrounded by other transparent glass flowers in the style of Nicoletta Ceccoli, creating a beautiful, surreal, and emotionally expressive movie scene with a white. transparent feel and a dreamyl atmosphere." --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" -W 480 -H 832 -i ..\..\ComfyUI\input\dance_girl.jpg --control-video ./post+depth --video-frames 33 --offload-to-cpu -``` - - - -### Wan2.2 VACE-Fun A14B - -VACE-Fun runs as a MoE pair: `--diffusion-model` takes the low-noise expert and -`--high-noise-diffusion-model` the high-noise one. Reference-to-video uses `-i` -for the reference image, same as Wan2.1 VACE. - -#### R2V - -``` -.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\Wan2.2-VACE-Fun-A14B-low-noise-Q8_0.gguf --high-noise-diffusion-model ..\models\diffusion_models\Wan2.2-VACE-Fun-A14B-high-noise-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf -p "a lovely cat" --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -n "色调艳丽,过曝,静态,细节模糊不清,字幕,风格,作品,画作,画面,静止,整体发灰,最差质量,低质量,JPEG压缩残留,丑陋的,残缺的,多余的手指,画得不好的手部,画得不好的脸部, 畸形的,毁容的,形态畸形的肢体,手指融合,静止不动的画面,杂乱的背景,三条腿,背景人很多,倒着走" -W 832 -H 480 -i ..\assets\cat_with_sd_cpp_42.png --video-frames 33 --offload-to-cpu -``` - - - -#### T2V - -Same command without `-i` (VACE context is synthesized from an empty control -video, like Wan2.1 VACE t2v). - -> On GPUs with ~12 GB VRAM, VACE also needs `--vae-tiling` — the control-video -> encode can exceed the budget otherwise. diff --git a/docs/z_image.md b/docs/z_image.md deleted file mode 100644 index 8dca31959b..0000000000 --- a/docs/z_image.md +++ /dev/null @@ -1,41 +0,0 @@ -# How to Use - -You can run Z-Image with stable-diffusion.cpp on GPUs with 4GB of VRAM — or even less. - -## Download weights - -- Download Z-Image-Turbo - - safetensors: https://huggingface.co/Comfy-Org/z_image_turbo/tree/main/split_files/diffusion_models - - gguf: https://huggingface.co/leejet/Z-Image-Turbo-GGUF/tree/main -- Download Z-Image - - safetensors: https://huggingface.co/Comfy-Org/z_image/tree/main/split_files/diffusion_models - - gguf: https://huggingface.co/unsloth/Z-Image-GGUF/tree/main -- Download vae - - safetensors: https://huggingface.co/black-forest-labs/FLUX.1-schnell/tree/main -- Download Qwen3 4b - - safetensors: https://huggingface.co/Comfy-Org/z_image_turbo/tree/main/split_files/text_encoders - - gguf: https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/tree/main - -## Examples - -### Z-Image-Turbo - -``` -.\bin\Release\sd-cli.exe --diffusion-model z_image_turbo-Q3_K.gguf --vae ..\models\vae\ae.sft --llm ..\models\text_encoders\Qwen3-4B-Instruct-2507-Q4_K_M.gguf -p "A cinematic, melancholic photograph of a solitary hooded figure walking through a sprawling, rain-slicked metropolis at night. The city lights are a chaotic blur of neon orange and cool blue, reflecting on the wet asphalt. The scene evokes a sense of being a single component in a vast machine. Superimposed over the image in a sleek, modern, slightly glitched font is the philosophical quote: 'THE CITY IS A CIRCUIT BOARD, AND I AM A BROKEN TRANSISTOR.' -- moody, atmospheric, profound, dark academic" --cfg-scale 1.0 -v --offload-to-cpu -H 1024 -W 512 --steps 8 -``` - -z-image example - -### Z-Image-Base - -``` -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\z_image_bf16.safetensors --vae ..\models\vae\ae.sft --llm ..\models\text_encoders\qwen_3_4b.safetensors -p "A cinematic, melancholic photograph of a solitary hooded figure walking through a sprawling, rain-slicked metropolis at night. The city lights are a chaotic blur of neon orange and cool blue, reflecting on the wet asphalt. The scene evokes a sense of being a single component in a vast machine. Superimposed over the image in a sleek, modern, slightly glitched font is the philosophical quote: 'THE CITY IS A CIRCUIT BOARD, AND I AM A BROKEN TRANSISTOR.' -- moody, atmospheric, profound, dark academic" --cfg-scale 5.0 -v --offload-to-cpu -H 1024 -W 512 -``` - -z-image example - -## Comparison of Different Quantization Types - -| bf16 | q8_0 | q6_K | q5_0 | q4_K | q4_0 | q3_K | q2_K| -|---|---|---|---|---|---|---|---| -| bf16 | q8_0 | q6_K | q5_0 | q4_K | q4_0 | q3_K | q2_K | \ No newline at end of file diff --git a/docs/z_image_l2p.md b/docs/z_image_l2p.md deleted file mode 100644 index a7836f54df..0000000000 --- a/docs/z_image_l2p.md +++ /dev/null @@ -1,21 +0,0 @@ -# Z-Image L2P - -[L2P](https://nju-pcalab.github.io/projects/L2P/) (Latent-to-Pixel) transfers Z-Image-Turbo to pixel space. The VAE is replaced by 16x16 patch tokenization on RGB pixels and a small convolutional local decoder that turns the last DiT hidden states back into pixels, so no VAE is needed. The text encoder is the Qwen3-4B used by Z-Image-Turbo. - -## Download weights - -- Download L2P (1K) - - safetensors: https://huggingface.co/zhen-nan/L2P/tree/main -- Download Qwen3 4b - - safetensors: https://huggingface.co/Comfy-Org/z_image_turbo/tree/main/split_files/text_encoders - - gguf: https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/tree/main - -## Text-to-image - -Do not pass `--vae`. Image dimensions must be multiples of 16. The reference pipeline uses 30 steps with a CFG scale of 2.0. - -The released checkpoint keeps part of the transformer in F32. Pass `--type bf16` to load it in BF16, as the reference pipeline runs it; this reduces the diffusion model from about 18.6 GB to 11.8 GB. - -```bash -.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\model-1k-merge.safetensors --llm ..\models\text_encoders\qwen_3_4b.safetensors -p "an origami pig on fire in the middle of a dark room with a pentagram on the floor" --cfg-scale 2.0 --steps 30 --sampling-method euler -W 1024 -H 1024 --type bf16 -v -o z_image_l2p.png -``` diff --git a/examples/cli/CMakeLists.txt b/examples/cli/CMakeLists.txt index 90164cfc02..36b9243e35 100644 --- a/examples/cli/CMakeLists.txt +++ b/examples/cli/CMakeLists.txt @@ -14,7 +14,6 @@ if(APPLE) endif() target_include_directories(${TARGET} PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/.." - "${PROJECT_SOURCE_DIR}/src" ) install(TARGETS ${TARGET} RUNTIME) target_link_libraries(${TARGET} PRIVATE stable-diffusion zip ${CMAKE_THREAD_LIBS_INIT}) diff --git a/examples/cli/README.md b/examples/cli/README.md index 5789455491..0316f7bd02 100644 --- a/examples/cli/README.md +++ b/examples/cli/README.md @@ -1,18 +1,36 @@ # Usage -For detailed command-line arguments, run: +From the build directory, show the full list of command-line arguments: -```bash +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe -h +``` + +**Linux/macOS** + +```sh ./bin/sd-cli -h ``` +See [sampling methods and schedulers](../../docs/guides/sampling.md) for supported +names, defaults, and custom sigma schedules. + +Omitted inference options use the [library defaults](../../docs/guides/c_api.md#keys). +In particular, an omitted seed requests a random seed; use `--seed 42` for a +repeatable run. Video generation defaults to 6 requested frames before model +alignment. SLG layers remain unset unless `--skip-layers` is supplied, and the +conditioning cache uses the library default. Zero and false are explicit inputs; +invalid values are reported rather than treated as omitted. + Logging defaults to `info`. Use `--log-level ` to select `debug`, `verbose`, `info`, `warn`, or `error` (from most to least detailed). Each level includes messages at that level and all less detailed levels. `-v` and `--verbose` are equivalent to `--log-level verbose`. If repeated, the last logging option wins. `--flash-attn on|off|auto` defaults to `auto`, selecting flash attention for each -compute backend during context creation. See [Backend selection](../../docs/backend.md#flash-attention---flash-attn-onoffauto) +compute backend during context creation. See [Backend selection](../../docs/guides/backend.md#flash-attention---flash-attn-onoffauto) for probing behavior. Use module assignments such as `--flash-attn auto,te=off,vae=on` to override individual modules. @@ -20,32 +38,69 @@ individual modules. `--conv-direct on|off|auto` also defaults to `auto`, enabling direct convolution on backends with implicit GEMM implementations. Use `--conv-direct auto,vae=off` for module overrides; supported modules are `diffusion`, `vae`, and `controlnet`. -See [Direct convolution](../../docs/backend.md#direct-convolution---conv-direct-onoffauto). +See [Direct convolution](../../docs/guides/backend.md#direct-convolution---conv-direct-onoffauto). For direct image repair or automatic post-generation YOLOv8 detection followed by cropped inpainting, see -[ADetailer](../../docs/adetailer.md). +[ADetailer](../../docs/models/adetailer.md). Use repeatable `--image-preprocess` rules to select resizing, cropping, padding, and resampling separately for each image input. Add `canny=true` to any input rule for edge detection. See -[Image preprocessing](../../docs/image_preprocessing.md) for input selectors, +[Image preprocessing](../../docs/guides/image_preprocessing.md) for input selectors, input defaults, downstream model processing, mask alignment, and examples. Metadata mode inspects PNG, JPEG, WebP (including animated WebP), AVI, WebM, and Matroska container metadata without loading any model. Use `--image` for both images and videos: -```bash +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-cli.exe -M metadata --image .\output.png +.\bin\Release\sd-cli.exe ` + -M metadata ` + --image .\output.jpg ` + --metadata-format json +.\bin\Release\sd-cli.exe -M metadata --image .\output.webm +.\bin\Release\sd-cli.exe ` + -M metadata ` + --image .\output.avi ` + --metadata-format json +.\bin\Release\sd-cli.exe ` + -M metadata ` + --image .\output.png ` + --metadata-raw +.\bin\Release\sd-cli.exe ` + -M metadata ` + --image .\output.png ` + --metadata-all +``` + +**Linux/macOS** + +```sh ./bin/sd-cli -M metadata --image ./output.png -./bin/sd-cli -M metadata --image ./output.jpg --metadata-format json +./bin/sd-cli \ + -M metadata \ + --image ./output.jpg \ + --metadata-format json ./bin/sd-cli -M metadata --image ./output.webm -./bin/sd-cli -M metadata --image ./output.avi --metadata-format json -./bin/sd-cli -M metadata --image ./output.png --metadata-raw -./bin/sd-cli -M metadata --image ./output.png --metadata-all +./bin/sd-cli \ + -M metadata \ + --image ./output.avi \ + --metadata-format json +./bin/sd-cli \ + -M metadata \ + --image ./output.png \ + --metadata-raw +./bin/sd-cli \ + -M metadata \ + --image ./output.png \ + --metadata-all ``` For completely black or white images or videos, NaNs, and the `--linear-scale` / `--attn-scale` workaround, see [Troubleshooting](../../docs/troubleshooting.md). -Generation outputs embed the core [generation record](../../docs/generation_metadata.md). +Generation outputs embed the core [generation record](../../docs/guides/generation_metadata.md). Images retain the native JSON and an additional WebUI-compatible infotext. `--disable-image-metadata` omits both from files. diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index 2bff0507f7..a29fd70b60 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -35,28 +35,28 @@ std::regex format_specifier_regex("(?:[^%]|^)(?:%%)*(%\\d{0,3}d)"); struct SDCliParams { std::string input_frames; - int video_upscale_factor = 0; - int vsr_window_size = 16; - int vsr_temporal_chunk_size = 5; - SDMode mode = IMG_GEN; - std::string output_path = "output.png"; - int output_begin_idx = -1; - int compression_quality = 90; + int video_upscale_factor = 0; + std::optional vsr_window_size; + std::optional vsr_temporal_chunk_size; + SDMode mode = IMG_GEN; + std::string output_path = "output.png"; + int output_begin_idx = -1; + int compression_quality = 90; std::string image_path; std::string metadata_format = "text"; sd_log_level_t log_level = SD_LOG_INFO; bool convert_name = false; - preview_t preview_method = PREVIEW_NONE; - int preview_interval = 1; - std::string preview_path = "preview.png"; - int preview_fps = 16; - bool preview_noisy = false; - bool color = false; - bool metadata_raw = false; - bool metadata_brief = false; - bool metadata_all = false; + std::string preview_method = "none"; + int preview_interval = 1; + std::string preview_path = "preview.png"; + int preview_fps = 16; + bool preview_noisy = false; + bool color = false; + bool metadata_raw = false; + bool metadata_brief = false; + bool metadata_all = false; std::string imatrix_out; std::vector imatrix_in; @@ -167,18 +167,7 @@ struct SDCliParams { if (++index >= argc) { return -1; } - const char* preview = argv[index]; - int preview_found = -1; - for (int m = 0; m < PREVIEW_COUNT; m++) { - if (!strcmp(preview, previews_str[m])) { - preview_found = m; - } - } - if (preview_found == -1) { - LOG_ERROR("error: preview method %s", preview); - return -1; - } - preview_method = (preview_t)preview_found; + preview_method = argv[index]; return 1; }; @@ -203,7 +192,7 @@ struct SDCliParams { on_mode_arg}, {"", "--preview", - std::string("preview method. must be one of the following [") + previews_str[0] + ", " + previews_str[1] + ", " + previews_str[2] + ", " + previews_str[3] + "] (default is " + previews_str[PREVIEW_NONE] + ")", + std::string("preview method. must be one of the following [") + previews_str[0] + ", " + previews_str[1] + ", " + previews_str[2] + ", " + previews_str[3] + "] (default is " + previews_str[0] + ")", on_preview_arg}, {"-h", "--help", @@ -272,7 +261,7 @@ struct SDCliParams { << " log_level: " << log_level_name(log_level) << ",\n" << " color: " << (color ? "true" : "false") << ",\n" << " convert_name: " << (convert_name ? "true" : "false") << ",\n" - << " preview_method: " << previews_str[preview_method] << ",\n" + << " preview_method: " << preview_method << ",\n" << " preview_interval: " << preview_interval << ",\n" << " preview_path: \"" << preview_path << "\",\n" << " preview_fps: " << preview_fps << ",\n" @@ -368,10 +357,10 @@ bool load_images_from_dir(const std::string dir, return false; } - images.emplace_back(sd_image_t{(uint32_t)width, - (uint32_t)height, - (uint32_t)loaded_channel, - image_buffer}); + images.emplace_back(SDImage{(uint32_t)width, + (uint32_t)height, + (uint32_t)loaded_channel, + image_buffer}); if (max_image_num > 0 && static_cast(images.size()) >= max_image_num) { break; @@ -402,7 +391,11 @@ std::string format_frame_idx(std::string pattern, int frame_idx) { int continuous_preview_counter = 0; -void step_callback(int step, int frame_count, sd_image_t* image, bool is_noisy, void* data) { +void step_callback(int step, int frame_count, const sd_tensor_t* const* tensors, bool is_noisy, void* data) { + std::vector views; + for (int i = 0; i < frame_count; ++i) + views.push_back(tensor_image_view(tensors[i])); + SDImage* image = views.data(); (void)step; (void)is_noisy; SDCliParams* cli_params = (SDCliParams*)data; @@ -424,7 +417,7 @@ void step_callback(int step, int frame_count, sd_image_t* image, bool is_noisy, LOG_ERROR("save preview image to '%s' failed", path.u8string().c_str()); } } else { - int fps = cli_params->preview_method == PREVIEW_PROJ ? cli_params->preview_fps / 4 : cli_params->preview_fps; + int fps = cli_params->preview_method == "proj" ? cli_params->preview_fps / 4 : cli_params->preview_fps; if (create_video_from_sd_images(cli_params->preview_path.c_str(), image, frame_count, fps, cli_params->compression_quality) != 0) { LOG_ERROR("save preview video to '%s' failed", cli_params->preview_path.c_str()); } @@ -456,7 +449,7 @@ bool save_results(const SDCliParams& cli_params, const SDGenerationParams& gen_params, SDImageVec& results, int num_results, - const sd_audio_t* generated_audio, + const SDAudio* generated_audio, const std::string& video_info) { if (results.empty() || num_results <= 0) { return false; @@ -497,7 +490,7 @@ bool save_results(const SDCliParams& cli_params, } auto write_image = [&](const fs::path& path, int idx) { - const sd_image_t& img = results[idx]; + const SDImage& img = results[idx]; if (!img.data) return false; @@ -556,7 +549,7 @@ bool save_results(const SDCliParams& cli_params, std::string final_ext_lower = ext.u8string(); std::transform(final_ext_lower.begin(), final_ext_lower.end(), final_ext_lower.begin(), ::tolower); const bool mux_audio = generated_audio != nullptr && (final_ext_lower == ".avi" || final_ext_lower == ".webm"); - if (create_video_from_sd_images(video_path.u8string().c_str(), results.data(), num_results, gen_params.fps, cli_params.compression_quality, mux_audio ? generated_audio : nullptr, params) == 0) { + if (create_video_from_sd_images(video_path.u8string().c_str(), results.data(), num_results, gen_params.fps.value(), cli_params.compression_quality, mux_audio ? generated_audio : nullptr, params) == 0) { LOG_INFO("save result video to '%s'", video_path.u8string().c_str()); if (generated_audio != nullptr && !mux_audio) { fs::path wav_path = video_path; @@ -591,10 +584,8 @@ bool save_results(const SDCliParams& cli_params, } static bool apply_adetailer(sd_ctx_t* sd_ctx, - const sd_ctx_params_t& sd_ctx_params, const SDContextParams& ctx_params, - const SDGenerationParams& gen_params, - const sd_img_gen_params_t& img_gen_params, + SDGenerationParams& gen_params, SDMode mode, SDImageVec& results, int num_results) { @@ -602,17 +593,12 @@ static bool apply_adetailer(sd_ctx_t* sd_ctx, return true; } - sd_adetailer_params_t ad_params{}; - ad_params.prompt = gen_params.ad_prompt.empty() ? nullptr : gen_params.ad_prompt.c_str(); - ad_params.negative_prompt = gen_params.ad_negative_prompt.empty() ? nullptr : gen_params.ad_negative_prompt.c_str(); - ad_params.extra_ad_args = gen_params.extra_ad_args.c_str(); - - ADetailerCtxPtr ad_ctx(new_adetailer_ctx(gen_params.ad_model_path.c_str(), - ctx_params.n_threads, - sd_ctx_params.backend, - sd_ctx_params.params_backend)); + ADetailerCtxPtr ad_ctx(create_sd_adetailer(gen_params.ad_model_path.c_str(), + ctx_params.n_threads, + ctx_params.effective_backend.c_str(), + ctx_params.effective_params_backend.c_str())); if (ad_ctx == nullptr) { - LOG_ERROR("new_adetailer_ctx failed"); + LOG_ERROR("sd_adetailer_create failed"); return false; } @@ -620,30 +606,29 @@ static bool apply_adetailer(sd_ctx_t* sd_ctx, if (results[i].data == nullptr) { continue; } - sd_img_gen_params_t ad_generation_params = img_gen_params; - ad_generation_params.seed = generation_seed(results.generation_info(i), img_gen_params.seed); + SDOptBuilder ad_options(gen_params.to_image_opt(true).release()); + ad_options.set_int("seed", generation_seed(results.generation_info(i), gen_params.seed.value_or(-1))); if (mode == IMG_GEN) { - ad_generation_params.width = 512; - ad_generation_params.height = 512; - ad_generation_params.strength = 0.4f; + ad_options.unset("width"); + ad_options.unset("height"); + ad_options.unset("strength"); } - sd_image_generation_result_t* detailed = nullptr; - if (!adetail_image(ad_ctx.get(), - sd_ctx, - results[i], - &ad_params, - &ad_generation_params, - &detailed) || - detailed == nullptr || detailed->count <= 0 || detailed->items[0].image.data == nullptr) { - free_sd_image_generation_result(detailed); + sd_result_t* detailed = nullptr; + if (!adetail_sd_image(ad_ctx.get(), + sd_ctx, + results[i], + ad_options.finish(), + &detailed) || + detailed == nullptr || sd_result_get_image_count(detailed) <= 0 || sd_tensor_data(sd_result_get_image(detailed, 0)) == nullptr) { + sd_result_free(detailed); LOG_ERROR("ADetailer failed for image %d", i + 1); return false; } - results.append_generation_info(i, detailed->items[0].generation_info, ctx_params.lora_model_dir); - free(results[i].data); - results[i] = detailed->items[0].image; - detailed->items[0].image = {}; - free_sd_image_generation_result(detailed); + results.append_generation_info(i, sd_result_get_generation_info(detailed, 0), ctx_params.lora_model_dir); + sd_image_free(&results[i]); + results[i] = take_result_image(detailed, 0); + + sd_result_free(detailed); } return true; } @@ -656,7 +641,6 @@ int main(int argc, const char* argv[]) { SDCliParams cli_params; SDContextParams ctx_params; - ctx_params.conditioning_cache_size = 0; SDGenerationParams gen_params; sd_set_log_callback(sd_log_cb, (void*)&cli_params); @@ -685,10 +669,6 @@ int main(int argc, const char* argv[]) { return 1; } ctx_params.prepare_backend_assignments(); - if (ctx_params.diffusion_model_path.empty() || ctx_params.vae_path.empty() || gen_params.fps <= 0) { - LOG_ERROR("vid_upscale requires --diffusion-model, --vae and positive --fps"); - return 1; - } std::vector owned_frames; if (!gen_params.init_image_path.empty()) { owned_frames.emplace_back(); @@ -702,44 +682,45 @@ int main(int argc, const char* argv[]) { } if (owned_frames.size() > size_t(INT_MAX)) return 1; - std::vector frames; + std::vector frames; for (const auto& frame : owned_frames) frames.push_back(frame.get()); - sd_video_upscaler_ctx_params_t params; - sd_video_upscaler_ctx_params_init(¶ms); + SDVideoUpscalerParams params; + params.diffusion_model_path = ctx_params.diffusion_model_path.c_str(); params.vae_path = ctx_params.vae_path.c_str(); params.upscale_factor = cli_params.video_upscale_factor; params.n_threads = ctx_params.n_threads; params.backend = ctx_params.effective_backend.c_str(); params.params_backend = ctx_params.effective_params_backend.c_str(); - params.flash_attn_mode = ctx_params.flash_attn_mode.c_str(); - std::string conv_direct_mode = ctx_params.conv_direct_mode; + params.flash_attn_mode = ctx_params.flash_attn_mode ? ctx_params.flash_attn_mode->c_str() : nullptr; + std::string conv_direct_mode = ctx_params.conv_direct_mode.value_or(""); if (conv_direct_mode.empty()) { conv_direct_mode = "auto"; - if (ctx_params.diffusion_conv_direct) + if (ctx_params.diffusion_conv_direct.value_or(false)) conv_direct_mode += ",diffusion=on"; - if (ctx_params.vae_conv_direct) + if (ctx_params.vae_conv_direct.value_or(false)) conv_direct_mode += ",vae=on"; } params.conv_direct_mode = conv_direct_mode.c_str(); - std::unique_ptr upscaler( - new_video_upscaler_ctx(¶ms), - free_video_upscaler_ctx); + std::unique_ptr upscaler( + create_sd_vid_upscaler(params), + sd_vid_upscaler_free); if (!upscaler) return 1; - sd_video_generation_result_t* output = nullptr; - sd_video_upscale_params_t upscale_params; - sd_video_upscale_params_init(&upscale_params); + sd_result_t* output = nullptr; + SDVideoUpscaleParams upscale_params; + upscale_params.frames = frames.data(); upscale_params.frame_count = int(frames.size()); upscale_params.fps = gen_params.fps; upscale_params.window_size = cli_params.vsr_window_size; upscale_params.temporal_chunk_size = cli_params.vsr_temporal_chunk_size; upscale_params.seed = gen_params.seed; - if (!upscale_video(upscaler.get(), &upscale_params, &output)) + if (!upscale_sd_video(upscaler.get(), upscale_params, &output)) return 1; - std::string video_info = output->generation_info; + std::string video_info = sd_result_get_generation_info(output, 0); + gen_params.fps = sd_result_get_fps(output); SDImageVec results; results.adopt_video(output); return save_results(cli_params, ctx_params, gen_params, results, int(results.size()), nullptr, video_info) ? 0 : 1; @@ -750,7 +731,7 @@ int main(int argc, const char* argv[]) { return 1; } - if (gen_params.video_frames > 4) { + if (cli_params.mode == VID_GEN || gen_params.video_frames > 4) { size_t last_dot_pos = cli_params.preview_path.find_last_of("."); std::string base_path = cli_params.preview_path; std::string file_ext = ""; @@ -763,14 +744,18 @@ int main(int argc, const char* argv[]) { cli_params.preview_path = base_path + ".avi"; } } - cli_params.preview_fps = gen_params.fps; - - sd_set_preview_callback(step_callback, - cli_params.preview_method, - cli_params.preview_interval, - !cli_params.preview_noisy, - cli_params.preview_noisy, - (void*)&cli_params); + if (gen_params.fps) + cli_params.preview_fps = *gen_params.fps; + + if (sd_set_preview_callback(step_callback, + cli_params.preview_method.c_str(), + cli_params.preview_interval, + !cli_params.preview_noisy, + cli_params.preview_noisy, + (void*)&cli_params) != SD_STATUS_OK) { + LOG_ERROR("%s", sd_get_last_error()); + return 1; + } LOG_VERBOSE("version: %s", version_string().c_str()); LOG_VERBOSE("%s", sd_get_system_info()); @@ -813,7 +798,7 @@ int main(int argc, const char* argv[]) { ctx_params.wtype, ctx_params.tensor_type_rules.c_str(), cli_params.convert_name, - ctx_params.n_threads, + ctx_params.n_threads.value_or(0), loras.empty() ? nullptr : loras.data(), (int)loras.size()); if (!success) { @@ -945,7 +930,7 @@ int main(int argc, const char* argv[]) { gen_params.control_frames.clear(); if (!load_images_from_dir(gen_params.control_video_path, gen_params.control_frames, - gen_params.video_frames)) { + gen_params.video_frames.value_or(0))) { return 1; } } @@ -958,46 +943,55 @@ int main(int argc, const char* argv[]) { } } - sd_ctx_params_t sd_ctx_params = ctx_params.to_sd_ctx_params_t(); + SDOptPtr sd_ctx_params = ctx_params.to_context_opt(); SDImageVec results; - int num_results = 0; - sd_audio_t* generated_audio = nullptr; + int num_results = 0; + SDAudio* generated_audio = nullptr; std::string video_info; if (cli_params.mode == UPSCALE) { num_results = 1; results.push_back(gen_params.init_image.release()); } else { - SDCtxPtr sd_ctx(new_sd_ctx(&sd_ctx_params)); + SDCtxPtr sd_ctx(create_sd_context(sd_ctx_params)); if (sd_ctx == nullptr) { - LOG_INFO("new_sd_ctx_t failed"); + LOG_INFO("sd_ctx_create failed"); return 1; } - sd_img_gen_params_t img_gen_params{}; - const bool use_img_gen_params = cli_params.mode == IMG_GEN || cli_params.mode == ADETAILER; - if (use_img_gen_params) { - img_gen_params = gen_params.to_sd_img_gen_params_t(); - } + SDImageOwner prompt_image; + sd_set_prompt_image_callback( + sd_ctx.get(), [](const char* source, void* data, const sd_tensor_t** image_out) -> bool { + auto& image = *static_cast(data); + SDImage loaded{}; + if (!load_sd_image_from_file(&loaded, source, 0, 0, 3)) { + return false; + } + image.reset(loaded); + *image_out = image.tensor(); + return true; + }, + &prompt_image); + const bool use_img_gen_params = cli_params.mode == IMG_GEN || cli_params.mode == ADETAILER; if (cli_params.mode == IMG_GEN) { - sd_image_generation_result_t* generated = nullptr; - generate_image(sd_ctx.get(), &img_gen_params, &generated); + sd_result_t* generated = nullptr; + generate_sd_image(sd_ctx.get(), gen_params.to_image_opt(), &generated); results.adopt(generated, ctx_params.lora_model_dir); num_results = results.count(); } else if (cli_params.mode == ADETAILER) { num_results = 1; results.push_back(gen_params.init_image.release()); } else if (cli_params.mode == VID_GEN) { - sd_vid_gen_params_t vid_gen_params = gen_params.to_sd_vid_gen_params_t(); - sd_video_generation_result_t* generated = nullptr; - if (generate_video(sd_ctx.get(), &vid_gen_params, &generated)) { - gen_params.fps = cli_params.preview_fps = generated->fps; - generated_audio = generated->audio; - generated->audio = nullptr; - num_results = generated->frame_count; + SDOptPtr vid_gen_params = gen_params.to_video_opt(); + sd_result_t* generated = nullptr; + if (generate_sd_video(sd_ctx.get(), vid_gen_params, &generated)) { + gen_params.fps = cli_params.preview_fps = sd_result_get_fps(generated); + generated_audio = retain_result_audio(generated); + + num_results = sd_result_get_image_count(generated); results.adopt_video(generated, ctx_params.lora_model_dir); video_info = results.generation_info(0); } @@ -1010,10 +1004,8 @@ int main(int argc, const char* argv[]) { if (use_img_gen_params && !apply_adetailer(sd_ctx.get(), - sd_ctx_params, ctx_params, gen_params, - img_gen_params, cli_params.mode, results, num_results)) { @@ -1021,17 +1013,16 @@ int main(int argc, const char* argv[]) { } } - int upscale_factor = 4; // unused for RealESRGAN_x4plus_anime_6B.pth if (ctx_params.esrgan_path.size() > 0 && gen_params.upscale_repeats > 0) { - UpscalerCtxPtr upscaler_ctx(new_upscaler_ctx(ctx_params.esrgan_path.c_str(), - ctx_params.diffusion_conv_direct, - ctx_params.n_threads, - gen_params.upscale_tile_size, - sd_ctx_params.backend, - sd_ctx_params.params_backend)); + UpscalerCtxPtr upscaler_ctx(create_sd_upscaler(ctx_params.esrgan_path.c_str(), + ctx_params.diffusion_conv_direct, + ctx_params.n_threads, + gen_params.upscale_tile_size, + ctx_params.effective_backend.c_str(), + ctx_params.effective_params_backend.c_str())); if (upscaler_ctx == nullptr) { - LOG_ERROR("new_upscaler_ctx failed"); + LOG_ERROR("sd_upscaler_create failed"); } else { for (int i = 0; i < num_results; i++) { if (results[i].data == nullptr) { @@ -1040,20 +1031,17 @@ int main(int argc, const char* argv[]) { SDImageOwner current_image(results[i]); results[i] = {0, 0, 0, nullptr}; for (int u = 0; u < gen_params.upscale_repeats; ++u) { - sd_image_generation_result_t* upscaled = nullptr; - bool upscale_ok = upscale(upscaler_ctx.get(), - current_image.get(), - upscale_factor, - &upscaled); - if (!upscale_ok || !upscaled || upscaled->count <= 0 || upscaled->items[0].image.data == nullptr) { - free_sd_image_generation_result(upscaled); + sd_result_t* upscaled = nullptr; + bool upscale_ok = upscale_sd_image(upscaler_ctx.get(), current_image.get(), &upscaled); + if (!upscale_ok || !upscaled || sd_result_get_image_count(upscaled) <= 0 || sd_tensor_data(sd_result_get_image(upscaled, 0)) == nullptr) { + sd_result_free(upscaled); LOG_ERROR("upscale failed"); break; } - results.append_generation_info(i, upscaled->items[0].generation_info); - sd_image_t upscaled_image = upscaled->items[0].image; - upscaled->items[0].image = {}; - free_sd_image_generation_result(upscaled); + results.append_generation_info(i, sd_result_get_generation_info(upscaled, 0)); + SDImage upscaled_image = take_result_image(upscaled, 0); + + sd_result_free(upscaled); current_image.reset(upscaled_image); } results[i] = current_image.release(); // Set the final upscaled image as the result diff --git a/examples/common/common.cpp b/examples/common/common.cpp index 102bf9507c..0c155c1b89 100644 --- a/examples/common/common.cpp +++ b/examples/common/common.cpp @@ -38,25 +38,6 @@ const char* const modes_str[] = { "vid_upscale", }; -static sd_vae_format_t str_to_vae_format(const std::string& value) { - if (value == "auto") { - return SD_VAE_FORMAT_AUTO; - } - if (value == "flux") { - return SD_VAE_FORMAT_FLUX; - } - if (value == "sd3") { - return SD_VAE_FORMAT_SD3; - } - if (value == "flux2") { - return SD_VAE_FORMAT_FLUX2; - } - if (value == "wan") { - return SD_VAE_FORMAT_WAN; - } - return SD_VAE_FORMAT_COUNT; -} - static void prepend_backend_assignment(std::string& spec, const char* assignment) { spec = spec.empty() ? assignment : std::string(assignment) + "," + spec; } @@ -286,13 +267,13 @@ bool parse_options(int argc, const char** argv, const std::vector& o invalid_arg = true; return; } - if (option.concat && !option.target->empty()) { + if (option.concat && !option.target.get().empty()) { if (option.concat > 0 && option.concat <= 0xff) { - *option.target += static_cast(option.concat); + option.target.set(option.target.get() + static_cast(option.concat)); } - *option.target += argv_to_utf8(i, argv); + option.target.set(option.target.get() + argv_to_utf8(i, argv)); } else { - *option.target = argv_to_utf8(i, argv); + option.target.set(argv_to_utf8(i, argv)); } found_arg = true; })) @@ -304,7 +285,7 @@ bool parse_options(int argc, const char** argv, const std::vector& o return; } try { - *option.target = std::stoi(argv[i]); + option.target.set(std::stoi(argv[i])); } catch (const std::invalid_argument&) { invalid_arg = true; } @@ -318,7 +299,7 @@ bool parse_options(int argc, const char** argv, const std::vector& o return; } try { - *option.target = std::stof(argv[i]); + option.target.set(std::stof(argv[i])); } catch (const std::invalid_argument&) { invalid_arg = true; } @@ -327,8 +308,8 @@ bool parse_options(int argc, const char** argv, const std::vector& o break; if (match_and_apply(options.bool_options, [&](auto& option) { - *option.target = option.keep_true ? true : false; - found_arg = true; + option.target.set(option.keep_true); + found_arg = true; })) break; @@ -360,7 +341,7 @@ bool parse_options(int argc, const char** argv, const std::vector& o return true; } -static int parse_scale_override(int argc, const char** argv, int index, float& scale) { +static int parse_scale_override(int argc, const char** argv, int index, std::optional& scale) { if (++index >= argc) { return -1; } @@ -379,7 +360,7 @@ static int parse_scale_override(int argc, const char** argv, int index, float& s return 1; } -static int parse_on_off_arg(int argc, const char** argv, int index, const char* option, bool& value) { +static int parse_on_off_arg(int argc, const char** argv, int index, const char* option, std::optional& value) { if (++index >= argc) { LOG_ERROR("%s requires 'on' or 'off'", option); return -1; @@ -587,12 +568,11 @@ ArgOptions SDContextParams::get_options() { options.int_options = { {"-t", "--threads", - "number of threads to use during computation (default: -1). " - "If threads <= 0, then threads will be set to the number of CPU physical cores", + "positive number of computation threads; unset uses the library CPU core selection", &n_threads}, {"", "--conditioning-cache-size", - "maximum number of conditioning results cached per model context (default: " + std::to_string(conditioning_cache_size) + ", 0 disables caching)", + "maximum cached conditioning results; unset uses the library default, 0 disables caching", &conditioning_cache_size}, }; @@ -667,12 +647,7 @@ ArgOptions SDContextParams::get_options() { return -1; } const char* arg = argv[index]; - rng_type = str_to_rng_type(arg); - if (rng_type == RNG_TYPE_COUNT) { - LOG_ERROR("error: invalid rng type %s", - arg); - return -1; - } + rng_type = arg; return 1; }; @@ -681,12 +656,7 @@ ArgOptions SDContextParams::get_options() { return -1; } const char* arg = argv[index]; - sampler_rng_type = str_to_rng_type(arg); - if (sampler_rng_type == RNG_TYPE_COUNT) { - LOG_ERROR("error: invalid sampler rng type %s", - arg); - return -1; - } + sampler_rng_type = arg; return 1; }; @@ -695,12 +665,7 @@ ArgOptions SDContextParams::get_options() { return -1; } const char* arg = argv[index]; - prediction = str_to_prediction(arg); - if (prediction == PREDICTION_COUNT) { - LOG_ERROR("error: invalid prediction type %s", - arg); - return -1; - } + prediction = arg; return 1; }; @@ -709,12 +674,7 @@ ArgOptions SDContextParams::get_options() { return -1; } const char* arg = argv[index]; - lora_apply_mode = str_to_lora_apply_mode(arg); - if (lora_apply_mode == LORA_APPLY_MODE_COUNT) { - LOG_ERROR("error: invalid lora apply model %s", - arg); - return -1; - } + lora_apply_mode = arg; return 1; }; @@ -835,7 +795,7 @@ ArgOptions SDContextParams::get_options() { } bool SDContextParams::print_devices() const { - if (!sd_add_rpc_servers(rpc_servers.c_str())) { + if (!sd_add_rpc_servers(rpc_servers ? rpc_servers->c_str() : nullptr)) { return false; } size_t device_list_size = sd_list_devices(nullptr, 0); @@ -878,20 +838,12 @@ void SDContextParams::build_embedding_map() { } bool SDContextParams::resolve(SDMode mode) { - if (n_threads <= 0) { - n_threads = sd_get_num_physical_cores(); - } - build_embedding_map(); return true; } bool SDContextParams::validate(SDMode mode) { - if (conditioning_cache_size < 0) { - LOG_ERROR("error: conditioning-cache-size must be non-negative"); - return false; - } if (mode == CONVERT) { const bool has_convert_input = model_path.length() != 0 || clip_l_path.length() != 0 || @@ -915,8 +867,8 @@ bool SDContextParams::validate(SDMode mode) { } } - if (str_to_vae_format(vae_format) == SD_VAE_FORMAT_COUNT) { - LOG_ERROR("error: vae_format must be 'auto', 'flux', 'sd3', 'flux2', or 'wan'"); + if (!to_context_opt()) { + LOG_ERROR("%s", sd_get_last_error()); return false; } @@ -934,8 +886,8 @@ bool SDContextParams::resolve_and_validate(SDMode mode) { } void SDContextParams::prepare_backend_assignments() { - effective_backend = backend; - effective_params_backend = params_backend; + effective_backend = backend.value_or(""); + effective_params_backend = params_backend.value_or(""); if (offload_params_to_cpu) { prepend_backend_assignment(effective_params_backend, "*=cpu"); @@ -995,39 +947,39 @@ std::string SDContextParams::to_string() const { << " lora_model_dir: \"" << lora_model_dir << "\",\n" << " hires_upscalers_dir: \"" << hires_upscalers_dir << "\",\n" << " photo_maker_path: \"" << photo_maker_path << "\",\n" - << " rng_type: " << sd_rng_type_name(rng_type) << ",\n" - << " sampler_rng_type: " << sd_rng_type_name(sampler_rng_type) << ",\n" + << " rng_type: " << rng_type << ",\n" + << " sampler_rng_type: " << sampler_rng_type << ",\n" << " offload_params_to_cpu: " << (offload_params_to_cpu ? "true" : "false") << ",\n" << " max_vram: \"" << max_vram << "\",\n" - << " disable_prefetch: " << (disable_prefetch ? "true" : "false") << ",\n" - << " disable_segmented_compute: " << (disable_segmented_compute ? "true" : "false") << ",\n" - << " batched_cfg: " << (batched_cfg ? "true" : "false") << ",\n" - << " eager_load: " << (eager_load ? "true" : "false") << ",\n" + << " disable_prefetch: " << disable_prefetch << ",\n" + << " disable_segmented_compute: " << disable_segmented_compute << ",\n" + << " batched_cfg: " << batched_cfg << ",\n" + << " eager_load: " << eager_load << ",\n" << " backend: \"" << backend << "\",\n" << " params_backend: \"" << params_backend << "\",\n" << " split_mode: \"" << split_mode << "\",\n" << " model_args: \"" << model_args << "\",\n" - << " auto_fit: " << (auto_fit ? "true" : "false") << ",\n" - << " enable_mmap: " << (enable_mmap ? "true" : "false") << ",\n" + << " auto_fit: " << auto_fit << ",\n" + << " enable_mmap: " << enable_mmap << ",\n" << " control_net_cpu: " << (control_net_cpu ? "true" : "false") << ",\n" << " clip_on_cpu: " << (clip_on_cpu ? "true" : "false") << ",\n" << " vae_on_cpu: " << (vae_on_cpu ? "true" : "false") << ",\n" << " flash_attn_mode: " << flash_attn_mode << ",\n" - << " conv_direct_mode: " << (conv_direct_mode.empty() ? "auto (legacy flags honored)" : conv_direct_mode) << ",\n" - << " sage_attn: " << (sage_attn ? "true" : "false") << ",\n" + << " conv_direct_mode: " << conv_direct_mode << ",\n" + << " sage_attn: " << sage_attn << ",\n" << " linear_scale: " << linear_scale << ",\n" << " attn_scale: " << attn_scale << ",\n" - << " diffusion_conv_direct: " << (diffusion_conv_direct ? "true" : "false") << ",\n" - << " vae_conv_direct: " << (vae_conv_direct ? "true" : "false") << ",\n" - << " taesd_preview_only: " << (taesd_preview_only ? "true" : "false") << ",\n" - << " prediction: " << sd_prediction_name(prediction) << ",\n" - << " lora_apply_mode: " << sd_lora_apply_mode_name(lora_apply_mode) << ",\n" - << " force_sdxl_vae_conv_scale: " << (force_sdxl_vae_conv_scale ? "true" : "false") << "\n" + << " diffusion_conv_direct: " << diffusion_conv_direct << ",\n" + << " vae_conv_direct: " << vae_conv_direct << ",\n" + << " taesd_preview_only: " << taesd_preview_only << ",\n" + << " prediction: " << prediction << ",\n" + << " lora_apply_mode: " << lora_apply_mode << ",\n" + << " force_sdxl_vae_conv_scale: " << force_sdxl_vae_conv_scale << "\n" << "}"; return oss.str(); } -sd_ctx_params_t SDContextParams::to_sd_ctx_params_t() { +SDOptPtr SDContextParams::to_context_opt() { prepare_backend_assignments(); embedding_vec.clear(); embedding_vec.reserve(embedding_map.size()); @@ -1038,67 +990,66 @@ sd_ctx_params_t SDContextParams::to_sd_ctx_params_t() { embedding_vec.emplace_back(item); } - sd_ctx_params_t sd_ctx_params; - sd_ctx_params_init(&sd_ctx_params); - sd_ctx_params.model_path = model_path.c_str(); - sd_ctx_params.clip_l_path = clip_l_path.c_str(); - sd_ctx_params.clip_g_path = clip_g_path.c_str(); - sd_ctx_params.clip_vision_path = clip_vision_path.c_str(); - sd_ctx_params.t5xxl_path = t5xxl_path.c_str(); - sd_ctx_params.llm_path = llm_path.c_str(); - sd_ctx_params.llm_vision_path = llm_vision_path.c_str(); - sd_ctx_params.tokenizer = tokenizer.c_str(); - sd_ctx_params.diffusion_model_path = diffusion_model_path.c_str(); - sd_ctx_params.high_noise_diffusion_model_path = high_noise_diffusion_model_path.c_str(); - sd_ctx_params.uncond_diffusion_model_path = uncond_diffusion_model_path.c_str(); - sd_ctx_params.embeddings_connectors_path = embeddings_connectors_path.c_str(); - sd_ctx_params.vae_path = vae_path.c_str(); - sd_ctx_params.audio_vae_path = audio_vae_path.c_str(); - sd_ctx_params.audio_encoder_path = audio_encoder_path.c_str(); - sd_ctx_params.taesd_path = taesd_path.c_str(); - sd_ctx_params.control_net_path = control_net_path.c_str(); - sd_ctx_params.ip_adapter_path = ip_adapter_path.c_str(); - sd_ctx_params.motion_module_path = motion_module_path.c_str(); - sd_ctx_params.embeddings = embedding_vec.data(); - sd_ctx_params.embedding_count = static_cast(embedding_vec.size()); - sd_ctx_params.photo_maker_path = photo_maker_path.c_str(); - sd_ctx_params.pulid_weights_path = pulid_weights_path.c_str(); - sd_ctx_params.tensor_type_rules = tensor_type_rules.c_str(); - sd_ctx_params.n_threads = n_threads; - sd_ctx_params.conditioning_cache_size = conditioning_cache_size; - sd_ctx_params.wtype = wtype; - sd_ctx_params.rng_type = rng_type; - sd_ctx_params.sampler_rng_type = sampler_rng_type; - sd_ctx_params.prediction = prediction; - sd_ctx_params.lora_apply_mode = lora_apply_mode; - sd_ctx_params.enable_mmap = enable_mmap; - sd_ctx_params.flash_attn_mode = flash_attn_mode.c_str(); - sd_ctx_params.conv_direct_mode = conv_direct_mode.c_str(); - sd_ctx_params.sage_attn = sage_attn; - sd_ctx_params.linear_scale = linear_scale; - sd_ctx_params.attn_scale = attn_scale; - sd_ctx_params.tae_preview_only = taesd_preview_only; - sd_ctx_params.diffusion_conv_direct = diffusion_conv_direct; - sd_ctx_params.vae_conv_direct = vae_conv_direct; - sd_ctx_params.force_sdxl_vae_conv_scale = force_sdxl_vae_conv_scale; - sd_ctx_params.vae_format = str_to_vae_format(vae_format); - sd_ctx_params.max_vram = max_vram.c_str(); - sd_ctx_params.disable_prefetch = disable_prefetch; - sd_ctx_params.disable_segmented_compute = disable_segmented_compute; - sd_ctx_params.batched_cfg = batched_cfg; - sd_ctx_params.eager_load = eager_load; - sd_ctx_params.backend = effective_backend.c_str(); - sd_ctx_params.params_backend = effective_params_backend.c_str(); - sd_ctx_params.split_mode = split_mode.c_str(); - sd_ctx_params.auto_fit = auto_fit; - sd_ctx_params.rpc_servers = rpc_servers.c_str(); - sd_ctx_params.model_args = model_args.empty() ? nullptr : model_args.c_str(); - return sd_ctx_params; + SDOptBuilder opt(sd_ctx_opt_create()); + opt.set_string("model_path", model_path); + opt.set_string("clip_l_path", clip_l_path); + opt.set_string("clip_g_path", clip_g_path); + opt.set_string("clip_vision_path", clip_vision_path); + opt.set_string("t5xxl_path", t5xxl_path); + opt.set_string("llm_path", llm_path); + opt.set_string("llm_vision_path", llm_vision_path); + opt.set_string("tokenizer", tokenizer); + opt.set_string("diffusion_model_path", diffusion_model_path); + opt.set_string("high_noise_diffusion_model_path", high_noise_diffusion_model_path); + opt.set_string("uncond_diffusion_model_path", uncond_diffusion_model_path); + opt.set_string("embeddings_connectors_path", embeddings_connectors_path); + opt.set_string("vae_path", vae_path); + opt.set_string("audio_vae_path", audio_vae_path); + opt.set_string("audio_encoder_path", audio_encoder_path); + opt.set_string("taesd_path", taesd_path); + opt.set_string("control_net_path", control_net_path); + opt.set_string("ip_adapter_path", ip_adapter_path); + opt.set_string("motion_module_path", motion_module_path); + opt.set_string("photo_maker_path", photo_maker_path); + opt.set_string("pulid_weights_path", pulid_weights_path); + opt.set_string("tensor_type_rules", tensor_type_rules); + opt.set_int("n_threads", n_threads); + opt.set_int("conditioning_cache_size", conditioning_cache_size); + if (wtype != SD_TYPE_COUNT) + opt.set_string("wtype", sd_type_name(wtype)); + opt.set_string("rng_type", rng_type); + opt.set_string("sampler_rng_type", sampler_rng_type); + opt.set_string("prediction", prediction); + opt.set_string("lora_apply_mode", lora_apply_mode); + opt.set_bool("enable_mmap", enable_mmap); + opt.set_string("flash_attn_mode", flash_attn_mode); + opt.set_string("conv_direct_mode", conv_direct_mode); + opt.set_bool("sage_attn", sage_attn); + opt.set_float("linear_scale", linear_scale); + opt.set_float("attn_scale", attn_scale); + opt.set_bool("tae_preview_only", taesd_preview_only); + opt.set_bool("diffusion_conv_direct", diffusion_conv_direct); + opt.set_bool("vae_conv_direct", vae_conv_direct); + opt.set_bool("force_sdxl_vae_conv_scale", force_sdxl_vae_conv_scale); + opt.set_string("vae_format", vae_format); + opt.set_string("max_vram", max_vram); + opt.set_bool("disable_prefetch", disable_prefetch); + opt.set_bool("disable_segmented_compute", disable_segmented_compute); + opt.set_bool("batched_cfg", batched_cfg); + opt.set_bool("eager_load", eager_load); + if (backend || !effective_backend.empty()) + opt.set_string("backend", effective_backend); + if (params_backend || !effective_params_backend.empty()) + opt.set_string("params_backend", effective_params_backend); + opt.set_string("split_mode", split_mode); + opt.set_bool("auto_fit", auto_fit); + opt.set_string("rpc_servers", rpc_servers); + opt.set_string("model_args", model_args); + opt.set_embeddings("embeddings", embedding_vec.data(), embedding_vec.size()); + return opt.finish(); } SDGenerationParams::SDGenerationParams() { - sd_sample_params_init(&sample_params); - sd_sample_params_init(&high_noise_sample_params); } ArgOptions SDGenerationParams::get_options() { @@ -1247,7 +1198,7 @@ ArgOptions SDGenerationParams::get_options() { &qwen_image_layers}, {"", "--video-frames", - "video frames (default: 1)", + "video frames; unset uses the library default", &video_frames}, {"", "--fps", @@ -1440,12 +1391,7 @@ ArgOptions SDGenerationParams::get_options() { return -1; } const char* arg = argv[index]; - sample_params.sample_method = str_to_sample_method(arg); - if (sample_params.sample_method == SAMPLE_METHOD_COUNT) { - LOG_ERROR("error: invalid sample method %s", - arg); - return -1; - } + sample_params.sample_method = arg; return 1; }; @@ -1454,12 +1400,7 @@ ArgOptions SDGenerationParams::get_options() { return -1; } const char* arg = argv[index]; - high_noise_sample_params.sample_method = str_to_sample_method(arg); - if (high_noise_sample_params.sample_method == SAMPLE_METHOD_COUNT) { - LOG_ERROR("error: invalid high noise sample method %s", - arg); - return -1; - } + high_noise_sample_params.sample_method = arg; return 1; }; @@ -1468,12 +1409,7 @@ ArgOptions SDGenerationParams::get_options() { return -1; } const char* arg = argv[index]; - sample_params.scheduler = str_to_scheduler(arg); - if (sample_params.scheduler == SCHEDULER_COUNT) { - LOG_ERROR("error: invalid scheduler %s", - arg); - return -1; - } + sample_params.scheduler = arg; return 1; }; @@ -1482,7 +1418,7 @@ ArgOptions SDGenerationParams::get_options() { return -1; } std::string layers_str = argv[index]; - if (layers_str[0] != '[' || layers_str[layers_str.size() - 1] != ']') { + if (layers_str.size() < 2 || layers_str.front() != '[' || layers_str.back() != ']') { return -1; } @@ -1495,8 +1431,12 @@ ArgOptions SDGenerationParams::get_options() { std::vector layers; for (const auto& token : tokens) { try { - layers.push_back(std::stoi(token)); - } catch (const std::invalid_argument&) { + size_t used = 0; + const int layer = std::stoi(token, &used); + if (used != token.size()) + return -1; + layers.push_back(layer); + } catch (const std::exception&) { return -1; } } @@ -1509,7 +1449,7 @@ ArgOptions SDGenerationParams::get_options() { return -1; } std::string layers_str = argv[index]; - if (layers_str[0] != '[' || layers_str[layers_str.size() - 1] != ']') { + if (layers_str.size() < 2 || layers_str.front() != '[' || layers_str.back() != ']') { return -1; } @@ -1522,8 +1462,12 @@ ArgOptions SDGenerationParams::get_options() { std::vector layers; for (const auto& token : tokens) { try { - layers.push_back(std::stoi(token)); - } catch (const std::invalid_argument&) { + size_t used = 0; + const int layer = std::stoi(token, &used); + if (used != token.size()) + return -1; + layers.push_back(layer); + } catch (const std::exception&) { return -1; } } @@ -1531,7 +1475,7 @@ ArgOptions SDGenerationParams::get_options() { return 1; }; - auto parse_sigmas_arg = [&](const char* value, std::vector* target, const char* option_name) { + auto parse_sigmas_arg = [&](const char* value, std::optional>* target, const char* option_name) { if (target == nullptr || value == nullptr) { return -1; } @@ -1556,7 +1500,7 @@ ArgOptions SDGenerationParams::get_options() { sigmas_str.pop_back(); } - size_t before = target->size(); + std::vector parsed; std::stringstream ss(sigmas_str); std::string item; while (std::getline(ss, item, ',')) { @@ -1564,7 +1508,7 @@ ArgOptions SDGenerationParams::get_options() { item.erase(item.find_last_not_of(" \t\n\r\f\v") + 1); if (!item.empty()) { try { - target->push_back(std::stof(item)); + parsed.push_back(std::stof(item)); } catch (const std::invalid_argument&) { LOG_ERROR("error: invalid float value '%s' in %s", item.c_str(), option_name); return -1; @@ -1575,10 +1519,11 @@ ArgOptions SDGenerationParams::get_options() { } } - if (target->size() == before && !sigmas_str.empty()) { + if (parsed.empty() && !sigmas_str.empty()) { LOG_ERROR("error: could not parse any sigma values from '%s'", value); return -1; } + *target = std::move(parsed); return 1; }; @@ -1763,20 +1708,23 @@ ArgOptions SDGenerationParams::get_options() { return 1; }; - std::string sample_methods = sample_method_to_str[0]; - for (int i = 1; i < SAMPLE_METHOD_COUNT; i++) { - sample_methods += ", " + std::string(sample_method_to_str[i]); + std::string sample_methods; + for (const char* name : example_options::sample_methods) { + if (!sample_methods.empty()) + sample_methods += ", "; + sample_methods += name; } - - std::string schedulers = scheduler_to_str[0]; - for (int i = 1; i < SCHEDULER_COUNT; i++) { - schedulers += ", " + std::string(scheduler_to_str[i]); + std::string schedulers; + for (const char* name : example_options::schedulers) { + if (!schedulers.empty()) + schedulers += ", "; + schedulers += name; } options.manual_options = { {"-s", "--seed", - "RNG seed (default: 42, use random seed for < 0)", + "RNG seed; unset or a negative value requests a random seed", on_seed_arg}, {"", "--sampling-method", @@ -1803,11 +1751,11 @@ ArgOptions SDGenerationParams::get_options() { on_hires_sigmas_arg}, {"", "--skip-layers", - "layers to skip for SLG steps (default: [7,8,9])", + "layers to skip for SLG steps; unset uses the library default", on_skip_layers_arg}, {"", "--high-noise-skip-layers", - "(high noise) layers to skip for SLG steps (default: [7,8,9])", + "(high noise) layers to skip for SLG steps; unset uses the library default", on_high_noise_skip_layers_arg}, {"-r", "--ref-image", @@ -2199,10 +2147,64 @@ bool SDGenerationParams::parse_image_preprocess_json(const std::string& json_str return true; } +bool SDGenerationParams::parse_prompt_images_json(const std::string& json_str) { + auto images = json::parse(json_str, nullptr, false); + if (!images.is_object()) { + return false; + } + std::map decoded; + for (auto it = images.begin(); it != images.end(); ++it) { + if (it.key().empty() || it.key().find('>') != std::string::npos || !it.value().is_string()) { + return false; + } + SDImageOwner image; + if (!decode_base64_image(it.value().get(), 3, 0, 0, image)) { + return false; + } + decoded.emplace(it.key(), std::move(image)); + } + prompt_images = std::move(decoded); + return true; +} + +template +static void load_json_input(const json& object, const char* key, Field& out) { + if (!object.contains(key)) + return; + const auto& value = object.at(key); + if (value.is_null()) { + out = Field{}; + return; + } + using T = typename InputType::type; + bool valid = false; + if constexpr (std::is_same_v) + valid = value.is_string(); + else if constexpr (std::is_same_v) + valid = value.is_boolean(); + else if constexpr (std::is_integral_v) { + valid = value.is_number_integer(); + if (valid) { + if (value.is_number_unsigned()) + valid = value.get() <= static_cast(std::numeric_limits::max()); + else { + const auto integer = value.get(); + valid = integer >= std::numeric_limits::min() && integer <= std::numeric_limits::max(); + } + } + } else if constexpr (std::is_floating_point_v) + valid = value.is_number(); + else + valid = value.is_array(); + if (!valid) + throw std::invalid_argument(std::string(key) + ": invalid value type or range"); + out = value.get(); +} + /** Populates generation params from a JSON payload (server APIs); returns false on any malformed field. */ bool SDGenerationParams::from_json_str( const std::string& json_str, - const std::function& lora_path_resolver) { + const std::function& lora_path_resolver) try { json j; try { j = json::parse(json_str); @@ -2213,34 +2215,10 @@ bool SDGenerationParams::from_json_str( if (j.contains("image_preprocess") && !parse_image_preprocess_json(j["image_preprocess"].dump())) return false; + if (j.contains("prompt_images") && !parse_prompt_images_json(j["prompt_images"].dump())) + return false; - auto load_if_exists = [&](const char* key, auto& out) { - if (j.contains(key)) { - using T = std::decay_t; - if constexpr (std::is_same_v) { - if (j[key].is_string()) - out = j[key]; - } else if constexpr (std::is_same_v || std::is_same_v) { - if (j[key].is_number_integer()) - out = j[key]; - } else if constexpr (std::is_same_v) { - if (j[key].is_number()) - out = j[key]; - } else if constexpr (std::is_same_v) { - if (j[key].is_boolean()) - out = j[key]; - } else if constexpr (std::is_same_v>) { - if (j[key].is_array()) - out = j[key].get>(); - } else if constexpr (std::is_same_v>) { - if (j[key].is_array()) - out = j[key].get>(); - } else if constexpr (std::is_same_v>) { - if (j[key].is_array()) - out = j[key].get>(); - } - } - }; + auto load_if_exists = [&](const char* key, auto& out) { load_json_input(j, key, out); }; load_if_exists("prompt", prompt); load_if_exists("negative_prompt", negative_prompt); @@ -2275,98 +2253,42 @@ bool SDGenerationParams::from_json_str( if (j.contains("hires") && j["hires"].is_object()) { const json& hires_json = j["hires"]; - if (hires_json.contains("enabled") && hires_json["enabled"].is_boolean()) { - hires_enabled = hires_json["enabled"]; - } - if (hires_json.contains("upscaler") && hires_json["upscaler"].is_string()) { - hires_upscaler = hires_json["upscaler"]; - } - if (hires_json.contains("scale") && hires_json["scale"].is_number()) { - hires_scale = hires_json["scale"]; - } - if (hires_json.contains("target_width") && hires_json["target_width"].is_number_integer()) { - hires_width = hires_json["target_width"]; - } - if (hires_json.contains("target_height") && hires_json["target_height"].is_number_integer()) { - hires_height = hires_json["target_height"]; - } - if (hires_json.contains("steps") && hires_json["steps"].is_number_integer()) { - hires_steps = hires_json["steps"]; - } - if (hires_json.contains("denoising_strength") && hires_json["denoising_strength"].is_number()) { - hires_denoising_strength = hires_json["denoising_strength"]; - } - if (hires_json.contains("custom_sigmas") && hires_json["custom_sigmas"].is_array()) { - hires_custom_sigmas = hires_json["custom_sigmas"].get>(); - } - if (hires_json.contains("upscale_tile_size") && hires_json["upscale_tile_size"].is_number_integer()) { - hires_upscale_tile_size = hires_json["upscale_tile_size"]; - } + load_json_input(hires_json, "enabled", hires_enabled); + load_json_input(hires_json, "upscaler", hires_upscaler); + load_json_input(hires_json, "scale", hires_scale); + load_json_input(hires_json, "target_width", hires_width); + load_json_input(hires_json, "target_height", hires_height); + load_json_input(hires_json, "steps", hires_steps); + load_json_input(hires_json, "denoising_strength", hires_denoising_strength); + load_json_input(hires_json, "custom_sigmas", hires_custom_sigmas); + load_json_input(hires_json, "upscale_tile_size", hires_upscale_tile_size); } auto parse_sample_params_json = [&](const json& sample_json, - sd_sample_params_t& target_params, + SDSampleParams& target_params, std::string& target_extra_sample_args, - std::vector& target_skip_layers, - std::vector* target_custom_sigmas) { - if (sample_json.contains("sample_steps") && sample_json["sample_steps"].is_number_integer()) { - target_params.sample_steps = sample_json["sample_steps"]; - } - if (sample_json.contains("eta") && sample_json["eta"].is_number()) { - target_params.eta = sample_json["eta"]; - } - if (sample_json.contains("shifted_timestep") && sample_json["shifted_timestep"].is_number_integer()) { - target_params.shifted_timestep = sample_json["shifted_timestep"]; - } - if (sample_json.contains("flow_shift") && sample_json["flow_shift"].is_number()) { - target_params.flow_shift = sample_json["flow_shift"]; - } - if (sample_json.contains("extra_sample_args") && sample_json["extra_sample_args"].is_string()) { - target_extra_sample_args = sample_json["extra_sample_args"].get(); - } - if (target_custom_sigmas != nullptr && - sample_json.contains("custom_sigmas") && - sample_json["custom_sigmas"].is_array() && - !sample_json["custom_sigmas"].empty()) { // empty [] = "not specified", keep server/CLI default - *target_custom_sigmas = sample_json["custom_sigmas"].get>(); - } - if (sample_json.contains("sample_method") && sample_json["sample_method"].is_string()) { - enum sample_method_t tmp = str_to_sample_method(sample_json["sample_method"].get().c_str()); - if (tmp != SAMPLE_METHOD_COUNT) { - target_params.sample_method = tmp; - } - } - if (sample_json.contains("scheduler") && sample_json["scheduler"].is_string()) { - enum scheduler_t tmp = str_to_scheduler(sample_json["scheduler"].get().c_str()); - if (tmp != SCHEDULER_COUNT) { - target_params.scheduler = tmp; - } - } + std::optional>& target_skip_layers, + std::optional>* target_custom_sigmas) { + load_json_input(sample_json, "sample_steps", target_params.sample_steps); + load_json_input(sample_json, "eta", target_params.eta); + load_json_input(sample_json, "shifted_timestep", target_params.shifted_timestep); + load_json_input(sample_json, "flow_shift", target_params.flow_shift); + load_json_input(sample_json, "extra_sample_args", target_extra_sample_args); + if (target_custom_sigmas) + load_json_input(sample_json, "custom_sigmas", *target_custom_sigmas); + load_json_input(sample_json, "sample_method", target_params.sample_method); + load_json_input(sample_json, "scheduler", target_params.scheduler); if (sample_json.contains("guidance") && sample_json["guidance"].is_object()) { const json& guidance_json = sample_json["guidance"]; - if (guidance_json.contains("txt_cfg") && guidance_json["txt_cfg"].is_number()) { - target_params.guidance.txt_cfg = guidance_json["txt_cfg"]; - } - if (guidance_json.contains("img_cfg") && guidance_json["img_cfg"].is_number()) { - target_params.guidance.img_cfg = guidance_json["img_cfg"]; - } - if (guidance_json.contains("distilled_guidance") && guidance_json["distilled_guidance"].is_number()) { - target_params.guidance.distilled_guidance = guidance_json["distilled_guidance"]; - } + load_json_input(guidance_json, "txt_cfg", target_params.guidance.txt_cfg); + load_json_input(guidance_json, "img_cfg", target_params.guidance.img_cfg); + load_json_input(guidance_json, "distilled_guidance", target_params.guidance.distilled_guidance); if (guidance_json.contains("slg") && guidance_json["slg"].is_object()) { const json& slg_json = guidance_json["slg"]; - if (slg_json.contains("layers") && slg_json["layers"].is_array()) { - target_skip_layers = slg_json["layers"].get>(); - } - if (slg_json.contains("layer_start") && slg_json["layer_start"].is_number()) { - target_params.guidance.slg.layer_start = slg_json["layer_start"]; - } - if (slg_json.contains("layer_end") && slg_json["layer_end"].is_number()) { - target_params.guidance.slg.layer_end = slg_json["layer_end"]; - } - if (slg_json.contains("scale") && slg_json["scale"].is_number()) { - target_params.guidance.slg.scale = slg_json["scale"]; - } + load_json_input(slg_json, "layers", target_skip_layers); + load_json_input(slg_json, "layer_start", target_params.guidance.slg.layer_start); + load_json_input(slg_json, "layer_end", target_params.guidance.slg.layer_end); + load_json_input(slg_json, "scale", target_params.guidance.slg.scale); } } }; @@ -2384,30 +2306,14 @@ bool SDGenerationParams::from_json_str( if (j.contains("vae_tiling_params") && j["vae_tiling_params"].is_object()) { const json& tiling_json = j["vae_tiling_params"]; - if (tiling_json.contains("enabled") && tiling_json["enabled"].is_boolean()) { - vae_tiling_params.enabled = tiling_json["enabled"]; - } - if (tiling_json.contains("temporal_tiling") && tiling_json["temporal_tiling"].is_boolean()) { - vae_tiling_params.temporal_tiling = tiling_json["temporal_tiling"]; - } - if (tiling_json.contains("tile_size_w") && tiling_json["tile_size_w"].is_number_integer()) { - vae_tiling_params.tile_size_w = tiling_json["tile_size_w"]; - } - if (tiling_json.contains("tile_size_h") && tiling_json["tile_size_h"].is_number_integer()) { - vae_tiling_params.tile_size_h = tiling_json["tile_size_h"]; - } - if (tiling_json.contains("target_overlap") && tiling_json["target_overlap"].is_number()) { - vae_tiling_params.target_overlap = tiling_json["target_overlap"]; - } - if (tiling_json.contains("rel_size_w") && tiling_json["rel_size_w"].is_number()) { - vae_tiling_params.rel_size_w = tiling_json["rel_size_w"]; - } - if (tiling_json.contains("rel_size_h") && tiling_json["rel_size_h"].is_number()) { - vae_tiling_params.rel_size_h = tiling_json["rel_size_h"]; - } - if (tiling_json.contains("extra_tiling_args") && tiling_json["extra_tiling_args"].is_string()) { - extra_tiling_args = tiling_json["extra_tiling_args"].get(); - } + load_json_input(tiling_json, "enabled", vae_tiling_params.enabled); + load_json_input(tiling_json, "temporal_tiling", vae_tiling_params.temporal_tiling); + load_json_input(tiling_json, "tile_size_w", vae_tiling_params.tile_size_w); + load_json_input(tiling_json, "tile_size_h", vae_tiling_params.tile_size_h); + load_json_input(tiling_json, "target_overlap", vae_tiling_params.target_overlap); + load_json_input(tiling_json, "rel_size_w", vae_tiling_params.rel_size_w); + load_json_input(tiling_json, "rel_size_h", vae_tiling_params.rel_size_h); + load_json_input(tiling_json, "extra_tiling_args", extra_tiling_args); } if (!parse_lora_json_field(j, lora_path_resolver, lora_map, high_noise_lora_map)) { @@ -2438,6 +2344,9 @@ bool SDGenerationParams::from_json_str( } return true; +} catch (const std::exception& error) { + LOG_ERROR("invalid generation parameters: %s", error.what()); + return false; } bool SDGenerationParams::extract_and_remove_lora(const std::string& lora_model_dir) { @@ -2521,27 +2430,26 @@ bool SDGenerationParams::extract_and_remove_lora(const std::string& lora_model_d } bool SDGenerationParams::width_and_height_are_set() const { - return width > 0 && height > 0; + return width.has_value() && height.has_value(); } void SDGenerationParams::set_width_and_height_if_unset(int w, int h) { - if (!width_and_height_are_set()) { - LOG_INFO("set width x height to %d x %d", w, h); - width = w; + if (!width) + width = w; + if (!height) height = h; - } } -int SDGenerationParams::get_resolved_width() const { - return (width > 0) ? width : 512; +int SDGenerationParams::get_requested_width() const { + return width.value_or(0); } -int SDGenerationParams::get_resolved_height() const { - return (height > 0) ? height : 512; +int SDGenerationParams::get_requested_height() const { + return height.value_or(0); } bool SDGenerationParams::initialize_cache_params() { - sd_cache_params_init(&cache_params); + cache_params = SDCacheParams{}; auto parse_named_params = [&](const std::string& opt_str) -> bool { std::stringstream ss(opt_str); @@ -2607,19 +2515,19 @@ bool SDGenerationParams::initialize_cache_params() { if (!cache_mode.empty()) { if (cache_mode == "disabled") { - cache_params.mode = SD_CACHE_DISABLED; + cache_params.mode = "disabled"; } else if (cache_mode == "easycache") { - cache_params.mode = SD_CACHE_EASYCACHE; + cache_params.mode = "easycache"; } else if (cache_mode == "ucache") { - cache_params.mode = SD_CACHE_UCACHE; + cache_params.mode = "ucache"; } else if (cache_mode == "dbcache") { - cache_params.mode = SD_CACHE_DBCACHE; + cache_params.mode = "dbcache"; } else if (cache_mode == "taylorseer") { - cache_params.mode = SD_CACHE_TAYLORSEER; + cache_params.mode = "taylorseer"; } else if (cache_mode == "cache-dit") { - cache_params.mode = SD_CACHE_CACHE_DIT; + cache_params.mode = "cache_dit"; } else if (cache_mode == "spectrum") { - cache_params.mode = SD_CACHE_SPECTRUM; + cache_params.mode = "spectrum"; } else { LOG_ERROR("error: invalid cache mode '%s'", cache_mode.c_str()); return false; @@ -2630,9 +2538,9 @@ bool SDGenerationParams::initialize_cache_params() { return false; } - if (cache_params.mode == SD_CACHE_DBCACHE || - cache_params.mode == SD_CACHE_TAYLORSEER || - cache_params.mode == SD_CACHE_CACHE_DIT) { + if (cache_params.mode == "dbcache" || + cache_params.mode == "taylorseer" || + cache_params.mode == "cache_dit") { cache_params.scm_policy_dynamic = scm_policy_dynamic; } @@ -2642,47 +2550,35 @@ bool SDGenerationParams::initialize_cache_params() { bool SDGenerationParams::resolve(const std::string& lora_model_dir, const std::string& hires_upscalers_dir, bool strict, bool validate_missing_loras) { vae_tiling_params.extra_tiling_args = extra_tiling_args.empty() ? nullptr : extra_tiling_args.c_str(); - if (high_noise_sample_params.sample_steps <= 0) { - high_noise_sample_params.sample_steps = -1; - } - if (!initialize_cache_params()) { return false; } - if (strict) { - batch_count = std::clamp(batch_count, 1, 8); - sample_params.sample_steps = std::clamp(sample_params.sample_steps, 1, 100); - } - - resolved_control_type = SD_CONTROL_TYPE_NONE; - if (!control_type.empty()) { - resolved_control_type = str_to_sd_control_type(control_type.c_str()); - if (resolved_control_type == SD_CONTROL_TYPE_COUNT) { - LOG_ERROR("invalid control type '%s' (openpose, depth, scribble, canny, normal, segment, tile, repaint)", control_type.c_str()); - return false; - } + if (strict && ((batch_count && (*batch_count < 1 || *batch_count > 8)) || + (sample_params.sample_steps && (*sample_params.sample_steps < 1 || *sample_params.sample_steps > 100)))) { + LOG_ERROR("server requests require batch_count in [1, 8] and sample_steps in [1, 100]"); + return false; } hires_upscaler_model_path.clear(); - if (hires_enabled) { - if (hires_upscaler.empty()) { - hires_upscaler = "Latent"; - } - resolved_hires_upscaler = str_to_sd_hires_upscaler(hires_upscaler.c_str()); - if (resolved_hires_upscaler == SD_HIRES_UPSCALER_NONE) { - hires_enabled = false; - } else if (resolved_hires_upscaler == SD_HIRES_UPSCALER_COUNT) { + resolved_hires_upscaler.clear(); + if (hires_upscaler) { + static const std::map names = { + {"None", "none"}, {"Latent", "latent"}, {"Latent (nearest)", "latent_nearest"}, {"Latent (nearest-exact)", "latent_nearest_exact"}, {"Latent (antialiased)", "latent_antialiased"}, {"Latent (bicubic)", "latent_bicubic"}, {"Latent (bicubic antialiased)", "latent_bicubic_antialiased"}, {"Lanczos", "lanczos"}, {"Nearest", "nearest"}, {"Model", "model"}}; + const auto it = names.find(*hires_upscaler); + if (it != names.end()) { + resolved_hires_upscaler = it->second; + } else if (std::any_of(names.begin(), names.end(), [&](const auto& name) { return name.second == *hires_upscaler; })) { + resolved_hires_upscaler = *hires_upscaler; + } else { static const std::vector valid_ext = {".gguf", ".safetensors", ".pt", ".pth"}; - if (!resolve_model_file_from_dir(hires_upscaler, - hires_upscalers_dir, - valid_ext, - "hires upscaler", - hires_upscaler_model_path)) { + if (hires_enabled.value_or(false) && + !resolve_model_file_from_dir(*hires_upscaler, hires_upscalers_dir, valid_ext, "hires upscaler", hires_upscaler_model_path)) return false; - } - resolved_hires_upscaler = SD_HIRES_UPSCALER_MODEL; + resolved_hires_upscaler = "model"; } + if (resolved_hires_upscaler == "none") + hires_enabled = false; } prompt_with_lora = prompt; @@ -2700,54 +2596,6 @@ bool SDGenerationParams::validate(SDMode mode) { LOG_ERROR("--image-preprocess requires img_gen or vid_gen mode"); return false; } - if (batch_count <= 0) { - LOG_ERROR("error: batch_count must be greater than 0"); - return false; - } - - if (qwen_image_layers < 0) { - LOG_ERROR("error: qwen_image_layers must be non-negative"); - return false; - } - - if (sample_params.sample_steps <= 0) { - LOG_ERROR("error: the sample_steps must be greater than 0\n"); - return false; - } - - if (strength < 0.f || strength > 1.f) { - LOG_ERROR("error: can only work with strength in [0.0, 1.0]\n"); - return false; - } - - if (sample_params.guidance.txt_cfg < 0.f) { - LOG_ERROR("error: cfg_scale must be positive"); - return false; - } - - if (!cache_mode.empty()) { - if (cache_mode == "easycache" || cache_mode == "ucache") { - if (cache_params.reuse_threshold < 0.0f) { - LOG_ERROR("error: cache threshold must be non-negative"); - return false; - } - if (cache_params.start_percent < 0.0f || cache_params.start_percent >= 1.0f || - cache_params.end_percent <= 0.0f || cache_params.end_percent > 1.0f || - cache_params.start_percent >= cache_params.end_percent) { - LOG_ERROR("error: cache start/end percents must satisfy 0.0 <= start < end <= 1.0"); - return false; - } - } - } - - if (mode == VID_GEN && video_frames <= 0) { - return false; - } - - if (mode == VID_GEN && fps <= 0) { - return false; - } - if (ref_video_audio_paths.size() + ref_video_audios.size() > ref_video_paths.size() + ref_videos.size()) { LOG_ERROR("error: each --ref-video-audio needs a corresponding --ref-video"); return false; @@ -2759,46 +2607,10 @@ bool SDGenerationParams::validate(SDMode mode) { return false; } - if (sample_params.shifted_timestep < 0 || sample_params.shifted_timestep > 1000) { - LOG_ERROR("error: shifted_timestep must be in range [0, 1000]"); - return false; - } - if (upscale_repeats < 1) { return false; } - if (upscale_tile_size < 1) { - return false; - } - - if (hires_enabled) { - if (hires_width < 0 || hires_height < 0) { - LOG_ERROR("error: hires target width and height must be >= 0"); - return false; - } - if (hires_scale <= 0.f && hires_width <= 0 && hires_height <= 0) { - LOG_ERROR("error: hires scale must be positive when target size is not set"); - return false; - } - if (hires_steps < 0) { - LOG_ERROR("error: hires steps must be >= 0"); - return false; - } - if (hires_denoising_strength <= 0.f || hires_denoising_strength > 1.f) { - LOG_ERROR("error: hires denoising strength must be in (0.0, 1.0]"); - return false; - } - if (!hires_custom_sigmas.empty() && hires_custom_sigmas.size() < 2) { - LOG_ERROR("error: hires custom sigmas must contain at least two values"); - return false; - } - if (hires_upscale_tile_size < 1) { - LOG_ERROR("error: hires upscale tile size must be positive"); - return false; - } - } - if (mode == UPSCALE || mode == ADETAILER) { if (init_image_path.length() == 0) { LOG_ERROR("error: %s mode needs an init image (--init-img)\n", @@ -2812,6 +2624,13 @@ bool SDGenerationParams::validate(SDMode mode) { return false; } + if (mode == IMG_GEN || mode == ADETAILER || mode == VID_GEN) { + auto opt = mode == VID_GEN ? to_video_opt() : to_image_opt(mode == ADETAILER); + if (!opt) { + LOG_ERROR("%s", sd_get_last_error()); + return false; + } + } return true; } @@ -2828,10 +2647,68 @@ bool SDGenerationParams::resolve_and_validate(SDMode mode, return true; } -sd_img_gen_params_t SDGenerationParams::to_sd_img_gen_params_t() { - sd_img_gen_params_t params; - sd_img_gen_params_init(¶ms); +static void set_sample_options(SDOptBuilder& opt, const std::string& prefix, const SDSampleParams& params, const std::optional>& layers, const std::optional>& sigmas, const std::string& extra_args) { + if (!extra_args.empty()) + opt.set_string((prefix + "args").c_str(), extra_args); + opt.set_float((prefix + "eta").c_str(), params.eta); + opt.set_float((prefix + "guidance.distilled_guidance").c_str(), params.guidance.distilled_guidance); + opt.set_float((prefix + "guidance.img_cfg").c_str(), params.guidance.img_cfg); + opt.set_float((prefix + "guidance.slg.layer_end").c_str(), params.guidance.slg.layer_end); + opt.set_float((prefix + "guidance.slg.layer_start").c_str(), params.guidance.slg.layer_start); + if (layers) + opt.set_ints((prefix + "guidance.slg.layers").c_str(), layers->data(), layers->size()); + opt.set_float((prefix + "guidance.slg.scale").c_str(), params.guidance.slg.scale); + opt.set_float((prefix + "guidance.txt_cfg").c_str(), params.guidance.txt_cfg); + opt.set_string((prefix + "method").c_str(), params.sample_method); + if (prefix == "sample.") { + opt.set_string("sample.scheduler", params.scheduler); + opt.set_float("sample.flow_shift", params.flow_shift); + opt.set_int("sample.shifted_timestep", params.shifted_timestep); + if (sigmas) + opt.set_floats((prefix + "sigmas").c_str(), sigmas->data(), sigmas->size()); + } + opt.set_int((prefix + "steps").c_str(), params.sample_steps); +} +static void set_cache_options(SDOptBuilder& opt, const SDCacheParams& params) { + opt.set_int("cache.bn_compute_blocks", params.Bn_compute_blocks); + opt.set_float("cache.end_percent", params.end_percent); + opt.set_float("cache.error_decay_rate", params.error_decay_rate); + opt.set_int("cache.fn_compute_blocks", params.Fn_compute_blocks); + opt.set_int("cache.max_cached_steps", params.max_cached_steps); + opt.set_int("cache.max_continuous_cached_steps", params.max_continuous_cached_steps); + opt.set_int("cache.max_warmup_steps", params.max_warmup_steps); + opt.set_string("cache.mode", params.mode); + opt.set_bool("cache.reset_error_on_compute", params.reset_error_on_compute); + opt.set_float("cache.residual_diff_threshold", params.residual_diff_threshold); + opt.set_float("cache.reuse_threshold", params.reuse_threshold); + opt.set_string("cache.scm_mask", params.scm_mask); + opt.set_bool("cache.scm_policy_dynamic", params.scm_policy_dynamic); + opt.set_float("cache.spectrum_flex_window", params.spectrum_flex_window); + opt.set_float("cache.spectrum_lam", params.spectrum_lam); + opt.set_int("cache.spectrum_m", params.spectrum_m); + opt.set_float("cache.spectrum_stop_percent", params.spectrum_stop_percent); + opt.set_float("cache.spectrum_w", params.spectrum_w); + opt.set_int("cache.spectrum_warmup_steps", params.spectrum_warmup_steps); + opt.set_int("cache.spectrum_window_size", params.spectrum_window_size); + opt.set_float("cache.start_percent", params.start_percent); + opt.set_int("cache.taylorseer_n_derivatives", params.taylorseer_n_derivatives); + opt.set_int("cache.taylorseer_skip_interval", params.taylorseer_skip_interval); + opt.set_bool("cache.use_relative_threshold", params.use_relative_threshold); +} + +static void set_tiling_options(SDOptBuilder& opt, const SDTilingParams& params) { + opt.set_bool("vae_tiling.enabled", params.enabled); + opt.set_string("vae_tiling.extra_tiling_args", params.extra_tiling_args); + opt.set_float("vae_tiling.rel_size_h", params.rel_size_h); + opt.set_float("vae_tiling.rel_size_w", params.rel_size_w); + opt.set_float("vae_tiling.target_overlap", params.target_overlap); + opt.set_bool("vae_tiling.temporal_tiling", params.temporal_tiling); + opt.set_int("vae_tiling.tile_size_h", params.tile_size_h); + opt.set_int("vae_tiling.tile_size_w", params.tile_size_w); +} + +SDOptPtr SDGenerationParams::to_image_opt(bool adetail) { lora_vec.clear(); lora_vec.reserve(lora_map.size() + high_noise_lora_map.size()); for (const auto& kv : lora_map) { @@ -2853,85 +2730,101 @@ sd_img_gen_params_t SDGenerationParams::to_sd_img_gen_params_t() { pm_id_image_views.push_back(image.get()); } - sample_params.guidance.slg.layers = skip_layers.empty() ? nullptr : skip_layers.data(); - sample_params.guidance.slg.layer_count = skip_layers.size(); - high_noise_sample_params.guidance.slg.layers = high_noise_skip_layers.empty() ? nullptr : high_noise_skip_layers.data(); - high_noise_sample_params.guidance.slg.layer_count = high_noise_skip_layers.size(); - sample_params.custom_sigmas = custom_sigmas.empty() ? nullptr : custom_sigmas.data(); - sample_params.custom_sigmas_count = static_cast(custom_sigmas.size()); - sample_params.extra_sample_args = extra_sample_args.empty() ? nullptr : extra_sample_args.c_str(); - high_noise_sample_params.extra_sample_args = high_noise_extra_sample_args.empty() ? nullptr : high_noise_extra_sample_args.c_str(); - vae_tiling_params.extra_tiling_args = extra_tiling_args.empty() ? nullptr : extra_tiling_args.c_str(); - cache_params.scm_mask = scm_mask.empty() ? nullptr : scm_mask.c_str(); - - sd_pm_params_t pm_params = { - pm_id_image_views.empty() ? nullptr : pm_id_image_views.data(), - static_cast(pm_id_image_views.size()), - pm_id_embed_path.empty() ? nullptr : pm_id_embed_path.c_str(), - pm_style_strength, - }; - - sd_pulid_params_t pulid_params = { - pulid_id_embedding_path.empty() ? nullptr : pulid_id_embedding_path.c_str(), - pulid_id_weight, - }; - - if (increase_ref_index) { - if (!ref_image_args.empty()) { - ref_image_args += ","; - } - ref_image_args += "ref_index_mode=increase"; - LOG_WARN("Notice: --increase-ref-index is deprecated. Use --ref-image-args \"ref_index_mode=increase\" instead."); - } - - params.loras = lora_vec.empty() ? nullptr : lora_vec.data(); - params.lora_count = static_cast(lora_vec.size()); - params.prompt = prompt.c_str(); - params.negative_prompt = negative_prompt.c_str(); - params.clip_skip = clip_skip; - params.init_image = init_image.get(); - params.ref_images = ref_image_views.empty() ? nullptr : ref_image_views.data(); - params.ref_images_count = static_cast(ref_image_views.size()); - params.ref_image_args = ref_image_args.c_str(); - params.mask_image = mask_image.get(); - params.width = get_resolved_width(); - params.height = get_resolved_height(); - params.sample_params = sample_params; - params.strength = strength; - params.seed = seed; - params.batch_count = batch_count; - params.qwen_image_layers = qwen_image_layers; - params.control_image = control_image.get(); - params.control_strength = control_strength; - params.control_type = resolved_control_type; - params.ip_adapter_image = ip_adapter_image.get(); - params.ip_adapter_strength = ip_adapter_strength; - params.pm_params = pm_params; - params.pulid_params = pulid_params; - params.vae_tiling_params = vae_tiling_params; - params.cache = cache_params; - - params.hires.enabled = hires_enabled; - params.hires.upscaler = resolved_hires_upscaler; - params.hires.model_path = hires_upscaler_model_path.empty() ? nullptr : hires_upscaler_model_path.c_str(); - params.hires.scale = hires_scale; - params.hires.target_width = hires_width; - params.hires.target_height = hires_height; - params.hires.steps = hires_steps; - params.hires.denoising_strength = hires_denoising_strength; - params.hires.upscale_tile_size = hires_upscale_tile_size; - params.hires.custom_sigmas = hires_custom_sigmas.empty() ? nullptr : hires_custom_sigmas.data(); - params.hires.custom_sigmas_count = static_cast(hires_custom_sigmas.size()); - params.circular_x = circular || circular_x; - params.circular_y = circular || circular_y; - params.image_preprocess = {image_preprocess.c_str()}; - return params; + vae_tiling_params.extra_tiling_args = extra_tiling_args.empty() ? nullptr : extra_tiling_args.c_str(); + cache_params.scm_mask = scm_mask.empty() ? nullptr : scm_mask.c_str(); + + std::string effective_ref_image_args = ref_image_args; + if (!adetail && increase_ref_index) { + if (!effective_ref_image_args.empty()) + effective_ref_image_args += ","; + effective_ref_image_args += "ref_index_mode=increase"; + } + + SDOptBuilder opt(adetail ? sd_adetail_opt_create() : sd_img_gen_opt_create()); + opt.set_string("prompt", prompt); + opt.set_string("negative_prompt", negative_prompt); + opt.set_int("clip_skip", clip_skip); + if (!adetail) + opt.set_image("init_image", init_image.get()); + if (!adetail) + opt.set_string("ref_image_args", effective_ref_image_args); + if (!adetail) + opt.set_image("mask_image", mask_image.get()); + opt.set_int("width", width); + opt.set_int("height", height); + opt.set_float("strength", strength); + opt.set_int("seed", seed); + if (!adetail) + opt.set_int("batch_count", batch_count); + if (!adetail) + opt.set_int("qwen_image_layers", qwen_image_layers); + if (!adetail) + opt.set_image("control_image", control_image.get()); + if (!adetail) + opt.set_float("control_strength", control_strength); + if (!adetail) + opt.set_string("control_type", control_type); + if (!adetail) + opt.set_image("ip_adapter_image", ip_adapter_image.get()); + if (!adetail) + opt.set_float("ip_adapter_strength", ip_adapter_strength); + if (!adetail) + opt.set_bool("hires.enabled", hires_enabled); + if (!adetail) + opt.set_string("hires.upscaler", resolved_hires_upscaler.empty() ? hires_upscaler : std::optional(resolved_hires_upscaler)); + if (!adetail) + opt.set_string("hires.model_path", hires_upscaler_model_path); + if (!adetail) + opt.set_float("hires.scale", hires_scale); + if (!adetail) + opt.set_int("hires.target_width", hires_width); + if (!adetail) + opt.set_int("hires.target_height", hires_height); + if (!adetail) + opt.set_int("hires.steps", hires_steps); + if (!adetail) + opt.set_float("hires.denoising_strength", hires_denoising_strength); + if (!adetail) + opt.set_int("hires.upscale_tile_size", hires_upscale_tile_size); + if (circular) + opt.set_bool("circular_x", true); + else + opt.set_bool("circular_x", circular_x); + if (circular) + opt.set_bool("circular_y", true); + else + opt.set_bool("circular_y", circular_y); + opt.set_loras("loras", lora_vec.data(), lora_vec.size()); + if (!adetail) + opt.set_images("ref_images", ref_image_views.data(), ref_image_views.size()); + if (!adetail) + opt.set_images("photomaker.id_images", pm_id_image_views.data(), pm_id_image_views.size()); + if (!adetail) + opt.set_string("photomaker.id_embed_path", pm_id_embed_path); + if (!adetail) + opt.set_float("photomaker.style_strength", pm_style_strength); + if (!adetail) + opt.set_string("pulid.id_embedding_path", pulid_id_embedding_path); + if (!adetail) + opt.set_float("pulid.id_weight", pulid_id_weight); + if (!adetail) + opt.set_floats("hires.sigmas", hires_custom_sigmas); + if (!adetail) + opt.set_string("image_preprocess.rules", image_preprocess); + set_sample_options(opt, "sample.", sample_params, skip_layers, custom_sigmas, extra_sample_args); + set_cache_options(opt, cache_params); + set_tiling_options(opt, vae_tiling_params); + if (adetail) { + if (!ad_prompt.empty()) + opt.set_string("adetail.prompt", ad_prompt); + if (!ad_negative_prompt.empty()) + opt.set_string("adetail.negative_prompt", ad_negative_prompt); + opt.set_string("adetail.args", extra_ad_args); + } + return opt.finish(); } -sd_vid_gen_params_t SDGenerationParams::to_sd_vid_gen_params_t() { - sd_vid_gen_params_t params; - sd_vid_gen_params_init(¶ms); - +SDOptPtr SDGenerationParams::to_video_opt() { lora_vec.clear(); lora_vec.reserve(lora_map.size() + high_noise_lora_map.size()); for (const auto& kv : lora_map) { @@ -2963,7 +2856,7 @@ sd_vid_gen_params_t SDGenerationParams::to_sd_vid_gen_params_t() { for (auto& frame : ref_videos[i]) { frame_views.push_back(frame.get()); } - sd_audio_t audio = i < ref_video_audios.size() ? ref_video_audios[i].get() : sd_audio_t{}; + SDAudio audio = i < ref_video_audios.size() ? ref_video_audios[i].get() : SDAudio{}; ref_video_views.push_back({frame_views.empty() ? nullptr : frame_views.data(), static_cast(frame_views.size()), 24, @@ -2976,65 +2869,61 @@ sd_vid_gen_params_t SDGenerationParams::to_sd_vid_gen_params_t() { ref_audio_views.push_back(audio.get()); } - sample_params.guidance.slg.layers = skip_layers.empty() ? nullptr : skip_layers.data(); - sample_params.guidance.slg.layer_count = skip_layers.size(); - high_noise_sample_params.guidance.slg.layers = high_noise_skip_layers.empty() ? nullptr : high_noise_skip_layers.data(); - high_noise_sample_params.guidance.slg.layer_count = high_noise_skip_layers.size(); - sample_params.custom_sigmas = custom_sigmas.empty() ? nullptr : custom_sigmas.data(); - sample_params.custom_sigmas_count = static_cast(custom_sigmas.size()); - sample_params.extra_sample_args = extra_sample_args.empty() ? nullptr : extra_sample_args.c_str(); - high_noise_sample_params.extra_sample_args = high_noise_extra_sample_args.empty() ? nullptr : high_noise_extra_sample_args.c_str(); - vae_tiling_params.extra_tiling_args = extra_tiling_args.empty() ? nullptr : extra_tiling_args.c_str(); - cache_params.scm_mask = scm_mask.empty() ? nullptr : scm_mask.c_str(); - - params.loras = lora_vec.empty() ? nullptr : lora_vec.data(); - params.lora_count = static_cast(lora_vec.size()); - params.prompt = prompt.c_str(); - params.negative_prompt = negative_prompt.c_str(); - params.clip_skip = clip_skip; - params.init_image = init_image.get(); - params.end_image = end_image.get(); - params.ref_images = ref_image_views.empty() ? nullptr : ref_image_views.data(); - params.ref_images_count = static_cast(ref_image_views.size()); - params.ref_videos = ref_video_views.empty() ? nullptr : ref_video_views.data(); - params.ref_videos_count = static_cast(ref_video_views.size()); - params.ref_audios = ref_audio_views.empty() ? nullptr : ref_audio_views.data(); - params.ref_audios_count = static_cast(ref_audio_views.size()); - params.control_frames = control_frame_views.empty() ? nullptr : control_frame_views.data(); - params.control_frames_size = static_cast(control_frame_views.size()); - params.width = get_resolved_width(); - params.height = get_resolved_height(); - params.sample_params = sample_params; - params.high_noise_sample_params = high_noise_sample_params; - params.moe_boundary = moe_boundary; - params.strength = strength; - params.seed = seed; - params.video_frames = video_frames; - params.fps = fps; - params.vace_strength = vace_strength; - params.vae_tiling_params = vae_tiling_params; - params.cache = cache_params; - params.hires.enabled = hires_enabled; - params.hires.upscaler = resolved_hires_upscaler; - params.hires.model_path = hires_upscaler_model_path.empty() ? nullptr : hires_upscaler_model_path.c_str(); - params.hires.scale = hires_scale; - params.hires.target_width = hires_width; - params.hires.target_height = hires_height; - params.hires.steps = hires_steps; - params.hires.denoising_strength = hires_denoising_strength; - params.hires.upscale_tile_size = hires_upscale_tile_size; - params.hires.custom_sigmas = hires_custom_sigmas.empty() ? nullptr : hires_custom_sigmas.data(); - params.hires.custom_sigmas_count = static_cast(hires_custom_sigmas.size()); - params.circular_x = circular || circular_x; - params.circular_y = circular || circular_y; - params.image_preprocess = {image_preprocess.c_str()}; - params.ref_image_args = ref_image_args.c_str(); - return params; + vae_tiling_params.extra_tiling_args = extra_tiling_args.empty() ? nullptr : extra_tiling_args.c_str(); + cache_params.scm_mask = scm_mask.empty() ? nullptr : scm_mask.c_str(); + + SDOptBuilder opt(sd_vid_gen_opt_create()); + opt.set_string("prompt", prompt); + opt.set_string("negative_prompt", negative_prompt); + opt.set_int("clip_skip", clip_skip); + opt.set_image("init_image", init_image.get()); + opt.set_image("end_image", end_image.get()); + opt.set_int("width", width); + opt.set_int("height", height); + opt.set_float("moe_boundary", moe_boundary); + opt.set_float("strength", strength); + opt.set_int("seed", seed); + opt.set_int("video_frames", video_frames); + opt.set_int("fps", fps); + opt.set_float("vace_strength", vace_strength); + opt.set_bool("hires.enabled", hires_enabled); + opt.set_string("hires.upscaler", resolved_hires_upscaler.empty() ? hires_upscaler : std::optional(resolved_hires_upscaler)); + opt.set_string("hires.model_path", hires_upscaler_model_path); + opt.set_float("hires.scale", hires_scale); + opt.set_int("hires.target_width", hires_width); + opt.set_int("hires.target_height", hires_height); + opt.set_int("hires.steps", hires_steps); + opt.set_float("hires.denoising_strength", hires_denoising_strength); + opt.set_int("hires.upscale_tile_size", hires_upscale_tile_size); + if (circular) + opt.set_bool("circular_x", true); + else + opt.set_bool("circular_x", circular_x); + if (circular) + opt.set_bool("circular_y", true); + else + opt.set_bool("circular_y", circular_y); + opt.set_string("ref_image_args", ref_image_args); + opt.set_loras("loras", lora_vec.data(), lora_vec.size()); + opt.set_images("ref_images", ref_image_views.data(), ref_image_views.size()); + opt.set_videos("ref_videos", ref_video_views.data(), ref_video_views.size()); + opt.set_audios("ref_audios", ref_audio_views.data(), ref_audio_views.size()); + opt.set_frames("control_frames", control_frame_views.data(), control_frame_views.size()); + opt.set_floats("hires.sigmas", hires_custom_sigmas); + opt.set_string("image_preprocess.rules", image_preprocess); + opt.set_string("sample.scheduler", high_noise_sample_params.scheduler); + opt.set_float("sample.flow_shift", high_noise_sample_params.flow_shift); + opt.set_int("sample.shifted_timestep", high_noise_sample_params.shifted_timestep); + set_sample_options(opt, "sample.", sample_params, skip_layers, custom_sigmas, extra_sample_args); + set_sample_options(opt, "high_noise_sample.", high_noise_sample_params, high_noise_skip_layers, {}, high_noise_extra_sample_args); + set_cache_options(opt, cache_params); + set_tiling_options(opt, vae_tiling_params); + return opt.finish(); } std::string SDGenerationParams::to_string() const { - FreeUniquePtr sample_params_str(sd_sample_params_to_str(&sample_params)); - FreeUniquePtr high_noise_sample_params_str(sd_sample_params_to_str(&high_noise_sample_params)); + const std::string sample_params_str = sample_params.sample_method.value_or("unset") + ", " + sample_params.scheduler.value_or("unset") + ", steps=" + (sample_params.sample_steps ? std::to_string(*sample_params.sample_steps) : "unset"); + const std::string high_noise_sample_params_str = high_noise_sample_params.sample_method.value_or("unset") + ", " + high_noise_sample_params.scheduler.value_or("unset") + ", steps=" + (high_noise_sample_params.sample_steps ? std::to_string(*high_noise_sample_params.sample_steps) : "unset"); std::ostringstream lora_ss; lora_ss << "{\n"; @@ -3091,15 +2980,15 @@ std::string SDGenerationParams::to_string() const { << " pm_id_images_dir: \"" << pm_id_images_dir << "\",\n" << " pm_id_embed_path: \"" << pm_id_embed_path << "\",\n" << " pm_style_strength: " << pm_style_strength << ",\n" - << " skip_layers: " << vec_to_string(skip_layers) << ",\n" - << " sample_params: " << SAFE_STR(sample_params_str.get()) << ",\n" - << " high_noise_skip_layers: " << vec_to_string(high_noise_skip_layers) << ",\n" - << " high_noise_sample_params: " << SAFE_STR(high_noise_sample_params_str.get()) << ",\n" - << " custom_sigmas: " << vec_to_string(custom_sigmas) << ",\n" + << " skip_layers: " << (skip_layers ? vec_to_string(*skip_layers) : "unset") << ",\n" + << " sample_params: " << SAFE_STR(sample_params_str.c_str()) << ",\n" + << " high_noise_skip_layers: " << (high_noise_skip_layers ? vec_to_string(*high_noise_skip_layers) : "unset") << ",\n" + << " high_noise_sample_params: " << SAFE_STR(high_noise_sample_params_str.c_str()) << ",\n" + << " custom_sigmas: " << (custom_sigmas ? vec_to_string(*custom_sigmas) : "unset") << ",\n" << " cache_mode: \"" << cache_mode << "\",\n" << " cache_option: \"" << cache_option << "\",\n" << " cache: " - << (cache_params.mode != SD_CACHE_DISABLED ? "enabled" : "disabled") + << cache_params.mode << " (threshold=" << cache_params.reuse_threshold << ", start=" << cache_params.start_percent << ", end=" << cache_params.end_percent << "),\n" @@ -3113,7 +3002,7 @@ std::string SDGenerationParams::to_string() const { << " seed: " << seed << ",\n" << " upscale_repeats: " << upscale_repeats << ",\n" << " upscale_tile_size: " << upscale_tile_size << ",\n" - << " hires: { enabled: " << (hires_enabled ? "true" : "false") + << " hires: { enabled: " << hires_enabled << ", upscaler: \"" << hires_upscaler << "\"" << ", model_path: \"" << hires_upscaler_model_path << "\"" << ", scale: " << hires_scale @@ -3121,7 +3010,7 @@ std::string SDGenerationParams::to_string() const { << ", target_height: " << hires_height << ", steps: " << hires_steps << ", denoising_strength: " << hires_denoising_strength - << ", custom_sigmas: " << vec_to_string(hires_custom_sigmas) + << ", custom_sigmas: " << (hires_custom_sigmas ? vec_to_string(*hires_custom_sigmas) : "unset") << ", upscale_tile_size: " << hires_upscale_tile_size << " },\n" << " vae_tiling_params: { " << vae_tiling_params.enabled << ", " diff --git a/examples/common/common.h b/examples/common/common.h index d4df4fad61..eb4a043a7f 100644 --- a/examples/common/common.h +++ b/examples/common/common.h @@ -2,6 +2,7 @@ #define __EXAMPLES_COMMON_COMMON_H__ #include "generation_metadata.h" +#include "options.h" #include #include @@ -40,21 +41,21 @@ struct StringOption { std::string long_name; std::string desc; int concat; - std::string* target; + OptionTarget target; }; struct IntOption { std::string short_name; std::string long_name; std::string desc; - int* target; + OptionTarget target; }; struct FloatOption { std::string short_name; std::string long_name; std::string desc; - float* target; + OptionTarget target; }; struct BoolOption { @@ -62,7 +63,7 @@ struct BoolOption { std::string long_name; std::string desc; bool keep_true; - bool* target; + OptionTarget target; }; struct ManualFunction { @@ -123,8 +124,8 @@ bool decode_base64_image(const std::string& encoded_input, SDImageOwner& out_image); struct SDContextParams { - int n_threads = -1; - int conditioning_cache_size = 4; + std::optional n_threads; + std::optional conditioning_cache_size; std::string model_path; std::string clip_l_path; std::string clip_g_path; @@ -138,7 +139,7 @@ struct SDContextParams { std::string uncond_diffusion_model_path; std::string embeddings_connectors_path; std::string vae_path; - std::string vae_format = "auto"; + std::optional vae_format; std::string audio_vae_path; std::string audio_encoder_path; std::string taesd_path; @@ -157,42 +158,41 @@ struct SDContextParams { std::map embedding_map; std::vector embedding_vec; - rng_type_t rng_type = CUDA_RNG; - rng_type_t sampler_rng_type = RNG_TYPE_COUNT; - bool offload_params_to_cpu = false; - std::string max_vram = "0"; - bool disable_prefetch = false; - bool disable_segmented_compute = false; - bool eager_load = false; - bool batched_cfg = true; - std::string backend; - std::string params_backend; - std::string split_mode; - std::string model_args; - bool auto_fit = true; - std::string rpc_servers; + std::optional rng_type; + std::optional sampler_rng_type; + bool offload_params_to_cpu = false; + std::optional max_vram; + std::optional disable_prefetch; + std::optional disable_segmented_compute; + std::optional eager_load; + std::optional batched_cfg; + std::optional backend; + std::optional params_backend; + std::optional split_mode; + std::optional model_args; + std::optional auto_fit; + std::optional rpc_servers; bool list_devices = false; std::string effective_backend; std::string effective_params_backend; - bool enable_mmap = false; - bool control_net_cpu = false; - bool clip_on_cpu = false; - bool vae_on_cpu = false; - std::string flash_attn_mode = "auto"; - std::string conv_direct_mode; // Empty selects auto while honoring legacy convolution flags. - bool sage_attn = false; - bool diffusion_conv_direct = false; - bool vae_conv_direct = false; - bool taesd_preview_only = false; - - prediction_t prediction = PREDICTION_COUNT; - lora_apply_mode_t lora_apply_mode = LORA_APPLY_AUTO; - - bool force_sdxl_vae_conv_scale = false; - float linear_scale = 0.f; - float attn_scale = 0.f; - - float flow_shift = INFINITY; + std::optional enable_mmap; + bool control_net_cpu = false; + bool clip_on_cpu = false; + bool vae_on_cpu = false; + std::optional flash_attn_mode; + std::optional conv_direct_mode; + std::optional sage_attn; + std::optional diffusion_conv_direct; + std::optional vae_conv_direct; + std::optional taesd_preview_only; + + std::optional prediction; + std::optional lora_apply_mode; + + std::optional force_sdxl_vae_conv_scale; + std::optional linear_scale; + std::optional attn_scale; + ArgOptions get_options(); bool print_devices() const; void build_embedding_map(); @@ -201,7 +201,7 @@ struct SDContextParams { bool validate(SDMode mode); bool resolve_and_validate(SDMode mode); std::string to_string() const; - sd_ctx_params_t to_sd_ctx_params_t(); + SDOptPtr to_context_opt(); }; struct SDGenerationParams { @@ -212,15 +212,15 @@ struct SDGenerationParams { std::string ad_prompt; std::string ad_negative_prompt; std::string extra_ad_args; - int clip_skip = -1; // <= 0 represents unspecified - int width = -1; - int height = -1; - int batch_count = 1; - int qwen_image_layers = 3; - int64_t seed = 42; - float strength = 0.75f; - float control_strength = 0.9f; - float ip_adapter_strength = 1.0f; + std::optional clip_skip; + std::optional width; + std::optional height; + std::optional batch_count; + std::optional qwen_image_layers; + std::optional seed; + std::optional strength; + std::optional control_strength; + std::optional ip_adapter_strength; bool increase_ref_index = false; bool embed_image_metadata = true; @@ -228,7 +228,7 @@ struct SDGenerationParams { std::string end_image_path; std::string mask_image_path; std::string control_image_path; - std::string control_type; + std::optional control_type; std::string ip_adapter_image_path; std::vector ref_image_paths; std::vector ref_video_paths; @@ -236,26 +236,26 @@ struct SDGenerationParams { std::vector ref_audio_paths; std::string control_video_path; - sd_sample_params_t sample_params; - sd_sample_params_t high_noise_sample_params; + SDSampleParams sample_params; + SDSampleParams high_noise_sample_params; std::string extra_sample_args; std::string high_noise_extra_sample_args; - std::vector skip_layers = {7, 8, 9}; - std::vector high_noise_skip_layers = {7, 8, 9}; + std::optional> skip_layers; + std::optional> high_noise_skip_layers; - std::vector custom_sigmas; + std::optional> custom_sigmas; std::string cache_mode; std::string cache_option; std::string scm_mask; - bool scm_policy_dynamic = true; - sd_cache_params_t cache_params{}; - - float moe_boundary = 0.875f; - int video_frames = 1; - int fps = 16; - float vace_strength = 1.f; - sd_tiling_params_t vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr}; + std::optional scm_policy_dynamic; + SDCacheParams cache_params{}; + + std::optional moe_boundary; + std::optional video_frames; + std::optional fps; + std::optional vace_strength; + SDTilingParams vae_tiling_params; std::string extra_tiling_args; std::string ref_image_args; @@ -263,28 +263,28 @@ struct SDGenerationParams { std::string pm_id_images_dir; std::string pm_id_embed_path; - float pm_style_strength = 20.f; + std::optional pm_style_strength; std::string pulid_id_embedding_path; - float pulid_id_weight = 1.0f; + std::optional pulid_id_weight; - int upscale_repeats = 1; - int upscale_tile_size = 128; + int upscale_repeats = 1; + std::optional upscale_tile_size; - bool circular = false; - bool circular_x = false; - bool circular_y = false; + bool circular = false; + std::optional circular_x; + std::optional circular_y; - bool hires_enabled = false; - std::string hires_upscaler = "Latent"; + std::optional hires_enabled; + std::optional hires_upscaler; std::string hires_upscaler_model_path; - float hires_scale = 2.f; - int hires_width = 0; - int hires_height = 0; - int hires_steps = 0; - float hires_denoising_strength = 0.7f; - int hires_upscale_tile_size = 128; - std::vector hires_custom_sigmas; + std::optional hires_scale; + std::optional hires_width; + std::optional hires_height; + std::optional hires_steps; + std::optional hires_denoising_strength; + std::optional hires_upscale_tile_size; + std::optional> hires_custom_sigmas; std::map lora_map; std::map high_noise_lora_map; @@ -292,13 +292,13 @@ struct SDGenerationParams { // Derived and normalized fields. std::string prompt_with_lora; // for metadata record only std::vector lora_vec; - sd_hires_upscaler_t resolved_hires_upscaler; - sd_control_type_t resolved_control_type = SD_CONTROL_TYPE_NONE; + std::string resolved_hires_upscaler; // Owned execution payload. SDImageOwner init_image; SDImageOwner end_image; std::vector ref_images; + std::map prompt_images; std::vector> ref_videos; std::vector ref_video_audios; std::vector ref_audios; @@ -308,13 +308,13 @@ struct SDGenerationParams { std::vector pm_id_images; std::vector control_frames; - // Backing storage for sd_img_gen_params_t view fields. - std::vector ref_image_views; - std::vector> ref_video_frame_views; - std::vector ref_video_views; - std::vector ref_audio_views; - std::vector pm_id_image_views; - std::vector control_frame_views; + // Media views used while populating the opaque options. + std::vector ref_image_views; + std::vector> ref_video_frame_views; + std::vector ref_video_views; + std::vector ref_audio_views; + std::vector pm_id_image_views; + std::vector control_frame_views; SDGenerationParams(); SDGenerationParams(const SDGenerationParams& other) = default; @@ -325,20 +325,21 @@ struct SDGenerationParams { bool from_json_str(const std::string& json_str, const std::function& lora_path_resolver = {}); bool parse_image_preprocess_json(const std::string& json_str); + bool parse_prompt_images_json(const std::string& json_str); bool initialize_cache_params(); bool extract_and_remove_lora(const std::string& lora_model_dir); bool width_and_height_are_set() const; void set_width_and_height_if_unset(int w, int h); - int get_resolved_width() const; - int get_resolved_height() const; + int get_requested_width() const; + int get_requested_height() const; bool resolve(const std::string& lora_model_dir, const std::string& hires_upscalers_dir, bool strict = false, bool validate_missing_loras = false); bool validate(SDMode mode); bool resolve_and_validate(SDMode mode, const std::string& lora_model_dir, const std::string& hires_upscalers_dir, bool strict = false); - sd_img_gen_params_t to_sd_img_gen_params_t(); - sd_vid_gen_params_t to_sd_vid_gen_params_t(); + SDOptPtr to_image_opt(bool adetail = false); + SDOptPtr to_video_opt(); std::string to_string() const; }; diff --git a/examples/common/media_io.cpp b/examples/common/media_io.cpp index e878c889e6..4159e46695 100644 --- a/examples/common/media_io.cpp +++ b/examples/common/media_io.cpp @@ -409,7 +409,7 @@ bool extract_vp8_frame_from_webp(const std::vector& webp_data, std::vec return false; } -bool encode_sd_image_to_vp8_frame(const sd_image_t& image, int quality, std::vector& vp8_frame) { +bool encode_sd_image_to_vp8_frame(const SDImage& image, int quality, std::vector& vp8_frame) { if (image.data == nullptr || image.width == 0 || image.height == 0) { return false; } @@ -659,7 +659,7 @@ void write_fourcc(std::vector& data, const char* fourcc) { data.insert(data.end(), fourcc, fourcc + 4); } -static std::vector audio_to_pcm16_bytes(const sd_audio_t* audio) { +static std::vector audio_to_pcm16_bytes(const SDAudio* audio) { if (audio == nullptr || audio->data == nullptr || audio->sample_count == 0 || audio->channels == 0 || audio->sample_rate == 0) { return {}; } @@ -674,7 +674,7 @@ static std::vector audio_to_pcm16_bytes(const sd_audio_t* audio) { return bytes; } -static std::pair audio_sample_range_for_video_frame(const sd_audio_t* audio, int frame_idx, int num_frames, int fps) { +static std::pair audio_sample_range_for_video_frame(const SDAudio* audio, int frame_idx, int num_frames, int fps) { if (audio == nullptr || fps <= 0 || num_frames <= 0) { return {0, 0}; } @@ -783,7 +783,7 @@ uint8_t* load_image_from_file(const char* image_path, return load_image_common(false, image_path, 0, width, height, expected_width, expected_height, expected_channel, out_channel); } -bool load_sd_image_from_file(sd_image_t* image, +bool load_sd_image_from_file(SDImage* image, const char* image_path, int expected_width, int expected_height, @@ -836,7 +836,7 @@ static void append_avi_metadata(std::vector& data, const std::string& p size_t start_pos = data.size(); } -std::vector create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) { +std::vector create_mjpg_avi_from_sd_images_to_vector(SDImage* images, int num_images, int fps, int quality, const SDAudio* audio, const std::string& parameters) { if (num_images == 0) { fprintf(stderr, "Error: Image array is empty.\n"); return {}; @@ -1043,7 +1043,7 @@ std::vector create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images return avi_data; } -int create_mjpg_avi_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) { +int create_mjpg_avi_from_sd_images(const char* filename, SDImage* images, int num_images, int fps, int quality, const SDAudio* audio, const std::string& parameters) { std::vector avi_data = create_mjpg_avi_from_sd_images_to_vector(images, num_images, fps, quality, audio, parameters); if (avi_data.empty()) { return -1; @@ -1056,7 +1056,7 @@ int create_mjpg_avi_from_sd_images(const char* filename, sd_image_t* images, int } #ifdef SD_USE_WEBP -std::vector create_animated_webp_from_sd_images_to_vector(sd_image_t* images, int num_images, int fps, int quality) { +std::vector create_animated_webp_from_sd_images_to_vector(SDImage* images, int num_images, int fps, int quality) { if (num_images == 0) { fprintf(stderr, "Error: Image array is empty.\n"); return {}; @@ -1102,7 +1102,7 @@ std::vector create_animated_webp_from_sd_images_to_vector(sd_image_t* i int timestamp_ms = 0; for (int i = 0; i < num_images; ++i) { - const sd_image_t& image = images[i]; + const SDImage& image = images[i]; if (static_cast(image.width) != width || static_cast(image.height) != height) { fprintf(stderr, "Error: Frame dimensions do not match.\n"); return {}; @@ -1160,7 +1160,7 @@ std::vector create_animated_webp_from_sd_images_to_vector(sd_image_t* i return std::vector(webp_data.data.bytes, webp_data.data.bytes + webp_data.data.size); } -int create_animated_webp_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality) { +int create_animated_webp_from_sd_images(const char* filename, SDImage* images, int num_images, int fps, int quality) { std::vector webp_data = create_animated_webp_from_sd_images_to_vector(images, num_images, fps, quality); if (webp_data.empty()) { return -1; @@ -1174,7 +1174,7 @@ int create_animated_webp_from_sd_images(const char* filename, sd_image_t* images #endif #ifdef SD_USE_WEBM -std::vector create_webm_from_sd_images_to_vector(sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) { +std::vector create_webm_from_sd_images_to_vector(SDImage* images, int num_images, int fps, int quality, const SDAudio* audio, const std::string& parameters) { if (num_images == 0) { fprintf(stderr, "Error: Image array is empty.\n"); return {}; @@ -1261,7 +1261,7 @@ std::vector create_webm_from_sd_images_to_vector(sd_image_t* images, in uint64_t timestamp_ns = 0; for (int i = 0; i < num_images; ++i) { - const sd_image_t& image = images[i]; + const SDImage& image = images[i]; if (static_cast(image.width) != width || static_cast(image.height) != height) { fprintf(stderr, "Error: Frame dimensions do not match.\n"); return -1; @@ -1314,7 +1314,7 @@ std::vector create_webm_from_sd_images_to_vector(sd_image_t* images, in return writer.data(); } -int create_webm_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) { +int create_webm_from_sd_images(const char* filename, SDImage* images, int num_images, int fps, int quality, const SDAudio* audio, const std::string& parameters) { std::vector webm_data = create_webm_from_sd_images_to_vector(images, num_images, fps, quality, audio, parameters); if (webm_data.empty()) { return -1; @@ -1328,11 +1328,11 @@ int create_webm_from_sd_images(const char* filename, sd_image_t* images, int num #endif std::vector create_video_from_sd_images_to_vector(const std::string& output_format, - sd_image_t* images, + SDImage* images, int num_images, int fps, int quality, - const sd_audio_t* audio, + const SDAudio* audio, const std::string& parameters) { std::string format = output_format; std::transform(format.begin(), format.end(), format.begin(), @@ -1359,7 +1359,7 @@ std::vector create_video_from_sd_images_to_vector(const std::string& ou return create_mjpg_avi_from_sd_images_to_vector(images, num_images, fps, quality, audio, parameters); } -int create_video_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) { +int create_video_from_sd_images(const char* filename, SDImage* images, int num_images, int fps, int quality, const SDAudio* audio, const std::string& parameters) { std::string path = filename ? filename : ""; auto pos = path.find_last_of('.'); std::string ext = pos == std::string::npos ? "" : path.substr(pos); diff --git a/examples/common/media_io.h b/examples/common/media_io.h index 7feb0ada25..d765db7fcc 100644 --- a/examples/common/media_io.h +++ b/examples/common/media_io.h @@ -5,7 +5,7 @@ #include #include -#include "stable-diffusion.h" +#include "resource_owners.hpp" enum class EncodedImageFormat { JPEG, @@ -43,7 +43,7 @@ uint8_t* load_image_from_file(const char* image_path, int expected_height = 0, int expected_channel = 3); -bool load_sd_image_from_file(sd_image_t* image, +bool load_sd_image_from_file(SDImage* image, const char* image_path, int expected_width = 0, int expected_height = 0, @@ -59,26 +59,26 @@ uint8_t* load_image_from_memory(const char* image_bytes, int expected_channel = 3); int create_mjpg_avi_from_sd_images(const char* filename, - sd_image_t* images, + SDImage* images, int num_images, int fps, int quality = 90, - const sd_audio_t* audio = nullptr, + const SDAudio* audio = nullptr, const std::string& parameters = ""); -std::vector create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images, +std::vector create_mjpg_avi_from_sd_images_to_vector(SDImage* images, int num_images, int fps, int quality = 90, - const sd_audio_t* audio = nullptr, + const SDAudio* audio = nullptr, const std::string& parameters = ""); #ifdef SD_USE_WEBP int create_animated_webp_from_sd_images(const char* filename, - sd_image_t* images, + SDImage* images, int num_images, int fps, int quality = 90); -std::vector create_animated_webp_from_sd_images_to_vector(sd_image_t* images, +std::vector create_animated_webp_from_sd_images_to_vector(SDImage* images, int num_images, int fps, int quality = 90); @@ -86,33 +86,33 @@ std::vector create_animated_webp_from_sd_images_to_vector(sd_image_t* i #ifdef SD_USE_WEBM int create_webm_from_sd_images(const char* filename, - sd_image_t* images, + SDImage* images, int num_images, int fps, int quality = 90, - const sd_audio_t* audio = nullptr, + const SDAudio* audio = nullptr, const std::string& parameters = ""); -std::vector create_webm_from_sd_images_to_vector(sd_image_t* images, +std::vector create_webm_from_sd_images_to_vector(SDImage* images, int num_images, int fps, int quality = 90, - const sd_audio_t* audio = nullptr, + const SDAudio* audio = nullptr, const std::string& parameters = ""); #endif int create_video_from_sd_images(const char* filename, - sd_image_t* images, + SDImage* images, int num_images, int fps, int quality = 90, - const sd_audio_t* audio = nullptr, + const SDAudio* audio = nullptr, const std::string& parameters = ""); std::vector create_video_from_sd_images_to_vector(const std::string& output_format, - sd_image_t* images, + SDImage* images, int num_images, int fps, int quality = 90, - const sd_audio_t* audio = nullptr, + const SDAudio* audio = nullptr, const std::string& parameters = ""); bool write_wav_to_file(const std::string& path, diff --git a/examples/common/options.h b/examples/common/options.h new file mode 100644 index 0000000000..3b860328f8 --- /dev/null +++ b/examples/common/options.h @@ -0,0 +1,185 @@ +#ifndef SD_EXAMPLES_OPTIONS_H +#define SD_EXAMPLES_OPTIONS_H + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace nlohmann { + template + struct adl_serializer> { + static void to_json(json& out, const std::optional& value) { + out = value ? json(*value) : json(nullptr); + } + static void from_json(const json& in, std::optional& value) { + value = in.is_null() ? std::nullopt : std::optional(in.get()); + } + }; +} + +template +std::ostream& operator<<(std::ostream& out, const std::optional& value) { + if (!value) + return out << "unset"; + if constexpr (std::is_same_v) + return out << (*value ? "true" : "false"); + else + return out << *value; +} + +template +class OptionTarget { + T* value_ = nullptr; + std::optional* input_ = nullptr; + +public: + OptionTarget(T* value) + : value_(value) {} + OptionTarget(std::optional* input) + : input_(input) {} + T get() const { return input_ ? input_->value_or(T{}) : *value_; } + void set(T value) const { + if (input_) + *input_ = std::move(value); + else + *value_ = std::move(value); + } +}; + +template +struct InputType { + using type = T; +}; +template +struct InputType> { + using type = T; +}; + +struct SDTilingParams { + std::optional enabled; + std::optional temporal_tiling; + std::optional tile_size_w; + std::optional tile_size_h; + std::optional target_overlap; + std::optional rel_size_w; + std::optional rel_size_h; + const char* extra_tiling_args = nullptr; +}; + +struct SDSLGParams { + std::optional layer_start; + std::optional layer_end; + std::optional scale; +}; + +struct SDGuidanceParams { + std::optional txt_cfg; + std::optional img_cfg; + std::optional distilled_guidance; + SDSLGParams slg; +}; + +struct SDSampleParams { + SDGuidanceParams guidance; + std::optional scheduler; + std::optional sample_method; + std::optional sample_steps; + std::optional eta; + std::optional shifted_timestep; + std::optional flow_shift; +}; + +struct SDCacheParams { + std::optional mode; + std::optional reuse_threshold; + std::optional start_percent; + std::optional end_percent; + std::optional error_decay_rate; + std::optional use_relative_threshold; + std::optional reset_error_on_compute; + std::optional Fn_compute_blocks; + std::optional Bn_compute_blocks; + std::optional residual_diff_threshold; + std::optional max_warmup_steps; + std::optional max_cached_steps; + std::optional max_continuous_cached_steps; + std::optional taylorseer_n_derivatives; + std::optional taylorseer_skip_interval; + const char* scm_mask = nullptr; + std::optional scm_policy_dynamic; + std::optional spectrum_w; + std::optional spectrum_m; + std::optional spectrum_lam; + std::optional spectrum_window_size; + std::optional spectrum_flex_window; + std::optional spectrum_warmup_steps; + std::optional spectrum_stop_percent; +}; + +namespace example_options { + inline constexpr const char* sample_methods[] = { + "euler", + "euler_a", + "heun", + "dpm2", + "dpm++2s_a", + "dpm++2m", + "dpm++2mv2", + "ipndm", + "ipndm_v", + "lcm", + "ddim_trailing", + "tcd", + "res_multistep", + "res_2s", + "er_sde", + "euler_cfg_pp", + "euler_a_cfg_pp", + "euler_ge", + "dpm++2m_sde", + "dpm++2m_sde_bt", + "lms", + }; + + inline constexpr const char* schedulers[] = { + "discrete", + "karras", + "exponential", + "ays", + "gits", + "sgm_uniform", + "simple", + "smoothstep", + "kl_optimal", + "lcm", + "bong_tangent", + "ltx2", + "logit_normal", + "flux2", + "flux", + "beta", + "llada_image", + }; + + inline constexpr const char* preview_modes[] = { + "none", + "proj", + "tae", + "vae", + }; + + template + bool contains(const char* const (&values)[N], const std::string& value) { + for (const char* entry : values) + if (value == entry) + return true; + return false; + } +} + +#endif diff --git a/examples/common/resource_owners.hpp b/examples/common/resource_owners.hpp index 8a8f843146..8ba69e7a9c 100644 --- a/examples/common/resource_owners.hpp +++ b/examples/common/resource_owners.hpp @@ -4,13 +4,136 @@ #include #include #include +#include #include +#include +#include #include #include +#include #include "generation_metadata.h" +#include "log.h" #include "stable-diffusion.h" +struct SDImage { + uint32_t width; + uint32_t height; + uint32_t channel; + uint8_t* data; + sd_tensor_t* storage = nullptr; +}; + +struct SDAudio { + uint32_t sample_rate; + uint32_t channels; + uint64_t sample_count; + float* data; + sd_tensor_t* storage = nullptr; +}; + +struct SDReferenceVideo { + SDImage* frames; + int frame_count; + int fps; + SDAudio audio; +}; + +struct SDTensorDeleter { + void operator()(sd_tensor_t* tensor) const { sd_tensor_free(tensor); } +}; +using SDTensorPtr = std::unique_ptr; + +static inline SDImage tensor_image_view(const sd_tensor_t* tensor) { + if (!tensor) + return {}; + return {uint32_t(sd_tensor_dim(tensor, 1)), uint32_t(sd_tensor_dim(tensor, 2)), uint32_t(sd_tensor_dim(tensor, 0)), + const_cast(static_cast(sd_tensor_data(tensor)))}; +} + +static inline void sd_image_free(SDImage* image) { + if (!image) + return; + if (image->storage) + sd_tensor_free(image->storage); + else + free(image->data); + *image = {}; +} + +static inline void free_sd_audio(SDAudio* audio) { + if (!audio) + return; + if (audio->storage) + sd_tensor_free(audio->storage); + else + free(audio->data); + free(audio); +} + +static inline SDTensorPtr image_tensor(const SDImage& image) { + if (!image.data) + return {}; + if (image.storage && sd_tensor_data(image.storage) == image.data && + sd_tensor_dim(image.storage, 0) == image.channel && sd_tensor_dim(image.storage, 1) == image.width && sd_tensor_dim(image.storage, 2) == image.height) + return SDTensorPtr(sd_tensor_retain(image.storage)); + const int64_t shape[] = {image.channel, image.width, image.height}; + return SDTensorPtr(sd_tensor_create(SD_TENSOR_U8, 3, shape, image.data, size_t(image.width) * image.height * image.channel)); +} + +static inline SDTensorPtr audio_tensor(const SDAudio& audio) { + if (!audio.data) + return {}; + if (audio.storage) + return SDTensorPtr(sd_tensor_retain(audio.storage)); + const int64_t shape[] = {audio.channels, static_cast(audio.sample_count)}; + return SDTensorPtr(sd_tensor_create(SD_TENSOR_F32, 2, shape, audio.data, size_t(audio.sample_count) * audio.channels * sizeof(float))); +} + +static inline SDTensorPtr frames_tensor(const SDImage* frames, size_t count) { + if (!frames || !count) + return {}; + const auto& first = frames[0]; + const size_t frame_bytes = size_t(first.width) * first.height * first.channel; + if (!frame_bytes || count > SIZE_MAX / frame_bytes) { + LOG_ERROR("invalid video frame storage size"); + return {}; + } + std::vector pixels(frame_bytes * count); + for (size_t i = 0; i < count; ++i) { + if (!frames[i].data || frames[i].width != first.width || frames[i].height != first.height || frames[i].channel != first.channel) { + LOG_ERROR("video frames must have matching dimensions and channels"); + return {}; + } + memcpy(pixels.data() + i * frame_bytes, frames[i].data, frame_bytes); + } + const int64_t shape[] = {first.channel, first.width, first.height, static_cast(count)}; + return SDTensorPtr(sd_tensor_create(SD_TENSOR_U8, 4, shape, pixels.data(), pixels.size())); +} + +static inline SDAudio* retain_result_audio(const sd_result_t* result) { + const auto* audio = sd_result_get_audio(result); + if (!audio) + return nullptr; + auto* view = static_cast(malloc(sizeof(SDAudio))); + if (!view) + return nullptr; + *view = {audio->sample_rate, uint32_t(sd_tensor_dim(audio->data, 0)), uint64_t(sd_tensor_dim(audio->data, 1)), + const_cast(static_cast(sd_tensor_data(audio->data))), sd_tensor_retain(audio->data)}; + return view; +} + +static inline bool preprocess_canny(SDImage& image, float high, float low, float weak, float strong, bool inverse) { + auto input = image_tensor(image); + sd_tensor_t* output = nullptr; + if (!input || sd_preprocess_canny(input.get(), high, low, weak, strong, inverse, &output) != SD_STATUS_OK) + return false; + sd_image_free(&image); + image = tensor_image_view(output); + image.storage = output; + return true; +} + struct FreeDeleter { void operator()(void* ptr) const { free(ptr); @@ -28,23 +151,23 @@ struct FileCloser { struct SDCtxDeleter { void operator()(sd_ctx_t* ctx) const { if (ctx != nullptr) { - free_sd_ctx(ctx); + sd_ctx_free(ctx); } } }; struct UpscalerCtxDeleter { - void operator()(upscaler_ctx_t* ctx) const { + void operator()(sd_upscaler_t* ctx) const { if (ctx != nullptr) { - free_upscaler_ctx(ctx); + sd_upscaler_free(ctx); } } }; struct ADetailerCtxDeleter { - void operator()(adetailer_ctx_t* ctx) const { + void operator()(sd_adetailer_t* ctx) const { if (ctx != nullptr) { - free_adetailer_ctx(ctx); + sd_adetailer_free(ctx); } } }; @@ -54,12 +177,280 @@ using FreeUniquePtr = std::unique_ptr; using FilePtr = std::unique_ptr; using SDCtxPtr = std::unique_ptr; -using UpscalerCtxPtr = std::unique_ptr; -using ADetailerCtxPtr = std::unique_ptr; +using UpscalerCtxPtr = std::unique_ptr; +using ADetailerCtxPtr = std::unique_ptr; + +struct SDOptDeleter { + void operator()(sd_opt_t* opt) const { sd_opt_free(opt); } +}; +struct SDResultDeleter { + void operator()(sd_result_t* result) const { sd_result_free(result); } +}; +using SDOptPtr = std::unique_ptr; +using SDResultPtr = std::unique_ptr; + +class SDOptBuilder { + SDOptPtr opt; + sd_status_t status; + +public: + explicit SDOptBuilder(sd_opt_t* value) + : opt(value), status(value ? SD_STATUS_OK : SD_STATUS_OUT_OF_MEMORY) {} + SDOptPtr finish() { return status == SD_STATUS_OK ? std::move(opt) : SDOptPtr{}; } + void unset(const char* key) { + if (status == SD_STATUS_OK) + status = sd_opt_unset(opt.get(), key); + } + void set_floats(const char* key, const std::optional>& value) { + if (value) + set_floats(key, value->data(), value->size()); + } + void set_bool(const char* key, const std::optional& value) { + if (value) + set_bool(key, *value); + } + template + void set_int(const char* key, const std::optional& value) { + if (value) + set_int(key, *value); + } + template + void set_float(const char* key, const std::optional& value) { + if (value) + set_float(key, *value); + } + void set_string(const char* key, const std::optional& value) { + if (value) + set_string(key, *value); + } + void set_bool(const char* key, bool value) { + if (status == SD_STATUS_OK) + status = sd_opt_set_bool(opt.get(), key, value); + } + void set_int(const char* key, int64_t value) { + if (status == SD_STATUS_OK) + status = sd_opt_set_int(opt.get(), key, value); + } + void set_float(const char* key, double value) { + if (status == SD_STATUS_OK) + status = sd_opt_set_float(opt.get(), key, value); + } + void set_string(const char* key, const std::string& value) { + if (status == SD_STATUS_OK) + status = sd_opt_set_string(opt.get(), key, value.c_str()); + } + void set_string(const char* key, const char* value) { + if (value && status == SD_STATUS_OK) + status = sd_opt_set_string(opt.get(), key, value); + } + void set_image(const char* key, const SDImage& value) { + if (!value.data || status != SD_STATUS_OK) + return; + auto tensor = image_tensor(value); + if (!tensor) { + status = SD_STATUS_INVALID_ARGUMENT; + return; + } + status = sd_opt_set_tensor(opt.get(), key, tensor.get()); + } + void set_images(const char* key, const SDImage* value, size_t count) { + if (status != SD_STATUS_OK) + return; + std::vector storage; + std::vector tensors; + storage.reserve(count); + tensors.reserve(count); + for (size_t i = 0; i < count; ++i) { + storage.push_back(image_tensor(value[i])); + if (!storage.back()) { + status = SD_STATUS_INVALID_ARGUMENT; + return; + } + tensors.push_back(storage.back().get()); + } + status = sd_opt_set_tensors(opt.get(), key, tensors.data(), count); + } + void set_frames(const char* key, const SDImage* value, size_t count) { + if (!count || status != SD_STATUS_OK) + return; + auto tensor = frames_tensor(value, count); + if (!tensor) { + status = SD_STATUS_INVALID_ARGUMENT; + return; + } + status = sd_opt_set_tensor(opt.get(), key, tensor.get()); + } + void set_floats(const char* key, const float* value, size_t count) { + if (status == SD_STATUS_OK) + status = sd_opt_set_floats(opt.get(), key, value, count); + } + void set_ints(const char* key, const int32_t* value, size_t count) { + if (status == SD_STATUS_OK) + status = sd_opt_set_ints(opt.get(), key, value, count); + } + void set_loras(const char* key, const sd_lora_t* value, size_t count) { + if (status == SD_STATUS_OK) + status = sd_opt_set_loras(opt.get(), key, value, count); + } + void set_embeddings(const char* key, const sd_embedding_t* value, size_t count) { + if (status == SD_STATUS_OK) + status = sd_opt_set_embeddings(opt.get(), key, value, count); + } + void set_audios(const char* key, const SDAudio* value, size_t count) { + if (status != SD_STATUS_OK) + return; + std::vector storage; + std::vector audios; + storage.reserve(count); + audios.reserve(count); + for (size_t i = 0; i < count; ++i) { + storage.push_back(audio_tensor(value[i])); + audios.push_back({value[i].sample_rate, storage.back().get()}); + } + status = sd_opt_set_audios(opt.get(), key, audios.data(), count); + } + void set_videos(const char* key, const SDReferenceVideo* value, size_t count) { + if (status != SD_STATUS_OK) + return; + std::vector storage; + std::vector videos; + storage.reserve(count * 2); + videos.reserve(count); + for (size_t i = 0; i < count; ++i) { + storage.push_back(frames_tensor(value[i].frames, value[i].frame_count)); + const auto* frames = storage.back().get(); + storage.push_back(audio_tensor(value[i].audio)); + videos.push_back({frames, value[i].fps, {value[i].audio.sample_rate, storage.back().get()}}); + } + status = sd_opt_set_videos(opt.get(), key, videos.data(), count); + } +}; + +static inline sd_ctx_t* create_sd_context(const SDOptPtr& opt) { + sd_ctx_t* ctx = nullptr; + if (opt) + sd_ctx_create(opt.get(), &ctx); + return ctx; +} + +static inline bool generate_sd_image(sd_ctx_t* ctx, const SDOptPtr& opt, sd_result_t** result) { + if (result) + *result = nullptr; + return opt && sd_generate_image(ctx, opt.get(), result) == SD_STATUS_OK; +} + +static inline bool generate_sd_video(sd_ctx_t* ctx, const SDOptPtr& opt, sd_result_t** result) { + if (result) + *result = nullptr; + return opt && sd_generate_video(ctx, opt.get(), result) == SD_STATUS_OK; +} + +static inline sd_upscaler_t* create_sd_upscaler(const char* path, std::optional direct, std::optional threads, std::optional tile, const char* backend, const char* params_backend) { + SDOptPtr opt(sd_upscaler_opt_create()); + if (!opt || sd_opt_set_string(opt.get(), "model_path", path) != SD_STATUS_OK || + (direct && sd_opt_set_bool(opt.get(), "conv_direct", *direct) != SD_STATUS_OK) || + (tile && sd_opt_set_int(opt.get(), "tile_size", *tile) != SD_STATUS_OK) || + (threads && sd_opt_set_int(opt.get(), "n_threads", *threads) != SD_STATUS_OK) || + (backend && sd_opt_set_string(opt.get(), "backend", backend) != SD_STATUS_OK) || + (params_backend && sd_opt_set_string(opt.get(), "params_backend", params_backend) != SD_STATUS_OK)) + return nullptr; + sd_upscaler_t* result = nullptr; + sd_upscaler_create(opt.get(), &result); + return result; +} + +static inline bool upscale_sd_image(sd_upscaler_t* ctx, SDImage image, sd_result_t** result) { + if (result) + *result = nullptr; + SDOptPtr opt(sd_upscale_opt_create()); + auto tensor = image_tensor(image); + return opt && tensor && sd_opt_set_tensor(opt.get(), "image", tensor.get()) == SD_STATUS_OK && + sd_upscale(ctx, opt.get(), result) == SD_STATUS_OK; +} + +static inline sd_adetailer_t* create_sd_adetailer(const char* path, std::optional threads, const char* backend, const char* params_backend) { + SDOptPtr opt(sd_adetailer_opt_create()); + if (!opt || sd_opt_set_string(opt.get(), "model_path", path) != SD_STATUS_OK || + (threads && sd_opt_set_int(opt.get(), "n_threads", *threads) != SD_STATUS_OK) || + (backend && sd_opt_set_string(opt.get(), "backend", backend) != SD_STATUS_OK) || + (params_backend && sd_opt_set_string(opt.get(), "params_backend", params_backend) != SD_STATUS_OK)) + return nullptr; + sd_adetailer_t* result = nullptr; + sd_adetailer_create(opt.get(), &result); + return result; +} + +static inline bool adetail_sd_image(sd_adetailer_t* detector, sd_ctx_t* ctx, SDImage image, SDOptPtr opt, sd_result_t** result) { + if (result) + *result = nullptr; + auto tensor = image_tensor(image); + return opt && tensor && sd_opt_set_tensor(opt.get(), "image", tensor.get()) == SD_STATUS_OK && + sd_adetail(detector, ctx, opt.get(), result) == SD_STATUS_OK; +} + +struct SDVideoUpscalerParams { + const char* diffusion_model_path{}; + const char* vae_path{}; + // Explicit output scale; RelayVSR requires 2 or 4 matching both checkpoints. + int upscale_factor{}; + std::optional n_threads; + const char* backend{}; + const char* params_backend{}; + // auto/on/off or diffusion/vae assignments; null selects auto. + const char* flash_attn_mode{}; + const char* conv_direct_mode{}; +}; + +struct SDVideoUpscaleParams { + const SDImage* frames{}; + int frame_count{}; + std::optional fps; + std::optional seed; + // RelayVSR reference window, including both endpoints; defaults to 16. + std::optional window_size; + // SeedVR2 independent temporal chunks in input frames (1 or 4n+1); defaults to 5. + std::optional temporal_chunk_size; +}; + +static inline sd_vid_upscaler_t* create_sd_vid_upscaler(const SDVideoUpscalerParams& config) { + SDOptPtr opt(sd_vid_upscaler_opt_create()); + if (!opt || sd_opt_set_string(opt.get(), "diffusion_model_path", config.diffusion_model_path) != SD_STATUS_OK || + sd_opt_set_string(opt.get(), "vae_path", config.vae_path) != SD_STATUS_OK || + sd_opt_set_int(opt.get(), "upscale_factor", config.upscale_factor) != SD_STATUS_OK || + (config.n_threads && sd_opt_set_int(opt.get(), "n_threads", *config.n_threads) != SD_STATUS_OK) || + (config.backend && sd_opt_set_string(opt.get(), "backend", config.backend) != SD_STATUS_OK) || + (config.params_backend && sd_opt_set_string(opt.get(), "params_backend", config.params_backend) != SD_STATUS_OK) || + (config.flash_attn_mode && sd_opt_set_string(opt.get(), "flash_attn_mode", config.flash_attn_mode) != SD_STATUS_OK) || + (config.conv_direct_mode && sd_opt_set_string(opt.get(), "conv_direct_mode", config.conv_direct_mode) != SD_STATUS_OK)) + return nullptr; + sd_vid_upscaler_t* result = nullptr; + sd_vid_upscaler_create(opt.get(), &result); + return result; +} + +static inline bool upscale_sd_video(sd_vid_upscaler_t* ctx, const SDVideoUpscaleParams& config, sd_result_t** result) { + if (result) + *result = nullptr; + SDOptBuilder opt(sd_vid_upscale_opt_create()); + opt.set_frames("frames", config.frames, config.frame_count); + opt.set_int("fps", config.fps); + opt.set_int("seed", config.seed); + opt.set_int("window_size", config.window_size); + opt.set_int("temporal_chunk_size", config.temporal_chunk_size); + auto params = opt.finish(); + return params && sd_upscale_video(ctx, params.get(), result) == SD_STATUS_OK; +} + +static inline SDImage take_result_image(sd_result_t* result, size_t index) { + const auto* tensor = sd_result_get_image(result, index); + auto image = tensor_image_view(tensor); + image.storage = sd_tensor_retain(tensor); + return image; +} class SDImageOwner { private: - static sd_image_t copy_image(const sd_image_t& image) { + static SDImage copy_image(const SDImage& image) { if (image.data == nullptr) { return {image.width, image.height, image.channel, nullptr}; } @@ -74,11 +465,12 @@ class SDImageOwner { return {image.width, image.height, image.channel, raw_copy}; } - sd_image_t image_ = {0, 0, 0, nullptr}; + SDImage image_ = {0, 0, 0, nullptr}; + SDTensorPtr public_tensor_; public: SDImageOwner() = default; - explicit SDImageOwner(sd_image_t image) + explicit SDImageOwner(SDImage image) : image_(image) { } @@ -109,9 +501,14 @@ class SDImageOwner { reset(); } - sd_image_t* put() { + const sd_tensor_t* tensor() { + public_tensor_ = image_tensor(image_); + return public_tensor_.get(); + } + + SDImage* put() { if (image_.data != nullptr) { - free(image_.data); + sd_image_free(&image_); image_.data = nullptr; } image_.width = 0; @@ -120,23 +517,23 @@ class SDImageOwner { return &image_; } - sd_image_t& get() { + SDImage& get() { return image_; } - const sd_image_t& get() const { + const SDImage& get() const { return image_; } - sd_image_t release() { - sd_image_t image = image_; - image_ = {0, 0, 0, nullptr}; + SDImage release() { + SDImage image = image_; + image_ = {0, 0, 0, nullptr}; return image; } - void reset(sd_image_t image = {0, 0, 0, nullptr}) { + void reset(SDImage image = {0, 0, 0, nullptr}) { if (image_.data != nullptr) { - free(image_.data); + sd_image_free(&image_); } image_ = image; } @@ -161,7 +558,7 @@ class SDAudioOwner { return samples_.empty(); } - sd_audio_t get() { + SDAudio get() { return {sample_rate_, channels_, channels_ == 0 ? 0 : static_cast(samples_.size() / channels_), @@ -175,7 +572,7 @@ class SDAudioOwner { class SDImageVec { private: - std::vector images_; + std::vector images_; std::vector generation_infos_; public: @@ -201,7 +598,7 @@ class SDImageVec { clear(); } - void push_back(sd_image_t image) { + void push_back(SDImage image) { images_.push_back(image); } @@ -223,32 +620,25 @@ class SDImageVec { generation_infos_.at(index) = merge_generation_info(generation_infos_.at(index), generation_info_with_relative_lora_paths(info, lora_model_dir)); } - void adopt(sd_image_generation_result_t* result, const std::string& lora_model_dir = {}) { + void adopt(sd_result_t* result, const std::string& lora_model_dir = {}) { clear(); + SDResultPtr owner(result); if (!result) return; - for (int i = 0; i < result->count; ++i) { - images_.push_back(result->items[i].image); - result->items[i].image = {}; - generation_infos_.push_back(generation_info_with_relative_lora_paths(result->items[i].generation_info, lora_model_dir)); + const size_t count = sd_result_get_image_count(result); + images_.reserve(count); + generation_infos_.reserve(count); + for (size_t i = 0; i < count; ++i) { + generation_infos_.push_back(generation_info_with_relative_lora_paths(sd_result_get_generation_info(result, i), lora_model_dir)); + images_.push_back(take_result_image(result, i)); } - free_sd_image_generation_result(result); } - void adopt_video(sd_video_generation_result_t* result, const std::string& lora_model_dir = {}) { - clear(); - if (!result) - return; - const auto info = generation_info_with_relative_lora_paths(result->generation_info, lora_model_dir); - for (int i = 0; i < result->frame_count; ++i) { - images_.push_back(result->frames[i]); - result->frames[i] = {}; - generation_infos_.push_back(info); - } - free_sd_video_generation_result(result); + void adopt_video(sd_result_t* result, const std::string& lora_model_dir = {}) { + adopt(result, lora_model_dir); } - void adopt(sd_image_t* images, int count) { + void adopt(SDImage* images, int count) { clear(); if (images == nullptr || count <= 0) { free(images); @@ -278,33 +668,33 @@ class SDImageVec { return !images_.empty(); } - sd_image_t* data() { + SDImage* data() { return images_.data(); } - const sd_image_t* data() const { + const SDImage* data() const { return images_.data(); } - sd_image_t& operator[](size_t index) { + SDImage& operator[](size_t index) { return images_[index]; } - const sd_image_t& operator[](size_t index) const { + const SDImage& operator[](size_t index) const { return images_[index]; } - std::vector& raw() { + std::vector& raw() { return images_; } - const std::vector& raw() const { + const std::vector& raw() const { return images_; } void clear() { - for (sd_image_t& image : images_) { - free(image.data); + for (SDImage& image : images_) { + sd_image_free(&image); image.data = nullptr; } images_.clear(); diff --git a/examples/server/README.md b/examples/server/README.md index 33b256eb61..8673eb80c4 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -1,70 +1,84 @@ -# Model selection +# stable-diffusion.cpp Server -Start without model arguments to select models in the web UI: +`sd-server` provides a Web UI for image and video generation, model management, +and HTTP APIs. Download or build the server before starting; see the +[build guide](../../docs/build.md) and [frontend build instructions](#build-with-frontend). -```sh -sd-server --model-dir ./models -``` +- [Web UI](#web-ui): start the server, load a model, and generate. +- [Model management](#model-management): model directories, recipes, and startup models. +- [Server configuration](#server-configuration): launcher settings, network, logging, and help. +- [API](#api): native and compatibility endpoints. +- [Frontend development](#frontend-development): build or customize the Web UI. -From a source checkout, you can also use `sh ./start-server.sh` on Linux/macOS -or `.\start-server.bat` on Windows. The launchers look for the server relative -to the script: beside the script, then in `bin/Release/`, `bin/`, -`build/bin/Release/`, and `build/bin/`, in that order. -For a different build directory, set `SD_SERVER_PATH` to the executable path: +Inference settings follow this priority: request fields, explicit server startup +options, model recipe presets, then [library defaults](../../docs/guides/c_api.md#keys). +Omitted request fields retain the configured override. In native JSON requests, +`null` clears an optional inference field so the library selects its value; +`0`, `false`, empty strings and empty numeric arrays are explicit values. +Capability `defaults` and `defaults_by_mode` contain configured overrides, with +`null` for optional settings left to the library. Effective settings are returned +in generation metadata. Output formats, queue limits and listen addresses keep +server-local defaults. -```sh -SD_SERVER_PATH="./build_vulkan/bin/sd-server" sh ./start-server.sh --listen-port 8080 -``` +## Web UI + +### Start the server + +From the repository root, start the launcher. It finds the server executable +and uses the repository's `models/` directory: + +**Windows (PowerShell)** ```powershell -$env:SD_SERVER_PATH = ".\build_vulkan\bin\Release\sd-server.exe" -.\start-server.bat --listen-port 8080 +.\start-server.bat ``` -Binary release archives include both launchers. Keep the launchers under version -control and put personal settings in a separate file beside them: +You can also double-click `start-server.bat` in File Explorer to start the +server. Keep the console window open while using the Web UI. -- Windows batch: `start-server.local.bat` -- Linux/macOS: `start-server.local.sh` +**Linux/macOS** -These local files are ignored by Git and excluded from release archives. They -are loaded automatically after the launcher's defaults. They can override -`SD_SERVER_PATH`, `MODEL_DIR`, and `COMMANDLINE_ARGS`; there is no need to edit -the tracked launchers. For example: +```sh +sh ./start-server.sh +``` -```bat -@echo off -set "MODEL_DIR=D:\stable-diffusion-webui\models" -set "COMMANDLINE_ARGS=--listen-port 8080 --offload-to-cpu" +To start the executable directly instead, run from the build directory: + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-server.exe --model-dir ..\models ``` +**Linux/macOS** + ```sh -MODEL_DIR="/data/stable-diffusion-webui/models" -COMMANDLINE_ARGS="--listen-port 8080 --offload-to-cpu" +./bin/sd-server --model-dir ../models ``` -Each launcher prints the resolved command before execution and forwards the -server's exit code. - -`SD_SERVER_PATH` inherits the environment value by default; leave it empty for -automatic discovery, or set it in the local file to select another executable. -Settings are applied in this order: launcher defaults, local configuration, -then invocation arguments. Repeated `--model-dir` arguments add roots, so they -do not replace the `MODEL_DIR` root. Local configuration files are shell scripts -and use the corresponding shell's syntax. +The Windows path above is for a Visual Studio Release build. Adjust the path +for another build configuration or an extracted release archive. Open + after the server starts, and keep the server running +while using the browser. + +### Load a model + +1. Download the weights and required components listed in the matching + [model guide](../../docs/models/README.md). Put them in the category folders + under `models/`, such as `checkpoints/`, `diffusion_models/`, `text_encoders/`, + `tokenizers/`, and `vae/`. See [Model directories](#model-directories) below. +2. Click **Model** below the prompt to open model selection. +3. Choose the matching **Recipe**. Keep **Apply recipe defaults** enabled to + start with that recipe's generation settings. +4. Select each component in its named field: for example, **Diffusion model**, + the model-specific text encoder field, and **VAE**. Required fields have an + asterisk. A VAE/TAE requirement needs one compatible decoder. Match all files + to the chosen checkpoint and variant, as described in its model guide. +5. If a file is missing from a dropdown, check its category folder and click + **Refresh files & recipes**. Selectors support searching by name or path. +6. Click **Load models** and wait for loading to finish. Selecting a recipe or + file alone does not reload the model. -The launchers preserve the current working directory, so the default -`MODEL_DIR="./models"` is relative to where you start the launcher. Arguments -passed when invoking the launcher follow these defaults. Repeated `--model-dir` -arguments add search roots rather than replacing `MODEL_DIR`; set `MODEL_DIR` in -the local configuration to change the first root. Other single-value options can override the defaults. -In the shell script, `COMMANDLINE_ARGS` is a -whitespace-separated list; pass values containing spaces as quoted arguments -when invoking the script. `MODEL_DIR` itself supports spaces in both scripts. -A leading `--version` bypasses the configured defaults. An explicit -`SD_SERVER_PATH` is used without falling back to another build if it is invalid. - -Choose a recipe, select the required component files, and click **Load models**. Recipe and component selectors support searching by name or relative file ID. Use the arrow keys and Enter to select, or Escape to cancel a search. With **Apply recipe defaults** enabled, selecting a recipe immediately applies its @@ -74,123 +88,56 @@ switching recipes. This preference defaults to enabled and is saved locally. Model loading and server-info refreshes do not reapply defaults or reset the form. Recipes declaring `video_frames` also switch the form to Video Generation; other recipes switch to Image Generation. Generation is enabled after the -selected recipe is loaded. Changing a selection does not reload the model -until you click Load. +selected recipe is loaded. + +The model button shows elapsed loading time and keeps the final time after +success or failure, including after a page refresh. + +### Generate an image or video + +1. Enter a prompt, for example `a lovely cat` for an image model. +2. Open **Sample & Guidance** to set **Method**, **Scheduler**, **Steps**, and + **CFG Scale** when the model guide specifies values different from the recipe + defaults. Set the output dimensions in **Aspect Ratio & Resolution**. +3. Supply inputs for the chosen task. Editing recipes use **Reference Images**; + IP-Adapter uses **IP-Adapter Image**. For image-to-video, use video generation + and upload a **Start Frame**; add an **End Frame** only for a compatible + first/last-frame workflow. Set the video length and FPS as required by the model. +4. Click **Generate**. Wait for denoising, decoding, and output encoding to + finish, then preview or save the result. Sampling reaching 100% does not mean + decoding has finished. The generation status bar shows the current stage and elapsed time even when Preview Mode is `none`. During sampling it shows completed steps for the current pass; prompt encoding and VAE decoding show an activity indicator when their -total work is unknown. Sampling reaching 100% does not mean the job is finished: -decoding, ADetailer, and output encoding may still follow. -The canvas keeps the submitted output aspect ratio, and stage changes keep the -same layout. Editing the next request's resolution does not resize the current -preview or result. - -The server can also start with the existing model arguments shown below; -existing API clients then work without selecting a recipe. The UI automatically -selects **Command-line model (all features)** and uses the startup generation -defaults. This runtime-only recipe exposes all controls for the supported -image/video mode. Selecting another recipe previews a model switch; selecting -the command-line entry again restores its generation defaults. The entry is -removed when the command-line context is unloaded or replaced. - -Built-in recipes cover SD 1/2/3/3.5, SDXL and distilled checkpoints; -FLUX 1/2/3, Chroma, Qwen Image and editing variants; Z-Image, Anima, ERNIE, -LENS, PiD and other supported image families; AnimateDiff, Wan, LTX, -HunyuanVideo, LingBot Video and MiniMax-H3. Model-specific text encoder -labels identify the expected encoder family and size. Loading a video model enables **Video Generation**; I2V also requires a -start image. The model button below the prompt opens model and recipe selection. -Its status summary shows elapsed loading time from the status API and keeps the final -time after success or failure, including after a page refresh. -Built-ins are embedded in the -binary from [recipes/](recipes/), with one JSON file per recipe. Adding or -editing a built-in requires rebuilding the server. User recipes are read from -`/recipes/*.json`. A valid user recipe with the same ID replaces -the entire built-in recipe. Across roots, the first valid user recipe wins; -overrides are logged. Duplicate user IDs within one root are rejected; invalid -recipes are reported without removing a valid recipe from another root or the -built-in version. Slots need only `id` and -`required`; the server supplies labels and search directories. Use `label` -only when a recipe needs a more specific display name. +total work is unknown. The canvas keeps the submitted output aspect ratio, and +stage changes keep the same layout. Editing the next request's resolution does +not resize the current preview or result. -Recipes use `features` to enable specialized controls and `disabled_features` -to turn off shared defaults. Disabled features take precedence. The UI hides -unsupported fields, empty sections and irrelevant sampler/cache options, and -omits hidden values from generation requests. See the -[feature declarations](api.md#recipe-feature-declarations) for the defaults -and complete list of supported names. +### LoRAs -Built-in additions and changes must follow the [recipe policy](recipes/README.md). -Split recipes by task weights or added capabilities, not by style or acceleration -LoRAs. Quantizations, file formats and compatible layer-count variants share a -recipe. Native encoders, projectors and connectors stay in the model's recipe. -Community editing recipes require selecting the matching LoRA separately; a -recipe does not enable it automatically. +To apply a LoRA, open **LoRA**, select a compatible file, and set its +**Strength**. Place files under `models/loras/` or `models/Lora/`. Browser +requests use this selector; prompt tags such as `` are CLI syntax +and are not parsed by the native server API. -See [generation defaults and sources](recipes/defaults.md) for the selected -publisher recommendations and reference-code defaults, with step defaults capped -at 20. Recipes link to the -original publisher's Hugging Face repository where one is available. DiT -recipes default to Euler, except for models requiring a specialized sampler. +## Model management -Compatible recipes offer a TAE slot alongside VAE. When the checkpoint does not -bundle a VAE, select at least one of VAE or TAE. The Models panel shows a reminder -below the model selectors until this requirement is met. A selected TAE is used for -encoding and decoding, including when a full VAE is also selected; use complete, -model-compatible TAE weights rather than decoder-only preview weights. Qwen -Image 2.1 uses the same recipe for its full VAE and TAEQI2.1 alternatives. +### Model directories -The following variants have dedicated recipes: +Repeat `--model-dir` to share multiple model directories, including an existing +SD WebUI model directory. Run these examples from the build directory: -| Model | Recipe IDs / notes | -| --- | --- | -| Z-Image L2P | `z-image-l2p`; no VAE | -| LLaDA-Image | `llada-image` (base) and `llada-image-turbo`; use matching encoders and connectors | -| PixArt Alpha | `pixart-alpha-512`, `pixart-alpha-1024`; Alpha VAE scale and positional settings | -| PiD / PiD 1.5 | `pid` / `pid-15` for Flux or Z-Image VAE; append `-sd3`, `-flux2`, or `-qwen-image` for other latent formats | -| SDXL adapters | `sdxl-ip-adapter` supports classic and Plus ViT-H adapters; `sdxl-controlnet` requires compatible SDXL weights | -| FLUX.1 Fill / Depth | `flux1-fill` uses Init Image and Mask Image; `flux1-depth` uses a precomputed depth map in Control Image | -| SD2 / SDXL inpainting | `sd2-inpainting`, `sdxl-inpainting` | -| SD-Turbo | `sd-turbo`; single-file SD2.1-derived checkpoint | -| Community editing LoRAs | `anima-edit`, `krea2-raw-edit`, `krea2-turbo-edit`; apply a matching LoRA after loading | -| HunyuanVideo 1.5 | `hunyuan-video-15-t2v`, `hunyuan-video-15-i2v`; choose the task's checkpoint | -| Wan 2.1 / compatible single Wan 2.2 experts | `wan21-t2v`, `wan21-i2v`; the I2V recipe also accepts compatible SkyReels V2 weights | -| Qwen Image 2.1 Turbo | `qwen-image-21-turbo`; standalone Turbo weights with the fixed 8-step schedule, Euler and CFG 1 | -| MOSS Image | `moss-image-base`, `moss-image-sft`; choose the corresponding checkpoint | - -PiD always requires a reference image and uses the first one. Its recipes expose -reference inputs rather than ordinary image-to-image inputs. IP-Adapter recipes -expose a separate IP-Adapter image and strength. FLUX Canny/Depth consume Control -Image directly without a separate ControlNet; the UI does not calculate edge -or depth maps. Their strength is controlled by guidance, not ControlNet strength. -Krea2 edit recipes use the default `krea2_ostris_edit` reference preset, so the -editing LoRA must match that convention. Ordinary Anima and Krea2 recipes do not -expose reference-image editing inputs. - -Fill/Depth guidance defaults follow the [Diffusers FLUX documentation](https://huggingface.co/docs/diffusers/api/pipelines/flux). -SD-Turbo uses the [model card's one-step configuration](https://huggingface.co/stabilityai/sd-turbo), -with CFG disabled using scale 1 in sd.cpp. - -SVD is not included: legacy model recognition and UNet branches remain, but the -current runner/conditioning setup does not provide a dedicated SVD image-conditioning -path. Adding a recipe alone would not establish a working SVD pipeline. +**Windows (PowerShell)** -Reference images, masks, audio, first/last video frames, and generation options -such as upscaling remain generation inputs. Hardware and memory options remain -startup arguments. Tiny SD/Tiny SD2 and Diffusers LCM directories must first be -converted into single-file checkpoints for the file catalog. LCM checkpoints -and LoRAs use the matching SD family recipe; select LCM sampling and adjust -steps/CFG for the chosen weights. -Metadata inspection, conversion, standalone upscaling and ADetailer are CLI -workflows rather than model-loading recipes. ESRGAN and LTX latent upscalers -are selected through generation upscaling settings with `--hires-upscalers-dir`; -textual inversion uses the startup `--embd-dir` option. +```powershell +.\bin\Release\sd-server.exe --model-dir ..\models --model-dir D:\stable-diffusion-webui\models +``` -Repeat `--model-dir` to share multiple model directories, including an existing -SD WebUI model directory: +**Linux/macOS** ```sh -sd-server --model-dir ./models --model-dir /data/stable-diffusion-webui/models +./bin/sd-server --model-dir ../models --model-dir /data/stable-diffusion-webui/models ``` With no `--model-dir`, the server uses `./models`. Explicit roots replace that @@ -216,6 +163,7 @@ models/ checkpoints/ diffusion_models/ text_encoders/ + tokenizers/ vae/ clip_vision/ loras/ @@ -242,6 +190,132 @@ model root. Use `--lora-model-dir` to replace automatic discovery with one explicit LoRA directory. Duplicate names remain separate selectable entries. The LoRA selectors support searching by name or path with the same keyboard controls as model selectors. +### Recipes + +A recipe describes a model's components, supported capabilities, and generation +defaults. See the [model guides](../../docs/models/README.md) for compatible +recipes and weights. Text encoder labels in the selectors identify the expected +encoder family and size. + +#### Built-in and user recipes + +Built-in recipes are embedded in the binary from [recipes/](recipes/), with one JSON file per recipe. Adding or +editing a built-in requires rebuilding the server. User recipes are read from +`/recipes/*.json`. A valid user recipe with the same ID replaces +the entire built-in recipe. Across roots, the first valid user recipe wins; +overrides are logged. Duplicate user IDs within one root are rejected; invalid +recipes are reported without removing a valid recipe from another root or the +built-in version. Slots need only `id` and +`required`; the server supplies labels and search directories. Use `label` +only when a recipe needs a more specific display name. + +Built-in additions and changes must follow the [recipe policy](recipes/README.md). +Split recipes by task weights or added capabilities, not by style or acceleration +LoRAs. Quantizations, file formats and compatible layer-count variants share a +recipe. Native encoders, projectors and connectors stay in the model's recipe. +Community editing recipes require selecting the matching LoRA separately; a +recipe does not enable it automatically. + +#### Generation defaults and controls + +See [generation defaults and sources](recipes/defaults.md) for the selected +publisher recommendations and reference-code defaults, with step defaults capped +at 20. Recipes link to the +original publisher's Hugging Face repository where one is available. DiT +recipes default to Euler, except for models requiring a specialized sampler. + +Recipes use `features` to enable specialized controls and `disabled_features` +to turn off shared defaults. Disabled features take precedence. The UI hides +unsupported fields, empty sections and irrelevant sampler/cache options, and +omits hidden values from generation requests. See the +[feature declarations](api.md#recipe-feature-declarations) for the defaults +and complete list of supported names. + +#### VAE and TAE + +Compatible recipes offer a TAE slot alongside VAE. When the checkpoint does not +bundle a VAE, select at least one of VAE or TAE. The Models panel shows a reminder +below the model selectors until this requirement is met. A selected TAE is used for +encoding and decoding, including when a full VAE is also selected; use complete, +model-compatible TAE weights rather than decoder-only preview weights. Qwen +Image 2.1 uses the same recipe for its full VAE and TAEQI2.1 alternatives. + +### Models loaded at startup + +You can pass the same model-loading arguments used by `sd-cli` to `sd-server`. +For example, run this Z-Image Turbo configuration from the build directory: + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-server.exe ` + --model-dir ..\models ` + --diffusion-model ..\models\diffusion_models\z_image_turbo_bf16.safetensors ` + --vae ..\models\vae\ae.sft ` + --llm ..\models\text_encoders\qwen_3_4b.safetensors ` + --offload-to-cpu ` + -v ` + --cfg-scale 1.0 +``` + +**Linux/macOS** + +```sh +./bin/sd-server \ + --model-dir ../models \ + --diffusion-model ../models/diffusion_models/z_image_turbo_bf16.safetensors \ + --vae ../models/vae/ae.sft \ + --llm ../models/text_encoders/qwen_3_4b.safetensors \ + --offload-to-cpu \ + -v \ + --cfg-scale 1.0 +``` + +`--diffusion-model`, `--vae`, and `--llm` select the model components. +`--offload-to-cpu` keeps weights in RAM when possible to reduce VRAM pressure, +`-v` enables verbose logging, and `--cfg-scale 1.0` sets the default CFG scale. + +API clients can use the model loaded at startup without selecting a recipe. +The UI automatically selects **Command-line model (all features)** and uses the +startup generation defaults. This runtime-only recipe exposes all controls for the supported +image/video mode. Selecting another recipe previews a model switch; selecting +the command-line entry again restores its generation defaults. The entry is +removed when the command-line context is unloaded or replaced. + +#### Identity inputs + +The browser currently has no PhotoMaker identity-image or PuLID identity-embedding +upload control. Prepare the inputs using the +[PhotoMaker](../../docs/models/photo_maker.md) or [PuLID](../../docs/models/pulid.md) +guide, then supply them when starting the server. From the build directory, for example: + +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-server.exe --model-dir ..\models --pm-id-images-dir ..\inputs\identity +``` + +**Linux/macOS** + +```sh +./bin/sd-server --model-dir ../models --pm-id-images-dir ../inputs/identity +``` + +Select the PhotoMaker recipe and weights in the browser. For PhotoMaker v2, also +pass `--pm-id-embed-path` with the prepared embedding file. For PuLID, use +`--pulid-id-embedding` instead of the PhotoMaker input argument, then select the +PuLID recipe and weights. These startup inputs are inherited by generation requests. + +#### Sharded checkpoints + +For a checkpoint loaded from a complete repository directory, pass its model +loading arguments when starting `sd-server`, as in the model's CLI example. +Keep the index, tokenizer, and all shards together. The browser then uses +**Command-line model (all features)** for the model loaded at startup; selecting +an individual shard in a recipe dropdown does not load the whole repository. + +### Switching models + The server keeps one model context loaded. Loading or unloading is rejected while generation requests or queued jobs are pending. A switch validates the selection before unloading the old context, then loads the replacement. @@ -253,146 +327,186 @@ request parameters take precedence. See [the API reference](api.md#model-management) for recipe fields, supported slots, management endpoints, and compatibility behavior. -# Example +### Limitations -The following example starts `sd-server` with a standalone diffusion model, VAE, and LLM text encoder: +Tiny SD/Tiny SD2 and Diffusers LCM directories must first be +converted into single-file checkpoints for the file catalog. LCM checkpoints +and LoRAs use the matching SD family recipe; select LCM sampling and adjust +steps/CFG for the chosen weights. +Metadata inspection, conversion, standalone upscaling and ADetailer are CLI +workflows rather than model-loading recipes. ESRGAN and LTX latent upscalers +are selected through generation upscaling settings with `--hires-upscalers-dir`; +textual inversion uses the startup `--embd-dir` option. -``` -.\bin\Release\sd-server.exe --diffusion-model ..\models\diffusion_models\z_image_turbo_bf16.safetensors --vae ..\models\vae\ae.sft --llm ..\models\text_encoders\qwen_3_4b.safetensors --offload-to-cpu -v --cfg-scale 1.0 -``` +SVD is not available through the current server pipeline. -What this example does: +## Server configuration -* `--diffusion-model` selects the standalone diffusion model -* `--vae` selects the VAE decoder -* `--llm` selects the text encoder / language model used by this pipeline -* `--offload-to-cpu` reduces VRAM pressure by keeping weights in RAM when possible -* `-v` enables verbose logging (equivalent to `--log-level verbose`) -* `--cfg-scale 1.0` sets the default CFG scale for generation +Reference images, masks, audio, first/last video frames, and generation options +such as upscaling are supplied with generation requests. Hardware and memory +options are startup arguments and apply to subsequent model loads. -Logging defaults to `info`. Use `--log-level ` to select `debug`, `verbose`, -`info`, `warn`, or `error` (from most to least detailed). Each level includes -messages at that level and all less detailed levels. `-v` and `--verbose` are -equivalent to `--log-level verbose`. If repeated, the last logging option wins. +### Launcher settings -After the server starts successfully: +Both source checkouts and binary release archives include `start-server.bat` +and `start-server.sh`. The launchers search for the server beside the script, +then in `bin/Release/`, `bin/`, `build/bin/Release/`, and `build/bin/`, in that order. +They print the resolved command and forward the server's exit code. -* the web UI is available at `http://127.0.0.1:1234/` -* the native async API is available under `/sdcpp/v1/...` -* the compatibility APIs are available under `/v1/...` and `/sdapi/v1/...` +#### Custom executable -If you want to use a different host or port, pass: +For a different build directory, set `SD_SERVER_PATH` to the executable path. +Run from the repository root: -```bash ---listen-ip --listen-port +**Windows (PowerShell)** + +```powershell +$env:SD_SERVER_PATH = ".\build_vulkan\bin\Release\sd-server.exe" +.\start-server.bat --listen-port 8080 ``` -The native async job queue can be tuned with: +**Linux/macOS** -```bash ---max-pending-jobs --completed-ttl-seconds +```sh +SD_SERVER_PATH="./build_vulkan/bin/sd-server" sh ./start-server.sh --listen-port 8080 ``` -`--max-pending-jobs` defaults to 64 and limits how many async jobs may be -queued or generating. `--completed-ttl-seconds` defaults to 600 and controls -how long completed jobs remain available through the async API. +`SD_SERVER_PATH` inherits the environment value by default. Leave it empty for +automatic discovery. An explicit invalid path does not fall back to another build. -# Frontend +#### Local configuration -## Build with Frontend +Keep personal settings in a file beside the launchers: `start-server.local.bat` +on Windows or `start-server.local.sh` on Linux/macOS. These files are ignored by +Git and excluded from release archives. They are loaded automatically and can +set `SD_SERVER_PATH`, `MODEL_DIR`, and `COMMANDLINE_ARGS`. -The server can optionally build the web frontend and embed it into the binary as `gen_index_html.h`. +**Windows (`start-server.local.bat`)** -### Requirements +```bat +@echo off +set "MODEL_DIR=D:\stable-diffusion-webui\models" +set "COMMANDLINE_ARGS=--listen-port 8080 --offload-to-cpu" +``` -Install the following tools: +**Linux/macOS (`start-server.local.sh`)** -* **Node.js** ≥ 20 - https://nodejs.org/ +```sh +MODEL_DIR="/data/stable-diffusion-webui/models" +COMMANDLINE_ARGS="--listen-port 8080 --offload-to-cpu" +``` -* **pnpm** ≥ 10 - Install via npm: +Settings are applied in this order: launcher defaults, local configuration, +then invocation arguments. Local files are shell scripts and use the +corresponding shell's syntax. -```bash -npm install -g pnpm -``` +The launchers preserve the current working directory, so the default +`MODEL_DIR="./models"` is relative to where you start the launcher. To change +the first model root, set `MODEL_DIR` in the local configuration. Repeated +`--model-dir` arguments add search roots; other single-value options can override +the defaults. -Verify installation: +In the shell script, `COMMANDLINE_ARGS` is a whitespace-separated list. Pass +values containing spaces as quoted invocation arguments instead. `MODEL_DIR` +itself supports spaces in both scripts. A leading `--version` bypasses the +configured defaults. -```bash -node -v -pnpm -v -``` +### Network and job queue -### Install frontend dependencies +| Option | Default | Purpose | +| --- | --- | --- | +| `--listen-ip ` | `127.0.0.1` | Server listen address | +| `--listen-port ` | `1234` | Server listen port | +| `--max-pending-jobs ` | `64` | Maximum async jobs queued or generating | +| `--completed-ttl-seconds ` | `600` | Retention time for completed async jobs | -Go to the frontend directory and install dependencies: +### Logging -```bash -cd examples/server/frontend -pnpm install -``` +Logging defaults to `info`. Use `--log-level ` to select `debug`, `verbose`, +`info`, `warn`, or `error` (from most to least detailed). Each level includes +messages at that level and all less detailed levels. `-v` and `--verbose` are +equivalent to `--log-level verbose`. If repeated, the last logging option wins. -### Build the server with CMake +### Command-line help -Enable the frontend build option when configuring CMake: +Run from the build directory for the complete list of server and generation options: -```bash -cmake -B build -DSD_SERVER_BUILD_FRONTEND=ON -cmake --build build --config Release +**Windows (PowerShell)** + +```powershell +.\bin\Release\sd-server.exe --help ``` -If `pnpm` is available, the build system will automatically run: +**Linux/macOS** -``` -pnpm run build -pnpm run build:header +```sh +./bin/sd-server --help ``` -and embed the generated frontend into the server binary. +## API -## Frontend Repository +The server exposes these API families at the same host and port as the Web UI: -The web frontend is maintained in a **separate repository**, https://github.com/leejet/sdcpp-webui. - -If you want to modify the UI or frontend logic, please submit pull requests to the **frontend repository**. +| API | Path prefix | +| --- | --- | +| Native async API | `/sdcpp/v1/...` | +| OpenAI-compatible API | `/v1/...` | +| Stable Diffusion WebUI-compatible API | `/sdapi/v1/...` | -This repository (`stable-diffusion.cpp`) only vendors the frontend periodically. Changes from the frontend repo are synchronized: +See the [API reference](api.md) for endpoints, request fields, and examples. -* approximately **every 1–2 weeks**, or -* when there are **major frontend updates** +## Frontend development -Because of this, frontend changes will **not appear here immediately** after being merged upstream. +### Build with frontend -## Using an external frontend +The server can build the frontend and embed it in the binary. Install +[Node.js](https://nodejs.org/) 20 or newer and pnpm 10 or newer: -By default, the server uses the **embedded frontend** generated during the build (`gen_index_html.h`). +```sh +npm install -g pnpm +node -v +pnpm -v +``` -You can also serve a custom frontend file instead of the embedded one by using: +Run the following commands from the repository root. They work in PowerShell +and POSIX shells. Initialize the frontend submodule if it is missing, then +configure and build with CMake: -```bash ---serve-html-path +```sh +git submodule update --init examples/server/frontend +cmake -S . -B build -DSD_SERVER_BUILD_FRONTEND=ON +cmake --build build --config Release ``` -For example: +Add the backend options from the [build guide](../../docs/build.md) when +configuring CMake. The build installs frontend dependencies, runs `pnpm run build` +and `pnpm run build:header`, and embeds `gen_index_html.h` in `sd-server`. +If pnpm is unavailable, CMake can use an existing generated header; without +either, the server builds without the embedded Web UI. -```bash -sd-server --serve-html-path ./index.html -``` +### Using an external frontend -In this case, the server will load and serve the specified `index.html` file instead of the embedded frontend. This is useful when: +Use `--serve-html-path` to serve a custom `index.html` instead of the embedded +frontend. For example, place it at the repository root and run from the build +directory: -* developing or testing frontend changes -* using a custom UI -* avoiding rebuilding the binary after frontend modifications +**Windows (PowerShell)** -# Usage +```powershell +.\bin\Release\sd-server.exe --model-dir ..\models --serve-html-path ..\index.html +``` -For detailed command-line arguments, run: +**Linux/macOS** -```bash -./bin/sd-server -h +```sh +./bin/sd-server --model-dir ../models --serve-html-path ../index.html ``` -For completely black or white images or videos, NaNs, and the `--linear-scale` / -`--attn-scale` startup options, see [Troubleshooting](../../docs/troubleshooting.md). +This lets you develop or test the frontend without rebuilding the server binary. + +### Frontend repository + +The Web UI is maintained in the separate +[sdcpp-webui repository](https://github.com/leejet/sdcpp-webui), included here as +a Git submodule. Submit UI and frontend logic changes to that repository. +Changes appear here when the submodule is updated. diff --git a/examples/server/api.md b/examples/server/api.md index f6096c67a1..9f48e247f8 100644 --- a/examples/server/api.md +++ b/examples/server/api.md @@ -11,6 +11,14 @@ The server currently exposes three API families: The `sdcpp API` is the native API surface. Its request schema is the same schema used by `sd_cpp_extra_args`. +Image and video requests can include interleaved images in `prompt` or +`negative_prompt` using ``. The native `sdcpp API` accepts a +`prompt_images` object mapping each ID to a base64 image or image data URI. +For `OpenAI API` and `sdapi`, pass `prompt_images` through `sd_cpp_extra_args` +embedded inside `prompt`. IDs are resolved against that request's images. +See [images in prompts](../../docs/guides/prompt_images.md) for examples and supported +LLM encoders. + Global LoRA rule: - Server APIs do not parse LoRA tags embedded inside `prompt`. @@ -221,6 +229,9 @@ values reflect these declarations. A loaded recipe is snapshotted: refreshing recipe files does not change the active context's capabilities until reloading. Command-line preloading uses the runtime-only all-features recipe instead of model-family inference and component paths; variant-specific recipe declarations are more precise. +Known high-noise sampling settings and `moe_boundary` are ignored when no +high-noise stage is loaded, and `vace_strength` is ignored by non-VACE models. +Actual control-frame inputs still require VACE. These declarations govern the Web UI and advertised capabilities, not API request validation. Existing generation APIs continue accepting their existing @@ -975,7 +986,7 @@ Shared default fields used by both `img_gen` and `vid_gen`: | `output_format` | `string` | | `output_compression` | `integer` | -`vae_tiling_params.tile_size_w` and `tile_size_h` are in **image pixels**, with `0` selecting the 256-pixel default. Both encode and decode use these sizes without an encoding multiplier. Positive `rel_size_w`/`rel_size_h` values override the corresponding absolute size: values up to 1 are dimension fractions, and values greater than 1 are target tile counts. Set `enabled` to use spatial tiling. Sizes are aligned down to the VAE scale factor and capped at the input dimensions; explicit sizes below the minimum supported tile size are rejected. These fields previously used latent units; see [VAE tiling](../../docs/performance.md#use-vae-tiling-to-reduce-encode-and-decode-memory-usage) for migration and OOM retry behavior. +`vae_tiling_params.tile_size_w` and `tile_size_h` are in **image pixels**, with `0` selecting the 256-pixel default. Both encode and decode use these sizes without an encoding multiplier. Positive `rel_size_w`/`rel_size_h` values override the corresponding absolute size: values up to 1 are dimension fractions, and values greater than 1 are target tile counts. Set `enabled` to use spatial tiling. Sizes are aligned down to the VAE scale factor and capped at the input dimensions; explicit sizes below the minimum supported tile size are rejected. These fields previously used latent units; see [VAE tiling](../../docs/guides/performance.md#use-vae-tiling-to-reduce-encode-and-decode-memory-usage) for migration and OOM retry behavior. `vae_tiling_params.extra_tiling_args` accepts a key=value list. Supported video VAEs accept `temporal_tile_frames` (alias `temporal_tile_size`, default `4`) and `temporal_tile_overlap` (default `1`). Wan2.2 TI2V 5B uses `temporal_decode_frames` (default `1`) to decode multiple latent frames per forward pass, trading additional memory for throughput; other model versions ignore this option. LTX and Wan preserve causal state between temporal tiles. Hunyuan Video and TAEHV use overlap blending. MiniMax H3 keeps its model-specific fixed temporal windows because its latent-to-frame mapping is non-linear. @@ -1247,7 +1258,7 @@ Example: Native image/video requests and SDAPI accept `image_preprocess` as a rule string or array of rule strings. OpenAI-compatible requests can supply it in -`sd_cpp_extra_args`. See [Image preprocessing](../../docs/image_preprocessing.md) +`sd_cpp_extra_args`. See [Image preprocessing](../../docs/guides/image_preprocessing.md) for one-time input geometry, native-resolution decoding, mask alignment, and `canny=true` for edge detection on any supported image input. @@ -1274,7 +1285,7 @@ and `ref_images[]`. RGB-only models drop it, so sending RGBA is safe for every m If omitted or null: -- single-image fields map to an empty `sd_image_t` +- single-image fields clear the corresponding image tensor - array fields map to an empty C-style array, represented as `pointer = nullptr` and `count = 0` ### Field Mapping Summary @@ -1608,7 +1619,7 @@ Example: - Other video model families currently ignore `ref_images`. - `features_by_mode.vid_gen` does not currently advertise `ref_images`; its absence is not an indication that MiniMax-H3 reference images are unsupported. -See [MiniMax-H3 reference conditioning](../../docs/minimax_h3.md#reference-to-audio-video-conditioning) for model requirements and prompt examples. +See [MiniMax-H3 reference conditioning](../../docs/models/minimax_h3.md#reference-to-audio-video-conditioning) for model requirements and prompt examples. ### LoRA Rules @@ -1637,7 +1648,7 @@ Frame ordering rules: If omitted or null: -- single-image fields map to an empty `sd_image_t` +- single-image fields clear the corresponding image tensor - array fields map to an empty C-style array, represented as `pointer = nullptr` and `count = 0` ### Field Mapping Summary @@ -1699,6 +1710,13 @@ Sampling fields: High-noise sampling fields: +The two video stages share the scheduler, flow shift and timestep shifting. +`high_noise_sample_params.scheduler`, `.flow_shift` and `.shifted_timestep` +are accepted as aliases for those shared settings in video requests. Image +requests ignore these high-noise fields. Within one video request, a +corresponding `sample_params` field takes precedence. Request aliases override +inherited startup and recipe values; `null` clears the inherited shared setting. + | Field | Type | | --- | --- | | `high_noise_sample_params.scheduler` | `string` | @@ -1903,5 +1921,5 @@ result items and OpenAI image items contain the parameter snapshot with each image's actual seed and without `batch_count`. Native video results contain the video request snapshot. SD API `info.generation_info` is an array aligned with `info.infotexts` and `info.all_seeds`; the outer `parameters` field -remains the request echo. See [generation metadata](../../docs/generation_metadata.md) +remains the request echo. See [generation metadata](../../docs/guides/generation_metadata.md) for field semantics, postprocessing and container storage. diff --git a/examples/server/async_jobs.cpp b/examples/server/async_jobs.cpp index c730b9c13d..bb88fb6aa5 100644 --- a/examples/server/async_jobs.cpp +++ b/examples/server/async_jobs.cpp @@ -43,14 +43,18 @@ class JobCancellationMonitor { } }; -static void preview_callback(int step, int frame_count, sd_image_t* frames, bool is_noisy, void* data) { +static void preview_callback(int step, int frame_count, const sd_tensor_t* const* tensors, bool is_noisy, void* data) { + std::vector views; + for (int i = 0; i < frame_count; ++i) + views.push_back(tensor_image_view(tensors[i])); + SDImage* frames = views.data(); (void)is_noisy; if (frame_count <= 0 || frames == nullptr || frames[0].data == nullptr) { return; } AsyncGenerationJob* job = static_cast(data); - const sd_image_t& frame = frames[0]; + const SDImage& frame = frames[0]; const sd_preview_info_t info = sd_get_preview_info(); auto image_bytes = encode_image_to_vector(EncodedImageFormat::PNG, @@ -73,23 +77,20 @@ static void preview_callback(int step, int frame_count, sd_image_t* frames, bool } static void clear_preview_callback() { - sd_set_preview_callback(nullptr, PREVIEW_NONE, 1, false, false, nullptr); + sd_set_preview_callback(nullptr, "none", 1, false, false, nullptr); } static void set_preview_callback(AsyncGenerationJob& job) { - preview_t mode = str_to_preview(job.kind == AsyncJobKind::ImgGen - ? job.img_gen.preview_mode.c_str() - : job.vid_gen.preview_mode.c_str()); - if (mode == PREVIEW_COUNT || mode == PREVIEW_NONE) { + const auto& mode = job.kind == AsyncJobKind::ImgGen ? job.img_gen.preview_mode : job.vid_gen.preview_mode; + if (mode == "none") return; - } int interval = job.kind == AsyncJobKind::ImgGen ? job.img_gen.preview_interval : job.vid_gen.preview_interval; if (interval <= 0) { interval = 1; } - sd_set_preview_callback(preview_callback, mode, interval, true, false, &job); + sd_set_preview_callback(preview_callback, mode.c_str(), interval, true, false, &job); } static void update_job_progress(AsyncGenerationJob& job, const char* stage, const char* detail = "", int sample_pass = -1, int64_t completed = 0, int64_t total = 0, const char* unit = "") { @@ -343,8 +344,8 @@ bool execute_img_gen_job(ServerRuntime& runtime, std::vector& output_images, json& output_info, std::string& error_message) { - output_info = json::array(); - sd_img_gen_params_t params = job.img_gen.to_sd_img_gen_params_t(); + output_info = json::array(); + SDOptPtr params = job.img_gen.to_image_opt(); SDImageVec results; @@ -355,15 +356,16 @@ bool execute_img_gen_job(ServerRuntime& runtime, } JobCancellationMonitor cancellation(runtime.sd_ctx, job); JobCallbacksGuard callbacks(runtime.sd_ctx, job); - sd_image_generation_result_t* generated = nullptr; - generate_image(runtime.sd_ctx, ¶ms, &generated); + PromptImageCallbackGuard prompt_images(runtime.sd_ctx, job.img_gen.gen_params); + sd_result_t* generated = nullptr; + generate_sd_image(runtime.sd_ctx, params, &generated); results.adopt(generated, runtime.ctx_params->lora_model_dir); if (job.cancel_requested.load()) { return false; } if (!results.empty() && job.img_gen.adetailer_enabled && !runtime.default_gen_params->ad_model_path.empty()) { callbacks.begin_detailing(); - if (!apply_server_adetailer(runtime, job.img_gen, params, results)) { + if (!apply_server_adetailer(runtime, job.img_gen, results)) { error_message = "ADetailer failed"; return false; } @@ -423,12 +425,12 @@ bool execute_vid_gen_job(ServerRuntime& runtime, int& output_fps, json& output_info, std::string& error_message) { - sd_vid_gen_params_t params = job.vid_gen.to_sd_vid_gen_params_t(); + SDOptPtr params = job.vid_gen.to_video_opt(); std::string str_params; SDImageVec results; - int num_results = 0; - sd_audio_t* generated_audio = nullptr; + int num_results = 0; + SDAudio* generated_audio = nullptr; { std::lock_guard lock(*runtime.sd_ctx_mutex); @@ -437,11 +439,12 @@ bool execute_vid_gen_job(ServerRuntime& runtime, } JobCancellationMonitor cancellation(runtime.sd_ctx, job); JobCallbacksGuard callbacks(runtime.sd_ctx, job); - sd_video_generation_result_t* generated = nullptr; - if (generate_video(runtime.sd_ctx, ¶ms, &generated)) { - output_fps = generated->fps; - generated_audio = generated->audio; - generated->audio = nullptr; + PromptImageCallbackGuard prompt_images(runtime.sd_ctx, job.vid_gen.gen_params); + sd_result_t* generated = nullptr; + if (generate_sd_video(runtime.sd_ctx, params, &generated)) { + output_fps = sd_result_get_fps(generated); + generated_audio = retain_result_audio(generated); + results.adopt_video(generated, runtime.ctx_params->lora_model_dir); output_info = json::parse(results.generation_info(0)); if (job.vid_gen.gen_params.embed_image_metadata) diff --git a/examples/server/main.cpp b/examples/server/main.cpp index 67d557a4e5..391a1ec9ad 100644 --- a/examples/server/main.cpp +++ b/examples/server/main.cpp @@ -52,8 +52,6 @@ static void parse_args(int argc, exit(ctx_params.print_devices() ? 0 : 1); } - const bool random_seed_requested = default_gen_params.seed < 0; - if (!svr_params.resolve_and_validate() || !ctx_params.resolve_and_validate(ctx_params.model_path.empty() && ctx_params.diffusion_model_path.empty() ? METADATA : IMG_GEN) || !default_gen_params.resolve_and_validate(IMG_GEN, @@ -62,10 +60,6 @@ static void parse_args(int argc, print_usage(argv[0], options_vec); exit(1); } - - if (random_seed_requested) { - default_gen_params.seed = -1; - } } void sd_log_cb(enum sd_log_level_t level, const char* log, void* data) { @@ -93,16 +87,16 @@ int main(int argc, const char** argv) { LOG_DEBUG("%s", ctx_params.to_string().c_str()); LOG_DEBUG("%s", default_gen_params.to_string().c_str()); - sd_ctx_params_t sd_ctx_params = ctx_params.to_sd_ctx_params_t(); + SDOptPtr sd_ctx_params = ctx_params.to_context_opt(); ServerModels models; if (!ctx_params.model_path.empty() || !ctx_params.diffusion_model_path.empty()) { const auto started = std::chrono::steady_clock::now(); - models.context.reset(new_sd_ctx(&sd_ctx_params)); + models.context.reset(create_sd_context(sd_ctx_params)); models.status["load_elapsed_seconds"] = std::chrono::duration(std::chrono::steady_clock::now() - started).count(); } if ((!ctx_params.model_path.empty() || !ctx_params.diffusion_model_path.empty()) && !models.context) { - LOG_ERROR("new_sd_ctx_t failed"); + LOG_ERROR("sd_ctx_create failed"); return 1; } diff --git a/examples/server/model_catalog.cpp b/examples/server/model_catalog.cpp index f665567096..5a538fd529 100644 --- a/examples/server/model_catalog.cpp +++ b/examples/server/model_catalog.cpp @@ -193,19 +193,26 @@ static void apply_recipe_defaults(const json& defaults, SDGenerationParams& para throw std::runtime_error(std::string(key) + " is out of range"); } if (std::string(key) == "width") - params.width = static_cast(value); + if (!params.width) + params.width = static_cast(value); if (std::string(key) == "height") - params.height = static_cast(value); + if (!params.height) + params.height = static_cast(value); if (std::string(key) == "steps") - params.sample_params.sample_steps = static_cast(value); + if (!params.sample_params.sample_steps) + params.sample_params.sample_steps = static_cast(value); if (field == "shifted_timestep") - params.sample_params.shifted_timestep = static_cast(value); + if (!params.sample_params.shifted_timestep) + params.sample_params.shifted_timestep = static_cast(value); if (field == "video_frames") - params.video_frames = static_cast(value); + if (!params.video_frames) + params.video_frames = static_cast(value); if (field == "fps") - params.fps = static_cast(value); + if (!params.fps) + params.fps = static_cast(value); if (field == "high_noise_steps") - params.high_noise_sample_params.sample_steps = static_cast(value); + if (!params.high_noise_sample_params.sample_steps) + params.high_noise_sample_params.sample_steps = static_cast(value); } for (const char* key : {"cfg_scale", "strength", "moe_boundary", "img_cfg_scale", "distilled_guidance", "flow_shift", "high_noise_cfg_scale", "high_noise_flow_shift"}) { if (!defaults.contains(key)) { @@ -219,21 +226,29 @@ static void apply_recipe_defaults(const json& defaults, SDGenerationParams& para throw std::runtime_error(std::string(key) + " is out of range"); } if (std::string(key) == "strength") - params.strength = value; + if (!params.strength) + params.strength = value; if (std::string(key) == "moe_boundary") - params.moe_boundary = value; + if (!params.moe_boundary) + params.moe_boundary = value; if (std::string(key) == "cfg_scale") - params.sample_params.guidance.txt_cfg = value; + if (!params.sample_params.guidance.txt_cfg) + params.sample_params.guidance.txt_cfg = value; if (std::string(key) == "img_cfg_scale") - params.sample_params.guidance.img_cfg = value; + if (!params.sample_params.guidance.img_cfg) + params.sample_params.guidance.img_cfg = value; if (std::string(key) == "distilled_guidance") - params.sample_params.guidance.distilled_guidance = value; + if (!params.sample_params.guidance.distilled_guidance) + params.sample_params.guidance.distilled_guidance = value; if (std::string(key) == "flow_shift") - params.sample_params.flow_shift = value; + if (!params.sample_params.flow_shift && !params.high_noise_sample_params.flow_shift) + params.sample_params.flow_shift = value; if (std::string(key) == "high_noise_cfg_scale") - params.high_noise_sample_params.guidance.txt_cfg = value; + if (!params.high_noise_sample_params.guidance.txt_cfg) + params.high_noise_sample_params.guidance.txt_cfg = value; if (std::string(key) == "high_noise_flow_shift") - params.high_noise_sample_params.flow_shift = value; + if (!params.sample_params.flow_shift && !params.high_noise_sample_params.flow_shift) + params.high_noise_sample_params.flow_shift = value; } if (defaults.contains("custom_sigmas")) { const auto& sigmas = defaults.at("custom_sigmas"); @@ -243,40 +258,28 @@ static void apply_recipe_defaults(const json& defaults, SDGenerationParams& para if (!sigma.is_number() || !std::isfinite(sigma.get()) || sigma.get() < 0) throw std::runtime_error("custom_sigmas must contain finite nonnegative numbers"); } - params.custom_sigmas = sigmas.get>(); + if (!params.custom_sigmas) + params.custom_sigmas = sigmas.get>(); } for (const char* key : {"sampling_method", "high_noise_sampling_method"}) { if (!defaults.contains(key)) continue; auto& sample = std::string(key) == "sampling_method" ? params.sample_params : params.high_noise_sample_params; const auto name = defaults.at(key).get(); - bool found = false; - for (int i = 0; i < SAMPLE_METHOD_COUNT; ++i) { - if (name == sd_sample_method_name(static_cast(i))) { - sample.sample_method = static_cast(i); - found = true; - break; - } - } - if (!found) + if (!example_options::contains(example_options::sample_methods, name)) throw std::runtime_error(std::string("unknown ") + key); + if (!sample.sample_method) + sample.sample_method = name; } for (const char* key : {"scheduler", "high_noise_scheduler"}) { if (!defaults.contains(key)) continue; auto& sample = std::string(key) == "scheduler" ? params.sample_params : params.high_noise_sample_params; const auto name = defaults.at(key).get(); - bool found = false; - for (int i = 0; i < SCHEDULER_COUNT; ++i) { - if (name == sd_scheduler_name(static_cast(i)) || - (name == "normal" && i == DISCRETE_SCHEDULER)) { - sample.scheduler = static_cast(i); - found = true; - break; - } - } - if (!found) + if (!example_options::contains(example_options::schedulers, name) && name != "normal") throw std::runtime_error(std::string("unknown ") + key); + if (!params.sample_params.scheduler && !params.high_noise_sample_params.scheduler) + sample.scheduler = name; } } @@ -286,16 +289,20 @@ static void apply_recipe_context(const json& settings, SDContextParams& params) const auto format = settings.at("vae_format").get(); if (format != "auto" && format != "flux" && format != "sd3" && format != "flux2" && format != "wan") throw std::runtime_error("unknown vae_format"); - params.vae_format = format; + if (!params.vae_format) + params.vae_format = format; } if (settings.contains("model_args")) { - params.model_args = settings.at("model_args").get(); + if (!params.model_args) + params.model_args = settings.at("model_args").get(); } if (settings.contains("backend")) { - params.backend = settings.at("backend").get(); + if (!params.backend) + params.backend = settings.at("backend").get(); } if (settings.contains("params_backend")) { - params.params_backend = settings.at("params_backend").get(); + if (!params.params_backend) + params.params_backend = settings.at("params_backend").get(); } } diff --git a/examples/server/recipes/README.md b/examples/server/recipes/README.md index dcf9d69ab3..9405fdb357 100644 --- a/examples/server/recipes/README.md +++ b/examples/server/recipes/README.md @@ -56,7 +56,7 @@ prompt, a style preset, or a collection of sampling settings. has different native inpainting/control conditioning, beyond its layer count ([model card](https://huggingface.co/ostris/Flex.2-preview)). SDXS uses an adjusted AutoEncoderTiny and a one-step model, rather than only a shallower - ordinary SD checkpoint ([runtime guide](../../../docs/distilled_sd.md#sdxs-512-dreamshaper)). + ordinary SD checkpoint ([runtime guide](../../../docs/models/distilled_sd.md#sdxs-512-dreamshaper)). Their native components stay in their own model recipe. - Quantization, file format, prompt wording, seeds, step counts and CFG values are not grounds for another recipe. Differences in required components, diff --git a/examples/server/recipes/defaults.md b/examples/server/recipes/defaults.md index 0e2b63cb1f..fdccc96a9c 100644 --- a/examples/server/recipes/defaults.md +++ b/examples/server/recipes/defaults.md @@ -70,11 +70,11 @@ For community edit adapters and other optional extensions, these are the base-mo | `ming-image` | 1 | 12 | 1024 × 1024 | [Source](https://huggingface.co/inclusionAI/Ming-Image-0.1-Design) | | `minimax-h3-fl2va` | 1 | — | 1344 × 768 | [Source](https://huggingface.co/docs/diffusers/main/en/api/pipelines/minimax_h3) Guidance-distilled; official 768p canvas. No fixed step count is prescribed here; the upstream API counts sigma grid points rather than sd.cpp denoising steps. | | `minimax-h3-ref2va` | 1 | — | 1344 × 768 | [Source](https://huggingface.co/docs/diffusers/main/en/api/pipelines/minimax_h3) Guidance-distilled; official 768p canvas. No fixed step count is prescribed here; the upstream API counts sigma grid points rather than sd.cpp denoising steps. | -| `minimax-h3-image` | 1 | 6 | 1024 × 1024 | [Runtime guide](../../../docs/minimax_h3_image.md) Dedicated image VAE and fixed PDD sigma schedule. | +| `minimax-h3-image` | 1 | 6 | 1024 × 1024 | [Runtime guide](../../../docs/models/minimax_h3_image.md) Dedicated image VAE and fixed PDD sigma schedule. | | `minit2i` | 2.5 | 20 | 512 × 512 | [Source](https://huggingface.co/MiniT2I/MiniT2I/raw/main/pipeline.py) B/16 example and native model size; L/16 uses CFG 6. | | `moss-image-base` | 3 | 20 | 1024 × 1024 | [Source](https://huggingface.co/OpenMOSS-Team/MOSS-Image-Base) | | `moss-image-sft` | 3 | 20 | 1024 × 1024 | [Source](https://huggingface.co/OpenMOSS-Team/MOSS-Image-SFT) | -| `p-h3-image` | 1 | 8 | 1024 × 1024 | [Runtime guide](../../../docs/p_h3_image.md) Fixed P-H3 sigma schedule; upstream 2912 × 2912 reduced to the catalog default. | +| `p-h3-image` | 1 | 8 | 1024 × 1024 | [Runtime guide](../../../docs/models/p_h3_image.md) Fixed P-H3 sigma schedule; upstream 2912 × 2912 reduced to the catalog default. | | `nitrosd-realism` | 1 | 1 | 1024 × 1024 | [Source](https://huggingface.co/ChenDY/NitroFusion) | | `ovis-image` | 5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/ATH-MaaS/Ovis-Image-7B) | | `pid` | 1 | 4 | 2048 × 2048 | [Source](https://huggingface.co/nvidia/PiD) Standalone PiD decoding at 2K; guidance is handled by the upstream generator. | diff --git a/examples/server/recipes/hyperflow-h3-flash-fl2va.json b/examples/server/recipes/hyperflow-h3-flash-fl2va.json index 64e79d5436..c8d9f74535 100644 --- a/examples/server/recipes/hyperflow-h3-flash-fl2va.json +++ b/examples/server/recipes/hyperflow-h3-flash-fl2va.json @@ -3,7 +3,7 @@ "id": "hyperflow-h3-flash-fl2va", "name": "HyperFlow H3 Flash FL2VA", "size_multiple": 32, - "description": "Generate video and audio from text, optionally with start/end images. Use the HyperFlow H3 Flash FL2VA transformer. Reuses MiniMax-H3 components with the HyperFlow 6-step schedule. Audio VAE is optional for silent video.", + "description": "Generate video and audio from text, optionally with start/end images. Use the HyperFlow H3 Flash FL2VA transformer. Reuses MiniMax-H3 components with either the MiniMax-H3 Qwen3-VL-32B text encoder or Qwen3-VL-4B with the P-H3 text connector and the HyperFlow 6-step schedule. Audio VAE is optional for silent video.", "model_url": "https://huggingface.co/VideoRebirthAI/hyperflow-h3-flash", "slots": [ { @@ -14,7 +14,17 @@ { "id": "llm", "required": true, - "label": "MiniMax-H3 Qwen 3 VL 32B Text Encoder" + "label": "MiniMax-H3 Qwen 3 VL 32B or P-H3 Qwen 3 VL 4B Text Encoder" + }, + { + "id": "llm_vision", + "required": false, + "label": "Vision tower (when not included in the text encoder)" + }, + { + "id": "embeddings_connectors", + "required": false, + "label": "P-H3 Text Connector (for Qwen 3 VL 4B, when not included in the text encoder)" }, { "id": "vae", diff --git a/examples/server/recipes/hyperflow-h3-flash-ref2va.json b/examples/server/recipes/hyperflow-h3-flash-ref2va.json index 871a69063d..bc7cde123f 100644 --- a/examples/server/recipes/hyperflow-h3-flash-ref2va.json +++ b/examples/server/recipes/hyperflow-h3-flash-ref2va.json @@ -3,7 +3,7 @@ "id": "hyperflow-h3-flash-ref2va", "name": "HyperFlow H3 Flash Ref2VA", "size_multiple": 32, - "description": "Generate video and audio with reference images. Use the HyperFlow H3 Flash Ref2VA transformer; video/audio references are not validated for these weights. Reuses MiniMax-H3 components with the HyperFlow 6-step schedule. Audio VAE is optional for silent video.", + "description": "Generate video and audio with reference images. Use the HyperFlow H3 Flash Ref2VA transformer; video/audio references are not validated for these weights. Reuses MiniMax-H3 components with either the MiniMax-H3 Qwen3-VL-32B text encoder or Qwen3-VL-4B with the P-H3 text connector and the HyperFlow 6-step schedule. Reference images need the matching vision tower. Audio VAE is optional for silent video.", "model_url": "https://huggingface.co/VideoRebirthAI/hyperflow-h3-flash", "slots": [ { @@ -14,7 +14,17 @@ { "id": "llm", "required": true, - "label": "MiniMax-H3 Qwen 3 VL 32B Text Encoder" + "label": "MiniMax-H3 Qwen 3 VL 32B or P-H3 Qwen 3 VL 4B Text Encoder" + }, + { + "id": "llm_vision", + "required": false, + "label": "Vision tower (required for reference images when not included in the text encoder)" + }, + { + "id": "embeddings_connectors", + "required": false, + "label": "P-H3 Text Connector (for Qwen 3 VL 4B, when not included in the text encoder)" }, { "id": "vae", diff --git a/examples/server/recipes/minimax-h3-fl2va.json b/examples/server/recipes/minimax-h3-fl2va.json index f0af22bd9d..63f03197d3 100644 --- a/examples/server/recipes/minimax-h3-fl2va.json +++ b/examples/server/recipes/minimax-h3-fl2va.json @@ -3,7 +3,7 @@ "id": "minimax-h3-fl2va", "name": "MiniMax H3 FL2VA", "size_multiple": 32, - "description": "Generate video and audio, optionally with start/end images. Supports full and pruned FL2VA weights. Audio VAE is optional for silent video.", + "description": "Generate video and audio, optionally with start/end images. Supports full and pruned FL2VA weights with either the MiniMax-H3 Qwen3-VL-32B text encoder or Qwen3-VL-4B with the P-H3 text connector. Audio VAE is optional for silent video.", "model_url": "https://huggingface.co/MiniMaxAI/MiniMax-H3", "slots": [ { @@ -13,7 +13,17 @@ { "id": "llm", "required": true, - "label": "MiniMax-H3 Qwen 3 VL 32B Text Encoder" + "label": "MiniMax-H3 Qwen 3 VL 32B or P-H3 Qwen 3 VL 4B Text Encoder" + }, + { + "id": "llm_vision", + "required": false, + "label": "Vision tower (when not included in the text encoder)" + }, + { + "id": "embeddings_connectors", + "required": false, + "label": "P-H3 Text Connector (for Qwen 3 VL 4B, when not included in the text encoder)" }, { "id": "vae", diff --git a/examples/server/recipes/minimax-h3-image.json b/examples/server/recipes/minimax-h3-image.json index 709cd4e55b..0774b5b19d 100644 --- a/examples/server/recipes/minimax-h3-image.json +++ b/examples/server/recipes/minimax-h3-image.json @@ -3,7 +3,7 @@ "id": "minimax-h3-image", "name": "MiniMax H3 Image", "size_multiple": 32, - "description": "Text-to-image and reference-image editing with the six-step PDD checkpoint. Requires the MiniMax-H3-Image 2D VAE and MiniMax-H3 Qwen3-VL-32B text encoder with its vision tower.", + "description": "Text-to-image and reference-image editing with the six-step PDD checkpoint. Requires the MiniMax-H3-Image 2D VAE and either the MiniMax-H3 Qwen3-VL-32B text encoder with its vision tower or the smaller P-H3-Image text encoder (Qwen3-VL-4B with the P-H3 text connector).", "model_url": "https://huggingface.co/MiniMaxAI/MiniMax-H3-Image", "slots": [ { @@ -13,13 +13,18 @@ { "id": "llm", "required": true, - "label": "MiniMax-H3 Qwen 3 VL 32B Text Encoder" + "label": "MiniMax-H3 Qwen 3 VL 32B or P-H3 Qwen 3 VL 4B Text Encoder" }, { "id": "llm_vision", "required": false, "label": "Vision tower (required when not included in the text encoder)" }, + { + "id": "embeddings_connectors", + "required": false, + "label": "P-H3 Text Connector (for Qwen 3 VL 4B, when not included in the text encoder)" + }, { "id": "vae", "required": true, diff --git a/examples/server/recipes/minimax-h3-ref2va.json b/examples/server/recipes/minimax-h3-ref2va.json index 9ee5f63d7d..61f29db33a 100644 --- a/examples/server/recipes/minimax-h3-ref2va.json +++ b/examples/server/recipes/minimax-h3-ref2va.json @@ -3,7 +3,7 @@ "id": "minimax-h3-ref2va", "name": "MiniMax H3 Ref2VA", "size_multiple": 32, - "description": "Supply image, video or audio references with compatible Ref2VA weights. Audio VAE is optional for silent video.", + "description": "Supply image, video or audio references with compatible Ref2VA weights. Use either the MiniMax-H3 Qwen3-VL-32B text encoder or Qwen3-VL-4B with the P-H3 text connector. Image/video references need the matching vision tower. Audio VAE is optional for silent video.", "model_url": "https://huggingface.co/MiniMaxAI/MiniMax-H3", "slots": [ { @@ -13,7 +13,17 @@ { "id": "llm", "required": true, - "label": "MiniMax-H3 Qwen 3 VL 32B Text Encoder" + "label": "MiniMax-H3 Qwen 3 VL 32B or P-H3 Qwen 3 VL 4B Text Encoder" + }, + { + "id": "llm_vision", + "required": false, + "label": "Vision tower (required for image/video references when not included in the text encoder)" + }, + { + "id": "embeddings_connectors", + "required": false, + "label": "P-H3 Text Connector (for Qwen 3 VL 4B, when not included in the text encoder)" }, { "id": "vae", diff --git a/examples/server/recipes/p-h3-image.json b/examples/server/recipes/p-h3-image.json index 1098bab9eb..a2b9e31cc8 100644 --- a/examples/server/recipes/p-h3-image.json +++ b/examples/server/recipes/p-h3-image.json @@ -3,7 +3,7 @@ "id": "p-h3-image", "name": "P-H3-Image", "size_multiple": 32, - "description": "Supports text-to-image and reference-image editing with the fixed 8-step schedule. Use the P-H3 single-frame VAE and Qwen3-VL-4B with its vision tower. Supply the P-H3 text connector separately unless it is included in the text encoder.", + "description": "Supports text-to-image and reference-image editing with the fixed 8-step schedule. Use the P-H3 single-frame VAE and either Qwen3-VL-4B with the P-H3 text connector or the MiniMax-H3 Qwen3-VL-32B text encoder. Reference images need the matching vision tower.", "model_url": "https://huggingface.co/PrunaAI/P-H3-Image", "slots": [ { @@ -13,7 +13,7 @@ { "id": "llm", "required": true, - "label": "Qwen 3 VL 4B Text Encoder" + "label": "MiniMax-H3 Qwen 3 VL 32B or P-H3 Qwen 3 VL 4B Text Encoder" }, { "id": "llm_vision", @@ -23,7 +23,7 @@ { "id": "embeddings_connectors", "required": false, - "label": "P-H3 Text Connector (required when not included in the text encoder)" + "label": "P-H3 Text Connector (for Qwen 3 VL 4B, when not included in the text encoder)" }, { "id": "vae", diff --git a/examples/server/routes_models.cpp b/examples/server/routes_models.cpp index f33057758c..9dbd6ed761 100644 --- a/examples/server/routes_models.cpp +++ b/examples/server/routes_models.cpp @@ -112,8 +112,8 @@ static void change_model(ServerRuntime& runtime, const httplib::Request& req, ht *runtime.ctx_params = unload ? models.base_context : std::move(context); *runtime.default_gen_params = unload ? models.base_defaults : std::move(defaults); if (!unload) { - auto params = runtime.ctx_params->to_sd_ctx_params_t(); - models.context.reset(new_sd_ctx(¶ms)); + auto params = runtime.ctx_params->to_context_opt(); + models.context.reset(create_sd_context(params)); if (!models.context) { throw std::runtime_error("Model initialization failed; check the server log for component compatibility and memory errors"); } diff --git a/examples/server/routes_openai.cpp b/examples/server/routes_openai.cpp index f721e1495d..e67bd20159 100644 --- a/examples/server/routes_openai.cpp +++ b/examples/server/routes_openai.cpp @@ -35,8 +35,8 @@ static bool build_openai_generation_request(const httplib::Request& req, std::string size = j.value("size", ""); std::string output_format = j.value("output_format", "png"); int output_compression = j.value("output_compression", 100); - int width = runtime.default_gen_params->width > 0 ? runtime.default_gen_params->width : 512; - int height = runtime.default_gen_params->height > 0 ? runtime.default_gen_params->height : 512; + auto width = runtime.default_gen_params->width; + auto height = runtime.default_gen_params->height; if (!size.empty()) { auto pos = size.find('x'); if (pos != std::string::npos) { @@ -94,10 +94,6 @@ static bool build_openai_edit_request(const httplib::Request& req, size_t image_count = req.form.get_file_count("image[]"); bool has_legacy_image = req.form.has_file("image"); - if (image_count == 0 && !has_legacy_image) { - error_message = "at least one image[] required"; - return false; - } std::vector> images_bytes; for (size_t i = 0; i < image_count; ++i) { @@ -124,8 +120,8 @@ static bool build_openai_edit_request(const httplib::Request& req, } std::string size = req.form.get_field("size"); - int width = -1; - int height = -1; + auto width = runtime.default_gen_params->width; + auto height = runtime.default_gen_params->height; if (!size.empty()) { auto pos = size.find('x'); if (pos != std::string::npos) { @@ -189,12 +185,12 @@ static bool build_openai_edit_request(const httplib::Request& req, mask_w, mask_h, mask_channel, 0, 0, 1); request.gen_params.mask_image.reset({(uint32_t)mask_w, (uint32_t)mask_h, 1, mask_raw}); - const sd_image_t& mask_image = request.gen_params.mask_image.get(); + const SDImage& mask_image = request.gen_params.mask_image.get(); request.gen_params.set_width_and_height_if_unset(mask_image.width, mask_image.height); } else { request.gen_params.mask_image.reset({ - (uint32_t)request.gen_params.get_resolved_width(), - (uint32_t)request.gen_params.get_resolved_height(), + (uint32_t)request.gen_params.get_requested_width(), + (uint32_t)request.gen_params.get_requested_height(), 1, nullptr, }); @@ -204,6 +200,10 @@ static bool build_openai_edit_request(const httplib::Request& req, error_message = "invalid sd_cpp_extra_args"; return false; } + if (images_bytes.empty() && request.gen_params.prompt_images.empty()) { + error_message = "at least one image[] or prompt_images in sd_cpp_extra_args required"; + return false; + } // Intentionally disable prompt-embedded LoRA tag parsing for server APIs. if (!request.gen_params.resolve_and_validate(IMG_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) { @@ -218,16 +218,17 @@ static bool execute_sync_img_gen_request(ServerRuntime& runtime, ImgGenJobRequest& request, SDImageVec& results, std::string& error_message) { - sd_img_gen_params_t img_gen_params = request.to_sd_img_gen_params_t(); - int num_results = 0; + SDOptPtr img_gen_params = request.to_image_opt(); + int num_results = 0; { std::lock_guard lock(*runtime.sd_ctx_mutex); - sd_image_generation_result_t* generated = nullptr; - generate_image(runtime.sd_ctx, &img_gen_params, &generated); + PromptImageCallbackGuard prompt_images(runtime.sd_ctx, request.gen_params); + sd_result_t* generated = nullptr; + generate_sd_image(runtime.sd_ctx, img_gen_params, &generated); results.adopt(generated, runtime.ctx_params->lora_model_dir); num_results = results.count(); - if (!results.empty() && !apply_server_adetailer(runtime, request, img_gen_params, results)) { + if (!results.empty() && !apply_server_adetailer(runtime, request, results)) { error_message = "ADetailer failed"; return false; } diff --git a/examples/server/routes_sdapi.cpp b/examples/server/routes_sdapi.cpp index 0d37898a8d..e9f60c7555 100644 --- a/examples/server/routes_sdapi.cpp +++ b/examples/server/routes_sdapi.cpp @@ -47,41 +47,38 @@ static std::string lower_ascii(std::string value) { return value; } -static enum sample_method_t get_sdapi_sample_method(std::string name) { - enum sample_method_t result = str_to_sample_method(name.c_str()); - if (result != SAMPLE_METHOD_COUNT) { - return result; - } - +static std::string get_sdapi_sample_method(std::string name) { + if (example_options::contains(example_options::sample_methods, name)) + return name; name = lower_ascii(name); - static const std::unordered_map hardcoded{ - {"euler a", EULER_A_SAMPLE_METHOD}, - {"k_euler_a", EULER_A_SAMPLE_METHOD}, - {"euler", EULER_SAMPLE_METHOD}, - {"k_euler", EULER_SAMPLE_METHOD}, - {"heun", HEUN_SAMPLE_METHOD}, - {"k_heun", HEUN_SAMPLE_METHOD}, - {"dpm2", DPM2_SAMPLE_METHOD}, - {"k_dpm_2", DPM2_SAMPLE_METHOD}, - {"lcm", LCM_SAMPLE_METHOD}, - {"ddim", DDIM_TRAILING_SAMPLE_METHOD}, - {"dpm++ 2m", DPMPP2M_SAMPLE_METHOD}, - {"k_dpmpp_2m", DPMPP2M_SAMPLE_METHOD}, - {"dpm++ 2m sde", DPMPP2M_SDE_SAMPLE_METHOD}, - {"k_dpmpp_2m_sde", DPMPP2M_SDE_SAMPLE_METHOD}, - {"dpm++ 2m sde gpu", DPMPP2M_SDE_BT_SAMPLE_METHOD}, - {"k_dpmpp_2m_sde_gpu", DPMPP2M_SDE_BT_SAMPLE_METHOD}, - {"res multistep", RES_MULTISTEP_SAMPLE_METHOD}, - {"k_res_multistep", RES_MULTISTEP_SAMPLE_METHOD}, - {"res 2s", RES_2S_SAMPLE_METHOD}, - {"k_res_2s", RES_2S_SAMPLE_METHOD}, - {"euler_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD}, - {"k_euler_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD}, - {"euler_a_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD}, - {"k_euler_a_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD}, + static const std::unordered_map hardcoded{ + {"euler a", "euler_a"}, + {"k_euler_a", "euler_a"}, + {"euler", "euler"}, + {"k_euler", "euler"}, + {"heun", "heun"}, + {"k_heun", "heun"}, + {"dpm2", "dpm2"}, + {"k_dpm_2", "dpm2"}, + {"lcm", "lcm"}, + {"ddim", "ddim_trailing"}, + {"dpm++ 2m", "dpm++2m"}, + {"k_dpmpp_2m", "dpm++2m"}, + {"dpm++ 2m sde", "dpm++2m_sde"}, + {"k_dpmpp_2m_sde", "dpm++2m_sde"}, + {"dpm++ 2m sde gpu", "dpm++2m_sde_bt"}, + {"k_dpmpp_2m_sde_gpu", "dpm++2m_sde_bt"}, + {"res multistep", "res_multistep"}, + {"k_res_multistep", "res_multistep"}, + {"res 2s", "res_2s"}, + {"k_res_2s", "res_2s"}, + {"euler_cfg_pp", "euler_cfg_pp"}, + {"k_euler_cfg_pp", "euler_cfg_pp"}, + {"euler_a_cfg_pp", "euler_a_cfg_pp"}, + {"k_euler_a_cfg_pp", "euler_a_cfg_pp"}, }; auto it = hardcoded.find(name); - return it != hardcoded.end() ? it->second : SAMPLE_METHOD_COUNT; + return it != hardcoded.end() ? it->second : std::string{}; } static bool parse_sdapi_adetailer(const json& body, @@ -191,10 +188,11 @@ static bool parse_sdapi_adetailer(const json& body, ServerADetailerPass pass; pass.prompt = text("ad_prompt", ""); pass.negative_prompt = text("ad_negative_prompt", ""); - pass.width = request.gen_params.get_resolved_width(); - pass.height = request.gen_params.get_resolved_height(); - pass.strength = static_cast(number("ad_denoising_strength", 0.4, 0, 1)); - auto append = [&](const char* key, const std::string& value) { + pass.width = request.gen_params.width; + pass.height = request.gen_params.height; + if (config.contains("ad_denoising_strength")) + pass.strength = static_cast(number("ad_denoising_strength", 0, 0, 1)); + auto append = [&](const char* key, const std::string& value) { if (!pass.extra_args.empty()) pass.extra_args += ","; pass.extra_args += std::string(key) + "=" + value; @@ -218,7 +216,7 @@ static bool parse_sdapi_adetailer(const json& body, append("mask_min_ratio", json(min_ratio).dump()); append("mask_max_ratio", json(max_ratio).dump()); // Bound pixel operations before converting them to the native mask parameters. - const int mask_limit = std::clamp(std::max(pass.width, pass.height), 32, 16384); + const int mask_limit = k_max_generation_dimension; append_integer("dilate_erode", "ad_dilate_erode", 4, -mask_limit, mask_limit); append_integer("x_offset", "ad_x_offset", 0, -mask_limit, mask_limit); append_integer("y_offset", "ad_y_offset", 0, -mask_limit, mask_limit); @@ -231,8 +229,12 @@ static bool parse_sdapi_adetailer(const json& body, append("mask_mode", merge == "None" ? "none" : merge == "Merge" ? "merge" : "merge_invert"); if (boolean("ad_use_inpaint_width_height", false)) { - pass.width = integer("ad_inpaint_width", 512, 1, k_max_generation_dimension); - pass.height = integer("ad_inpaint_height", 512, 1, k_max_generation_dimension); + pass.width.reset(); + pass.height.reset(); + if (config.contains("ad_inpaint_width")) + pass.width = integer("ad_inpaint_width", 0, 1, k_max_generation_dimension); + if (config.contains("ad_inpaint_height")) + pass.height = integer("ad_inpaint_height", 0, 1, k_max_generation_dimension); } if (boolean("ad_use_steps", false)) append_integer("steps", "ad_steps", 28, 1, k_max_adetailer_steps); @@ -251,20 +253,20 @@ static bool parse_sdapi_adetailer(const json& body, if (legacy_karras) sampler.resize(sampler.size() - suffix.size()); const auto method = get_sdapi_sample_method(sampler); - if (method == SAMPLE_METHOD_COUNT) + if (method.empty()) throw std::invalid_argument("unsupported ADetailer sampler"); - append("sample_method", sd_sample_method_name(method)); + append("sample_method", method); if (legacy_karras && text("ad_scheduler", "Use same scheduler") == "Use same scheduler") - append("scheduler", sd_scheduler_name(KARRAS_SCHEDULER)); + append("scheduler", "karras"); } const auto scheduler = text("ad_scheduler", "Use same scheduler"); if (scheduler == "Automatic") { pass.automatic_scheduler = true; } else if (scheduler != "Use same scheduler") { - const auto value = str_to_scheduler(scheduler.c_str()); - if (value == SCHEDULER_COUNT) + const auto value = scheduler; + if (!example_options::contains(example_options::schedulers, value) && value != "normal") throw std::invalid_argument("unsupported ADetailer scheduler"); - append("scheduler", sd_scheduler_name(value)); + append("scheduler", value == "normal" ? "discrete" : value); } } request.adetailer_passes.push_back(std::move(pass)); @@ -287,21 +289,9 @@ static bool build_sdapi_img_gen_request(const json& j, std::string& error_message) { std::string prompt = j.value("prompt", ""); std::string negative_prompt = j.value("negative_prompt", ""); - int width = j.value("width", 512); - int height = j.value("height", 512); - int steps = j.value("steps", runtime.default_gen_params->sample_params.sample_steps); - float cfg_scale = j.value("cfg_scale", runtime.default_gen_params->sample_params.guidance.txt_cfg); - int64_t seed = j.value("seed", -1); - int batch_size = j.value("batch_size", 1); - int clip_skip = j.value("clip_skip", -1); std::string sampler_name = j.value("sampler_name", ""); std::string scheduler_name = j.value("scheduler", ""); - if (width <= 0 || height <= 0) { - error_message = "width and height must be positive"; - return false; - } - if (prompt.empty()) { error_message = "prompt required"; return false; @@ -309,15 +299,20 @@ static bool build_sdapi_img_gen_request(const json& j, request.gen_params = *runtime.default_gen_params; - request.gen_params.prompt = prompt; - request.gen_params.negative_prompt = negative_prompt; - request.gen_params.seed = seed; - request.gen_params.sample_params.sample_steps = steps; - request.gen_params.batch_count = batch_size; - request.gen_params.sample_params.guidance.txt_cfg = cfg_scale; - const bool recipe_dimensions = !img2img && model_status(runtime)["recipe"].is_string(); - request.gen_params.width = j.value("width", recipe_dimensions ? runtime.default_gen_params->width : -1); - request.gen_params.height = j.value("height", recipe_dimensions ? runtime.default_gen_params->height : -1); + request.gen_params.prompt = prompt; + if (j.contains("negative_prompt")) + request.gen_params.negative_prompt = negative_prompt; + auto load_input = [&](const char* key, auto& value) { + if (j.contains(key)) + j.at(key).get_to(value); + }; + load_input("seed", request.gen_params.seed); + load_input("steps", request.gen_params.sample_params.sample_steps); + load_input("batch_size", request.gen_params.batch_count); + load_input("cfg_scale", request.gen_params.sample_params.guidance.txt_cfg); + load_input("width", request.gen_params.width); + load_input("height", request.gen_params.height); + load_input("clip_skip", request.gen_params.clip_skip); if (!img2img && j.value("enable_hr", false)) { request.gen_params.hires_enabled = true; @@ -337,17 +332,13 @@ static bool build_sdapi_img_gen_request(const json& j, return false; } - if (clip_skip > 0) { - request.gen_params.clip_skip = clip_skip; - } - - enum sample_method_t sample_method = get_sdapi_sample_method(sampler_name); - if (sample_method != SAMPLE_METHOD_COUNT) { + const auto sample_method = get_sdapi_sample_method(sampler_name); + if (!sample_method.empty()) { request.gen_params.sample_params.sample_method = sample_method; } - enum scheduler_t scheduler = str_to_scheduler(scheduler_name.c_str()); - if (scheduler != SCHEDULER_COUNT) { + const auto scheduler = scheduler_name; + if (example_options::contains(example_options::schedulers, scheduler) || scheduler == "normal") { request.gen_params.sample_params.scheduler = scheduler; } @@ -404,7 +395,7 @@ static bool build_sdapi_img_gen_request(const json& j, 0, 0, request.gen_params.init_image)) { - const sd_image_t& image = request.gen_params.init_image.get(); + const SDImage& image = request.gen_params.init_image.get(); request.gen_params.set_width_and_height_if_unset(image.width, image.height); } } @@ -415,10 +406,10 @@ static bool build_sdapi_img_gen_request(const json& j, 0, 0, request.gen_params.mask_image)) { - const sd_image_t& image = request.gen_params.mask_image.get(); + const SDImage& image = request.gen_params.mask_image.get(); request.gen_params.set_width_and_height_if_unset(image.width, image.height); } - sd_image_t& mask_image = request.gen_params.mask_image.get(); + SDImage& mask_image = request.gen_params.mask_image.get(); bool inpainting_mask_invert = j.value("inpainting_mask_invert", 0) != 0; if (inpainting_mask_invert && mask_image.data != nullptr) { for (uint32_t i = 0; i < mask_image.width * mask_image.height; ++i) { @@ -445,7 +436,7 @@ static bool build_sdapi_img_gen_request(const json& j, 0, 0, 0, image_owner)) { - const sd_image_t& image = image_owner.get(); + const SDImage& image = image_owner.get(); request.gen_params.set_width_and_height_if_unset(image.width, image.height); request.gen_params.ref_images.push_back(std::move(image_owner)); } @@ -499,14 +490,18 @@ struct SDAPIPreviewState { static SDAPIPreviewState g_preview; -static void preview_callback(int step, int frame_count, sd_image_t* frames, bool is_noisy, void* data) { +static void preview_callback(int step, int frame_count, const sd_tensor_t* const* tensors, bool is_noisy, void* data) { + std::vector views; + for (int i = 0; i < frame_count; ++i) + views.push_back(tensor_image_view(tensors[i])); + SDImage* frames = views.data(); (void)is_noisy; (void)data; if (frame_count <= 0 || frames == nullptr || frames[0].data == nullptr) { return; } - const sd_image_t& frame = frames[0]; + const SDImage& frame = frames[0]; const sd_preview_info_t info = sd_get_preview_info(); auto image_bytes = encode_image_to_vector(EncodedImageFormat::JPEG, @@ -530,7 +525,7 @@ static void preview_callback(int step, int frame_count, sd_image_t* frames, bool } struct SDAPIPreviewGuard { - SDAPIPreviewGuard(const std::string& id_task, const sd_img_gen_params_t& params, const std::string& job_type, bool detailing = false) { + SDAPIPreviewGuard(const std::string& id_task, const SDGenerationParams& params, const std::string& job_type, bool detailing = false) { std::lock_guard lock(g_preview.mutex); g_preview.id_task = id_task; g_preview.total = 0; @@ -539,13 +534,13 @@ struct SDAPIPreviewGuard { g_preview.step = 0; g_preview.active = true; // Each batch item has a base pass and, when enabled, a hires pass. - g_preview.job_count = static_cast(std::max(1, params.batch_count)) * (params.hires.enabled ? 2 : 1) + (detailing ? 1 : 0); + g_preview.job_count = static_cast(std::max(1, params.batch_count.value_or(1))) * (params.hires_enabled.value_or(false) ? 2 : 1) + (detailing ? 1 : 0); g_preview.job_no = 0; - sd_set_preview_callback(preview_callback, PREVIEW_PROJ, 1, true, false, nullptr); + sd_set_preview_callback(preview_callback, "proj", 1, true, false, nullptr); } void begin_detailing() { - sd_set_preview_callback(nullptr, PREVIEW_NONE, 1, false, false, nullptr); + sd_set_preview_callback(nullptr, "none", 1, false, false, nullptr); std::lock_guard lock(g_preview.mutex); g_preview.job_type = "adetailer"; g_preview.job_no = static_cast(g_preview.job_count - 1); @@ -554,7 +549,7 @@ struct SDAPIPreviewGuard { } ~SDAPIPreviewGuard() { - sd_set_preview_callback(nullptr, PREVIEW_NONE, 1, false, false, nullptr); + sd_set_preview_callback(nullptr, "none", 1, false, false, nullptr); std::lock_guard lock(g_preview.mutex); g_preview.active = false; g_preview.b64.clear(); @@ -594,21 +589,22 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) { LOG_DEBUG("%s\n", request.gen_params.to_string().c_str()); - sd_img_gen_params_t img_gen_params = request.to_sd_img_gen_params_t(); + SDOptPtr img_gen_params = request.to_image_opt(); SDImageVec results; int num_results = 0; { std::lock_guard lock(*runtime->sd_ctx_mutex); + PromptImageCallbackGuard prompt_images(runtime->sd_ctx, request.gen_params); const bool detailing = request.adetailer_enabled && !runtime->default_gen_params->ad_model_path.empty(); - SDAPIPreviewGuard preview_guard(id_task, img_gen_params, img2img ? "img2img" : "txt2img", detailing); - sd_image_generation_result_t* generated = nullptr; - generate_image(runtime->sd_ctx, &img_gen_params, &generated); + SDAPIPreviewGuard preview_guard(id_task, request.gen_params, img2img ? "img2img" : "txt2img", detailing); + sd_result_t* generated = nullptr; + generate_sd_image(runtime->sd_ctx, img_gen_params, &generated); results.adopt(generated, runtime->ctx_params->lora_model_dir); num_results = results.count(); if (detailing && !results.empty()) preview_guard.begin_detailing(); - if (!results.empty() && !apply_server_adetailer(*runtime, request, img_gen_params, results)) { + if (!results.empty() && !apply_server_adetailer(*runtime, request, results)) { res.status = 500; res.set_content(R"({"error":"ADetailer failed"})", "application/json"); return; @@ -747,8 +743,8 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) { routes.get("/sdapi/v1/samplers", [runtime](const httplib::Request&, httplib::Response& res) { std::vector sampler_names; sampler_names.push_back("default"); - for (int i = 0; i < SAMPLE_METHOD_COUNT; i++) { - sampler_names.push_back(sd_sample_method_name((sample_method_t)i)); + for (const char* name : example_options::sample_methods) { + sampler_names.push_back(name); } json r = json::array(); for (auto name : sampler_names) { @@ -764,9 +760,9 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) { routes.get("/sdapi/v1/schedulers", [runtime](const httplib::Request&, httplib::Response& res) { std::vector scheduler_names; scheduler_names.push_back("default"); - for (int i = 0; i < SCHEDULER_COUNT; i++) { - scheduler_names.push_back(sd_scheduler_name((scheduler_t)i)); - if (i == DISCRETE_SCHEDULER) { + for (const std::string name : example_options::schedulers) { + scheduler_names.push_back(name); + if (name == "discrete") { scheduler_names.push_back("normal"); } } diff --git a/examples/server/routes_sdcpp.cpp b/examples/server/routes_sdcpp.cpp index 1d11b89adc..c6e49710b1 100644 --- a/examples/server/routes_sdcpp.cpp +++ b/examples/server/routes_sdcpp.cpp @@ -15,7 +15,7 @@ namespace fs = std::filesystem; static constexpr uint32_t k_max_upscale_dimension = 8192; -static bool valid_upscale_dimensions(const sd_image_t& image, int factor, int repeats) { +static bool valid_upscale_dimensions(const SDImage& image, int factor, int repeats) { if (image.width == 0 || image.height == 0 || factor < 1 || repeats < 1 || repeats > 4) { return false; } @@ -31,51 +31,20 @@ static bool valid_upscale_dimensions(const sd_image_t& image, int factor, int re return true; } -static bool parse_cache_mode(const std::string& mode_str, sd_cache_mode_t& mode_out) { - if (mode_str == "disabled") { - mode_out = SD_CACHE_DISABLED; - return true; - } - if (mode_str == "easycache") { - mode_out = SD_CACHE_EASYCACHE; - return true; - } - if (mode_str == "ucache") { - mode_out = SD_CACHE_UCACHE; - return true; - } - if (mode_str == "dbcache") { - mode_out = SD_CACHE_DBCACHE; - return true; - } - if (mode_str == "taylorseer") { - mode_out = SD_CACHE_TAYLORSEER; - return true; - } - if (mode_str == "cache-dit") { - mode_out = SD_CACHE_CACHE_DIT; - return true; - } - if (mode_str == "spectrum") { - mode_out = SD_CACHE_SPECTRUM; - return true; - } - return false; -} - -static json finite_number_or_null(float value) { - return std::isfinite(value) ? json(value) : json(nullptr); -} - -static const char* capability_scheduler_name(enum scheduler_t scheduler) { - return scheduler < SCHEDULER_COUNT ? sd_scheduler_name(scheduler) : "default"; +static bool parse_cache_mode(const std::string& value, std::string& mode) { + const std::string canonical = value == "cache-dit" ? "cache_dit" : value; + SDOptPtr opt(sd_img_gen_opt_create()); + if (!opt || sd_opt_set_string(opt.get(), "cache.mode", canonical.c_str()) != SD_STATUS_OK) + return false; + mode = canonical; + return true; } -static const char* capability_sample_method_name(enum sample_method_t sample_method) { - return sample_method < SAMPLE_METHOD_COUNT ? sd_sample_method_name(sample_method) : "default"; +static json finite_number_or_null(const std::optional& value) { + return value && std::isfinite(*value) ? json(*value) : json(nullptr); } -static json make_vae_tiling_json(const sd_tiling_params_t& params) { +static json make_vae_tiling_json(const SDTilingParams& params) { return { {"enabled", params.enabled}, {"temporal_tiling", params.temporal_tiling}, @@ -99,11 +68,11 @@ static fs::path resolve_display_model_path(const ServerRuntime& runtime) { return {}; } -static json make_sample_params_json(const sd_sample_params_t& sample_params, const std::vector& skip_layers, const std::vector& custom_sigmas = {}) { +static json make_sample_params_json(const SDSampleParams& sample_params, const std::optional>& skip_layers, const std::optional>& custom_sigmas = {}) { const auto& guidance = sample_params.guidance; return { - {"scheduler", capability_scheduler_name(sample_params.scheduler)}, - {"sample_method", capability_sample_method_name(sample_params.sample_method)}, + {"scheduler", sample_params.scheduler}, + {"sample_method", sample_params.sample_method}, {"sample_steps", sample_params.sample_steps}, {"custom_sigmas", custom_sigmas}, {"eta", finite_number_or_null(sample_params.eta)}, @@ -144,8 +113,8 @@ static json make_img_gen_defaults_json(const SDGenerationParams& defaults, const {"prompt", defaults.prompt}, {"negative_prompt", defaults.negative_prompt}, {"clip_skip", defaults.clip_skip}, - {"width", defaults.width > 0 ? defaults.width : 512}, - {"height", defaults.height > 0 ? defaults.height : 512}, + {"width", defaults.width}, + {"height", defaults.height}, {"strength", defaults.strength}, {"seed", defaults.seed}, {"batch_count", defaults.batch_count}, @@ -173,8 +142,8 @@ static json make_vid_gen_defaults_json(const SDGenerationParams& defaults, const {"prompt", defaults.prompt}, {"negative_prompt", defaults.negative_prompt}, {"clip_skip", defaults.clip_skip}, - {"width", defaults.width > 0 ? defaults.width : 512}, - {"height", defaults.height > 0 ? defaults.height : 512}, + {"width", defaults.width}, + {"height", defaults.height}, {"strength", defaults.strength}, {"seed", defaults.seed}, {"video_frames", defaults.video_frames}, @@ -234,8 +203,8 @@ static json make_vid_gen_features_json() { static json make_preview_modes_json() { json modes = json::array(); - for (int i = 0; i < PREVIEW_COUNT; ++i) { - modes.push_back(sd_preview_name((preview_t)i)); + for (const char* name : example_options::preview_modes) { + modes.push_back(name); } return modes; } @@ -258,13 +227,13 @@ static json make_capabilities_json(ServerRuntime& runtime) { json available_upscalers = json::array(); json supported_modes = json::array(); - for (int i = 0; i < SAMPLE_METHOD_COUNT; ++i) { - samplers.push_back(sd_sample_method_name((sample_method_t)i)); + for (const char* name : example_options::sample_methods) { + samplers.push_back(name); } - for (int i = 0; i < SCHEDULER_COUNT; ++i) { - schedulers.push_back(sd_scheduler_name((scheduler_t)i)); - if (i == DISCRETE_SCHEDULER) { + for (const std::string name : example_options::schedulers) { + schedulers.push_back(name); + if (name == "discrete") { schedulers.push_back("normal"); } } @@ -483,6 +452,21 @@ static bool parse_vid_gen_request(const json& body, return false; } + auto& params = request.gen_params; + auto merge_shared_sample_input = [&](const char* key, auto member) { + const auto base = body.find("sample_params"); + const auto high = body.find("high_noise_sample_params"); + if (base != body.end() && base->is_object() && base->contains(key)) { + if (base->at(key).is_null()) + (params.high_noise_sample_params.*member).reset(); + } else if (high != body.end() && high->is_object() && high->contains(key)) { + params.sample_params.*member = params.high_noise_sample_params.*member; + } + }; + merge_shared_sample_input("scheduler", &SDSampleParams::scheduler); + merge_shared_sample_input("flow_shift", &SDSampleParams::flow_shift); + merge_shared_sample_input("shifted_timestep", &SDSampleParams::shifted_timestep); + std::string output_format = body.value("output_format", "webm"); int output_compression = body.value("output_compression", 100); if (!assign_output_options(request, output_format, output_compression, error_message)) { @@ -546,7 +530,12 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { res.set_content(json({{"error", error_message}}).dump(), "application/json"); return; } - const int tile_size = std::max(32, body.value("tile_size", runtime->default_gen_params->upscale_tile_size)); + const auto tile_size = body.value("tile_size", runtime->default_gen_params->upscale_tile_size); + if (tile_size && *tile_size < 32) { + res.status = 400; + res.set_content(json({{"error", "tile_size must be at least 32"}}).dump(), "application/json"); + return; + } const int repeats = std::clamp(body.value("repeats", 1), 1, 4); const std::string wanted = body.value("upscaler", std::string()); @@ -597,18 +586,18 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { // One GPU: an upscale must not run while a generation is using it. std::lock_guard ctx_lock(*runtime->sd_ctx_mutex); - UpscalerCtxPtr upscaler_ctx(new_upscaler_ctx(model_path.c_str(), - runtime->ctx_params->diffusion_conv_direct, - runtime->ctx_params->n_threads, - tile_size, - runtime->ctx_params->backend.c_str(), - runtime->ctx_params->params_backend.c_str())); + UpscalerCtxPtr upscaler_ctx(create_sd_upscaler(model_path.c_str(), + runtime->ctx_params->diffusion_conv_direct, + runtime->ctx_params->n_threads, + tile_size, + runtime->ctx_params->backend ? runtime->ctx_params->backend->c_str() : nullptr, + runtime->ctx_params->params_backend ? runtime->ctx_params->params_backend->c_str() : nullptr)); if (upscaler_ctx == nullptr) { res.status = 500; res.set_content(R"({"error":"the upscaler model could not be loaded"})", "application/json"); return; } - const int factor = get_upscale_factor(upscaler_ctx.get()); + const int factor = sd_upscaler_get_scale(upscaler_ctx.get()); // The model file may have changed since its metadata was cached. if (!valid_upscale_dimensions(input.get(), factor, repeats)) { res.status = 400; @@ -619,24 +608,24 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) { SDImageOwner current(input.release()); std::string generation_info; for (int i = 0; i < repeats; ++i) { - sd_image_generation_result_t* generated = nullptr; - if (!upscale(upscaler_ctx.get(), current.get(), (uint32_t)factor, &generated) || - !generated || generated->count <= 0 || generated->items[0].image.data == nullptr) { - free_sd_image_generation_result(generated); + sd_result_t* generated = nullptr; + if (!upscale_sd_image(upscaler_ctx.get(), current.get(), &generated) || + !generated || sd_result_get_image_count(generated) <= 0 || sd_tensor_data(sd_result_get_image(generated, 0)) == nullptr) { + sd_result_free(generated); res.status = 500; res.set_content(R"({"error":"upscale failed"})", "application/json"); return; } - generation_info = merge_generation_info(generation_info, generated->items[0].generation_info); - sd_image_t produced = generated->items[0].image; - generated->items[0].image = {}; - free_sd_image_generation_result(generated); + generation_info = merge_generation_info(generation_info, sd_result_get_generation_info(generated, 0)); + SDImage produced = take_result_image(generated, 0); + + sd_result_free(generated); current.reset(produced); } const std::string& format = output_options.output_format; const int compression = output_options.output_compression; - const sd_image_t result = current.get(); + const SDImage result = current.get(); auto image_bytes = encode_image_to_vector(format == "jpeg" ? EncodedImageFormat::JPEG : format == "webp" ? EncodedImageFormat::WEBP : EncodedImageFormat::PNG, diff --git a/examples/server/runtime.cpp b/examples/server/runtime.cpp index 5d08bba305..6ac3866c77 100644 --- a/examples/server/runtime.cpp +++ b/examples/server/runtime.cpp @@ -19,37 +19,60 @@ namespace fs = std::filesystem; +bool PromptImageCallbackGuard::load_image(const char* source, void* data, const sd_tensor_t** image_out) { + auto& guard = *static_cast(data); + const auto& params = guard.params_; + const auto it = params.prompt_images.find(source); + if (it == params.prompt_images.end()) { + LOG_ERROR("Unknown prompt image ID: %s", source); + return false; + } + guard.image_tensor_ = image_tensor(it->second.get()); + *image_out = guard.image_tensor_.get(); + return *image_out != nullptr; +} + +PromptImageCallbackGuard::PromptImageCallbackGuard(sd_ctx_t* ctx, const SDGenerationParams& params) + : ctx_(ctx), params_(params) { + sd_set_prompt_image_callback(ctx_, load_image, this); +} + +PromptImageCallbackGuard::~PromptImageCallbackGuard() { + sd_set_prompt_image_callback(ctx_, nullptr, nullptr); +} + bool validate_server_img_gen_request(const ServerRuntime& runtime, const ImgGenJobRequest& request, std::string& error_message) { - auto dimensions_valid = [&](double width, double height) { - if (!std::isfinite(width) || !std::isfinite(height) || - width < 1 || height < 1 || width > k_max_generation_dimension || height > k_max_generation_dimension) { - error_message = "generation width and height must be between 1 and " + std::to_string(k_max_generation_dimension); + auto within_dimension_limit = [&](double value) { + if (!std::isfinite(value) || value > k_max_generation_dimension) { + error_message = "generation dimensions must not exceed " + std::to_string(k_max_generation_dimension); return false; } return true; }; const auto& params = request.gen_params; - const int width = params.get_resolved_width(); - const int height = params.get_resolved_height(); - if (!dimensions_valid(width, height)) { + if ((params.width && !within_dimension_limit(*params.width)) || + (params.height && !within_dimension_limit(*params.height))) return false; - } - if (params.hires_enabled) { - const double hires_width = params.hires_width > 0 ? params.hires_width : params.hires_height > 0 ? params.hires_height - : std::round(width * params.hires_scale); - const double hires_height = params.hires_height > 0 ? params.hires_height : params.hires_width > 0 ? params.hires_width - : std::round(height * params.hires_scale); - if (!dimensions_valid(hires_width, hires_height)) { + if (params.hires_enabled.value_or(false)) { + if ((params.hires_width && !within_dimension_limit(*params.hires_width)) || + (params.hires_height && !within_dimension_limit(*params.hires_height))) return false; + if (params.hires_width.value_or(0) <= 0 && params.hires_height.value_or(0) <= 0) { + // Use the documented library defaults for this resource-policy check only. + const float scale = params.hires_scale.value_or(2.f); + if (!within_dimension_limit(std::round(params.width.value_or(512) * scale)) || + !within_dimension_limit(std::round(params.height.value_or(512) * scale))) + return false; } } if (!request.adetailer_enabled || runtime.default_gen_params->ad_model_path.empty()) { return true; } auto pass_valid = [&](const ServerADetailerPass& pass) { - if (!dimensions_valid(pass.width, pass.height)) { + if ((pass.width && !within_dimension_limit(*pass.width)) || + (pass.height && !within_dimension_limit(*pass.height))) { return false; } // Match the native key normalization, but leave full option parsing to ADetailer. @@ -94,10 +117,9 @@ bool validate_server_img_gen_request(const ServerRuntime& runtime, } bool apply_server_adetailer(ServerRuntime& runtime, - const ImgGenJobRequest& request, - const sd_img_gen_params_t& params, + ImgGenJobRequest& request, SDImageVec& results) { - const auto& gen_params = request.gen_params; + auto& gen_params = request.gen_params; const auto& ad_model_path = runtime.default_gen_params->ad_model_path; if (!request.adetailer_enabled || ad_model_path.empty()) { return true; @@ -117,53 +139,54 @@ bool apply_server_adetailer(ServerRuntime& runtime, passes.push_back(std::move(pass)); } - ADetailerCtxPtr ad_ctx(new_adetailer_ctx(ad_model_path.c_str(), - runtime.ctx_params->n_threads, - runtime.ctx_params->effective_backend.c_str(), - runtime.ctx_params->effective_params_backend.c_str())); + ADetailerCtxPtr ad_ctx(create_sd_adetailer(ad_model_path.c_str(), + runtime.ctx_params->n_threads, + runtime.ctx_params->effective_backend.c_str(), + runtime.ctx_params->effective_params_backend.c_str())); if (ad_ctx == nullptr) { - LOG_ERROR("new_adetailer_ctx failed"); + LOG_ERROR("sd_adetailer_create failed"); return false; } for (const auto& pass : passes) { - sd_adetailer_params_t ad_params{}; - ad_params.prompt = pass.prompt.empty() ? nullptr : pass.prompt.c_str(); - ad_params.negative_prompt = pass.negative_prompt.empty() ? nullptr : pass.negative_prompt.c_str(); - ad_params.extra_ad_args = pass.extra_args.c_str(); for (int i = 0; i < results.count(); ++i) { if (results[i].data == nullptr) { continue; } - sd_img_gen_params_t ad_generation_params = params; - ad_generation_params.seed = generation_seed(results.generation_info(i)); - ad_generation_params.width = pass.width; - ad_generation_params.height = pass.height; - ad_generation_params.strength = pass.strength; - if (pass.clip_skip > 0) { - ad_generation_params.clip_skip = pass.clip_skip; + SDOptBuilder ad_options(gen_params.to_image_opt(true).release()); + ad_options.unset("width"); + ad_options.unset("height"); + ad_options.unset("strength"); + ad_options.set_string("adetail.prompt", pass.prompt); + ad_options.set_string("adetail.negative_prompt", pass.negative_prompt); + ad_options.set_string("adetail.args", pass.extra_args); + ad_options.set_int("seed", generation_seed(results.generation_info(i))); + ad_options.set_int("width", pass.width); + ad_options.set_int("height", pass.height); + ad_options.set_float("strength", pass.strength); + if (pass.clip_skip) { + ad_options.set_int("clip_skip", pass.clip_skip); } if (pass.automatic_scheduler) { - ad_generation_params.sample_params.scheduler = SCHEDULER_COUNT; + ad_options.set_string("sample.scheduler", "auto"); } - sd_image_generation_result_t* detailed = nullptr; - if (!adetail_image(ad_ctx.get(), - runtime.sd_ctx, - results[i], - &ad_params, - &ad_generation_params, - &detailed) || - detailed == nullptr || detailed->count <= 0 || detailed->items[0].image.data == nullptr) { - free_sd_image_generation_result(detailed); + sd_result_t* detailed = nullptr; + if (!adetail_sd_image(ad_ctx.get(), + runtime.sd_ctx, + results[i], + ad_options.finish(), + &detailed) || + detailed == nullptr || sd_result_get_image_count(detailed) <= 0 || sd_tensor_data(sd_result_get_image(detailed, 0)) == nullptr) { + sd_result_free(detailed); LOG_ERROR("ADetailer failed for image %d", i + 1); return false; } - results.append_generation_info(i, detailed->items[0].generation_info, runtime.ctx_params->lora_model_dir); - free(results[i].data); - results[i] = detailed->items[0].image; - detailed->items[0].image = {}; - free_sd_image_generation_result(detailed); + results.append_generation_info(i, sd_result_get_generation_info(detailed, 0), runtime.ctx_params->lora_model_dir); + sd_image_free(&results[i]); + results[i] = take_result_image(detailed, 0); + + sd_result_free(detailed); } } return true; @@ -562,7 +585,7 @@ void refresh_upscaler_cache(ServerRuntime& rt) { }); upscaler_entry.image_upscale_factor = previous != previous_cache.end() ? previous->image_upscale_factor - : get_upscaler_model_scale(upscaler_entry.fullpath.c_str()); + : sd_upscaler_probe_scale(upscaler_entry.fullpath.c_str()); if (upscaler_entry.image_upscale_factor > 0) { upscaler_entry.scale = upscaler_entry.image_upscale_factor; upscaler_entry.model_name = "ESRGAN_" + std::to_string(upscaler_entry.scale) + "x"; diff --git a/examples/server/runtime.h b/examples/server/runtime.h index ba9ddc416b..abb9766f71 100644 --- a/examples/server/runtime.h +++ b/examples/server/runtime.h @@ -73,10 +73,10 @@ struct ServerADetailerPass { std::string prompt; std::string negative_prompt; std::string extra_args; - int width = 512; - int height = 512; - float strength = 0.4f; - int clip_skip = -1; + std::optional width; + std::optional height; + std::optional strength; + std::optional clip_skip; bool automatic_scheduler = false; }; @@ -89,9 +89,21 @@ struct ImgGenJobRequest { bool adetailer_enabled = true; std::vector adetailer_passes; - sd_img_gen_params_t to_sd_img_gen_params_t() { - return gen_params.to_sd_img_gen_params_t(); - } + SDOptPtr to_image_opt() { return gen_params.to_image_opt(); } +}; + +class PromptImageCallbackGuard { + sd_ctx_t* ctx_; + const SDGenerationParams& params_; + SDTensorPtr image_tensor_; + + static bool load_image(const char* source, void* data, const sd_tensor_t** image_out); + +public: + PromptImageCallbackGuard(sd_ctx_t* ctx, const SDGenerationParams& params); + ~PromptImageCallbackGuard(); + PromptImageCallbackGuard(const PromptImageCallbackGuard&) = delete; + PromptImageCallbackGuard& operator=(const PromptImageCallbackGuard&) = delete; }; constexpr int k_max_generation_dimension = 4096; @@ -108,9 +120,7 @@ struct VidGenJobRequest { std::string preview_mode = "none"; int preview_interval = 1; - sd_vid_gen_params_t to_sd_vid_gen_params_t() { - return gen_params.to_sd_vid_gen_params_t(); - } + SDOptPtr to_video_opt() { return gen_params.to_video_opt(); } }; std::string base64_encode(const std::vector& bytes); @@ -134,6 +144,5 @@ std::string get_lora_full_path(ServerRuntime& rt, const std::string& path); void refresh_upscaler_cache(ServerRuntime& rt); int64_t unix_timestamp_now_ms(); bool apply_server_adetailer(ServerRuntime& runtime, - const ImgGenJobRequest& request, - const sd_img_gen_params_t& params, + ImgGenJobRequest& request, SDImageVec& results); diff --git a/ggml b/ggml index d803c238f6..eeda9ec48f 160000 --- a/ggml +++ b/ggml @@ -1 +1 @@ -Subproject commit d803c238f67915fb825a9aa4df27b99356d51597 +Subproject commit eeda9ec48fc84a3df0bb9dcc19ae9053075a2539 diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h index 7453a3f81a..c5a25c6d10 100644 --- a/include/stable-diffusion.h +++ b/include/stable-diffusion.h @@ -19,487 +19,222 @@ #endif #endif -#ifdef __cplusplus -extern "C" { -#endif - #include #include #include #include -enum rng_type_t { - STD_DEFAULT_RNG, - CUDA_RNG, - CPU_RNG, - RNG_TYPE_COUNT -}; - -enum sample_method_t { - EULER_SAMPLE_METHOD, - EULER_A_SAMPLE_METHOD, - HEUN_SAMPLE_METHOD, - DPM2_SAMPLE_METHOD, - DPMPP2S_A_SAMPLE_METHOD, - DPMPP2M_SAMPLE_METHOD, - DPMPP2Mv2_SAMPLE_METHOD, - IPNDM_SAMPLE_METHOD, - IPNDM_V_SAMPLE_METHOD, - LCM_SAMPLE_METHOD, - DDIM_TRAILING_SAMPLE_METHOD, - TCD_SAMPLE_METHOD, - RES_MULTISTEP_SAMPLE_METHOD, - RES_2S_SAMPLE_METHOD, - ER_SDE_SAMPLE_METHOD, - EULER_CFG_PP_SAMPLE_METHOD, - EULER_A_CFG_PP_SAMPLE_METHOD, - EULER_GE_SAMPLE_METHOD, - DPMPP2M_SDE_SAMPLE_METHOD, - DPMPP2M_SDE_BT_SAMPLE_METHOD, - LMS_SAMPLE_METHOD, - SAMPLE_METHOD_COUNT -}; - -extern SD_API const char* sample_method_to_str[]; - -enum scheduler_t { - DISCRETE_SCHEDULER, - KARRAS_SCHEDULER, - EXPONENTIAL_SCHEDULER, - AYS_SCHEDULER, - GITS_SCHEDULER, - SGM_UNIFORM_SCHEDULER, - SIMPLE_SCHEDULER, - SMOOTHSTEP_SCHEDULER, - KL_OPTIMAL_SCHEDULER, - LCM_SCHEDULER, - BONG_TANGENT_SCHEDULER, - LTX2_SCHEDULER, - LOGIT_NORMAL_SCHEDULER, - FLUX2_SCHEDULER, - FLUX_SCHEDULER, - BETA_SCHEDULER, - LLADA_IMAGE_SCHEDULER, - SCHEDULER_COUNT -}; - -extern SD_API const char* scheduler_to_str[]; - -enum prediction_t { - EPS_PRED, - V_PRED, - EDM_V_PRED, - FLOW_PRED, - FLUX_FLOW_PRED, - SEFI_FLOW_PRED, - MINIT2I_FLOW_PRED, - SENSENOVA_U1_FLOW_PRED, - FLUX3_FLOW_PRED, - H3_AV_FLOW_PRED, - MING_IMAGE_FLOW_PRED, - PREDICTION_COUNT -}; - -// same as enum ggml_type -enum sd_type_t { - SD_TYPE_F32 = 0, - SD_TYPE_F16 = 1, - SD_TYPE_Q4_0 = 2, - SD_TYPE_Q4_1 = 3, - // SD_TYPE_Q4_2 = 4, support has been removed - // SD_TYPE_Q4_3 = 5, support has been removed - SD_TYPE_Q5_0 = 6, - SD_TYPE_Q5_1 = 7, - SD_TYPE_Q8_0 = 8, - SD_TYPE_Q8_1 = 9, - SD_TYPE_Q2_K = 10, - SD_TYPE_Q3_K = 11, - SD_TYPE_Q4_K = 12, - SD_TYPE_Q5_K = 13, - SD_TYPE_Q6_K = 14, - SD_TYPE_Q8_K = 15, - SD_TYPE_IQ2_XXS = 16, - SD_TYPE_IQ2_XS = 17, - SD_TYPE_IQ3_XXS = 18, - SD_TYPE_IQ1_S = 19, - SD_TYPE_IQ4_NL = 20, - SD_TYPE_IQ3_S = 21, - SD_TYPE_IQ2_S = 22, - SD_TYPE_IQ4_XS = 23, - SD_TYPE_I8 = 24, - SD_TYPE_I16 = 25, - SD_TYPE_I32 = 26, - SD_TYPE_I64 = 27, - SD_TYPE_F64 = 28, - SD_TYPE_IQ1_M = 29, - SD_TYPE_BF16 = 30, - // SD_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files - // SD_TYPE_Q4_0_4_8 = 32, - // SD_TYPE_Q4_0_8_8 = 33, - SD_TYPE_TQ1_0 = 34, - SD_TYPE_TQ2_0 = 35, - // SD_TYPE_IQ4_NL_4_4 = 36, - // SD_TYPE_IQ4_NL_4_8 = 37, - // SD_TYPE_IQ4_NL_8_8 = 38, - SD_TYPE_MXFP4 = 39, // MXFP4 (1 block) - SD_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) - SD_TYPE_Q1_0 = 41, - SD_TYPE_Q2_0 = 42, - SD_TYPE_F8_E4M3 = 43, - SD_TYPE_F8_E5M2 = 44, - SD_TYPE_COUNT = 45, -}; - -enum sd_log_level_t { - SD_LOG_DEBUG, - SD_LOG_VERBOSE, - SD_LOG_INFO, - SD_LOG_WARN, - SD_LOG_ERROR -}; - -enum preview_t { - PREVIEW_NONE, - PREVIEW_PROJ, - PREVIEW_TAE, - PREVIEW_VAE, - PREVIEW_COUNT -}; - -enum lora_apply_mode_t { - LORA_APPLY_AUTO, - LORA_APPLY_IMMEDIATELY, - LORA_APPLY_AT_RUNTIME, - LORA_APPLY_MODE_COUNT, -}; - -typedef struct { - bool enabled; - bool temporal_tiling; - // Spatial tile dimensions in image pixels for both encode and decode; 0 uses 256. - int tile_size_w; - int tile_size_h; - float target_overlap; - // Positive values override tile_size: <= 1 is a dimension fraction, > 1 a target tile count. - float rel_size_w; - float rel_size_h; - const char* extra_tiling_args; -} sd_tiling_params_t; - -typedef struct { - const char* name; - const char* path; -} sd_embedding_t; +#ifdef __cplusplus +extern "C" { +#endif -enum sd_vae_format_t { - SD_VAE_FORMAT_AUTO = -1, - SD_VAE_FORMAT_FLUX, - SD_VAE_FORMAT_SD3, - SD_VAE_FORMAT_FLUX2, - SD_VAE_FORMAT_WAN, - SD_VAE_FORMAT_COUNT, -}; +// Opaque handles -typedef struct { - const char* model_path; - const char* clip_l_path; - const char* clip_g_path; - const char* clip_vision_path; - const char* t5xxl_path; - const char* llm_path; - const char* llm_vision_path; - const char* diffusion_model_path; - const char* high_noise_diffusion_model_path; - const char* uncond_diffusion_model_path; - const char* embeddings_connectors_path; - const char* vae_path; - const char* audio_vae_path; - const char* audio_encoder_path; - const char* taesd_path; - const char* control_net_path; - const char* ip_adapter_path; - const char* motion_module_path; - const sd_embedding_t* embeddings; - uint32_t embedding_count; - const char* photo_maker_path; - const char* pulid_weights_path; - const char* tensor_type_rules; - int n_threads; - enum sd_type_t wtype; - enum rng_type_t rng_type; - enum rng_type_t sampler_rng_type; - enum prediction_t prediction; - enum lora_apply_mode_t lora_apply_mode; - bool enable_mmap; - bool flash_attn; - bool diffusion_flash_attn; - bool tae_preview_only; - bool diffusion_conv_direct; - bool vae_conv_direct; - bool force_sdxl_vae_conv_scale; - enum sd_vae_format_t vae_format; - const char* max_vram; // Optional per-device GiB budget for managed weights and runner buffers; 0 uses live free VRAM without an explicit budget - bool disable_prefetch; // Disable asynchronous next-segment weight prefetch - bool eager_load; // Load all params into the params backend at model-load time instead of lazily on first use - const char* backend; - const char* params_backend; - const char* split_mode; // weight distribution for multi-device modules: layer (default) or row, or per-module assignments e.g. "diffusion=row" - bool auto_fit; - const char* rpc_servers; - const char* model_args; - bool disable_segmented_compute; // Force monolithic graph execution even when automatic graph cutting would fit memory better - bool batched_cfg; // Run the conditional and unconditional CFG branches in one batched UNet forward when supported - float linear_scale; // Override linear input scaling; 0 keeps the model default - float attn_scale; // Override flash-attention K/V scaling; 0 keeps the model default - const char* tokenizer; // tokenizer.json path or main=FILE,clip-l=FILE,clip-g=FILE assignments; required for PiD and Lens - bool sage_attn; - int conditioning_cache_size; // Maximum cached conditioning entries per context; 0 disables caching (default: 4) - const char* flash_attn_mode; // on/off/auto or module assignments (e.g. "auto,te=off"); NULL/empty honors legacy flags. - const char* conv_direct_mode; // on/off/auto or diffusion/vae/controlnet assignments; NULL/empty honors legacy flags. -} sd_ctx_params_t; +typedef struct sd_tensor_t sd_tensor_t; +typedef struct sd_ctx_t sd_ctx_t; +typedef struct sd_result sd_result_t; +typedef struct sd_opt sd_opt_t; +typedef struct sd_upscaler sd_upscaler_t; +typedef struct sd_vid_upscaler sd_vid_upscaler_t; +typedef struct sd_adetailer sd_adetailer_t; + +// Status and diagnostics + +typedef enum { + SD_STATUS_OK = 0, + SD_STATUS_INVALID_ARGUMENT = 1, + SD_STATUS_UNKNOWN_KEY = 2, + SD_STATUS_TYPE_MISMATCH = 3, + SD_STATUS_OUT_OF_RANGE = 4, + SD_STATUS_UNSUPPORTED = 5, + SD_STATUS_OUT_OF_MEMORY = 6, + SD_STATUS_LOAD_FAILED = 7, + SD_STATUS_GENERATION_FAILED = 8, + SD_STATUS_BUSY = 9, + SD_STATUS_CANCELLED = 10, +} sd_status_t; + +// Thread-local diagnostic, cleared by successful status/accessor calls. +// Valid until the next API call; free functions and tensor retain preserve it. +SD_API const char* sd_get_last_error(void); + +// Tensors and media + +typedef int32_t sd_tensor_type_t; +#define SD_TENSOR_U8 1 +#define SD_TENSOR_I32 2 +#define SD_TENSOR_I64 3 +#define SD_TENSOR_F32 4 +#define SD_TENSOR_F64 5 typedef struct { uint32_t sample_rate; - uint32_t channels; - uint64_t sample_count; - float* data; + const sd_tensor_t* data; } sd_audio_t; typedef struct { - uint32_t width; - uint32_t height; - uint32_t channel; - uint8_t* data; -} sd_image_t; - -typedef struct { - sd_image_t image; - // Owned UTF-8 JSON using the sdcpp.generation/v1 schema. - char* generation_info; -} sd_image_result_t; - -typedef struct { - sd_image_result_t* items; - int count; -} sd_image_generation_result_t; - -typedef struct { - sd_image_t* frames; - int frame_count; - sd_audio_t* audio; + const sd_tensor_t* data; int fps; - char* generation_info; -} sd_video_generation_result_t; + sd_audio_t audio; +} sd_video_t; + +// Contiguous dimension first. Images: U8 [C,W,H], audio: F32 [C,N], video: U8 [C,W,H,T]. +// create copies data; data_size must exactly match type and shape. Rank <= 8. +// Rank zero is a scalar; zero dimensions represent empty tensors. +SD_API sd_tensor_t* sd_tensor_create( + sd_tensor_type_t type, + size_t rank, + const int64_t* shape, + const void* data, + size_t data_size); + +// retain returns the same immutable handle with one additional ownership reference. +SD_API sd_tensor_t* sd_tensor_retain(const sd_tensor_t* tensor); +SD_API void sd_tensor_free(sd_tensor_t* tensor); +SD_API sd_tensor_type_t sd_tensor_type(const sd_tensor_t* tensor); +SD_API size_t sd_tensor_rank(const sd_tensor_t* tensor); +SD_API int64_t sd_tensor_dim(const sd_tensor_t* tensor, size_t axis); +SD_API size_t sd_tensor_byte_size(const sd_tensor_t* tensor); +SD_API const void* sd_tensor_data(const sd_tensor_t* tensor); + +// Options typedef struct { - // Semicolon-separated target=...,key=value rules. NULL preserves defaults. - const char* rules; -} sd_image_preprocess_params_t; + const char* name; + const char* path; +} sd_embedding_t; typedef struct { - sd_image_t* frames; - int frame_count; - int fps; - sd_audio_t audio; -} sd_ref_video_t; + bool is_high_noise; + float multiplier; + const char* path; +} sd_lora_t; -typedef struct { - int* layers; - size_t layer_count; - float layer_start; - float layer_end; - float scale; -} sd_slg_params_t; +typedef sd_opt_t sd_ctx_opt_t; +typedef sd_opt_t sd_img_gen_opt_t; +typedef sd_opt_t sd_vid_gen_opt_t; +typedef sd_opt_t sd_upscaler_opt_t; +typedef sd_opt_t sd_upscale_opt_t; +typedef sd_opt_t sd_vid_upscaler_opt_t; +typedef sd_opt_t sd_vid_upscale_opt_t; +typedef sd_opt_t sd_adetailer_opt_t; +typedef sd_opt_t sd_adetail_opt_t; + +// Parameter keys and types are documented in docs/guides/c_api.md. +// Setters replace values atomically; a failed setter preserves the previous value. +// Strings, arrays and media descriptors are copied. Tensor storage is retained; +// callers may free their tensor handles after a successful setter. +// Do not mutate/free params concurrently with an operation using them. +// The aliases share one C type; constructors bind their purpose at runtime. +SD_API sd_ctx_opt_t* sd_ctx_opt_create(void); +SD_API sd_img_gen_opt_t* sd_img_gen_opt_create(void); +SD_API sd_vid_gen_opt_t* sd_vid_gen_opt_create(void); +SD_API sd_upscaler_opt_t* sd_upscaler_opt_create(void); +SD_API sd_upscale_opt_t* sd_upscale_opt_create(void); +SD_API sd_vid_upscaler_opt_t* sd_vid_upscaler_opt_create(void); +SD_API sd_vid_upscale_opt_t* sd_vid_upscale_opt_create(void); +SD_API sd_adetailer_opt_t* sd_adetailer_opt_create(void); +SD_API sd_adetail_opt_t* sd_adetail_opt_create(void); +SD_API void sd_opt_free(sd_opt_t* params); + +SD_API sd_status_t sd_opt_set_bool(sd_opt_t* params, const char* key, bool value); +SD_API sd_status_t sd_opt_set_int(sd_opt_t* params, const char* key, int64_t value); +SD_API sd_status_t sd_opt_set_float(sd_opt_t* params, const char* key, double value); +SD_API sd_status_t sd_opt_set_string(sd_opt_t* params, const char* key, const char* value); +SD_API sd_status_t sd_opt_set_tensor(sd_opt_t* params, const char* key, const sd_tensor_t* tensor); +SD_API sd_status_t sd_opt_set_tensors( + sd_opt_t* params, + const char* key, + const sd_tensor_t* const* tensors, + size_t count); +SD_API sd_status_t sd_opt_set_floats(sd_opt_t* params, const char* key, const float* values, size_t count); +SD_API sd_status_t sd_opt_set_ints(sd_opt_t* params, const char* key, const int32_t* values, size_t count); +SD_API sd_status_t sd_opt_set_loras(sd_opt_t* params, const char* key, const sd_lora_t* values, size_t count); +SD_API sd_status_t sd_opt_set_embeddings( + sd_opt_t* params, + const char* key, + const sd_embedding_t* values, + size_t count); +SD_API sd_status_t sd_opt_set_audios(sd_opt_t* params, const char* key, const sd_audio_t* values, size_t count); +SD_API sd_status_t sd_opt_set_videos(sd_opt_t* params, const char* key, const sd_video_t* values, size_t count); + +// Setters/unset reject keys for another purpose. Unsetting an applicable known +// absent key succeeds; unknown keys fail. NULL is not a string value. +SD_API sd_status_t sd_opt_unset(sd_opt_t* params, const char* key); + +// Context and generation + +// Output pointers are required and set to NULL on failure. Context creation retains +// its own configuration; params may be freed immediately after the call returns. +SD_API sd_status_t sd_ctx_create(const sd_ctx_opt_t* params, sd_ctx_t** ctx_out); +SD_API void sd_ctx_free(sd_ctx_t* ctx); + +SD_API sd_status_t sd_generate_image(sd_ctx_t* ctx, const sd_img_gen_opt_t* params, sd_result_t** result_out); +SD_API sd_status_t sd_generate_video(sd_ctx_t* ctx, const sd_vid_gen_opt_t* params, sd_result_t** result_out); -typedef struct { - float txt_cfg; - float img_cfg; - float distilled_guidance; - sd_slg_params_t slg; -} sd_guidance_params_t; +SD_API bool sd_ctx_supports_image_generation(const sd_ctx_t* sd_ctx); +SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx); -typedef struct { - sd_guidance_params_t guidance; - enum scheduler_t scheduler; - enum sample_method_t sample_method; - int sample_steps; - float eta; - int shifted_timestep; - float* custom_sigmas; - int custom_sigmas_count; - float flow_shift; - const char* extra_sample_args; -} sd_sample_params_t; +// ControlNet hot-swap APIs are not safe to call while generation is in flight. +SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path); +SD_API bool sd_ctx_unload_control_net(sd_ctx_t* sd_ctx); +SD_API bool sd_ctx_has_control_net(const sd_ctx_t* sd_ctx); -typedef struct { - sd_image_t* id_images; - int id_images_count; - const char* id_embed_path; - float style_strength; -} sd_pm_params_t; // photo maker +// Requires a loaded context; returns a static string owned by the library, or "Unknown". +SD_API const char* sd_get_model_version_name(const sd_ctx_t* sd_ctx); -typedef struct { - const char* id_embedding_path; - float id_weight; -} sd_pulid_params_t; - -enum sd_cache_mode_t { - SD_CACHE_DISABLED = 0, - SD_CACHE_EASYCACHE, - SD_CACHE_UCACHE, - SD_CACHE_DBCACHE, - SD_CACHE_TAYLORSEER, - SD_CACHE_CACHE_DIT, - SD_CACHE_SPECTRUM, +enum sd_cancel_mode_t { + // Stop the current generation as soon as possible. + SD_CANCEL_ALL, + // Finish the current image sample, then skip additional batch latents and return completed images. + SD_CANCEL_NEW_LATENTS, + // Clear a pending cancellation request. + SD_CANCEL_RESET }; -typedef struct { - enum sd_cache_mode_t mode; - float reuse_threshold; - float start_percent; - float end_percent; - float error_decay_rate; - bool use_relative_threshold; - bool reset_error_on_compute; - int Fn_compute_blocks; - int Bn_compute_blocks; - float residual_diff_threshold; - int max_warmup_steps; - int max_cached_steps; - int max_continuous_cached_steps; - int taylorseer_n_derivatives; - int taylorseer_skip_interval; - const char* scm_mask; - bool scm_policy_dynamic; - float spectrum_w; - int spectrum_m; - float spectrum_lam; - int spectrum_window_size; - float spectrum_flex_window; - int spectrum_warmup_steps; - float spectrum_stop_percent; -} sd_cache_params_t; +SD_API void sd_cancel_generation(sd_ctx_t* sd_ctx, enum sd_cancel_mode_t mode); -typedef struct { - bool is_high_noise; - float multiplier; - const char* path; -} sd_lora_t; +// Results -enum sd_hires_upscaler_t { - SD_HIRES_UPSCALER_NONE, - SD_HIRES_UPSCALER_LATENT, - SD_HIRES_UPSCALER_LATENT_NEAREST, - SD_HIRES_UPSCALER_LATENT_NEAREST_EXACT, - SD_HIRES_UPSCALER_LATENT_ANTIALIASED, - SD_HIRES_UPSCALER_LATENT_BICUBIC, - SD_HIRES_UPSCALER_LATENT_BICUBIC_ANTIALIASED, - SD_HIRES_UPSCALER_LANCZOS, - SD_HIRES_UPSCALER_NEAREST, - SD_HIRES_UPSCALER_MODEL, - SD_HIRES_UPSCALER_COUNT, -}; +typedef enum { + SD_RESULT_INVALID = 0, + SD_RESULT_IMAGE = 1, + SD_RESULT_VIDEO = 2, +} sd_result_kind_t; -enum sd_control_type_t { - SD_CONTROL_TYPE_NONE, - SD_CONTROL_TYPE_OPENPOSE, - SD_CONTROL_TYPE_DEPTH, - SD_CONTROL_TYPE_SCRIBBLE, // thick lines: scribble, hed, softedge, ted - SD_CONTROL_TYPE_CANNY, // thin lines: canny, lineart, anime lineart, mlsd - SD_CONTROL_TYPE_NORMAL, - SD_CONTROL_TYPE_SEGMENT, - SD_CONTROL_TYPE_TILE, - SD_CONTROL_TYPE_REPAINT, // inpaint, outpaint - SD_CONTROL_TYPE_COUNT, -}; +// Results own all outputs independently of ctx and params. Views are read-only and +// borrowed until sd_result_free(). Retain a tensor to keep its data independently. +// Media descriptors contain metadata and borrowed immutable tensor handles. +SD_API sd_result_kind_t sd_result_get_kind(const sd_result_t* result); +SD_API size_t sd_result_get_image_count(const sd_result_t* result); +SD_API const sd_tensor_t* sd_result_get_image(const sd_result_t* result, size_t index); -typedef struct { - bool enabled; - enum sd_hires_upscaler_t upscaler; - const char* model_path; - float scale; - int target_width; - int target_height; - int steps; - float denoising_strength; - int upscale_tile_size; - float* custom_sigmas; - int custom_sigmas_count; -} sd_hires_params_t; +// Materializes contiguous U8 [C,W,H,T] storage on first access. +SD_API const sd_video_t* sd_result_get_video(const sd_result_t* result); -typedef struct { - const sd_lora_t* loras; - uint32_t lora_count; - const char* prompt; - const char* negative_prompt; - int clip_skip; - sd_image_t init_image; - sd_image_t* ref_images; - int ref_images_count; - const char* ref_image_args; - sd_image_t mask_image; - int width; - int height; - sd_sample_params_t sample_params; - float strength; - int64_t seed; - int batch_count; - sd_image_t control_image; - float control_strength; - sd_image_t ip_adapter_image; - float ip_adapter_strength; - sd_pm_params_t pm_params; - sd_pulid_params_t pulid_params; - sd_tiling_params_t vae_tiling_params; - sd_cache_params_t cache; - sd_hires_params_t hires; - int qwen_image_layers; - bool circular_x; - bool circular_y; - sd_image_preprocess_params_t image_preprocess; - enum sd_control_type_t control_type; -} sd_img_gen_params_t; +// Image metadata is per image; video metadata is shared by all frame indices. +SD_API const char* sd_result_get_generation_info(const sd_result_t* result, size_t index); +SD_API const sd_audio_t* sd_result_get_audio(const sd_result_t* result); +SD_API int sd_result_get_fps(const sd_result_t* result); -typedef struct { - const sd_lora_t* loras; - uint32_t lora_count; - const char* prompt; - const char* negative_prompt; - int clip_skip; - sd_image_t init_image; - sd_image_t end_image; - sd_image_t* ref_images; - int ref_images_count; - sd_ref_video_t* ref_videos; - int ref_videos_count; - sd_audio_t* ref_audios; - int ref_audios_count; - sd_image_t* control_frames; - int control_frames_size; - int width; - int height; - sd_sample_params_t sample_params; - sd_sample_params_t high_noise_sample_params; - float moe_boundary; - float strength; - int64_t seed; - int video_frames; - int fps; - float vace_strength; - sd_tiling_params_t vae_tiling_params; - sd_cache_params_t cache; - sd_hires_params_t hires; - bool circular_x; - bool circular_y; - sd_image_preprocess_params_t image_preprocess; - const char* ref_image_args; -} sd_vid_gen_params_t; +SD_API void sd_result_free(sd_result_t* result); -typedef struct sd_ctx_t sd_ctx_t; -struct ggml_tensor; +// Callbacks and progress + +enum sd_log_level_t { + SD_LOG_DEBUG, + SD_LOG_VERBOSE, + SD_LOG_INFO, + SD_LOG_WARN, + SD_LOG_ERROR +}; typedef void (*sd_log_cb_t)(enum sd_log_level_t level, const char* text, void* data); +SD_API void sd_set_log_callback(sd_log_cb_t sd_log_cb, void* data); + typedef void (*sd_progress_cb_t)(int step, int steps, float time, void* data); +SD_API void sd_set_progress_callback(sd_progress_cb_t cb, void* data); + typedef struct { const char* stage; const char* detail; @@ -509,217 +244,174 @@ typedef struct { int64_t total; } sd_generation_progress_t; typedef void (*sd_generation_progress_cb_t)(const sd_generation_progress_t* progress, void* data); -typedef void (*sd_preview_cb_t)(int step, int frame_count, sd_image_t* frames, bool is_noisy, void* data); + +// Context-local generation events, independent of preview and the legacy progress callback. +// Strings and the event are valid only during the synchronous callback. Callbacks must not throw. +// total == 0 means unknown; sample_pass counts from 1 since callback registration. +// Register/clear only while the context is idle. Passing NULL clears the callback. +SD_API void sd_set_generation_progress_callback(sd_ctx_t* ctx, sd_generation_progress_cb_t cb, void* data); typedef struct { int sample_pass; int total_steps; } sd_preview_info_t; -typedef bool (*sd_graph_eval_callback_t)(struct ggml_tensor* t, bool ask, void* user_data); -SD_API void sd_set_log_callback(sd_log_cb_t sd_log_cb, void* data); -SD_API void sd_set_progress_callback(sd_progress_cb_t cb, void* data); -// Context-local generation events, independent of preview and the legacy progress callback. -// Strings and the event are valid only during the synchronous callback. Callbacks must not throw. -// total == 0 means unknown; sample_pass counts from 1 since callback registration. -// Register/clear only while the context is idle. Passing nullptr clears the callback. -SD_API void sd_set_generation_progress_callback(sd_ctx_t* ctx, sd_generation_progress_cb_t cb, void* data); +// Frames are borrowed during the callback; retain a tensor to keep its snapshot. +typedef void (*sd_preview_cb_t)(int step, int frame_count, const sd_tensor_t* const* frames, bool is_noisy, void* data); + // In each sampling pass, a positive interval previews every Nth denoiser step, while a // negative interval previews only completed logical step -interval. Zero previews the final // completed step of the first sampling pass (base-resolution or high-noise). -SD_API void sd_set_preview_callback(sd_preview_cb_t cb, enum preview_t mode, int interval, bool denoised, bool noisy, void* data); +// Mode is "none", "proj", "tae" or "vae"; invalid input preserves registration. +SD_API sd_status_t sd_set_preview_callback( + sd_preview_cb_t cb, + const char* mode, + int interval, + bool denoised, + bool noisy, + void* data); + // Query from a preview callback. Passes are numbered from 1 since the last // sd_set_preview_callback call; total_steps is the actual count in that pass. // Both fields are zero before sampling starts. SD_API sd_preview_info_t sd_get_preview_info(); -SD_API void sd_set_backend_eval_callback(sd_graph_eval_callback_t cb, void* data); -SD_API int32_t sd_get_num_physical_cores(); -SD_API const char* sd_get_system_info(); -SD_API bool sd_ctx_supports_image_generation(const sd_ctx_t* sd_ctx); -SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx); -// ControlNet hot-swap APIs are not safe to call while generation is in flight. -SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path); -SD_API bool sd_ctx_unload_control_net(sd_ctx_t* sd_ctx); -SD_API bool sd_ctx_has_control_net(const sd_ctx_t* sd_ctx); +// Resolves markers. The returned U8 [C,W,H] tensor is borrowed +// and copied before the next callback. Register/clear while ctx is idle. +typedef bool (*sd_prompt_image_cb_t)(const char* source, void* data, const sd_tensor_t** image_out); +SD_API void sd_set_prompt_image_callback(sd_ctx_t* ctx, sd_prompt_image_cb_t cb, void* data); -SD_API const char* sd_type_name(enum sd_type_t type); -SD_API enum sd_type_t str_to_sd_type(const char* str); -SD_API const char* sd_rng_type_name(enum rng_type_t rng_type); -SD_API enum rng_type_t str_to_rng_type(const char* str); -SD_API const char* sd_sample_method_name(enum sample_method_t sample_method); -SD_API enum sample_method_t str_to_sample_method(const char* str); -SD_API const char* sd_scheduler_name(enum scheduler_t scheduler); -SD_API enum scheduler_t str_to_scheduler(const char* str); -SD_API const char* sd_prediction_name(enum prediction_t prediction); -SD_API enum prediction_t str_to_prediction(const char* str); -SD_API const char* sd_preview_name(enum preview_t preview); -SD_API enum preview_t str_to_preview(const char* str); -SD_API const char* sd_lora_apply_mode_name(enum lora_apply_mode_t mode); -SD_API enum lora_apply_mode_t str_to_lora_apply_mode(const char* str); -SD_API const char* sd_hires_upscaler_name(enum sd_hires_upscaler_t upscaler); -SD_API enum sd_hires_upscaler_t str_to_sd_hires_upscaler(const char* str); -SD_API const char* sd_control_type_name(enum sd_control_type_t control_type); -// Also accepts the names of the kinds a type stands for (hed, lineart, mlsd, ...); -// returns SD_CONTROL_TYPE_COUNT when the name is unknown. -SD_API enum sd_control_type_t str_to_sd_control_type(const char* str); - -SD_API void sd_cache_params_init(sd_cache_params_t* cache_params); -SD_API void sd_hires_params_init(sd_hires_params_t* hires_params); - -SD_API void sd_ctx_params_init(sd_ctx_params_t* sd_ctx_params); -SD_API char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params); - -SD_API sd_ctx_t* new_sd_ctx(const sd_ctx_params_t* sd_ctx_params); -SD_API void free_sd_ctx(sd_ctx_t* sd_ctx); -SD_API void free_sd_audio(sd_audio_t* audio); - -SD_API void sd_sample_params_init(sd_sample_params_t* sample_params); -SD_API char* sd_sample_params_to_str(const sd_sample_params_t* sample_params); +struct ggml_tensor; -// Requires a loaded context; returns a static string owned by the library, or "Unknown". -SD_API const char* sd_get_model_version_name(const sd_ctx_t* sd_ctx); +typedef bool (*sd_graph_eval_callback_t)(struct ggml_tensor* t, bool ask, void* user_data); +SD_API void sd_set_backend_eval_callback(sd_graph_eval_callback_t cb, void* data); -SD_API enum sample_method_t sd_get_default_sample_method(const sd_ctx_t* sd_ctx); -SD_API enum scheduler_t sd_get_default_scheduler(const sd_ctx_t* sd_ctx, enum sample_method_t sample_method); +// Image upscaling -SD_API void sd_img_gen_params_init(sd_img_gen_params_t* sd_img_gen_params); -SD_API char* sd_img_gen_params_to_str(const sd_img_gen_params_t* sd_img_gen_params); -SD_API bool generate_image(sd_ctx_t* sd_ctx, - const sd_img_gen_params_t* sd_img_gen_params, - sd_image_generation_result_t** result_out); +SD_API sd_status_t sd_upscaler_create(const sd_upscaler_opt_t* opt, sd_upscaler_t** out); +SD_API void sd_upscaler_free(sd_upscaler_t* upscaler); +SD_API sd_status_t sd_upscale(sd_upscaler_t* upscaler, const sd_upscale_opt_t* opt, sd_result_t** out); +SD_API int sd_upscaler_get_scale(const sd_upscaler_t* upscaler); -// Results own their pixels, audio and generation_info, independently of the context. -// Generation calls require result_out and set it to NULL on failure. -SD_API void free_sd_image_generation_result(sd_image_generation_result_t* result); -SD_API void free_sd_video_generation_result(sd_video_generation_result_t* result); +// Reads model metadata only; returns 0 if it is not a recognized RGB upscaler. +SD_API int sd_upscaler_probe_scale(const char* model_path); -enum sd_cancel_mode_t { - // Stop the current generation as soon as possible. - SD_CANCEL_ALL, - // Finish the current image sample, then skip additional batch latents and return completed images. - SD_CANCEL_NEW_LATENTS, - // Clear a pending cancellation request. - SD_CANCEL_RESET -}; - -SD_API void sd_cancel_generation(sd_ctx_t* sd_ctx, enum sd_cancel_mode_t mode); +// Video upscaling -SD_API void sd_vid_gen_params_init(sd_vid_gen_params_t* sd_vid_gen_params); -SD_API bool generate_video(sd_ctx_t* sd_ctx, - const sd_vid_gen_params_t* sd_vid_gen_params, - sd_video_generation_result_t** result_out); +SD_API sd_status_t sd_vid_upscaler_create(const sd_vid_upscaler_opt_t* opt, sd_vid_upscaler_t** out); +SD_API void sd_vid_upscaler_free(sd_vid_upscaler_t* upscaler); +SD_API sd_status_t sd_upscale_video( + sd_vid_upscaler_t* upscaler, + const sd_vid_upscale_opt_t* opt, + sd_result_t** out); -typedef struct upscaler_ctx_t upscaler_ctx_t; +// ADetailer -SD_API upscaler_ctx_t* new_upscaler_ctx(const char* esrgan_path, - bool direct, - int n_threads, - int tile_size, - const char* backend, - const char* params_backend); -SD_API void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx); +SD_API sd_status_t sd_adetailer_create(const sd_adetailer_opt_t* opt, sd_adetailer_t** out); +SD_API void sd_adetailer_free(sd_adetailer_t* adetailer); +SD_API sd_status_t sd_adetail( + sd_adetailer_t* adetailer, + sd_ctx_t* ctx, + const sd_adetail_opt_t* opt, + sd_result_t** out); -SD_API bool upscale(upscaler_ctx_t* upscaler_ctx, - sd_image_t input_image, - uint32_t upscale_factor, - sd_image_generation_result_t** result_out); +// Image preprocessing -SD_API int get_upscale_factor(upscaler_ctx_t* upscaler_ctx); -// Reads model metadata only; returns 0 if the file is not a recognized RGB ESRGAN model. -SD_API int get_upscaler_model_scale(const char* model_path); +SD_API sd_status_t sd_preprocess_canny( + const sd_tensor_t* image, + float high_threshold, + float low_threshold, + float weak, + float strong, + bool inverse, + sd_tensor_t** image_out); -typedef struct video_upscaler_ctx_t video_upscaler_ctx_t; +// Weight types and model conversion -typedef struct { - const char* diffusion_model_path; - const char* vae_path; - // Explicit output scale; RelayVSR requires 2 or 4 matching both checkpoints. - int upscale_factor; - int n_threads; - const char* backend; - const char* params_backend; - // auto/on/off or diffusion/vae assignments; null selects auto. - const char* flash_attn_mode; - const char* conv_direct_mode; -} sd_video_upscaler_ctx_params_t; +// same as enum ggml_type +enum sd_type_t { + SD_TYPE_F32 = 0, + SD_TYPE_F16 = 1, + SD_TYPE_Q4_0 = 2, + SD_TYPE_Q4_1 = 3, + // SD_TYPE_Q4_2 = 4, support has been removed + // SD_TYPE_Q4_3 = 5, support has been removed + SD_TYPE_Q5_0 = 6, + SD_TYPE_Q5_1 = 7, + SD_TYPE_Q8_0 = 8, + SD_TYPE_Q8_1 = 9, + SD_TYPE_Q2_K = 10, + SD_TYPE_Q3_K = 11, + SD_TYPE_Q4_K = 12, + SD_TYPE_Q5_K = 13, + SD_TYPE_Q6_K = 14, + SD_TYPE_Q8_K = 15, + SD_TYPE_IQ2_XXS = 16, + SD_TYPE_IQ2_XS = 17, + SD_TYPE_IQ3_XXS = 18, + SD_TYPE_IQ1_S = 19, + SD_TYPE_IQ4_NL = 20, + SD_TYPE_IQ3_S = 21, + SD_TYPE_IQ2_S = 22, + SD_TYPE_IQ4_XS = 23, + SD_TYPE_I8 = 24, + SD_TYPE_I16 = 25, + SD_TYPE_I32 = 26, + SD_TYPE_I64 = 27, + SD_TYPE_F64 = 28, + SD_TYPE_IQ1_M = 29, + SD_TYPE_BF16 = 30, + // SD_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files + // SD_TYPE_Q4_0_4_8 = 32, + // SD_TYPE_Q4_0_8_8 = 33, + SD_TYPE_TQ1_0 = 34, + SD_TYPE_TQ2_0 = 35, + // SD_TYPE_IQ4_NL_4_4 = 36, + // SD_TYPE_IQ4_NL_4_8 = 37, + // SD_TYPE_IQ4_NL_8_8 = 38, + SD_TYPE_MXFP4 = 39, // MXFP4 (1 block) + SD_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) + SD_TYPE_Q1_0 = 41, + SD_TYPE_Q2_0 = 42, + SD_TYPE_F8_E4M3 = 43, + SD_TYPE_F8_E5M2 = 44, + SD_TYPE_COUNT = 45, +}; -typedef struct { - const sd_image_t* frames; - int frame_count; - int fps; - int64_t seed; - // RelayVSR reference window, including both endpoints; defaults to 16. - int window_size; - // SeedVR2 independent temporal chunks in input frames (1 or 4n+1); defaults to 5. - int temporal_chunk_size; -} sd_video_upscale_params_t; - -SD_API void sd_video_upscaler_ctx_params_init(sd_video_upscaler_ctx_params_t* params); -SD_API void sd_video_upscale_params_init(sd_video_upscale_params_t* params); -SD_API video_upscaler_ctx_t* new_video_upscaler_ctx(const sd_video_upscaler_ctx_params_t* params); -SD_API void free_video_upscaler_ctx(video_upscaler_ctx_t* ctx); -// Frames are RGB, have equal dimensions, and remain owned by the caller. -// Free the result with free_sd_video_generation_result(). -SD_API bool upscale_video(video_upscaler_ctx_t* ctx, - const sd_video_upscale_params_t* params, - sd_video_generation_result_t** result_out); - -typedef struct adetailer_ctx_t adetailer_ctx_t; +SD_API const char* sd_type_name(enum sd_type_t type); +SD_API enum sd_type_t str_to_sd_type(const char* str); -typedef struct { - const char* prompt; - const char* negative_prompt; - const char* extra_ad_args; -} sd_adetailer_params_t; - -SD_API adetailer_ctx_t* new_adetailer_ctx(const char* detector_path, - int n_threads, - const char* backend, - const char* params_backend); -SD_API void free_adetailer_ctx(adetailer_ctx_t* adetailer_ctx); -SD_API bool adetail_image(adetailer_ctx_t* adetailer_ctx, - sd_ctx_t* sd_ctx, - sd_image_t input_image, - const sd_adetailer_params_t* adetailer_params, - const sd_img_gen_params_t* inpaint_params, - sd_image_generation_result_t** result_out); - -SD_API bool convert(const char* input_path, - const char* vae_path, - const char* output_path, - enum sd_type_t output_type, - const char* tensor_type_rules, - bool convert_name); - -SD_API bool convert_with_components(const char* model_path, - const char* clip_l_path, - const char* clip_g_path, - const char* t5xxl_path, - const char* diffusion_model_path, - const char* vae_path, - const char* output_path, - enum sd_type_t output_type, - const char* tensor_type_rules, - bool convert_name, - int n_threads, - const sd_lora_t* loras, - int lora_count); - -SD_API bool preprocess_canny(sd_image_t image, - float high_threshold, - float low_threshold, - float weak, - float strong, - bool inverse); +SD_API bool convert( + const char* input_path, + const char* vae_path, + const char* output_path, + enum sd_type_t output_type, + const char* tensor_type_rules, + bool convert_name); + +SD_API bool convert_with_components( + const char* model_path, + const char* clip_l_path, + const char* clip_g_path, + const char* t5xxl_path, + const char* diffusion_model_path, + const char* vae_path, + const char* output_path, + enum sd_type_t output_type, + const char* tensor_type_rules, + bool convert_name, + int n_threads, + const sd_lora_t* loras, + int lora_count); + +// Importance matrix SD_API bool load_imatrix(const char* imatrix_path); SD_API void save_imatrix(const char* imatrix_path); SD_API void enable_imatrix_collection(void); SD_API void disable_imatrix_collection(void); -SD_API const char* sd_commit(void); -SD_API const char* sd_version(void); +// Devices and system information // List available ggml backend devices, one `namedescription` per line. // The names are the device names accepted by the --backend / --params-backend @@ -732,9 +424,13 @@ SD_API size_t sd_list_devices(char* buffer, size_t buffer_size); // Returns false if RPC is unavailable or an endpoint has no usable devices. SD_API bool sd_add_rpc_servers(const char* servers); -// for C API, caller needs to call free_sd_images to free the memory after use -// This helps avoid CRT problems on Windows when memory is allocated in the library but freed in the caller, which may use a different CRT. -SD_API void free_sd_images(sd_image_t* result_images, int num_images); +SD_API int32_t sd_get_num_physical_cores(); +SD_API const char* sd_get_system_info(); + +// Build information + +SD_API const char* sd_commit(void); +SD_API const char* sd_version(void); #ifdef __cplusplus } diff --git a/scripts/format-code.ps1 b/scripts/format-code.ps1 index 02097e6bb9..782e09205e 100644 --- a/scripts/format-code.ps1 +++ b/scripts/format-code.ps1 @@ -2,6 +2,9 @@ $patterns = @( "src/*.cpp" "src/*.h" "src/*.hpp" + "src/api/*.cpp" + "src/api/*.h" + "src/api/*.hpp" "src/conditioning/*.cpp" "src/conditioning/*.h" "src/conditioning/*.hpp" diff --git a/scripts/format-code.sh b/scripts/format-code.sh index dd4d0fa95b..c2806f6473 100644 --- a/scripts/format-code.sh +++ b/scripts/format-code.sh @@ -6,6 +6,7 @@ repo_root="$(cd "$(dirname "$0")/.." && pwd)" cd "$repo_root" for f in src/*.cpp src/*.h src/*.hpp \ + src/api/*.cpp src/api/*.h src/api/*.hpp \ src/conditioning/*.cpp src/conditioning/*.h src/conditioning/*.hpp \ src/core/*.cpp src/core/*.h src/core/*.hpp \ src/core/cpu/*.cpp src/core/cpu/*.h \ diff --git a/src/api/api.cpp b/src/api/api.cpp new file mode 100644 index 0000000000..72cd6d4049 --- /dev/null +++ b/src/api/api.cpp @@ -0,0 +1,245 @@ +#include "api.h" +#include "options.h" +#include "result.h" +#include "tensor.h" + +#include +#include +#include + +#include "core/ggml_extend.h" +#include "core/option_types.h" +#include "core/util.h" +#include "pipeline/diffusion_engine.h" +#include "pipeline/generation.h" + +static thread_local char last_error[512] = {}; + +void sd::api::clear_error() { + last_error[0] = '\0'; +} + +sd_status_t sd::api::error(sd_status_t status, const char* key, const char* reason) { + snprintf(last_error, sizeof(last_error), "%s%s%s", key ? key : "", key ? ": " : "", reason); + return status; +} + +const char* sd_get_last_error(void) { + return last_error; +} + +const char* sd_type_name(enum sd_type_t type) { + if (type == SD_TYPE_F8_E4M3) { + return "f8_e4m3"; + } + if (type == SD_TYPE_F8_E5M2) { + return "f8_e5m2"; + } + const auto ggml_type = sd_type_to_ggml_type(type); + if (ggml_type != GGML_TYPE_COUNT) { + return ggml_type_name(ggml_type); + } + return "NONE"; +} + +enum sd_type_t str_to_sd_type(const char* str) { + if (!strcmp(str, "f8_e4m3")) { + return SD_TYPE_F8_E4M3; + } + if (!strcmp(str, "f8_e5m2")) { + return SD_TYPE_F8_E5M2; + } + for (int i = 0; i < std::min(SD_TYPE_COUNT, GGML_TYPE_COUNT); i++) { + auto trait = ggml_get_type_traits((ggml_type)i); + if (!strcmp(str, trait->type_name)) { + return (enum sd_type_t)i; + } + } + return SD_TYPE_COUNT; +} + +sd_status_t sd_set_preview_callback(sd_preview_cb_t cb, const char* mode, int interval, bool denoised, bool noisy, void* data) { + sd::api::clear_error(); + if (!mode) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, "preview.mode", "mode is required"); + const auto parsed = str_to_preview(mode); + if (parsed == PREVIEW_COUNT) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, "preview.mode", "unknown preview mode"); + sd_set_preview_callback_impl(cb, parsed, interval, denoised, noisy, data); + return SD_STATUS_OK; +} + +void sd_set_prompt_image_callback(sd_ctx_t* ctx, sd_prompt_image_cb_t cb, void* data) { + if (ctx == nullptr || ctx->sd == nullptr) { + return; + } + std::lock_guard lock(ctx->sd->execution_mutex); + if (!ctx->sd->executing_) { + ctx->sd->prompt_image_callback = cb; + ctx->sd->prompt_image_callback_data = data; + } +} + +void sd_set_generation_progress_callback(sd_ctx_t* ctx, sd_generation_progress_cb_t cb, void* data) { + if (ctx == nullptr || ctx->sd == nullptr) { + return; + } + std::lock_guard lock(ctx->sd->execution_mutex); + if (!ctx->sd->executing_) { + ctx->sd->generation_progress.set_callback(cb, data); + } +} + +static sd_ctx_t* create_context(const sd::ContextConfig& params) { + auto engine = std::make_unique(); + if (!engine->init(¶ms)) + return nullptr; + auto context = std::make_unique(); + context->sd = engine.release(); + return context.release(); +} + +sd_status_t sd_ctx_create(const sd_ctx_opt_t* params, sd_ctx_t** ctx_out) { + if (ctx_out) + *ctx_out = nullptr; + return sd::api::call([&] { + if (!ctx_out) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "ctx_out is required"); + sd::OptConfig config; + auto status = sd::api::resolve(params, sd::api::Context, config); + const auto& context = config.context; + if (status != SD_STATUS_OK) + return status; + *ctx_out = create_context(context); + return *ctx_out ? SD_STATUS_OK : sd::api::error(SD_STATUS_LOAD_FAILED, nullptr, "context creation failed; see log for details"); + }, + SD_STATUS_LOAD_FAILED); +} + +void sd_ctx_free(sd_ctx_t* sd_ctx) { + if (!sd_ctx) + return; + if (sd_ctx->sd != nullptr) { + delete sd_ctx->sd; + sd_ctx->sd = nullptr; + } + delete sd_ctx; +} + +SD_API void sd_cancel_generation(sd_ctx_t* sd_ctx, enum sd_cancel_mode_t mode) { + if (sd_ctx && sd_ctx->sd) { + if (mode < SD_CANCEL_ALL || mode > SD_CANCEL_RESET) { + mode = SD_CANCEL_ALL; + } + sd_ctx->sd->set_cancel_flag(mode); + } +} + +SD_API bool sd_ctx_supports_image_generation(const sd_ctx_t* sd_ctx) { + if (sd_ctx == nullptr || sd_ctx->sd == nullptr) { + return false; + } + return sd_version_supports_image_generation(sd_ctx->sd->version); +} + +SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx) { + if (sd_ctx == nullptr || sd_ctx->sd == nullptr) { + return false; + } + if (sd_ctx->sd->config_->animatediff_loaded && sd_version_supports_animatediff(sd_ctx->sd->version)) { + return true; + } + return sd_version_supports_video_generation(sd_ctx->sd->version); +} + +SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path) { + if (sd_ctx == nullptr || sd_ctx->sd == nullptr || path == nullptr) { + return false; + } + return sd_ctx->sd->load_control_net_from_file(path); +} + +SD_API bool sd_ctx_unload_control_net(sd_ctx_t* sd_ctx) { + if (sd_ctx == nullptr || sd_ctx->sd == nullptr) { + return false; + } + return sd_ctx->sd->unload_control_net(); +} + +SD_API bool sd_ctx_has_control_net(const sd_ctx_t* sd_ctx) { + if (sd_ctx == nullptr || sd_ctx->sd == nullptr) { + return false; + } + if (sd_ctx->sd->control_net != nullptr) { + return true; + } + // z-image keeps control net weights inside the diffusion runner, registered + // under ModelComponent::ControlNet instead of a separate control_net runner. + return sd_ctx->sd->model_manager != nullptr && + sd_ctx->sd->model_manager->registered_params_size({ModelComponent::ControlNet}) > 0; +} + +const char* sd_get_model_version_name(const sd_ctx_t* sd_ctx) { + if (sd_ctx == nullptr || sd_ctx->sd == nullptr || sd_ctx->sd->version >= VERSION_COUNT) { + return "Unknown"; + } + return sd_version_name(sd_ctx->sd->version); +} + +sd_status_t sd_generate_image(sd_ctx_t* ctx, const sd_img_gen_opt_t* params, sd_result_t** result_out) { + if (result_out) + *result_out = nullptr; + return sd::api::call([&] { + if (!result_out) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "result_out is required"); + sd::OptConfig config; + auto status = sd::api::resolve(params, sd::api::Image, config); + const auto& image = config.image; + if (status != SD_STATUS_OK) + return status; + sd::ImageGenerationResult* images = nullptr; + status = sd::generate_image(ctx, image, &images, params); + if (status != SD_STATUS_OK) + return status; + return sd::make_image_result(images, result_out) ? SD_STATUS_OK + : sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate result"); + }, + SD_STATUS_GENERATION_FAILED); +} + +sd_status_t sd_generate_video(sd_ctx_t* ctx, const sd_vid_gen_opt_t* params, sd_result_t** result_out) { + if (result_out) + *result_out = nullptr; + return sd::api::call([&] { + if (!result_out) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "result_out is required"); + sd::OptConfig config; + auto status = sd::api::resolve(params, sd::api::Video, config); + const auto& video = config.video; + if (status != SD_STATUS_OK) + return status; + sd::VideoGenerationResult* frames = nullptr; + status = sd::generate_video(ctx, video, &frames, params); + if (status != SD_STATUS_OK) + return status; + return sd::make_video_result(frames, result_out) ? SD_STATUS_OK + : sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate result"); + }, + SD_STATUS_GENERATION_FAILED); +} + +sd_status_t sd_preprocess_canny(const sd_tensor_t* image, float high_threshold, float low_threshold, float weak, float strong, bool inverse, sd_tensor_t** image_out) { + if (image_out) + *image_out = nullptr; + return sd::api::call([&] { + if (!image || !image_out) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "image and image_out are required"); + sd::api::image_view(*image); + auto output = sd::api::allocate_tensor(image->type, image->shape); + memcpy(output.get()->mutable_data(), image->data(), image->bytes); + if (!sd::preprocess_canny(sd::api::image_view(*output.get()), high_threshold, low_threshold, weak, strong, inverse)) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "Canny preprocessing failed"); + *image_out = sd_tensor_retain(output.get()); + return SD_STATUS_OK; + }); +} diff --git a/src/api/api.h b/src/api/api.h new file mode 100644 index 0000000000..545859e960 --- /dev/null +++ b/src/api/api.h @@ -0,0 +1,34 @@ +#ifndef SD_INTERNAL_API_H +#define SD_INTERNAL_API_H + +#include +#include + +#include "stable-diffusion.h" + +class StableDiffusionGGML; + +struct sd_ctx_t { + StableDiffusionGGML* sd = nullptr; +}; + +namespace sd::api { + void clear_error(); + sd_status_t error(sd_status_t status, const char* key, const char* reason); + + template + sd_status_t call(Function&& function, sd_status_t failure = SD_STATUS_INVALID_ARGUMENT) noexcept { + clear_error(); + try { + return function(); + } catch (const std::bad_alloc&) { + return error(SD_STATUS_OUT_OF_MEMORY, nullptr, "out of memory"); + } catch (const std::exception& exception) { + return error(failure, nullptr, exception.what()); + } catch (...) { + return error(failure, nullptr, "unexpected exception"); + } + } +} + +#endif // SD_INTERNAL_API_H diff --git a/src/api/options.cpp b/src/api/options.cpp new file mode 100644 index 0000000000..591cce3448 --- /dev/null +++ b/src/api/options.cpp @@ -0,0 +1,897 @@ +#include "options.h" +#include "core/option_types.h" +#include "tensor.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum class ParamType { Bool, + Int, + Float, + String, + Tensor, + Tensors, + Floats, + Ints, + Loras, + Embeddings, + Audios, + Videos }; + +struct LoraValues { + std::vector paths; + std::vector items; +}; + +struct EmbeddingValues { + std::vector strings; + std::vector items; +}; + +struct TensorValue { + sd::api::TensorRef tensor; + std::vector frames; +}; + +struct ImageValues { + std::vector tensors; + std::vector items; +}; + +struct AudioValues { + std::vector tensors; + std::vector items; +}; + +struct VideoValues { + std::vector tensors; + std::vector> frames; + std::vector items; +}; + +using ParamValue = std::variant, std::vector, LoraValues, EmbeddingValues, AudioValues, VideoValues>; + +struct sd_opt { + sd::api::Operation operation; + std::map values; +}; + +struct ParamSpec { + const char* key; + ParamType type; + int operations; + std::function apply; + int64_t int_min = std::numeric_limits::min(); + int64_t int_max = std::numeric_limits::max(); + double float_min = -std::numeric_limits::max(); + double float_max = std::numeric_limits::max(); + std::function valid; + const char* invalid_reason = "invalid parameter value"; + bool video_tensor = false; +}; + +static sd_cache_mode_t parse_cache(const char* text) { + static const char* names[] = {"disabled", "easycache", "ucache", "dbcache", "taylorseer", "cache_dit", "spectrum"}; + for (int i = 0; i < 7; ++i) + if (strcmp(text, names[i]) == 0) + return static_cast(i); + return static_cast(-1); +} + +static sd_vae_format_t parse_vae_format(const char* text) { + static const char* names[] = {"auto", "flux", "sd3", "flux2", "wan"}; + for (int i = 0; i < 5; ++i) + if (strcmp(text, names[i]) == 0) + return static_cast(i - 1); + return SD_VAE_FORMAT_COUNT; +} + +static sd_hires_upscaler_t parse_hires_upscaler(const char* text) { + static const char* names[] = {"none", "latent", "latent_nearest", "latent_nearest_exact", + "latent_antialiased", "latent_bicubic", "latent_bicubic_antialiased", + "lanczos", "nearest", "model"}; + for (int i = 0; i < SD_HIRES_UPSCALER_COUNT; ++i) + if (strcmp(text, names[i]) == 0) + return static_cast(i); + return SD_HIRES_UPSCALER_COUNT; +} + +static bool valid_sigmas(const ParamValue& value) { + const auto& sigmas = std::get>(value); + if (sigmas.empty()) + return true; + if (sigmas.size() < 2 || sigmas.back() != 0.f) + return false; + for (size_t i = 0; i < sigmas.size(); ++i) + if (sigmas[i] < 0.f || (i && sigmas[i] >= sigmas[i - 1])) + return false; + return true; +} + +static bool valid_layers(const ParamValue& value) { + const auto& layers = std::get>(value); + return std::all_of(layers.begin(), layers.end(), [](int index) { return index >= 0; }); +} + +template +static ParamSpec parameter(const char* key, ParamType type, int operations, Apply apply) { + return {key, type, operations, [apply](const ParamValue& value, sd::OptConfig& config) { + apply(std::get(value), config); + }}; +} + +template +static ParamSpec integer_parameter(const char* key, int operations, Apply apply, int64_t min = INT64_MIN, int64_t max = INT64_MAX) { + auto spec = parameter(key, ParamType::Int, operations, apply); + spec.int_min = min; + spec.int_max = max; + return spec; +} + +template +static ParamSpec float_parameter(const char* key, int operations, Apply apply, double min = -std::numeric_limits::max(), double max = std::numeric_limits::max()) { + auto spec = parameter(key, ParamType::Float, operations, apply); + spec.float_min = min; + spec.float_max = max; + return spec; +} + +template +struct ArrayStorage; +template <> +struct ArrayStorage { + using Value = std::vector; + static constexpr auto type = ParamType::Floats; +}; +template <> +struct ArrayStorage { + using Value = std::vector; + static constexpr auto type = ParamType::Ints; +}; +template <> +struct ArrayStorage { + using Value = ImageValues; + static constexpr auto type = ParamType::Tensors; +}; +template <> +struct ArrayStorage { + using Value = AudioValues; + static constexpr auto type = ParamType::Audios; +}; +template <> +struct ArrayStorage { + using Value = VideoValues; + static constexpr auto type = ParamType::Videos; +}; +template <> +struct ArrayStorage { + using Value = LoraValues; + static constexpr auto type = ParamType::Loras; +}; +template <> +struct ArrayStorage { + using Value = EmbeddingValues; + static constexpr auto type = ParamType::Embeddings; +}; + +template +static const auto& array_items(const std::vector& value) { + return value; +} +static const auto& array_items(const ImageValues& value) { + return value.items; +} +static const auto& array_items(const AudioValues& value) { + return value.items; +} +static const auto& array_items(const VideoValues& value) { + return value.items; +} +static const auto& array_items(const LoraValues& value) { + return value.items; +} +static const auto& array_items(const EmbeddingValues& value) { + return value.items; +} + +template +class ParamGroup { + std::vector& specs; + int operations; + Select select; + + template + ParamSpec binding(const char* key, ParamType type, Field Object::* field, Convert convert) const { + return parameter(key, type, operations, [select = select, field, convert](const Value& value, sd::OptConfig& config) { + const auto decoded = convert(value); + std::apply([&](auto&... target) { ((target.*field = decoded), ...); }, select(config)); + }); + } + +public: + ParamGroup(std::vector& specs, int operations, Select select) + : specs(specs), operations(operations), select(select) {} + + ParamGroup for_operations(int mask) const { return {specs, mask, select}; } + + template + void boolean(const char* key, bool Object::* field) { + specs.push_back(binding(key, ParamType::Bool, field, [](bool value) { return value; })); + } + + template + void integer(const char* key, Field Object::* field, int64_t min = INT64_MIN, int64_t max = INT64_MAX) { + auto spec = binding(key, ParamType::Int, field, [](int64_t value) { return static_cast(value); }); + spec.int_min = min; + spec.int_max = max; + specs.push_back(std::move(spec)); + } + + template + void number(const char* key, float Object::* field, double min = -std::numeric_limits::max(), double max = std::numeric_limits::max()) { + auto spec = binding(key, ParamType::Float, field, [](double value) { return static_cast(value); }); + spec.float_min = min; + spec.float_max = max; + specs.push_back(std::move(spec)); + } + + template + void string(const char* key, const char* Object::* field) { + specs.push_back(binding(key, ParamType::String, field, [](const std::string& value) { return value.c_str(); })); + } + + template + void image(const char* key, sd::Image Object::* field) { + specs.push_back(binding(key, ParamType::Tensor, field, [](const TensorValue& value) { return sd::api::image_view(*value.tensor.get()); })); + } + + template + void choice(const char* key, Enum Object::* field, Enum (*parse)(const char*), Enum invalid, bool allow_auto = false) { + auto spec = binding(key, ParamType::String, field, [=](const std::string& value) { + return allow_auto && value == "auto" ? invalid : parse(value.c_str()); + }); + spec.valid = [=](const ParamValue& value) { + const auto& text = std::get(value); + return (allow_auto && text == "auto") || parse(text.c_str()) != invalid; + }; + specs.push_back(std::move(spec)); + } + + template + void frames(const char* key, Element* Object::* data, Count Object::* count) { + auto spec = parameter(key, ParamType::Tensor, operations, [select = select, data, count](const TensorValue& value, sd::OptConfig& config) { + std::apply([&](auto&... target) { + ((target.*data = const_cast(value.frames.data()), target.*count = static_cast(value.frames.size())), ...); + }, + select(config)); + }); + spec.video_tensor = true; + specs.push_back(std::move(spec)); + } + + template + void array(const char* key, Element* Object::* data, Count Object::* count, bool (*valid)(const ParamValue&) = nullptr, const char* reason = "invalid parameter value") { + using Storage = ArrayStorage>; + using Value = typename Storage::Value; + auto spec = parameter(key, Storage::type, operations, [select = select, data, count](const Value& value, sd::OptConfig& config) { + const auto& items = array_items(value); + std::apply([&](auto&... target) { + ((target.*data = const_cast(items.data()), target.*count = static_cast(items.size())), ...); + }, + select(config)); + }); + spec.valid = valid; + spec.invalid_reason = reason; + specs.push_back(std::move(spec)); + } +}; + +static void add_context_specs(std::vector& specs) { + using namespace sd::api; + ParamGroup ctx(specs, Context, [](sd::OptConfig& config) { return std::tie(config.context); }); + + ctx.number("attn_scale", &sd::ContextConfig::attn_scale, 0, std::numeric_limits::max()); + ctx.string("audio_encoder_path", &sd::ContextConfig::audio_encoder_path); + ctx.string("audio_vae_path", &sd::ContextConfig::audio_vae_path); + ctx.boolean("auto_fit", &sd::ContextConfig::auto_fit); + ctx.boolean("batched_cfg", &sd::ContextConfig::batched_cfg); + ctx.string("clip_g_path", &sd::ContextConfig::clip_g_path); + ctx.string("clip_l_path", &sd::ContextConfig::clip_l_path); + ctx.string("clip_vision_path", &sd::ContextConfig::clip_vision_path); + ctx.integer("conditioning_cache_size", &sd::ContextConfig::conditioning_cache_size, 0, INT32_MAX); + ctx.string("control_net_path", &sd::ContextConfig::control_net_path); + ctx.boolean("diffusion_conv_direct", &sd::ContextConfig::diffusion_conv_direct); + ctx.boolean("diffusion_flash_attn", &sd::ContextConfig::diffusion_flash_attn); + ctx.boolean("disable_prefetch", &sd::ContextConfig::disable_prefetch); + ctx.boolean("disable_segmented_compute", &sd::ContextConfig::disable_segmented_compute); + ctx.boolean("eager_load", &sd::ContextConfig::eager_load); + ctx.array("embeddings", &sd::ContextConfig::embeddings, &sd::ContextConfig::embedding_count); + ctx.string("embeddings_connectors_path", &sd::ContextConfig::embeddings_connectors_path); + ctx.boolean("enable_mmap", &sd::ContextConfig::enable_mmap); + ctx.boolean("flash_attn", &sd::ContextConfig::flash_attn); + ctx.boolean("force_sdxl_vae_conv_scale", &sd::ContextConfig::force_sdxl_vae_conv_scale); + ctx.string("high_noise_diffusion_model_path", &sd::ContextConfig::high_noise_diffusion_model_path); + ctx.string("ip_adapter_path", &sd::ContextConfig::ip_adapter_path); + ctx.number("linear_scale", &sd::ContextConfig::linear_scale, 0, std::numeric_limits::max()); + ctx.string("llm_path", &sd::ContextConfig::llm_path); + ctx.string("llm_vision_path", &sd::ContextConfig::llm_vision_path); + ctx.choice("lora_apply_mode", &sd::ContextConfig::lora_apply_mode, str_to_lora_apply_mode, LORA_APPLY_MODE_COUNT); + ctx.string("max_vram", &sd::ContextConfig::max_vram); + ctx.string("model_args", &sd::ContextConfig::model_args); + ctx.for_operations(Context | Upscaler | ADetailer).string("model_path", &sd::ContextConfig::model_path); + ctx.string("motion_module_path", &sd::ContextConfig::motion_module_path); + ctx.string("photo_maker_path", &sd::ContextConfig::photo_maker_path); + ctx.choice("prediction", &sd::ContextConfig::prediction, str_to_prediction, PREDICTION_COUNT, true); + ctx.string("pulid_weights_path", &sd::ContextConfig::pulid_weights_path); + ctx.choice("rng_type", &sd::ContextConfig::rng_type, str_to_rng_type, RNG_TYPE_COUNT, true); + ctx.string("rpc_servers", &sd::ContextConfig::rpc_servers); + ctx.boolean("sage_attn", &sd::ContextConfig::sage_attn); + ctx.choice("sampler_rng_type", &sd::ContextConfig::sampler_rng_type, str_to_rng_type, RNG_TYPE_COUNT, true); + ctx.string("split_mode", &sd::ContextConfig::split_mode); + ctx.string("t5xxl_path", &sd::ContextConfig::t5xxl_path); + ctx.boolean("tae_preview_only", &sd::ContextConfig::tae_preview_only); + ctx.string("taesd_path", &sd::ContextConfig::taesd_path); + ctx.string("tensor_type_rules", &sd::ContextConfig::tensor_type_rules); + ctx.string("tokenizer", &sd::ContextConfig::tokenizer); + ctx.string("uncond_diffusion_model_path", &sd::ContextConfig::uncond_diffusion_model_path); + ctx.boolean("vae_conv_direct", &sd::ContextConfig::vae_conv_direct); + ctx.choice("vae_format", &sd::ContextConfig::vae_format, parse_vae_format, SD_VAE_FORMAT_COUNT); + ctx.choice("wtype", &sd::ContextConfig::wtype, str_to_sd_type, SD_TYPE_COUNT, true); + + specs.push_back(parameter("backend", ParamType::String, Context | Upscaler | VidUpscaler | ADetailer, [](const std::string& value, sd::OptConfig& config) { config.context.backend = value.c_str(); config.post.video_upscaler.backend = value.c_str(); })); + specs.push_back(parameter("conv_direct_mode", ParamType::String, Context | VidUpscaler, [](const std::string& value, sd::OptConfig& config) { config.context.conv_direct_mode = value.c_str(); config.post.video_upscaler.conv_direct_mode = value.c_str(); })); + specs.push_back(parameter("diffusion_model_path", ParamType::String, Context | VidUpscaler, [](const std::string& value, sd::OptConfig& config) { config.context.diffusion_model_path = value.c_str(); config.post.video_upscaler.diffusion_model_path = value.c_str(); })); + specs.push_back(parameter("flash_attn_mode", ParamType::String, Context | VidUpscaler, [](const std::string& value, sd::OptConfig& config) { config.context.flash_attn_mode = value.c_str(); config.post.video_upscaler.flash_attn_mode = value.c_str(); })); + specs.push_back(integer_parameter( + "n_threads", Context | Upscaler | VidUpscaler | ADetailer, [](const int64_t& value, sd::OptConfig& config) { config.context.n_threads = static_cast(value); config.post.video_upscaler.n_threads = static_cast(value); }, 1, INT32_MAX)); + specs.push_back(parameter("params_backend", ParamType::String, Context | Upscaler | VidUpscaler | ADetailer, [](const std::string& value, sd::OptConfig& config) { config.context.params_backend = value.c_str(); config.post.video_upscaler.params_backend = value.c_str(); })); + specs.push_back(parameter("vae_path", ParamType::String, Context | VidUpscaler, [](const std::string& value, sd::OptConfig& config) { config.context.vae_path = value.c_str(); config.post.video_upscaler.vae_path = value.c_str(); })); +} + +static void add_generation_specs(std::vector& specs) { + using namespace sd::api; + ParamGroup img(specs, Image, [](sd::OptConfig& config) { return std::tie(config.image); }); + ParamGroup vid(specs, Video, [](sd::OptConfig& config) { return std::tie(config.video); }); + ParamGroup preprocess(specs, Image | Video, [](sd::OptConfig& config) { return std::tie(config.image.image_preprocess, config.video.image_preprocess); }); + ParamGroup photomaker(specs, Image, [](sd::OptConfig& config) { return std::tie(config.image.pm_params); }); + ParamGroup pulid(specs, Image, [](sd::OptConfig& config) { return std::tie(config.image.pulid_params); }); + + img.integer("batch_count", &sd::ImageConfig::batch_count, 1, INT32_MAX); + img.image("control_image", &sd::ImageConfig::control_image); + img.number("control_strength", &sd::ImageConfig::control_strength, 0, std::numeric_limits::max()); + img.choice("control_type", &sd::ImageConfig::control_type, str_to_sd_control_type, SD_CONTROL_TYPE_COUNT); + img.image("ip_adapter_image", &sd::ImageConfig::ip_adapter_image); + img.number("ip_adapter_strength", &sd::ImageConfig::ip_adapter_strength, 0, std::numeric_limits::max()); + img.image("mask_image", &sd::ImageConfig::mask_image); + img.integer("qwen_image_layers", &sd::ImageConfig::qwen_image_layers, 0, INT32_MAX); + + vid.frames("control_frames", &sd::VideoConfig::control_frames, &sd::VideoConfig::control_frames_size); + vid.image("end_image", &sd::VideoConfig::end_image); + vid.number("moe_boundary", &sd::VideoConfig::moe_boundary, 0, 1); + vid.array("ref_audios", &sd::VideoConfig::ref_audios, &sd::VideoConfig::ref_audios_count); + vid.array("ref_videos", &sd::VideoConfig::ref_videos, &sd::VideoConfig::ref_videos_count); + vid.number("vace_strength", &sd::VideoConfig::vace_strength, 0, std::numeric_limits::max()); + vid.integer("video_frames", &sd::VideoConfig::video_frames, 1, INT32_MAX); + + preprocess.string("image_preprocess.rules", &sd::ImagePreprocessConfig::rules); + + photomaker.string("photomaker.id_embed_path", &sd::PhotoMakerConfig::id_embed_path); + photomaker.array("photomaker.id_images", &sd::PhotoMakerConfig::id_images, &sd::PhotoMakerConfig::id_images_count); + photomaker.number("photomaker.style_strength", &sd::PhotoMakerConfig::style_strength, 0, 100); + + pulid.string("pulid.id_embedding_path", &sd::PuLIDConfig::id_embedding_path); + pulid.number("pulid.id_weight", &sd::PuLIDConfig::id_weight, 0, std::numeric_limits::max()); + + specs.push_back(parameter("circular_x", ParamType::Bool, Image | Video | ADetail, [](const bool& value, sd::OptConfig& config) { config.image.circular_x = value; config.video.circular_x = value; })); + specs.push_back(parameter("circular_y", ParamType::Bool, Image | Video | ADetail, [](const bool& value, sd::OptConfig& config) { config.image.circular_y = value; config.video.circular_y = value; })); + specs.push_back(integer_parameter( + "clip_skip", Image | Video | ADetail, [](const int64_t& value, sd::OptConfig& config) { config.image.clip_skip = static_cast(value); config.video.clip_skip = static_cast(value); }, INT32_MIN, INT32_MAX)); + specs.push_back(integer_parameter( + "fps", Video | VidUpscale, [](const int64_t& value, sd::OptConfig& config) { config.video.fps = static_cast(value); config.post.video_upscale.fps = static_cast(value); }, 1, INT32_MAX)); + specs.push_back(integer_parameter( + "height", Image | Video | ADetail, [](const int64_t& value, sd::OptConfig& config) { config.image.height = static_cast(value); config.video.height = static_cast(value); }, 1, INT32_MAX)); + specs.push_back(parameter("init_image", ParamType::Tensor, Image | Video, [](const TensorValue& value, sd::OptConfig& config) { config.image.init_image = sd::api::image_view(*value.tensor.get()); config.video.init_image = config.image.init_image; })); + specs.push_back(parameter("loras", ParamType::Loras, Image | Video | ADetail, [](const LoraValues& value, sd::OptConfig& config) { config.image.loras = value.items.data(); config.image.lora_count = static_cast(value.items.size()); config.video.loras = value.items.data(); config.video.lora_count = static_cast(value.items.size()); })); + specs.push_back(parameter("negative_prompt", ParamType::String, Image | Video | ADetail, [](const std::string& value, sd::OptConfig& config) { config.image.negative_prompt = value.c_str(); config.video.negative_prompt = value.c_str(); })); + specs.push_back(parameter("prompt", ParamType::String, Image | Video | ADetail, [](const std::string& value, sd::OptConfig& config) { config.image.prompt = value.c_str(); config.video.prompt = value.c_str(); })); + specs.push_back(parameter("ref_image_args", ParamType::String, Image | Video, [](const std::string& value, sd::OptConfig& config) { config.image.ref_image_args = value.c_str(); config.video.ref_image_args = value.c_str(); })); + specs.push_back(parameter("ref_images", ParamType::Tensors, Image | Video, [](const ImageValues& value, sd::OptConfig& config) { config.image.ref_images = const_cast(value.items.data()); config.image.ref_images_count = static_cast(value.items.size()); config.video.ref_images = config.image.ref_images; config.video.ref_images_count = config.image.ref_images_count; })); + specs.push_back(integer_parameter("seed", Image | Video | VidUpscale | ADetail, [](const int64_t& value, sd::OptConfig& config) { config.image.seed = static_cast(value); config.video.seed = static_cast(value); config.post.video_upscale.seed = value; })); + specs.push_back(float_parameter( + "strength", Image | Video | ADetail, [](const double& value, sd::OptConfig& config) { config.image.strength = static_cast(value); config.video.strength = static_cast(value); }, 0, 1)); + specs.push_back(integer_parameter( + "width", Image | Video | ADetail, [](const int64_t& value, sd::OptConfig& config) { config.image.width = static_cast(value); config.video.width = static_cast(value); }, 1, INT32_MAX)); +} + +static void add_sampling_specs(std::vector& specs) { + using namespace sd::api; + ParamGroup high_noise(specs, Video, [](sd::OptConfig& config) { return std::tie(config.video.high_noise_sample_params); }); + ParamGroup high_noise_guidance(specs, Video, [](sd::OptConfig& config) { return std::tie(config.video.high_noise_sample_params.guidance); }); + ParamGroup high_noise_slg(specs, Video, [](sd::OptConfig& config) { return std::tie(config.video.high_noise_sample_params.guidance.slg); }); + ParamGroup sample(specs, Image | Video | ADetail, [](sd::OptConfig& config) { return std::tie(config.image.sample_params, config.video.sample_params); }); + ParamGroup guidance(specs, Image | Video | ADetail, [](sd::OptConfig& config) { return std::tie(config.image.sample_params.guidance, config.video.sample_params.guidance); }); + ParamGroup slg(specs, Image | Video | ADetail, [](sd::OptConfig& config) { return std::tie(config.image.sample_params.guidance.slg, config.video.sample_params.guidance.slg); }); + + high_noise.string("high_noise_sample.args", &sd::SampleConfig::extra_sample_args); + high_noise.number("high_noise_sample.eta", &sd::SampleConfig::eta, 0, std::numeric_limits::max()); + high_noise.choice("high_noise_sample.method", &sd::SampleConfig::sample_method, str_to_sample_method, SAMPLE_METHOD_COUNT, true); + high_noise.integer("high_noise_sample.steps", &sd::SampleConfig::sample_steps, -1, INT32_MAX); + + high_noise_guidance.number("high_noise_sample.guidance.distilled_guidance", &sd::GuidanceConfig::distilled_guidance); + high_noise_guidance.number("high_noise_sample.guidance.img_cfg", &sd::GuidanceConfig::img_cfg); + high_noise_guidance.number("high_noise_sample.guidance.txt_cfg", &sd::GuidanceConfig::txt_cfg); + + high_noise_slg.number("high_noise_sample.guidance.slg.layer_end", &sd::SLGConfig::layer_end, 0, 1); + high_noise_slg.number("high_noise_sample.guidance.slg.layer_start", &sd::SLGConfig::layer_start, 0, 1); + high_noise_slg.array("high_noise_sample.guidance.slg.layers", &sd::SLGConfig::layers, &sd::SLGConfig::layer_count, valid_layers, "layer indices must be nonnegative"); + high_noise_slg.number("high_noise_sample.guidance.slg.scale", &sd::SLGConfig::scale, 0, std::numeric_limits::max()); + + sample.string("sample.args", &sd::SampleConfig::extra_sample_args); + sample.number("sample.eta", &sd::SampleConfig::eta, 0, std::numeric_limits::max()); + sample.number("sample.flow_shift", &sd::SampleConfig::flow_shift, 0, std::numeric_limits::max()); + sample.choice("sample.method", &sd::SampleConfig::sample_method, str_to_sample_method, SAMPLE_METHOD_COUNT, true); + sample.choice("sample.scheduler", &sd::SampleConfig::scheduler, str_to_scheduler, SCHEDULER_COUNT, true); + sample.integer("sample.shifted_timestep", &sd::SampleConfig::shifted_timestep, 0, INT32_MAX); + sample.array("sample.sigmas", &sd::SampleConfig::custom_sigmas, &sd::SampleConfig::custom_sigmas_count, valid_sigmas, "sigmas require at least two strictly decreasing nonnegative entries ending in zero, or an empty array"); + sample.integer("sample.steps", &sd::SampleConfig::sample_steps, 1, INT32_MAX); + + guidance.number("sample.guidance.distilled_guidance", &sd::GuidanceConfig::distilled_guidance); + guidance.number("sample.guidance.img_cfg", &sd::GuidanceConfig::img_cfg); + guidance.number("sample.guidance.txt_cfg", &sd::GuidanceConfig::txt_cfg); + + slg.number("sample.guidance.slg.layer_end", &sd::SLGConfig::layer_end, 0, 1); + slg.number("sample.guidance.slg.layer_start", &sd::SLGConfig::layer_start, 0, 1); + slg.array("sample.guidance.slg.layers", &sd::SLGConfig::layers, &sd::SLGConfig::layer_count, valid_layers, "layer indices must be nonnegative"); + slg.number("sample.guidance.slg.scale", &sd::SLGConfig::scale, 0, std::numeric_limits::max()); +} + +static void add_cache_specs(std::vector& specs) { + using namespace sd::api; + ParamGroup cache(specs, Image | Video | ADetail, [](sd::OptConfig& config) { return std::tie(config.image.cache, config.video.cache); }); + + cache.integer("cache.bn_compute_blocks", &sd::CacheConfig::Bn_compute_blocks, 0, INT32_MAX); + cache.number("cache.end_percent", &sd::CacheConfig::end_percent, 0, 1); + cache.number("cache.error_decay_rate", &sd::CacheConfig::error_decay_rate, 0, 1); + cache.integer("cache.fn_compute_blocks", &sd::CacheConfig::Fn_compute_blocks, 0, INT32_MAX); + cache.integer("cache.max_cached_steps", &sd::CacheConfig::max_cached_steps, INT32_MIN, INT32_MAX); + cache.integer("cache.max_continuous_cached_steps", &sd::CacheConfig::max_continuous_cached_steps, INT32_MIN, INT32_MAX); + cache.integer("cache.max_warmup_steps", &sd::CacheConfig::max_warmup_steps, 0, INT32_MAX); + cache.choice("cache.mode", &sd::CacheConfig::mode, parse_cache, static_cast(-1)); + cache.boolean("cache.reset_error_on_compute", &sd::CacheConfig::reset_error_on_compute); + cache.number("cache.residual_diff_threshold", &sd::CacheConfig::residual_diff_threshold, 0, std::numeric_limits::max()); + cache.number("cache.reuse_threshold", &sd::CacheConfig::reuse_threshold, 0, std::numeric_limits::max()); + cache.string("cache.scm_mask", &sd::CacheConfig::scm_mask); + cache.boolean("cache.scm_policy_dynamic", &sd::CacheConfig::scm_policy_dynamic); + cache.number("cache.spectrum_flex_window", &sd::CacheConfig::spectrum_flex_window, 0, std::numeric_limits::max()); + cache.number("cache.spectrum_lam", &sd::CacheConfig::spectrum_lam, 0, std::numeric_limits::max()); + cache.integer("cache.spectrum_m", &sd::CacheConfig::spectrum_m, 1, INT32_MAX); + cache.number("cache.spectrum_stop_percent", &sd::CacheConfig::spectrum_stop_percent, 0, 1); + cache.number("cache.spectrum_w", &sd::CacheConfig::spectrum_w, 0, std::numeric_limits::max()); + cache.integer("cache.spectrum_warmup_steps", &sd::CacheConfig::spectrum_warmup_steps, 0, INT32_MAX); + cache.integer("cache.spectrum_window_size", &sd::CacheConfig::spectrum_window_size, 1, INT32_MAX); + cache.number("cache.start_percent", &sd::CacheConfig::start_percent, 0, 1); + cache.integer("cache.taylorseer_n_derivatives", &sd::CacheConfig::taylorseer_n_derivatives, 0, INT32_MAX); + cache.integer("cache.taylorseer_skip_interval", &sd::CacheConfig::taylorseer_skip_interval, 1, INT32_MAX); + cache.boolean("cache.use_relative_threshold", &sd::CacheConfig::use_relative_threshold); +} + +static void add_hires_specs(std::vector& specs) { + using namespace sd::api; + ParamGroup hires(specs, Image | Video, [](sd::OptConfig& config) { return std::tie(config.image.hires, config.video.hires); }); + + hires.number("hires.denoising_strength", &sd::HiresConfig::denoising_strength, 0, 1); + hires.boolean("hires.enabled", &sd::HiresConfig::enabled); + hires.string("hires.model_path", &sd::HiresConfig::model_path); + hires.number("hires.scale", &sd::HiresConfig::scale, 0, std::numeric_limits::max()); + hires.array("hires.sigmas", &sd::HiresConfig::custom_sigmas, &sd::HiresConfig::custom_sigmas_count, valid_sigmas, "sigmas require at least two strictly decreasing nonnegative entries ending in zero, or an empty array"); + hires.integer("hires.steps", &sd::HiresConfig::steps, 0, INT32_MAX); + hires.integer("hires.target_height", &sd::HiresConfig::target_height, 0, INT32_MAX); + hires.integer("hires.target_width", &sd::HiresConfig::target_width, 0, INT32_MAX); + hires.integer("hires.upscale_tile_size", &sd::HiresConfig::upscale_tile_size, 1, INT32_MAX); + hires.choice("hires.upscaler", &sd::HiresConfig::upscaler, parse_hires_upscaler, SD_HIRES_UPSCALER_COUNT); +} + +static void add_tiling_specs(std::vector& specs) { + using namespace sd::api; + ParamGroup tiling(specs, Image | Video | ADetail, [](sd::OptConfig& config) { return std::tie(config.image.vae_tiling_params, config.video.vae_tiling_params); }); + + tiling.boolean("vae_tiling.enabled", &sd::TilingConfig::enabled); + tiling.string("vae_tiling.extra_tiling_args", &sd::TilingConfig::extra_tiling_args); + tiling.number("vae_tiling.rel_size_h", &sd::TilingConfig::rel_size_h, 0, std::numeric_limits::max()); + tiling.number("vae_tiling.rel_size_w", &sd::TilingConfig::rel_size_w, 0, std::numeric_limits::max()); + tiling.number("vae_tiling.target_overlap", &sd::TilingConfig::target_overlap, 0, 0.99); + tiling.boolean("vae_tiling.temporal_tiling", &sd::TilingConfig::temporal_tiling); + tiling.integer("vae_tiling.tile_size_h", &sd::TilingConfig::tile_size_h, 0, INT32_MAX); + tiling.integer("vae_tiling.tile_size_w", &sd::TilingConfig::tile_size_w, 0, INT32_MAX); +} + +static void add_postprocess_specs(std::vector& specs) { + using namespace sd::api; + ParamGroup input(specs, Upscale | ADetail, [](sd::OptConfig& config) { return std::tie(config.post); }); + ParamGroup upscaler(specs, Upscaler, [](sd::OptConfig& config) { return std::tie(config.post); }); + ParamGroup video_upscaler(specs, VidUpscaler, [](sd::OptConfig& config) { return std::tie(config.post.video_upscaler); }); + ParamGroup video_upscale(specs, VidUpscale, [](sd::OptConfig& config) { return std::tie(config.post.video_upscale); }); + ParamGroup adetail(specs, ADetail, [](sd::OptConfig& config) { return std::tie(config.post.adetail); }); + + input.image("image", &sd::PostprocessConfig::image); + + upscaler.boolean("conv_direct", &sd::PostprocessConfig::conv_direct); + upscaler.integer("tile_size", &sd::PostprocessConfig::tile_size, 1, INT32_MAX); + + video_upscaler.integer("upscale_factor", &sd::VideoUpscalerConfig::upscale_factor, 1, INT32_MAX); + + video_upscale.frames("frames", &sd::VideoUpscaleConfig::frames, &sd::VideoUpscaleConfig::frame_count); + video_upscale.integer("window_size", &sd::VideoUpscaleConfig::window_size, 1, INT32_MAX); + video_upscale.integer("temporal_chunk_size", &sd::VideoUpscaleConfig::temporal_chunk_size, 1, INT32_MAX); + + adetail.string("adetail.prompt", &sd::ADetailConfig::prompt); + adetail.string("adetail.negative_prompt", &sd::ADetailConfig::negative_prompt); + adetail.string("adetail.args", &sd::ADetailConfig::extra_ad_args); +} + +static std::vector make_specs() { + std::vector specs; + add_context_specs(specs); + add_generation_specs(specs); + add_sampling_specs(specs); + add_cache_specs(specs); + add_hires_specs(specs); + add_tiling_specs(specs); + add_postprocess_specs(specs); + return specs; +} + +static const ParamSpec* find_spec(const char* key) { + static const auto specs = make_specs(); + if (!key) + return nullptr; + for (const auto& spec : specs) + if (strcmp(key, spec.key) == 0) + return &spec; + return nullptr; +} + +static sd_status_t check_key(sd_opt_t* params, const char* key, ParamType type, const ParamSpec** out) { + if (!params || !key) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "params and key are required"); + *out = find_spec(key); + if (!*out) + return sd::api::error(SD_STATUS_UNKNOWN_KEY, key, "unknown parameter"); + if (!((*out)->operations & params->operation)) + return sd::api::error(SD_STATUS_UNSUPPORTED, key, "parameter does not apply to this option object"); + if ((*out)->type != type) + return sd::api::error(SD_STATUS_TYPE_MISMATCH, key, "incorrect setter type"); + return SD_STATUS_OK; +} + +static sd_status_t store_value(sd_opt_t* params, const ParamSpec& spec, ParamValue value) { + if (spec.type == ParamType::Int) { + const auto number = std::get(value); + if (number < spec.int_min || number > spec.int_max) + return sd::api::error(SD_STATUS_OUT_OF_RANGE, spec.key, "integer outside supported range"); + } + if (spec.type == ParamType::Float) { + const auto number = std::get(value); + if (!std::isfinite(number) || number < spec.float_min || number > spec.float_max) + return sd::api::error(SD_STATUS_OUT_OF_RANGE, spec.key, "float must be finite and within supported range"); + } + if (spec.valid && !spec.valid(value)) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, spec.key, spec.invalid_reason); + params->values.insert_or_assign(spec.key, std::move(value)); + return SD_STATUS_OK; +} + +static sd_opt_t* create_opt(sd::api::Operation operation) { + sd_opt_t* params = nullptr; + sd::api::call([&] { params = new sd_opt_t{operation, {}}; return SD_STATUS_OK; }); + return params; +} + +sd_ctx_opt_t* sd_ctx_opt_create(void) { + return create_opt(sd::api::Context); +} +sd_img_gen_opt_t* sd_img_gen_opt_create(void) { + return create_opt(sd::api::Image); +} +sd_vid_gen_opt_t* sd_vid_gen_opt_create(void) { + return create_opt(sd::api::Video); +} +sd_upscaler_opt_t* sd_upscaler_opt_create(void) { + return create_opt(sd::api::Upscaler); +} + +sd_upscale_opt_t* sd_upscale_opt_create(void) { + return create_opt(sd::api::Upscale); +} + +sd_vid_upscaler_opt_t* sd_vid_upscaler_opt_create(void) { + return create_opt(sd::api::VidUpscaler); +} + +sd_vid_upscale_opt_t* sd_vid_upscale_opt_create(void) { + return create_opt(sd::api::VidUpscale); +} + +sd_adetailer_opt_t* sd_adetailer_opt_create(void) { + return create_opt(sd::api::ADetailer); +} + +sd_adetail_opt_t* sd_adetail_opt_create(void) { + return create_opt(sd::api::ADetail); +} + +void sd_opt_free(sd_opt_t* params) { + delete params; +} + +sd_status_t sd_opt_set_bool(sd_opt_t* params, const char* key, bool value) { + return sd::api::call([&] { + const ParamSpec* spec = nullptr; + auto status = check_key(params, key, ParamType::Bool, &spec); + if (status != SD_STATUS_OK) + return status; + return store_value(params, *spec, ParamValue(value)); + }); +} + +sd_status_t sd_opt_set_int(sd_opt_t* params, const char* key, int64_t value) { + return sd::api::call([&] { + const ParamSpec* spec = nullptr; + auto status = check_key(params, key, ParamType::Int, &spec); + if (status != SD_STATUS_OK) + return status; + return store_value(params, *spec, ParamValue(value)); + }); +} + +sd_status_t sd_opt_set_float(sd_opt_t* params, const char* key, double value) { + return sd::api::call([&] { + const ParamSpec* spec = nullptr; + auto status = check_key(params, key, ParamType::Float, &spec); + if (status != SD_STATUS_OK) + return status; + return store_value(params, *spec, ParamValue(value)); + }); +} + +sd_status_t sd_opt_set_string(sd_opt_t* params, const char* key, const char* value) { + return sd::api::call([&] { + const ParamSpec* spec = nullptr; + auto status = check_key(params, key, ParamType::String, &spec); + if (status != SD_STATUS_OK) + return status; + if (!value) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "string is NULL; use unset to restore defaults"); + return store_value(params, *spec, std::string(value)); + }); +} + +sd_status_t sd_opt_set_tensor(sd_opt_t* params, const char* key, const sd_tensor_t* tensor) { + return sd::api::call([&] { + const ParamSpec* spec = nullptr; + auto status = check_key(params, key, ParamType::Tensor, &spec); + if (status != SD_STATUS_OK) + return status; + if (!tensor) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "tensor is required"); + TensorValue value; + value.tensor = sd::api::TensorRef(tensor); + try { + if (spec->video_tensor) + value.frames = sd::api::video_frames(*tensor); + else + sd::api::image_view(*tensor); + } catch (const std::invalid_argument& error) { + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, error.what()); + } + return store_value(params, *spec, std::move(value)); + }); +} + +template +static sd_status_t set_array(sd_opt_t* params, const char* key, const T* values, size_t count, ParamType type, Function build) { + return sd::api::call([&] { + const ParamSpec* spec = nullptr; + auto status = check_key(params, key, type, &spec); + if (status != SD_STATUS_OK) + return status; + if (count > INT32_MAX) + return sd::api::error(SD_STATUS_OUT_OF_RANGE, key, "too many array elements"); + if (!values && count) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "array is NULL with nonzero count"); + return build(*spec); + }); +} + +sd_status_t sd_opt_set_tensors(sd_opt_t* params, const char* key, const sd_tensor_t* const* values, size_t count) { + return set_array(params, key, values, count, ParamType::Tensors, [&](const ParamSpec& spec) { + ImageValues copy; + copy.tensors.reserve(count); + copy.items.reserve(count); + for (size_t i = 0; i < count; ++i) { + if (!values[i]) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "tensor is required"); + copy.items.push_back(sd::api::image_view(*values[i])); + copy.tensors.emplace_back(values[i]); + } + return store_value(params, spec, std::move(copy)); + }); +} + +sd_status_t sd_opt_set_floats(sd_opt_t* params, const char* key, const float* values, size_t count) { + return set_array(params, key, values, count, ParamType::Floats, [&](const ParamSpec& spec) { + std::vector copy; + for (size_t i = 0; i < count; ++i) { + if (!std::isfinite(values[i])) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "array values must be finite"); + copy.push_back(values[i]); + } + return store_value(params, spec, std::move(copy)); + }); +} + +sd_status_t sd_opt_set_ints(sd_opt_t* params, const char* key, const int32_t* values, size_t count) { + return set_array(params, key, values, count, ParamType::Ints, [&](const ParamSpec& spec) { + std::vector copy; + for (size_t i = 0; i < count; ++i) { + copy.push_back(values[i]); + } + return store_value(params, spec, std::move(copy)); + }); +} + +sd_status_t sd_opt_set_loras(sd_opt_t* params, const char* key, const sd_lora_t* values, size_t count) { + return set_array(params, key, values, count, ParamType::Loras, [&](const ParamSpec& spec) { + LoraValues copy; + copy.paths.reserve(count); + for (size_t i = 0; i < count; ++i) { + if (!values[i].path || !*values[i].path || !std::isfinite(values[i].multiplier)) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "LoRA requires a path and finite multiplier"); + copy.paths.emplace_back(values[i].path); + copy.items.push_back({values[i].is_high_noise, values[i].multiplier, copy.paths.back().c_str()}); + } + return store_value(params, spec, std::move(copy)); + }); +} + +sd_status_t sd_opt_set_embeddings(sd_opt_t* params, const char* key, const sd_embedding_t* values, size_t count) { + return set_array(params, key, values, count, ParamType::Embeddings, [&](const ParamSpec& spec) { + EmbeddingValues copy; + copy.strings.reserve(count * 2); + for (size_t i = 0; i < count; ++i) { + if (!values[i].name || !*values[i].name || !values[i].path || !*values[i].path) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "embedding requires a name and path"); + copy.strings.emplace_back(values[i].name); + const char* name = copy.strings.back().c_str(); + copy.strings.emplace_back(values[i].path); + copy.items.push_back({name, copy.strings.back().c_str()}); + } + return store_value(params, spec, std::move(copy)); + }); +} + +sd_status_t sd_opt_set_audios(sd_opt_t* params, const char* key, const sd_audio_t* values, size_t count) { + return set_array(params, key, values, count, ParamType::Audios, [&](const ParamSpec& spec) { + AudioValues copy; + copy.tensors.reserve(count); + copy.items.reserve(count); + for (size_t i = 0; i < count; ++i) { + copy.items.push_back(sd::api::audio_view(values[i])); + copy.tensors.emplace_back(values[i].data); + } + return store_value(params, spec, std::move(copy)); + }); +} + +sd_status_t sd_opt_set_videos(sd_opt_t* params, const char* key, const sd_video_t* values, size_t count) { + return set_array(params, key, values, count, ParamType::Videos, [&](const ParamSpec& spec) { + VideoValues copy; + copy.frames.reserve(count); + copy.items.reserve(count); + for (size_t i = 0; i < count; ++i) { + const auto& video = values[i]; + if (!video.data || video.fps <= 0) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "video data and positive fps are required"); + copy.frames.push_back(sd::api::video_frames(*video.data)); + copy.tensors.emplace_back(video.data); + sd::Audio audio{}; + if (video.audio.data) { + audio = sd::api::audio_view(video.audio); + copy.tensors.emplace_back(video.audio.data); + } + copy.items.push_back({copy.frames.back().data(), static_cast(copy.frames.back().size()), video.fps, audio}); + } + return store_value(params, spec, std::move(copy)); + }); +} + +sd_status_t sd_opt_unset(sd_opt_t* params, const char* key) { + return sd::api::call([&] { + if (!params || !key) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "params and key are required"); + const auto* spec = find_spec(key); + if (!spec) + return sd::api::error(SD_STATUS_UNKNOWN_KEY, key, "unknown parameter"); + if (!(spec->operations & params->operation)) + return sd::api::error(SD_STATUS_UNSUPPORTED, key, "parameter does not apply to this option object"); + params->values.erase(key); + return SD_STATUS_OK; + }); +} + +bool sd::api::is_set(const sd_opt_t* params, const char* key) { + return params && params->values.count(key) != 0; +} + +bool sd::api::has_prefix(const sd_opt_t* params, const char* prefix) { + if (!params) + return false; + const auto it = params->values.lower_bound(prefix); + return it != params->values.end() && it->first.compare(0, strlen(prefix), prefix) == 0; +} + +sd_status_t sd::api::resolve(const sd_opt_t* params, Operation operation, OptConfig& config) { + if (!params) + return error(SD_STATUS_INVALID_ARGUMENT, nullptr, "params is required"); + if (params->operation != operation) + return error(SD_STATUS_UNSUPPORTED, nullptr, "option object has the wrong purpose for this operation"); + config = OptConfig{}; + if (operation == ADetail) + config.image.strength = 0.4f; + for (const auto& entry : params->values) { + const auto* spec = find_spec(entry.first.c_str()); + if (!(spec->operations & operation)) + return error(SD_STATUS_UNSUPPORTED, spec->key, "parameter does not apply to this operation"); + spec->apply(entry.second, config); + } + const auto& image = config.image; + const auto& video = config.video; + const bool sampling = operation == Image || operation == Video || operation == ADetail; + const auto& cache = operation == Video ? video.cache : image.cache; + if (sampling && cache.mode != SD_CACHE_DISABLED) { + if (cache.start_percent >= cache.end_percent) + return error(SD_STATUS_INVALID_ARGUMENT, "cache", "start_percent must precede end_percent"); + } + if (sampling) { + const auto& sample = operation == Video ? video.sample_params : image.sample_params; + if (sample.guidance.slg.layer_start > sample.guidance.slg.layer_end) + return error(SD_STATUS_INVALID_ARGUMENT, "sample.guidance.slg", "layer_start must not exceed layer_end"); + const auto& hires = operation == Video ? video.hires : image.hires; + if (hires.enabled && hires.upscaler != SD_HIRES_UPSCALER_NONE && + hires.scale <= 0.f && hires.target_width <= 0 && hires.target_height <= 0) + return error(SD_STATUS_INVALID_ARGUMENT, "hires", "enabled hires requires a positive scale or target size"); + } + if (operation == Video && video.high_noise_sample_params.guidance.slg.layer_start > video.high_noise_sample_params.guidance.slg.layer_end) + return error(SD_STATUS_INVALID_ARGUMENT, "high_noise_sample.guidance.slg", "layer_start must not exceed layer_end"); + if ((operation == Upscaler || operation == ADetailer) && (!config.context.model_path || !*config.context.model_path)) + return error(SD_STATUS_INVALID_ARGUMENT, "model_path", "model path is required"); + if (operation == VidUpscaler) { + const auto& load = config.post.video_upscaler; + if (!load.diffusion_model_path || !*load.diffusion_model_path || !load.vae_path || !*load.vae_path || load.upscale_factor <= 0) + return error(SD_STATUS_INVALID_ARGUMENT, nullptr, "diffusion_model_path, vae_path and upscale_factor are required"); + } + if ((operation == Upscale || operation == ADetail) && !config.post.image.data) + return error(SD_STATUS_INVALID_ARGUMENT, "image", "input image is required"); + if (operation == Upscale && config.post.image.channel != 3) + return error(SD_STATUS_INVALID_ARGUMENT, "image", "upscaling requires an RGB image"); + if (operation == VidUpscale) { + const auto& input = config.post.video_upscale; + if (!input.frames || input.frame_count <= 0) + return error(SD_STATUS_INVALID_ARGUMENT, "frames", "video upscaling requires nonempty frames"); + for (int i = 0; i < input.frame_count; ++i) + if (input.frames[i].channel != 3 || input.frames[i].width != input.frames[0].width || input.frames[i].height != input.frames[0].height) + return error(SD_STATUS_INVALID_ARGUMENT, "frames", "frames must be RGB and have equal dimensions"); + } + return SD_STATUS_OK; +} diff --git a/src/api/options.h b/src/api/options.h new file mode 100644 index 0000000000..6517829939 --- /dev/null +++ b/src/api/options.h @@ -0,0 +1,25 @@ +#ifndef SD_OPTIONS_H +#define SD_OPTIONS_H + +#include "api.h" +#include "config.h" + +namespace sd::api { + enum Operation { + Context = 1, + Image = 2, + Video = 4, + Upscaler = 8, + Upscale = 16, + VidUpscaler = 32, + VidUpscale = 64, + ADetailer = 128, + ADetail = 256 + }; + + sd_status_t resolve(const sd_opt_t* params, Operation operation, OptConfig& config); + bool is_set(const sd_opt_t* params, const char* key); + bool has_prefix(const sd_opt_t* params, const char* prefix); +} + +#endif // SD_OPTIONS_H diff --git a/src/api/result.cpp b/src/api/result.cpp new file mode 100644 index 0000000000..388f948cee --- /dev/null +++ b/src/api/result.cpp @@ -0,0 +1,243 @@ +#include "result.h" +#include "api.h" + +#include +#include +#include + +namespace sd { + using nlohmann::json; + + char* copy_generation_info(const json& info) { + const std::string text = info.dump(-1, ' ', false, json::error_handler_t::replace); + char* result = static_cast(malloc(text.size() + 1)); + if (result) + memcpy(result, text.c_str(), text.size() + 1); + return result; + } + + bool make_image_generation_result(sd::Image* images, int count, const std::vector& infos, sd::ImageGenerationResult** result_out) { + if (!result_out || !images || count <= 0 || infos.size() != static_cast(count)) { + sd::free_images(images, count); + return false; + } + auto* result = static_cast(calloc(1, sizeof(sd::ImageGenerationResult))); + if (!result) { + sd::free_images(images, count); + return false; + } + result->items = static_cast(calloc(count, sizeof(sd::ImageResult))); + if (!result->items) { + free(result); + sd::free_images(images, count); + return false; + } + result->count = count; + try { + for (int i = 0; i < count; ++i) { + result->items[i].image = images[i]; + images[i] = {}; + result->items[i].generation_info = copy_generation_info(infos[i]); + if (!result->items[i].generation_info) { + sd::free_images(images, count); + sd::free_image_generation_result(result); + return false; + } + } + } catch (...) { + sd::free_images(images, count); + sd::free_image_generation_result(result); + return false; + } + free(images); + *result_out = result; + return true; + } +} + +void sd::free_image_generation_result(sd::ImageGenerationResult* result) { + if (!result) + return; + for (int i = 0; i < result->count; ++i) { + free(result->items[i].image.data); + free(result->items[i].generation_info); + } + free(result->items); + free(result); +} + +void sd::free_video_generation_result(sd::VideoGenerationResult* result) { + if (!result) + return; + sd::free_images(result->frames, result->frame_count); + sd::free_audio(result->audio); + free(result->generation_info); + free(result); +} + +static sd::api::TensorRef consume_image(sd::Image& image) { + std::vector shape{image.channel, image.width, image.height}; + auto* data = image.data; + image.data = nullptr; + return sd::api::own_tensor(SD_TENSOR_U8, std::move(shape), data); +} + +bool sd::make_image_result(ImageGenerationResult* images, sd_result_t** result_out) { + if (result_out) + *result_out = nullptr; + std::unique_ptr owner(images, free_image_generation_result); + if (!result_out || !images) + return false; + auto result = std::make_unique(); + result->kind = SD_RESULT_IMAGE; + result->images.reserve(images->count); + result->generation_infos.reserve(images->count); + for (int i = 0; i < images->count; ++i) { + result->images.push_back(consume_image(images->items[i].image)); + result->generation_infos.emplace_back(images->items[i].generation_info ? images->items[i].generation_info : ""); + } + *result_out = result.release(); + return true; +} + +bool sd::make_image_result(sd::Image* images, int count, const std::vector& infos, sd_result_t** result_out) { + if (result_out) + *result_out = nullptr; + ImageGenerationResult* raw = nullptr; + if (!make_image_generation_result(images, count, infos, &raw)) + return false; + return make_image_result(raw, result_out); +} + +bool sd::make_video_result(VideoGenerationResult* video, sd_result_t** result_out) { + if (result_out) + *result_out = nullptr; + std::unique_ptr owner(video, free_video_generation_result); + if (!result_out || !video) + return false; + auto result = std::make_unique(); + result->kind = SD_RESULT_VIDEO; + result->fps = video->fps; + result->images.reserve(video->frame_count); + for (int i = 0; i < video->frame_count; ++i) + result->images.push_back(consume_image(video->frames[i])); + result->generation_infos.emplace_back(video->generation_info ? video->generation_info : ""); + if (video->audio) { + auto& audio = *video->audio; + std::vector shape{audio.channels, static_cast(audio.sample_count)}; + auto* data = audio.data; + audio.data = nullptr; + result->audio_tensor = sd::api::own_tensor(SD_TENSOR_F32, std::move(shape), data); + result->audio = {audio.sample_rate, result->audio_tensor.get()}; + } + *result_out = result.release(); + return true; +} + +void sd::free_images(sd::Image* result_images, int num_images) { + if (result_images == nullptr) { + return; + } + + for (int i = 0; i < num_images; ++i) { + if (result_images[i].data != nullptr) { + free(result_images[i].data); + result_images[i].data = nullptr; + } + } + + free(result_images); +} + +void sd::free_audio(sd::Audio* audio) { + if (audio == nullptr) { + return; + } + free(audio->data); + audio->data = nullptr; + free(audio); +} + +void sd::free_image(sd::Image* image) { + if (image) { + free(image->data); + *image = {}; + } +} + +void sd_result_free(sd_result_t* result) { + delete result; +} + +sd_result_kind_t sd_result_get_kind(const sd_result_t* result) { + sd::api::clear_error(); + if (!result) { + sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "result is required"); + return SD_RESULT_INVALID; + } + return result->kind; +} + +size_t sd_result_get_image_count(const sd_result_t* result) { + if (!sd_result_get_kind(result)) + return 0; + return result->images.size(); +} + +const sd_tensor_t* sd_result_get_image(const sd_result_t* result, size_t index) { + if (index >= sd_result_get_image_count(result)) { + sd::api::error(result ? SD_STATUS_OUT_OF_RANGE : SD_STATUS_INVALID_ARGUMENT, nullptr, "invalid result or image index"); + return nullptr; + } + return result->images[index].get(); +} + +const char* sd_result_get_generation_info(const sd_result_t* result, size_t index) { + if (!sd_result_get_image(result, index)) + return nullptr; + return result->generation_infos[result->kind == SD_RESULT_VIDEO ? 0 : index].c_str(); +} + +const sd_audio_t* sd_result_get_audio(const sd_result_t* result) { + if (!sd_result_get_kind(result)) + return nullptr; + return result->audio.data ? &result->audio : nullptr; +} + +int sd_result_get_fps(const sd_result_t* result) { + if (!sd_result_get_kind(result)) + return 0; + return result->fps; +} + +const sd_video_t* sd_result_get_video(const sd_result_t* result) { + const sd_video_t* output = nullptr; + sd::api::call([&] { + if (!result) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "result is required"); + if (result->kind != SD_RESULT_VIDEO) + return sd::api::error(SD_STATUS_TYPE_MISMATCH, nullptr, "result is not a video"); + std::lock_guard lock(result->video_mutex); + if (!result->video_tensor) { + if (result->images.empty()) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "video has no frames"); + const auto& first = *result->images.front().get(); + auto shape = first.shape; + shape.push_back(static_cast(result->images.size())); + for (const auto& frame : result->images) + if (frame->type != first.type || frame->shape != first.shape) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "video frames must have identical shapes and types"); + auto packed = sd::api::allocate_tensor(first.type, std::move(shape)); + auto* destination = static_cast(packed.get()->mutable_data()); + for (const auto& frame : result->images) { + memcpy(destination, frame->data(), frame->bytes); + destination += frame->bytes; + } + result->video_tensor = std::move(packed); + result->video = {result->video_tensor.get(), result->fps, result->audio}; + } + output = &result->video; + return SD_STATUS_OK; + }); + return output; +} diff --git a/src/api/result.h b/src/api/result.h new file mode 100644 index 0000000000..452055b478 --- /dev/null +++ b/src/api/result.h @@ -0,0 +1,57 @@ +#ifndef SD_RESULT_H +#define SD_RESULT_H + +#include +#include +#include +#include +#include "tensor.h" + +#include "json.hpp" +#include "stable-diffusion.h" + +namespace sd { + struct ImageResult { + sd::Image image; + // Owned UTF-8 JSON using the sdcpp.generation/v1 schema. + char* generation_info; + }; + + struct ImageGenerationResult { + sd::ImageResult* items; + int count; + }; + + struct VideoGenerationResult { + sd::Image* frames; + int frame_count; + sd::Audio* audio; + int fps; + char* generation_info; + }; + + void free_image_generation_result(ImageGenerationResult* result); + void free_video_generation_result(VideoGenerationResult* result); + void free_images(sd::Image* images, int count); + + char* copy_generation_info(const nlohmann::json& info); + // Consumes images even on failure. + bool make_image_generation_result(sd::Image* images, int count, const std::vector& infos, sd::ImageGenerationResult** result_out); + bool make_image_result(ImageGenerationResult* images, sd_result_t** result_out); + bool make_image_result(sd::Image* images, int count, const std::vector& infos, sd_result_t** result_out); + bool make_video_result(VideoGenerationResult* video, sd_result_t** result_out); +} + +struct sd_result { + sd_result_kind_t kind = SD_RESULT_INVALID; + std::vector images; + std::vector generation_infos; + sd::api::TensorRef audio_tensor; + sd_audio_t audio{}; + int fps = 0; + mutable std::mutex video_mutex; + mutable sd::api::TensorRef video_tensor; + mutable sd_video_t video{}; +}; + +#endif // SD_RESULT_H diff --git a/src/api/tensor.cpp b/src/api/tensor.cpp new file mode 100644 index 0000000000..756753560f --- /dev/null +++ b/src/api/tensor.cpp @@ -0,0 +1,191 @@ +#include "tensor.h" +#include "api.h" + +#include +#include +#include +#include +#include + +static size_t element_size(sd_tensor_type_t type) { + switch (type) { + case SD_TENSOR_U8: + return sizeof(uint8_t); + case SD_TENSOR_I32: + return sizeof(int32_t); + case SD_TENSOR_I64: + return sizeof(int64_t); + case SD_TENSOR_F32: + return sizeof(float); + case SD_TENSOR_F64: + return sizeof(double); + default: + throw std::invalid_argument("invalid tensor type"); + } +} + +size_t sd::api::tensor_byte_size(sd_tensor_type_t type, const std::vector& shape) { + if (shape.size() > 8) + throw std::invalid_argument("tensor rank must not exceed 8"); + size_t count = 1; + for (int64_t dim : shape) { + if (dim < 0 || (count && uint64_t(dim) > SIZE_MAX / count)) + throw std::invalid_argument("negative or overflowing tensor dimensions"); + count *= static_cast(dim); + } + const size_t size = element_size(type); + if (count > uint64_t(INT64_MAX) || count > SIZE_MAX / size) + throw std::invalid_argument("tensor byte size overflow"); + return count * size; +} + +static sd_tensor_t::Storage allocate_storage(sd_tensor_type_t type, std::vector shape, size_t bytes) { + if (bytes && shape.empty()) + shape.push_back(1); + if (!bytes) + shape.clear(); + switch (type) { + case SD_TENSOR_U8: + return sd::Tensor(std::move(shape)); + case SD_TENSOR_I32: + return sd::Tensor(std::move(shape)); + case SD_TENSOR_I64: + return sd::Tensor(std::move(shape)); + case SD_TENSOR_F32: + return sd::Tensor(std::move(shape)); + case SD_TENSOR_F64: + return sd::Tensor(std::move(shape)); + default: + throw std::invalid_argument("invalid tensor type"); + } +} + +sd_tensor_t::sd_tensor_t(sd_tensor_type_t type, std::vector shape) + : type(type), shape(std::move(shape)), bytes(sd::api::tensor_byte_size(type, this->shape)), storage_(allocate_storage(type, this->shape, bytes)) {} + +static std::vector validated_shape(sd_tensor_type_t type, std::vector shape, const void* data, size_t size) { + if (size != sd::api::tensor_byte_size(type, shape) || (size && !data)) + throw std::invalid_argument("tensor data size does not match type and shape"); + return shape; +} + +sd_tensor_t::sd_tensor_t(sd_tensor_type_t type, std::vector shape, const void* data, size_t size) + : sd_tensor_t(type, validated_shape(type, std::move(shape), data, size)) { + if (bytes) + memcpy(mutable_data(), data, bytes); +} + +sd::api::TensorRef sd::api::allocate_tensor(sd_tensor_type_t type, std::vector shape) { + return TensorRef::adopt(new sd_tensor_t(type, std::move(shape))); +} + +sd::api::TensorRef sd::api::own_tensor(sd_tensor_type_t type, std::vector shape, void* data) { + std::unique_ptr owner(data, std::free); + const size_t bytes = tensor_byte_size(type, shape); + return TensorRef::adopt(new sd_tensor_t(type, std::move(shape), data, bytes)); +} + +sd::Image sd::api::image_view(const sd_tensor_t& tensor) { + const auto& s = tensor.shape; + if (tensor.type != SD_TENSOR_U8 || s.size() != 3 || s[0] < 1 || s[0] > 4 || + s[1] <= 0 || s[2] <= 0 || s[1] > INT32_MAX || s[2] > INT32_MAX || !tensor.data()) + throw std::invalid_argument("image requires U8 [channels,width,height], channels in [1,4], and positive dimensions"); + return {uint32_t(s[1]), uint32_t(s[2]), uint32_t(s[0]), const_cast(static_cast(tensor.data()))}; +} + +sd::Audio sd::api::audio_view(const sd_audio_t& audio) { + if (!audio.data || audio.sample_rate == 0) + throw std::invalid_argument("audio requires samples and a positive sample rate"); + const auto& t = *audio.data; + if (t.type != SD_TENSOR_F32 || t.shape.size() != 2 || t.shape[0] <= 0 || t.shape[0] > UINT32_MAX || t.shape[1] <= 0 || !t.data()) + throw std::invalid_argument("audio requires F32 [channels,samples] with positive dimensions"); + return {audio.sample_rate, uint32_t(t.shape[0]), uint64_t(t.shape[1]), const_cast(static_cast(t.data()))}; +} + +std::vector sd::api::video_frames(const sd_tensor_t& tensor) { + const auto& s = tensor.shape; + if (tensor.type != SD_TENSOR_U8 || s.size() != 4 || s[0] < 1 || s[0] > 4 || s[1] <= 0 || s[2] <= 0 || s[3] <= 0 || + s[1] > INT32_MAX || s[2] > INT32_MAX || s[3] > INT32_MAX || !tensor.data()) + throw std::invalid_argument("video requires U8 [channels,width,height,frames] with positive dimensions"); + const size_t stride = size_t(s[0]) * size_t(s[1]) * size_t(s[2]); + auto* data = const_cast(static_cast(tensor.data())); + std::vector result; + result.reserve(size_t(s[3])); + for (int64_t i = 0; i < s[3]; ++i) + result.push_back({uint32_t(s[1]), uint32_t(s[2]), uint32_t(s[0]), data + size_t(i) * stride}); + return result; +} + +void sd::api::preview_images(const std::function& callback, int step, int count, const sd::Image* images, bool noisy, void* data) { + std::vector tensors; + std::vector views; + tensors.reserve(count); + views.reserve(count); + for (int i = 0; i < count; ++i) { + const auto& image = images[i]; + const int64_t shape[] = {image.channel, image.width, image.height}; + auto* tensor = sd_tensor_create(SD_TENSOR_U8, 3, shape, image.data, size_t(image.channel) * image.width * image.height); + if (!tensor) + throw std::bad_alloc(); + tensors.push_back(TensorRef::adopt(tensor)); + views.push_back(tensors.back().get()); + } + callback(step, count, views.data(), noisy, data); +} + +sd_tensor_t* sd_tensor_create(sd_tensor_type_t type, size_t rank, const int64_t* shape, const void* data, size_t data_size) { + sd_tensor_t* result = nullptr; + sd::api::call([&] { + if (rank > 8 || (rank && !shape)) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "invalid tensor shape"); + std::vector dimensions; + if (rank) + dimensions.assign(shape, shape + rank); + const size_t bytes = sd::api::tensor_byte_size(type, dimensions); + if (data_size != bytes || (bytes && !data)) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "tensor data size does not match type and shape"); + result = new sd_tensor_t(type, std::move(dimensions), data, data_size); + return SD_STATUS_OK; + }); + return result; +} + +sd_tensor_t* sd_tensor_retain(const sd_tensor_t* tensor) { + if (tensor) + tensor->references.fetch_add(1, std::memory_order_relaxed); + return const_cast(tensor); +} + +void sd_tensor_free(sd_tensor_t* tensor) { + if (tensor && tensor->references.fetch_sub(1, std::memory_order_acq_rel) == 1) + delete tensor; +} + +static bool valid_tensor(const sd_tensor_t* tensor) { + sd::api::clear_error(); + if (!tensor) + sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "tensor is required"); + return tensor != nullptr; +} + +sd_tensor_type_t sd_tensor_type(const sd_tensor_t* tensor) { + return valid_tensor(tensor) ? tensor->type : 0; +} +size_t sd_tensor_rank(const sd_tensor_t* tensor) { + return valid_tensor(tensor) ? tensor->shape.size() : 0; +} +size_t sd_tensor_byte_size(const sd_tensor_t* tensor) { + return valid_tensor(tensor) ? tensor->bytes : 0; +} +const void* sd_tensor_data(const sd_tensor_t* tensor) { + return valid_tensor(tensor) ? tensor->data() : nullptr; +} +int64_t sd_tensor_dim(const sd_tensor_t* tensor, size_t axis) { + if (!valid_tensor(tensor)) + return 0; + if (axis >= tensor->shape.size()) { + sd::api::error(SD_STATUS_OUT_OF_RANGE, nullptr, "invalid tensor axis"); + return 0; + } + return tensor->shape[axis]; +} diff --git a/src/api/tensor.h b/src/api/tensor.h new file mode 100644 index 0000000000..9d5728708a --- /dev/null +++ b/src/api/tensor.h @@ -0,0 +1,93 @@ +#ifndef __SD_API_TENSOR_H__ +#define __SD_API_TENSOR_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "core/tensor.hpp" + +#include "core/media.h" +#include "stable-diffusion.h" + +namespace sd::api { + size_t tensor_byte_size(sd_tensor_type_t type, const std::vector& shape); +} + +struct sd_tensor_t { + using Storage = std::variant, sd::Tensor, sd::Tensor, sd::Tensor, sd::Tensor>; + mutable std::atomic references{1}; + sd_tensor_type_t type; + std::vector shape; + size_t bytes; + +private: + Storage storage_; + +public: + sd_tensor_t(sd_tensor_type_t type, std::vector shape); + sd_tensor_t(sd_tensor_type_t type, std::vector shape, const void* data, size_t size); + + template + explicit sd_tensor_t(sd::Tensor tensor) + : type(std::is_same_v ? SD_TENSOR_U8 : std::is_same_v ? SD_TENSOR_I32 + : std::is_same_v ? SD_TENSOR_I64 + : std::is_same_v ? SD_TENSOR_F32 + : SD_TENSOR_F64), + shape(tensor.shape()), + bytes(sd::api::tensor_byte_size(type, shape)), + storage_(std::move(tensor)) { + static_assert(std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v, "unsupported tensor type"); + if (std::get>(storage_).numel() != static_cast(bytes / sizeof(T))) + throw std::invalid_argument("tensor shape and storage size do not match"); + } + + const void* data() const { + return bytes ? std::visit([](const auto& tensor) -> const void* { return tensor.data(); }, storage_) : nullptr; + } + void* mutable_data() { + return bytes ? std::visit([](auto& tensor) -> void* { return tensor.data(); }, storage_) : nullptr; + } +}; + +namespace sd::api { + class TensorRef { + sd_tensor_t* tensor_ = nullptr; + + public: + TensorRef() = default; + explicit TensorRef(const sd_tensor_t* tensor) + : tensor_(sd_tensor_retain(tensor)) {} + static TensorRef adopt(sd_tensor_t* tensor) { + TensorRef result; + result.tensor_ = tensor; + return result; + } + TensorRef(const TensorRef& other) + : TensorRef(other.tensor_) {} + TensorRef(TensorRef&& other) noexcept + : tensor_(other.tensor_) { other.tensor_ = nullptr; } + TensorRef& operator=(TensorRef other) noexcept { + std::swap(tensor_, other.tensor_); + return *this; + } + ~TensorRef() { sd_tensor_free(tensor_); } + sd_tensor_t* get() const { return tensor_; } + const sd_tensor_t* operator->() const { return tensor_; } + explicit operator bool() const { return tensor_ != nullptr; } + }; + + size_t tensor_byte_size(sd_tensor_type_t type, const std::vector& shape); + TensorRef own_tensor(sd_tensor_type_t type, std::vector shape, void* data); + TensorRef allocate_tensor(sd_tensor_type_t type, std::vector shape); + sd::Image image_view(const sd_tensor_t& tensor); + sd::Audio audio_view(const sd_audio_t& audio); + std::vector video_frames(const sd_tensor_t& tensor); + void preview_images(const std::function& callback, int step, int count, const sd::Image* images, bool noisy, void* data); +} + +#endif // __SD_API_TENSOR_H__ diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp index 12f4dc9a06..808375c4e6 100644 --- a/src/conditioning/conditioner.hpp +++ b/src/conditioning/conditioner.hpp @@ -4,10 +4,12 @@ #include #include #include +#include #include #include #include #include +#include "conditioning/prompt_input.h" #include "core/ggml_tensor_utils.h" #include "core/tensor_ggml.hpp" @@ -132,6 +134,7 @@ static inline sd::Tensor apply_token_weights(sd::Tensor hidden_sta struct ConditionerParams { std::string text; + const sd::PromptInput* prompt_input = nullptr; int clip_skip = -1; int width = -1; int height = -1; @@ -143,6 +146,7 @@ struct ConditionerParams { struct Conditioner { virtual int resolve_clip_skip(int) const { return 0; } + virtual bool supports_prompt_images() const { return false; } virtual ~Conditioner() = default; public: @@ -1979,6 +1983,10 @@ struct LLMEmbedder : public Conditioner { std::shared_ptr byt5; std::shared_ptr ph3_connector; + bool supports_prompt_images() const override { + return llm->supports_prompt_images(); + } + LLMEmbedder(ggml_backend_t backend, const String2TensorStorage& tensor_storage_map = {}, SDVersion version = VERSION_QWEN_IMAGE, @@ -1989,13 +1997,13 @@ struct LLMEmbedder : public Conditioner { : version(version) { if (!tokenizers.has(TokenizerConfig::MAIN)) { if (sd_version_is_lens(version)) { - throw std::runtime_error("Lens requires an external GPT-OSS tokenizer.json; pass --tokenizer FILE or set sd_ctx_params_t::tokenizer"); + throw std::runtime_error("Lens requires an external GPT-OSS tokenizer.json; pass --tokenizer FILE or set the tokenizer context option"); } if (sd_version_is_pid(version)) { - throw std::runtime_error("PiD requires an external Gemma 2 tokenizer.json; pass --tokenizer FILE or set sd_ctx_params_t::tokenizer"); + throw std::runtime_error("PiD requires an external Gemma 2 tokenizer.json; pass --tokenizer FILE or set the tokenizer context option"); } if (version == VERSION_MOSS_IMAGE) { - throw std::runtime_error("MOSS-Image requires the official Qwen3.5 tokenizer.json; pass --tokenizer FILE or set sd_ctx_params_t::tokenizer"); + throw std::runtime_error("MOSS-Image requires the official Qwen3.5 tokenizer.json; pass --tokenizer FILE or set the tokenizer context option"); } } LLM::LLMArch arch = LLM::LLMArch::QWEN2_5_VL; @@ -2024,17 +2032,37 @@ struct LLMEmbedder : public Conditioner { } else if (version == VERSION_MOSS_IMAGE) { arch = LLM::LLMArch::QWEN3_5; } + if (arch == LLM::LLMArch::QWEN2_5_VL || arch == LLM::LLMArch::QWEN3_VL || arch == LLM::LLMArch::QWEN3_5) { + for (const auto& entry : tensor_storage_map) { + if (starts_with(entry.first, "text_encoders.llm.visual.")) { + enable_vision = true; + break; + } + } + } llm = std::make_shared(arch, backend, tensor_storage_map, "text_encoders.llm", enable_vision, weight_manager); - if (version == VERSION_P_H3_IMAGE) { - if (llm->config.num_layers != 36 || llm->config.hidden_size != 2560) { - throw std::runtime_error("P-H3-Image requires the Qwen3-VL-4B text encoder"); + if (sd_version_is_minimax_h3(version)) { + const bool has_ph3_connector = tensor_storage_map.count(std::string(PH3ImageTextConnectorRunner::prefix) + ".proj.weight") != 0; + if (has_ph3_connector) { + if (llm->config.num_layers != 36 || llm->config.hidden_size != 2560) { + throw std::runtime_error("The P-H3-Image text connector requires the Qwen3-VL-4B text encoder"); + } + ph3_connector = std::make_shared(backend, tensor_storage_map, weight_manager); + } else if (llm->config.hidden_size == 2560) { + throw std::runtime_error( + sd_format("%s with Qwen3-VL-4B requires the P-H3-Image text connector; " + "pass it with --embeddings-connectors or use the MiniMax-H3 Qwen3-VL-32B text encoder", + sd_version_name(version))); + } else if (llm->config.hidden_size != 5120 || (llm->config.num_layers != 50 && llm->config.num_layers != 64)) { + throw std::runtime_error( + sd_format("%s requires the MiniMax-H3 Qwen3-VL-32B text encoder or Qwen3-VL-4B with the P-H3-Image text connector", + sd_version_name(version))); } - ph3_connector = std::make_shared(backend, tensor_storage_map, weight_manager); } int pad_id = 151643; if (arch == LLM::LLMArch::MISTRAL_SMALL_3_2 || arch == LLM::LLMArch::MINISTRAL_3_3B) { @@ -2172,13 +2200,9 @@ struct LLMEmbedder : public Conditioner { } } - std::tuple, std::vector, std::vector> tokenize(std::string text, - const std::pair& attn_range, - size_t min_length = 0, - size_t max_length = 100000000, - bool spell_quotes = false, - const std::string& suffix = "", - const std::vector>>& image_embeds = {}) { + static std::vector> parse_text_attention(const std::string& text, + const std::pair& attn_range, + bool spell_quotes) { std::vector> parsed_attention; if (attn_range.first >= 0 && attn_range.second > 0) { if (attn_range.first > 0) { @@ -2200,6 +2224,18 @@ struct LLMEmbedder : public Conditioner { parsed_attention.emplace_back(text, 1.f); } + return parsed_attention; + } + + std::tuple, std::vector, std::vector> tokenize(std::string text, + const std::pair& attn_range, + size_t min_length = 0, + size_t max_length = 100000000, + bool spell_quotes = false, + const std::string& suffix = "", + const std::vector>>& image_embeds = {}) { + auto parsed_attention = parse_text_attention(text, attn_range, spell_quotes); + { std::stringstream ss; ss << "["; @@ -2275,6 +2311,57 @@ struct LLMEmbedder : public Conditioner { return {tokens, weights, mask}; } + bool prepare_prompt_tokens(int n_threads, + const sd::PromptInputTemplate& input_template, + const std::string& prompt, + const std::pair& attn_range, + int min_length, + int max_length, + bool spell_quotes, + const std::string& suffix, + const LLM::ImageEmbeds& images, + const LLM::DeepStackImageEmbeds& deepstack, + const std::vector& grids, + LLM::PromptTokens& result) { + sd::PromptInput input; + auto text = parse_text_attention(prompt, attn_range, spell_quotes); + text.emplace_back(suffix, 1.f); + for (const auto& span : text) { + auto parts = input_template.expand(span.first, span.second); + input.insert(input.end(), std::make_move_iterator(parts.begin()), std::make_move_iterator(parts.end())); + } + max_length = std::min(max_length, static_cast(llm->config.max_position_embeddings)); + if (!llm->prepare_prompt(n_threads, *tokenizer, input, max_length, result, images, deepstack, grids)) { + return false; + } + auto expanded = result.tokens; + for (const auto& image : result.image_embeds) { + expanded.insert(expanded.begin() + image.first, image.second.shape()[1] - 1, expanded[image.first]); + } + if (sd_version_is_flux3(version)) { + min_length = std::min(max_length, ((result.expanded_length + 79) / 80) * 80); + } + tokenizer->pad_tokens(expanded, &result.weights, &result.mask, min_length, max_length); + const int leading_padding = static_cast(std::find(result.mask.begin(), result.mask.end(), 1.f) - result.mask.begin()); + for (auto& image : result.image_embeds) { + image.first += leading_padding; + } + for (auto& layer : result.deepstack_image_embeds) { + for (auto& image : layer) { + image.first += leading_padding; + } + } + for (auto& grid : result.image_grids) { + grid.index += leading_padding; + } + for (auto it = result.image_embeds.rbegin(); it != result.image_embeds.rend(); ++it) { + expanded.erase(expanded.begin() + it->first + 1, expanded.begin() + it->first + it->second.shape()[1]); + } + result.tokens = std::move(expanded); + result.expanded_length = static_cast(result.weights.size()); + return true; + } + sd::Tensor encode_prompt(int n_threads, const std::string prompt, const std::pair& prompt_attn_range, @@ -2288,8 +2375,18 @@ struct LLMEmbedder : public Conditioner { const LLM::DeepStackImageEmbeds& deepstack_image_embeds = {}, const std::vector& image_grids = {}, const std::string& prompt_suffix = "", - sd::Tensor* output_mask = nullptr) { - auto tokens_weights_mask = tokenize(prompt, prompt_attn_range, min_length, max_length, spell_quotes, prompt_suffix, image_embeds); + sd::Tensor* output_mask = nullptr, + const sd::PromptInputTemplate* input_template = nullptr, + LLM::PromptTokens* output_prompt_tokens = nullptr) { + LLM::PromptTokens prepared; + if (input_template != nullptr && + !prepare_prompt_tokens(n_threads, *input_template, prompt, prompt_attn_range, min_length, max_length, + spell_quotes, prompt_suffix, image_embeds, deepstack_image_embeds, image_grids, prepared)) { + return {}; + } + auto tokens_weights_mask = input_template == nullptr + ? tokenize(prompt, prompt_attn_range, min_length, max_length, spell_quotes, prompt_suffix, image_embeds) + : std::make_tuple(prepared.tokens, prepared.weights, prepared.mask); auto& tokens = std::get<0>(tokens_weights_mask); auto& weights = std::get<1>(tokens_weights_mask); auto& mask = std::get<2>(tokens_weights_mask); @@ -2320,21 +2417,24 @@ struct LLMEmbedder : public Conditioner { sd::Tensor position_ids; if (sd_version_is_flux3(version)) { position_ids = sd::Tensor::zeros({static_cast(weights.size() * 4)}); + if (input_template != nullptr) { + position_ids.values() = LLM::LLMRunner::build_position_ids(llm->config, weights.size(), prepared.image_grids); + } for (size_t axis = 0; axis < 4; ++axis) { for (size_t i = 0; i < weights.size(); ++i) { - position_ids[axis * weights.size() + i] = mask[i] == 0.f ? 1 : static_cast(i); + position_ids[axis * weights.size() + i] = mask[i] == 0.f ? 1 : (input_template == nullptr ? static_cast(i) : position_ids[axis * weights.size() + i]); } } } auto hidden_states = llm->compute(n_threads, input_ids, attention_mask, - image_embeds, + input_template == nullptr ? image_embeds : prepared.image_embeds, out_layers, false, false, - deepstack_image_embeds, - image_grids, + input_template == nullptr ? deepstack_image_embeds : prepared.deepstack_image_embeds, + input_template == nullptr ? image_grids : prepared.image_grids, position_ids); if (hidden_states.empty()) { LOG_ERROR("LLM prompt encoding failed"); @@ -2374,6 +2474,9 @@ struct LLMEmbedder : public Conditioner { output_mask->data()[i - prompt_template_encode_start_idx] = mask[i]; } } + if (output_prompt_tokens != nullptr && input_template != nullptr) { + *output_prompt_tokens = std::move(prepared); + } return new_hidden_states; } @@ -2470,7 +2573,16 @@ struct LLMEmbedder : public Conditioner { } SDCondition get_learned_condition(int n_threads, - const ConditionerParams& conditioner_params) override { + const ConditionerParams& params) override { + ConditionerParams conditioner_params = params; + sd::PromptInputTemplate input_template(params.prompt_input); + if (params.prompt_input != nullptr) { + if (!supports_prompt_images()) { + LOG_ERROR("This conditioner does not support inline prompt images"); + return {}; + } + conditioner_params.text = input_template.text; + } std::string prompt; std::pair prompt_attn_range; std::vector extra_prompts; @@ -2493,7 +2605,7 @@ struct LLMEmbedder : public Conditioner { if (sd_version_is_minimax_h3(version)) { prompt_template_encode_start_idx = 0; // The final tap is normalized; LLMRunner exposes it at num_layers + 1. - out_layers = version == VERSION_P_H3_IMAGE ? std::set{13, 19, 25, 31, 37} : std::set{50}; + out_layers = ph3_connector ? std::set{13, 19, 25, 31, 37} : std::set{50}; prompt_attn_range = {0, 0}; if (llm->enable_vision) { @@ -3258,6 +3370,7 @@ struct LLMEmbedder : public Conditioner { GGML_ABORT("unknown version %d", version); } + LLM::PromptTokens prompt_tokens; auto hidden_states = encode_prompt(n_threads, prompt, prompt_attn_range, @@ -3271,7 +3384,9 @@ struct LLMEmbedder : public Conditioner { deepstack_image_embeds, image_grids, prompt_suffix, - version == VERSION_IRIS ? &text_mask : nullptr); + version == VERSION_IRIS ? &text_mask : nullptr, + params.prompt_input == nullptr ? nullptr : &input_template, + &prompt_tokens); if (hidden_states.empty()) { return {}; } @@ -3338,10 +3453,11 @@ struct LLMEmbedder : public Conditioner { result.extra_c_crossattns = std::move(extra_hidden_states_vec); result.c_vector = std::move(text_mask); if (version == VERSION_QWEN_IMAGE_2_1) { - auto slots = sd::Tensor::zeros({result.c_crossattn.shape()[1]}); + auto slots = sd::Tensor::zeros({result.c_crossattn.shape()[1]}); + const auto& references = params.prompt_input == nullptr ? image_embeds : prompt_tokens.image_embeds; for (size_t i = 0; i < image_embeds.size(); ++i) { - int64_t begin = image_embeds[i].first - prompt_template_encode_start_idx; - int64_t end = begin + image_embeds[i].second.shape()[1]; + int64_t begin = references[i].first - prompt_template_encode_start_idx; + int64_t end = begin + references[i].second.shape()[1]; if (begin < 0 || end > slots.numel()) { LOG_ERROR("Qwen Image 2.1 image slots exceed the encoded prompt"); return {}; @@ -3354,7 +3470,8 @@ struct LLMEmbedder : public Conditioner { } if (sd_version_is_minimax_h3(version)) { std::vector tags(static_cast(result.c_crossattn.shape()[1]), 1); - for (const auto& [index, image_embed] : image_embeds) { + const auto& images = params.prompt_input == nullptr ? image_embeds : prompt_tokens.image_embeds; + for (const auto& [index, image_embed] : images) { int64_t begin = std::max(0, index - 1); int64_t end = std::min(static_cast(tags.size()), index + image_embed.shape()[1] + 1); @@ -3526,7 +3643,7 @@ struct LLaDAImageEmbedder : public Conditioner { text_projection_prefix(text_projection_prefix), sigvq_prefix(sigvq_prefix) { if (!tokenizers.has(TokenizerConfig::MAIN)) { - throw std::runtime_error("LLaDA-Image requires an external LLaDA2 tokenizer.json; pass --tokenizer FILE or set sd_ctx_params_t::tokenizer"); + throw std::runtime_error("LLaDA-Image requires an external LLaDA2 tokenizer.json; pass --tokenizer FILE or set the tokenizer context option"); } llm = std::make_shared(LLM::LLMArch::LLADA2_MOE, backend, diff --git a/src/conditioning/conditioning_cache.h b/src/conditioning/conditioning_cache.h index 34648c9ad1..58f539a388 100644 --- a/src/conditioning/conditioning_cache.h +++ b/src/conditioning/conditioning_cache.h @@ -11,11 +11,16 @@ class ConditioningCache { struct Entry { ConditionerParams params; std::vector> ref_images; + sd::PromptInput prompt_input; std::vector references; SDCondition condition; Entry(const ConditionerParams& input, const SDCondition& output) : params(input), condition(output) { + if (input.prompt_input != nullptr) { + prompt_input = *input.prompt_input; + params.prompt_input = &prompt_input; + } // Request-owned reference pointers must not outlive the request. if (input.ref_images != nullptr) { ref_images = *input.ref_images; @@ -52,10 +57,21 @@ class ConditioningCache { r.ref_image_size); }; if (fields(a) != fields(b) || + (a.prompt_input == nullptr) != (b.prompt_input == nullptr) || (a.ref_images == nullptr) != (b.ref_images == nullptr) || (a.minimax_h3_references == nullptr) != (b.minimax_h3_references == nullptr)) { return false; } + if (a.prompt_input != nullptr && + !std::equal(a.prompt_input->begin(), a.prompt_input->end(), + b.prompt_input->begin(), b.prompt_input->end(), + [](const sd::PromptPart& x, const sd::PromptPart& y) { + return x.type == y.type && x.content == y.content && x.weight == y.weight && + (x.image == y.image || + (x.image && y.image && x.image->shape() == y.image->shape() && x.image->values() == y.image->values())); + })) { + return false; + } if (a.ref_images != nullptr && !same_images(*a.ref_images, *b.ref_images)) { return false; } diff --git a/src/conditioning/fibo2.hpp b/src/conditioning/fibo2.hpp index 43b2ee0845..db124ea3a7 100644 --- a/src/conditioning/fibo2.hpp +++ b/src/conditioning/fibo2.hpp @@ -132,7 +132,12 @@ struct Fibo2Conditioner : public LLMEmbedder { return result; } - SDCondition get_learned_condition(int n_threads, const ConditionerParams& p) override { + SDCondition get_learned_condition(int n_threads, const ConditionerParams& params) override { + auto p = params; + sd::PromptInputTemplate input_template(p.prompt_input); + if (p.prompt_input != nullptr) { + p.text = input_template.text; + } int image_count = p.ref_images ? static_cast(p.ref_images->size()) : 0; if (image_count && !llm->enable_vision) { LOG_ERROR("Fibo 2.0 editing requires vision weights"); @@ -169,7 +174,8 @@ struct Fibo2Conditioner : public LLMEmbedder { } } const std::set layers = {5, 7, 9, 11, 13, 16, 18, 20, 22, 24, 27, 29, 31, 33, 35}; - auto states = encode_prompt(n_threads, prompt, {0, 0}, 0, 0, images, layers, 0, false, 4096, deepstack, grids); + auto states = encode_prompt(n_threads, prompt, {0, 0}, 0, 0, images, layers, 0, false, 4096, deepstack, grids, + "", nullptr, p.prompt_input == nullptr ? nullptr : &input_template); if (states.empty()) { return {}; } diff --git a/src/conditioning/prompt_input.cpp b/src/conditioning/prompt_input.cpp new file mode 100644 index 0000000000..0748e9c1e7 --- /dev/null +++ b/src/conditioning/prompt_input.cpp @@ -0,0 +1,155 @@ +#include "conditioning/prompt_input.h" + +#include +#include +#include + +#include "core/util.h" + +namespace sd { + + PromptInputTemplate::PromptInputTemplate(const PromptInput* input) + : input_(input) { + if (input == nullptr) { + return; + } + uint32_t codepoint = 0xf0000; + for (size_t i = 0; i < input->size(); ++i) { + const auto& part = (*input)[i]; + if (part.type == PromptPart::Type::TEXT) { + text += part.content; + continue; + } + // One private-use code point survives JSON formatting and quote spelling. + std::string marker; + do { + if (codepoint > 0xffffd) { + throw std::invalid_argument("Too many prompt image markers"); + } + marker = {static_cast(0xf0 | (codepoint >> 18)), + static_cast(0x80 | ((codepoint >> 12) & 0x3f)), + static_cast(0x80 | ((codepoint >> 6) & 0x3f)), + static_cast(0x80 | (codepoint & 0x3f))}; + ++codepoint; + } while (std::any_of(input->begin(), input->end(), [&](const PromptPart& source) { + return source.content.find(marker) != std::string::npos; + })); + image_markers_.emplace(marker, i); + text += marker; + } + } + + PromptInput PromptInputTemplate::expand(const std::string& formatted_text, float weight) const { + PromptInput result; + size_t cursor = 0; + size_t search = 0; + while (input_ != nullptr && search < formatted_text.size()) { + const size_t begin = formatted_text.find(static_cast(0xf3), search); + if (begin == std::string::npos) { + break; + } + const auto marker = image_markers_.find(formatted_text.substr(begin, 4)); + search = begin + 1; + if (marker == image_markers_.end()) { + continue; + } + if (begin > cursor) { + result.push_back({PromptPart::Type::TEXT, formatted_text.substr(cursor, begin - cursor), {}, weight}); + } + result.push_back((*input_)[marker->second]); + cursor = begin + marker->first.size(); + search = cursor; + } + if (cursor < formatted_text.size()) { + result.push_back({PromptPart::Type::TEXT, formatted_text.substr(cursor), {}, weight}); + } + return result; + } + + PromptInput parse_prompt_input(const std::string& text) { + PromptInput parts; + std::string plain_text; + size_t cursor = 0; + while (cursor < text.size()) { + const size_t begin = text.find(" slashes && text[begin - slashes - 1] == '\\') { + ++slashes; + } + if (slashes % 2 != 0) { + plain_text += text.substr(cursor, begin - cursor - 1); + plain_text += "', begin + 7); + if (end == std::string::npos || end == begin + 7) { + throw std::invalid_argument("Invalid prompt marker at byte " + std::to_string(begin)); + } + plain_text += text.substr(cursor, begin - cursor); + if (!plain_text.empty()) { + parts.push_back({PromptPart::Type::TEXT, std::move(plain_text), {}}); + plain_text.clear(); + } + parts.push_back({PromptPart::Type::IMAGE, text.substr(begin + 7, end - begin - 7), {}}); + cursor = end + 1; + } + if (!plain_text.empty() || parts.empty()) { + parts.push_back({PromptPart::Type::TEXT, std::move(plain_text), {}}); + } + return parts; + } + + bool prompt_has_images(const PromptInput& input) { + return std::any_of(input.begin(), input.end(), [](const PromptPart& part) { + return part.type == PromptPart::Type::IMAGE; + }); + } + + std::string prompt_input_text(const PromptInput& input) { + std::string text; + for (const auto& part : input) { + if (part.type == PromptPart::Type::TEXT) { + text += part.content; + } + } + return text; + } + + bool resolve_prompt_images(PromptInput& input, const PromptImageLoader& loader) { + constexpr size_t max_image_sources = 32; + std::map>> images; + for (const auto& part : input) { + if (part.type == PromptPart::Type::IMAGE) { + images.try_emplace(part.content); + if (images.size() > max_image_sources) { + LOG_ERROR("A prompt supports at most %zu unique image sources", max_image_sources); + return false; + } + } + } + for (auto& part : input) { + if (part.type != PromptPart::Type::IMAGE) { + continue; + } + if (!loader) { + return false; + } + auto& resolved = images.at(part.content); + if (!resolved) { + auto image = loader(part.content); + if (image.empty()) { + return false; + } + resolved = std::make_shared>(std::move(image)); + } + part.image = resolved; + } + return true; + } + +} // namespace sd diff --git a/src/conditioning/prompt_input.h b/src/conditioning/prompt_input.h new file mode 100644 index 0000000000..ed5fa2d7f9 --- /dev/null +++ b/src/conditioning/prompt_input.h @@ -0,0 +1,45 @@ +#ifndef __SD_CONDITIONING_PROMPT_INPUT_H__ +#define __SD_CONDITIONING_PROMPT_INPUT_H__ + +#include +#include +#include +#include +#include + +#include "core/tensor.hpp" + +namespace sd { + + struct PromptPart { + enum class Type { TEXT, + IMAGE }; + + Type type = Type::TEXT; + std::string content; + std::shared_ptr> image; + float weight = 1.f; + }; + + using PromptInput = std::vector; + using PromptImageLoader = std::function(const std::string&)>; + + class PromptInputTemplate { + const PromptInput* input_; + std::map image_markers_; + + public: + std::string text; + + explicit PromptInputTemplate(const PromptInput* input); + PromptInput expand(const std::string& formatted_text, float weight = 1.f) const; + }; + + PromptInput parse_prompt_input(const std::string& text); + bool prompt_has_images(const PromptInput& input); + std::string prompt_input_text(const PromptInput& input); + bool resolve_prompt_images(PromptInput& input, const PromptImageLoader& loader); + +} // namespace sd + +#endif // __SD_CONDITIONING_PROMPT_INPUT_H__ diff --git a/src/config.cpp b/src/config.cpp new file mode 100644 index 0000000000..5cffb854eb --- /dev/null +++ b/src/config.cpp @@ -0,0 +1,133 @@ +#include "config.h" + +#include + +namespace sd { + + SampleConfig::SampleConfig() { + guidance.txt_cfg = 7.0f; + guidance.img_cfg = INFINITY; + guidance.distilled_guidance = 3.5f; + guidance.slg.layers = nullptr; + guidance.slg.layer_count = 0; + guidance.slg.layer_start = 0.01f; + guidance.slg.layer_end = 0.2f; + guidance.slg.scale = 0.f; + scheduler = SCHEDULER_COUNT; + sample_method = SAMPLE_METHOD_COUNT; + shifted_timestep = 0; + sample_steps = 20; + eta = INFINITY; + custom_sigmas = nullptr; + custom_sigmas_count = 0; + flow_shift = INFINITY; + extra_sample_args = nullptr; + } + + CacheConfig::CacheConfig() { + mode = SD_CACHE_DISABLED; + reuse_threshold = INFINITY; + start_percent = 0.15f; + end_percent = 0.95f; + error_decay_rate = 1.0f; + use_relative_threshold = true; + reset_error_on_compute = true; + Fn_compute_blocks = 8; + Bn_compute_blocks = 0; + residual_diff_threshold = 0.08f; + max_warmup_steps = 8; + max_cached_steps = -1; + max_continuous_cached_steps = -1; + taylorseer_n_derivatives = 1; + taylorseer_skip_interval = 1; + scm_mask = nullptr; + scm_policy_dynamic = true; + spectrum_w = 0.40f; + spectrum_m = 3; + spectrum_lam = 1.0f; + spectrum_window_size = 2; + spectrum_flex_window = 0.50f; + spectrum_warmup_steps = 4; + spectrum_stop_percent = 0.9f; + } + + HiresConfig::HiresConfig() { + enabled = false; + upscaler = SD_HIRES_UPSCALER_LATENT; + model_path = nullptr; + scale = 2.f; + target_width = 0; + target_height = 0; + steps = 0; + denoising_strength = 0.7f; + upscale_tile_size = 128; + custom_sigmas = nullptr; + custom_sigmas_count = 0; + } + + ContextConfig::ContextConfig() { + n_threads = sd_get_num_physical_cores(); + conditioning_cache_size = 4; + wtype = SD_TYPE_COUNT; + rng_type = CUDA_RNG; + sampler_rng_type = RNG_TYPE_COUNT; + prediction = PREDICTION_COUNT; + lora_apply_mode = LORA_APPLY_AUTO; + max_vram = nullptr; + disable_prefetch = false; + disable_segmented_compute = false; + batched_cfg = true; + eager_load = false; + enable_mmap = false; + flash_attn_mode = nullptr; + conv_direct_mode = nullptr; + diffusion_flash_attn = false; + sage_attn = false; + linear_scale = 0.f; + attn_scale = 0.f; + vae_format = SD_VAE_FORMAT_AUTO; + backend = nullptr; + params_backend = nullptr; + split_mode = nullptr; + auto_fit = true; + rpc_servers = nullptr; + model_args = nullptr; + pulid_weights_path = nullptr; + } + + ImageConfig::ImageConfig() { + clip_skip = -1; + ref_images_count = 0; + ref_image_args = ""; + width = 512; + height = 512; + strength = 0.75f; + seed = -1; + batch_count = 1; + control_strength = 0.9f; + ip_adapter_strength = 1.0f; + qwen_image_layers = 3; + circular_x = false; + circular_y = false; + pm_params = {nullptr, 0, nullptr, 20.f}; + pulid_params = {nullptr, 1.0f}; + vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr}; + } + + VideoConfig::VideoConfig() { + high_noise_sample_params.sample_steps = -1; + width = 512; + height = 512; + strength = 0.75f; + seed = -1; + video_frames = 6; + fps = 16; + moe_boundary = 0.875f; + vace_strength = 1.f; + vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr}; + circular_x = false; + circular_y = false; + ref_image_args = ""; + } + +} // namespace sd diff --git a/src/config.h b/src/config.h new file mode 100644 index 0000000000..7331ccc4b9 --- /dev/null +++ b/src/config.h @@ -0,0 +1,295 @@ +#ifndef SD_CONFIG_H +#define SD_CONFIG_H + +#include "core/media.h" + +#include "core/option_types.h" +#include "stable-diffusion.h" + +namespace sd { + struct TilingConfig { + bool enabled; + bool temporal_tiling; + // Spatial tile dimensions in image pixels for both encode and decode; 0 uses 256. + int tile_size_w; + int tile_size_h; + float target_overlap; + // Positive values override tile_size: <= 1 is a dimension fraction, > 1 a target tile count. + float rel_size_w; + float rel_size_h; + const char* extra_tiling_args; + }; + + struct ImagePreprocessConfig { + // Semicolon-separated target=...,key=value rules. NULL preserves defaults. + const char* rules; + }; + + struct SLGConfig { + int* layers; + size_t layer_count; + float layer_start; + float layer_end; + float scale; + }; + + struct GuidanceConfig { + float txt_cfg; + float img_cfg; + float distilled_guidance; + sd::SLGConfig slg; + }; + + struct SampleConfig { + sd::GuidanceConfig guidance; + enum scheduler_t scheduler; + enum sample_method_t sample_method; + int sample_steps; + float eta; + int shifted_timestep; + float* custom_sigmas; + int custom_sigmas_count; + float flow_shift; + const char* extra_sample_args; + + SampleConfig(); + }; + + struct PhotoMakerConfig { + sd::Image* id_images; + int id_images_count; + const char* id_embed_path; + float style_strength; + }; + + struct PuLIDConfig { + const char* id_embedding_path; + float id_weight; + }; + + struct CacheConfig { + enum sd_cache_mode_t mode; + float reuse_threshold; + float start_percent; + float end_percent; + float error_decay_rate; + bool use_relative_threshold; + bool reset_error_on_compute; + int Fn_compute_blocks; + int Bn_compute_blocks; + float residual_diff_threshold; + int max_warmup_steps; + int max_cached_steps; + int max_continuous_cached_steps; + int taylorseer_n_derivatives; + int taylorseer_skip_interval; + const char* scm_mask; + bool scm_policy_dynamic; + float spectrum_w; + int spectrum_m; + float spectrum_lam; + int spectrum_window_size; + float spectrum_flex_window; + int spectrum_warmup_steps; + float spectrum_stop_percent; + + CacheConfig(); + }; + + struct HiresConfig { + bool enabled; + enum sd_hires_upscaler_t upscaler; + const char* model_path; + float scale; + int target_width; + int target_height; + int steps; + float denoising_strength; + int upscale_tile_size; + float* custom_sigmas; + int custom_sigmas_count; + + HiresConfig(); + }; + + struct ContextConfig { + const char* model_path{}; + const char* clip_l_path{}; + const char* clip_g_path{}; + const char* clip_vision_path{}; + const char* t5xxl_path{}; + const char* llm_path{}; + const char* llm_vision_path{}; + const char* diffusion_model_path{}; + const char* high_noise_diffusion_model_path{}; + const char* uncond_diffusion_model_path{}; + const char* embeddings_connectors_path{}; + const char* vae_path{}; + const char* audio_vae_path{}; + const char* audio_encoder_path{}; + const char* taesd_path{}; + const char* control_net_path{}; + const char* ip_adapter_path{}; + const char* motion_module_path{}; + const sd_embedding_t* embeddings{}; + uint32_t embedding_count{}; + const char* photo_maker_path{}; + const char* pulid_weights_path{}; + const char* tensor_type_rules{}; + int n_threads{}; + enum sd_type_t wtype {}; + enum rng_type_t rng_type {}; + enum rng_type_t sampler_rng_type {}; + enum prediction_t prediction {}; + enum lora_apply_mode_t lora_apply_mode {}; + bool enable_mmap{}; + bool flash_attn{}; + bool diffusion_flash_attn{}; + bool tae_preview_only{}; + bool diffusion_conv_direct{}; + bool vae_conv_direct{}; + bool force_sdxl_vae_conv_scale{}; + enum sd_vae_format_t vae_format {}; + const char* max_vram{}; // Optional per-device GiB budget for managed weights and runner buffers; 0 uses live free VRAM without an explicit budget + bool disable_prefetch{}; // Disable asynchronous next-segment weight prefetch + bool eager_load{}; // Load all params into the params backend at model-load time instead of lazily on first use + const char* backend{}; + const char* params_backend{}; + const char* split_mode{}; // weight distribution for multi-device modules: layer (default) or row, or per-module assignments e.g. "diffusion=row" + bool auto_fit{}; + const char* rpc_servers{}; + const char* model_args{}; + bool disable_segmented_compute{}; // Force monolithic graph execution even when automatic graph cutting would fit memory better + bool batched_cfg{}; // Run the conditional and unconditional CFG branches in one batched UNet forward when supported + float linear_scale{}; // Override linear input scaling; 0 keeps the model default + float attn_scale{}; // Override flash-attention K/V scaling; 0 keeps the model default + const char* tokenizer{}; // tokenizer.json path or main=FILE,clip-l=FILE,clip-g=FILE assignments; required for PiD and Lens + bool sage_attn{}; + int conditioning_cache_size{}; // Maximum cached conditioning entries per context; 0 disables caching (default: 4) + const char* flash_attn_mode{}; // on/off/auto or module assignments (e.g. "auto,te=off"); NULL/empty honors legacy flags. + const char* conv_direct_mode{}; // on/off/auto or diffusion/vae/controlnet assignments; NULL/empty honors legacy flags. + + ContextConfig(); + }; + + struct ImageConfig { + const sd_lora_t* loras{}; + uint32_t lora_count{}; + const char* prompt{}; + const char* negative_prompt{}; + int clip_skip{}; + sd::Image init_image{}; + sd::Image* ref_images{}; + int ref_images_count{}; + const char* ref_image_args{}; + sd::Image mask_image{}; + int width{}; + int height{}; + sd::SampleConfig sample_params{}; + float strength{}; + int64_t seed{}; + int batch_count{}; + sd::Image control_image{}; + float control_strength{}; + sd::Image ip_adapter_image{}; + float ip_adapter_strength{}; + sd::PhotoMakerConfig pm_params{}; + sd::PuLIDConfig pulid_params{}; + sd::TilingConfig vae_tiling_params{}; + sd::CacheConfig cache{}; + sd::HiresConfig hires{}; + int qwen_image_layers{}; + bool circular_x{}; + bool circular_y{}; + sd::ImagePreprocessConfig image_preprocess{}; + enum sd_control_type_t control_type {}; + + ImageConfig(); + }; + + struct VideoConfig { + const sd_lora_t* loras{}; + uint32_t lora_count{}; + const char* prompt{}; + const char* negative_prompt{}; + int clip_skip{}; + sd::Image init_image{}; + sd::Image end_image{}; + sd::Image* ref_images{}; + int ref_images_count{}; + sd::ReferenceVideo* ref_videos{}; + int ref_videos_count{}; + sd::Audio* ref_audios{}; + int ref_audios_count{}; + sd::Image* control_frames{}; + int control_frames_size{}; + int width{}; + int height{}; + sd::SampleConfig sample_params{}; + sd::SampleConfig high_noise_sample_params{}; + float moe_boundary{}; + float strength{}; + int64_t seed{}; + int video_frames{}; + int fps{}; + float vace_strength{}; + sd::TilingConfig vae_tiling_params{}; + sd::CacheConfig cache{}; + sd::HiresConfig hires{}; + bool circular_x{}; + bool circular_y{}; + sd::ImagePreprocessConfig image_preprocess{}; + const char* ref_image_args{}; + + VideoConfig(); + }; + + struct VideoUpscalerConfig { + const char* diffusion_model_path{}; + const char* vae_path{}; + // Explicit output scale; RelayVSR requires 2 or 4 matching both checkpoints. + int upscale_factor{}; + int n_threads{}; + const char* backend{}; + const char* params_backend{}; + // auto/on/off or diffusion/vae assignments; null selects auto. + const char* flash_attn_mode{}; + const char* conv_direct_mode{}; + }; + + struct VideoUpscaleConfig { + const sd::Image* frames{}; + int frame_count{}; + int fps = 24; + int64_t seed = 42; + // RelayVSR reference window, including both endpoints; defaults to 16. + int window_size = 16; + // SeedVR2 independent temporal chunks in input frames (1 or 4n+1); defaults to 5. + int temporal_chunk_size = 5; + }; + + struct ADetailConfig { + const char* prompt{}; + const char* negative_prompt{}; + const char* extra_ad_args{}; + }; + + struct PostprocessConfig { + sd::Image image{}; + bool conv_direct = false; + int tile_size = 128; + VideoUpscalerConfig video_upscaler; + VideoUpscaleConfig video_upscale; + ADetailConfig adetail; + }; + + struct OptConfig { + ContextConfig context; + ImageConfig image; + VideoConfig video; + PostprocessConfig post; + }; + +} // namespace sd + +#endif // SD_CONFIG_H diff --git a/src/core/backend_fit.cpp b/src/core/backend_fit.cpp index 120cbe423f..123894de21 100644 --- a/src/core/backend_fit.cpp +++ b/src/core/backend_fit.cpp @@ -1,4 +1,5 @@ #include "backend_fit.h" +#include "config.h" #include #include @@ -489,7 +490,7 @@ namespace sd::backend_fit { return true; } - bool prepare_vae_decode_retry_tiling(sd_tiling_params_t& tiling_params, + bool prepare_vae_decode_retry_tiling(sd::TilingConfig& tiling_params, bool prefer_temporal_tiling, ggml_status status, int latent_tile_size_w, diff --git a/src/core/backend_fit.h b/src/core/backend_fit.h index ad1b60c614..61393ad6b2 100644 --- a/src/core/backend_fit.h +++ b/src/core/backend_fit.h @@ -3,6 +3,7 @@ #include +#include "config.h" #include "core/ggml_graph_cut.h" #include "model_loader.h" #include "stable-diffusion.h" @@ -15,7 +16,7 @@ namespace sd::backend_fit { std::string& runtime_spec, std::string& params_spec); - bool prepare_vae_decode_retry_tiling(sd_tiling_params_t& tiling_params, + bool prepare_vae_decode_retry_tiling(sd::TilingConfig& tiling_params, bool prefer_temporal_tiling, ggml_status status, int latent_tile_size_w, diff --git a/src/core/compute_workspace.cpp b/src/core/compute_workspace.cpp index cf9d04c576..128e0c3ad3 100644 --- a/src/core/compute_workspace.cpp +++ b/src/core/compute_workspace.cpp @@ -53,10 +53,12 @@ namespace sd { if (measurement.buffers.empty()) { return false; } - const bool grows = std::any_of(measurement.buffers.begin(), measurement.buffers.end(), - [&](const BackendBufferSize& size) { return size.bytes > bytes(size.backend); }); - if (grows) { - release(); + GGML_ASSERT(peak_scheduler_bytes_.empty() || peak_scheduler_bytes_.size() == measurement.buffers.size()); + // The reported size includes per-graph scratch, which can shrink while the scheduler is retained. + for (size_t i = 0; i < peak_scheduler_bytes_.size(); ++i) { + if (measurement.buffers[i].bytes > peak_scheduler_bytes_[i]) { + return release_scheduler(); + } } return true; } @@ -64,14 +66,14 @@ namespace sd { bool ComputeWorkspace::release_excess(const Measurement& measurement) { return std::any_of(measurement.buffers.begin(), measurement.buffers.end(), [&](const BackendBufferSize& size) { return bytes(size.backend) > size.bytes; }) && - release(); + release_scheduler(); } - bool ComputeWorkspace::allocate(ggml_cgraph* graph, const AssignNodes& assign_nodes) { + bool ComputeWorkspace::allocate(ggml_cgraph* graph, const Measurement& measurement, const AssignNodes& assign_nodes) { GGML_ASSERT(!active_); const size_t capacity = static_cast(ggml_graph_n_nodes(graph) + ggml_graph_n_leafs(graph)) + 8; if (scheduler_ == nullptr || capacity > scheduler_capacity_) { - release(); + release_scheduler(); scheduler_ = make_scheduler(capacity); scheduler_capacity_ = capacity; } @@ -85,6 +87,10 @@ namespace sd { release(); return false; } + peak_scheduler_bytes_.resize(measurement.buffers.size()); + for (size_t i = 0; i < measurement.buffers.size(); ++i) { + peak_scheduler_bytes_[i] = std::max(peak_scheduler_bytes_[i], bytes(measurement.buffers[i].backend)); + } active_ = true; return true; } @@ -154,8 +160,13 @@ namespace sd { append(uses); } // Match the full allocation metadata, including topology and external placement. - if (key == measurement_key_) { - return {measured_buffers_}; + for (auto& cached : measurement_cache_) { + if (key == cached.key) { + if (&cached != &measurement_cache_.front()) { + std::swap(cached, measurement_cache_.front()); + } + return {measurement_cache_.front().buffers}; + } } } const size_t graph_size = tensors.size() + 8; @@ -229,8 +240,8 @@ namespace sd { } ggml_free(context); if (cache_measurement && !result.buffers.empty()) { - measurement_key_ = std::move(key); - measured_buffers_ = result.buffers; + measurement_cache_.back() = std::move(measurement_cache_.front()); + measurement_cache_.front() = {std::move(key), result.buffers}; } return result; } @@ -263,14 +274,21 @@ namespace sd { } bool ComputeWorkspace::release() { + if (!release_scheduler()) { + return false; + } + measurement_cache_ = {}; + return true; + } + + bool ComputeWorkspace::release_scheduler() { if (active_) { return false; } - measurement_key_.clear(); - measured_buffers_.clear(); ggml_backend_sched_free(scheduler_); scheduler_ = nullptr; scheduler_capacity_ = 0; + peak_scheduler_bytes_.clear(); return true; } diff --git a/src/core/compute_workspace.h b/src/core/compute_workspace.h index 8dee5da556..6cfbd5c939 100644 --- a/src/core/compute_workspace.h +++ b/src/core/compute_workspace.h @@ -1,6 +1,7 @@ #ifndef __SD_CORE_COMPUTE_WORKSPACE_H__ #define __SD_CORE_COMPUTE_WORKSPACE_H__ +#include #include #include @@ -19,10 +20,15 @@ namespace sd { ggml_backend_sched_t scheduler_ = nullptr; size_t scheduler_capacity_ = 0; bool active_ = false; - std::vector measurement_key_; - std::vector measured_buffers_; + std::vector peak_scheduler_bytes_; + struct CachedMeasurement { + std::vector key; + std::vector buffers; + }; + std::array measurement_cache_; ggml_backend_sched_t make_scheduler(size_t graph_size); + bool release_scheduler(); public: struct Measurement { @@ -39,7 +45,7 @@ namespace sd { void set_extra_backends(const std::vector& backends); bool prepare(const Measurement& measurement); bool release_excess(const Measurement& measurement); - bool allocate(ggml_cgraph* graph, const AssignNodes& assign_nodes); + bool allocate(ggml_cgraph* graph, const Measurement& measurement, const AssignNodes& assign_nodes); Measurement measure( ggml_cgraph* graph, const std::function& external_backend, diff --git a/src/core/cuda/CMakeLists.txt b/src/core/cuda/CMakeLists.txt index e2dd3f92c3..a1290e7372 100644 --- a/src/core/cuda/CMakeLists.txt +++ b/src/core/cuda/CMakeLists.txt @@ -49,16 +49,27 @@ if(SD_HIPBLAS) find_package(hipblas REQUIRED) if(WIN32 OR CMAKE_CXX_COMPILER MATCHES "hipcc") sd_cuda_sources(hip CXX) + set_source_files_properties(hip_attention.cu PROPERTIES LANGUAGE CXX) target_link_libraries(${SD_CUDA_TARGET} PRIVATE hip::device) else() sd_cuda_sources(hip HIP) + set_source_files_properties(hip_attention.cu PROPERTIES LANGUAGE HIP) set_target_properties(${SD_CUDA_TARGET} PROPERTIES HIP_STANDARD 17 HIP_STANDARD_REQUIRED ON) endif() + target_sources(${SD_CUDA_TARGET} PRIVATE hip_attention.cu) set_source_files_properties(${SD_CUDA_SOURCES} PROPERTIES COMPILE_OPTIONS "-ffast-math") # W4A8 WMMA uses the wave64 fragment layout on both GFX11 and GFX12. set_property(SOURCE "${CMAKE_CURRENT_BINARY_DIR}/w4_hip.cu" APPEND PROPERTY COMPILE_OPTIONS "-mwavefrontsize64") target_link_libraries(${SD_CUDA_TARGET} PRIVATE hip::host roc::hipblas) target_link_libraries(${SD_LIB} PRIVATE hip::host roc::hipblas) + find_package(hipblaslt CONFIG QUIET) + if(hipblaslt_FOUND AND TARGET roc::hipblaslt AND hipblaslt_VERSION VERSION_GREATER_EQUAL 1.0) + target_sources(${SD_CUDA_TARGET} PRIVATE hip_i8_gemm.cpp) + target_include_directories(${SD_CUDA_TARGET} PRIVATE "${PROJECT_SOURCE_DIR}/include") + target_compile_definitions(${SD_CUDA_TARGET} PRIVATE SD_HIP_I8_LT) + target_link_libraries(${SD_CUDA_TARGET} PRIVATE roc::hipblaslt) + target_link_libraries(${SD_LIB} PRIVATE roc::hipblaslt) + endif() endif() if(SD_MUSA) diff --git a/src/core/cuda/affine.cu b/src/core/cuda/affine.cu index 2cd660b722..0bd7989735 100644 --- a/src/core/cuda/affine.cu +++ b/src/core/cuda/affine.cu @@ -29,6 +29,12 @@ namespace SD_CUDA_NAMESPACE { static bool bounded(const ggml_tensor* tensor) { return ggml_nelements(tensor) > 0 && ggml_nelements(tensor) <= int64_t(INT_MAX) * 1024; } + static bool cast_supports(ggml_backend_dev_t, const ggml_tensor* node, void*) { + const auto* src = node->src[0]; + return node->src[1] == node && !node->view_src && node->type == GGML_TYPE_F16 && ggml_is_contiguous(node) && + src && src->type == GGML_TYPE_F32 && ggml_are_same_shape(src, node) && !ggml_is_contiguous(src) && src->ne[0] % 4 == 0 && src->nb[0] == sizeof(float) && + (src->nb[1] | src->nb[2] | src->nb[3]) % 16 == 0 && aligned(src, 16) && aligned(node, 4) && bounded(node); + } static int match(ggml_backend_dev_t, const ggml_cgraph* graph, int first, void*) { auto* g = const_cast(graph); if (disabled() || first + 1 >= ggml_graph_n_nodes(g)) @@ -106,15 +112,19 @@ namespace SD_CUDA_NAMESPACE { dst[2 * i] = __floats2half2_rn(value.x, value.y); dst[2 * i + 1] = __floats2half2_rn(value.z, value.w); } - static __global__ void scale_permute(const float* src, half2* dst, int64_t dim, int64_t tokens, int64_t heads, int64_t count, int64_t nb1, int64_t nb2, int64_t nb3, float scale) { + template + static __global__ void cast_strided(const float* src, half2* dst, int64_t dim, int64_t ne1, int64_t ne2, int64_t count, int64_t stride1, int64_t stride2, int64_t stride3, float scale) { const int64_t index = (int64_t(blockIdx.x) * blockDim.x + threadIdx.x) * 4; if (index >= count) return; const int64_t col = index % dim, row = index / dim; - const int64_t token = row % tokens, head = row / tokens % heads, batch = row / (tokens * heads); - const float4 value = *reinterpret_cast(src + col + head * nb1 + token * nb2 + batch * nb3); - dst[index / 2] = __floats2half2_rn(__fmul_rn(value.x, scale), __fmul_rn(value.y, scale)); - dst[index / 2 + 1] = __floats2half2_rn(__fmul_rn(value.z, scale), __fmul_rn(value.w, scale)); + const int64_t i1 = row % ne1, i2 = row / ne1 % ne2, i3 = row / (ne1 * ne2); + float4 value = *reinterpret_cast(src + col + i1 * stride1 + i2 * stride2 + i3 * stride3); + if constexpr (Scaled) { + value = make_float4(__fmul_rn(value.x, scale), __fmul_rn(value.y, scale), __fmul_rn(value.z, scale), __fmul_rn(value.w, scale)); + } + dst[index / 2] = __floats2half2_rn(value.x, value.y); + dst[index / 2 + 1] = __floats2half2_rn(value.z, value.w); } static bool overlaps(const ggml_tensor* a, const ggml_tensor* b) { const auto x = reinterpret_cast(a->data), y = reinterpret_cast(b->data); @@ -129,15 +139,17 @@ namespace SD_CUDA_NAMESPACE { ggml_ext_gpu_context context; if (!cuda_context(execution->backend, &context)) return GGML_STATUS_FAILED; - auto stream = static_cast(context.stream); - const auto* node = nodes[0]; - const auto* a = node->src[0]; - const auto* b = node->op == GGML_OP_CONT ? nullptr : node->src[1]; - auto* dst = nodes[count - 1]; - const auto* residual = node->op == GGML_OP_MUL ? (dst->src[0] == node ? dst->src[1] : dst->src[0]) : nullptr; - const bool gate = residual != nullptr; - const bool alias = (overlaps(dst, a) && !(gate && dst->data == a->data)) || (b && overlaps(dst, b)) || - (residual && overlaps(dst, residual) && dst->data != residual->data); + auto stream = static_cast(context.stream); + const auto* node = nodes[0]; + const auto* a = node->src[0]; + const auto* b = node->op == GGML_OP_CONT || node->op == GGML_OP_CPY ? nullptr : node->src[1]; + auto* dst = nodes[count - 1]; + const auto* residual = node->op == GGML_OP_MUL ? (dst->src[0] == node ? dst->src[1] : dst->src[0]) : nullptr; + const bool gate = residual != nullptr; + const bool alias = (overlaps(dst, a) && !(gate && dst->data == a->data)) || (b && overlaps(dst, b)) || + (residual && overlaps(dst, residual) && dst->data != residual->data); + if (alias && !execution->workspace) + return GGML_STATUS_FAILED; void* output = alias ? ggml_backend_buffer_get_base(execution->workspace) : dst->data; const int64_t count4 = ggml_nelements(dst) / 4; const unsigned blocks = unsigned((count4 + 255) / 256); @@ -150,9 +162,12 @@ namespace SD_CUDA_NAMESPACE { kernel<<>>(static_cast(a->data), static_cast(b->data), static_cast(output), node->ne[0] / 4, a->ne[permuted ? 2 : 1], node->ne[permuted ? 2 : 1], node->ne[permuted ? 1 : 2], count4); + } else if (node->op == GGML_OP_CONT) { + cast_strided<<>>(static_cast(a->data), static_cast(output), + a->ne[0], a->ne[2], a->ne[1], ggml_nelements(dst), a->nb[2] / 4, a->nb[1] / 4, a->nb[3] / 4, param(nodes[1], 0)); } else { - scale_permute<<>>(static_cast(a->data), static_cast(output), - a->ne[0], a->ne[2], a->ne[1], ggml_nelements(dst), a->nb[1] / 4, a->nb[2] / 4, a->nb[3] / 4, param(nodes[1], 0)); + cast_strided<<>>(static_cast(a->data), static_cast(output), + a->ne[0], a->ne[1], a->ne[2], ggml_nelements(dst), a->nb[1] / 4, a->nb[2] / 4, a->nb[3] / 4, 1.f); } auto error = cuda_last_error(); if (error == cuda_success && alias) @@ -178,6 +193,12 @@ namespace SD_CUDA_NAMESPACE { desc.name = "sd.cont_scale_permute_cast." SD_CUDA_NAME ".v1"; desc.op = GGML_OP_CONT; GGML_ASSERT(ggml_ext_register_kernel_ex(device, &desc, GGML_EXT_KERNEL_CAPTURE_SAFE | GGML_EXT_KERNEL_REPLAY_REUSABLE)); + desc.name = "sd.cast_strided_f32_f16." SD_CUDA_NAME ".v1"; + desc.op = GGML_OP_CPY; + desc.match = nullptr; + desc.supports = sd_affine_cuda::cast_supports; + desc.preferred = sd_affine_cuda::cast_supports; + GGML_ASSERT(ggml_ext_register_kernel_ex(device, &desc, GGML_EXT_KERNEL_CAPTURE_SAFE | GGML_EXT_KERNEL_REPLAY_REUSABLE)); } } diff --git a/src/core/cuda/bf16.cu b/src/core/cuda/bf16.cu index 59f5073947..8389323800 100644 --- a/src/core/cuda/bf16.cu +++ b/src/core/cuda/bf16.cu @@ -29,7 +29,7 @@ namespace SD_CUDA_NAMESPACE { } static bool projection(const ggml_tensor* mat, const ggml_tensor* input) { - return mat->op == GGML_OP_MUL_MAT && mat->src[1] == input && mat->op_params[0] == GGML_PREC_DEFAULT && + return mat->op == GGML_OP_MUL_MAT && mat->src[1] == input && ggml_prec_get_acc(mat) == GGML_PREC_UNDEFINED && matrix(input, GGML_TYPE_F32) && input->ne[1] >= 32 && ggml_nelements(input) <= INT_MAX && matrix(mat, GGML_TYPE_F32) && matrix(mat->src[0], GGML_TYPE_BF16) && !mat->view_src; } diff --git a/src/core/cuda/convrot.cu b/src/core/cuda/convrot.cu index 26920ae934..c069d46b91 100644 --- a/src/core/cuda/convrot.cu +++ b/src/core/cuda/convrot.cu @@ -30,6 +30,7 @@ namespace SD_CUDA_NAMESPACE { MOD, RMS_MOD, SWIGLU, + SITU, SILU, SIGMOID }; struct Operation { @@ -81,8 +82,11 @@ namespace SD_CUDA_NAMESPACE { op.eps = param_f32(first, 0); if (op.k < 1024 || !vector(op.weight, op.k) || !vector(op.modulation, op.k) || (op.shift && !vector(op.shift, op.k))) return false; - } else if (count == 2 && first->op == GGML_OP_GLU && ggml_get_glu_op(first) == GGML_GLU_OP_SWIGLU && value == first) { - op.kind = SWIGLU; + } else if (count == 2 && first->op == GGML_OP_GLU && value == first) { + const auto glu = ggml_get_glu_op(first); + if (glu != GGML_GLU_OP_SWIGLU && glu != GGML_GLU_OP_SITU) + return false; + op.kind = glu == GGML_GLU_OP_SITU ? SITU : SWIGLU; op.up = first->src[1] ? first->src[1] : op.input; if (!f32(op.up)) return false; @@ -217,26 +221,26 @@ namespace SD_CUDA_NAMESPACE { const auto x = reinterpret_cast(a->data), y = reinterpret_cast(b->data); return x <= y ? y - x < ggml_nbytes(a) : x - y < ggml_nbytes(b); } - template + template static CUDAError launch(Plan& p, const Operation& op, int8_t* output, CUDAStream stream) { constexpr bool warp = Scaled || RMS || Modulate; constexpr int extra = Normalize ? 384 : RMS ? 256 : 128; if (!p.configured) { - auto status = cuda_set_shared_memory(quantize, + auto status = cuda_set_shared_memory(quantize, int(p.device.shared_memory - extra)); if (status != cuda_success) return status; p.configured = true; } - const int threads = Normalize ? 1024 : RMS ? (op.k <= 1024 ? 128 : op.k <= 6144 ? 256 - : op.k <= 8192 ? 512 - : 1024) - : Swiglu && !Sigmoid && op.k >= 16384 ? 512 - : 256; + const int threads = Normalize ? 1024 : RMS ? (op.k <= 1024 ? 128 : op.k <= 6144 ? 256 + : op.k <= 8192 ? 512 + : 1024) + : (Situ ? op.k >= 8192 : Swiglu && !Sigmoid && op.k >= 16384) ? 512 + : 256; const int64_t padded = (op.rows + 3) / 4 * 4; auto data = [](const ggml_tensor* t) { return t ? static_cast(t->data) : nullptr; }; - quantize<<>>( + quantize<<>>( data(op.input) + op.gate_offset, output, reinterpret_cast(output + op.k * padded), op.k, op.rows, op.k, op.up ? data(op.up) + op.up_offset : nullptr, op.stride, op.up_stride, nullptr, nullptr, data(op.modulation), op.eps, op.scale, op.bias, data(op.weight), data(op.shift)); return cuda_last_error(); @@ -274,6 +278,9 @@ namespace SD_CUDA_NAMESPACE { case SILU: error = launch(plan, op, output, stream); break; + case SITU: + error = launch(plan, op, output, stream); + break; case SIGMOID: error = launch(plan, op, output, stream); break; diff --git a/src/core/cuda/convrot_kernel.h b/src/core/cuda/convrot_kernel.h index 04abf97ac3..fa9e4daf0e 100644 --- a/src/core/cuda/convrot_kernel.h +++ b/src/core/cuda/convrot_kernel.h @@ -40,7 +40,13 @@ namespace SD_CUDA_NAMESPACE { return __fadd_rn(__fadd_rn(__fadd_rn(a, b), c), d); } - template + static __device__ __forceinline__ float situ(float gate, float up) { + const float a = __fmul_rn(tanhf(__fmul_rn(0.25f, gate)), 4.0f); + const float b = __fmul_rn(tanhf(__fmul_rn(1.0f / 25.0f, up)), 25.0f); + return __fmul_rn(__fmul_rn(a, 1.0f / (1.0f + expf(-gate))), b); + } + + template __global__ void quantize( const T* src, int8_t* dst, @@ -119,14 +125,18 @@ namespace SD_CUDA_NAMESPACE { if constexpr (dequantize) { gate_value = (float)((const int32_t*)src_row)[i] * activation_scales[row]; up_value = (float)((const int32_t*)up)[row * up_stride + i] * activation_scales[row]; - gate_value *= weight_scales[i]; - up_value *= weight_scales[k + i]; + gate_value *= weight_scales[(situ_gate ? k : 0) + i]; + up_value *= weight_scales[(situ_gate ? 0 : k) + i]; } else { gate_value = src_row[i]; up_value = up[row * up_stride + i]; } - const float activated = sigmoid_gate ? 1.0f / (1.0f + expf(-gate_value)) : (gate_value / (1.0f + expf(-gate_value))); - v[j] = (activated * up_value) * (1.0f / 16.0f); + if constexpr (situ_gate) { + v[j] = situ(gate_value, up_value) * (1.0f / 16.0f); + } else { + const float activated = sigmoid_gate ? 1.0f / (1.0f + expf(-gate_value)) : (gate_value / (1.0f + expf(-gate_value))); + v[j] = (activated * up_value) * (1.0f / 16.0f); + } } } // One logical warp owns H256, keeping every butterfly in registers. diff --git a/src/core/cuda/cute_attention.cu b/src/core/cuda/cute_attention.cu index 32bc891023..a4224321e9 100644 --- a/src/core/cuda/cute_attention.cu +++ b/src/core/cuda/cute_attention.cu @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -242,7 +243,7 @@ namespace sd_fattn_cute { bool scaled_output = false; }; - template + template __global__ __launch_bounds__(Warps * 32) void kernel(Params p) { using T = Traits; using Element = typename T::Element; @@ -294,6 +295,11 @@ namespace sd_fattn_cute { copy_rows(global_copy, last_k, tKsK, tKVcKV, p.nk - block * BN, p.dim); cute::cp_async_fence(); clear(acc_o); + if constexpr (QRegs) { + cute::cp_async_wait<0>(); + __syncthreads(); + cute::copy(copyQ, tSsQ, threadQ.retile_D(rQ)); + } Softmax<2 * size<1>(acc_o)> softmax; auto step = [&](auto first_flag, int n) { constexpr bool First = decltype(first_flag)::value; @@ -304,7 +310,7 @@ namespace sd_fattn_cute { auto v_tile = tVgV(_, _, _, n); copy_rows(global_copy, v_tile, tVsV, tKVcKV, p.nk - n * BN, p.dim); cute::cp_async_fence(); - gemm(acc_s, rQ, rK, tSsQ, tSsK, mma, copyQ, copyK, threadQ, threadK); + gemm(acc_s, rQ, rK, tSsQ, tSsK, mma, copyQ, copyK, threadQ, threadK); if constexpr (First) { #pragma unroll for (int i = 0; i < size(acc_s); ++i) { @@ -680,9 +686,9 @@ static ggml_status attention_prepare(ggml_backend_t, ggml_tensor* const* nodes, return GGML_STATUS_SUCCESS; } -template +template static cudaError_t attention_launch(const ggml_ext_cuda_context& context, const sd_fattn_cute::Params& p, int64_t batches) { - auto kernel = sd_fattn_cute::kernel<64, 64, D, FullDim>; + auto kernel = sd_fattn_cute::kernel<64, 64, D, FullDim, 4, QRegs>; constexpr int shared_bytes = sd_fattn_cute::Traits<64, 64, D>::SharedBytes; static std::mutex mutex; static std::unordered_map initialized; @@ -705,11 +711,12 @@ static cudaError_t attention_launch(const ggml_ext_cuda_context& context, const } static ggml_status attention_execute(const ggml_ext_execution_context* execution, ggml_tensor* const* nodes, int count, void* state, void*) { - const auto& plan = *static_cast(state); - const auto* attn = nodes[plan.index]; - auto device = ggml_backend_get_device(execution->backend); - auto reg = ggml_backend_dev_backend_reg(device); - auto query = reinterpret_cast(ggml_backend_reg_get_proc_address(reg, "ggml_ext_cuda_context_v1")); + const auto& plan = *static_cast(state); + const auto* attn = nodes[plan.index]; + auto device = ggml_backend_get_device(execution->backend); + const auto& properties = cuda_device(device)->properties; + auto reg = ggml_backend_dev_backend_reg(device); + auto query = reinterpret_cast(ggml_backend_reg_get_proc_address(reg, "ggml_ext_cuda_context_v1")); ggml_ext_cuda_context context; if (!query || !query(execution->backend, &context)) return GGML_STATUS_FAILED; @@ -774,7 +781,10 @@ static ggml_status attention_execute(const ggml_ext_execution_context* execution error = attention_launch<112, true>(context, p, q->ne[3]); break; case 128: - error = attention_launch<128>(context, p, q->ne[3]); + error = p.dim == 128 && p.nq >= 4096 && p.nq < 8192 && p.nk >= 4096 && p.nk < 8192 && + properties.major == 8 && properties.minor == 9 + ? attention_launch<128, true, true>(context, p, q->ne[3]) + : attention_launch<128>(context, p, q->ne[3]); break; case 192: error = attention_launch<192>(context, p, q->ne[3]); @@ -794,7 +804,7 @@ static ggml_status attention_execute(const ggml_ext_execution_context* execution if (rows != padded_rows && cudaMemsetAsync(packed + width * rows, 0, width * (padded_rows - rows), stream) != cudaSuccess) return GGML_STATUS_FAILED; if (cudaFuncSetAttribute(quantize_attention, cudaFuncAttributeMaxDynamicSharedMemorySize, - int(cuda_device(device)->properties.sharedMemPerBlockOptin - 128)) != cudaSuccess) + int(properties.sharedMemPerBlockOptin - 128)) != cudaSuccess) return GGML_STATUS_FAILED; quantize_attention<<>>(work[3], packed, reinterpret_cast(packed + width * padded_rows), width, rows, scale, bias); @@ -809,7 +819,18 @@ static void attention_destroy(void* state, void*) { } void sd_ggml_register_cuda_attention(ggml_backend_dev_t device) { - if (!cuda_device(device)) + const auto* info = cuda_device(device); + if (!info || info->properties.major < 8) + return; + int previous; + if (cudaGetDevice(&previous) != cudaSuccess || cudaSetDevice(info->ordinal) != cudaSuccess) + return; + cudaFuncAttributes attributes{}; + const auto error = cudaFuncGetAttributes(&attributes, sd_fattn_cute::kernel<64, 64>); + const auto restored = cudaSetDevice(previous); + // JIT preserves instruction guards from the original PTX architecture. + if (restored != cudaSuccess || error != cudaSuccess || + std::min(attributes.binaryVersion, attributes.ptxVersion) < 80) return; ggml_ext_kernel_desc desc = {}; desc.abi_version = GGML_EXT_ABI_VERSION; diff --git a/src/core/cuda/hip_attention.cu b/src/core/cuda/hip_attention.cu new file mode 100644 index 0000000000..adff9fb1ea --- /dev/null +++ b/src/core/cuda/hip_attention.cu @@ -0,0 +1,390 @@ +#define SD_CUDA_VENDOR_HIP 1 +#include "runtime.h" + +#include +#include +#include +#include + +namespace sd_hip { + static constexpr int WAVE_SIZE = 32; + static constexpr int WMMA_TILE = 16; + + template + struct AttentionTile { + static constexpr int channels = (HeadDim + WMMA_TILE - 1) / WMMA_TILE * WMMA_TILE; + static constexpr int keys = HeadDim > 128 ? 32 : 64; + static constexpr int query_waves = HeadDim > 128 ? 4 : 8; + static constexpr int queries = query_waves * WMMA_TILE; + }; + +#if __has_builtin(__builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12) +#define SD_HIP_ATTN_WMMA_128B +#elif __has_builtin(__builtin_amdgcn_wmma_f32_16x16x16_f16_w32) +#define SD_HIP_ATTN_WMMA_256B +#endif + +#if defined(SD_HIP_ATTN_WMMA_128B) || defined(SD_HIP_ATTN_WMMA_256B) + using Half8 = _Float16 __attribute__((ext_vector_type(8))); + using Accumulator = float __attribute__((ext_vector_type(8))); +#ifdef SD_HIP_ATTN_WMMA_256B + using HalfFragment = _Float16 __attribute__((ext_vector_type(16))); + static constexpr bool replicate_inputs = true; +#else + using HalfFragment = Half8; + static constexpr bool replicate_inputs = false; +#endif + + static __device__ __forceinline__ HalfFragment load_fragment(const _Float16* ptr) { + const auto low = *reinterpret_cast(ptr); +#ifdef SD_HIP_ATTN_WMMA_256B + // LDS rows are only 16-byte aligned, including when the fragment spans 32 bytes. + const auto high = *reinterpret_cast(ptr + 8); + return __builtin_shufflevector(low, high, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); +#else + return low; +#endif + } + + static __device__ __forceinline__ Accumulator attention_wmma(HalfFragment a, HalfFragment b, Accumulator acc) { +#ifdef SD_HIP_ATTN_WMMA_256B + return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a, b, acc); +#else + return __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(a, b, acc); +#endif + } +#endif + + struct AttentionInput { + const void* data; + size_t token_stride; + size_t head_stride; + size_t batch_stride; + + template + __device__ const T* head(int index, int batch) const { + return reinterpret_cast(static_cast(data) + index * head_stride + batch * batch_stride); + } + }; + + template + static __global__ void pack_attention_values(AttentionInput value, _Float16* packed, int keys) { + constexpr int channels = AttentionTile::channels; + constexpr int key_tile = AttentionTile::keys; + constexpr int channel_tiles = (HeadDim + 31) / 32; + // Each block transposes a 32x32 tile. The extra column avoids LDS bank conflicts. + __shared__ _Float16 tile[32][33]; + const int head = blockIdx.y; + const int batch = blockIdx.z; + const int token = (blockIdx.x / channel_tiles) * 32; + const int channel = (blockIdx.x % channel_tiles) * 32; + const int x = threadIdx.x; + const int y = threadIdx.y; + const auto* input = value.head<_Float16>(head, batch); + const size_t stride = value.token_stride / sizeof(_Float16); + const size_t head_size = ((size_t(keys) + key_tile - 1) / key_tile) * key_tile * channels; + auto* output = packed + (size_t(batch) * gridDim.y + head) * head_size + + size_t(token / key_tile) * key_tile * channels + channel * key_tile + token % key_tile; +#pragma unroll + for (int row = y; row < 32; row += 8) { + tile[row][x] = token + row < keys && channel + x < HeadDim ? input[(token + row) * stride + channel + x] : _Float16(0); + } + __syncthreads(); +#pragma unroll + for (int row = y; row < 32; row += 8) { + if (channel + row < channels) + output[row * key_tile + x] = tile[x][row]; + } + } + + template + static __global__ __launch_bounds__(AttentionTile::query_waves* WAVE_SIZE) void wmma_attention( + AttentionInput query, + AttentionInput key, + const _Float16* packed_value, + float* dst, + int queries, + int keys, + int kv_heads, + int queries_per_kv, + float scale) { +#if defined(SD_HIP_ATTN_WMMA_128B) || defined(SD_HIP_ATTN_WMMA_256B) + constexpr int channels = AttentionTile::channels; + constexpr int key_tile = AttentionTile::keys; + constexpr int query_tile = AttentionTile::queries; + const int lane = threadIdx.x % WMMA_TILE; + const int half = (threadIdx.x % WAVE_SIZE) / WMMA_TILE; + // The 256-bit operand layout duplicates both fragments across the two half-waves + // and distributes accumulator rows as even and odd indices. + const int input_offset = replicate_inputs ? 0 : half * 8; + const int acc_offset = replicate_inputs ? half : half * 8; + constexpr int acc_stride = replicate_inputs ? 2 : 1; + const int wave = threadIdx.x / WAVE_SIZE; + const int head = blockIdx.y; + const int kv_head = queries_per_kv == 1 ? head : head / queries_per_kv; + const int batch = blockIdx.z; + const int q_index = blockIdx.x * query_tile + wave * WMMA_TILE + lane; + const float* q = query.head(head, batch); + const _Float16* k = key.head<_Float16>(kv_head, batch); + const size_t q_stride = query.token_stride / sizeof(float); + const size_t k_stride = key.token_stride / sizeof(_Float16); + const size_t value_head_size = ((size_t(keys) + key_tile - 1) / key_tile) * key_tile * channels; + const _Float16* v = packed_value + (size_t(batch) * kv_heads + kv_head) * value_head_size; + + __shared__ __align__(16) _Float16 k_tile[key_tile][channels + 8]; + HalfFragment q_fragments[channels / WMMA_TILE] = {}; + Accumulator output[channels / WMMA_TILE] = {}; + if (q_index < queries) { +#pragma unroll + for (int d = 0; d < channels / WMMA_TILE; ++d) { +#pragma unroll + for (int i = 0; i < (replicate_inputs ? 16 : 8); ++i) { + const int channel = d * WMMA_TILE + input_offset + i; + if (channel < HeadDim) + q_fragments[d][i] = q[q_index * q_stride + channel] * scale; + } + } + } + + float maximum = -INFINITY; + float sum = 0.f; + for (int base = 0; base < keys; base += key_tile) { + const int tile_keys = min(keys - base, key_tile); + for (int index = threadIdx.x; index < key_tile * channels / 8; index += blockDim.x) { + const int token = index / (channels / 8); + const int channel = (index % (channels / 8)) * 8; + Half8 k_fragment = {}; + if (token < tile_keys && channel < HeadDim) { + const auto* row = k + (base + token) * k_stride; + if constexpr (HeadDim % 8 == 0) { + k_fragment = *reinterpret_cast(row + channel); + } else { +#pragma unroll + for (int i = 0; i < 8; ++i) + if (channel + i < HeadDim) + k_fragment[i] = row[channel + i]; + } + } + *reinterpret_cast(&k_tile[token][channel]) = k_fragment; + } + __syncthreads(); + + // K * Q^T leaves each query in a pair of lanes. + Accumulator scores[key_tile / WMMA_TILE] = {}; +#pragma unroll + for (int d = 0; d < channels / WMMA_TILE; ++d) { +#pragma unroll + for (int t = 0; t < key_tile / WMMA_TILE; ++t) { + const auto k_fragment = load_fragment(&k_tile[t * WMMA_TILE + lane][d * WMMA_TILE + input_offset]); + scores[t] = attention_wmma(k_fragment, q_fragments[d], scores[t]); + } + } + + float tile_max = -INFINITY; +#pragma unroll + for (int t = 0; t < key_tile / WMMA_TILE; ++t) { +#pragma unroll + for (int i = 0; i < 8; ++i) { + scores[t][i] = t * WMMA_TILE + acc_offset + acc_stride * i < tile_keys ? scores[t][i] : -INFINITY; + tile_max = fmaxf(tile_max, scores[t][i]); + } + } + const float next_max = fmaxf(maximum, fmaxf(tile_max, cuda_shuffle_xor(tile_max, 16))); + const float rescale = __expf(maximum - next_max); + maximum = next_max; + float tile_sum = 0.f; + HalfFragment probabilities[key_tile / WMMA_TILE]; +#pragma unroll + for (int t = 0; t < key_tile / WMMA_TILE; ++t) { +#pragma unroll + for (int i = 0; i < 8; ++i) { + const float p = __expf(scores[t][i] - maximum); + if constexpr (replicate_inputs) { + // Reassemble the 16 probabilities in key order for both + // half-waves before using them as the B operand of V^T * P. + const float peer = cuda_shuffle_xor(p, WMMA_TILE); + probabilities[t][2 * i] = half == 0 ? p : peer; + probabilities[t][2 * i + 1] = half == 0 ? peer : p; + } else { + probabilities[t][i] = p; + } + tile_sum += p; + } + } + sum = fmaf(sum, rescale, tile_sum + cuda_shuffle_xor(tile_sum, 16)); +#pragma unroll + for (int d = 0; d < channels / WMMA_TILE; ++d) { + output[d] *= rescale; +#pragma unroll + for (int t = 0; t < key_tile / WMMA_TILE; ++t) { + const size_t offset = size_t(base) * channels + (d * WMMA_TILE + lane) * key_tile + t * WMMA_TILE + input_offset; + const auto v_fragment = load_fragment(v + offset); + output[d] = attention_wmma(v_fragment, probabilities[t], output[d]); + } + } + __syncthreads(); + } + + if (q_index < queries) { + const float inverse = 1.f / sum; + float* out = dst + ((size_t(batch) * queries + q_index) * gridDim.y + head) * HeadDim; +#pragma unroll + for (int d = 0; d < channels / WMMA_TILE; ++d) { + output[d] *= inverse; + if constexpr (replicate_inputs || HeadDim % 8 != 0) { +#pragma unroll + for (int i = 0; i < 8; ++i) { + const int channel = d * WMMA_TILE + acc_offset + acc_stride * i; + if (channel < HeadDim) + out[channel] = output[d][i]; + } + } else { + const int channel = d * WMMA_TILE + acc_offset; + if (channel < HeadDim) + *reinterpret_cast(out + channel) = output[d]; + } + } + } +#elif defined(__HIP_DEVICE_COMPILE__) + __builtin_trap(); +#endif + } + + static bool attention_aligned(const ggml_tensor* tensor, size_t alignment) { + size_t offset = 0; + while (tensor->view_src) { + offset += tensor->view_offs; + tensor = tensor->view_src; + } + return (reinterpret_cast(tensor->data) + offset) % alignment == 0; + } + + static bool attention_layout(const ggml_tensor* tensor, ggml_type type, size_t alignment) { + return tensor->type == type && tensor->nb[0] == ggml_type_size(type) && + tensor->nb[1] % alignment == 0 && tensor->nb[2] % alignment == 0 && tensor->nb[3] % alignment == 0 && + attention_aligned(tensor, alignment); + } + + template + static ggml_status launch_attention(const ggml_ext_execution_context* execution, ggml_tensor* const* nodes, CUDAStream stream) { + using Tile = AttentionTile; + auto* dst = nodes[0]; + auto* q = dst->src[0]; + auto* k = dst->src[1]; + auto* v = dst->src[2]; + auto input = [](const ggml_tensor* t) { return AttentionInput{t->data, t->nb[1], t->nb[2], t->nb[3]}; }; + float scale; + std::memcpy(&scale, dst->op_params, sizeof(scale)); + auto* packed = static_cast<_Float16*>(ggml_backend_buffer_get_base(execution->workspace)); + const size_t value_tiles = ((v->ne[1] + Tile::keys - 1) / Tile::keys) * (Tile::keys / 32) * ((HeadDim + 31) / 32); + const dim3 pack_grid(value_tiles, v->ne[2], v->ne[3]); + pack_attention_values<<>>(input(v), packed, v->ne[1]); + if (cuda_last_error() != cuda_success) + return GGML_STATUS_FAILED; + + const dim3 grid((q->ne[1] + Tile::queries - 1) / Tile::queries, q->ne[2], q->ne[3]); + wmma_attention<<>>( + input(q), input(k), packed, static_cast(dst->data), q->ne[1], k->ne[1], k->ne[2], q->ne[2] / k->ne[2], scale); + return cuda_last_error() == cuda_success ? GGML_STATUS_SUCCESS : GGML_STATUS_FAILED; + } + + struct AttentionKernel { + int head_dim; + int channels; + int key_tile; + int query_tile; + ggml_status (*launch)(const ggml_ext_execution_context*, ggml_tensor* const*, CUDAStream); + }; + + template + static constexpr AttentionKernel attention_kernel() { + using Tile = AttentionTile; + return {HeadDim, Tile::channels, Tile::keys, Tile::queries, launch_attention}; + } + + static const AttentionKernel* select_attention_kernel(int64_t head_dim) { + static constexpr AttentionKernel kernels[] = { + attention_kernel<40>(), + attention_kernel<52>(), + attention_kernel<64>(), + attention_kernel<72>(), + attention_kernel<80>(), + attention_kernel<120>(), + attention_kernel<128>(), + attention_kernel<160>(), + attention_kernel<256>(), + attention_kernel<320>(), + }; + for (const auto& kernel : kernels) + if (head_dim == kernel.head_dim) + return &kernel; + return nullptr; + } + + static bool wmma_attention_supports(ggml_backend_dev_t, const ggml_tensor* dst, void*) { + const auto* q = dst->src[0]; + const auto* k = dst->src[1]; + const auto* v = dst->src[2]; + const auto* kernel = select_attention_kernel(q->ne[0]); + float params[3]; + std::memcpy(params, dst->op_params, sizeof(params)); + return kernel && !dst->src[3] && !dst->src[4] && std::isfinite(params[0]) && params[1] == 0.f && params[2] == 0.f && + q->ne[0] == k->ne[0] && ggml_are_same_shape(k, v) && + k->ne[2] > 0 && q->ne[2] % k->ne[2] == 0 && q->ne[3] == k->ne[3] && q->ne[2] <= 65535 && q->ne[3] <= 65535 && + q->ne[1] >= 512 && k->ne[1] >= 512 && q->ne[1] <= INT_MAX - kernel->query_tile && + k->ne[1] <= INT_MAX - kernel->key_tile && + attention_layout(q, GGML_TYPE_F32, sizeof(float)) && + attention_layout(k, GGML_TYPE_F16, q->ne[0] % 8 == 0 ? 16 : sizeof(_Float16)) && + attention_layout(v, GGML_TYPE_F16, sizeof(_Float16)) && + dst->type == GGML_TYPE_F32 && ggml_is_contiguous(dst) && !dst->view_src && attention_aligned(dst, 32); + } + + static ggml_status wmma_attention_prepare(ggml_backend_t, ggml_tensor* const* nodes, int, void**, size_t* workspace, void*) { + const auto* v = nodes[0]->src[2]; + const auto* kernel = select_attention_kernel(v->ne[0]); + if (!kernel) + return GGML_STATUS_FAILED; + const size_t tile = kernel->key_tile; + *workspace = ((size_t(v->ne[1]) + tile - 1) / tile) * tile * kernel->channels * + v->ne[2] * v->ne[3] * sizeof(_Float16); + return GGML_STATUS_SUCCESS; + } + + static void wmma_attention_destroy(void*, void*) {} + + static ggml_status wmma_attention_execute(const ggml_ext_execution_context* execution, ggml_tensor* const* nodes, int, void*, void*) { + ggml_ext_gpu_context context; + if (!execution->workspace || !cuda_context(execution->backend, &context)) { + return GGML_STATUS_FAILED; + } + const auto* kernel = select_attention_kernel(nodes[0]->src[0]->ne[0]); + return kernel ? kernel->launch(execution, nodes, static_cast(context.stream)) : GGML_STATUS_FAILED; + } + + void register_wmma_attention(ggml_backend_dev_t device) { + static const bool disabled = std::getenv("SD_HIP_DISABLE_WMMA_ATTN") != nullptr; + const auto info = cuda_device_info(device); + int previous; + if (disabled || info.device < 0 || hipGetDevice(&previous) != hipSuccess || hipSetDevice(info.device) != hipSuccess) { + return; + } + hipFuncAttributes attributes{}; + const auto status = hipFuncGetAttributes(&attributes, reinterpret_cast(wmma_attention<128>)); + const auto restored = hipSetDevice(previous); + // The device image uses a trap stub with no LDS when it lacks compatible wave32 WMMA. + if (status != hipSuccess || restored != hipSuccess || attributes.sharedSizeBytes == 0) { + return; + } + ggml_ext_kernel_desc desc = {}; + desc.abi_version = GGML_EXT_ABI_VERSION; + desc.name = "sd.wmma_attention.hip.v1"; + desc.op = GGML_OP_FLASH_ATTN_EXT; + desc.priority = 200; + desc.supports = wmma_attention_supports; + desc.preferred = wmma_attention_supports; + desc.prepare = wmma_attention_prepare; + desc.execute = wmma_attention_execute; + desc.destroy = wmma_attention_destroy; + GGML_ASSERT(ggml_ext_register_kernel_ex(device, &desc, GGML_EXT_KERNEL_CAPTURE_SAFE | GGML_EXT_KERNEL_REPLAY_REUSABLE)); + } +} diff --git a/src/core/cuda/hip_i8_gemm.cpp b/src/core/cuda/hip_i8_gemm.cpp new file mode 100644 index 0000000000..61b5fbb295 --- /dev/null +++ b/src/core/cuda/hip_i8_gemm.cpp @@ -0,0 +1,220 @@ +#include "hip_i8_gemm.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/util.h" + +namespace sd_hip { + using GemmShape = std::tuple; + + struct HipLtPlan { + GemmShape shape; + hipblasLtMatmulDesc_t op = nullptr; + hipblasLtMatrixLayout_t a = nullptr; + hipblasLtMatrixLayout_t b = nullptr; + hipblasLtMatrixLayout_t c = nullptr; + hipblasLtMatmulAlgo_t algo{}; + bool use_lt = false; + + explicit HipLtPlan(GemmShape shape) + : shape(shape) {} + + ~HipLtPlan() { + if (c) + hipblasLtMatrixLayoutDestroy(c); + if (b) + hipblasLtMatrixLayoutDestroy(b); + if (a) + hipblasLtMatrixLayoutDestroy(a); + if (op) + hipblasLtMatmulDescDestroy(op); + } + + bool init(const HipI8Gemm& gemm) { + const hipblasOperation_t transpose = HIPBLAS_OP_T; + return hipblasLtMatmulDescCreate(&op, HIPBLAS_COMPUTE_32I, HIP_R_32I) == HIPBLAS_STATUS_SUCCESS && + hipblasLtMatmulDescSetAttribute(op, HIPBLASLT_MATMUL_DESC_TRANSA, &transpose, sizeof(transpose)) == HIPBLAS_STATUS_SUCCESS && + hipblasLtMatrixLayoutCreate(&a, HIP_R_8I, gemm.k, gemm.n, gemm.k) == HIPBLAS_STATUS_SUCCESS && + hipblasLtMatrixLayoutCreate(&b, HIP_R_8I, gemm.k, gemm.rows, gemm.k) == HIPBLAS_STATUS_SUCCESS && + hipblasLtMatrixLayoutCreate(&c, HIP_R_32I, gemm.n, gemm.rows, gemm.n) == HIPBLAS_STATUS_SUCCESS; + } + + hipblasStatus_t run(hipblasLtHandle_t handle, const HipI8Gemm& gemm, const hipblasLtMatmulAlgo_t& choice) const { + const int32_t alpha = 1, beta = 0; + return hipblasLtMatmul(handle, op, &alpha, gemm.weight, a, gemm.input, b, + &beta, gemm.output, c, gemm.output, c, &choice, nullptr, 0, gemm.stream); + } + }; + + struct HipLtDevice { + int ordinal; + std::mutex mutex; + std::map handles; + std::list plans; + + explicit HipLtDevice(int ordinal) + : ordinal(ordinal) {} + + ~HipLtDevice() { + if (handles.empty()) + return; + int previous; + if (hipGetDevice(&previous) == hipSuccess && hipSetDevice(ordinal) == hipSuccess) { + for (const auto& entry : handles) + (void)hipblasLtDestroy(entry.second); + (void)hipSetDevice(previous); + } + } + }; + + struct HipGemmTimer { + hipEvent_t start = nullptr; + hipEvent_t end = nullptr; + + ~HipGemmTimer() { + if (end) + (void)hipEventDestroy(end); + if (start) + (void)hipEventDestroy(start); + } + + template + float measure(hipStream_t stream, Launch launch) const { + constexpr int trials = 3; + if (hipEventRecord(start, stream) != hipSuccess) + return 0.f; + for (int i = 0; i < trials; ++i) { + if (launch() != HIPBLAS_STATUS_SUCCESS) + return 0.f; + } + float ms; + return hipEventRecord(end, stream) == hipSuccess && hipEventSynchronize(end) == hipSuccess && + hipEventElapsedTime(&ms, start, end) == hipSuccess + ? ms / trials + : 0.f; + } + }; + + static void tune(int device, hipblasLtHandle_t handle, const HipI8Gemm& gemm, HipLtPlan& plan) { + if (!plan.init(gemm)) + return; + + hipblasLtMatmulPreference_t preference = nullptr; + if (hipblasLtMatmulPreferenceCreate(&preference) != HIPBLAS_STATUS_SUCCESS) + return; + // A zero-workspace kernel can use the existing ggml accumulator buffer. + constexpr uint64_t workspace = 0; + std::array choices{}; + int count = 0; + if (hipblasLtMatmulPreferenceSetAttribute(preference, HIPBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspace, sizeof(workspace)) == HIPBLAS_STATUS_SUCCESS) { + const auto status = hipblasLtMatmulAlgoGetHeuristic(handle, plan.op, plan.a, plan.b, plan.c, plan.c, + preference, int(choices.size()), choices.data(), &count); + if (status != HIPBLAS_STATUS_SUCCESS) + count = 0; + } + hipblasLtMatmulPreferenceDestroy(preference); + if (count == 0) + return; + + HipGemmTimer timer; + if (hipEventCreate(&timer.start) != hipSuccess || hipEventCreate(&timer.end) != hipSuccess) + return; + const auto blas_status = gemm.run_hipblas(); + const bool blas_unsupported = blas_status == HIPBLAS_STATUS_NOT_SUPPORTED || blas_status == HIPBLAS_STATUS_ARCH_MISMATCH; + float baseline = 0.f; + if (blas_status == HIPBLAS_STATUS_SUCCESS) { + baseline = timer.measure(gemm.stream, [&] { return gemm.run_hipblas(); }); + if (baseline <= 0.f) + return; + } else if (!blas_unsupported) { + return; + } + + float best = blas_unsupported ? std::numeric_limits::infinity() : baseline * 0.95f; + for (int i = 0; i < count; ++i) { + const auto& candidate = choices[i]; + if (candidate.state != HIPBLAS_STATUS_SUCCESS || candidate.workspaceSize != 0 || + plan.run(handle, gemm, candidate.algo) != HIPBLAS_STATUS_SUCCESS) + continue; + const float ms = timer.measure(gemm.stream, [&] { return plan.run(handle, gemm, candidate.algo); }); + if (ms > 0.f && ms < best) { + best = ms; + plan.algo = candidate.algo; + plan.use_lt = true; + } + } + if (plan.use_lt) { + if (blas_unsupported) { + LOG_VERBOSE("hipBLASLt INT8 GEMM %d x %d x %d on HIP device %d: %.3f ms (hipBLAS unsupported)", + gemm.n, gemm.rows, gemm.k, device, best); + } else { + LOG_VERBOSE("hipBLASLt INT8 GEMM %d x %d x %d on HIP device %d: %.3f -> %.3f ms", + gemm.n, gemm.rows, gemm.k, device, baseline, best); + } + } + } + + static HipLtDevice& lt_device(int ordinal) { + static std::mutex mutex; + static std::map devices; + std::lock_guard lock(mutex); + return devices.try_emplace(ordinal, ordinal).first->second; + } + + bool hip_i8_gemm_lt(int device, const HipI8Gemm& gemm) { + static const bool disabled = std::getenv("SD_HIP_DISABLE_I8_LT") != nullptr; + if (disabled || gemm.n < 1024 || gemm.rows < 1024 || gemm.k < 1024 || gemm.k > INT32_MAX / 16384 || + (gemm.n | gemm.k) % 16 != 0 || + (reinterpret_cast(gemm.weight) | reinterpret_cast(gemm.input) | + reinterpret_cast(gemm.output)) % 16 != 0) { + return false; + } + auto& state = lt_device(device); + std::lock_guard lock(state.mutex); + const GemmShape shape(gemm.n, gemm.rows, gemm.k); + auto plan = std::find_if(state.plans.begin(), state.plans.end(), [&](const auto& entry) { return entry.shape == shape; }); + const bool needs_tuning = plan == state.plans.end(); + if (!needs_tuning) { + state.plans.splice(state.plans.begin(), state.plans, plan); + if (!plan->use_lt) + return false; + } + + const auto stream_handle = state.handles.find(gemm.stream); + hipblasLtHandle_t handle = stream_handle == state.handles.end() ? nullptr : stream_handle->second; + if (needs_tuning || !handle) { + // Handle creation and tuning must run outside graph capture. + hipStreamCaptureStatus capture; + if (hipStreamIsCapturing(gemm.stream, &capture) != hipSuccess || capture != hipStreamCaptureStatusNone) + return false; + } + if (!handle) { + // A host lock does not wait for work submitted on another stream. + if (hipblasLtCreate(&handle) != HIPBLAS_STATUS_SUCCESS) + return false; + state.handles.emplace(gemm.stream, handle); + } + + if (needs_tuning) { + state.plans.emplace_front(shape); + plan = state.plans.begin(); + tune(device, handle, gemm, *plan); + if (state.plans.size() > 64) + state.plans.pop_back(); + if (!plan->use_lt) + return false; + } + if (plan->run(handle, gemm, plan->algo) == HIPBLAS_STATUS_SUCCESS) + return true; + plan->use_lt = false; + return false; + } +} diff --git a/src/core/cuda/hip_i8_gemm.h b/src/core/cuda/hip_i8_gemm.h new file mode 100644 index 0000000000..1c24b32f3d --- /dev/null +++ b/src/core/cuda/hip_i8_gemm.h @@ -0,0 +1,35 @@ +#ifndef __SD_CORE_CUDA_HIP_I8_GEMM_H__ +#define __SD_CORE_CUDA_HIP_I8_GEMM_H__ + +#include +#include +#include + +namespace sd_hip { + struct HipI8Gemm { + int n, rows, k; + const void* weight; + const int8_t* input; + int32_t* output; + hipblasHandle_t blas; + hipStream_t stream; + + hipblasStatus_t run_hipblas() const { +#if HIP_VERSION >= 60500000 + constexpr auto compute_type = HIPBLAS_COMPUTE_32I; +#else + constexpr auto compute_type = HIPBLAS_R_32I; +#endif + const int32_t alpha = 1, beta = 0; + return hipblasGemmEx(blas, HIPBLAS_OP_T, HIPBLAS_OP_N, n, rows, k, &alpha, + weight, HIPBLAS_R_8I, k, input, HIPBLAS_R_8I, k, + &beta, output, HIPBLAS_R_32I, n, compute_type, HIPBLAS_GEMM_DEFAULT); + } + }; + +#ifdef SD_HIP_I8_LT + bool hip_i8_gemm_lt(int device, const HipI8Gemm& gemm); +#endif +} + +#endif diff --git a/src/core/cuda/i8_gated_residual.cu b/src/core/cuda/i8_gated_residual.cu index 71477d7432..d5764c6065 100644 --- a/src/core/cuda/i8_gated_residual.cu +++ b/src/core/cuda/i8_gated_residual.cu @@ -35,10 +35,12 @@ namespace sd_i8_gated { } }; - template + template struct TileConfig { - using Tile = cutlass::gemm::GemmShape; - using Warp = cutlass::gemm::GemmShape; + using Tile = cutlass::gemm::GemmShape; + using Warp = cutlass::gemm::GemmShape; + static constexpr int kStages = Stages; + static constexpr int kBand = Band; }; enum Tile { @@ -100,7 +102,7 @@ namespace sd_i8_gated { struct KernelTypes { using Tile = typename Config::Tile; using Warp = typename Config::Warp; - using Swizzle = std::conditional_t>; + using Swizzle = std::conditional_t>; using Map = cutlass::epilogue::threadblock::OutputTileThreadLayout; using Col = cutlass::epilogue::threadblock::VisitorColBroadcast>; using Row = cutlass::epilogue::threadblock::VisitorRowBroadcast>; @@ -139,7 +141,7 @@ namespace sd_i8_gated { cutlass::gemm::GemmShape<16, 8, 32>, Epilogue, Swizzle, - 4, + Config::kStages, cutlass::arch::OpMultiplyAddSaturate>::GemmKernel; using Gemm = cutlass::gemm::device::GemmUniversalAdapter; }; @@ -147,6 +149,7 @@ namespace sd_i8_gated { struct DeviceInfo { int ordinal = -1; int major = 0; + int minor = 0; int sms = 0; int threads_per_sm = 0; size_t shared = 0; @@ -164,7 +167,7 @@ namespace sd_i8_gated { int ordinal = query ? query(device) : -1; cudaDeviceProp props = {}; if (ordinal >= 0 && cudaGetDeviceProperties(&props, ordinal) == cudaSuccess) - info = {ordinal, props.major, props.multiProcessorCount, props.maxThreadsPerMultiProcessor, size_t(props.sharedMemPerBlockOptin)}; + info = {ordinal, props.major, props.minor, props.multiProcessorCount, props.maxThreadsPerMultiProcessor, size_t(props.sharedMemPerBlockOptin)}; cache.emplace(device, info); return info; } @@ -286,6 +289,10 @@ namespace sd_i8_gated { return op.rows >= 8192 ? configure(op, plan) : configure(op, plan); } static bool configure(const Operation& op, Plan& plan) { + if (plan.device.major == 8 && plan.device.minor == 9 && op.scale == 1.f && + op.rows >= 4096 && op.rows < 8192 && op.n == 3072 && (op.k == 3072 || op.k == 8192) && + configure, false, false>(op, plan)) + return true; auto try_tile = [&](Tile tile) { switch (tile) { case TILE_128_256: diff --git a/src/core/cuda/i8_gemm.cu b/src/core/cuda/i8_gemm.cu index 7c2702d479..ee681997cb 100644 --- a/src/core/cuda/i8_gemm.cu +++ b/src/core/cuda/i8_gemm.cu @@ -7,6 +7,7 @@ #include #include #include +#include "convrot_kernel.h" #include "i8_gemm.h" namespace sd_i8_cutlass { @@ -33,10 +34,12 @@ namespace sd_i8_cutlass { } }; - template + template struct TileConfig { - using Tile = cutlass::gemm::GemmShape; - using Warp = cutlass::gemm::GemmShape; + using Tile = cutlass::gemm::GemmShape; + using Warp = cutlass::gemm::GemmShape; + static constexpr int kStages = Stages; + static constexpr int kBand = Band; }; enum Tile { @@ -73,6 +76,7 @@ namespace sd_i8_cutlass { struct DeviceInfo { int ordinal = -1; int major = 0; + int minor = 0; int sms = 0; int threads_per_sm = 0; size_t shared = 0; @@ -90,7 +94,7 @@ namespace sd_i8_cutlass { int ordinal = query ? query(device) : -1; cudaDeviceProp props = {}; if (ordinal >= 0 && cudaGetDeviceProperties(&props, ordinal) == cudaSuccess) - info = {ordinal, props.major, props.multiProcessorCount, props.maxThreadsPerMultiProcessor, size_t(props.sharedMemPerBlockOptin)}; + info = {ordinal, props.major, props.minor, props.multiProcessorCount, props.maxThreadsPerMultiProcessor, size_t(props.sharedMemPerBlockOptin)}; cache.emplace(device, info); return info; } @@ -139,13 +143,30 @@ namespace sd_i8_cutlass { }; template - struct SwigluStore : cutlass::epilogue::threadblock::VisitorAuxStore> { + struct GatedStore : cutlass::epilogue::threadblock::VisitorAuxStore> { using Base = cutlass::epilogue::threadblock::VisitorAuxStore>; - using Base::Base; + struct Arguments : Base::Arguments { + bool situ = false; + }; + using Params = Arguments; + + template + static constexpr Params to_underlying_arguments(Shape const&, Arguments const& args, void*) { + return args; + } + + GatedStore() = default; + CUTLASS_HOST_DEVICE GatedStore(Params const& params, typename Base::SharedStorage const& shared) + : Base(params, shared), situ(params.situ) {} + + bool situ = false; + template struct Callbacks : CB { - CUTLASS_DEVICE Callbacks(CB const& base) - : CB(base) {} + bool situ; + + CUTLASS_DEVICE Callbacks(CB const& base, bool situ) + : CB(base), situ(situ) {} CUTLASS_DEVICE void end_step(int step) { auto src = cute::filter(this->tC_rAux); auto coords = cute::filter(this->tC_cAux(cute::_, cute::_, cute::_, step)); @@ -153,9 +174,10 @@ namespace sd_i8_cutlass { for (int i = 0; i < cute::size(src); ++i) { auto coord = coords(i); if (cute::elem_less(coord, this->problem_shape)) { - // Each vector contains two gate/up pairs; only their SwiGLU results are stored. - const float4 a = *reinterpret_cast(&src(i)); - float2 value = make_float2(__fmul_rn(a.x / (1.f + expf(-a.x)), a.y), __fmul_rn(a.z / (1.f + expf(-a.z)), a.w)); + // SwiGLU packs [gate | up] and SiTU packs [up | gate]. + const float4 a = *reinterpret_cast(&src(i)); + const float2 value = situ ? make_float2(sd_cuda::sd_convrot::situ(a.y, a.x), sd_cuda::sd_convrot::situ(a.w, a.z)) + : make_float2(__fmul_rn(a.x / (1.f + expf(-a.x)), a.y), __fmul_rn(a.z / (1.f + expf(-a.z)), a.w)); int64_t n = cute::get<1>(this->problem_shape), m = cute::get<0>(this->problem_shape); int64_t offset = cute::get<0>(coord) * (n / 2) + cute::get<1>(coord) / 2 + cute::get<2>(coord) * m * (n / 2); *reinterpret_cast(this->params_ptr->ptr_aux + offset) = value; @@ -166,7 +188,7 @@ namespace sd_i8_cutlass { template CUTLASS_DEVICE auto get_callbacks(cutlass::gemm::GemmCoord tile, int tid, Shape shape) { auto cb = Base::get_callbacks(tile, tid, shape); - return Callbacks(cb); + return Callbacks(cb, situ); } }; @@ -191,6 +213,7 @@ namespace sd_i8_cutlass { template struct DenseTypes { static constexpr bool kSplit = false; + static constexpr bool kGated = false; static constexpr bool kBias = !PackedW4; using Weight = std::conditional_t; using Tile = typename Config::Tile; @@ -232,22 +255,23 @@ namespace sd_i8_cutlass { using Gemm = cutlass::gemm::device::GemmUniversalAdapter; }; template - struct SwigluTypes { + struct GatedTypes { static constexpr bool kSplit = Split; static constexpr bool kBias = false; + static constexpr bool kGated = true; using Weight = int8_t; using Tile = typename Config::Tile; using Warp = typename Config::Warp; - using Swizzle = std::conditional_t>; + using Swizzle = std::conditional_t>; using Map = cutlass::epilogue::threadblock::OutputTileThreadLayout; using Col = cutlass::epilogue::threadblock::VisitorColBroadcast>; using Row = cutlass::epilogue::threadblock::VisitorRowBroadcast>; using Mul = cutlass::epilogue::threadblock::VisitorCompute; using ScaleX = cutlass::epilogue::threadblock::Sm80EVT; using ScaleW = cutlass::epilogue::threadblock::Sm80EVT; - using Store = SwigluStore; + using Store = GatedStore; using Epilogue = cutlass::epilogue::threadblock::Sm80EVT; - using GemmBase = typename cutlass::gemm::kernel::DefaultGemmUniversal, cutlass::epilogue::thread::LinearCombination, Swizzle, 4, cutlass::arch::OpMultiplyAddSaturate, cutlass::gemm::SharedMemoryClearOption::kNone, false, false, false, cutlass::layout::NoPermute, cutlass::layout::NoPermute, WeightLayout>::GemmKernel; + using GemmBase = typename cutlass::gemm::kernel::DefaultGemmUniversal, cutlass::epilogue::thread::LinearCombination, Swizzle, Config::kStages, cutlass::arch::OpMultiplyAddSaturate, cutlass::gemm::SharedMemoryClearOption::kNone, false, false, false, cutlass::layout::NoPermute, cutlass::layout::NoPermute, WeightLayout>::GemmKernel; using VisitorEpilogue = cutlass::epilogue::threadblock::EpilogueWithVisitorCallbacks; using BaseMma = typename GemmBase::Mma; using SplitMma = cutlass::gemm::threadblock::MmaMultistage< @@ -271,7 +295,7 @@ namespace sd_i8_cutlass { struct Plan { DeviceInfo device; int64_t rows, n, k; - bool swiglu; + SDI8GemmActivation activation; bool packed_w4; bool split; size_t kernel_bytes = 0, scratch_offset = 0; @@ -285,6 +309,8 @@ namespace sd_i8_cutlass { if constexpr (Types::kBias) { // A negative zero default preserves signed-zero outputs. epilogue = {{scales, {bias, -0.f, {}}, {}}, {out, {p.n, cute::_1{}, p.rows * p.n}}}; + } else if constexpr (Types::kGated) { + epilogue = {scales, {{out, {p.n, cute::_1{}, p.rows * p.n}}, p.activation == SDI8GemmActivation::Situ}}; } else { epilogue = {scales, {out, {p.n, cute::_1{}, p.rows * p.n}}}; } @@ -298,7 +324,7 @@ namespace sd_i8_cutlass { 16) return GGML_STATUS_FAILED; auto* paired = static_cast(scratch); - auto args = arguments(p, x, w, xs, p.swiglu ? paired : ws, out, bias); + auto args = arguments(p, x, w, xs, p.activation != SDI8GemmActivation::None ? paired : ws, out, bias); if (Kernel::can_implement(args) != cutlass::Status::kSuccess) return GGML_STATUS_FAILED; if (!p.occupancy) { @@ -318,7 +344,7 @@ namespace sd_i8_cutlass { } if (params.get_workspace_size() > p.kernel_bytes) return GGML_STATUS_FAILED; - if (p.swiglu) { + if (p.activation != SDI8GemmActivation::None) { pair_scales<<>>(ws, up_scales, paired, int(p.n)); if (cudaGetLastError() != cudaSuccess) return GGML_STATUS_FAILED; @@ -346,7 +372,7 @@ namespace sd_i8_cutlass { bytes = std::max(bytes, params.get_workspace_size()); } p.kernel_bytes = bytes; - p.scratch_offset = p.swiglu ? (size_t(p.n) * sizeof(float) + 255) / 256 * 256 : 0; + p.scratch_offset = p.activation != SDI8GemmActivation::None ? (size_t(p.n) * sizeof(float) + 255) / 256 * 256 : 0; p.launch = launch; return true; } @@ -355,12 +381,17 @@ namespace sd_i8_cutlass { if (p.packed_w4) return configure_kernel>(p); if (p.split) - return p.rows >= 8192 ? configure_kernel>(p) : configure_kernel>(p); - if (p.swiglu) - return p.rows >= 8192 ? configure_kernel>(p) : configure_kernel>(p); + return p.rows >= 8192 ? configure_kernel>(p) : configure_kernel>(p); + if (p.activation != SDI8GemmActivation::None) + return p.rows >= 8192 ? configure_kernel>(p) : configure_kernel>(p); return p.rows >= 8192 ? configure_kernel>(p) : configure_kernel>(p); } static bool configure(Plan& p) { + // This SiTU shape fits two resident CTAs per SM with the narrower, three-stage tile on Ada. + if (p.device.major == 8 && p.device.minor == 9 && p.activation == SDI8GemmActivation::Situ && !p.split && + p.rows >= 4096 && p.rows < 8192 && p.n == 16384 && p.k == 3072 && + configure_kernel, false>>(p)) + return true; auto try_tile = [&](Tile tile) { switch (tile) { case TILE_128_256: @@ -384,22 +415,23 @@ namespace sd_i8_cutlass { } } struct SDI8CudaGemmPlan : sd_i8_cutlass::Plan {}; -static SDI8CudaGemmPlan* prepare_gemm(ggml_backend_dev_t device, int64_t rows, int64_t n, int64_t k, bool swiglu, size_t* workspace, bool packed_w4, bool split) { +static SDI8CudaGemmPlan* prepare_gemm(ggml_backend_dev_t device, int64_t rows, int64_t n, int64_t k, SDI8GemmActivation activation, size_t* workspace, bool packed_w4, bool split) { #ifdef SD_CUDA_FORCE_CUBLAS return nullptr; #else - const auto info = sd_i8_cutlass::device_info(device); - if ((packed_w4 && (swiglu || k > INT_MAX / 1024)) || info.major < 8 || rows < (sd_i8_cutlass::small_rows_enabled(rows, n, k) ? 1 : 128) || - rows > INT_MAX || n < 1 || n > INT_MAX || k < 1 || k > INT_MAX || k % 64 || n % (swiglu ? 256 : 128)) + const auto info = sd_i8_cutlass::device_info(device); + const bool gated = activation != SDI8GemmActivation::None; + if ((packed_w4 && (gated || k > INT_MAX / 1024)) || info.major < 8 || rows < (sd_i8_cutlass::small_rows_enabled(rows, n, k) ? 1 : 128) || + rows > INT_MAX || n < 1 || n > INT_MAX || k < 1 || k > INT_MAX || k % 64 || n % (gated ? 256 : 128)) return nullptr; - auto* p = new SDI8CudaGemmPlan; - p->device = info; - p->rows = rows; - p->n = n; - p->k = k; - p->swiglu = swiglu; - p->packed_w4 = packed_w4; - p->split = split; + auto* p = new SDI8CudaGemmPlan; + p->device = info; + p->rows = rows; + p->n = n; + p->k = k; + p->activation = activation; + p->packed_w4 = packed_w4; + p->split = split; if (!sd_i8_cutlass::configure(*p)) { delete p; return nullptr; @@ -408,16 +440,16 @@ static SDI8CudaGemmPlan* prepare_gemm(ggml_backend_dev_t device, int64_t rows, i return p; #endif } -SDI8CudaGemmPlan* sd_cuda_i8_gemm_prepare(ggml_backend_dev_t device, int64_t rows, int64_t n, int64_t k, bool swiglu, size_t* workspace, bool packed_w4) { - return prepare_gemm(device, rows, n, k, swiglu, workspace, packed_w4, false); +SDI8CudaGemmPlan* sd_cuda_i8_gemm_prepare(ggml_backend_dev_t device, int64_t rows, int64_t n, int64_t k, SDI8GemmActivation activation, size_t* workspace, bool packed_w4) { + return prepare_gemm(device, rows, n, k, activation, workspace, packed_w4, false); } SDI8CudaGemmPlan* sd_cuda_i8_gemm_prepare_split(ggml_backend_dev_t device, int64_t rows, int64_t n, int64_t k, size_t* workspace) { if (n > INT_MAX / 2 || k > INT_MAX / 16384) return nullptr; - return prepare_gemm(device, rows, 2 * n, k, true, workspace, false, true); + return prepare_gemm(device, rows, 2 * n, k, SDI8GemmActivation::Swiglu, workspace, false, true); } ggml_status sd_cuda_i8_gemm_execute(SDI8CudaGemmPlan* plan, const int8_t* x, const int8_t* w, const float* xs, const float* ws, float* out, void* scratch, cudaStream_t stream, const float* bias) { - if (bias && (plan->swiglu || plan->packed_w4)) + if (bias && (plan->activation != SDI8GemmActivation::None || plan->packed_w4)) return GGML_STATUS_FAILED; return plan->launch(*plan, x, w, xs, ws, out, scratch, stream, nullptr, nullptr, bias); } diff --git a/src/core/cuda/i8_gemm.h b/src/core/cuda/i8_gemm.h index 756238ef11..db9bfe75ba 100644 --- a/src/core/cuda/i8_gemm.h +++ b/src/core/cuda/i8_gemm.h @@ -3,7 +3,10 @@ #include #include "ggml-ext.h" struct SDI8CudaGemmPlan; -SDI8CudaGemmPlan* sd_cuda_i8_gemm_prepare(ggml_backend_dev_t device, int64_t rows, int64_t n, int64_t k, bool swiglu, size_t* workspace, bool packed_w4 = false); +enum class SDI8GemmActivation { None, + Swiglu, + Situ }; +SDI8CudaGemmPlan* sd_cuda_i8_gemm_prepare(ggml_backend_dev_t device, int64_t rows, int64_t n, int64_t k, SDI8GemmActivation activation, size_t* workspace, bool packed_w4 = false); ggml_status sd_cuda_i8_gemm_execute(SDI8CudaGemmPlan* plan, const int8_t* x, const int8_t* w, const float* xs, const float* ws, float* out, void* scratch, cudaStream_t stream, const float* bias = nullptr); // n is the width of each branch; gate/up weights and scales remain separate. SDI8CudaGemmPlan* sd_cuda_i8_gemm_prepare_split(ggml_backend_dev_t device, int64_t rows, int64_t n, int64_t k, size_t* workspace); diff --git a/src/core/cuda/int8.cu b/src/core/cuda/int8.cu index 8e92468482..46afe433a4 100644 --- a/src/core/cuda/int8.cu +++ b/src/core/cuda/int8.cu @@ -9,7 +9,7 @@ struct SDI8CudaGemmPlan; #include "runtime.h" #if defined(SD_CUDA_VENDOR_HIP) -#include +#include "hip_i8_gemm.h" #elif !defined(SD_CUDA_VENDOR_MUSA) #include #endif @@ -164,8 +164,9 @@ namespace SD_CUDA_NAMESPACE { } } + template static __global__ void convrot_group_amax_i8_cuda( - const float* src, + Input input, float* partial_max, int64_t k, int64_t groups, @@ -180,11 +181,10 @@ namespace SD_CUDA_NAMESPACE { __shared__ float maxima[2]; const int64_t row = index / groups; const int64_t group = index - row * groups; - const float* input = src + row * k + group * 256; - values[tid] = input[tid] * (1.0f / 16.0f); - values[tid + 64] = input[tid + 64] * (1.0f / 16.0f); - values[tid + 128] = input[tid + 128] * (1.0f / 16.0f); - values[tid + 192] = input[tid + 192] * (1.0f / 16.0f); +#pragma unroll + for (int i = tid; i < 256; i += 64) { + values[i] = input.load(row, group * 256 + i, k) * (1.0f / 16.0f); + } __syncthreads(); #pragma unroll @@ -205,6 +205,9 @@ namespace SD_CUDA_NAMESPACE { __syncthreads(); } +#pragma unroll + for (int i = tid; i < 256; i += 64) + input.store(row * k + group * 256 + i, values[i]); float value = fmaxf(fabsf(values[tid]), fabsf(values[tid + 64])); value = fmaxf(value, fabsf(values[tid + 128])); value = fmaxf(value, fabsf(values[tid + 192])); @@ -218,6 +221,39 @@ namespace SD_CUDA_NAMESPACE { } } + struct ConvrotInput { + const float* values; + + __device__ float load(int64_t row, int64_t col, int64_t width) const { + return values[row * width + col]; + } + __device__ void store(int64_t, float) const {} + }; + +#if defined(SD_CUDA_VENDOR_HIP) + struct SplitSwiGLUInput { + const int32_t* gate; + int32_t* up; + const float* row_scales; + const float* gate_scales; + const float* up_scales; + + __device__ float load(int64_t row, int64_t col, int64_t width) const { + const int64_t index = row * width + col; + const float scale = row_scales[row]; + const float g = __fmul_rn(__fmul_rn(float(gate[index]), scale), gate_scales[col]); + const float u = __fmul_rn(__fmul_rn(float(up[index]), scale), up_scales[col]); + const float silu = __fdiv_rn(g, __fadd_rn(1.0f, __ocml_exp_f32(-g))); + return __fmul_rn(silu, u); + } + + __device__ void store(int64_t index, float value) const { + // Reuse the up accumulator after the block has read its entire tile. + up[index] = __float_as_int(value); + } + }; +#endif + static __global__ void reduce_convrot_row_amax_i8_cuda( const float* partial_max, float* scales, @@ -247,6 +283,7 @@ namespace SD_CUDA_NAMESPACE { } } + template static __global__ void convrot_group_quantize_i8_cuda( const float* src, int8_t* dst, @@ -266,28 +303,29 @@ namespace SD_CUDA_NAMESPACE { const int64_t group = index - row * groups; const float* input = src + row * k + group * 256; int8_t* output = dst + row * dst_stride + group * 256; - values[tid] = input[tid] * (1.0f / 16.0f); - values[tid + 64] = input[tid + 64] * (1.0f / 16.0f); - values[tid + 128] = input[tid + 128] * (1.0f / 16.0f); - values[tid + 192] = input[tid + 192] * (1.0f / 16.0f); - __syncthreads(); - + if constexpr (!pre_rotated) { #pragma unroll - for (int stride = 1; stride < 256; stride *= 4) { - const int base = (tid / stride) * 4 * stride + tid % stride; - const int i0 = base; - const int i1 = i0 + stride; - const int i2 = i1 + stride; - const int i3 = i2 + stride; - const float a = values[i0]; - const float b = values[i1]; - const float c = values[i2]; - const float d = values[i3]; - values[i0] = a + b + c - d; - values[i1] = a + b - c + d; - values[i2] = a - b + c + d; - values[i3] = -a + b + c + d; + for (int i = tid; i < 256; i += 64) + values[i] = input[i] * (1.0f / 16.0f); __syncthreads(); + +#pragma unroll + for (int stride = 1; stride < 256; stride *= 4) { + const int base = (tid / stride) * 4 * stride + tid % stride; + const int i0 = base; + const int i1 = i0 + stride; + const int i2 = i1 + stride; + const int i3 = i2 + stride; + const float a = values[i0]; + const float b = values[i1]; + const float c = values[i2]; + const float d = values[i3]; + values[i0] = a + b + c - d; + values[i1] = a + b - c + d; + values[i2] = a - b + c + d; + values[i3] = -a + b + c + d; + __syncthreads(); + } } const float row_scale = scales[row]; @@ -295,7 +333,13 @@ namespace SD_CUDA_NAMESPACE { const int indices[4] = {tid, tid + 64, tid + 128, tid + 192}; #pragma unroll for (int i = 0; i < 4; ++i) { - int value = row_scale == 0.0f ? 0 : __float2int_rn(values[indices[i]] * inv_scale); + float rotated; + if constexpr (pre_rotated) { + rotated = __int_as_float(reinterpret_cast(input)[indices[i]]); + } else { + rotated = values[indices[i]]; + } + int value = row_scale == 0.0f ? 0 : __float2int_rn(rotated * inv_scale); value = max(-127, min(127, value)); output[indices[i]] = (int8_t)value; } @@ -362,6 +406,39 @@ namespace SD_CUDA_NAMESPACE { dst[index] = value; } +#if defined(SD_CUDA_VENDOR_HIP) + static __device__ __forceinline__ float dequantize_gated_residual(int32_t accum, float row_scale, float weight_scale, float gate, float residual) { + const float value = __fmul_rn(__fmul_rn(float(accum), row_scale), weight_scale); + return __fadd_rn(__fmul_rn(value, gate), residual); + } + + static __global__ void dequantize_gated_residual_vec4( + const int4* accum, + const float* scales, + const float4* weight_scales, + const float4* gates, + const float4* residual, + float4* dst, + int64_t n4, + int64_t rows) { + const int64_t col = int64_t(blockIdx.x) * blockDim.x + threadIdx.x; + const int64_t row = blockIdx.y + int64_t(blockIdx.z) * 65535; + if (col >= n4 || row >= rows) + return; + + const int64_t index = row * n4 + col; + const int4 a = accum[index]; + const float scale = scales[row]; + const float4 w = weight_scales[col]; + const float4 g = gates[col]; + const float4 r = residual[index]; + dst[index] = make_float4(dequantize_gated_residual(a.x, scale, w.x, g.x, r.x), + dequantize_gated_residual(a.y, scale, w.y, g.y, r.y), + dequantize_gated_residual(a.z, scale, w.z, g.z, r.z), + dequantize_gated_residual(a.w, scale, w.w, g.w, r.w)); + } +#endif + static bool aligned(const ggml_tensor* tensor, size_t alignment = 4) { if (!tensor) return true; @@ -438,7 +515,8 @@ namespace SD_CUDA_NAMESPACE { } enum Kind { MAT, QUANTIZE, - SWIGLU, + GATED, + GATED_RESIDUAL, SWIGLU_QUANTIZE, CONCAT }; static bool fusion_disabled() { @@ -452,6 +530,52 @@ namespace SD_CUDA_NAMESPACE { (op.input->type == GGML_TYPE_F32 || (aligned(op.input, 16) && aligned(op.input_scale, 16))); } #endif + static const ggml_tensor* residual_input(const ggml_tensor* mul, const ggml_tensor* add) { + return add->src[0] == mul ? add->src[1] : add->src[1] == mul ? add->src[0] + : nullptr; + } + +#if defined(SD_CUDA_VENDOR_HIP) + static bool match_gated_residual(const ggml_tensor* mat, const ggml_tensor* mul, const ggml_tensor* add, const SDI8Operation& op) { + if (mul->op != GGML_OP_MUL || mul->src[0] != mat || add->op != GGML_OP_ADD || !aligned(op.weight_scale, 16)) + return false; + const auto* gate = mul->src[1]; + const auto* residual = residual_input(mul, add); + if (!gate || gate == mat || gate->ne[0] != mat->ne[0] || ggml_nelements(gate) != mat->ne[0] || + !residual || residual == mat || residual == mul) + return false; + for (const auto* tensor : {mat, mul, add, gate, residual}) { + if (tensor->type != GGML_TYPE_F32 || !ggml_is_contiguous(tensor) || !aligned(tensor, 16) || + (tensor != gate && !ggml_are_same_shape(mat, tensor))) + return false; + } + return true; + } + + static void optimize_gated_residual(ggml_backend_dev_t device, ggml_cgraph* graph, void*) { + if (fusion_disabled()) + return; + const int total = ggml_graph_n_nodes(graph); + for (int i = 2; i + 1 < total; ++i) { + const auto* mul = ggml_graph_node(graph, i); + if (mul->op != GGML_OP_MUL) + continue; + const auto* mat = mul->src[0]; + const auto* add = ggml_graph_node(graph, i + 1); + SDI8Operation op; + if (!sd_ggml_i8_describe(mat, &op) || !op.weight_scale || op.bias || + !match_gated_residual(mat, mul, add, op) || !supports(device, mat, nullptr)) + continue; + for (int j = i - 2; j >= 0 && j >= i - 32; --j) { + if (ggml_graph_node(graph, j) == mat) { + ggml_ext_graph_delay(graph, j, 1, i); + break; + } + } + } + } +#endif + static int match(ggml_backend_dev_t device, const ggml_cgraph* graph, int first, void*) { auto* g = const_cast(graph); if (fusion_disabled() || first + 1 >= ggml_graph_n_nodes(g)) @@ -461,14 +585,36 @@ namespace SD_CUDA_NAMESPACE { if (!supports(device, mat, nullptr) || !sd_ggml_i8_describe(mat, &op) || !op.weight_scale || op.bias) return 0; const auto* next = ggml_graph_node(g, first + 1); +#if defined(SD_CUDA_VENDOR_HIP) + if (first + 2 < ggml_graph_n_nodes(g) && match_gated_residual(mat, next, ggml_graph_node(g, first + 2), op)) + return 3; +#endif if (next->op == GGML_OP_CONCAT && next->src[1] == mat && next->src[0] != mat && next->op_params[0] == 1 && next->type == GGML_TYPE_F32 && mat->ne[2] == 1 && mat->ne[3] == 1 && next->ne[2] == 1 && next->ne[3] == 1 && ggml_is_contiguous(next) && ggml_is_contiguous(next->src[0])) return 2; - if (next->op != GGML_OP_GLU || ggml_get_glu_op(next) != GGML_GLU_OP_SWIGLU || next->op_params[1] != 0 || + if (next->op != GGML_OP_GLU || next->src[0] != mat || next->src[1] || next->type != GGML_TYPE_F32 || !ggml_is_contiguous(next) || ggml_nelements(next) > int64_t(INT_MAX) * 256) return 0; + const auto activation = ggml_get_glu_op(next); + const bool situ = activation == GGML_GLU_OP_SITU && next->op_params[1] != 0; + if (situ) { +#ifdef SD_CUDA_I8_CUTLASS + if (!next->data && !next->buffer && !next->view_src && cutlass_compatible(op, next)) { + size_t workspace = 0; + auto* gemm = sd_cuda_i8_gemm_prepare(device, ggml_nrows(mat), mat->ne[0], op.weight->ne[0], SDI8GemmActivation::Situ, &workspace); + const bool supported = gemm != nullptr; + sd_cuda_i8_gemm_free(gemm); + return supported ? 2 : 0; + } +#endif +#if !defined(SD_CUDA_VENDOR_HIP) + return 0; +#endif + } + if (!situ && (activation != GGML_GLU_OP_SWIGLU || next->op_params[1] != 0)) + return 0; if (first + 2 < ggml_graph_n_nodes(g)) { const auto* quant = ggml_graph_node(g, first + 2); int group; @@ -477,7 +623,7 @@ namespace SD_CUDA_NAMESPACE { #ifdef SD_CUDA_I8_CUTLASS if (!next->data && !next->buffer && !next->view_src && cutlass_compatible(op, next)) { size_t workspace = 0; - auto* gemm = sd_cuda_i8_gemm_prepare(device, ggml_nrows(mat), mat->ne[0], op.weight->ne[0], true, &workspace); + auto* gemm = sd_cuda_i8_gemm_prepare(device, ggml_nrows(mat), mat->ne[0], op.weight->ne[0], SDI8GemmActivation::Swiglu, &workspace); const bool fused_epilogue = gemm != nullptr; sd_cuda_i8_gemm_free(gemm); // ConvRot is a separate launch either way. Keep its input in the @@ -489,7 +635,7 @@ namespace SD_CUDA_NAMESPACE { return 3; } } - return 2; + return situ ? 0 : 2; } struct Plan { ggml_ext_gpu_device_info device; @@ -498,7 +644,8 @@ namespace SD_CUDA_NAMESPACE { int64_t k, n, rows, padded; size_t quant, scales, accum, partial, output, gemm_offset; SDI8CudaGemmPlan* gemm = nullptr; - bool fast_swiglu = false; + bool fused_activation = false; + bool situ = false; }; static size_t reserve(size_t& total, size_t bytes) { const size_t offset = total; @@ -513,15 +660,16 @@ namespace SD_CUDA_NAMESPACE { *state = p; auto device = ggml_backend_get_device(backend); p->device = cuda_device_info(device); - p->kind = !mat ? QUANTIZE : count == 3 ? SWIGLU_QUANTIZE + p->kind = !mat ? QUANTIZE : count == 3 ? (dst->op == GGML_OP_ADD ? GATED_RESIDUAL : SWIGLU_QUANTIZE) : count == 1 ? MAT : dst->op == GGML_OP_CONCAT ? CONCAT - : SWIGLU; + : GATED; p->k = mat ? op.weight->ne[0] : dst->ne[0]; p->n = mat ? nodes[0]->ne[0] : 0; p->rows = mat ? ggml_nrows(nodes[0]) : ggml_nrows(dst->src[0]); p->padded = (p->rows + 3) / 4 * 4; p->group = mat ? op.group_size : 0; + p->situ = (p->kind == GATED || p->kind == SWIGLU_QUANTIZE) && ggml_get_glu_op(nodes[1]) == GGML_GLU_OP_SITU; if (!mat) sd_ggml_convrot_describe(dst, &p->group); size_t total = 0; @@ -533,17 +681,18 @@ namespace SD_CUDA_NAMESPACE { // Retain cuBLAS when I32 accumulation can overflow. if (cutlass_compatible(op, dst) && (!op.bias || p->kind == MAT) && (p->kind != CONCAT || ggml_nbytes(dst->src[0]) % 16 == 0)) { - if (p->kind == SWIGLU || p->kind == SWIGLU_QUANTIZE) { - p->gemm = sd_cuda_i8_gemm_prepare(device, p->rows, p->n, p->k, true, &gemm_bytes); - p->fast_swiglu = p->gemm != nullptr; + if (p->kind == GATED || p->kind == SWIGLU_QUANTIZE) { + const auto activation = p->situ ? SDI8GemmActivation::Situ : SDI8GemmActivation::Swiglu; + p->gemm = sd_cuda_i8_gemm_prepare(device, p->rows, p->n, p->k, activation, &gemm_bytes); + p->fused_activation = p->gemm != nullptr; } if (!p->gemm) - p->gemm = sd_cuda_i8_gemm_prepare(device, p->rows, p->n, p->k, false, &gemm_bytes); + p->gemm = sd_cuda_i8_gemm_prepare(device, p->rows, p->n, p->k, SDI8GemmActivation::None, &gemm_bytes); } p->gemm_offset = reserve(total, gemm_bytes); #endif - const bool intermediate = !p->gemm || p->kind == SWIGLU_QUANTIZE || (p->kind == SWIGLU && !p->fast_swiglu); - p->accum = reserve(total, intermediate ? size_t(p->fast_swiglu ? p->n / 2 : p->n) * p->padded * sizeof(float) : 0); + const bool intermediate = !p->gemm || p->kind == SWIGLU_QUANTIZE || (p->kind == GATED && !p->fused_activation); + p->accum = reserve(total, intermediate ? size_t(p->fused_activation ? p->n / 2 : p->n) * p->padded * sizeof(float) : 0); } const bool large_rotation = (!mat || (op.input->type == GGML_TYPE_F32 && op.group_size == 256)) && size_t(p->k) * sizeof(float) + 128 > p->device.shared_memory; p->partial = reserve(total, p->group == 64 ? size_t(p->rows) * (p->k / 64) * sizeof(float) : large_rotation ? size_t(p->rows) * (p->k / 256) * sizeof(float) @@ -570,7 +719,7 @@ namespace SD_CUDA_NAMESPACE { const auto x = reinterpret_cast(a->data), y = reinterpret_cast(b->data); return x <= y ? y - x < ggml_nbytes(a) : x - y < ggml_nbytes(b); } -#ifdef SD_CUDA_I8_CUTLASS +#if defined(SD_CUDA_I8_CUTLASS) || defined(SD_CUDA_VENDOR_HIP) static bool split_describe(ggml_backend_dev_t device, ggml_tensor* const* nodes, SDI8Operation& gate, SDI8Operation& up) { const auto* glu = nodes[2]; if (glu->op != GGML_OP_GLU || ggml_get_glu_op(glu) != GGML_GLU_OP_SWIGLU || glu->op_params[1] != 0 || @@ -591,6 +740,8 @@ namespace SD_CUDA_NAMESPACE { return false; return true; } +#endif +#ifdef SD_CUDA_I8_CUTLASS static int split_match(ggml_backend_dev_t device, const ggml_cgraph* graph, int first, void*) { auto* g = const_cast(graph); if (fusion_disabled() || first + 2 >= ggml_graph_n_nodes(g)) @@ -698,14 +849,14 @@ namespace SD_CUDA_NAMESPACE { quantize_convrot<<>>(src, dst, scales, p.k, p.rows, p.k); } else { const int64_t groups = p.k / 256, total_groups = p.rows * groups; - convrot_group_amax_i8_cuda<<>>(src, partial, p.k, groups, total_groups); + convrot_group_amax_i8_cuda<<>>(ConvrotInput{src}, partial, p.k, groups, total_groups); reduce_convrot_row_amax_i8_cuda<<>>(partial, scales, groups, p.rows); - convrot_group_quantize_i8_cuda<<>>(src, dst, scales, p.k, groups, total_groups, p.k); + convrot_group_quantize_i8_cuda<><<>>(src, dst, scales, p.k, groups, total_groups, p.k); } return cuda_last_error(); } template - static __global__ void swiglu(const float* src, const float* xs, const float* ws, float* dst, int64_t n, int64_t rows) { + static __global__ void gated_activation(const float* src, const float* xs, const float* ws, float* dst, int64_t n, int64_t rows, bool situ) { const int64_t i = int64_t(blockIdx.x) * blockDim.x + threadIdx.x; const int64_t half = n / 2; if (i >= rows * half) @@ -721,12 +872,12 @@ namespace SD_CUDA_NAMESPACE { gate = src[row * n + col]; up = src[row * n + half + col]; } - dst[i] = __fmul_rn(gate / (1.f + expf(-gate)), up); + dst[i] = situ ? sd_convrot::situ(up, gate) : __fmul_rn(gate / (1.f + expf(-gate)), up); } - template + template static CUDAError quantize_epilogue(const Plan& p, const float* src, const float* xs, const float* ws, int8_t* output, CUDAStream stream) { const int64_t hidden = p.n / 2; - auto kernel = sd_convrot::quantize; + auto kernel = sd_convrot::quantize; static std::mutex mutex; static std::unordered_map initialized; { @@ -743,12 +894,17 @@ namespace SD_CUDA_NAMESPACE { if (error != cuda_success) return error; } - const int threads = Rotated && !Swiglu && !Dequantize ? (hidden <= 1024 ? 128 : hidden <= 8192 ? 256 - : 1024) - : 256; - kernel<<>>(src, output, - reinterpret_cast(output + hidden * p.padded), hidden, p.rows, hidden, - Swiglu ? src + hidden : nullptr, Swiglu ? p.n : hidden, Swiglu ? p.n : hidden, xs, ws, nullptr, 0.f, 1.f, 0.f, nullptr, nullptr); + int threads = Situ && hidden >= 8192 ? 512 : 256; + if constexpr (Rotated && !Swiglu && !Dequantize) { + threads = hidden <= 1024 ? 128 : hidden <= 8192 ? 256 + : 1024; + } + const auto* gate = Situ ? src + hidden : src; + const auto* up = Swiglu ? src + (Situ ? 0 : hidden) : nullptr; + const auto stride = Swiglu ? p.n : hidden; + kernel<<>>( + gate, output, reinterpret_cast(output + hidden * p.padded), hidden, p.rows, hidden, + up, stride, stride, xs, ws, nullptr, 0.f, 1.f, 0.f, nullptr, nullptr); return cuda_last_error(); } static ggml_status execute(const ggml_ext_execution_context* execution, ggml_tensor* const* nodes, int count, void* state, void*) { @@ -762,14 +918,15 @@ namespace SD_CUDA_NAMESPACE { auto scratch_at = [scratch](size_t offset) { return scratch ? scratch + offset : nullptr; }; auto* dst = nodes[count - 1]; SDI8Operation op; - const bool mat = sd_ggml_i8_describe(nodes[0], &op); - bool alias = false; + const bool mat = sd_ggml_i8_describe(nodes[0], &op); + const auto* residual = p.kind == GATED_RESIDUAL ? residual_input(nodes[1], dst) : nullptr; + bool alias = false; for (int i = 0; i < count; ++i) for (auto* input : nodes[i]->src) { bool internal = false; for (int j = 0; j < count; ++j) internal |= input == nodes[j]; - if (!internal) + if (!internal && !(residual && i == count - 1 && input == residual && dst->data == residual->data)) alias |= overlaps(dst, input); } // CUTLASS quantization epilogues read only the workspace intermediate after GEMM completes. @@ -815,7 +972,7 @@ namespace SD_CUDA_NAMESPACE { const size_t prefix = p.kind == CONCAT ? ggml_nbytes(dst->src[0]) : 0; auto* gemm_output = reinterpret_cast(static_cast(output) + prefix); auto* accum = reinterpret_cast(scratch_at(p.accum)); - const bool intermediate = p.kind == SWIGLU_QUANTIZE || (p.kind == SWIGLU && !p.fast_swiglu); + const bool intermediate = p.kind == SWIGLU_QUANTIZE || (p.kind == GATED && !p.fused_activation); #ifdef SD_CUDA_I8_CUTLASS if (p.gemm) { auto status = sd_cuda_i8_gemm_execute(p.gemm, quant, static_cast(op.weight->data), scales, ws, @@ -832,19 +989,16 @@ namespace SD_CUDA_NAMESPACE { #if defined(SD_CUDA_VENDOR_MUSA) return GGML_STATUS_FAILED; #else - const int32_t alpha = 1, beta = 0; #if defined(SD_CUDA_VENDOR_HIP) -#if HIP_VERSION >= 60500000 - constexpr auto compute_type = HIPBLAS_COMPUTE_32I; + const HipI8Gemm gemm{int(p.n), int(p.padded), int(p.k), op.weight->data, quant, accum, + static_cast(handle), stream}; +#ifdef SD_HIP_I8_LT + if (!hip_i8_gemm_lt(context.device, gemm) && gemm.run_hipblas() != HIPBLAS_STATUS_SUCCESS) #else - constexpr auto compute_type = HIPBLAS_R_32I; + if (gemm.run_hipblas() != HIPBLAS_STATUS_SUCCESS) #endif - const auto status = hipblasGemmEx(static_cast(handle), HIPBLAS_OP_T, HIPBLAS_OP_N, - int(p.n), int(p.padded), int(p.k), &alpha, - op.weight->data, HIPBLAS_R_8I, int(p.k), quant, HIPBLAS_R_8I, int(p.k), - &beta, accum, HIPBLAS_R_32I, int(p.n), compute_type, HIPBLAS_GEMM_DEFAULT); - if (status != HIPBLAS_STATUS_SUCCESS) #else + const int32_t alpha = 1, beta = 0; const auto status = cublasGemmEx(static_cast(handle), CUBLAS_OP_T, CUBLAS_OP_N, int(p.n), int(p.padded), int(p.k), &alpha, op.weight->data, CUDA_R_8I, int(p.k), quant, CUDA_R_8I, int(p.k), @@ -856,22 +1010,34 @@ namespace SD_CUDA_NAMESPACE { } const auto* values = reinterpret_cast(accum); if (p.kind == SWIGLU_QUANTIZE) { - if (p.fast_swiglu) + if (p.fused_activation) error = quantize_epilogue(p, values, nullptr, nullptr, static_cast(output), stream); else if (p.gemm) error = quantize_epilogue(p, values, nullptr, nullptr, static_cast(output), stream); +#if defined(SD_CUDA_VENDOR_HIP) + else if (p.situ) + error = quantize_epilogue(p, values, scales, ws, static_cast(output), stream); +#endif else error = quantize_epilogue(p, values, scales, ws, static_cast(output), stream); - } else if (p.kind == SWIGLU && !p.fast_swiglu) { + } else if (p.kind == GATED && !p.fused_activation) { const unsigned blocks = unsigned((p.rows * (p.n / 2) + 255) / 256); if (p.gemm) - swiglu<<>>(values, scales, ws, static_cast(output), p.n, p.rows); + gated_activation<<>>(values, scales, ws, static_cast(output), p.n, p.rows, p.situ); else - swiglu<<>>(values, scales, ws, static_cast(output), p.n, p.rows); + gated_activation<<>>(values, scales, ws, static_cast(output), p.n, p.rows, p.situ); error = cuda_last_error(); } else if (!p.gemm) { const unsigned gy = unsigned(std::min(p.rows, 65535)), gz = unsigned((p.rows + 65534) / 65535); - if ((reinterpret_cast(ws) | reinterpret_cast(bias) | reinterpret_cast(gemm_output)) % 16 == 0) { +#if defined(SD_CUDA_VENDOR_HIP) + if (p.kind == GATED_RESIDUAL) { + dequantize_gated_residual_vec4<<>>( + reinterpret_cast(accum), scales, reinterpret_cast(ws), + static_cast(nodes[1]->src[1]->data), static_cast(residual->data), + reinterpret_cast(gemm_output), p.n / 4, p.rows); + } else +#endif + if ((reinterpret_cast(ws) | reinterpret_cast(bias) | reinterpret_cast(gemm_output)) % 16 == 0) { dequantize_i32_rows_vec4_cuda<<>>( reinterpret_cast(accum), scales, reinterpret_cast(ws), reinterpret_cast(bias), reinterpret_cast(gemm_output), p.n / 4, p.rows); } else { @@ -887,6 +1053,108 @@ namespace SD_CUDA_NAMESPACE { return error == cuda_success ? GGML_STATUS_SUCCESS : GGML_STATUS_FAILED; } +#if defined(SD_CUDA_VENDOR_HIP) + struct SplitConvrotPlan { + int64_t k, n, rows, padded; + size_t partial, output; + }; + + static int split_convrot_match(ggml_backend_dev_t device, const ggml_cgraph* graph, int first, void*) { + auto* g = const_cast(graph); + if (fusion_disabled() || first + 3 >= ggml_graph_n_nodes(g)) + return 0; + ggml_tensor* nodes[] = {ggml_graph_node(g, first), ggml_graph_node(g, first + 1), ggml_graph_node(g, first + 2), ggml_graph_node(g, first + 3)}; + SDI8Operation gate, up; + int group; + if (!split_describe(device, nodes, gate, up) || !gate.packed || + nodes[3]->src[0] != nodes[2] || !sd_ggml_convrot_describe(nodes[3], &group) || group != 256 || + !convrot_supports(device, nodes[3], nullptr) || + size_t(nodes[2]->ne[0]) * sizeof(float) + 128 <= cuda_device_info(device).shared_memory) + return 0; + return 4; + } + + static ggml_status split_convrot_prepare(ggml_backend_t, ggml_tensor* const* nodes, int, void** state, size_t* workspace, void*) { + SDI8Operation op; + if (!sd_ggml_i8_describe(nodes[0], &op)) + return GGML_STATUS_FAILED; + auto* p = new SplitConvrotPlan{}; + *state = p; + p->rows = ggml_nrows(nodes[2]); + p->padded = (p->rows + 3) / 4 * 4; + p->n = nodes[2]->ne[0]; + p->k = op.weight->ne[0]; + size_t total = 2 * size_t(p->n) * p->padded * sizeof(int32_t); + p->partial = reserve(total, size_t(p->rows) * (p->n / 256) * sizeof(float)); + const auto* dst = nodes[3]; + p->output = dst->data || dst->buffer || dst->view_src ? reserve(total, ggml_nbytes(dst)) : SIZE_MAX; + *workspace = total; + return GGML_STATUS_SUCCESS; + } + + static ggml_status split_convrot_execute(const ggml_ext_execution_context* execution, ggml_tensor* const* nodes, int, void* state, void*) { + const auto& p = *static_cast(state); + auto* dst = nodes[3]; + SDI8Operation gate, up; + if (!sd_ggml_i8_describe(nodes[2]->src[0], &gate) || !sd_ggml_i8_describe(nodes[2]->src[1], &up)) + return GGML_STATUS_FAILED; + ggml_ext_gpu_context context; + if (!cuda_context(execution->backend, &context)) + return GGML_STATUS_FAILED; + auto reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(execution->backend)); + auto blas_query = reinterpret_cast(ggml_backend_reg_get_proc_address(reg, "ggml_ext_gpu_blas_v1")); + auto handle = blas_query ? static_cast(blas_query(execution->backend)) : nullptr; + if (!handle) + return GGML_STATUS_FAILED; + + auto stream = static_cast(context.stream); + auto* scratch = static_cast(ggml_backend_buffer_get_base(execution->workspace)); + auto* gate_accum = reinterpret_cast(scratch); + auto* up_accum = gate_accum + p.n * p.padded; + const auto* input = static_cast(gate.input->data); + auto gemm = [&](const SDI8Operation& op, int32_t* output) { + const HipI8Gemm g{int(p.n), int(p.padded), int(p.k), op.weight->data, input, output, handle, stream}; +#ifdef SD_HIP_I8_LT + if (hip_i8_gemm_lt(context.device, g)) + return true; +#endif + return g.run_hipblas() == HIPBLAS_STATUS_SUCCESS; + }; + if (!gemm(gate, gate_accum) || !gemm(up, up_accum)) + return GGML_STATUS_FAILED; + + bool alias = false; + for (const auto* tensor : {gate.input, gate.weight, up.weight, gate.weight_scale, up.weight_scale}) + alias |= overlaps(dst, tensor); + if (alias && p.output == SIZE_MAX) + return GGML_STATUS_FAILED; + auto* output = static_cast(alias ? scratch + p.output : dst->data); + if (p.padded > p.rows && cuda_memset_async(output + p.n * p.rows, 0, size_t(p.n) * (p.padded - p.rows), stream) != cuda_success) + return GGML_STATUS_FAILED; + auto* row_scales = reinterpret_cast(output + p.n * p.padded); + auto* partial = reinterpret_cast(scratch + p.partial); + const auto groups = p.n / 256; + const auto total_groups = p.rows * groups; + const SplitSwiGLUInput glu{gate_accum, up_accum, reinterpret_cast(input + p.k * p.padded), + static_cast(gate.weight_scale->data), static_cast(up.weight_scale->data)}; + convrot_group_amax_i8_cuda<<>>(glu, partial, p.n, groups, total_groups); + if (cuda_last_error() != cuda_success) + return GGML_STATUS_FAILED; + reduce_convrot_row_amax_i8_cuda<<>>(partial, row_scales, groups, p.rows); + if (cuda_last_error() != cuda_success) + return GGML_STATUS_FAILED; + convrot_group_quantize_i8_cuda<<>>( + reinterpret_cast(up_accum), output, row_scales, p.n, groups, total_groups, p.n); + if (cuda_last_error() != cuda_success || (alias && cuda_copy_async(dst->data, output, ggml_nbytes(dst), stream) != cuda_success)) + return GGML_STATUS_FAILED; + return GGML_STATUS_SUCCESS; + } + + static void split_convrot_destroy(void* state, void*) { + delete static_cast(state); + } +#endif + } void register_i8(ggml_backend_dev_t device) { @@ -905,9 +1173,13 @@ namespace SD_CUDA_NAMESPACE { desc.priority = 240; desc.supports = nullptr; desc.match = sd_i8_cuda::match; +#if defined(SD_CUDA_VENDOR_HIP) + desc.optimize = sd_i8_cuda::optimize_gated_residual; +#endif GGML_ASSERT(ggml_ext_register_kernel_ex(device, &desc, GGML_EXT_KERNEL_CAPTURE_SAFE | GGML_EXT_KERNEL_REPLAY_REUSABLE)); desc.priority = 100; desc.match = nullptr; + desc.optimize = nullptr; desc.name = "sd.quantize_i8_convrot." SD_CUDA_NAME ".v1"; desc.custom_op = sd_ggml_convrot_op(); desc.supports = sd_i8_cuda::convrot_supports; @@ -922,6 +1194,17 @@ namespace SD_CUDA_NAMESPACE { desc.execute = sd_i8_cuda::split_execute; desc.destroy = sd_i8_cuda::split_destroy; GGML_ASSERT(ggml_ext_register_kernel_ex(device, &desc, GGML_EXT_KERNEL_CAPTURE_SAFE | GGML_EXT_KERNEL_REPLAY_REUSABLE)); +#endif +#if defined(SD_CUDA_VENDOR_HIP) + desc.name = "sd.linear_i8.swiglu_split_convrot.hip.v1"; + desc.custom_op = sd_ggml_i8_op(); + desc.priority = 260; + desc.supports = nullptr; + desc.match = sd_i8_cuda::split_convrot_match; + desc.prepare = sd_i8_cuda::split_convrot_prepare; + desc.execute = sd_i8_cuda::split_convrot_execute; + desc.destroy = sd_i8_cuda::split_convrot_destroy; + GGML_ASSERT(ggml_ext_register_kernel_ex(device, &desc, GGML_EXT_KERNEL_CAPTURE_SAFE | GGML_EXT_KERNEL_REPLAY_REUSABLE)); #endif } diff --git a/src/core/cuda/registration.cu b/src/core/cuda/registration.cu index d92be448ce..2755264ea6 100644 --- a/src/core/cuda/registration.cu +++ b/src/core/cuda/registration.cu @@ -16,6 +16,9 @@ namespace SD_CUDA_NAMESPACE { void register_modulate(ggml_backend_dev_t device); void register_bf16(ggml_backend_dev_t device); void register_rms_affine(ggml_backend_dev_t device); +#if defined(SD_CUDA_VENDOR_HIP) + void register_wmma_attention(ggml_backend_dev_t device); +#endif #if !defined(SD_CUDA_VENDOR_HIP) && !defined(SD_CUDA_VENDOR_MUSA) void register_sage_attention(ggml_backend_dev_t device); void register_sol_attention(ggml_backend_dev_t device); @@ -32,6 +35,9 @@ namespace SD_CUDA_NAMESPACE { register_rope(device); register_bf16(device); register_rms_affine(device); +#if defined(SD_CUDA_VENDOR_HIP) + register_wmma_attention(device); +#endif #if !defined(SD_CUDA_VENDOR_HIP) && !defined(SD_CUDA_VENDOR_MUSA) register_sage_attention(device); register_sol_attention(device); diff --git a/src/core/cuda/w4.cu b/src/core/cuda/w4.cu index 98f810e136..5ccb3ee9a4 100644 --- a/src/core/cuda/w4.cu +++ b/src/core/cuda/w4.cu @@ -856,7 +856,7 @@ namespace SD_CUDA_NAMESPACE { #ifdef SD_CUDA_W4_CUTLASS if (plan->mma && op.kind == SDW4Kind::W4A4) { if (!op.bias && aligned(dst, 16) && aligned(op.weight_scale, 16)) - plan->gemm = sd_cuda_i8_gemm_prepare(ggml_backend_get_device(backend), ggml_nrows(dst), dst->ne[0], op.input->ne[0], false, &gemm_bytes, true); + plan->gemm = sd_cuda_i8_gemm_prepare(ggml_backend_get_device(backend), ggml_nrows(dst), dst->ne[0], op.input->ne[0], SDI8GemmActivation::None, &gemm_bytes, true); plan->mma = plan->gemm != nullptr; if (plan->gemm && gemm_bytes == 0) gemm_bytes = 256; diff --git a/src/core/execution_engine.cpp b/src/core/execution_engine.cpp index 5ad241a1cd..867fc730ef 100644 --- a/src/core/execution_engine.cpp +++ b/src/core/execution_engine.cpp @@ -84,9 +84,12 @@ namespace sd { if (sd_backend_is(backend, "Vulkan")) { return true; } - auto buft = ggml_backend_get_default_buffer_type(backend); + auto buft = ggml_backend_get_default_buffer_type(backend); + size_t max_size = ggml_backend_buft_get_max_size(buft); + if (max_size == SIZE_MAX) { + return true; + } size_t alignment = ggml_backend_buft_get_alignment(buft); - size_t max_size = ggml_backend_buft_get_max_size(buft); max_size -= max_size % alignment; if (ggml_backend_buft_get_alloc_size(buft, node) > max_size) { return false; @@ -159,19 +162,21 @@ namespace sd { if (sched == nullptr || gf == nullptr) { return; } - ggml_backend_t current = session_.backend_; - const int n_nodes = ggml_graph_n_nodes(gf); + ggml_backend_t current = session_.backend_; + const int n_nodes = ggml_graph_n_nodes(gf); + const bool multi_device = is_multi_device(); for (int i = 0; i < n_nodes; i++) { - ggml_tensor* node = ggml_graph_node(gf, i); - auto node_assignment = session_.node_assignments_.find(original_graph == nullptr ? node : ggml_graph_node(original_graph, i)); - if (node_assignment != session_.node_assignments_.end()) { - current = node_assignment->second; - } - for (int s = 0; s < GGML_MAX_SRC; s++) { - ggml_backend_t weight_backend = backend_for_weight(node->src[s]); - if (weight_backend != nullptr) { - if (node_assignment == session_.node_assignments_.end()) { - current = weight_backend; + ggml_tensor* node = ggml_graph_node(gf, i); + if (multi_device) { + auto node_assignment = session_.node_assignments_.find(original_graph == nullptr ? node : ggml_graph_node(original_graph, i)); + if (node_assignment != session_.node_assignments_.end()) { + current = node_assignment->second; + } else { + for (auto* source : node->src) { + auto weight_backend = backend_for_weight(source); + if (weight_backend != nullptr) { + current = weight_backend; + } } } } @@ -665,7 +670,7 @@ namespace sd { return fail_segment("output cache allocation"); } new_cache_bytes = add_bytes(last ? 0 : session_.cut_cache_.estimate_output_bytes(graph, segment), session_.cache_.pending_bytes(segment_graph)); - if (!session_.workspace_.allocate(segment_graph, [&](ggml_backend_sched_t scheduler, ggml_cgraph* current) { + if (!session_.workspace_.allocate(segment_graph, measurement, [&](ggml_backend_sched_t scheduler, ggml_cgraph* current) { pin_multi_device_nodes(scheduler, current); })) { last_compute_status_ = GGML_STATUS_ALLOC_FAILED; diff --git a/src/core/ggml_tensor_utils.cpp b/src/core/ggml_tensor_utils.cpp index 5639c7ed98..44846c6b65 100644 --- a/src/core/ggml_tensor_utils.cpp +++ b/src/core/ggml_tensor_utils.cpp @@ -200,7 +200,7 @@ uint8_t* ggml_tensor_to_sd_image(ggml_tensor* input, int idx, bool video) { return image_data; } -void sd_image_to_ggml_tensor(sd_image_t image, +void sd_image_to_ggml_tensor(sd::Image image, ggml_tensor* tensor, bool scale) { GGML_ASSERT(image.width == tensor->ne[0]); diff --git a/src/core/ggml_tensor_utils.h b/src/core/ggml_tensor_utils.h index 65cc5f2f48..f7b101ce1b 100644 --- a/src/core/ggml_tensor_utils.h +++ b/src/core/ggml_tensor_utils.h @@ -1,6 +1,8 @@ #ifndef __SD_CORE_GGML_TENSOR_UTILS_H__ #define __SD_CORE_GGML_TENSOR_UTILS_H__ +#include "core/media.h" + #include #include #include @@ -58,7 +60,7 @@ __STATIC_INLINE__ ggml_fp16_t ggml_ext_tensor_get_f16(const ggml_tensor* tensor, return *(ggml_fp16_t*)((char*)(tensor->data) + i3 * tensor->nb[3] + i2 * tensor->nb[2] + i1 * tensor->nb[1] + i0 * tensor->nb[0]); } -__STATIC_INLINE__ float sd_image_get_f32(sd_image_t image, int64_t iw, int64_t ih, int64_t ic, bool scale = true) { +__STATIC_INLINE__ float sd_image_get_f32(sd::Image image, int64_t iw, int64_t ih, int64_t ic, bool scale = true) { float value = *(image.data + ih * image.width * image.channel + iw * image.channel + ic); if (scale) { value /= 255.f; @@ -146,7 +148,7 @@ uint8_t* ggml_tensor_to_sd_image(ggml_tensor* input, uint8_t* image_data = nullp uint8_t* ggml_tensor_to_sd_image(ggml_tensor* input, int idx, bool video = false); -void sd_image_to_ggml_tensor(sd_image_t image, +void sd_image_to_ggml_tensor(sd::Image image, ggml_tensor* tensor, bool scale = true); diff --git a/src/core/media.h b/src/core/media.h new file mode 100644 index 0000000000..3ee1d4ebe2 --- /dev/null +++ b/src/core/media.h @@ -0,0 +1,33 @@ +#ifndef __SD_CORE_MEDIA_H__ +#define __SD_CORE_MEDIA_H__ + +#include + +namespace sd { + struct Image { + uint32_t width; + uint32_t height; + uint32_t channel; + uint8_t* data; + }; + + struct Audio { + uint32_t sample_rate; + uint32_t channels; + uint64_t sample_count; + float* data; + }; + + struct ReferenceVideo { + Image* frames; + int frame_count; + int fps; + Audio audio; + }; + + bool preprocess_canny(Image image, float high_threshold, float low_threshold, float weak, float strong, bool inverse); + void free_image(Image* image); + void free_audio(Audio* audio); +} + +#endif // __SD_CORE_MEDIA_H__ diff --git a/src/core/option_types.cpp b/src/core/option_types.cpp new file mode 100644 index 0000000000..2c9a1c1bc9 --- /dev/null +++ b/src/core/option_types.cpp @@ -0,0 +1,265 @@ +#include "option_types.h" + +#include + +static constexpr const char* rng_type_to_str[] = { + "std_default", + "cuda", + "cpu", +}; + +const char* sd_rng_type_name(enum rng_type_t rng_type) { + if (rng_type < RNG_TYPE_COUNT) { + return rng_type_to_str[rng_type]; + } + return "NONE"; +} + +enum rng_type_t str_to_rng_type(const char* str) { + for (int i = 0; i < RNG_TYPE_COUNT; i++) { + if (!strcmp(str, rng_type_to_str[i])) { + return (enum rng_type_t)i; + } + } + return RNG_TYPE_COUNT; +} + +static constexpr const char* sample_method_to_str[] = { + "euler", + "euler_a", + "heun", + "dpm2", + "dpm++2s_a", + "dpm++2m", + "dpm++2mv2", + "ipndm", + "ipndm_v", + "lcm", + "ddim_trailing", + "tcd", + "res_multistep", + "res_2s", + "er_sde", + "euler_cfg_pp", + "euler_a_cfg_pp", + "euler_ge", + "dpm++2m_sde", + "dpm++2m_sde_bt", + "lms", +}; + +static_assert(SAMPLE_METHOD_COUNT == sizeof(sample_method_to_str) / sizeof(sample_method_to_str[0]), + "\nnumber of elements in sample_method_to_str[] != SAMPLE_METHOD_COUNT"); + +const char* sd_sample_method_name(enum sample_method_t sample_method) { + if (sample_method < SAMPLE_METHOD_COUNT) { + return sample_method_to_str[sample_method]; + } + return "NONE"; +} + +enum sample_method_t str_to_sample_method(const char* str) { + for (int i = 0; i < SAMPLE_METHOD_COUNT; i++) { + if (!strcmp(str, sample_method_to_str[i])) { + return (enum sample_method_t)i; + } + } + return SAMPLE_METHOD_COUNT; +} + +static constexpr const char* scheduler_to_str[] = { + "discrete", + "karras", + "exponential", + "ays", + "gits", + "sgm_uniform", + "simple", + "smoothstep", + "kl_optimal", + "lcm", + "bong_tangent", + "ltx2", + "logit_normal", + "flux2", + "flux", + "beta", + "llada_image", +}; + +static_assert(SCHEDULER_COUNT == sizeof(scheduler_to_str) / sizeof(scheduler_to_str[0]), + "\nnumber of elements in scheduler_to_str[] != SCHEDULER_COUNT"); + +const char* sd_scheduler_name(enum scheduler_t scheduler) { + if (scheduler < SCHEDULER_COUNT) { + return scheduler_to_str[scheduler]; + } + return "NONE"; +} + +enum scheduler_t str_to_scheduler(const char* str) { + if (!strcmp(str, "normal")) { + return DISCRETE_SCHEDULER; + } + for (int i = 0; i < SCHEDULER_COUNT; i++) { + if (!strcmp(str, scheduler_to_str[i])) { + return (enum scheduler_t)i; + } + } + return SCHEDULER_COUNT; +} + +static constexpr const char* prediction_to_str[] = { + "eps", + "v", + "edm_v", + "sd3_flow", + "flux_flow", + "sefi_flow", + "minit2i_flow", + "sensenova_u1_flow", + "flux3_flow", + "h3_av_flow", + "ming_image_flow", +}; + +const char* sd_prediction_name(enum prediction_t prediction) { + if (prediction >= EPS_PRED && prediction < PREDICTION_COUNT) { + return prediction_to_str[prediction]; + } + return "NONE"; +} + +enum prediction_t str_to_prediction(const char* str) { + for (int i = 0; i < PREDICTION_COUNT; i++) { + if (!strcmp(str, prediction_to_str[i])) { + return (enum prediction_t)i; + } + } + return PREDICTION_COUNT; +} + +static constexpr const char* preview_to_str[] = { + "none", + "proj", + "tae", + "vae", +}; + +const char* sd_preview_name(enum preview_t preview) { + if (preview < PREVIEW_COUNT) { + return preview_to_str[preview]; + } + return "NONE"; +} + +enum preview_t str_to_preview(const char* str) { + for (int i = 0; i < PREVIEW_COUNT; i++) { + if (!strcmp(str, preview_to_str[i])) { + return (enum preview_t)i; + } + } + return PREVIEW_COUNT; +} + +static constexpr const char* lora_apply_mode_to_str[] = { + "auto", + "immediately", + "at_runtime", +}; + +const char* sd_lora_apply_mode_name(enum lora_apply_mode_t mode) { + if (mode < LORA_APPLY_MODE_COUNT) { + return lora_apply_mode_to_str[mode]; + } + return "NONE"; +} + +enum lora_apply_mode_t str_to_lora_apply_mode(const char* str) { + for (int i = 0; i < LORA_APPLY_MODE_COUNT; i++) { + if (!strcmp(str, lora_apply_mode_to_str[i])) { + return (enum lora_apply_mode_t)i; + } + } + return LORA_APPLY_MODE_COUNT; +} + +static constexpr const char* hires_upscaler_to_str[] = { + "None", + "Latent", + "Latent (nearest)", + "Latent (nearest-exact)", + "Latent (antialiased)", + "Latent (bicubic)", + "Latent (bicubic antialiased)", + "Lanczos", + "Nearest", + "Model", +}; + +const char* sd_hires_upscaler_name(enum sd_hires_upscaler_t upscaler) { + if (upscaler >= SD_HIRES_UPSCALER_NONE && upscaler < SD_HIRES_UPSCALER_COUNT) { + return hires_upscaler_to_str[upscaler]; + } + return "NONE"; +} + +enum sd_hires_upscaler_t str_to_sd_hires_upscaler(const char* str) { + for (int i = 0; i < SD_HIRES_UPSCALER_COUNT; i++) { + if (!strcmp(str, hires_upscaler_to_str[i])) { + return (enum sd_hires_upscaler_t)i; + } + } + return SD_HIRES_UPSCALER_COUNT; +} + +static constexpr const char* control_type_to_str[] = { + "none", + "openpose", + "depth", + "scribble", + "canny", + "normal", + "segment", + "tile", + "repaint", +}; + +static_assert(SD_CONTROL_TYPE_COUNT == sizeof(control_type_to_str) / sizeof(control_type_to_str[0]), + "\nnumber of elements in control_type_to_str[] != SD_CONTROL_TYPE_COUNT"); + +const char* sd_control_type_name(enum sd_control_type_t control_type) { + if (control_type >= SD_CONTROL_TYPE_NONE && control_type < SD_CONTROL_TYPE_COUNT) { + return control_type_to_str[control_type]; + } + return "NONE"; +} + +enum sd_control_type_t str_to_sd_control_type(const char* str) { + for (int i = 0; i < SD_CONTROL_TYPE_COUNT; i++) { + if (!strcmp(str, control_type_to_str[i])) { + return (enum sd_control_type_t)i; + } + } + static const struct { + const char* alias; + enum sd_control_type_t type; + } aliases[] = { + {"hed", SD_CONTROL_TYPE_SCRIBBLE}, + {"softedge", SD_CONTROL_TYPE_SCRIBBLE}, + {"pidi", SD_CONTROL_TYPE_SCRIBBLE}, + {"ted", SD_CONTROL_TYPE_SCRIBBLE}, + {"lineart", SD_CONTROL_TYPE_CANNY}, + {"anime_lineart", SD_CONTROL_TYPE_CANNY}, + {"mlsd", SD_CONTROL_TYPE_CANNY}, + {"seg", SD_CONTROL_TYPE_SEGMENT}, + {"inpaint", SD_CONTROL_TYPE_REPAINT}, + {"outpaint", SD_CONTROL_TYPE_REPAINT}, + }; + for (const auto& entry : aliases) { + if (!strcmp(str, entry.alias)) { + return entry.type; + } + } + return SD_CONTROL_TYPE_COUNT; +} diff --git a/src/core/option_types.h b/src/core/option_types.h new file mode 100644 index 0000000000..280cde237d --- /dev/null +++ b/src/core/option_types.h @@ -0,0 +1,165 @@ +#ifndef __SD_CORE_OPTION_TYPES_H__ +#define __SD_CORE_OPTION_TYPES_H__ + +#include "stable-diffusion.h" + +enum rng_type_t { + STD_DEFAULT_RNG, + CUDA_RNG, + CPU_RNG, + RNG_TYPE_COUNT +}; + +enum sample_method_t { + EULER_SAMPLE_METHOD, + EULER_A_SAMPLE_METHOD, + HEUN_SAMPLE_METHOD, + DPM2_SAMPLE_METHOD, + DPMPP2S_A_SAMPLE_METHOD, + DPMPP2M_SAMPLE_METHOD, + DPMPP2Mv2_SAMPLE_METHOD, + IPNDM_SAMPLE_METHOD, + IPNDM_V_SAMPLE_METHOD, + LCM_SAMPLE_METHOD, + DDIM_TRAILING_SAMPLE_METHOD, + TCD_SAMPLE_METHOD, + RES_MULTISTEP_SAMPLE_METHOD, + RES_2S_SAMPLE_METHOD, + ER_SDE_SAMPLE_METHOD, + EULER_CFG_PP_SAMPLE_METHOD, + EULER_A_CFG_PP_SAMPLE_METHOD, + EULER_GE_SAMPLE_METHOD, + DPMPP2M_SDE_SAMPLE_METHOD, + DPMPP2M_SDE_BT_SAMPLE_METHOD, + LMS_SAMPLE_METHOD, + SAMPLE_METHOD_COUNT +}; + +enum scheduler_t { + DISCRETE_SCHEDULER, + KARRAS_SCHEDULER, + EXPONENTIAL_SCHEDULER, + AYS_SCHEDULER, + GITS_SCHEDULER, + SGM_UNIFORM_SCHEDULER, + SIMPLE_SCHEDULER, + SMOOTHSTEP_SCHEDULER, + KL_OPTIMAL_SCHEDULER, + LCM_SCHEDULER, + BONG_TANGENT_SCHEDULER, + LTX2_SCHEDULER, + LOGIT_NORMAL_SCHEDULER, + FLUX2_SCHEDULER, + FLUX_SCHEDULER, + BETA_SCHEDULER, + LLADA_IMAGE_SCHEDULER, + SCHEDULER_COUNT +}; + +enum prediction_t { + EPS_PRED, + V_PRED, + EDM_V_PRED, + FLOW_PRED, + FLUX_FLOW_PRED, + SEFI_FLOW_PRED, + MINIT2I_FLOW_PRED, + SENSENOVA_U1_FLOW_PRED, + FLUX3_FLOW_PRED, + H3_AV_FLOW_PRED, + MING_IMAGE_FLOW_PRED, + PREDICTION_COUNT +}; + +enum lora_apply_mode_t { + LORA_APPLY_AUTO, + LORA_APPLY_IMMEDIATELY, + LORA_APPLY_AT_RUNTIME, + LORA_APPLY_MODE_COUNT, +}; + +enum sd_vae_format_t { + SD_VAE_FORMAT_AUTO = -1, + SD_VAE_FORMAT_FLUX, + SD_VAE_FORMAT_SD3, + SD_VAE_FORMAT_FLUX2, + SD_VAE_FORMAT_WAN, + SD_VAE_FORMAT_COUNT, +}; + +enum sd_cache_mode_t { + SD_CACHE_DISABLED = 0, + SD_CACHE_EASYCACHE, + SD_CACHE_UCACHE, + SD_CACHE_DBCACHE, + SD_CACHE_TAYLORSEER, + SD_CACHE_CACHE_DIT, + SD_CACHE_SPECTRUM, +}; + +enum sd_hires_upscaler_t { + SD_HIRES_UPSCALER_NONE, + SD_HIRES_UPSCALER_LATENT, + SD_HIRES_UPSCALER_LATENT_NEAREST, + SD_HIRES_UPSCALER_LATENT_NEAREST_EXACT, + SD_HIRES_UPSCALER_LATENT_ANTIALIASED, + SD_HIRES_UPSCALER_LATENT_BICUBIC, + SD_HIRES_UPSCALER_LATENT_BICUBIC_ANTIALIASED, + SD_HIRES_UPSCALER_LANCZOS, + SD_HIRES_UPSCALER_NEAREST, + SD_HIRES_UPSCALER_MODEL, + SD_HIRES_UPSCALER_COUNT, +}; + +enum sd_control_type_t { + SD_CONTROL_TYPE_NONE, + SD_CONTROL_TYPE_OPENPOSE, + SD_CONTROL_TYPE_DEPTH, + SD_CONTROL_TYPE_SCRIBBLE, // thick lines: scribble, hed, softedge, ted + SD_CONTROL_TYPE_CANNY, // thin lines: canny, lineart, anime lineart, mlsd + SD_CONTROL_TYPE_NORMAL, + SD_CONTROL_TYPE_SEGMENT, + SD_CONTROL_TYPE_TILE, + SD_CONTROL_TYPE_REPAINT, // inpaint, outpaint + SD_CONTROL_TYPE_COUNT, +}; + +const char* sd_rng_type_name(enum rng_type_t rng_type); + +enum rng_type_t str_to_rng_type(const char* str); + +const char* sd_sample_method_name(enum sample_method_t sample_method); + +enum sample_method_t str_to_sample_method(const char* str); + +const char* sd_scheduler_name(enum scheduler_t scheduler); + +enum scheduler_t str_to_scheduler(const char* str); + +const char* sd_prediction_name(enum prediction_t prediction); + +enum prediction_t str_to_prediction(const char* str); + +const char* sd_lora_apply_mode_name(enum lora_apply_mode_t mode); + +enum lora_apply_mode_t str_to_lora_apply_mode(const char* str); + +const char* sd_hires_upscaler_name(enum sd_hires_upscaler_t upscaler); + +enum sd_hires_upscaler_t str_to_sd_hires_upscaler(const char* str); + +const char* sd_control_type_name(enum sd_control_type_t control_type); + +enum sd_control_type_t str_to_sd_control_type(const char* str); + +enum preview_t { + PREVIEW_NONE, + PREVIEW_PROJ, + PREVIEW_TAE, + PREVIEW_VAE, + PREVIEW_COUNT +}; +const char* sd_preview_name(enum preview_t preview); +enum preview_t str_to_preview(const char* str); + +#endif // __SD_CORE_OPTION_TYPES_H__ diff --git a/src/core/pe_cache.cpp b/src/core/pe_cache.cpp index df4aa62cb4..3f53a0a5eb 100644 --- a/src/core/pe_cache.cpp +++ b/src/core/pe_cache.cpp @@ -20,11 +20,17 @@ namespace sd { } const Tensor& PECache::get_or_create(const PECacheKey& key, const std::function()>& build) { - auto entry = entries_.find(key.name()); - if (entry != entries_.end() && entry->second.key == key) { - return entry->second.value; + auto& entries = entries_[key.name()]; + for (auto entry = entries.begin(); entry != entries.end(); ++entry) { + if (entry->key == key) { + entries.splice(entries.begin(), entries, entry); + return entries.front().value; + } + } + entries.push_front({key, build()}); + if (entries.size() > entries_per_name_) { + entries.pop_back(); } - auto value = build(); - return entries_.insert_or_assign(key.name(), Entry{key, std::move(value)}).first->second.value; + return entries.front().value; } } diff --git a/src/core/pe_cache.h b/src/core/pe_cache.h index 3539707bdb..90b5d3af37 100644 --- a/src/core/pe_cache.h +++ b/src/core/pe_cache.h @@ -2,6 +2,7 @@ #define __SD_CORE_PE_CACHE_H__ #include +#include #include #include #include @@ -71,9 +72,14 @@ namespace sd { Tensor value; }; - std::map entries_; + std::map> entries_; + size_t entries_per_name_; public: + explicit PECache(size_t entries_per_name = 1) + : entries_per_name_(entries_per_name) { + GGML_ASSERT(entries_per_name_ > 0); + } const Tensor& get_or_create(const PECacheKey& key, const std::function()>& build); }; diff --git a/src/core/util.cpp b/src/core/util.cpp index 9de1ace24f..220134f1f0 100644 --- a/src/core/util.cpp +++ b/src/core/util.cpp @@ -17,6 +17,7 @@ #include #include #include "core/ggml_tensor_utils.h" +#include "core/option_types.h" #include "runtime/preprocessing.hpp" #if defined(__APPLE__) && defined(__MACH__) @@ -721,7 +722,7 @@ void sd_set_progress_callback(sd_progress_cb_t cb, void* data) { sd_progress_cb = cb; sd_progress_cb_data = data; } -void sd_set_preview_callback(sd_preview_cb_t cb, preview_t mode, int interval, bool denoised, bool noisy, void* data) { +void sd_set_preview_callback_impl(sd_preview_cb_t cb, preview_t mode, int interval, bool denoised, bool noisy, void* data) { sd_preview_info = {}; sd_preview_cb = cb; sd_preview_cb_data = data; @@ -782,7 +783,7 @@ void* sd_get_progress_callback_data() { return sd_progress_cb_data; } -sd_image_t tensor_to_sd_image(const sd::Tensor& tensor, int frame_index) { +sd::Image tensor_to_sd_image(const sd::Tensor& tensor, int frame_index) { const auto& shape = tensor.shape(); GGML_ASSERT(shape.size() == 4 || shape.size() == 5); int width = static_cast(shape[0]); @@ -799,7 +800,7 @@ sd_image_t tensor_to_sd_image(const sd::Tensor& tensor, int frame_index) }; } -sd::Tensor sd_image_to_tensor(sd_image_t image, +sd::Tensor sd_image_to_tensor(sd::Image image, int target_width, int target_height, bool scale) { diff --git a/src/core/util.h b/src/core/util.h index 0929f60e01..082565a4e3 100644 --- a/src/core/util.h +++ b/src/core/util.h @@ -1,12 +1,15 @@ #ifndef __SD_CORE_UTIL_H__ #define __SD_CORE_UTIL_H__ +#include "core/media.h" + #include #include #include #include #include +#include "core/option_types.h" #include "core/tensor.hpp" #include "ggml-backend.h" #include "stable-diffusion.h" @@ -41,9 +44,9 @@ std::string utf32_to_utf8(const std::u32string& utf32_str); std::u32string unicode_value_to_utf32(int unicode_value); // std::string sd_basename(const std::string& path); -sd_image_t tensor_to_sd_image(const sd::Tensor& tensor, int frame_index = 0); +sd::Image tensor_to_sd_image(const sd::Tensor& tensor, int frame_index = 0); -sd::Tensor sd_image_to_tensor(sd_image_t image, +sd::Tensor sd_image_to_tensor(sd::Image image, int target_width = -1, int target_height = -1, bool scale = true); @@ -105,6 +108,7 @@ void* sd_get_progress_callback_data(); void sd_begin_preview_pass(int total_steps); sd_preview_cb_t sd_get_preview_callback(); void* sd_get_preview_callback_data(); +void sd_set_preview_callback_impl(sd_preview_cb_t cb, preview_t mode, int interval, bool denoised, bool noisy, void* data); preview_t sd_get_preview_mode(); int sd_get_preview_interval(); bool sd_should_preview_denoised(); diff --git a/src/detailer.cpp b/src/detailer.cpp index 9c48ca4787..234afce2fe 100644 --- a/src/detailer.cpp +++ b/src/detailer.cpp @@ -1,5 +1,9 @@ #include "detailer.h" +#include "api/options.h" +#include "api/result.h" +#include "core/option_types.h" #include "generation_info.h" +#include "pipeline/generation.h" #include "pipeline/request.h" #include @@ -15,8 +19,8 @@ #include "json.hpp" #include "model_loader.h" -struct adetailer_ctx_t { - ADetailerGGML* detailer = nullptr; +struct sd_adetailer { + std::unique_ptr detailer; }; struct LetterboxInput { @@ -96,7 +100,7 @@ static bool parse_float(const std::string& value, float* result) { } } -static float image_channel(const sd_image_t& image, int x, int y, int channel) { +static float image_channel(const sd::Image& image, int x, int y, int channel) { if (image.data == nullptr || image.width == 0 || image.height == 0 || image.channel == 0) { return 0.f; } @@ -107,7 +111,7 @@ static float image_channel(const sd_image_t& image, int x, int y, int channel) { return image.data[index] / 255.f; } -static float bilinear_channel(const sd_image_t& image, float x, float y, int channel) { +static float bilinear_channel(const sd::Image& image, float x, float y, int channel) { int x0 = static_cast(std::floor(x)); int y0 = static_cast(std::floor(y)); int x1 = x0 + 1; @@ -119,7 +123,7 @@ static float bilinear_channel(const sd_image_t& image, float x, float y, int cha return a * (1.f - wy) + b * wy; } -static LetterboxInput make_letterbox_input(sd_image_t image, int input_size) { +static LetterboxInput make_letterbox_input(sd::Image image, int input_size) { LetterboxInput result; result.tensor = sd::full({input_size, input_size, 3, 1}, 114.f / 255.f); result.scale = std::min(static_cast(input_size) / image.width, @@ -180,7 +184,7 @@ static float box_iou(const ADetailerDetection& a, const ADetailerDetection& b) { static std::vector decode_detections(const sd::Tensor& raw, const YOLOv8Config& config, const LetterboxInput& letterbox, - sd_image_t image, + sd::Image image, const ADetailerParams& params) { std::vector candidates; if (raw.empty() || raw.dim() < 2) { @@ -510,9 +514,9 @@ static CropRegion expand_crop(CropRegion crop, return crop; } -static sd_image_t resize_crop_image(const sd_image_t& source, const CropRegion& crop, int width, int height) { - sd_image_t output = {static_cast(width), static_cast(height), 3, nullptr}; - output.data = static_cast(malloc(static_cast(width) * height * 3)); +static sd::Image resize_crop_image(const sd::Image& source, const CropRegion& crop, int width, int height) { + sd::Image output = {static_cast(width), static_cast(height), 3, nullptr}; + output.data = static_cast(malloc(static_cast(width) * height * 3)); if (output.data == nullptr) { return {0, 0, 0, nullptr}; } @@ -531,9 +535,9 @@ static sd_image_t resize_crop_image(const sd_image_t& source, const CropRegion& return output; } -static sd_image_t resize_crop_mask(const Mask& source, const CropRegion& crop, int width, int height) { - sd_image_t output = {static_cast(width), static_cast(height), 1, nullptr}; - output.data = static_cast(malloc(static_cast(width) * height)); +static sd::Image resize_crop_mask(const Mask& source, const CropRegion& crop, int width, int height) { + sd::Image output = {static_cast(width), static_cast(height), 1, nullptr}; + output.data = static_cast(malloc(static_cast(width) * height)); if (output.data == nullptr) { return {0, 0, 0, nullptr}; } @@ -549,11 +553,11 @@ static sd_image_t resize_crop_mask(const Mask& source, const CropRegion& crop, i return output; } -static sd_image_t copy_for_detailing(const sd_image_t& source) { +static sd::Image copy_for_detailing(const sd::Image& source) { if (source.channel == 4) { - sd_image_t output = source; - size_t bytes = static_cast(source.width) * source.height * source.channel; - output.data = static_cast(malloc(bytes)); + sd::Image output = source; + size_t bytes = static_cast(source.width) * source.height * source.channel; + output.data = static_cast(malloc(bytes)); if (output.data != nullptr) { memcpy(output.data, source.data, bytes); } @@ -563,8 +567,8 @@ static sd_image_t copy_for_detailing(const sd_image_t& source) { return resize_crop_image(source, full, source.width, source.height); } -static void composite_crop(sd_image_t* destination, - const sd_image_t& generated, +static void composite_crop(sd::Image* destination, + const sd::Image& generated, const Mask& feather_mask, const CropRegion& crop) { int crop_width = crop.x2 - crop.x1; @@ -724,7 +728,7 @@ bool ADetailerGGML::load_from_file(const std::string& detector_path) { return true; } -std::vector ADetailerGGML::predict(sd_image_t image, +std::vector ADetailerGGML::predict(sd::Image image, const ADetailerParams& params) { LetterboxInput input = make_letterbox_input(image, params.input_size); int64_t start = ggml_time_ms(); @@ -758,8 +762,8 @@ std::vector ADetailerGGML::predict(sd_image_t image, return detections; } -static bool parse_adetailer_params(const sd_adetailer_params_t& public_params, - const sd_img_gen_params_t& inpaint_params, +static bool parse_adetailer_params(const sd::ADetailConfig& public_params, + const sd::ImageConfig& inpaint_params, ADetailerParams* params) { if (params == nullptr) { return false; @@ -883,41 +887,32 @@ static bool parse_adetailer_params(const sd_adetailer_params_t& public_params, return true; } -adetailer_ctx_t* new_adetailer_ctx(const char* detector_path, - int n_threads, - const char* backend, - const char* params_backend) { - if (detector_path == nullptr || detector_path[0] == '\0') { - return nullptr; - } - auto* context = static_cast(calloc(1, sizeof(adetailer_ctx_t))); - if (context == nullptr) { - return nullptr; - } - context->detailer = new ADetailerGGML(n_threads, SAFE_STR(backend), SAFE_STR(params_backend)); - if (context->detailer == nullptr || !context->detailer->load_from_file(detector_path)) { - delete context->detailer; - free(context); - return nullptr; - } - return context; +sd_status_t sd_adetailer_create(const sd_adetailer_opt_t* opt, sd_adetailer_t** out) { + if (out) + *out = nullptr; + return sd::api::call([&] { + if (!out) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required"); + sd::OptConfig config; + auto status = sd::api::resolve(opt, sd::api::ADetailer, config); + if (status != SD_STATUS_OK) + return status; + auto result = std::make_unique(); + const auto& ctx = config.context; + result->detailer = std::make_unique(ctx.n_threads, SAFE_STR(ctx.backend), SAFE_STR(ctx.params_backend)); + if (!result->detailer->load_from_file(ctx.model_path)) + return sd::api::error(SD_STATUS_LOAD_FAILED, "model_path", "could not load ADetailer"); + *out = result.release(); + return SD_STATUS_OK; + }, + SD_STATUS_LOAD_FAILED); } -void free_adetailer_ctx(adetailer_ctx_t* context) { - if (context == nullptr) { - return; - } - delete context->detailer; - context->detailer = nullptr; - free(context); +void sd_adetailer_free(sd_adetailer_t* adetailer) { + delete adetailer; } -bool adetail_image(adetailer_ctx_t* context, - sd_ctx_t* sd_ctx, - sd_image_t input_image, - const sd_adetailer_params_t* public_params, - const sd_img_gen_params_t* inpaint_params, - sd_image_generation_result_t** result_out) try { +static bool run_adetail(sd_adetailer_t* context, sd_ctx_t* sd_ctx, sd::Image input_image, const sd::ADetailConfig* public_params, const sd::ImageConfig* inpaint_params, sd_result_t** result_out, sd_status_t& failure) { if (result_out) *result_out = nullptr; if (!result_out) @@ -929,6 +924,7 @@ bool adetail_image(adetailer_ctx_t* context, ADetailerParams params; if (!parse_adetailer_params(*public_params, *inpaint_params, ¶ms)) { + failure = SD_STATUS_INVALID_ARGUMENT; return false; } @@ -939,8 +935,9 @@ bool adetail_image(adetailer_ctx_t* context, filter_and_sort_detections(&detections, input_image.width, input_image.height, params); auto masks = make_masks(detections, input_image.width, input_image.height, params); - sd_image_t current = copy_for_detailing(input_image); + sd::Image current = copy_for_detailing(input_image); if (current.data == nullptr) { + failure = SD_STATUS_OUT_OF_MEMORY; return false; } @@ -959,38 +956,41 @@ bool adetail_image(adetailer_ctx_t* context, if (!mask_bbox(masks[i], &crop)) { continue; } - crop = params.inpaint_only_masked ? expand_crop(crop, - current.width, - current.height, - params.inpaint_padding, - params.inpaint_width, - params.inpaint_height) - : CropRegion{0, 0, static_cast(current.width), static_cast(current.height)}; - sd_image_t local_image = resize_crop_image(current, crop, params.inpaint_width, params.inpaint_height); - sd_image_t local_mask = resize_crop_mask(masks[i], crop, params.inpaint_width, params.inpaint_height); + crop = params.inpaint_only_masked ? expand_crop(crop, + current.width, + current.height, + params.inpaint_padding, + params.inpaint_width, + params.inpaint_height) + : CropRegion{0, 0, static_cast(current.width), static_cast(current.height)}; + sd::Image local_image = resize_crop_image(current, crop, params.inpaint_width, params.inpaint_height); + sd::Image local_mask = resize_crop_mask(masks[i], crop, params.inpaint_width, params.inpaint_height); if (local_image.data == nullptr || local_mask.data == nullptr) { + failure = SD_STATUS_OUT_OF_MEMORY; free(local_image.data); free(local_mask.data); return false; } - sd_img_gen_params_t generation = resolved_inpaint; - generation.prompt = prompt.c_str(); - generation.negative_prompt = negative_prompt.c_str(); - generation.init_image = local_image; - generation.mask_image = local_mask; - generation.width = params.inpaint_width; - generation.height = params.inpaint_height; - generation.strength = params.denoising_strength; - generation.seed = static_cast((static_cast(resolved_inpaint.seed) + i) & INT64_MAX); - generation.batch_count = 1; - generation.ref_images = nullptr; - generation.ref_images_count = 0; - generation.control_image = {}; - generation.pm_params = {}; - generation.pulid_params = {}; - generation.hires.enabled = false; - generation.image_preprocess = {}; + sd::ImageConfig generation = resolved_inpaint; + generation.prompt = prompt.c_str(); + generation.negative_prompt = negative_prompt.c_str(); + generation.init_image = local_image; + generation.mask_image = local_mask; + generation.width = params.inpaint_width; + generation.height = params.inpaint_height; + generation.strength = params.denoising_strength; + generation.seed = static_cast((static_cast(resolved_inpaint.seed) + i) & INT64_MAX); + generation.batch_count = 1; + generation.ref_images = nullptr; + generation.ref_images_count = 0; + generation.control_image = {}; + generation.pm_params = {}; + generation.pulid_params = {}; + generation.hires.enabled = false; + generation.image_preprocess = {}; + + generation.sample_params.guidance.img_cfg = INFINITY; if (params.steps > 0) { generation.sample_params.sample_steps = params.steps; generation.sample_params.custom_sigmas = nullptr; @@ -1006,32 +1006,62 @@ bool adetail_image(adetailer_ctx_t* context, generation.sample_params.scheduler = params.scheduler; } - sd_image_generation_result_t* generated = nullptr; - bool success = generate_image(sd_ctx, &generation, &generated); + sd::ImageGenerationResult* generated = nullptr; + failure = sd::generate_image(sd_ctx, generation, &generated); + std::unique_ptr generated_owner(generated, sd::free_image_generation_result); free(local_image.data); free(local_mask.data); - if (!success || generated == nullptr || generated->count <= 0 || generated->items[0].image.data == nullptr) { - free_sd_image_generation_result(generated); + if (failure != SD_STATUS_OK) + return false; + if (!generated || generated->count <= 0 || !generated->items[0].image.data) { + failure = sd::api::error(SD_STATUS_GENERATION_FAILED, nullptr, "inpainting returned no image"); return false; } - std::unique_ptr generated_owner(generated, &free_sd_image_generation_result); Mask feather = gaussian_blur_mask(masks[i], params.mask_blur); composite_crop(¤t, generated->items[0].image, feather, crop); ++applied; } const std::vector infos{info}; - auto* outputs = static_cast(calloc(1, sizeof(sd_image_t))); - if (!outputs) + auto* outputs = static_cast(calloc(1, sizeof(sd::Image))); + if (!outputs) { + failure = SD_STATUS_OUT_OF_MEMORY; return false; + } outputs[0] = current; current_owner.release(); - if (!sd::make_image_generation_result(outputs, 1, infos, result_out)) + if (!sd::make_image_result(outputs, 1, infos, result_out)) { + failure = SD_STATUS_OUT_OF_MEMORY; return false; + } LOG_INFO("ADetailer applied %d mask(s)", applied); return true; -} catch (const std::exception& e) { - LOG_ERROR("ADetailer failed: %s", e.what()); - return false; +} + +sd_status_t sd_adetail(sd_adetailer_t* adetailer, sd_ctx_t* ctx, const sd_adetail_opt_t* opt, sd_result_t** out) { + if (out) + *out = nullptr; + return sd::api::call([&] { + if (!out) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required"); + sd::OptConfig config; + auto status = sd::api::resolve(opt, sd::api::ADetail, config); + if (status != SD_STATUS_OK) + return status; + if (!adetailer || !ctx) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "adetailer and diffusion context are required"); + if (!sd_ctx_supports_image_generation(ctx)) + return sd::api::error(SD_STATUS_UNSUPPORTED, nullptr, "inpainting requires an image-generation model"); + sd_status_t failure = SD_STATUS_GENERATION_FAILED; + if (!run_adetail(adetailer, ctx, config.post.image, &config.post.adetail, &config.image, out, failure)) { + if (failure == SD_STATUS_OK) + failure = SD_STATUS_GENERATION_FAILED; + if (*sd_get_last_error()) + return failure; + return sd::api::error(failure, nullptr, "ADetailer failed; see log for details"); + } + return SD_STATUS_OK; + }, + SD_STATUS_GENERATION_FAILED); } diff --git a/src/detailer.h b/src/detailer.h index 0e3ae50f5a..67ede5bdad 100644 --- a/src/detailer.h +++ b/src/detailer.h @@ -1,6 +1,10 @@ #ifndef __SD_DETAILER_H__ #define __SD_DETAILER_H__ +#include "core/media.h" + +#include "core/option_types.h" + #include #include #include @@ -70,7 +74,7 @@ struct ADetailerGGML { ~ADetailerGGML(); bool load_from_file(const std::string& detector_path); - std::vector predict(sd_image_t image, + std::vector predict(sd::Image image, const ADetailerParams& params); }; diff --git a/src/extensions/generation_extension.h b/src/extensions/generation_extension.h index e5ffd770e1..018092e2de 100644 --- a/src/extensions/generation_extension.h +++ b/src/extensions/generation_extension.h @@ -9,6 +9,7 @@ #include #include "conditioning/conditioner.hpp" +#include "config.h" #include "core/ggml_extend_backend.h" #include "model/diffusion/model.hpp" #include "model_loader.h" @@ -16,7 +17,7 @@ #include "stable-diffusion.h" struct GenerationExtensionInitContext { - const sd_ctx_params_t* params; + const sd::ContextConfig* params; SDVersion version; const String2TensorStorage& tensor_storage_map; bool photomaker_source_available; @@ -30,8 +31,8 @@ struct GenerationExtensionInitContext { struct GenerationExtensionConditionContext { Conditioner* conditioner; ConditionerParams& condition_params; - const sd_pm_params_t& pm_params; - const sd_pulid_params_t& pulid_params; + const sd::PhotoMakerConfig& pm_params; + const sd::PuLIDConfig& pulid_params; int n_threads; int total_steps; }; diff --git a/src/generation_info.cpp b/src/generation_info.cpp index 0d4829139d..f56ef98840 100644 --- a/src/generation_info.cpp +++ b/src/generation_info.cpp @@ -1,9 +1,10 @@ #include "generation_info.h" +#include "config.h" +#include "core/option_types.h" #include #include #include -#include #include "pipeline/request.h" #include "runtime/guidance.h" @@ -30,7 +31,7 @@ namespace sd { return std::strtod(buf, nullptr); } - static json sampling_info(const sd_sample_params_t& p, bool high_noise = false) { + static json sampling_info(const sd::SampleConfig& p, bool high_noise = false) { const auto& g = p.guidance; json info = {{"sample_steps", p.sample_steps}, {"guidance", {{"txt_cfg", g.txt_cfg}, {"distilled_guidance", g.distilled_guidance}}}}; if (!high_noise && p.shifted_timestep > 0) @@ -55,7 +56,7 @@ namespace sd { return info; } - static json hires_info(const sd_hires_params_t& p) { + static json hires_info(const sd::HiresConfig& p) { json info = {{"enabled", p.enabled}}; if (!p.enabled) return info; @@ -69,7 +70,7 @@ namespace sd { return info; } - static json cache_info(const sd_cache_params_t& p) { + static json cache_info(const sd::CacheConfig& p) { static const char* modes[] = {"disabled", "easycache", "ucache", "dbcache", "taylorseer", "cache_dit", "spectrum"}; if (p.mode < SD_CACHE_DISABLED || p.mode > SD_CACHE_SPECTRUM) return {{"mode", static_cast(p.mode)}}; @@ -144,7 +145,7 @@ namespace sd { return info; } - static json image_info(const sd_img_gen_params_t& p) { + static json image_info(const sd::ImageConfig& p) { auto info = common_info(p, "image"); if (p.init_image.data) info["strength"] = p.strength; @@ -177,7 +178,7 @@ namespace sd { return p; } - json generation_info(const pipeline::GenerationRequest& request, const sd_img_gen_params_t& params) { + json generation_info(const pipeline::GenerationRequest& request, const sd::ImageConfig& params) { auto p = resolved_params(request, params); p.qwen_image_layers = request.qwen_image_layers; auto info = image_info(p); @@ -188,7 +189,7 @@ namespace sd { return info; } - json generation_info(const pipeline::GenerationRequest& request, const sd_vid_gen_params_t& params) { + json generation_info(const pipeline::GenerationRequest& request, const sd::VideoConfig& params) { const auto p = resolved_params(request, params); auto info = common_info(p, "video"); if (!request.models.empty()) @@ -208,14 +209,13 @@ namespace sd { return info; } - json adetailer_generation_info(const sd_adetailer_params_t& params, const sd_img_gen_params_t& inpaint_params) { + json adetailer_generation_info(const sd::ADetailConfig& params, const sd::ImageConfig& inpaint_params) { auto inpaint = image_info(inpaint_params); inpaint["strength"] = inpaint_params.strength; // ADetailer clears these inputs before generating each crop. for (const auto* key : {"control_strength", "control_type", "pm_params", "pulid_params", "hires", "image_preprocess"}) inpaint.erase(key); - sd_img_gen_params_t defaults; - sd_img_gen_params_init(&defaults); + sd::ImageConfig defaults; if (inpaint_params.qwen_image_layers != defaults.qwen_image_layers) inpaint["qwen_image_layers"] = inpaint_params.qwen_image_layers; json adetailer = json::object(); @@ -232,69 +232,4 @@ namespace sd { return {{"schema", "sdcpp.generation/v1"}, {"kind", "image"}, {"upscale", {{"upscale_factor", upscale_factor}}}}; } - char* copy_generation_info(const json& info) { - const std::string text = info.dump(-1, ' ', false, json::error_handler_t::replace); - char* result = static_cast(malloc(text.size() + 1)); - if (result) - memcpy(result, text.c_str(), text.size() + 1); - return result; - } - - bool make_image_generation_result(sd_image_t* images, int count, const std::vector& infos, sd_image_generation_result_t** result_out) { - if (!result_out || !images || count <= 0 || infos.size() != static_cast(count)) { - free_sd_images(images, count); - return false; - } - auto* result = static_cast(calloc(1, sizeof(sd_image_generation_result_t))); - if (!result) { - free_sd_images(images, count); - return false; - } - result->items = static_cast(calloc(count, sizeof(sd_image_result_t))); - if (!result->items) { - free(result); - free_sd_images(images, count); - return false; - } - result->count = count; - try { - for (int i = 0; i < count; ++i) { - result->items[i].image = images[i]; - images[i] = {}; - result->items[i].generation_info = copy_generation_info(infos[i]); - if (!result->items[i].generation_info) { - free_sd_images(images, count); - free_sd_image_generation_result(result); - return false; - } - } - } catch (...) { - free_sd_images(images, count); - free_sd_image_generation_result(result); - return false; - } - free(images); - *result_out = result; - return true; - } -} - -void free_sd_image_generation_result(sd_image_generation_result_t* result) { - if (!result) - return; - for (int i = 0; i < result->count; ++i) { - free(result->items[i].image.data); - free(result->items[i].generation_info); - } - free(result->items); - free(result); -} - -void free_sd_video_generation_result(sd_video_generation_result_t* result) { - if (!result) - return; - free_sd_images(result->frames, result->frame_count); - free_sd_audio(result->audio); - free(result->generation_info); - free(result); -} +} // namespace sd diff --git a/src/generation_info.h b/src/generation_info.h index 56688ee812..d84673f7a2 100644 --- a/src/generation_info.h +++ b/src/generation_info.h @@ -3,6 +3,7 @@ #include +#include "config.h" #include "json.hpp" #include "stable-diffusion.h" @@ -11,13 +12,11 @@ namespace sd::pipeline { } namespace sd { - nlohmann::json generation_info(const pipeline::GenerationRequest& request, const sd_img_gen_params_t& params); - nlohmann::json generation_info(const pipeline::GenerationRequest& request, const sd_vid_gen_params_t& params); - nlohmann::json adetailer_generation_info(const sd_adetailer_params_t& params, const sd_img_gen_params_t& inpaint_params); + nlohmann::json generation_info(const pipeline::GenerationRequest& request, const sd::ImageConfig& params); + nlohmann::json generation_info(const pipeline::GenerationRequest& request, const sd::VideoConfig& params); + nlohmann::json adetailer_generation_info(const sd::ADetailConfig& params, const sd::ImageConfig& inpaint_params); nlohmann::json upscale_generation_info(uint32_t upscale_factor); - char* copy_generation_info(const nlohmann::json& info); - // Consumes images even on failure. - bool make_image_generation_result(sd_image_t* images, int count, const std::vector& infos, sd_image_generation_result_t** result_out); + } #endif // __SD_GENERATION_INFO_H__ diff --git a/src/model/diffusion/control.hpp b/src/model/diffusion/control.hpp index 4ad85ecf8c..2fc3dee36b 100644 --- a/src/model/diffusion/control.hpp +++ b/src/model/diffusion/control.hpp @@ -1,6 +1,8 @@ #ifndef __SD_MODEL_DIFFUSION_CONTROL_HPP__ #define __SD_MODEL_DIFFUSION_CONTROL_HPP__ +#include "core/option_types.h" + #include "model/common/block.hpp" #include "stable-diffusion.h" diff --git a/src/model/diffusion/fibo2.hpp b/src/model/diffusion/fibo2.hpp index 1ac7684320..091e74c0ea 100644 --- a/src/model/diffusion/fibo2.hpp +++ b/src/model/diffusion/fibo2.hpp @@ -106,29 +106,33 @@ namespace Fibo2 { auto project = [&](const char* name, ggml_tensor* h) { return std::dynamic_pointer_cast(blocks[name])->forward(ctx, h); }; + auto split_views = [&](ggml_tensor* qkv, int n, int i) { + int64_t slice = qkv->ne[0] / n; + return ggml_view_4d(c, qkv, head_dim, slice / head_dim, qkv->ne[1], qkv->ne[2] * qkv->ne[3], + head_dim * qkv->nb[0], qkv->nb[1], qkv->nb[2], i * slice * qkv->nb[0]); + }; + auto reshape = [&](ggml_tensor* h) { + return ggml_reshape_4d(c, h, head_dim, heads, h->ne[1], h->ne[2]); + }; ggml_tensor *q, *k, *v; if (fused_qkv) { - auto parts = ggml_ext_chunk(ctx, project("qkv", x), 3, 0); - q = parts[0]; - k = parts[1]; - v = parts[2]; + auto qkv = project("qkv", x); + q = split_views(qkv, 3, 0); + k = split_views(qkv, 3, 1); + v = split_views(qkv, 3, 2); } else { - q = project("to_q", x); + q = reshape(project("to_q", x)); if (fused_kv) { - auto parts = ggml_ext_chunk(ctx, project("to_kv", kv), 2, 0); - k = parts[0]; - v = parts[1]; + auto kv_out = project("to_kv", kv); + k = split_views(kv_out, 2, 0); + v = split_views(kv_out, 2, 1); } else { - k = project("to_k", kv); - v = project("to_v", kv); + k = reshape(project("to_k", kv)); + v = reshape(project("to_v", kv)); } } - auto reshape = [&](ggml_tensor* h) { - return ggml_reshape_4d(c, h, head_dim, heads, h->ne[1], h->ne[2]); - }; - q = std::dynamic_pointer_cast(blocks["norm_q"])->forward(ctx, reshape(q)); - k = std::dynamic_pointer_cast(blocks["norm_k"])->forward(ctx, reshape(k)); - v = reshape(v); + q = std::dynamic_pointer_cast(blocks["norm_q"])->forward(ctx, q); + k = std::dynamic_pointer_cast(blocks["norm_k"])->forward(ctx, k); auto prepare = [&](ggml_tensor* h) { if (pe) { return Rope::apply_rope(ctx, h, pe); diff --git a/src/model/diffusion/ideogram4.hpp b/src/model/diffusion/ideogram4.hpp index 8d7cf2ae23..bd1abebebf 100644 --- a/src/model/diffusion/ideogram4.hpp +++ b/src/model/diffusion/ideogram4.hpp @@ -213,19 +213,20 @@ namespace Ideogram4 { ggml_tensor* x, ggml_tensor* pe, ggml_tensor* mask = nullptr) { - int64_t n_token = x->ne[1]; - int64_t N = x->ne[2]; - auto qkv_proj = std::dynamic_pointer_cast(blocks["qkv"]); auto norm_q = std::dynamic_pointer_cast(blocks["norm_q"]); auto norm_k = std::dynamic_pointer_cast(blocks["norm_k"]); auto out_proj = std::dynamic_pointer_cast(blocks["o"]); - auto qkv = qkv_proj->forward(ctx, x); - auto qkv_vec = ggml_ext_chunk(ctx, qkv, 3, 0); - auto q = ggml_reshape_4d(ctx->ggml_ctx, qkv_vec[0], head_dim, num_heads, n_token, N); - auto k = ggml_reshape_4d(ctx->ggml_ctx, qkv_vec[1], head_dim, num_heads, n_token, N); - auto v = ggml_reshape_4d(ctx->ggml_ctx, qkv_vec[2], head_dim, num_heads, n_token, N); + auto qkv = qkv_proj->forward(ctx, x); + auto qkv_dim = qkv->ne[0] / 3; + auto split_qkv = [&](int i) { + return ggml_view_4d(ctx->ggml_ctx, qkv, head_dim, qkv_dim / head_dim, qkv->ne[1], qkv->ne[2] * qkv->ne[3], + head_dim * qkv->nb[0], qkv->nb[1], qkv->nb[2], i * qkv_dim * qkv->nb[0]); + }; + auto q = split_qkv(0); + auto k = split_qkv(1); + auto v = split_qkv(2); q = norm_q->forward(ctx, q); k = norm_k->forward(ctx, k); diff --git a/src/model/diffusion/iris.hpp b/src/model/diffusion/iris.hpp index 662704e237..7ab0cd5727 100644 --- a/src/model/diffusion/iris.hpp +++ b/src/model/diffusion/iris.hpp @@ -91,11 +91,17 @@ namespace Iris { } ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* pe = nullptr, ggml_tensor* mask = nullptr) { - auto c = ctx->ggml_ctx; - auto qkv = ggml_ext_chunk(ctx, std::dynamic_pointer_cast(blocks["qkv"])->forward(ctx, x), 3, 0); - auto q = ggml_reshape_4d(c, qkv[0], dim / heads, heads, x->ne[1], x->ne[2]); - auto k = ggml_reshape_4d(c, qkv[1], dim / heads, heads, x->ne[1], x->ne[2]); - auto v = ggml_reshape_4d(c, qkv[2], dim / heads, heads, x->ne[1], x->ne[2]); + auto c = ctx->ggml_ctx; + auto head_dim = dim / heads; + auto qkv = std::dynamic_pointer_cast(blocks["qkv"])->forward(ctx, x); + auto qkv_dim = qkv->ne[0] / 3; + auto split_qkv = [&](int i) { + return ggml_view_4d(c, qkv, head_dim, qkv_dim / head_dim, qkv->ne[1], qkv->ne[2] * qkv->ne[3], + head_dim * qkv->nb[0], qkv->nb[1], qkv->nb[2], i * qkv_dim * qkv->nb[0]); + }; + auto q = split_qkv(0); + auto k = split_qkv(1); + auto v = split_qkv(2); if (qk_norm) { q = std::dynamic_pointer_cast(blocks["q_norm"])->forward(ctx, q); k = std::dynamic_pointer_cast(blocks["k_norm"])->forward(ctx, k); @@ -103,10 +109,9 @@ namespace Iris { if (pe) { x = Rope::attention(ctx, q, k, v, pe, mask); } else { - q = ggml_reshape_3d(c, q, dim, x->ne[1], x->ne[2]); - k = ggml_reshape_3d(c, k, dim, x->ne[1], x->ne[2]); - v = ggml_reshape_3d(c, v, dim, x->ne[1], x->ne[2]); - x = ggml_ext_attention_ext(ctx, q, k, v, heads, mask, false, ctx->flash_attn_enabled); + q = ggml_reshape_3d(c, ggml_cont(c, ggml_permute(c, q, 0, 2, 1, 3)), head_dim, x->ne[1], heads * x->ne[2]); + k = ggml_reshape_3d(c, ggml_cont(c, ggml_permute(c, k, 0, 2, 1, 3)), head_dim, x->ne[1], heads * x->ne[2]); + x = ggml_ext_attention_ext(ctx, q, k, v, heads, mask, true, ctx->flash_attn_enabled); } return std::dynamic_pointer_cast(blocks["proj"])->forward(ctx, x); } diff --git a/src/model/diffusion/lens.hpp b/src/model/diffusion/lens.hpp index 92e81d31d3..85c987fc3a 100644 --- a/src/model/diffusion/lens.hpp +++ b/src/model/diffusion/lens.hpp @@ -151,22 +151,24 @@ namespace Lens { int64_t N = img->ne[2]; int64_t inner = dim_head * num_heads; - auto img_qkv_vec = ggml_ext_chunk(ctx, img_qkv->forward(ctx, img), 3, 0); - auto txt_qkv_vec = ggml_ext_chunk(ctx, txt_qkv->forward(ctx, txt), 3, 0); - - auto img_q = ggml_reshape_4d(ctx->ggml_ctx, img_qkv_vec[0], dim_head, num_heads, n_img, N); - auto img_k = ggml_reshape_4d(ctx->ggml_ctx, img_qkv_vec[1], dim_head, num_heads, n_img, N); - auto img_v = ggml_reshape_4d(ctx->ggml_ctx, img_qkv_vec[2], dim_head, num_heads, n_img, N); - - img_q = norm_q->forward(ctx, img_q); - img_k = norm_k->forward(ctx, img_k); - - auto txt_q = ggml_reshape_4d(ctx->ggml_ctx, txt_qkv_vec[0], dim_head, num_heads, n_txt, N); - auto txt_k = ggml_reshape_4d(ctx->ggml_ctx, txt_qkv_vec[1], dim_head, num_heads, n_txt, N); - auto txt_v = ggml_reshape_4d(ctx->ggml_ctx, txt_qkv_vec[2], dim_head, num_heads, n_txt, N); - - txt_q = norm_add_q->forward(ctx, txt_q); - txt_k = norm_add_k->forward(ctx, txt_k); + auto img_qkv_f = img_qkv->forward(ctx, img); + auto txt_qkv_f = txt_qkv->forward(ctx, txt); + auto split_qkv = [&](ggml_tensor* qkv, int i) { + int64_t qkv_dim = qkv->ne[0] / 3; + return ggml_view_4d(ctx->ggml_ctx, qkv, dim_head, qkv_dim / dim_head, qkv->ne[1], qkv->ne[2] * qkv->ne[3], + dim_head * qkv->nb[0], qkv->nb[1], qkv->nb[2], i * qkv_dim * qkv->nb[0]); + }; + auto img_q = split_qkv(img_qkv_f, 0); + auto img_k = split_qkv(img_qkv_f, 1); + auto img_v = split_qkv(img_qkv_f, 2); + img_q = norm_q->forward(ctx, img_q); + img_k = norm_k->forward(ctx, img_k); + + auto txt_q = split_qkv(txt_qkv_f, 0); + auto txt_k = split_qkv(txt_qkv_f, 1); + auto txt_v = split_qkv(txt_qkv_f, 2); + txt_q = norm_add_q->forward(ctx, txt_q); + txt_k = norm_add_k->forward(ctx, txt_k); auto q = ggml_concat(ctx->ggml_ctx, img_q, txt_q, 2); auto k = ggml_concat(ctx->ggml_ctx, img_k, txt_k, 2); diff --git a/src/model/diffusion/mmdit.hpp b/src/model/diffusion/mmdit.hpp index ea653273bf..250f5f3120 100644 --- a/src/model/diffusion/mmdit.hpp +++ b/src/model/diffusion/mmdit.hpp @@ -331,14 +331,19 @@ class SelfAttention : public GGMLBlock { } std::vector pre_attention(GGMLRunnerContext* ctx, ggml_tensor* x) { + auto c = ctx->ggml_ctx; auto qkv_proj = std::dynamic_pointer_cast(blocks["qkv"]); - auto qkv = qkv_proj->forward(ctx, x); - auto qkv_vec = ggml_ext_chunk(ctx, qkv, 3, 0); - int64_t head_dim = qkv_vec[0]->ne[0] / num_heads; - auto q = ggml_reshape_4d(ctx->ggml_ctx, qkv_vec[0], head_dim, num_heads, qkv_vec[0]->ne[1], qkv_vec[0]->ne[2]); // [N, n_token, n_head, d_head] - auto k = ggml_reshape_4d(ctx->ggml_ctx, qkv_vec[1], head_dim, num_heads, qkv_vec[1]->ne[1], qkv_vec[1]->ne[2]); // [N, n_token, n_head, d_head] - auto v = qkv_vec[2]; // [N, n_token, n_head*d_head] + auto qkv = qkv_proj->forward(ctx, x); + auto qkv_dim = qkv->ne[0] / 3; + auto head_dim = qkv_dim / num_heads; + auto split_qkv = [&](int i) { + return ggml_view_4d(c, qkv, head_dim, qkv_dim / head_dim, qkv->ne[1], qkv->ne[2] * qkv->ne[3], + head_dim * qkv->nb[0], qkv->nb[1], qkv->nb[2], i * qkv_dim * qkv->nb[0]); + }; + auto q = split_qkv(0); + auto k = split_qkv(1); + auto v = split_qkv(2); if (qk_norm == "rms" || qk_norm == "ln") { auto ln_q = std::dynamic_pointer_cast(blocks["ln_q"]); @@ -347,8 +352,9 @@ class SelfAttention : public GGMLBlock { k = ln_k->forward(ctx, k); } - q = ggml_reshape_3d(ctx->ggml_ctx, q, q->ne[0] * q->ne[1], q->ne[2], q->ne[3]); // [N, n_token, n_head*d_head] - k = ggml_reshape_3d(ctx->ggml_ctx, k, k->ne[0] * k->ne[1], k->ne[2], k->ne[3]); // [N, n_token, n_head*d_head] + // skip_reshape=true contract: q/k become [d_head, L, n_head*N]; v stays a 4-D view. + q = ggml_reshape_3d(c, ggml_cont(c, ggml_permute(c, q, 0, 2, 1, 3)), q->ne[0], q->ne[2], num_heads * q->ne[3]); + k = ggml_reshape_3d(c, ggml_cont(c, ggml_permute(c, k, 0, 2, 1, 3)), k->ne[0], k->ne[2], num_heads * k->ne[3]); return {q, k, v}; } @@ -366,8 +372,8 @@ class SelfAttention : public GGMLBlock { ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) { auto qkv = pre_attention(ctx, x); - x = ggml_ext_attention_ext(ctx, qkv[0], qkv[1], qkv[2], num_heads, nullptr, false, ctx->flash_attn_enabled); // [N, n_token, dim] - x = post_attention(ctx, x); // [N, n_token, dim] + x = ggml_ext_attention_ext(ctx, qkv[0], qkv[1], qkv[2], num_heads, nullptr, true, ctx->flash_attn_enabled); // [N, n_token, dim] + x = post_attention(ctx, x); // [N, n_token, dim] return x; } }; @@ -586,8 +592,8 @@ struct DismantledBlock : public GGMLBlock { auto qkv2 = std::get<1>(qkv_intermediates); auto intermediates = std::get<2>(qkv_intermediates); - auto attn_out = ggml_ext_attention_ext(ctx, qkv[0], qkv[1], qkv[2], num_heads, nullptr, false, ctx->flash_attn_enabled); // [N, n_token, dim] - auto attn2_out = ggml_ext_attention_ext(ctx, qkv2[0], qkv2[1], qkv2[2], num_heads, nullptr, false, ctx->flash_attn_enabled); // [N, n_token, dim] + auto attn_out = ggml_ext_attention_ext(ctx, qkv[0], qkv[1], qkv[2], num_heads, nullptr, true, ctx->flash_attn_enabled); // [N, n_token, dim] + auto attn2_out = ggml_ext_attention_ext(ctx, qkv2[0], qkv2[1], qkv2[2], num_heads, nullptr, true, ctx->flash_attn_enabled); // [N, n_token, dim] x = post_attention_x(ctx, attn_out, attn2_out, @@ -603,7 +609,7 @@ struct DismantledBlock : public GGMLBlock { auto qkv = qkv_intermediates.first; auto intermediates = qkv_intermediates.second; - auto attn_out = ggml_ext_attention_ext(ctx, qkv[0], qkv[1], qkv[2], num_heads, nullptr, false, ctx->flash_attn_enabled); // [N, n_token, dim] + auto attn_out = ggml_ext_attention_ext(ctx, qkv[0], qkv[1], qkv[2], num_heads, nullptr, true, ctx->flash_attn_enabled); // [N, n_token, dim] x = post_attention(ctx, attn_out, intermediates[0], @@ -644,10 +650,11 @@ block_mixing(GGMLRunnerContext* ctx, } std::vector qkv; for (int i = 0; i < 3; i++) { - qkv.push_back(ggml_concat(ctx->ggml_ctx, context_qkv[i], x_qkv[i], 1)); + // q/k are 3-D [dim, L, N] (token axis = 1); v is a 4-D view [d_head, heads, L, N] (token axis = 2) + qkv.push_back(ggml_concat(ctx->ggml_ctx, context_qkv[i], x_qkv[i], i == 2 ? 2 : 1)); } - auto attn = ggml_ext_attention_ext(ctx, qkv[0], qkv[1], qkv[2], x_block->num_heads, nullptr, false, ctx->flash_attn_enabled); // [N, n_context + n_token, hidden_size] + auto attn = ggml_ext_attention_ext(ctx, qkv[0], qkv[1], qkv[2], x_block->num_heads, nullptr, true, ctx->flash_attn_enabled); // [N, n_context + n_token, hidden_size] auto context_attn = ggml_view_3d(ctx->ggml_ctx, attn, @@ -679,7 +686,7 @@ block_mixing(GGMLRunnerContext* ctx, } if (x_block->self_attn) { - auto attn2 = ggml_ext_attention_ext(ctx, x_qkv2[0], x_qkv2[1], x_qkv2[2], x_block->num_heads, nullptr, false, ctx->flash_attn_enabled); // [N, n_token, hidden_size] + auto attn2 = ggml_ext_attention_ext(ctx, x_qkv2[0], x_qkv2[1], x_qkv2[2], x_block->num_heads, nullptr, true, ctx->flash_attn_enabled); // [N, n_token, hidden_size] x = x_block->post_attention_x(ctx, x_attn, diff --git a/src/model/diffusion/moss_image.hpp b/src/model/diffusion/moss_image.hpp index bcca79b89f..4bb9077c42 100644 --- a/src/model/diffusion/moss_image.hpp +++ b/src/model/diffusion/moss_image.hpp @@ -9,7 +9,6 @@ #include "model/common/ggml_block.hpp" #include "model/common/rope.hpp" -#include "model/diffusion/dit.hpp" #include "model/diffusion/model.hpp" #include "model_loader.h" @@ -164,9 +163,10 @@ namespace MOSSImage { auto c = ctx->ggml_ctx; auto linear = [&](const char* name, ggml_tensor* h) { return std::dynamic_pointer_cast(blocks[name])->forward(ctx, h); }; int64_t image_len = image->ne[0] * image->ne[1]; - auto x = linear("video_embedder", DiT::patchify(ctx, image, 1, 1)); + auto pixels = ggml_reshape_3d(c, image, image_len, image->ne[2], image->ne[3]); + auto x = linear("video_embedder", ggml_cont(c, ggml_transpose(c, pixels))); text = linear("text_embedder.proj", text); - auto text_pe = ggml_ext_slice(ctx, pe, 3, image_len, image_len + text->ne[1]); + auto text_pe = ggml_ext_slice(ctx, pe, 3, image_len, image_len + text->ne[1], false); for (int i = 0; i < 2; ++i) { text = std::dynamic_pointer_cast(blocks["text_embedder.blocks." + std::to_string(i)])->forward(ctx, text, text_pe); } @@ -175,7 +175,7 @@ namespace MOSSImage { auto head = linear("video_head_modulation", time); std::array parts; for (int i = 0; i < 6; ++i) { - parts[i] = ggml_ext_slice(ctx, mod, 0, i * config.dim, (i + 1) * config.dim); + parts[i] = ggml_ext_slice(ctx, mod, 0, i * config.dim, (i + 1) * config.dim, false); if (ctx->graph != nullptr) { ggml_build_forward_expand(ctx->graph, parts[i]); } @@ -187,18 +187,18 @@ namespace MOSSImage { sd::ggml_graph_cut::mark_graph_cut(x, "moss_image.layers." + std::to_string(i), "x"); } } - x = ggml_ext_slice(ctx, x, 1, 0, image_len); + x = ggml_ext_slice(ctx, x, 1, 0, image_len, false); x = std::dynamic_pointer_cast(blocks["final_norm_video"])->forward(ctx, x); - auto shift = ggml_ext_slice(ctx, head, 0, 0, config.dim); - auto scale = ggml_ext_slice(ctx, head, 0, config.dim, 2 * config.dim); + auto shift = ggml_ext_slice(ctx, head, 0, 0, config.dim, false); + auto scale = ggml_ext_slice(ctx, head, 0, config.dim, 2 * config.dim, false); x = ggml_ext_modulate(ctx, x, shift, scale); x = linear("final_linear_video", x); - return DiT::unpatchify(ctx, x, image->ne[1], image->ne[0], 1, 1); + return ggml_cont_4d(c, ggml_transpose(c, x), image->ne[0], image->ne[1], image->ne[2], image->ne[3]); } }; struct MOSSImageRunner : public DiffusionModelRunner { - sd::PECache pe_cache_; + sd::PECache pe_cache_{2}; MOSSImageConfig config; Model model; std::vector time_data; diff --git a/src/model/diffusion/pid.hpp b/src/model/diffusion/pid.hpp index 372e15729c..1002bf7306 100644 --- a/src/model/diffusion/pid.hpp +++ b/src/model/diffusion/pid.hpp @@ -293,16 +293,18 @@ namespace Pid { auto proj = std::dynamic_pointer_cast(blocks["proj"]); auto qkv = qkv_proj->forward(ctx, x); - auto qkv_vec = ggml_ext_chunk(ctx, qkv, 3, 0); - int64_t L = x->ne[1]; - int64_t N = x->ne[2]; int64_t head_dim = dim / num_heads; - auto q = ggml_reshape_4d(ctx->ggml_ctx, qkv_vec[0], head_dim, num_heads, L, N); - auto k = ggml_reshape_4d(ctx->ggml_ctx, qkv_vec[1], head_dim, num_heads, L, N); - auto v = ggml_reshape_4d(ctx->ggml_ctx, qkv_vec[2], head_dim, num_heads, L, N); - q = q_norm->forward(ctx, q); - k = k_norm->forward(ctx, k); - x = Rope::attention(ctx, q, k, v, pos, nullptr, 1.0f / 128.f, true); + auto qkv_dim = qkv->ne[0] / 3; + auto split_qkv = [&](int i) { + return ggml_view_4d(ctx->ggml_ctx, qkv, head_dim, qkv_dim / head_dim, qkv->ne[1], qkv->ne[2] * qkv->ne[3], + head_dim * qkv->nb[0], qkv->nb[1], qkv->nb[2], i * qkv_dim * qkv->nb[0]); + }; + auto q = split_qkv(0); + auto k = split_qkv(1); + auto v = split_qkv(2); + q = q_norm->forward(ctx, q); + k = k_norm->forward(ctx, k); + x = Rope::attention(ctx, q, k, v, pos, nullptr, 1.0f / 128.f, true); return proj->forward(ctx, x); } }; @@ -340,20 +342,25 @@ namespace Pid { int64_t Nx = x->ne[1]; int64_t Ny = y->ne[1]; - int64_t N = x->ne[2]; int64_t head_dim = dim / num_heads; - auto qkv_x = ggml_ext_chunk(ctx, qkv_x_proj->forward(ctx, x), 3, 0); - auto qx = ggml_reshape_4d(ctx->ggml_ctx, qkv_x[0], head_dim, num_heads, Nx, N); - auto kx = ggml_reshape_4d(ctx->ggml_ctx, qkv_x[1], head_dim, num_heads, Nx, N); - auto vx = ggml_reshape_4d(ctx->ggml_ctx, qkv_x[2], head_dim, num_heads, Nx, N); + auto split_qkv = [&](ggml_tensor* qkv, int i) { + int64_t qkv_dim = qkv->ne[0] / 3; + return ggml_view_4d(ctx->ggml_ctx, qkv, head_dim, qkv_dim / head_dim, qkv->ne[1], qkv->ne[2] * qkv->ne[3], + head_dim * qkv->nb[0], qkv->nb[1], qkv->nb[2], i * qkv_dim * qkv->nb[0]); + }; + + auto qkv_x = qkv_x_proj->forward(ctx, x); + auto qx = split_qkv(qkv_x, 0); + auto kx = split_qkv(qkv_x, 1); + auto vx = split_qkv(qkv_x, 2); qx = q_norm_x->forward(ctx, qx); kx = k_norm_x->forward(ctx, kx); - auto qkv_y = ggml_ext_chunk(ctx, qkv_y_proj->forward(ctx, y), 3, 0); - auto qy = ggml_reshape_4d(ctx->ggml_ctx, qkv_y[0], head_dim, num_heads, Ny, N); - auto ky = ggml_reshape_4d(ctx->ggml_ctx, qkv_y[1], head_dim, num_heads, Ny, N); - auto vy = ggml_reshape_4d(ctx->ggml_ctx, qkv_y[2], head_dim, num_heads, Ny, N); + auto qkv_y = qkv_y_proj->forward(ctx, y); + auto qy = split_qkv(qkv_y, 0); + auto ky = split_qkv(qkv_y, 1); + auto vy = split_qkv(qkv_y, 2); qy = q_norm_y->forward(ctx, qy); ky = k_norm_y->forward(ctx, ky); diff --git a/src/model/diffusion/seedvr2.hpp b/src/model/diffusion/seedvr2.hpp index 0f0a62e9d9..dad1e7a8ea 100644 --- a/src/model/diffusion/seedvr2.hpp +++ b/src/model/diffusion/seedvr2.hpp @@ -194,19 +194,26 @@ namespace SeedVR2 { return x; return std::dynamic_pointer_cast(blocks["ada." + branch(text)])->forward(ctx, x, emb, layer, gate); }; - auto vqkv = linear("attn.proj_qkv", false, ada(ggml_rms_norm(c, vid, 1e-5f), false, "attn")); - auto tqkv = linear("attn.proj_qkv", true, ada(ggml_rms_norm(c, txt, 1e-5f), true, "attn")); - vqkv = ggml_get_rows(c, vqkv, inputs.input(ctx, plan.order)); - auto vpe = rotation(ctx, inputs.input(ctx, plan.positions)); - auto tpe = rotation(ctx, inputs.input(ctx, plan.text_positions)); - auto qkv = [&](ggml_tensor* x, bool text, ggml_tensor* pe) { - auto parts = ggml_ext_chunk(ctx, x, 3, 0); + auto vqkv = linear("attn.proj_qkv", false, ada(ggml_rms_norm(c, vid, 1e-5f), false, "attn")); + auto tqkv = linear("attn.proj_qkv", true, ada(ggml_rms_norm(c, txt, 1e-5f), true, "attn")); + vqkv = ggml_get_rows(c, vqkv, inputs.input(ctx, plan.order)); + auto vpe = rotation(ctx, inputs.input(ctx, plan.positions)); + auto tpe = rotation(ctx, inputs.input(ctx, plan.text_positions)); + auto split_qkv = [&](ggml_tensor* x, int i) { + int64_t qkv_dim = x->ne[0] / 3; + return ggml_view_4d(c, x, 128, qkv_dim / 128, x->ne[1], x->ne[2] * x->ne[3], + 128 * x->nb[0], x->nb[1], x->nb[2], i * qkv_dim * x->nb[0]); + }; + auto qkv = [&](ggml_tensor* x, bool text, ggml_tensor* pe) { + std::vector parts{nullptr, nullptr, nullptr}; for (int i = 0; i < 2; ++i) { - auto h = ggml_reshape_3d(c, parts[i], 128, heads, x->ne[1]); - h = std::dynamic_pointer_cast(blocks[std::string(i == 0 ? "attn.norm_q." : "attn.norm_k.") + branch(text)])->forward(ctx, h); + auto h = std::dynamic_pointer_cast(blocks[std::string(i == 0 ? "attn.norm_q." : "attn.norm_k.") + branch(text)])->forward(ctx, split_qkv(x, i)); h = ggml_reshape_2d(c, h, dim, x->ne[1]); parts[i] = VSR::apply_rope(ctx, h, pe, heads); } + // v must stay 3-D [dim, L, N]: the window loop slices dim 1 as tokens. + auto v = split_qkv(x, 2); + parts[2] = ggml_view_3d(c, v, dim, x->ne[1], x->ne[2], v->nb[2], v->nb[3], 0); return parts; }; auto vp = qkv(vqkv, false, vpe), tp = qkv(tqkv, true, tpe); diff --git a/src/model/integrations/moss_image.cpp b/src/model/integrations/moss_image.cpp index 4a85d169a5..6a2114da2c 100644 --- a/src/model/integrations/moss_image.cpp +++ b/src/model/integrations/moss_image.cpp @@ -8,7 +8,7 @@ static bool build_moss_image(const sd::model_builders::Context& ctx, const TokenizerConfig& tokenizers, sd::model_builders::CoreRunners& runners) { runners.conditioner = std::make_shared(ctx.backends.runtime_backend(SDBackendModule::TE), - ctx.tensor_storage_map, ctx.version, "", false, + ctx.tensor_storage_map, ctx.version, "", true, ctx.weight_manager, tokenizers); runners.diffusion = std::make_shared(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION), ctx.tensor_storage_map, "model.diffusion_model", ctx.weight_manager); diff --git a/src/model/integrations/sd.cpp b/src/model/integrations/sd.cpp index a36ff8e36d..24a9acfc17 100644 --- a/src/model/integrations/sd.cpp +++ b/src/model/integrations/sd.cpp @@ -1,4 +1,5 @@ #include "conditioning/conditioner.hpp" +#include "core/option_types.h" #include "model/diffusion/unet.hpp" #include "model/vae/auto_encoder_kl.hpp" #include "model_definition.h" diff --git a/src/model/te/llm.hpp b/src/model/te/llm.hpp index e6d5626029..647ff517ee 100644 --- a/src/model/te/llm.hpp +++ b/src/model/te/llm.hpp @@ -20,6 +20,7 @@ #include #include +#include "conditioning/prompt_input.h" #include "core/ggml_extend.h" #include "core/ggml_extend_backend.h" #include "core/ggml_runner.h" @@ -110,6 +111,16 @@ namespace LLM { using ImageEmbeds = std::vector>>; using DeepStackImageEmbeds = std::vector; + struct PromptTokens { + std::vector tokens; + int expanded_length = 0; + std::vector weights; + std::vector mask; + ImageEmbeds image_embeds; + DeepStackImageEmbeds deepstack_image_embeds; + std::vector image_grids; + }; + struct LLMConfig { LLMArch arch = LLMArch::QWEN2_5_VL; int64_t num_layers = 28; @@ -176,18 +187,25 @@ namespace LLM { config.qkv_bias = false; config.rms_norm_eps = 1e-5f; } else if (arch == LLMArch::QWEN3_5) { - config.hidden_size = 4096; - config.intermediate_size = 12288; - config.head_dim = 256; - config.num_heads = 16; - config.num_kv_heads = 4; - config.qkv_bias = false; - config.qk_norm = true; - config.rms_norm_add = true; - config.partial_rotary = 0.25f; - config.rope_thetas = {10000000.f}; - config.max_position_embeddings = 262144; - config.vocab_size = 248320; + config.hidden_size = 4096; + config.intermediate_size = 12288; + config.head_dim = 256; + config.num_heads = 16; + config.num_kv_heads = 4; + config.qkv_bias = false; + config.qk_norm = true; + config.rms_norm_add = true; + config.partial_rotary = 0.25f; + config.rope_thetas = {10000000.f}; + config.max_position_embeddings = 262144; + config.vocab_size = 248320; + config.vision.arch = LLMVisionArch::QWEN3_VL; + config.vision.num_layers = 27; + config.vision.hidden_size = 1152; + config.vision.intermediate_size = 4304; + config.vision.out_hidden_size = 4096; + config.vision.patch_size = 16; + config.vision.num_position_embeddings = 2304; } else if (arch == LLMArch::QWEN3 || arch == LLMArch::QWEN3_VL) { config.head_dim = 128; config.num_heads = 32; @@ -1234,31 +1252,30 @@ namespace LLM { ggml_tensor* pe, ggml_tensor* mask = nullptr) { // x: [N, n_token, hidden_size] - int64_t n_token = x->ne[1]; - int64_t N = x->ne[2]; - auto proj = std::dynamic_pointer_cast(blocks["proj"]); + auto proj = std::dynamic_pointer_cast(blocks["proj"]); - std::vector qkv_vec; + ggml_tensor *q, *k, *v; if (llama_cpp_style) { auto q_proj = std::dynamic_pointer_cast(blocks["q_proj"]); auto k_proj = std::dynamic_pointer_cast(blocks["k_proj"]); auto v_proj = std::dynamic_pointer_cast(blocks["v_proj"]); - auto q = q_proj->forward(ctx, x); - auto k = k_proj->forward(ctx, x); - auto v = v_proj->forward(ctx, x); - - qkv_vec = {q, k, v}; + q = ggml_reshape_4d(ctx->ggml_ctx, q_proj->forward(ctx, x), head_dim, num_heads, x->ne[1], x->ne[2]); + k = ggml_reshape_4d(ctx->ggml_ctx, k_proj->forward(ctx, x), head_dim, num_heads, x->ne[1], x->ne[2]); + v = ggml_reshape_4d(ctx->ggml_ctx, v_proj->forward(ctx, x), head_dim, num_heads, x->ne[1], x->ne[2]); } else { - auto qkv_proj = std::dynamic_pointer_cast(blocks["qkv"]); - auto qkv = qkv_proj->forward(ctx, x); - qkv_vec = ggml_ext_chunk(ctx, qkv, 3, 0); + auto qkv_proj = std::dynamic_pointer_cast(blocks["qkv"]); + auto qkv = qkv_proj->forward(ctx, x); + auto qkv_dim = qkv->ne[0] / 3; + auto split_qkv = [&](int i) { + return ggml_view_4d(ctx->ggml_ctx, qkv, head_dim, qkv_dim / head_dim, qkv->ne[1], qkv->ne[2] * qkv->ne[3], + head_dim * qkv->nb[0], qkv->nb[1], qkv->nb[2], i * qkv_dim * qkv->nb[0]); + }; + q = split_qkv(0); + k = split_qkv(1); + v = split_qkv(2); } - auto q = ggml_reshape_4d(ctx->ggml_ctx, qkv_vec[0], head_dim, num_heads, qkv_vec[0]->ne[1], qkv_vec[0]->ne[2]); // [N, n_token, n_head, d_head] - auto k = ggml_reshape_4d(ctx->ggml_ctx, qkv_vec[1], head_dim, num_heads, qkv_vec[1]->ne[1], qkv_vec[1]->ne[2]); // [N, n_token, n_head, d_head] - auto v = ggml_reshape_4d(ctx->ggml_ctx, qkv_vec[2], head_dim, num_heads, qkv_vec[2]->ne[1], qkv_vec[2]->ne[2]); // [N, n_token, n_head, d_head] - x = Rope::attention(ctx, q, k, v, pe, mask, 1.f, false); // [N, n_token, hidden_size] x = proj->forward(ctx, x); // [N, n_token, hidden_size] @@ -1801,6 +1818,11 @@ namespace LLM { 1.f, 32.f, 1.f); + } else if (arch == LLMArch::QWEN3_5 && input_pos->ne[0] == n_token * 4) { + int sections[4] = {11, 11, 10, 0}; + int rotary_dim = rope_pairs * 2; + q = ggml_rope_multi(ctx->ggml_ctx, q, input_pos, nullptr, rotary_dim, sections, GGML_ROPE_TYPE_IMROPE, static_cast(max_position_embeddings), rope_thetas[0], 1.f, 0.f, 1.f, 32.f, 1.f); + k = ggml_rope_multi(ctx->ggml_ctx, k, input_pos, nullptr, rotary_dim, sections, GGML_ROPE_TYPE_IMROPE, static_cast(max_position_embeddings), rope_thetas[0], 1.f, 0.f, 1.f, 32.f, 1.f); } else if (arch == LLMArch::LLADA2_MOE || arch == LLMArch::QWEN3_5) { // LLaDA2 slices the head (query[..., :rotary_dim]) instead of zero-padding // inv_freq like gemma does, so rotate_half pairs i with i + rotary_dim/2 and the @@ -2208,6 +2230,16 @@ namespace LLM { std::array, 4> pos_embed_idx_data_; std::array, 4> pos_embed_weight_data_; + static bool check_window_mask_size(int64_t grid_h, int64_t grid_w) { + // The dense float mask must fit within 256 MiB, including rectangular grids. + constexpr int64_t max_patches = 8192; + if (grid_h <= 0 || grid_w <= 0 || grid_h > max_patches || grid_w > max_patches / grid_h) { + LOG_ERROR("Qwen2.5-VL images support at most %" PRId64 " patches (256 MiB window mask)", max_patches); + return false; + } + return true; + } + static ggml_tensor* process_image_common(GGMLRunnerContext* runner_ctx, ggml_tensor* image, const LLMVisionConfig& vision_params) { @@ -2318,8 +2350,12 @@ namespace LLM { GGML_ASSERT(image->ne[1] % (vision_params.patch_size * vision_params.spatial_merge_size) == 0); GGML_ASSERT(image->ne[0] % (vision_params.patch_size * vision_params.spatial_merge_size) == 0); - int grid_h = static_cast(image->ne[1]) / vision_params.patch_size; - int grid_w = static_cast(image->ne[0]) / vision_params.patch_size; + if (vision_params.arch == LLMVisionArch::QWEN2_5_VL && + !check_window_mask_size(image->ne[1] / vision_params.patch_size, image->ne[0] / vision_params.patch_size)) { + throw std::invalid_argument("Qwen2.5-VL window mask exceeds its memory limit"); + } + int grid_h = static_cast(image->ne[1] / vision_params.patch_size); + int grid_w = static_cast(image->ne[0] / vision_params.patch_size); auto pixel_values = process_image_common(runner_ctx, image, vision_params); int head_dim = static_cast(vision_params.hidden_size / vision_params.num_heads); @@ -2381,7 +2417,8 @@ namespace LLM { runner->set_backend_tensor_data(window_index, window_index_vec.data()); runner->set_backend_tensor_data(window_inverse_index, window_inverse_index_vec.data()); - window_mask_vec.resize((grid_h * grid_w) * (grid_h * grid_w)); + const size_t n_patches = static_cast(grid_h) * grid_w; + window_mask_vec.resize(n_patches * n_patches); int window_start_index = 0; for (int seq_index = 0; seq_index < seqlens.size(); seq_index++) { int window_end_index = window_start_index + seqlens[seq_index]; @@ -2392,8 +2429,9 @@ namespace LLM { if (j >= window_start_index && j < window_end_index) { mask_value = 0; } - GGML_ASSERT((i * (grid_h * grid_w) + j) < window_mask_vec.size()); - window_mask_vec[i * (grid_h * grid_w) + j] = mask_value; + const size_t index = static_cast(i) * n_patches + j; + GGML_ASSERT(index < window_mask_vec.size()); + window_mask_vec[index] = mask_value; } } window_start_index = window_end_index; @@ -2487,38 +2525,14 @@ namespace LLM { return hidden_states; } - ggml_cgraph* build_graph(const sd::Tensor& input_ids_tensor, - const sd::Tensor& attention_mask_tensor, - const ImageEmbeds& image_embeds_tensor, - const DeepStackImageEmbeds& deepstack_image_embeds_tensor, - const std::vector& image_grids, - std::set out_layers, - bool return_all_hidden_states = false, - const sd::Tensor& position_ids = {}) { - ggml_cgraph* gf = new_graph_custom(LLM_GRAPH_SIZE); - ggml_tensor* input_ids = make_input(input_ids_tensor); - std::vector> image_embeds; - image_embeds.reserve(image_embeds_tensor.size()); - for (const auto& [idx, embed_tensor] : image_embeds_tensor) { - ggml_tensor* embed = make_input(embed_tensor); - image_embeds.emplace_back(idx, embed); - } - std::vector>> deepstack_image_embeds(deepstack_image_embeds_tensor.size()); - for (size_t layer = 0; layer < deepstack_image_embeds_tensor.size(); ++layer) { - deepstack_image_embeds[layer].reserve(deepstack_image_embeds_tensor[layer].size()); - for (const auto& [idx, embed_tensor] : deepstack_image_embeds_tensor[layer]) { - deepstack_image_embeds[layer].emplace_back(idx, make_input(embed_tensor)); - } - } - - int64_t n_tokens = input_ids->ne[0]; - for (const auto& image_embed : image_embeds) { - n_tokens += image_embed.second->ne[1] - 1; - } + static std::vector build_position_ids(const LLMConfig& config, + int64_t n_tokens, + const std::vector& image_grids) { + std::vector input_pos_vec; if (config.arch == LLMArch::MISTRAL_SMALL_3_2 || config.arch == LLMArch::MINISTRAL_3_3B || config.arch == LLMArch::QWEN3 || - config.arch == LLMArch::QWEN3_5 || + (config.arch == LLMArch::QWEN3_5 && image_grids.empty()) || config.arch == LLMArch::GEMMA3_12B || config.arch == LLMArch::GEMMA4_12B || config.arch == LLMArch::GEMMA2_2B || @@ -2536,7 +2550,7 @@ namespace LLM { input_pos_vec[2 * n_tokens + i] = i; input_pos_vec[3 * n_tokens + i] = 0; } - if (config.arch == LLMArch::QWEN3_VL && !image_grids.empty()) { + if ((config.arch == LLMArch::QWEN2_5_VL || config.arch == LLMArch::QWEN3_VL || config.arch == LLMArch::QWEN3_5) && !image_grids.empty()) { int offset = 0; for (const auto& grid : image_grids) { int end = grid.index + grid.size; @@ -2562,6 +2576,39 @@ namespace LLM { } } + return input_pos_vec; + } + + ggml_cgraph* build_graph(const sd::Tensor& input_ids_tensor, + const sd::Tensor& attention_mask_tensor, + const ImageEmbeds& image_embeds_tensor, + const DeepStackImageEmbeds& deepstack_image_embeds_tensor, + const std::vector& image_grids, + std::set out_layers, + bool return_all_hidden_states = false, + const sd::Tensor& position_ids = {}) { + ggml_cgraph* gf = new_graph_custom(LLM_GRAPH_SIZE); + ggml_tensor* input_ids = make_input(input_ids_tensor); + std::vector> image_embeds; + image_embeds.reserve(image_embeds_tensor.size()); + for (const auto& [idx, embed_tensor] : image_embeds_tensor) { + ggml_tensor* embed = make_input(embed_tensor); + image_embeds.emplace_back(idx, embed); + } + std::vector>> deepstack_image_embeds(deepstack_image_embeds_tensor.size()); + for (size_t layer = 0; layer < deepstack_image_embeds_tensor.size(); ++layer) { + deepstack_image_embeds[layer].reserve(deepstack_image_embeds_tensor[layer].size()); + for (const auto& [idx, embed_tensor] : deepstack_image_embeds_tensor[layer]) { + deepstack_image_embeds[layer].emplace_back(idx, make_input(embed_tensor)); + } + } + + int64_t n_tokens = input_ids->ne[0]; + for (const auto& image_embed : image_embeds) { + n_tokens += image_embed.second->ne[1] - 1; + } + input_pos_vec = build_position_ids(config, n_tokens, image_grids); + if (!position_ids.empty()) { GGML_ASSERT(position_ids.numel() == static_cast(input_pos_vec.size())); input_pos_vec = position_ids.values(); @@ -2878,6 +2925,149 @@ namespace LLM { return outputs; } + bool supports_prompt_images() const { + return enable_vision && (config.arch == LLMArch::QWEN2_5_VL || config.arch == LLMArch::QWEN3_5 || config.arch == LLMArch::QWEN3_VL); + } + + bool prepare_prompt(int n_threads, + Tokenizer& tokenizer, + const sd::PromptInput& input, + int max_length, + PromptTokens& result, + const ImageEmbeds& reference_embeds = {}, + const DeepStackImageEmbeds& reference_deepstack = {}, + const std::vector& reference_grids = {}) { + result = {}; + if (!supports_prompt_images()) { + LOG_ERROR("%s requires compatible vision weights for inline prompt images", get_desc().c_str()); + return false; + } + std::vector image_token, vision_start, vision_end; + if (!tokenizer.encode("<|image_pad|>", image_token) || image_token.size() != 1 || + !tokenizer.encode("<|vision_start|>", vision_start) || vision_start.size() != 1 || + !tokenizer.encode("<|vision_end|>", vision_end) || vision_end.size() != 1) { + LOG_ERROR("The LLM tokenizer must encode vision markers as single special tokens"); + return false; + } + size_t reference_index = 0; + result.deepstack_image_embeds.resize(reference_deepstack.size()); + auto append_tokens = [&](const std::vector& tokens, float weight) { + for (int token : tokens) { + const bool reference = reference_index < reference_embeds.size() && + reference_embeds[reference_index].first == result.expanded_length; + const int count = reference ? static_cast(reference_embeds[reference_index].second.shape()[1]) : 1; + if (count > max_length - result.expanded_length) { + LOG_ERROR("Multimodal prompt exceeds the %d-token context", max_length); + return false; + } + if (reference) { + const auto& embed = reference_embeds[reference_index++]; + result.image_embeds.push_back(embed); + for (size_t layer = 0; layer < reference_deepstack.size(); ++layer) { + for (const auto& extra : reference_deepstack[layer]) { + if (extra.first == embed.first) { + result.deepstack_image_embeds[layer].push_back(extra); + } + } + } + for (const auto& grid : reference_grids) { + if (grid.index == embed.first) { + result.image_grids.push_back(grid); + } + } + } + result.tokens.push_back(token); + result.weights.insert(result.weights.end(), count, reference ? 1.f : weight); + result.expanded_length += count; + } + return true; + }; + for (const auto& part : input) { + if (part.type == sd::PromptPart::Type::TEXT) { + std::vector tokens; + if (!tokenizer.encode(part.content, tokens) || !append_tokens(tokens, part.weight)) { + return false; + } + continue; + } + if (!append_tokens(vision_start, 1.f)) { + return false; + } + if (!part.image || part.image->empty() || part.image->dim() != 4 || part.image->shape()[2] != 3 || part.image->shape()[3] != 1) { + LOG_ERROR("Prompt images must be RGB tensors with one frame"); + return false; + } + const auto& source = *part.image; + const int factor = config.vision.patch_size * config.vision.spatial_merge_size; + const double height = static_cast(source.shape()[1]); + const double width = static_cast(source.shape()[0]); + if (std::max(height, width) / std::min(height, width) > 200.0) { + LOG_ERROR("Prompt image aspect ratio exceeds 200"); + return false; + } + int h = std::max(factor, static_cast(std::round(height / factor)) * factor); + int w = std::max(factor, static_cast(std::round(width / factor)) * factor); + const bool qwen25 = config.arch == LLMArch::QWEN2_5_VL; + const double min_pixels = qwen25 ? 3136.0 : 65536.0; + const double max_pixels = qwen25 ? 12845056.0 : 16777216.0; + if (static_cast(h) * w > max_pixels) { + const double scale = std::sqrt(height * width / max_pixels); + h = std::max(factor, static_cast(std::floor(height / scale / factor)) * factor); + w = std::max(factor, static_cast(std::floor(width / scale / factor)) * factor); + } else if (static_cast(h) * w < min_pixels) { + const double scale = std::sqrt(min_pixels / (height * width)); + h = static_cast(std::ceil(height * scale / factor)) * factor; + w = static_cast(std::ceil(width * scale / factor)) * factor; + } + const int count = (h / factor) * (w / factor); + if (count > max_length - result.expanded_length) { + LOG_ERROR("Prompt images exceed the %d-token context", max_length); + return false; + } + if (qwen25 && !check_window_mask_size(h / config.vision.patch_size, w / config.vision.patch_size)) { + return false; + } + auto image = sd::ops::interpolate(source, {w, h, 3, 1}, sd::ops::InterpolateMode::Bicubic, false, true); + const float means[] = {0.48145466f, 0.4578275f, 0.40821073f}; + const float stds[] = {0.26862954f, 0.26130258f, 0.27577711f}; + for (int channel = 0; channel < 3; ++channel) { + for (int64_t pixel = 0; pixel < int64_t(w) * h; ++pixel) { + float& value = image[pixel + int64_t(channel) * w * h]; + value = (std::clamp(value, 0.f, 1.f) - (qwen25 ? means[channel] : 0.5f)) / + (qwen25 ? stds[channel] : 0.5f); + } + } + auto outputs = encode_image_outputs(n_threads, image, false); + if (outputs.empty() || outputs[0].shape()[0] != config.hidden_size || outputs[0].shape()[1] != count) { + LOG_ERROR("Prompt image encoder returned an invalid embedding shape"); + return false; + } + const int index = result.expanded_length; + result.tokens.push_back(image_token[0]); + result.weights.insert(result.weights.end(), count, 1.f); + result.expanded_length += count; + result.image_embeds.emplace_back(index, std::move(outputs[0])); + if (result.deepstack_image_embeds.empty()) { + result.deepstack_image_embeds.resize(outputs.size() - 1); + } + if (result.deepstack_image_embeds.size() != outputs.size() - 1) { + return false; + } + for (size_t layer = 1; layer < outputs.size(); ++layer) { + result.deepstack_image_embeds[layer - 1].emplace_back(index, std::move(outputs[layer])); + } + result.image_grids.push_back({index, count, h / config.vision.patch_size, w / config.vision.patch_size}); + if (!append_tokens(vision_end, 1.f)) { + return false; + } + } + if (reference_index != reference_embeds.size()) { + LOG_ERROR("Reference images do not match the tokenized prompt"); + return false; + } + return true; + } + std::vector> encode_video_block_outputs(const int n_threads, const sd::Tensor& frames, bool auto_runner_end = false) { diff --git a/src/model/vae/minimax_h3_audio_vae.hpp b/src/model/vae/minimax_h3_audio_vae.hpp index d5eb0d0fbf..d83b992e86 100644 --- a/src/model/vae/minimax_h3_audio_vae.hpp +++ b/src/model/vae/minimax_h3_audio_vae.hpp @@ -171,9 +171,9 @@ namespace MiniMaxH3 { auto v = ggml_add(ctx->ggml_ctx, qkv[2], bias_shape(params["v_bias"])); int64_t sequence = x->ne[1]; - auto mask = ggml_diag_mask_inf(ctx->ggml_ctx, - ggml_ext_zeros(ctx, sequence, sequence, 1, 1), - 0); + auto mask = ggml_tri(ctx->ggml_ctx, + ggml_ext_full(ctx, -INFINITY, sequence, sequence, 1, 1), + GGML_TRI_TYPE_UPPER); auto attn_out = ggml_ext_attention_ext(ctx, q, k, diff --git a/src/model/vae/minimax_h3_vae.hpp b/src/model/vae/minimax_h3_vae.hpp index c43c2c7152..3010cb82b8 100644 --- a/src/model/vae/minimax_h3_vae.hpp +++ b/src/model/vae/minimax_h3_vae.hpp @@ -1,6 +1,8 @@ #ifndef __SD_MODEL_VAE_MINIMAX_H3_VAE_HPP__ #define __SD_MODEL_VAE_MINIMAX_H3_VAE_HPP__ +#include "config.h" + #include #include #include @@ -719,11 +721,11 @@ namespace MiniMaxH3VAE { tensor.shape()[3]}); } - sd_tiling_params_t resolve_tiling_params(sd_tiling_params_t params) const override { + sd::TilingConfig resolve_tiling_params(sd::TilingConfig params) const override { return config.sol ? params : resolve_original_tiling_params(params); } - static sd_tiling_params_t resolve_original_tiling_params(sd_tiling_params_t params) { + static sd::TilingConfig resolve_original_tiling_params(sd::TilingConfig params) { if (!params.enabled) { params.target_overlap = 0.25f; } @@ -774,7 +776,7 @@ namespace MiniMaxH3VAE { sd::Tensor encode(int n_threads, const sd::Tensor& x, - sd_tiling_params_t tiling_params, + sd::TilingConfig tiling_params, bool circular_x = false, bool circular_y = false) override { auto input = ensure_video_shape(x); @@ -860,7 +862,7 @@ namespace MiniMaxH3VAE { sd::Tensor decode(int n_threads, const sd::Tensor& x, - sd_tiling_params_t tiling_params, + sd::TilingConfig tiling_params, bool decode_video = false, bool circular_x = false, bool circular_y = false, diff --git a/src/model/vae/vae.hpp b/src/model/vae/vae.hpp index 3c66965f69..f0a51c60a8 100644 --- a/src/model/vae/vae.hpp +++ b/src/model/vae/vae.hpp @@ -1,6 +1,8 @@ #ifndef __SD_MODEL_VAE_VAE_HPP__ #define __SD_MODEL_VAE_VAE_HPP__ +#include "config.h" + #include #include @@ -82,7 +84,7 @@ struct VAE : public GGMLRunner { sd::Tensor compute_with_temporal_tiling(const int n_threads, const sd::Tensor& input, VAETemporalDirection direction, - const sd_tiling_params_t& tiling_params) { + const sd::TilingConfig& tiling_params) { if (!tiling_params.temporal_tiling || input.dim() != 5 || input.shape()[2] <= 1) { return _compute(n_threads, input, direction == VAETemporalDirection::DECODE); } @@ -126,7 +128,7 @@ struct VAE : public GGMLRunner { bool circular_x, bool circular_y, bool decode_graph, - const sd_tiling_params_t& tiling_params, + const sd::TilingConfig& tiling_params, const char* error_message, bool silent = false) { auto on_processing = [&](const sd::Tensor& input_tile) { @@ -201,14 +203,14 @@ struct VAE : public GGMLRunner { return supports_temporal_tiling(VAETemporalDirection::DECODE); } - virtual sd_tiling_params_t resolve_tiling_params(sd_tiling_params_t params) const { + virtual sd::TilingConfig resolve_tiling_params(sd::TilingConfig params) const { return params; } bool get_tile_sizes(int& tile_size_w, int& tile_size_h, float& tile_overlap, - const sd_tiling_params_t& params, + const sd::TilingConfig& params, int64_t latent_w, int64_t latent_h) { const auto tiling = resolve_tiling_params(params); @@ -247,7 +249,7 @@ struct VAE : public GGMLRunner { virtual sd::Tensor encode(int n_threads, const sd::Tensor& x, - sd_tiling_params_t tiling_params, + sd::TilingConfig tiling_params, bool circular_x = false, bool circular_y = false) { return encode(n_threads, x, tiling_params, circular_x, circular_y, true); @@ -255,7 +257,7 @@ struct VAE : public GGMLRunner { sd::Tensor encode(int n_threads, const sd::Tensor& x, - sd_tiling_params_t tiling_params, + sd::TilingConfig tiling_params, bool circular_x, bool circular_y, bool auto_end) { @@ -313,7 +315,7 @@ struct VAE : public GGMLRunner { virtual sd::Tensor decode(int n_threads, const sd::Tensor& x, - sd_tiling_params_t tiling_params, + sd::TilingConfig tiling_params, bool decode_video = false, bool circular_x = false, bool circular_y = false, @@ -323,7 +325,7 @@ struct VAE : public GGMLRunner { sd::Tensor decode(int n_threads, const sd::Tensor& x, - sd_tiling_params_t tiling_params, + sd::TilingConfig tiling_params, bool decode_video, bool circular_x, bool circular_y, diff --git a/src/model/vae/vae_tiling.hpp b/src/model/vae/vae_tiling.hpp index c88ff9f056..a343084327 100644 --- a/src/model/vae/vae_tiling.hpp +++ b/src/model/vae/vae_tiling.hpp @@ -1,6 +1,8 @@ #ifndef __SD_MODEL_VAE_VAE_TILING_HPP__ #define __SD_MODEL_VAE_VAE_TILING_HPP__ +#include "config.h" + #include #include #include @@ -36,7 +38,7 @@ struct VAETemporalTilePlan { std::vector tiles; }; -inline VAETemporalTilingConfig resolve_vae_temporal_tiling_config(const sd_tiling_params_t& params, +inline VAETemporalTilingConfig resolve_vae_temporal_tiling_config(const sd::TilingConfig& params, int default_tile_frames, int default_overlap) { VAETemporalTilingConfig config; diff --git a/src/model_definition.h b/src/model_definition.h index 2b49994958..1a0f2df488 100644 --- a/src/model_definition.h +++ b/src/model_definition.h @@ -1,6 +1,8 @@ #ifndef __SD_MODEL_DEFINITION_H__ #define __SD_MODEL_DEFINITION_H__ +#include "core/option_types.h" + #include #include "model.h" #include "stable-diffusion.h" diff --git a/src/model_loader.cpp b/src/model_loader.cpp index 5987146c20..9e887117ea 100644 --- a/src/model_loader.cpp +++ b/src/model_loader.cpp @@ -581,7 +581,7 @@ SDVersion ModelLoader::get_sd_version() const { if (tensor_storage_map.count("model.diffusion_model.audio_patch_proj.weight") == 0) { return VERSION_MINIMAX_H3_IMAGE; } - // P-H3 uses the same DiT architecture as H3; its VAE or connector identifies the image pipeline. + // The connector is shared with H3 video; the single-frame VAE identifies P-H3. auto decoder = tensor_storage_map.find("first_stage_model.decoder.proj_out.weight"); if (decoder == tensor_storage_map.end()) { decoder = tensor_storage_map.find("vae.decoder.proj_out.weight"); @@ -589,17 +589,6 @@ SDVersion ModelLoader::get_sd_version() const { if (decoder != tensor_storage_map.end() && decoder->second.ne[1] == 768) { return VERSION_P_H3_IMAGE; } - for (const std::string prefix : {"", "embeddings_connectors."}) { - if (tensor_storage_map.count(prefix + "text_encoders.ph3_image_connector.proj.weight") || - tensor_storage_map.count(prefix + "text_encoders.llm.text_connector.proj.weight")) { - return VERSION_P_H3_IMAGE; - } - } - auto connector = tensor_storage_map.find("embeddings_connectors.proj.weight"); - if (connector != tensor_storage_map.end() && connector->second.logical_ne0() == 12800 && - connector->second.ne[1] == 5120 && tensor_storage_map.count("embeddings_connectors.sink_embedding")) { - return VERSION_P_H3_IMAGE; - } return VERSION_MINIMAX_H3; } if (name == "model.diffusion_model.y_embedder.layer_pool.weight" && diff --git a/src/name_conversion.cpp b/src/name_conversion.cpp index 79179f64a9..bd232319c6 100644 --- a/src/name_conversion.cpp +++ b/src/name_conversion.cpp @@ -1741,7 +1741,7 @@ std::string convert_tensor_name(std::string name, SDVersion version) { const std::vector> connector_prefix_map = { {"embeddings_connectors.text_encoders.", "text_encoders."}, - {"embeddings_connectors.", version == VERSION_P_H3_IMAGE ? "text_encoders.ph3_image_connector." : ""}, + {"embeddings_connectors.", sd_version_is_minimax_h3(version) ? "text_encoders.ph3_image_connector." : ""}, }; replace_with_prefix_map(name, connector_prefix_map); @@ -1800,7 +1800,10 @@ std::string convert_tensor_name(std::string name, SDVersion version) { } } - if (version == VERSION_FIBO2 || version == VERSION_QWEN_IMAGE_2_1 || sd_version_is_boogu_image(version) || sd_version_is_krea2(version) || sd_version_is_mage_flow(version) || sd_version_is_minimax_h3(version)) { + if (version == VERSION_FIBO2 || version == VERSION_QWEN_IMAGE_2_1 || version == VERSION_MOSS_IMAGE || + version == VERSION_IRIS || sd_version_is_flux3(version) || sd_version_is_lingbot_video(version) || + sd_version_is_ideogram4(version) || sd_version_is_sefi_image(version) || sd_version_is_boogu_image(version) || + sd_version_is_krea2(version) || sd_version_is_mage_flow(version) || sd_version_is_minimax_h3(version)) { const std::string hf_vision_prefix = "text_encoders.llm.model.visual."; if (starts_with(name, hf_vision_prefix)) { name = "text_encoders.llm.visual." + name.substr(hf_vision_prefix.size()); diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp index 5666680e30..c2ae6b83e6 100644 --- a/src/pipeline/diffusion_engine.cpp +++ b/src/pipeline/diffusion_engine.cpp @@ -1,10 +1,13 @@ #include "diffusion_engine.h" +#include "api/tensor.h" +#include "core/option_types.h" #include #include #include #include #include +#include #include #include #include @@ -23,6 +26,7 @@ #include "model.h" #include "model_definition.h" +#include "config.h" #include "core/rng.hpp" #include "core/rng_mt19937.hpp" #include "core/rng_philox.hpp" @@ -890,7 +894,7 @@ bool StableDiffusionGGML::set_sage_attention_enabled(bool enabled) { return true; } -bool StableDiffusionGGML::init(const sd_ctx_params_t* sd_ctx_params) { +bool StableDiffusionGGML::init(const sd::ContextConfig* sd_ctx_params) { #ifdef SD_USE_UPSTREAM_GGML LOG_WARN( "Using upstream GGML: INT8 tensorwise/convrot is disabled and FP8 weights are " @@ -1875,7 +1879,41 @@ bool StableDiffusionGGML::apply_loras(const sd_lora_t* loras, uint32_t lora_coun } SDCondition StableDiffusionGGML::get_learned_condition(const ConditionerParams& params) { - return conditioning_cache_->get(*cond_stage_model, n_threads, params); + if (params.prompt_input != nullptr) { + return conditioning_cache_->get(*cond_stage_model, n_threads, params); + } + auto input = sd::parse_prompt_input(params.text); + auto resolved = params; + if (sd::prompt_has_images(input)) { + if (!cond_stage_model->supports_prompt_images()) { + LOG_ERROR("Inline prompt images require a compatible LLM and vision weights"); + return {}; + } + if (prompt_image_callback == nullptr) { + LOG_ERROR("Inline prompt images require sd_set_prompt_image_callback()"); + return {}; + } + if (!sd::resolve_prompt_images(input, [&](const std::string& source) -> sd::Tensor { + const sd_tensor_t* tensor = nullptr; + if (!prompt_image_callback(source.c_str(), prompt_image_callback_data, &tensor) || !tensor) { + LOG_ERROR("Failed to resolve prompt image: %s", source.c_str()); + return {}; + } + auto image = sd::api::image_view(*tensor); + auto pixels = sd_image_to_tensor(image); + if (image.channel <= 2) { + auto gray = sd::ops::slice(pixels, 2, 0, 1); + return sd::ops::concat(sd::ops::concat(gray, gray, 2), gray, 2); + } + return image.channel == 3 ? pixels : sd::ops::slice(pixels, 2, 0, 3); + })) { + return {}; + } + resolved.prompt_input = &input; + } else { + resolved.text = sd::prompt_input_text(input); + } + return conditioning_cache_->get(*cond_stage_model, n_threads, resolved); } void StableDiffusionGGML::reset_generation_extensions() { @@ -1884,8 +1922,8 @@ void StableDiffusionGGML::reset_generation_extensions() { } } -void StableDiffusionGGML::prepare_generation_extensions(const sd_pm_params_t& pm_params, - const sd_pulid_params_t& pulid_params, +void StableDiffusionGGML::prepare_generation_extensions(const sd::PhotoMakerConfig& pm_params, + const sd::PuLIDConfig& pulid_params, ConditionerParams& condition_params, int total_steps) { reset_generation_extensions(); @@ -1927,7 +1965,7 @@ sd::Tensor StableDiffusionGGML::get_clip_vision_output(const sd::Tensor StableDiffusionGGML::get_audio_embedding(const sd_audio_t& audio) { +sd::Tensor StableDiffusionGGML::get_audio_embedding(const sd::Audio& audio) { if (audio_encoder == nullptr) { LOG_ERROR("audio encoder model is not loaded"); return {}; @@ -1949,7 +1987,7 @@ sd::Tensor StableDiffusionGGML::get_audio_embedding(const sd_audio_t& aud return audio_encoder->compute(n_threads, mono); } -void StableDiffusionGGML::compute_ip_adapter_tokens(const sd_image_t& image, float strength) { +void StableDiffusionGGML::compute_ip_adapter_tokens(const sd::Image& image, float strength) { ip_adapter_tokens = {}; ip_adapter_uncond_tokens = {}; ip_adapter_strength = strength; @@ -2049,7 +2087,7 @@ void StableDiffusionGGML::preview_image(int step, const sd::Tensor& latents, enum SDVersion version, preview_t preview_mode, - std::function step_callback, + std::function step_callback, void* step_callback_data, bool is_noisy) { bool is_video = preview_latent_tensor_is_video(latents); @@ -2138,17 +2176,14 @@ void StableDiffusionGGML::preview_image(int step, uint32_t img_height = static_cast(_latents.shape()[1]) * patch_sz; uint32_t img_channels = latent_alpha_proj != nullptr ? 4 : 3; - uint8_t* data = (uint8_t*)malloc(frames * img_width * img_height * img_channels * sizeof(uint8_t)); + std::unique_ptr data(static_cast(malloc(size_t(frames) * img_width * img_height * img_channels)), free); GGML_ASSERT(data != nullptr); - preview_latent_video(data, _latents, latent_rgb_proj, latent_rgb_bias, patch_sz, latent_alpha_proj, latent_alpha_bias); - sd_image_t* images = (sd_image_t*)malloc(frames * sizeof(sd_image_t)); - GGML_ASSERT(images != nullptr); + preview_latent_video(data.get(), _latents, latent_rgb_proj, latent_rgb_bias, patch_sz, latent_alpha_proj, latent_alpha_bias); + std::vector images(frames); for (uint32_t i = 0; i < frames; i++) { - images[i] = {img_width, img_height, img_channels, data + i * img_width * img_height * img_channels}; + images[i] = {img_width, img_height, img_channels, data.get() + size_t(i) * img_width * img_height * img_channels}; } - step_callback(step, frames, images, is_noisy, step_callback_data); - free(data); - free(images); + sd::api::preview_images(step_callback, step, frames, images.data(), is_noisy, step_callback_data); return; } @@ -2167,19 +2202,20 @@ void StableDiffusionGGML::preview_image(int step, return; } - is_video = preview_latent_tensor_is_video(decoded); - uint32_t frames = is_video ? static_cast(decoded.shape()[2]) : 1; - sd_image_t* images = (sd_image_t*)malloc(frames * sizeof(sd_image_t)); + is_video = preview_latent_tensor_is_video(decoded); + uint32_t frames = is_video ? static_cast(decoded.shape()[2]) : 1; + auto release_images = [frames](sd::Image* images) { + for (uint32_t i = 0; i < frames; ++i) + free(images[i].data); + free(images); + }; + std::unique_ptr images(static_cast(calloc(frames, sizeof(sd::Image))), release_images); GGML_ASSERT(images != nullptr); for (uint32_t i = 0; i < frames; ++i) { - images[i] = tensor_to_sd_image(decoded, static_cast(i)); + images.get()[i] = tensor_to_sd_image(decoded, static_cast(i)); } - step_callback(step, frames, images, is_noisy, step_callback_data); - for (uint32_t i = 0; i < frames; ++i) { - free(images[i].data); - } - free(images); + sd::api::preview_images(step_callback, step, frames, images.get(), is_noisy, step_callback_data); return; } @@ -2318,7 +2354,7 @@ sd::Tensor StableDiffusionGGML::sample(const std::shared_ptr& control_image, float control_strength, - const sd_guidance_params_t& guidance, + const sd::GuidanceConfig& guidance, float eta, int shifted_timestep, sample_method_t method, @@ -2332,7 +2368,7 @@ sd::Tensor StableDiffusionGGML::sample(const std::shared_ptr& video_positions) { struct RunnerEndOnExit { @@ -2448,15 +2484,6 @@ sd::Tensor StableDiffusionGGML::sample(const std::shared_ptr denoised = x_t; -#ifdef SD_USE_UPSTREAM_GGML - // workaround for sd.cpp #2149 - const auto diffusion_backends = backend_manager.runtime_backends(SDBackendModule::DIFFUSION); - const bool batch_cfg_backend_ok = std::none_of(diffusion_backends.begin(), diffusion_backends.end(), - [](ggml_backend_t backend) { return sd_backend_is(backend, "MTL"); }); -#else - const bool batch_cfg_backend_ok = true; -#endif - auto denoise = [&](const sd::Tensor& x, float sigma, int step) -> sd::guidance::GuiderOutput { if (get_cancel_flag() == SD_CANCEL_ALL) { LOG_VERBOSE("cancelling generation"); @@ -2779,7 +2806,6 @@ sd::Tensor StableDiffusionGGML::sample(const std::shared_ptrparams.batched_cfg && - batch_cfg_backend_ok && sd_version_is_unet(version) && !uncond.empty() && img_uncond.empty() && diff --git a/src/pipeline/diffusion_engine.h b/src/pipeline/diffusion_engine.h index 990a8b6515..ae05dc9cd3 100644 --- a/src/pipeline/diffusion_engine.h +++ b/src/pipeline/diffusion_engine.h @@ -1,6 +1,10 @@ #ifndef __SD_PIPELINE_DIFFUSION_ENGINE_H__ #define __SD_PIPELINE_DIFFUSION_ENGINE_H__ +#include "core/media.h" + +#include "core/option_types.h" + #include #include #include @@ -14,6 +18,7 @@ #include #include +#include "config.h" #include "core/ggml_extend_backend.h" #include "core/ggml_graph_cut.h" #include "core/tensor.hpp" @@ -79,6 +84,8 @@ class StableDiffusionGGML { std::shared_ptr control_net; std::shared_ptr ip_adapter; sd::Tensor ip_adapter_tokens; + sd_prompt_image_cb_t prompt_image_callback = nullptr; + void* prompt_image_callback_data = nullptr; sd::Tensor ip_adapter_uncond_tokens; float ip_adapter_strength = 1.0f; std::vector> generation_extensions; @@ -97,8 +104,8 @@ class StableDiffusionGGML { int animatediff_num_frames = 0; std::string taesd_path; - sd_tiling_params_t vae_tiling_params = {false, false, 0, 0, 0.5f, 0, 0, nullptr}; - bool enable_mmap = false; + sd::TilingConfig vae_tiling_params = {false, false, 0, 0, 0.5f, 0, 0, nullptr}; + bool enable_mmap = false; sd::ggml_graph_cut::MaxVramAssignment max_vram_assignment; bool disable_prefetch = false; bool disable_segmented_compute = false; @@ -122,7 +129,7 @@ class StableDiffusionGGML { using RunnerGroups = std::set; struct ModelConfig { - sd_ctx_params_t params{}; + sd::ContextConfig params{}; std::list strings; std::vector embeddings; std::map models; @@ -132,22 +139,22 @@ class StableDiffusionGGML { bool photomaker_source_available = false; bool animatediff_loaded = false; - explicit ModelConfig(const sd_ctx_params_t& initial) + explicit ModelConfig(const sd::ContextConfig& initial) : params(initial) { - for (auto member : {&sd_ctx_params_t::model_path, &sd_ctx_params_t::clip_l_path, - &sd_ctx_params_t::clip_g_path, &sd_ctx_params_t::clip_vision_path, - &sd_ctx_params_t::t5xxl_path, &sd_ctx_params_t::llm_path, - &sd_ctx_params_t::llm_vision_path, &sd_ctx_params_t::diffusion_model_path, - &sd_ctx_params_t::tokenizer, &sd_ctx_params_t::flash_attn_mode, &sd_ctx_params_t::conv_direct_mode, - &sd_ctx_params_t::high_noise_diffusion_model_path, &sd_ctx_params_t::uncond_diffusion_model_path, - &sd_ctx_params_t::embeddings_connectors_path, &sd_ctx_params_t::vae_path, - &sd_ctx_params_t::audio_vae_path, &sd_ctx_params_t::taesd_path, - &sd_ctx_params_t::control_net_path, &sd_ctx_params_t::ip_adapter_path, - &sd_ctx_params_t::motion_module_path, &sd_ctx_params_t::photo_maker_path, - &sd_ctx_params_t::pulid_weights_path, &sd_ctx_params_t::tensor_type_rules, - &sd_ctx_params_t::max_vram, &sd_ctx_params_t::backend, - &sd_ctx_params_t::params_backend, &sd_ctx_params_t::split_mode, - &sd_ctx_params_t::rpc_servers, &sd_ctx_params_t::model_args}) { + for (auto member : {&sd::ContextConfig::model_path, &sd::ContextConfig::clip_l_path, + &sd::ContextConfig::clip_g_path, &sd::ContextConfig::clip_vision_path, + &sd::ContextConfig::t5xxl_path, &sd::ContextConfig::llm_path, + &sd::ContextConfig::llm_vision_path, &sd::ContextConfig::diffusion_model_path, + &sd::ContextConfig::tokenizer, &sd::ContextConfig::flash_attn_mode, &sd::ContextConfig::conv_direct_mode, + &sd::ContextConfig::high_noise_diffusion_model_path, &sd::ContextConfig::uncond_diffusion_model_path, + &sd::ContextConfig::embeddings_connectors_path, &sd::ContextConfig::vae_path, + &sd::ContextConfig::audio_vae_path, &sd::ContextConfig::audio_encoder_path, &sd::ContextConfig::taesd_path, + &sd::ContextConfig::control_net_path, &sd::ContextConfig::ip_adapter_path, + &sd::ContextConfig::motion_module_path, &sd::ContextConfig::photo_maker_path, + &sd::ContextConfig::pulid_weights_path, &sd::ContextConfig::tensor_type_rules, + &sd::ContextConfig::max_vram, &sd::ContextConfig::backend, + &sd::ContextConfig::params_backend, &sd::ContextConfig::split_mode, + &sd::ContextConfig::rpc_servers, &sd::ContextConfig::model_args}) { strings.emplace_back(SAFE_STR(initial.*member)); params.*member = strings.back().c_str(); } @@ -336,7 +343,7 @@ class StableDiffusionGGML { bool init_model_loader(ModelLoader& model_loader, ModelConfig& configuration); - bool init(const sd_ctx_params_t* sd_ctx_params); + bool init(const sd::ContextConfig* sd_ctx_params); bool set_sage_attention_enabled(bool enabled); bool uses_tae() const; @@ -392,8 +399,8 @@ class StableDiffusionGGML { void reset_generation_extensions(); - void prepare_generation_extensions(const sd_pm_params_t& pm_params, - const sd_pulid_params_t& pulid_params, + void prepare_generation_extensions(const sd::PhotoMakerConfig& pm_params, + const sd::PuLIDConfig& pulid_params, ConditionerParams& condition_params, int total_steps); @@ -402,9 +409,9 @@ class StableDiffusionGGML { int clip_skip = -1, bool zero_out_masked = false); - sd::Tensor get_audio_embedding(const sd_audio_t& audio); + sd::Tensor get_audio_embedding(const sd::Audio& audio); - void compute_ip_adapter_tokens(const sd_image_t& image, float strength); + void compute_ip_adapter_tokens(const sd::Image& image, float strength); std::vector process_timesteps(const std::vector& timesteps, const sd::Tensor& init_latent, @@ -419,7 +426,7 @@ class StableDiffusionGGML { const sd::Tensor& latents, enum SDVersion version, preview_t preview_mode, - std::function step_callback, + std::function step_callback, void* step_callback_data, bool is_noisy); @@ -464,7 +471,7 @@ class StableDiffusionGGML { const SDCondition& img_uncond, const sd::Tensor& control_image, float control_strength, - const sd_guidance_params_t& guidance, + const sd::GuidanceConfig& guidance, float eta, int shifted_timestep, sample_method_t method, @@ -478,7 +485,7 @@ class StableDiffusionGGML { float vace_strength, int audio_length, float frame_rate, - const sd_cache_params_t* cache_params, + const sd::CacheConfig* cache_params, bool preview_final_step, const sd::Tensor& video_positions = {}); diff --git a/src/pipeline/generation.h b/src/pipeline/generation.h index 0bc3913150..08511f2afb 100644 --- a/src/pipeline/generation.h +++ b/src/pipeline/generation.h @@ -1,12 +1,22 @@ #ifndef __SD_PIPELINE_GENERATION_H__ #define __SD_PIPELINE_GENERATION_H__ +#include "core/media.h" + #include "conditioning/conditioner.hpp" +#include "config.h" #include "json.hpp" #include "stable-diffusion.h" class StableDiffusionGGML; +namespace sd { + struct ImageGenerationResult; + struct VideoGenerationResult; + sd_status_t generate_image(sd_ctx_t* ctx, const ImageConfig& config, ImageGenerationResult** result_out, const sd_opt_t* source = nullptr) noexcept; + sd_status_t generate_video(sd_ctx_t* ctx, const VideoConfig& config, VideoGenerationResult** result_out, const sd_opt_t* source = nullptr) noexcept; +} + static inline bool sd_version_supports_animatediff(SDVersion version) { return version == VERSION_SD1 || version == VERSION_SD1_INPAINT || version == VERSION_SD1_PIX2PIX; } @@ -53,16 +63,16 @@ namespace sd::pipeline { // Callers hold ExecutionScope; AnimateDiff reuses the image path within the same scope. bool generate_image(StableDiffusionGGML* sd, - const sd_img_gen_params_t* sd_img_gen_params, - sd_image_t** images_out, + const sd::ImageConfig* sd_img_gen_params, + sd::Image** images_out, int* num_images_out, std::vector* infos_out = nullptr); bool generate_video(StableDiffusionGGML* sd, - const sd_vid_gen_params_t* sd_vid_gen_params, - sd_image_t** frames_out, + const sd::VideoConfig* sd_vid_gen_params, + sd::Image** frames_out, int* num_frames_out, - sd_audio_t** audio_out, + sd::Audio** audio_out, int* fps_out, nlohmann::json* info_out = nullptr); diff --git a/src/pipeline/image.cpp b/src/pipeline/image.cpp index 5e7a349583..f2cdcea9c5 100644 --- a/src/pipeline/image.cpp +++ b/src/pipeline/image.cpp @@ -1,3 +1,5 @@ +#include "api/result.h" +#include "core/option_types.h" #include "generation.h" #include "generation_info.h" @@ -22,7 +24,7 @@ namespace sd::pipeline { }; static CircularAxesState configure_image_vae_axes(StableDiffusionGGML* sd, - const sd_img_gen_params_t* sd_img_gen_params, + const sd::ImageConfig* sd_img_gen_params, const GenerationRequest& request) { CircularAxesState original_axes = {sd->circular_x, sd->circular_y}; @@ -77,7 +79,7 @@ namespace sd::pipeline { public: ImageVaeAxesGuard(StableDiffusionGGML* sd, - const sd_img_gen_params_t* sd_img_gen_params, + const sd::ImageConfig* sd_img_gen_params, const GenerationRequest& request) : sd(sd), original_axes(configure_image_vae_axes(sd, sd_img_gen_params, request)) {} @@ -184,7 +186,7 @@ namespace sd::pipeline { } static std::optional prepare_image_generation_latents(StableDiffusionGGML* sd, - const sd_img_gen_params_t* sd_img_gen_params, + const sd::ImageConfig* sd_img_gen_params, GenerationRequest* request, SamplePlan* plan, const RefImageParams& ref_image_params) { @@ -541,7 +543,7 @@ namespace sd::pipeline { } static std::optional prepare_image_generation_embeds(StableDiffusionGGML* sd, - const sd_img_gen_params_t* sd_img_gen_params, + const sd::ImageConfig* sd_img_gen_params, GenerationRequest* request, SamplePlan* plan, ImageGenerationLatents* latents, @@ -669,10 +671,10 @@ namespace sd::pipeline { return embeds; } - static sd_image_t* decode_image_outputs(StableDiffusionGGML* sd, - const GenerationRequest& request, - const std::vector>& final_latents, - int* num_images_out) { + static sd::Image* decode_image_outputs(StableDiffusionGGML* sd, + const GenerationRequest& request, + const std::vector>& final_latents, + int* num_images_out) { if (final_latents.empty()) { LOG_ERROR("no latent images to decode"); return nullptr; @@ -760,8 +762,8 @@ namespace sd::pipeline { return nullptr; } - int image_count = static_cast(decoded_images.size()); - sd_image_t* result_images = (sd_image_t*)calloc(image_count, sizeof(sd_image_t)); + int image_count = static_cast(decoded_images.size()); + sd::Image* result_images = (sd::Image*)calloc(image_count, sizeof(sd::Image)); if (result_images == nullptr) { return nullptr; } @@ -915,8 +917,8 @@ namespace sd::pipeline { } bool generate_image(StableDiffusionGGML* sd, - const sd_img_gen_params_t* sd_img_gen_params, - sd_image_t** images_out, + const sd::ImageConfig* sd_img_gen_params, + sd::Image** images_out, int* num_images_out, std::vector* infos_out) { if (infos_out) @@ -944,7 +946,7 @@ namespace sd::pipeline { GenerationRequest request(sd, sd_img_gen_params); const auto info = sd::generation_info(request, *sd_img_gen_params); sd::ImagePreprocessor preprocessing(sd_img_gen_params->image_preprocess.rules); - sd_img_gen_params_t processed_params = *sd_img_gen_params; + sd::ImageConfig processed_params = *sd_img_gen_params; if (!preprocessing.prepare_inputs(processed_params, request.width, request.height)) return false; sd_img_gen_params = &processed_params; @@ -953,7 +955,6 @@ namespace sd::pipeline { sd->rng->manual_seed(request.seed); sd->sampler_rng->manual_seed(request.seed); - sd->set_flow_shift(sd_img_gen_params->sample_params.flow_shift); if (!sd->apply_loras(sd_img_gen_params->loras, sd_img_gen_params->lora_count)) return false; sd->apply_circular_axes(sd_img_gen_params->circular_x, sd_img_gen_params->circular_y); @@ -1084,7 +1085,6 @@ namespace sd::pipeline { make_hires_sigma_schedule(sd, request.hires, request.sample_params, - plan.sample_method, plan.sample_steps, sd->get_image_seq_len(request.hires.target_height, request.hires.target_width), &hires_scheduler_steps); @@ -1217,7 +1217,7 @@ namespace sd::pipeline { (*infos_out)[i]["seed"] = std::to_string(request.seed_for_batch(i / images_per_latent)); } } catch (...) { - free_sd_images(result, num_images); + sd::free_images(result, num_images); throw; } } @@ -1231,7 +1231,7 @@ namespace sd::pipeline { if (images_out != nullptr) { *images_out = result; } else { - free_sd_images(result, num_images); + sd::free_images(result, num_images); } return true; } diff --git a/src/pipeline/model_builders.cpp b/src/pipeline/model_builders.cpp index afc0ce8d7f..389b199829 100644 --- a/src/pipeline/model_builders.cpp +++ b/src/pipeline/model_builders.cpp @@ -1,4 +1,5 @@ #include "model_builders.h" +#include "core/option_types.h" #include #include diff --git a/src/pipeline/model_builders.h b/src/pipeline/model_builders.h index bb51ccc14a..1f56e4a1d2 100644 --- a/src/pipeline/model_builders.h +++ b/src/pipeline/model_builders.h @@ -4,6 +4,7 @@ #include #include +#include "config.h" #include "core/ggml_extend_backend.h" #include "model.h" #include "stable-diffusion.h" @@ -28,7 +29,7 @@ namespace IPAdapter { namespace sd::model_builders { struct Context { - const sd_ctx_params_t& params; + const sd::ContextConfig& params; SDVersion version; const String2TensorStorage& tensor_storage_map; SDBackendManager& backends; diff --git a/src/pipeline/relay_vsr.cpp b/src/pipeline/relay_vsr.cpp index c8b0d72fdf..20d70eaecb 100644 --- a/src/pipeline/relay_vsr.cpp +++ b/src/pipeline/relay_vsr.cpp @@ -1,3 +1,4 @@ +#include "api/result.h" #include "pipeline/video_upscale.h" #include @@ -11,7 +12,7 @@ namespace sd { namespace pipeline { - static sd::Tensor prepare_vsr_frame(const sd_image_t& frame, int width, int height) { + static sd::Tensor prepare_vsr_frame(const sd::Image& frame, int width, int height) { auto result = sd_image_to_tensor(frame, -1, -1, false); result /= 127.5f; result -= 1.f; @@ -103,7 +104,7 @@ namespace sd { flash_->runner_end(); } - bool upscale(const sd_video_upscale_params_t& params, sd_video_generation_result_t* result) override { + bool upscale(const sd::VideoUpscaleConfig& params, sd::VideoGenerationResult* result) override { const auto* frames = params.frames; const int frame_count = params.frame_count, fps = params.fps, window_size = params.window_size; const int64_t seed = params.seed; diff --git a/src/pipeline/request.cpp b/src/pipeline/request.cpp index 739b8e43a1..bbf3ab778f 100644 --- a/src/pipeline/request.cpp +++ b/src/pipeline/request.cpp @@ -1,5 +1,7 @@ #include "request.h" #include "conditioning/conditioner.hpp" +#include "config.h" +#include "core/option_types.h" #include #include @@ -93,9 +95,9 @@ namespace sd::pipeline { return scheduler; } - float resolve_eta(StableDiffusionGGML* sd, - float eta, - enum sample_method_t sample_method) { + static float resolve_eta(StableDiffusionGGML* sd, + float eta, + enum sample_method_t sample_method) { if (eta == INFINITY) { if (sd->version == VERSION_HIDREAM_O1) { return 8.f; @@ -120,7 +122,7 @@ namespace sd::pipeline { return eta; } - GenerationRequest::GenerationRequest(StableDiffusionGGML* sd, const sd_img_gen_params_t* sd_img_gen_params) { + GenerationRequest::GenerationRequest(StableDiffusionGGML* sd, const sd::ImageConfig* sd_img_gen_params) { prompt = SAFE_STR(sd_img_gen_params->prompt); negative_prompt = SAFE_STR(sd_img_gen_params->negative_prompt); width = sd_img_gen_params->width; @@ -136,7 +138,6 @@ namespace sd::pipeline { strength = sd_img_gen_params->strength; control_strength = sd_img_gen_params->control_strength; control_type = sd_img_gen_params->control_type; - eta = sd_img_gen_params->sample_params.eta; has_ref_images = sd_img_gen_params->ref_images_count > 0; guidance = sd_img_gen_params->sample_params.guidance; pm_params = sd_img_gen_params->pm_params; @@ -156,7 +157,7 @@ namespace sd::pipeline { models.erase("pulid_weights"); } - GenerationRequest::GenerationRequest(StableDiffusionGGML* sd, const sd_vid_gen_params_t* sd_vid_gen_params) { + GenerationRequest::GenerationRequest(StableDiffusionGGML* sd, const sd::VideoConfig* sd_vid_gen_params) { prompt = SAFE_STR(sd_vid_gen_params->prompt); negative_prompt = SAFE_STR(sd_vid_gen_params->negative_prompt); width = sd_vid_gen_params->width; @@ -293,7 +294,7 @@ namespace sd::pipeline { } void GenerationRequest::resolve_guidance(StableDiffusionGGML* sd, - sd_guidance_params_t* guidance, + sd::GuidanceConfig* guidance, bool* use_uncond, bool* use_img_uncond, bool has_ref_images, @@ -388,16 +389,21 @@ namespace sd::pipeline { sample_params.sample_method = resolve_sample_method(sd, sample_params.sample_method); sample_params.scheduler = resolve_scheduler(sd, sample_params.scheduler, sample_params.sample_method); sample_params.eta = resolve_eta(sd, sample_params.eta, sample_params.sample_method); + sd->set_flow_shift(sample_params.flow_shift); + if (sd->is_flow_denoiser()) + sample_params.flow_shift = sd->active_flow_shift; if (frames > 0) { - high_noise_sample_params.guidance = high_noise_guidance; - high_noise_sample_params.sample_method = resolve_sample_method(sd, high_noise_sample_params.sample_method); - high_noise_sample_params.scheduler = resolve_scheduler(sd, high_noise_sample_params.scheduler, high_noise_sample_params.sample_method); - high_noise_sample_params.eta = resolve_eta(sd, high_noise_sample_params.eta, high_noise_sample_params.sample_method); + high_noise_sample_params.guidance = high_noise_guidance; + high_noise_sample_params.sample_method = resolve_sample_method(sd, high_noise_sample_params.sample_method); + high_noise_sample_params.scheduler = sample_params.scheduler; + high_noise_sample_params.flow_shift = sample_params.flow_shift; + high_noise_sample_params.shifted_timestep = sample_params.shifted_timestep; + high_noise_sample_params.eta = resolve_eta(sd, high_noise_sample_params.eta, high_noise_sample_params.sample_method); } } SamplePlan::SamplePlan(StableDiffusionGGML* sd, - const sd_img_gen_params_t* sd_img_gen_params, + const sd::ImageConfig* sd_img_gen_params, const GenerationRequest& request) { sample_method = request.sample_params.sample_method; extra_sample_args = request.sample_params.extra_sample_args; @@ -407,7 +413,7 @@ namespace sd::pipeline { } SamplePlan::SamplePlan(StableDiffusionGGML* sd, - const sd_vid_gen_params_t* sd_vid_gen_params, + const sd::VideoConfig* sd_vid_gen_params, const GenerationRequest& request) { sample_method = request.sample_params.sample_method; extra_sample_args = request.sample_params.extra_sample_args; @@ -425,9 +431,8 @@ namespace sd::pipeline { void SamplePlan::resolve(StableDiffusionGGML* sd, const GenerationRequest* request, - const sd_sample_params_t* sample_params) { - sample_method = resolve_sample_method(sd, sample_method); - + const sd::SampleConfig* sample_params) { + scheduler = sample_params->scheduler; total_steps = sample_steps + std::max(0, high_noise_sample_steps); if (sample_params->custom_sigmas_count > 0) { @@ -446,9 +451,6 @@ namespace sd::pipeline { } } } else { - scheduler = resolve_scheduler(sd, - sample_params->scheduler, - sample_method); int sample_seq_len = sd->get_image_seq_len(request->height, request->width); if (sd_version_is_ltxav(sd->version) && request->frames > 0) { int latent_frames = ((request->frames - 1) / 8) + 1; @@ -463,8 +465,6 @@ namespace sd::pipeline { sample_params->extra_sample_args); } - eta = resolve_eta(sd, eta, sample_method); - if (high_noise_sample_steps < 0) { for (size_t i = 0; i < sigmas.size(); ++i) { if (sigmas[i] < moe_boundary) { @@ -477,17 +477,13 @@ namespace sd::pipeline { LOG_INFO("sampling using %s method", sampling_methods_str[sample_method]); if (high_noise_sample_steps > 0) { - high_noise_sample_method = resolve_sample_method(sd, - high_noise_sample_method); - high_noise_eta = resolve_eta(sd, high_noise_eta, high_noise_sample_method); LOG_INFO("sampling(high noise) using %s method", sampling_methods_str[high_noise_sample_method]); } } std::vector make_hires_sigma_schedule(StableDiffusionGGML* sd, - const sd_hires_params_t& hires, - const sd_sample_params_t& sample_params, - sample_method_t sample_method, + const sd::HiresConfig& hires, + const sd::SampleConfig& sample_params, int default_steps, int sample_seq_len, int* scheduler_steps_out) { @@ -512,9 +508,7 @@ namespace sd::pipeline { int scheduler_steps = static_cast(effective_steps / hires.denoising_strength); scheduler_steps = std::max(1, scheduler_steps); - scheduler_t scheduler = resolve_scheduler(sd, - sample_params.scheduler, - sample_method); + scheduler_t scheduler = sample_params.scheduler; std::vector sigmas = sd->denoiser->get_sigmas(scheduler_steps, sample_seq_len, scheduler, diff --git a/src/pipeline/request.h b/src/pipeline/request.h index b1ab6404cb..1e15db8349 100644 --- a/src/pipeline/request.h +++ b/src/pipeline/request.h @@ -1,10 +1,13 @@ #ifndef __SD_PIPELINE_REQUEST_H__ #define __SD_PIPELINE_REQUEST_H__ +#include "core/option_types.h" + #include #include #include +#include "config.h" #include "stable-diffusion.h" class StableDiffusionGGML; @@ -19,52 +22,47 @@ namespace sd::pipeline { int64_t resolve_seed(int64_t seed); - float resolve_eta(StableDiffusionGGML* sd, - float eta, - enum sample_method_t sample_method); - struct GenerationRequest { std::string prompt; std::string negative_prompt; std::map models; - int width = -1; - int height = -1; - int clip_skip = -1; - int vae_scale_factor = -1; - int diffusion_model_down_factor = -1; - int64_t seed = -1; - bool use_uncond = false; - bool use_img_uncond = false; - bool use_high_noise_uncond = false; - bool use_high_noise_img_uncond = false; - bool has_ref_images = false; - bool is_layered_image = false; - bool use_high_noise = false; - bool use_vace = false; - const sd_cache_params_t* cache_params = nullptr; - int batch_count = 1; - int qwen_image_layers = 3; - int shifted_timestep = 0; - float strength = 1.f; - float control_strength = 0.f; - sd_control_type_t control_type = SD_CONTROL_TYPE_NONE; - float eta = 0.f; - sd_sample_params_t sample_params = {}; - sd_sample_params_t high_noise_sample_params = {}; - sd_guidance_params_t guidance = {}; - sd_guidance_params_t high_noise_guidance = {}; - sd_pm_params_t pm_params = {}; - sd_pulid_params_t pulid_params = {}; - sd_hires_params_t hires = {}; - sd_tiling_params_t vae_tiling_params = {}; - int frames = -1; - int requested_frames = -1; - int fps = 16; - float vace_strength = 1.f; - - GenerationRequest(StableDiffusionGGML* sd, const sd_img_gen_params_t* sd_img_gen_params); - - GenerationRequest(StableDiffusionGGML* sd, const sd_vid_gen_params_t* sd_vid_gen_params); + int width = -1; + int height = -1; + int clip_skip = -1; + int vae_scale_factor = -1; + int diffusion_model_down_factor = -1; + int64_t seed = -1; + bool use_uncond = false; + bool use_img_uncond = false; + bool use_high_noise_uncond = false; + bool use_high_noise_img_uncond = false; + bool has_ref_images = false; + bool is_layered_image = false; + bool use_high_noise = false; + bool use_vace = false; + const sd::CacheConfig* cache_params = nullptr; + int batch_count = 1; + int qwen_image_layers = 3; + int shifted_timestep = 0; + float strength = 1.f; + float control_strength = 0.f; + sd_control_type_t control_type = SD_CONTROL_TYPE_NONE; + sd::SampleConfig sample_params = {}; + sd::SampleConfig high_noise_sample_params = {}; + sd::GuidanceConfig guidance = {}; + sd::GuidanceConfig high_noise_guidance = {}; + sd::PhotoMakerConfig pm_params = {}; + sd::PuLIDConfig pulid_params = {}; + sd::HiresConfig hires = {}; + sd::TilingConfig vae_tiling_params = {}; + int frames = -1; + int requested_frames = -1; + int fps = 16; + float vace_strength = 1.f; + + GenerationRequest(StableDiffusionGGML* sd, const sd::ImageConfig* sd_img_gen_params); + + GenerationRequest(StableDiffusionGGML* sd, const sd::VideoConfig* sd_vid_gen_params); int64_t seed_for_batch(int index) const; @@ -75,7 +73,7 @@ namespace sd::pipeline { void resolve_hires(); static void resolve_guidance(StableDiffusionGGML* sd, - sd_guidance_params_t* guidance, + sd::GuidanceConfig* guidance, bool* use_uncond, bool* use_img_uncond, bool has_ref_images, @@ -99,22 +97,21 @@ namespace sd::pipeline { std::vector sigmas; SamplePlan(StableDiffusionGGML* sd, - const sd_img_gen_params_t* sd_img_gen_params, + const sd::ImageConfig* sd_img_gen_params, const GenerationRequest& request); SamplePlan(StableDiffusionGGML* sd, - const sd_vid_gen_params_t* sd_vid_gen_params, + const sd::VideoConfig* sd_vid_gen_params, const GenerationRequest& request); void resolve(StableDiffusionGGML* sd, const GenerationRequest* request, - const sd_sample_params_t* sample_params); + const sd::SampleConfig* sample_params); }; std::vector make_hires_sigma_schedule(StableDiffusionGGML* sd, - const sd_hires_params_t& hires, - const sd_sample_params_t& sample_params, - sample_method_t sample_method, + const sd::HiresConfig& hires, + const sd::SampleConfig& sample_params, int default_steps, int sample_seq_len, int* scheduler_steps_out); diff --git a/src/pipeline/seedvr2.cpp b/src/pipeline/seedvr2.cpp index 5fe2eaa444..cb07876fd9 100644 --- a/src/pipeline/seedvr2.cpp +++ b/src/pipeline/seedvr2.cpp @@ -1,3 +1,4 @@ +#include "api/result.h" #include "pipeline/video_upscale.h" #include @@ -54,7 +55,7 @@ namespace sd { vae_->runner_end(); } - bool upscale(const sd_video_upscale_params_t& params, sd_video_generation_result_t* result) override { + bool upscale(const sd::VideoUpscaleConfig& params, sd::VideoGenerationResult* result) override { const int chunk = params.temporal_chunk_size; if (chunk < 1 || chunk > INT_MAX - 3 || (chunk - 1) % 4) { LOG_ERROR("SeedVR2 temporal chunk size must be 1 or 4n+1 input frames"); diff --git a/src/pipeline/video.cpp b/src/pipeline/video.cpp index 915dc54b39..529bbb60ce 100644 --- a/src/pipeline/video.cpp +++ b/src/pipeline/video.cpp @@ -1,3 +1,4 @@ +#include "core/option_types.h" #include "generation.h" #include "generation_info.h" @@ -22,8 +23,8 @@ namespace sd::pipeline { - static sd_audio_t* waveform_to_sd_audio(const StableDiffusionGGML* sd, - const sd::Tensor& waveform) { + static sd::Audio* waveform_to_sd_audio(const StableDiffusionGGML* sd, + const sd::Tensor& waveform) { if (sd == nullptr || waveform.empty()) { return nullptr; } @@ -34,7 +35,7 @@ namespace sd::pipeline { return nullptr; } - sd_audio_t* audio = (sd_audio_t*)malloc(sizeof(sd_audio_t)); + sd::Audio* audio = (sd::Audio*)malloc(sizeof(sd::Audio)); if (audio == nullptr) { return nullptr; } @@ -374,7 +375,7 @@ namespace sd::pipeline { return sd::zeros({audio_length, 2, 32, 1}); } - static sd::Tensor prepare_minimax_h3_reference_waveform(const sd_audio_t& audio, + static sd::Tensor prepare_minimax_h3_reference_waveform(const sd::Audio& audio, int target_sample_rate = 32000) { if (audio.data == nullptr || audio.sample_count == 0 || audio.channels == 0 || audio.sample_rate == 0) { return {}; @@ -490,7 +491,7 @@ namespace sd::pipeline { } static std::optional prepare_video_generation_latents(StableDiffusionGGML* sd, - const sd_vid_gen_params_t* sd_vid_gen_params, + const sd::VideoConfig* sd_vid_gen_params, GenerationRequest* request, const RefImageParams& ref_image_params) { ImageGenerationLatents latents; @@ -576,7 +577,7 @@ namespace sd::pipeline { std::vector{width, height, image.shape()[2], image.shape()[3]}); }; - auto encode_reference_audio = [&](const sd_audio_t& audio, + auto encode_reference_audio = [&](const sd::Audio& audio, int32_t* audio_index) -> bool { if (sd->audio_vae_model == nullptr) { LOG_ERROR("MiniMax-H3 Ref2VA audio requires --audio-vae with encoder weights"); @@ -1194,7 +1195,7 @@ namespace sd::pipeline { } static std::optional prepare_video_generation_embeds(StableDiffusionGGML* sd, - const sd_vid_gen_params_t* sd_vid_gen_params, + const sd::VideoConfig* sd_vid_gen_params, const GenerationRequest& request, const ImageGenerationLatents& latents, const RefImageParams& ref_image_params) { @@ -1264,10 +1265,10 @@ namespace sd::pipeline { return embeds; } - static sd_image_t* decode_video_outputs(StableDiffusionGGML* sd, - const GenerationRequest& request, - const sd::Tensor& final_latent, - int* num_frames_out) { + static sd::Image* decode_video_outputs(StableDiffusionGGML* sd, + const GenerationRequest& request, + const sd::Tensor& final_latent, + int* num_frames_out) { if (final_latent.empty()) { LOG_ERROR("no latent video to decode"); return nullptr; @@ -1305,7 +1306,7 @@ namespace sd::pipeline { vid = sd::ops::slice(vid, 2, 0, request.frames); } - sd_image_t* result_images = (sd_image_t*)calloc(vid.shape()[2], sizeof(sd_image_t)); + sd::Image* result_images = (sd::Image*)calloc(vid.shape()[2], sizeof(sd::Image)); if (result_images == nullptr) { return nullptr; } @@ -1661,7 +1662,7 @@ namespace sd::pipeline { } static bool apply_ltxv_refine_image_conditioning(StableDiffusionGGML* sd, - const sd_vid_gen_params_t* sd_vid_gen_params, + const sd::VideoConfig* sd_vid_gen_params, const GenerationRequest& request, const ImageGenerationLatents& latents, sd::Tensor* latent, @@ -1765,8 +1766,8 @@ namespace sd::pipeline { } static bool generate_animatediff_video(StableDiffusionGGML* sd, - const sd_vid_gen_params_t* sd_vid_gen_params, - sd_image_t** frames_out, + const sd::VideoConfig* sd_vid_gen_params, + sd::Image** frames_out, int* num_frames_out) { int n_frames = sd_vid_gen_params->video_frames; if (n_frames < 1) { @@ -1778,8 +1779,8 @@ namespace sd::pipeline { n_frames = 32; } - sd_img_gen_params_t img_gen_params; - sd_img_gen_params_init(&img_gen_params); + sd::ImageConfig img_gen_params; + img_gen_params.loras = sd_vid_gen_params->loras; img_gen_params.lora_count = sd_vid_gen_params->lora_count; img_gen_params.prompt = sd_vid_gen_params->prompt; @@ -1815,10 +1816,10 @@ namespace sd::pipeline { } bool generate_video(StableDiffusionGGML* sd, - const sd_vid_gen_params_t* sd_vid_gen_params, - sd_image_t** frames_out, + const sd::VideoConfig* sd_vid_gen_params, + sd::Image** frames_out, int* num_frames_out, - sd_audio_t** audio_out, + sd::Audio** audio_out, int* fps_out, nlohmann::json* info_out) { if (info_out) @@ -1847,7 +1848,7 @@ namespace sd::pipeline { sd->vae_tiling_params = sd_vid_gen_params->vae_tiling_params; sd->apply_circular_axes(sd_vid_gen_params->circular_x, sd_vid_gen_params->circular_y); sd::ImagePreprocessor preprocessing(sd_vid_gen_params->image_preprocess.rules); - sd_vid_gen_params_t processed_params = *sd_vid_gen_params; + sd::VideoConfig processed_params = *sd_vid_gen_params; if (!preprocessing.prepare_inputs(processed_params, request.width, request.height)) return false; sd_vid_gen_params = &processed_params; @@ -1897,7 +1898,6 @@ namespace sd::pipeline { sd->rng->manual_seed(request.seed); sd->sampler_rng->manual_seed(request.seed); - sd->set_flow_shift(sd_vid_gen_params->sample_params.flow_shift); if (!sd->apply_loras(sd_vid_gen_params->loras, sd_vid_gen_params->lora_count)) return false; sd->reset_generation_extensions(); @@ -2141,13 +2141,10 @@ namespace sd::pipeline { make_hires_sigma_schedule(sd, request.hires, request.sample_params, - hires_sample_method, plan.sample_steps, sd->get_image_seq_len(hires_request.height, hires_request.width) * T, &hires_scheduler_steps); - float hires_eta = resolve_eta(sd, - sd_vid_gen_params->sample_params.eta, - hires_sample_method); + float hires_eta = plan.eta; LOG_VERBOSE("sample(latent upscale) %dx%dx%d", W, H, T); LOG_INFO("video latent spatial upscale refine: scheduler_steps=%d, denoising_strength=%.2f, sampler=%s, sigma_sched_size=%zu%s", @@ -2197,11 +2194,11 @@ namespace sd::pipeline { int64_t latent_end = ggml_time_ms(); LOG_INFO("generating latent video completed, taking %.2fs", (latent_end - latent_start) * 1.0f / 1000); - sd_audio_t* generated_audio = nullptr; + sd::Audio* generated_audio = nullptr; if (sd->version == VERSION_WAN2_2_S2V && sd_vid_gen_params->ref_audios_count > 0) { // Return the driving track for muxing with the generated video. - const sd_audio_t& driving = sd_vid_gen_params->ref_audios[0]; - generated_audio = (sd_audio_t*)malloc(sizeof(sd_audio_t)); + const sd::Audio& driving = sd_vid_gen_params->ref_audios[0]; + generated_audio = (sd::Audio*)malloc(sizeof(sd::Audio)); if (generated_audio != nullptr) { generated_audio->sample_rate = driving.sample_rate; generated_audio->channels = driving.channels; @@ -2270,13 +2267,13 @@ namespace sd::pipeline { if (sd->get_cancel_flag() == SD_CANCEL_ALL) { LOG_ERROR("cancelling generation before video decode"); - free_sd_audio(generated_audio); + sd::free_audio(generated_audio); return false; } auto result = decode_video_outputs(sd, latent_upscale_enabled ? hires_request : request, final_latent, num_frames_out); LOG_DEBUG("decode_video_outputs returned %s", result == nullptr ? "nullptr (failed)" : "frames"); if (result == nullptr) { - free_sd_audio(generated_audio); + sd::free_audio(generated_audio); return false; } @@ -2287,12 +2284,12 @@ namespace sd::pipeline { if (frames_out != nullptr) { *frames_out = result; } - if (sd->version == VERSION_WAN2_2_S2V && generated_audio != nullptr) { - // Limit the driving track to the generated video's duration. + if ((sd->version == VERSION_WAN2_2_S2V || sd_version_is_minimax_h3(sd->version)) && generated_audio != nullptr) { + // Limit the audio track to the generated video's duration. int fps = request.fps; uint64_t video_frames = num_frames_out != nullptr ? (uint64_t)*num_frames_out : 0; uint64_t want_samples = (uint64_t)((double)video_frames / fps * generated_audio->sample_rate); - LOG_DEBUG("s2v audio truncate: %llu samples -> %llu (video %llu frames @ %d fps)", + LOG_DEBUG("output audio truncate: %llu samples -> %llu (video %llu frames @ %d fps)", (unsigned long long)generated_audio->sample_count, (unsigned long long)want_samples, (unsigned long long)video_frames, @@ -2305,14 +2302,14 @@ namespace sd::pipeline { try { *info_out = info; } catch (...) { - free_sd_audio(generated_audio); + sd::free_audio(generated_audio); throw; } } if (audio_out != nullptr) { *audio_out = generated_audio; } else { - free_sd_audio(generated_audio); + sd::free_audio(generated_audio); } return true; } diff --git a/src/pipeline/video_upscale.cpp b/src/pipeline/video_upscale.cpp index f9b631e637..98837e8e7a 100644 --- a/src/pipeline/video_upscale.cpp +++ b/src/pipeline/video_upscale.cpp @@ -1,4 +1,7 @@ #include "pipeline/video_upscale.h" +#include "api/options.h" +#include "api/result.h" +#include "generation_info.h" #include @@ -66,7 +69,7 @@ namespace sd { model_.reset(); } - bool VideoUpscaler::init(const sd_video_upscaler_ctx_params_t& params) { + bool VideoUpscaler::init(const sd::VideoUpscalerConfig& params) { if (!params.diffusion_model_path || !*params.diffusion_model_path || !params.vae_path || !*params.vae_path || params.upscale_factor <= 0) { LOG_ERROR("video upscale requires diffusion and decoder model paths and a positive upscale factor"); @@ -110,7 +113,7 @@ namespace sd { return model_->init(runtime_, weights, upscale_factor_); } - bool VideoUpscaler::upscale(const sd_video_upscale_params_t& params, sd_video_generation_result_t** result_out) { + bool VideoUpscaler::upscale(const sd::VideoUpscaleConfig& params, sd::VideoGenerationResult** result_out) { if (!result_out) return false; *result_out = nullptr; @@ -130,11 +133,11 @@ namespace sd { } } sd::ParallelScope scope(executor_.get()); - auto result = std::unique_ptr( - static_cast(calloc(1, sizeof(sd_video_generation_result_t))), free_sd_video_generation_result); + auto result = std::unique_ptr( + static_cast(calloc(1, sizeof(sd::VideoGenerationResult))), sd::free_video_generation_result); if (!result) return false; - result->frames = static_cast(calloc(params.frame_count, sizeof(sd_image_t))); + result->frames = static_cast(calloc(params.frame_count, sizeof(sd::Image))); if (!result->frames) return false; result->frame_count = params.frame_count; @@ -148,55 +151,50 @@ namespace sd { } // namespace pipeline } // namespace sd -struct video_upscaler_ctx_t { +struct sd_vid_upscaler { sd::pipeline::VideoUpscaler upscaler; }; -void sd_video_upscaler_ctx_params_init(sd_video_upscaler_ctx_params_t* params) { - if (!params) - return; - *params = {}; - params->n_threads = sd_get_num_physical_cores(); +sd_status_t sd_vid_upscaler_create(const sd_vid_upscaler_opt_t* opt, sd_vid_upscaler_t** out) { + if (out) + *out = nullptr; + return sd::api::call([&] { + if (!out) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required"); + sd::OptConfig config; + auto status = sd::api::resolve(opt, sd::api::VidUpscaler, config); + if (status != SD_STATUS_OK) + return status; + auto result = std::make_unique(); + if (!result->upscaler.init(config.post.video_upscaler)) + return sd::api::error(SD_STATUS_LOAD_FAILED, nullptr, "could not load video upscaler"); + *out = result.release(); + return SD_STATUS_OK; + }, + SD_STATUS_LOAD_FAILED); } -void sd_video_upscale_params_init(sd_video_upscale_params_t* params) { - if (!params) - return; - *params = {}; - params->fps = 24; - params->seed = 42; - params->window_size = 16; - params->temporal_chunk_size = 5; +void sd_vid_upscaler_free(sd_vid_upscaler_t* upscaler) { + delete upscaler; } -video_upscaler_ctx_t* new_video_upscaler_ctx(const sd_video_upscaler_ctx_params_t* params) { - if (!params) - return nullptr; - try { - auto ctx = std::make_unique(); - if (!ctx->upscaler.init(*params)) - return nullptr; - return ctx.release(); - } catch (const std::exception& error) { - LOG_ERROR("video upscaler initialization failed: %s", error.what()); - return nullptr; - } -} - -bool upscale_video(video_upscaler_ctx_t* ctx, const sd_video_upscale_params_t* params, sd_video_generation_result_t** result_out) { - if (!result_out) - return false; - *result_out = nullptr; - if (!ctx || !params) - return false; - try { - return ctx->upscaler.upscale(*params, result_out); - } catch (const std::exception& error) { - LOG_ERROR("video upscale failed: %s", error.what()); - return false; - } -} - -void free_video_upscaler_ctx(video_upscaler_ctx_t* ctx) { - delete ctx; +sd_status_t sd_upscale_video(sd_vid_upscaler_t* upscaler, const sd_vid_upscale_opt_t* opt, sd_result_t** out) { + if (out) + *out = nullptr; + return sd::api::call([&] { + if (!out) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required"); + sd::OptConfig config; + auto status = sd::api::resolve(opt, sd::api::VidUpscale, config); + if (status != SD_STATUS_OK) + return status; + if (!upscaler) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "upscaler is required"); + sd::VideoGenerationResult* video = nullptr; + if (!upscaler->upscaler.upscale(config.post.video_upscale, &video)) + return sd::api::error(SD_STATUS_GENERATION_FAILED, nullptr, "video upscale failed"); + return sd::make_video_result(video, out) ? SD_STATUS_OK + : sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate result"); + }, + SD_STATUS_GENERATION_FAILED); } diff --git a/src/pipeline/video_upscale.h b/src/pipeline/video_upscale.h index ec5815c44f..51ab7681ed 100644 --- a/src/pipeline/video_upscale.h +++ b/src/pipeline/video_upscale.h @@ -4,6 +4,8 @@ #include #include +#include "api/result.h" +#include "config.h" #include "core/ggml_extend_backend.h" #include "core/tensor.hpp" #include "model_manager.h" @@ -34,7 +36,7 @@ namespace sd { virtual ~VideoUpscaleModel() = default; virtual bool init(VideoUpscaleRuntime& runtime, const String2TensorStorage& weights, int upscale_factor) = 0; virtual void end() = 0; - virtual bool upscale(const sd_video_upscale_params_t& params, sd_video_generation_result_t* result) = 0; + virtual bool upscale(const sd::VideoUpscaleConfig& params, sd::VideoGenerationResult* result) = 0; }; bool is_relay_vsr_model(const String2TensorStorage& weights); @@ -50,8 +52,8 @@ namespace sd { public: ~VideoUpscaler(); - bool init(const sd_video_upscaler_ctx_params_t& params); - bool upscale(const sd_video_upscale_params_t& params, sd_video_generation_result_t** result_out); + bool init(const sd::VideoUpscalerConfig& params); + bool upscale(const sd::VideoUpscaleConfig& params, sd::VideoGenerationResult** result_out); }; } // namespace pipeline diff --git a/src/runtime/denoiser.hpp b/src/runtime/denoiser.hpp index 701d8d9983..b947074694 100644 --- a/src/runtime/denoiser.hpp +++ b/src/runtime/denoiser.hpp @@ -1,6 +1,8 @@ #ifndef __SD_RUNTIME_DENOISER_HPP__ #define __SD_RUNTIME_DENOISER_HPP__ +#include "core/option_types.h" + #include #include #include diff --git a/src/runtime/image_preprocess.cpp b/src/runtime/image_preprocess.cpp index 445f0d1fc8..d5e17d9984 100644 --- a/src/runtime/image_preprocess.cpp +++ b/src/runtime/image_preprocess.cpp @@ -196,7 +196,7 @@ namespace sd { return options; } - bool ImagePreprocessor::validate_inputs(const sd_img_gen_params_t& params) const { + bool ImagePreprocessor::validate_inputs(const sd::ImageConfig& params) const { const std::map counts = { {ImageTarget::Init, params.init_image.data != nullptr}, {ImageTarget::Mask, params.mask_image.data != nullptr}, @@ -215,7 +215,7 @@ namespace sd { return valid_; } - bool ImagePreprocessor::validate_inputs(const sd_vid_gen_params_t& params) const { + bool ImagePreprocessor::validate_inputs(const sd::VideoConfig& params) const { const std::map counts = { {ImageTarget::Init, params.init_image.data != nullptr}, {ImageTarget::End, params.end_image.data != nullptr}, @@ -370,7 +370,7 @@ namespace sd { return apply_transform(image, options, p, label, default_filter); } - Tensor ImagePreprocessor::preprocess_input(sd_image_t image, ImageTarget target, int index, int width, int height) { + Tensor ImagePreprocessor::preprocess_input(sd::Image image, ImageTarget target, int index, int width, int height) { if (image.data == nullptr || image.width == 0 || image.height == 0 || image.width > INT_MAX || image.height > INT_MAX || image.channel < 1 || image.channel > 4) { fail(std::string(enum_name(target, image_targets)) + ": invalid input image"); return {}; @@ -419,7 +419,7 @@ namespace sd { std::free(image.data); } - bool ImagePreprocessor::prepare_image(sd_image_t& image, ImageTarget target, int index, int width, int height) { + bool ImagePreprocessor::prepare_image(sd::Image& image, ImageTarget target, int index, int width, int height) { if (image.data == nullptr) return true; auto options = resolve_options(target, index); @@ -431,13 +431,13 @@ namespace sd { if (output.data == nullptr) return fail("could not allocate input preprocessing buffer"); owned_images_.push_back(output); - if (canny && !preprocess_canny(output, 0.08f, 0.08f, 0.8f, 1.f, false)) + if (canny && !sd::preprocess_canny(output, 0.08f, 0.08f, 0.8f, 1.f, false)) return fail("Canny preprocessing failed"); image = output; return true; } - bool ImagePreprocessor::prepare_array(sd_image_t*& images, int count, ImageTarget target, std::vector& storage, int width, int height) { + bool ImagePreprocessor::prepare_array(sd::Image*& images, int count, ImageTarget target, std::vector& storage, int width, int height) { if (count < 0 || (count > 0 && images == nullptr)) return fail(std::string("invalid image array: ") + enum_name(target, image_targets)); if (count == 0) @@ -453,7 +453,7 @@ namespace sd { return true; } - bool ImagePreprocessor::prepare_inputs(sd_img_gen_params_t& params, int width, int height) { + bool ImagePreprocessor::prepare_inputs(sd::ImageConfig& params, int width, int height) { if (prepared_) return fail("inputs have already been prepared"); prepared_ = true; @@ -470,7 +470,7 @@ namespace sd { return true; } - bool ImagePreprocessor::prepare_inputs(sd_vid_gen_params_t& params, int width, int height) { + bool ImagePreprocessor::prepare_inputs(sd::VideoConfig& params, int width, int height) { if (prepared_) return fail("inputs have already been prepared"); prepared_ = true; diff --git a/src/runtime/image_preprocess.h b/src/runtime/image_preprocess.h index 7dfc458992..078befdb9c 100644 --- a/src/runtime/image_preprocess.h +++ b/src/runtime/image_preprocess.h @@ -1,11 +1,14 @@ #ifndef __SD_RUNTIME_IMAGE_PREPROCESS_H__ #define __SD_RUNTIME_IMAGE_PREPROCESS_H__ +#include "core/media.h" + #include #include #include #include +#include "config.h" #include "core/tensor.hpp" #include "stable-diffusion.h" @@ -61,30 +64,30 @@ namespace sd { ImageTransform init_transform_; bool has_init_transform_ = false; bool prepared_ = false; - std::vector owned_images_; - std::vector ref_images_; - std::vector id_images_; - std::vector control_frames_; + std::vector owned_images_; + std::vector ref_images_; + std::vector id_images_; + std::vector control_frames_; bool fail(const std::string& message) const; std::map resolve_options(ImageTarget target, int index) const; Tensor apply_transform(const Tensor& image, const std::map& options, ImageTransform plan, const std::string& label, ops::InterpolateMode default_filter) const; - bool prepare_image(sd_image_t& image, ImageTarget target, int index, int width, int height); - bool prepare_array(sd_image_t*& images, int count, ImageTarget target, std::vector& storage, int width = -1, int height = -1); + bool prepare_image(sd::Image& image, ImageTarget target, int index, int width, int height); + bool prepare_array(sd::Image*& images, int count, ImageTarget target, std::vector& storage, int width = -1, int height = -1); public: explicit ImagePreprocessor(const char* rules = nullptr); ~ImagePreprocessor(); ImagePreprocessor(const ImagePreprocessor&) = delete; ImagePreprocessor& operator=(const ImagePreprocessor&) = delete; - bool prepare_inputs(sd_img_gen_params_t& params, int width, int height); - bool prepare_inputs(sd_vid_gen_params_t& params, int width, int height); + bool prepare_inputs(sd::ImageConfig& params, int width, int height); + bool prepare_inputs(sd::VideoConfig& params, int width, int height); bool is_valid() const { return valid_; } - bool validate_inputs(const sd_img_gen_params_t& params) const; - bool validate_inputs(const sd_vid_gen_params_t& params) const; + bool validate_inputs(const sd::ImageConfig& params) const; + bool validate_inputs(const sd::VideoConfig& params) const; Tensor apply_geometry(const Tensor& image, ImageTarget target, int index, int width, int height, ImageResizeMode default_mode = ImageResizeMode::Stretch, ops::InterpolateMode default_filter = ops::InterpolateMode::Nearest, ImageTransform* plan_out = nullptr) const; - Tensor preprocess_input(sd_image_t image, ImageTarget target, int index = 0, int width = -1, int height = -1); + Tensor preprocess_input(sd::Image image, ImageTarget target, int index = 0, int width = -1, int height = -1); }; } // namespace sd diff --git a/src/runtime/preprocessing.hpp b/src/runtime/preprocessing.hpp index ab4ca29b64..da91e4666b 100644 --- a/src/runtime/preprocessing.hpp +++ b/src/runtime/preprocessing.hpp @@ -1,6 +1,8 @@ #ifndef __SD_RUNTIME_PREPROCESSING_HPP__ #define __SD_RUNTIME_PREPROCESSING_HPP__ +#include "core/media.h" + #include #include @@ -93,7 +95,7 @@ static inline void preprocessing_tensor_frame_to_sd_image(const sd::Tensor sd_image_to_preprocessing_tensor(sd_image_t image) { +static inline sd::Tensor sd_image_to_preprocessing_tensor(sd::Image image) { sd::Tensor tensor({static_cast(image.width), static_cast(image.height), static_cast(image.channel), 1}); for (uint32_t y = 0; y < image.height; ++y) { for (uint32_t x = 0; x < image.width; ++x) { @@ -293,7 +295,7 @@ static inline void threshold_hystersis(sd::Tensor* img, float high_thresh } } -bool preprocess_canny(sd_image_t img, float high_threshold, float low_threshold, float weak, float strong, bool inverse) { +bool sd::preprocess_canny(sd::Image img, float high_threshold, float low_threshold, float weak, float strong, bool inverse) { float kX[9] = { -1, 0, 1, -2, 0, 2, diff --git a/src/runtime/sample-cache.cpp b/src/runtime/sample-cache.cpp index 0b92c73358..c8ea2c12ce 100644 --- a/src/runtime/sample-cache.cpp +++ b/src/runtime/sample-cache.cpp @@ -1,8 +1,9 @@ #include "runtime/sample-cache.h" +#include "config.h" namespace sd_sample { - static float get_cache_reuse_threshold(const sd_cache_params_t& params) { + static float get_cache_reuse_threshold(const sd::CacheConfig& params) { float reuse_threshold = params.reuse_threshold; if (reuse_threshold == INFINITY) { if (params.mode == SD_CACHE_EASYCACHE) { @@ -26,7 +27,7 @@ namespace sd_sample { return mode == SampleCacheMode::CACHEDIT; } - static bool has_valid_cache_percent_range(const sd_cache_params_t& cache_params) { + static bool has_valid_cache_percent_range(const sd::CacheConfig& cache_params) { if (cache_params.mode != SD_CACHE_EASYCACHE && cache_params.mode != SD_CACHE_UCACHE) { return true; } @@ -40,7 +41,7 @@ namespace sd_sample { static void init_easycache_runtime(SampleCacheRuntime& runtime, SDVersion version, - const sd_cache_params_t& cache_params, + const sd::CacheConfig& cache_params, Denoiser* denoiser) { if (!sd_version_is_dit(version)) { LOG_WARN("EasyCache requested but not supported for this model type"); @@ -68,7 +69,7 @@ namespace sd_sample { static void init_ucache_runtime(SampleCacheRuntime& runtime, SDVersion version, - const sd_cache_params_t& cache_params, + const sd::CacheConfig& cache_params, Denoiser* denoiser, const std::vector& sigmas) { if (!sd_version_is_unet(version)) { @@ -104,7 +105,7 @@ namespace sd_sample { static void init_cachedit_runtime(SampleCacheRuntime& runtime, SDVersion version, - const sd_cache_params_t& cache_params, + const sd::CacheConfig& cache_params, const std::vector& sigmas) { if (!sd_version_is_dit(version)) { LOG_WARN("CacheDIT requested but not supported for this model type (only DiT models)"); @@ -147,7 +148,7 @@ namespace sd_sample { static void init_spectrum_runtime(SampleCacheRuntime& runtime, SDVersion version, - const sd_cache_params_t& cache_params, + const sd::CacheConfig& cache_params, const std::vector& sigmas) { if (!sd_version_is_unet(version) && !sd_version_is_dit(version)) { LOG_WARN("Spectrum requested but not supported for this model type (only UNET and DiT models)"); @@ -174,7 +175,7 @@ namespace sd_sample { } SampleCacheRuntime init_sample_cache_runtime(SDVersion version, - const sd_cache_params_t* cache_params, + const sd::CacheConfig* cache_params, Denoiser* denoiser, const std::vector& sigmas) { SampleCacheRuntime runtime; diff --git a/src/runtime/sample-cache.h b/src/runtime/sample-cache.h index 30ff66e5fe..fd22edc819 100644 --- a/src/runtime/sample-cache.h +++ b/src/runtime/sample-cache.h @@ -1,6 +1,8 @@ #ifndef __SD_RUNTIME_SAMPLE_CACHE_H__ #define __SD_RUNTIME_SAMPLE_CACHE_H__ +#include "config.h" + #include #include "core/tensor.hpp" @@ -51,7 +53,7 @@ namespace sd_sample { }; SampleCacheRuntime init_sample_cache_runtime(SDVersion version, - const sd_cache_params_t* cache_params, + const sd::CacheConfig* cache_params, Denoiser* denoiser, const std::vector& sigmas); diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 4a7af2a2ee..acc3672cf0 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -1,890 +1,135 @@ -#include "stable-diffusion.h" -#include "generation_info.h" - -#include -#include -#include -#include - -#include "core/ggml_extend.h" -#include "core/util.h" +#include "api/api.h" +#include "api/options.h" +#include "api/result.h" #include "pipeline/diffusion_engine.h" #include "pipeline/generation.h" -#include "pipeline/request.h" - -#define NONE_STR "NONE" - -static float get_cache_reuse_threshold(const sd_cache_params_t& params) { - float reuse_threshold = params.reuse_threshold; - if (reuse_threshold == INFINITY) { - if (params.mode == SD_CACHE_EASYCACHE) { - reuse_threshold = 0.2f; - } else if (params.mode == SD_CACHE_UCACHE) { - reuse_threshold = 1.0f; - } - } - return std::max(0.0f, reuse_threshold); -} - -const char* sd_type_name(enum sd_type_t type) { - if (type == SD_TYPE_F8_E4M3) { - return "f8_e4m3"; - } - if (type == SD_TYPE_F8_E5M2) { - return "f8_e5m2"; - } - const auto ggml_type = sd_type_to_ggml_type(type); - if (ggml_type != GGML_TYPE_COUNT) { - return ggml_type_name(ggml_type); - } - return NONE_STR; -} - -enum sd_type_t str_to_sd_type(const char* str) { - if (!strcmp(str, "f8_e4m3")) { - return SD_TYPE_F8_E4M3; - } - if (!strcmp(str, "f8_e5m2")) { - return SD_TYPE_F8_E5M2; - } - for (int i = 0; i < std::min(SD_TYPE_COUNT, GGML_TYPE_COUNT); i++) { - auto trait = ggml_get_type_traits((ggml_type)i); - if (!strcmp(str, trait->type_name)) { - return (enum sd_type_t)i; - } - } - return SD_TYPE_COUNT; -} - -const char* rng_type_to_str[] = { - "std_default", - "cuda", - "cpu", -}; - -const char* sd_rng_type_name(enum rng_type_t rng_type) { - if (rng_type < RNG_TYPE_COUNT) { - return rng_type_to_str[rng_type]; - } - return NONE_STR; -} - -enum rng_type_t str_to_rng_type(const char* str) { - for (int i = 0; i < RNG_TYPE_COUNT; i++) { - if (!strcmp(str, rng_type_to_str[i])) { - return (enum rng_type_t)i; - } - } - return RNG_TYPE_COUNT; -} - -const char* sample_method_to_str[] = { - "euler", - "euler_a", - "heun", - "dpm2", - "dpm++2s_a", - "dpm++2m", - "dpm++2mv2", - "ipndm", - "ipndm_v", - "lcm", - "ddim_trailing", - "tcd", - "res_multistep", - "res_2s", - "er_sde", - "euler_cfg_pp", - "euler_a_cfg_pp", - "euler_ge", - "dpm++2m_sde", - "dpm++2m_sde_bt", - "lms", -}; - -static_assert(SAMPLE_METHOD_COUNT == sizeof(sample_method_to_str) / sizeof(sample_method_to_str[0]), - "\nnumber of elements in sample_method_to_str[] != SAMPLE_METHOD_COUNT"); - -const char* sd_sample_method_name(enum sample_method_t sample_method) { - if (sample_method < SAMPLE_METHOD_COUNT) { - return sample_method_to_str[sample_method]; - } - return NONE_STR; -} - -enum sample_method_t str_to_sample_method(const char* str) { - for (int i = 0; i < SAMPLE_METHOD_COUNT; i++) { - if (!strcmp(str, sample_method_to_str[i])) { - return (enum sample_method_t)i; - } - } - return SAMPLE_METHOD_COUNT; -} - -const char* scheduler_to_str[] = { - "discrete", - "karras", - "exponential", - "ays", - "gits", - "sgm_uniform", - "simple", - "smoothstep", - "kl_optimal", - "lcm", - "bong_tangent", - "ltx2", - "logit_normal", - "flux2", - "flux", - "beta", - "llada_image", -}; - -static_assert(SCHEDULER_COUNT == sizeof(scheduler_to_str) / sizeof(scheduler_to_str[0]), - "\nnumber of elements in scheduler_to_str[] != SCHEDULER_COUNT"); - -const char* sd_scheduler_name(enum scheduler_t scheduler) { - if (scheduler < SCHEDULER_COUNT) { - return scheduler_to_str[scheduler]; - } - return NONE_STR; -} - -enum scheduler_t str_to_scheduler(const char* str) { - if (!strcmp(str, "normal")) { - return DISCRETE_SCHEDULER; - } - for (int i = 0; i < SCHEDULER_COUNT; i++) { - if (!strcmp(str, scheduler_to_str[i])) { - return (enum scheduler_t)i; - } - } - return SCHEDULER_COUNT; -} - -const char* prediction_to_str[] = { - "eps", - "v", - "edm_v", - "sd3_flow", - "flux_flow", - "sefi_flow", - "minit2i_flow", - "sensenova_u1_flow", - "flux3_flow", - "h3_av_flow", - "ming_image_flow", -}; - -const char* sd_prediction_name(enum prediction_t prediction) { - if (prediction >= EPS_PRED && prediction < PREDICTION_COUNT) { - return prediction_to_str[prediction]; - } - return NONE_STR; -} - -enum prediction_t str_to_prediction(const char* str) { - for (int i = 0; i < PREDICTION_COUNT; i++) { - if (!strcmp(str, prediction_to_str[i])) { - return (enum prediction_t)i; - } - } - return PREDICTION_COUNT; -} - -const char* preview_to_str[] = { - "none", - "proj", - "tae", - "vae", -}; - -const char* sd_preview_name(enum preview_t preview) { - if (preview < PREVIEW_COUNT) { - return preview_to_str[preview]; - } - return NONE_STR; -} - -enum preview_t str_to_preview(const char* str) { - for (int i = 0; i < PREVIEW_COUNT; i++) { - if (!strcmp(str, preview_to_str[i])) { - return (enum preview_t)i; - } - } - return PREVIEW_COUNT; -} - -const char* lora_apply_mode_to_str[] = { - "auto", - "immediately", - "at_runtime", -}; - -const char* sd_lora_apply_mode_name(enum lora_apply_mode_t mode) { - if (mode < LORA_APPLY_MODE_COUNT) { - return lora_apply_mode_to_str[mode]; - } - return NONE_STR; -} - -enum lora_apply_mode_t str_to_lora_apply_mode(const char* str) { - for (int i = 0; i < LORA_APPLY_MODE_COUNT; i++) { - if (!strcmp(str, lora_apply_mode_to_str[i])) { - return (enum lora_apply_mode_t)i; - } - } - return LORA_APPLY_MODE_COUNT; -} - -const char* hires_upscaler_to_str[] = { - "None", - "Latent", - "Latent (nearest)", - "Latent (nearest-exact)", - "Latent (antialiased)", - "Latent (bicubic)", - "Latent (bicubic antialiased)", - "Lanczos", - "Nearest", - "Model", -}; - -const char* sd_hires_upscaler_name(enum sd_hires_upscaler_t upscaler) { - if (upscaler >= SD_HIRES_UPSCALER_NONE && upscaler < SD_HIRES_UPSCALER_COUNT) { - return hires_upscaler_to_str[upscaler]; - } - return NONE_STR; -} - -enum sd_hires_upscaler_t str_to_sd_hires_upscaler(const char* str) { - for (int i = 0; i < SD_HIRES_UPSCALER_COUNT; i++) { - if (!strcmp(str, hires_upscaler_to_str[i])) { - return (enum sd_hires_upscaler_t)i; - } - } - return SD_HIRES_UPSCALER_COUNT; -} - -const char* control_type_to_str[] = { - "none", - "openpose", - "depth", - "scribble", - "canny", - "normal", - "segment", - "tile", - "repaint", -}; - -static_assert(SD_CONTROL_TYPE_COUNT == sizeof(control_type_to_str) / sizeof(control_type_to_str[0]), - "\nnumber of elements in control_type_to_str[] != SD_CONTROL_TYPE_COUNT"); - -const char* sd_control_type_name(enum sd_control_type_t control_type) { - if (control_type >= SD_CONTROL_TYPE_NONE && control_type < SD_CONTROL_TYPE_COUNT) { - return control_type_to_str[control_type]; - } - return NONE_STR; -} - -enum sd_control_type_t str_to_sd_control_type(const char* str) { - for (int i = 0; i < SD_CONTROL_TYPE_COUNT; i++) { - if (!strcmp(str, control_type_to_str[i])) { - return (enum sd_control_type_t)i; - } - } - static const struct { - const char* alias; - enum sd_control_type_t type; - } aliases[] = { - {"hed", SD_CONTROL_TYPE_SCRIBBLE}, - {"softedge", SD_CONTROL_TYPE_SCRIBBLE}, - {"pidi", SD_CONTROL_TYPE_SCRIBBLE}, - {"ted", SD_CONTROL_TYPE_SCRIBBLE}, - {"lineart", SD_CONTROL_TYPE_CANNY}, - {"anime_lineart", SD_CONTROL_TYPE_CANNY}, - {"mlsd", SD_CONTROL_TYPE_CANNY}, - {"seg", SD_CONTROL_TYPE_SEGMENT}, - {"inpaint", SD_CONTROL_TYPE_REPAINT}, - {"outpaint", SD_CONTROL_TYPE_REPAINT}, - }; - for (const auto& entry : aliases) { - if (!strcmp(str, entry.alias)) { - return entry.type; - } - } - return SD_CONTROL_TYPE_COUNT; -} -const char* sd_vae_format_name(enum sd_vae_format_t format) { - switch (format) { - case SD_VAE_FORMAT_AUTO: - return "auto"; - case SD_VAE_FORMAT_FLUX: - return "flux"; - case SD_VAE_FORMAT_SD3: - return "sd3"; - case SD_VAE_FORMAT_FLUX2: - return "flux2"; - case SD_VAE_FORMAT_WAN: - return "wan"; - default: - return NONE_STR; - } -} - -void sd_cache_params_init(sd_cache_params_t* cache_params) { - *cache_params = {}; - cache_params->mode = SD_CACHE_DISABLED; - cache_params->reuse_threshold = INFINITY; - cache_params->start_percent = 0.15f; - cache_params->end_percent = 0.95f; - cache_params->error_decay_rate = 1.0f; - cache_params->use_relative_threshold = true; - cache_params->reset_error_on_compute = true; - cache_params->Fn_compute_blocks = 8; - cache_params->Bn_compute_blocks = 0; - cache_params->residual_diff_threshold = 0.08f; - cache_params->max_warmup_steps = 8; - cache_params->max_cached_steps = -1; - cache_params->max_continuous_cached_steps = -1; - cache_params->taylorseer_n_derivatives = 1; - cache_params->taylorseer_skip_interval = 1; - cache_params->scm_mask = nullptr; - cache_params->scm_policy_dynamic = true; - cache_params->spectrum_w = 0.40f; - cache_params->spectrum_m = 3; - cache_params->spectrum_lam = 1.0f; - cache_params->spectrum_window_size = 2; - cache_params->spectrum_flex_window = 0.50f; - cache_params->spectrum_warmup_steps = 4; - cache_params->spectrum_stop_percent = 0.9f; -} - -void sd_hires_params_init(sd_hires_params_t* hires_params) { - *hires_params = {}; - hires_params->enabled = false; - hires_params->upscaler = SD_HIRES_UPSCALER_LATENT; - hires_params->model_path = nullptr; - hires_params->scale = 2.0f; - hires_params->target_width = 0; - hires_params->target_height = 0; - hires_params->steps = 0; - hires_params->denoising_strength = 0.7f; - hires_params->upscale_tile_size = 128; - hires_params->custom_sigmas = nullptr; - hires_params->custom_sigmas_count = 0; -} - -void sd_ctx_params_init(sd_ctx_params_t* sd_ctx_params) { - *sd_ctx_params = {}; - sd_ctx_params->n_threads = sd_get_num_physical_cores(); - sd_ctx_params->conditioning_cache_size = 4; - sd_ctx_params->wtype = SD_TYPE_COUNT; - sd_ctx_params->rng_type = CUDA_RNG; - sd_ctx_params->sampler_rng_type = RNG_TYPE_COUNT; - sd_ctx_params->prediction = PREDICTION_COUNT; - sd_ctx_params->lora_apply_mode = LORA_APPLY_AUTO; - sd_ctx_params->max_vram = nullptr; - sd_ctx_params->disable_prefetch = false; - sd_ctx_params->disable_segmented_compute = false; - sd_ctx_params->batched_cfg = true; - sd_ctx_params->eager_load = false; - sd_ctx_params->enable_mmap = false; - sd_ctx_params->flash_attn_mode = nullptr; - sd_ctx_params->conv_direct_mode = nullptr; - sd_ctx_params->diffusion_flash_attn = false; - sd_ctx_params->sage_attn = false; - sd_ctx_params->linear_scale = 0.f; - sd_ctx_params->attn_scale = 0.f; - sd_ctx_params->vae_format = SD_VAE_FORMAT_AUTO; - sd_ctx_params->backend = nullptr; - sd_ctx_params->params_backend = nullptr; - sd_ctx_params->split_mode = nullptr; - sd_ctx_params->auto_fit = true; - sd_ctx_params->rpc_servers = nullptr; - sd_ctx_params->model_args = nullptr; - sd_ctx_params->pulid_weights_path = nullptr; -} - -char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params) { - char* buf = (char*)malloc(8192); - if (!buf) - return nullptr; - buf[0] = '\0'; - - snprintf(buf + strlen(buf), 8192 - strlen(buf), - "model_path: %s\n" - "clip_l_path: %s\n" - "clip_g_path: %s\n" - "clip_vision_path: %s\n" - "t5xxl_path: %s\n" - "llm_path: %s\n" - "llm_vision_path: %s\n" - "tokenizer: %s\n" - "diffusion_model_path: %s\n" - "high_noise_diffusion_model_path: %s\n" - "uncond_diffusion_model_path: %s\n" - "embeddings_connectors_path: %s\n" - "vae_path: %s\n" - "audio_vae_path: %s\n" - "audio_encoder_path: %s\n" - "taesd_path: %s\n" - "control_net_path: %s\n" - "photo_maker_path: %s\n" - "pulid_weights_path: %s\n" - "tensor_type_rules: %s\n" - "n_threads: %d\n" - "conditioning_cache_size: %d\n" - "wtype: %s\n" - "rng_type: %s\n" - "sampler_rng_type: %s\n" - "prediction: %s\n" - "max_vram: %s\n" - "disable_prefetch: %s\n" - "disable_segmented_compute: %s\n" - "eager_load: %s\n" - "backend: %s\n" - "params_backend: %s\n" - "split_mode: %s\n" - "model_args: %s\n" - "auto_fit: %s\n" - "flash_attn: %s\n" - "flash_attn_mode: %s\n" - "conv_direct_mode: %s\n" - "diffusion_flash_attn: %s\n" - "sage_attn: %s\n" - "linear_scale: %g\n" - "attn_scale: %g\n" - "vae_format: %s\n", - SAFE_STR(sd_ctx_params->model_path), - SAFE_STR(sd_ctx_params->clip_l_path), - SAFE_STR(sd_ctx_params->clip_g_path), - SAFE_STR(sd_ctx_params->clip_vision_path), - SAFE_STR(sd_ctx_params->t5xxl_path), - SAFE_STR(sd_ctx_params->llm_path), - SAFE_STR(sd_ctx_params->llm_vision_path), - SAFE_STR(sd_ctx_params->tokenizer), - SAFE_STR(sd_ctx_params->diffusion_model_path), - SAFE_STR(sd_ctx_params->high_noise_diffusion_model_path), - SAFE_STR(sd_ctx_params->uncond_diffusion_model_path), - SAFE_STR(sd_ctx_params->embeddings_connectors_path), - SAFE_STR(sd_ctx_params->vae_path), - SAFE_STR(sd_ctx_params->audio_vae_path), - SAFE_STR(sd_ctx_params->audio_encoder_path), - SAFE_STR(sd_ctx_params->taesd_path), - SAFE_STR(sd_ctx_params->control_net_path), - SAFE_STR(sd_ctx_params->photo_maker_path), - SAFE_STR(sd_ctx_params->pulid_weights_path), - SAFE_STR(sd_ctx_params->tensor_type_rules), - sd_ctx_params->n_threads, - sd_ctx_params->conditioning_cache_size, - sd_type_name(sd_ctx_params->wtype), - sd_rng_type_name(sd_ctx_params->rng_type), - sd_rng_type_name(sd_ctx_params->sampler_rng_type), - sd_prediction_name(sd_ctx_params->prediction), - SAFE_STR(sd_ctx_params->max_vram), - BOOL_STR(sd_ctx_params->disable_prefetch), - BOOL_STR(sd_ctx_params->disable_segmented_compute), - BOOL_STR(sd_ctx_params->eager_load), - SAFE_STR(sd_ctx_params->backend), - SAFE_STR(sd_ctx_params->params_backend), - SAFE_STR(sd_ctx_params->split_mode), - SAFE_STR(sd_ctx_params->model_args), - BOOL_STR(sd_ctx_params->auto_fit), - BOOL_STR(sd_ctx_params->flash_attn), - sd_ctx_params->flash_attn_mode && sd_ctx_params->flash_attn_mode[0] ? sd_ctx_params->flash_attn_mode : "auto (legacy flags honored)", - sd_ctx_params->conv_direct_mode && sd_ctx_params->conv_direct_mode[0] ? sd_ctx_params->conv_direct_mode : "auto (legacy flags honored)", - BOOL_STR(sd_ctx_params->diffusion_flash_attn), - BOOL_STR(sd_ctx_params->sage_attn), - sd_ctx_params->linear_scale, - sd_ctx_params->attn_scale, - sd_vae_format_name(sd_ctx_params->vae_format)); - - return buf; -} - -void sd_sample_params_init(sd_sample_params_t* sample_params) { - *sample_params = {}; - sample_params->guidance.txt_cfg = 7.0f; - sample_params->guidance.img_cfg = INFINITY; - sample_params->guidance.distilled_guidance = 3.5f; - sample_params->guidance.slg.layer_count = 0; - sample_params->guidance.slg.layer_start = 0.01f; - sample_params->guidance.slg.layer_end = 0.2f; - sample_params->guidance.slg.scale = 0.f; - sample_params->scheduler = SCHEDULER_COUNT; - sample_params->sample_method = SAMPLE_METHOD_COUNT; - sample_params->sample_steps = 20; - sample_params->eta = INFINITY; - sample_params->custom_sigmas = nullptr; - sample_params->custom_sigmas_count = 0; - sample_params->flow_shift = INFINITY; - sample_params->extra_sample_args = nullptr; -} - -char* sd_sample_params_to_str(const sd_sample_params_t* sample_params) { - char* buf = (char*)malloc(4096); - if (!buf) - return nullptr; - buf[0] = '\0'; - - snprintf(buf + strlen(buf), 4096 - strlen(buf), - "(txt_cfg: %.2f, " - "img_cfg: %.2f, " - "distilled_guidance: %.2f, " - "slg.layer_count: %zu, " - "slg.layer_start: %.2f, " - "slg.layer_end: %.2f, " - "slg.scale: %.2f, " - "scheduler: %s, " - "sample_method: %s, " - "sample_steps: %d, " - "eta: %.2f, " - "shifted_timestep: %d, " - "flow_shift: %.2f, " - "extra_sample_args: %s)", - sample_params->guidance.txt_cfg, - std::isfinite(sample_params->guidance.img_cfg) - ? sample_params->guidance.img_cfg - : sample_params->guidance.txt_cfg, - sample_params->guidance.distilled_guidance, - sample_params->guidance.slg.layer_count, - sample_params->guidance.slg.layer_start, - sample_params->guidance.slg.layer_end, - sample_params->guidance.slg.scale, - sd_scheduler_name(sample_params->scheduler), - sd_sample_method_name(sample_params->sample_method), - sample_params->sample_steps, - sample_params->eta, - sample_params->shifted_timestep, - sample_params->flow_shift, - SAFE_STR(sample_params->extra_sample_args)); - - return buf; -} - -void sd_img_gen_params_init(sd_img_gen_params_t* sd_img_gen_params) { - *sd_img_gen_params = {}; - sd_sample_params_init(&sd_img_gen_params->sample_params); - sd_img_gen_params->clip_skip = -1; - sd_img_gen_params->ref_images_count = 0; - sd_img_gen_params->ref_image_args = ""; - sd_img_gen_params->width = 512; - sd_img_gen_params->height = 512; - sd_img_gen_params->strength = 0.75f; - sd_img_gen_params->seed = -1; - sd_img_gen_params->batch_count = 1; - sd_img_gen_params->control_strength = 0.9f; - sd_img_gen_params->ip_adapter_strength = 1.0f; - sd_img_gen_params->qwen_image_layers = 3; - sd_img_gen_params->circular_x = false; - sd_img_gen_params->circular_y = false; - sd_img_gen_params->pm_params = {nullptr, 0, nullptr, 20.f}; - sd_img_gen_params->pulid_params = {nullptr, 1.0f}; - sd_img_gen_params->vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr}; - sd_cache_params_init(&sd_img_gen_params->cache); - sd_hires_params_init(&sd_img_gen_params->hires); -} - -char* sd_img_gen_params_to_str(const sd_img_gen_params_t* sd_img_gen_params) { - char* buf = (char*)malloc(4096); - if (!buf) - return nullptr; - buf[0] = '\0'; - - char* sample_params_str = sd_sample_params_to_str(&sd_img_gen_params->sample_params); - - snprintf(buf + strlen(buf), 4096 - strlen(buf), - "prompt: %s\n" - "negative_prompt: %s\n" - "clip_skip: %d\n" - "width: %d\n" - "height: %d\n" - "sample_params: %s\n" - "strength: %.2f\n" - "seed: %" PRId64 - "\n" - "batch_count: %d\n" - "qwen_image_layers: %d\n" - "ref_images_count: %d\n" - "ref_image_args: %s\n" - "control_strength: %.2f\n" - "control_type: %s\n" - "photo maker: {style_strength = %.2f, id_images_count = %d, id_embed_path = %s}\n" - "VAE tiling: %s (temporal=%s, extra_tiling_args=%s)\n" - "circular_x: %s\n" - "circular_y: %s\n" - "hires: {enabled=%s, upscaler=%s, model_path=%s, scale=%.2f, target=%dx%d, steps=%d, denoising_strength=%.2f}\n", - SAFE_STR(sd_img_gen_params->prompt), - SAFE_STR(sd_img_gen_params->negative_prompt), - sd_img_gen_params->clip_skip, - sd_img_gen_params->width, - sd_img_gen_params->height, - SAFE_STR(sample_params_str), - sd_img_gen_params->strength, - sd_img_gen_params->seed, - sd_img_gen_params->batch_count, - sd_img_gen_params->qwen_image_layers, - sd_img_gen_params->ref_images_count, - SAFE_STR(sd_img_gen_params->ref_image_args), - sd_img_gen_params->control_strength, - sd_control_type_name(sd_img_gen_params->control_type), - sd_img_gen_params->pm_params.style_strength, - sd_img_gen_params->pm_params.id_images_count, - SAFE_STR(sd_img_gen_params->pm_params.id_embed_path), - BOOL_STR(sd_img_gen_params->vae_tiling_params.enabled), - BOOL_STR(sd_img_gen_params->vae_tiling_params.temporal_tiling), - SAFE_STR(sd_img_gen_params->vae_tiling_params.extra_tiling_args), - BOOL_STR(sd_img_gen_params->circular_x), - BOOL_STR(sd_img_gen_params->circular_y), - BOOL_STR(sd_img_gen_params->hires.enabled), - sd_hires_upscaler_name(sd_img_gen_params->hires.upscaler), - SAFE_STR(sd_img_gen_params->hires.model_path), - sd_img_gen_params->hires.scale, - sd_img_gen_params->hires.target_width, - sd_img_gen_params->hires.target_height, - sd_img_gen_params->hires.steps, - sd_img_gen_params->hires.denoising_strength); - const char* cache_mode_str = "disabled"; - if (sd_img_gen_params->cache.mode == SD_CACHE_EASYCACHE) { - cache_mode_str = "easycache"; - } else if (sd_img_gen_params->cache.mode == SD_CACHE_UCACHE) { - cache_mode_str = "ucache"; - } - snprintf(buf + strlen(buf), 4096 - strlen(buf), - "cache: %s (threshold=%.3f, start=%.2f, end=%.2f)\n", - cache_mode_str, - get_cache_reuse_threshold(sd_img_gen_params->cache), - sd_img_gen_params->cache.start_percent, - sd_img_gen_params->cache.end_percent); - free(sample_params_str); - return buf; -} - -void sd_vid_gen_params_init(sd_vid_gen_params_t* sd_vid_gen_params) { - *sd_vid_gen_params = {}; - sd_sample_params_init(&sd_vid_gen_params->sample_params); - sd_sample_params_init(&sd_vid_gen_params->high_noise_sample_params); - sd_vid_gen_params->high_noise_sample_params.sample_steps = -1; - sd_vid_gen_params->width = 512; - sd_vid_gen_params->height = 512; - sd_vid_gen_params->strength = 0.75f; - sd_vid_gen_params->seed = -1; - sd_vid_gen_params->video_frames = 6; - sd_vid_gen_params->fps = 16; - sd_vid_gen_params->moe_boundary = 0.875f; - sd_vid_gen_params->vace_strength = 1.f; - sd_vid_gen_params->vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr}; - sd_vid_gen_params->hires.enabled = false; - sd_vid_gen_params->hires.upscaler = SD_HIRES_UPSCALER_LATENT; - sd_vid_gen_params->hires.scale = 2.f; - sd_vid_gen_params->hires.target_width = 0; - sd_vid_gen_params->hires.target_height = 0; - sd_vid_gen_params->hires.steps = 0; - sd_vid_gen_params->hires.denoising_strength = 0.7f; - sd_vid_gen_params->hires.upscale_tile_size = 128; - sd_vid_gen_params->hires.custom_sigmas = nullptr; - sd_vid_gen_params->hires.custom_sigmas_count = 0; - sd_vid_gen_params->circular_x = false; - sd_vid_gen_params->circular_y = false; - sd_vid_gen_params->ref_image_args = ""; - sd_cache_params_init(&sd_vid_gen_params->cache); -} - -struct sd_ctx_t { - StableDiffusionGGML* sd = nullptr; -}; - -void sd_set_generation_progress_callback(sd_ctx_t* ctx, sd_generation_progress_cb_t cb, void* data) { - if (ctx == nullptr || ctx->sd == nullptr) { - return; - } - std::lock_guard lock(ctx->sd->execution_mutex); - if (!ctx->sd->executing_) { - ctx->sd->generation_progress.set_callback(cb, data); - } -} - -sd_ctx_t* new_sd_ctx(const sd_ctx_params_t* sd_ctx_params) { - sd_ctx_t* sd_ctx = (sd_ctx_t*)malloc(sizeof(sd_ctx_t)); - if (sd_ctx == nullptr) { - return nullptr; - } - - sd_ctx->sd = new StableDiffusionGGML(); - if (sd_ctx->sd == nullptr) { - free(sd_ctx); - return nullptr; - } - - if (!sd_ctx->sd->init(sd_ctx_params)) { - delete sd_ctx->sd; - sd_ctx->sd = nullptr; - free(sd_ctx); - return nullptr; - } - return sd_ctx; -} - -void free_sd_ctx(sd_ctx_t* sd_ctx) { - if (sd_ctx->sd != nullptr) { - delete sd_ctx->sd; - sd_ctx->sd = nullptr; - } - free(sd_ctx); -} - -SD_API void sd_cancel_generation(sd_ctx_t* sd_ctx, enum sd_cancel_mode_t mode) { - if (sd_ctx && sd_ctx->sd) { - if (mode < SD_CANCEL_ALL || mode > SD_CANCEL_RESET) { - mode = SD_CANCEL_ALL; - } - sd_ctx->sd->set_cancel_flag(mode); - } -} - -void free_sd_audio(sd_audio_t* audio) { - if (audio == nullptr) { - return; - } - free(audio->data); - audio->data = nullptr; - free(audio); -} - -SD_API bool sd_ctx_supports_image_generation(const sd_ctx_t* sd_ctx) { - if (sd_ctx == nullptr || sd_ctx->sd == nullptr) { - return false; - } - return sd_version_supports_image_generation(sd_ctx->sd->version); -} - -SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx) { - if (sd_ctx == nullptr || sd_ctx->sd == nullptr) { - return false; - } - if (sd_ctx->sd->config_->animatediff_loaded && sd_version_supports_animatediff(sd_ctx->sd->version)) { - return true; - } - return sd_version_supports_video_generation(sd_ctx->sd->version); -} - -SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path) { - if (sd_ctx == nullptr || sd_ctx->sd == nullptr || path == nullptr) { - return false; - } - return sd_ctx->sd->load_control_net_from_file(path); -} - -SD_API bool sd_ctx_unload_control_net(sd_ctx_t* sd_ctx) { - if (sd_ctx == nullptr || sd_ctx->sd == nullptr) { - return false; - } - return sd_ctx->sd->unload_control_net(); -} - -SD_API bool sd_ctx_has_control_net(const sd_ctx_t* sd_ctx) { - if (sd_ctx == nullptr || sd_ctx->sd == nullptr) { - return false; - } - if (sd_ctx->sd->control_net != nullptr) { - return true; - } - // z-image keeps control net weights inside the diffusion runner, registered - // under ModelComponent::ControlNet instead of a separate control_net runner. - return sd_ctx->sd->model_manager != nullptr && - sd_ctx->sd->model_manager->registered_params_size({ModelComponent::ControlNet}) > 0; -} - -const char* sd_get_model_version_name(const sd_ctx_t* sd_ctx) { - if (sd_ctx == nullptr || sd_ctx->sd == nullptr || sd_ctx->sd->version >= VERSION_COUNT) { - return "Unknown"; - } - return sd_version_name(sd_ctx->sd->version); -} - -enum sample_method_t sd_get_default_sample_method(const sd_ctx_t* sd_ctx) { - return sd::pipeline::default_sample_method(sd_ctx != nullptr ? sd_ctx->sd : nullptr); -} - -enum scheduler_t sd_get_default_scheduler(const sd_ctx_t* sd_ctx, enum sample_method_t sample_method) { - return sd::pipeline::default_scheduler(sd_ctx != nullptr ? sd_ctx->sd : nullptr, sample_method); -} - -SD_API bool generate_image(sd_ctx_t* sd_ctx, - const sd_img_gen_params_t* params, - sd_image_generation_result_t** result_out) { +#include +#include +#include +#include + +static sd_status_t validate_generation(StableDiffusionGGML* engine, const sd::CacheConfig& cache, bool video) { + if (video ? !sd_version_supports_video_generation(engine->version) && + !(engine->config_->animatediff_loaded && sd_version_supports_animatediff(engine->version)) + : !sd_version_supports_image_generation(engine->version)) + return sd::api::error(SD_STATUS_UNSUPPORTED, nullptr, "model does not support this operation"); + if (cache.mode == SD_CACHE_UCACHE && !sd_version_is_unet(engine->version)) + return sd::api::error(SD_STATUS_UNSUPPORTED, "cache.mode", "ucache requires a UNet model"); + if ((cache.mode == SD_CACHE_EASYCACHE || cache.mode == SD_CACHE_DBCACHE || + cache.mode == SD_CACHE_TAYLORSEER || cache.mode == SD_CACHE_CACHE_DIT) && + !sd_version_is_dit(engine->version)) + return sd::api::error(SD_STATUS_UNSUPPORTED, "cache.mode", "selected cache requires a DiT model"); + if (cache.mode == SD_CACHE_SPECTRUM && !sd_version_is_unet(engine->version) && !sd_version_is_dit(engine->version)) + return sd::api::error(SD_STATUS_UNSUPPORTED, "cache.mode", "spectrum requires a UNet or DiT model"); + return SD_STATUS_OK; +} + +static sd_status_t generation_failure(StableDiffusionGGML* engine) { + if (engine->get_cancel_flag() == SD_CANCEL_ALL) + return sd::api::error(SD_STATUS_CANCELLED, nullptr, "generation cancelled"); + return sd::api::error(SD_STATUS_GENERATION_FAILED, nullptr, "generation failed; see log for details"); +} + +static sd_status_t run_image(sd_ctx_t* ctx, const sd::ImageConfig* params, sd::ImageGenerationResult** result_out, const sd_opt_t* keys = nullptr) { if (result_out) *result_out = nullptr; - if (!result_out || !sd_ctx || !sd_ctx->sd || !params) - return false; - StableDiffusionGGML::ExecutionScope execution(*sd_ctx->sd); + if (!result_out || !ctx || !ctx->sd || !params) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "context, params and result_out are required"); + StableDiffusionGGML::ExecutionScope execution(*ctx->sd); if (!execution.ready) - return false; - sd_image_t* images = nullptr; - int count = 0; + return execution.operation.acquired + ? sd::api::error(SD_STATUS_LOAD_FAILED, nullptr, "could not refresh model sources") + : sd::api::error(SD_STATUS_BUSY, nullptr, "context is already executing"); + auto status = validate_generation(ctx->sd, params->cache, false); + if (status != SD_STATUS_OK) + return status; + if (sd::api::is_set(keys, "qwen_image_layers") && ctx->sd->version != VERSION_QWEN_IMAGE_LAYERED) + return sd::api::error(SD_STATUS_UNSUPPORTED, "qwen_image_layers", "requires Qwen Image Layered"); + const bool native_control = sd_version_is_control(ctx->sd->version) || ctx->sd->version == VERSION_FLEX_2 || + (sd_version_is_z_image(ctx->sd->version) && ctx->sd->model_manager && + ctx->sd->model_manager->registered_params_size({ModelComponent::ControlNet}) > 0); + if (params->control_image.data && !ctx->sd->control_net && !native_control) + return sd::api::error(SD_STATUS_UNSUPPORTED, "control_image", "model has no active control-image component"); + if (params->ip_adapter_image.data && !ctx->sd->ip_adapter) + return sd::api::error(SD_STATUS_UNSUPPORTED, "ip_adapter_image", "no IP-Adapter is loaded"); + if (params->pm_params.id_images_count > 0 && !ctx->sd->config_->photomaker_source_available) + return sd::api::error(SD_STATUS_UNSUPPORTED, "photomaker.id_images", "no PhotoMaker component is loaded"); + if (params->pulid_params.id_embedding_path && *params->pulid_params.id_embedding_path && + (ctx->sd->version != VERSION_FLUX || !ctx->sd->config_->params.pulid_weights_path || + !*ctx->sd->config_->params.pulid_weights_path)) + return sd::api::error(SD_STATUS_UNSUPPORTED, "pulid.id_embedding_path", "requires a Flux model with PuLID weights"); + if (params->sample_params.guidance.img_cfg != INFINITY && params->sample_params.guidance.img_cfg != 1.f && + !sd_version_is_inpaint_or_unet_edit(ctx->sd->version) && + !(params->ref_images_count > 0 && sd_version_supports_ref_latent_img_cfg(ctx->sd->version))) + return sd::api::error(SD_STATUS_UNSUPPORTED, "sample.guidance.img_cfg", "model does not support image CFG for these inputs"); + sd::Image* images = nullptr; + int count = 0; std::vector infos; try { - if (!sd::pipeline::generate_image(sd_ctx->sd, params, &images, &count, &infos)) { - free_sd_images(images, count); - return false; + if (!sd::pipeline::generate_image(ctx->sd, params, &images, &count, &infos)) { + sd::free_images(images, count); + return generation_failure(ctx->sd); } - } catch (const std::exception& e) { - LOG_ERROR("image generation failed: %s", e.what()); - free_sd_images(images, count); - return false; + } catch (...) { + sd::free_images(images, count); + throw; } - return sd::make_image_generation_result(images, count, infos, result_out); + return sd::make_image_generation_result(images, count, infos, result_out) ? SD_STATUS_OK + : sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate image result"); } -SD_API bool generate_video(sd_ctx_t* sd_ctx, - const sd_vid_gen_params_t* params, - sd_video_generation_result_t** result_out) { +static sd_status_t run_video(sd_ctx_t* ctx, const sd::VideoConfig* params, sd::VideoGenerationResult** result_out, const sd_opt_t* keys = nullptr) { if (result_out) *result_out = nullptr; - if (!result_out || !sd_ctx || !sd_ctx->sd || !params) - return false; - StableDiffusionGGML::ExecutionScope execution(*sd_ctx->sd); + if (!result_out || !ctx || !ctx->sd || !params) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "context, params and result_out are required"); + StableDiffusionGGML::ExecutionScope execution(*ctx->sd); if (!execution.ready) - return false; - auto* result = static_cast(calloc(1, sizeof(sd_video_generation_result_t))); + return execution.operation.acquired + ? sd::api::error(SD_STATUS_LOAD_FAILED, nullptr, "could not refresh model sources") + : sd::api::error(SD_STATUS_BUSY, nullptr, "context is already executing"); + auto status = validate_generation(ctx->sd, params->cache, true); + if (status != SD_STATUS_OK) + return status; + const bool vace = ctx->sd->diffusion_model && + (ctx->sd->diffusion_model->get_desc() == "Wan2.1-VACE-1.3B" || + ctx->sd->diffusion_model->get_desc() == "Wan2.x-VACE-14B"); + if (params->control_frames_size > 0 && !vace) + return sd::api::error(SD_STATUS_UNSUPPORTED, "control_frames", "requires a VACE model"); + if (params->ref_videos_count > 0 && !sd_version_is_minimax_h3(ctx->sd->version)) + return sd::api::error(SD_STATUS_UNSUPPORTED, "ref_videos", "requires a MiniMax-H3 model"); + if (params->ref_audios_count > 0 && !sd_version_is_minimax_h3(ctx->sd->version) && ctx->sd->version != VERSION_WAN2_2_S2V) + return sd::api::error(SD_STATUS_UNSUPPORTED, "ref_audios", "requires MiniMax-H3 or Wan2.2 S2V"); + if (ctx->sd->version == VERSION_WAN2_2_S2V) { + if (params->end_image.data) + return sd::api::error(SD_STATUS_UNSUPPORTED, "end_image", "Wan2.2 S2V does not support an end image"); + if (params->ref_audios_count > 1 || (params->ref_audios_count && !ctx->sd->audio_encoder)) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, "ref_audios", "Wan2.2 S2V requires one driving track and an audio encoder"); + } + auto result = std::unique_ptr( + static_cast(calloc(1, sizeof(sd::VideoGenerationResult))), sd::free_video_generation_result); if (!result) - return false; - try { - nlohmann::json info; - if (!sd::pipeline::generate_video(sd_ctx->sd, params, &result->frames, &result->frame_count, - &result->audio, &result->fps, &info)) { - free_sd_video_generation_result(result); - return false; - } - result->generation_info = sd::copy_generation_info(info); - if (!result->generation_info) { - free_sd_video_generation_result(result); - return false; - } - } catch (const std::exception& e) { - LOG_ERROR("video generation failed: %s", e.what()); - free_sd_video_generation_result(result); - return false; - } - *result_out = result; - return true; + return sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate video result"); + nlohmann::json info; + if (!sd::pipeline::generate_video(ctx->sd, params, &result->frames, &result->frame_count, + &result->audio, &result->fps, &info)) + return generation_failure(ctx->sd); + result->generation_info = sd::copy_generation_info(info); + if (!result->generation_info) + return sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate video metadata"); + *result_out = result.release(); + return SD_STATUS_OK; +} + +sd_status_t sd::generate_image(sd_ctx_t* ctx, const ImageConfig& config, ImageGenerationResult** result_out, const sd_opt_t* source) noexcept { + if (result_out) + *result_out = nullptr; + return sd::api::call([&] { return run_image(ctx, &config, result_out, source); }, SD_STATUS_GENERATION_FAILED); } -SD_API void free_sd_images(sd_image_t* result_images, int num_images) { - if (result_images == nullptr) { - return; - } - - for (int i = 0; i < num_images; ++i) { - if (result_images[i].data != nullptr) { - free(result_images[i].data); - result_images[i].data = nullptr; - } - } - - free(result_images); +sd_status_t sd::generate_video(sd_ctx_t* ctx, const VideoConfig& config, VideoGenerationResult** result_out, const sd_opt_t* source) noexcept { + if (result_out) + *result_out = nullptr; + return sd::api::call([&] { return run_video(ctx, &config, result_out, source); }, SD_STATUS_GENERATION_FAILED); } diff --git a/src/upscaler.cpp b/src/upscaler.cpp index 82aa1f464d..9a16c0adef 100644 --- a/src/upscaler.cpp +++ b/src/upscaler.cpp @@ -1,4 +1,6 @@ #include "upscaler.h" +#include "api/options.h" +#include "api/result.h" #include "core/ggml_extend_backend.h" #include "core/util.h" #include "generation_info.h" @@ -60,13 +62,12 @@ sd::Tensor UpscalerGGML::upscale_tensor(const sd::Tensor& input_te return sd::pipeline::upscale_image_tensor(*model, input_tensor, n_threads, tile_size); } -sd_image_t UpscalerGGML::upscale(sd_image_t input_image, uint32_t upscale_factor) { +sd::Image UpscalerGGML::upscale(sd::Image input_image) { sd::ParallelScope tensor_scope(&tensor_executor); - // upscale_factor, unused for RealESRGAN_x4plus_anime_6B.pth - sd_image_t upscaled_image = {0, 0, 0, nullptr}; - const int scale = get_scale(); - int output_width = (int)input_image.width * scale; - int output_height = (int)input_image.height * scale; + sd::Image upscaled_image = {0, 0, 0, nullptr}; + const int scale = get_scale(); + int output_width = (int)input_image.width * scale; + int output_height = (int)input_image.height * scale; LOG_INFO("upscaling from (%i x %i) to (%i x %i)", input_image.width, input_image.height, output_width, output_height); @@ -77,81 +78,83 @@ sd_image_t UpscalerGGML::upscale(sd_image_t input_image, uint32_t upscale_factor if (upscaled.empty()) { return upscaled_image; } - sd_image_t upscaled_data = tensor_to_sd_image(upscaled); - int64_t t3 = ggml_time_ms(); + sd::Image upscaled_data = tensor_to_sd_image(upscaled); + int64_t t3 = ggml_time_ms(); LOG_INFO("input_image_tensor upscaled, taking %.2fs", (t3 - t0) / 1000.0f); upscaled_image = upscaled_data; return upscaled_image; } -struct upscaler_ctx_t { - UpscalerGGML* upscaler = nullptr; +struct sd_upscaler { + std::unique_ptr upscaler; }; -upscaler_ctx_t* new_upscaler_ctx(const char* esrgan_path_c_str, - bool direct, - int n_threads, - int tile_size, - const char* backend, - const char* params_backend) { - upscaler_ctx_t* upscaler_ctx = (upscaler_ctx_t*)malloc(sizeof(upscaler_ctx_t)); - if (upscaler_ctx == nullptr) { - return nullptr; - } - std::string esrgan_path(esrgan_path_c_str); - - upscaler_ctx->upscaler = new UpscalerGGML(n_threads, direct, tile_size, SAFE_STR(backend), SAFE_STR(params_backend)); - if (upscaler_ctx->upscaler == nullptr) { - return nullptr; - } - - if (!upscaler_ctx->upscaler->load_from_file(esrgan_path, n_threads)) { - delete upscaler_ctx->upscaler; - upscaler_ctx->upscaler = nullptr; - free(upscaler_ctx); - return nullptr; - } - return upscaler_ctx; +sd_status_t sd_upscaler_create(const sd_upscaler_opt_t* opt, sd_upscaler_t** out) { + if (out) + *out = nullptr; + return sd::api::call([&] { + if (!out) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required"); + sd::OptConfig config; + auto status = sd::api::resolve(opt, sd::api::Upscaler, config); + if (status != SD_STATUS_OK) + return status; + auto result = std::make_unique(); + const auto& ctx = config.context; + result->upscaler = std::make_unique(ctx.n_threads, config.post.conv_direct, config.post.tile_size, + SAFE_STR(ctx.backend), SAFE_STR(ctx.params_backend)); + if (!result->upscaler->load_from_file(ctx.model_path, ctx.n_threads)) + return sd::api::error(SD_STATUS_LOAD_FAILED, "model_path", "could not load upscaler"); + *out = result.release(); + return SD_STATUS_OK; + }, + SD_STATUS_LOAD_FAILED); } -bool upscale(upscaler_ctx_t* upscaler_ctx, - sd_image_t input_image, - uint32_t upscale_factor, - sd_image_generation_result_t** result_out) try { - if (result_out) - *result_out = nullptr; - if (!result_out) - return false; - if (upscaler_ctx == nullptr || upscaler_ctx->upscaler == nullptr) { - return false; - } - - sd_image_t* result_images = (sd_image_t*)calloc(1, sizeof(sd_image_t)); - if (result_images == nullptr) { - return false; - } - - auto release_images = [](sd_image_t* images) { free_sd_images(images, 1); }; - std::unique_ptr image_owner(result_images, release_images); - const std::vector infos{sd::upscale_generation_info(upscale_factor)}; - result_images[0] = upscaler_ctx->upscaler->upscale(input_image, upscale_factor); - if (!result_images[0].data) - return false; +void sd_upscaler_free(sd_upscaler_t* upscaler) { + delete upscaler; +} - return sd::make_image_generation_result(image_owner.release(), 1, infos, result_out); -} catch (const std::exception& e) { - LOG_ERROR("upscale failed: %s", e.what()); - return false; +sd_status_t sd_upscale(sd_upscaler_t* upscaler, const sd_upscale_opt_t* opt, sd_result_t** out) { + if (out) + *out = nullptr; + return sd::api::call([&] { + if (!out) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required"); + sd::OptConfig config; + auto status = sd::api::resolve(opt, sd::api::Upscale, config); + if (status != SD_STATUS_OK) + return status; + if (!upscaler || !upscaler->upscaler) + return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "upscaler is required"); + const auto& image = config.post.image; + const auto scale = upscaler->upscaler->get_scale(); + if (scale <= 0 || image.width > uint32_t(INT32_MAX / scale) || image.height > uint32_t(INT32_MAX / scale)) + return sd::api::error(SD_STATUS_OUT_OF_RANGE, "image", "upscaled dimensions exceed the supported range"); + auto release = [](sd::Image* images) { sd::free_images(images, 1); }; + std::unique_ptr images(static_cast(calloc(1, sizeof(sd::Image))), release); + if (!images) + return sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate result"); + images.get()[0] = upscaler->upscaler->upscale(image); + if (!images->data) + return sd::api::error(SD_STATUS_GENERATION_FAILED, nullptr, "upscale failed"); + const std::vector infos{sd::upscale_generation_info(scale)}; + return sd::make_image_result(images.release(), 1, infos, out) ? SD_STATUS_OK + : sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate result metadata"); + }, + SD_STATUS_GENERATION_FAILED); } -int get_upscale_factor(upscaler_ctx_t* upscaler_ctx) { - if (upscaler_ctx == nullptr || upscaler_ctx->upscaler == nullptr) { - return 1; +int sd_upscaler_get_scale(const sd_upscaler_t* upscaler) { + sd::api::clear_error(); + if (!upscaler || !upscaler->upscaler) { + sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "upscaler is required"); + return 0; } - return upscaler_ctx->upscaler->get_scale(); + return upscaler->upscaler->get_scale(); } -int get_upscaler_model_scale(const char* model_path) { +int sd_upscaler_probe_scale(const char* model_path) { if (model_path == nullptr || model_path[0] == '\0') { return 0; } @@ -178,13 +181,3 @@ int get_upscaler_model_scale(const char* model_path) { return 0; } } - -void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx) { - if (!upscaler_ctx) - return; - if (upscaler_ctx->upscaler != nullptr) { - delete upscaler_ctx->upscaler; - upscaler_ctx->upscaler = nullptr; - } - free(upscaler_ctx); -} diff --git a/src/upscaler.h b/src/upscaler.h index f3d09384e6..e3cf09fd2d 100644 --- a/src/upscaler.h +++ b/src/upscaler.h @@ -1,6 +1,8 @@ #ifndef __SD_UPSCALER_H__ #define __SD_UPSCALER_H__ +#include "core/media.h" + #include "core/ggml_extend_backend.h" #include "core/tensor.hpp" #include "model_manager.h" @@ -35,7 +37,7 @@ struct UpscalerGGML { void set_max_graph_vram_bytes(size_t max_vram_bytes); int get_scale(); sd::Tensor upscale_tensor(const sd::Tensor& input_tensor); - sd_image_t upscale(sd_image_t input_image, uint32_t upscale_factor); + sd::Image upscale(sd::Image input_image); }; #endif // __SD_UPSCALER_H__