feat: add IP-Adapter support for SD 1.5 and SDXL

README.md

@@ -68,6 +68,7 @@ API and command-line option may change frequently.***
     - [HunyuanVideo 1.5](./docs/hunyuan_video.md)
     - [LingBot-Video](./docs/lingbot_video.md)
   - [PhotoMaker](./docs/photo_maker.md) support.
+  - [IP-Adapter](./docs/ip_adapter.md) support (SD 1.5 and SDXL)
   - Control Net support with SD 1.5
   - [ADetailer](./docs/adetailer.md)
   - LoRA support, same as [stable-diffusion-webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Features#lora)

docs/ip_adapter.md

@@ -0,0 +1,55 @@
+# IP-Adapter
+
+stable-diffusion.cpp supports [IP-Adapter](https://github.com/tencent-ailab/IP-Adapter)
+image-prompt conditioning for SD 1.5 and SDXL. Given a reference image,
+IP-Adapter transfers the subject and appearance of that image into the
+generation, alongside the text prompt.
+
+IP-Adapter encodes the reference image with a CLIP-Vision (ViT-H/14)
+encoder, projects the embedding into a few image tokens, and injects them
+through a decoupled cross-attention added to every attn2 layer of the
+UNet. It composes with Control Net, so a reference image (appearance) and
+an OpenPose hint (pose) can be combined in a single generation.
+
+## Required weights
+
+1. A base SD 1.5 or SDXL model.
+2. A CLIP-Vision (ViT-H/14) image encoder, passed with `--clip_vision`
+   (for example `clip_vision_h.safetensors`).
+3. An IP-Adapter weight file, passed with `--ip-adapter`. The `vit-h`
+   variants reuse the same ViT-H encoder as above. From
+   [h94/IP-Adapter](https://huggingface.co/h94/IP-Adapter):
+   - SD 1.5: `models/ip-adapter_sd15.safetensors`
+   - SDXL: `sdxl_models/ip-adapter_sdxl_vit-h.safetensors`
+
+## Options
+
+- `--ip-adapter <path>` path to the IP-Adapter weight file.
+- `--ip-adapter-image <path>` path to the reference image.
+- `--ip-adapter-strength <float>` strength of the IP-Adapter injection
+  (default 1.0). Lower values let the text prompt dominate; 0.6 to 0.8 is
+  a good starting range.
+
+## Example (SD 1.5)
+
+```
+sd-cli -m ..\models\sd_v1.5.safetensors --clip_vision ..\models\clip_vision_h.safetensors --ip-adapter ..\models\ip-adapter_sd15.safetensors --ip-adapter-image ..\assets\reference.png --ip-adapter-strength 0.8 -p "a woman, best quality" -n "lowres, bad anatomy" --cfg-scale 7 --steps 30 --sampling-method dpm++2m --scheduler karras -W 512 -H 512
+```
+
+## Example (SDXL)
+
+```
+sd-cli -m ..\models\sdxl.safetensors --clip_vision ..\models\clip_vision_h.safetensors --ip-adapter ..\models\ip-adapter_sdxl_vit-h.safetensors --ip-adapter-image ..\assets\reference.png --ip-adapter-strength 0.8 -p "a woman, best quality" -n "lowres, bad anatomy" --cfg-scale 6 --steps 25 --sampling-method dpm++2m --scheduler karras -W 1024 -H 1024 --diffusion-fa --vae-tiling
+```
+
+The SDXL VAE decode at 1024x1024 is memory heavy; add `--vae-tiling` (and
+`--offload-to-cpu`) on GPUs with limited VRAM.
+
+## Combining with Control Net
+
+Add the usual Control Net options to keep the reference appearance while
+controlling the pose:
+
+```
+sd-cli -m ..\models\sdxl.safetensors --clip_vision ..\models\clip_vision_h.safetensors --ip-adapter ..\models\ip-adapter_sdxl_vit-h.safetensors --ip-adapter-image ..\assets\character.png --ip-adapter-strength 0.9 --control-net ..\models\OpenPoseXL2.safetensors --control-image ..\assets\pose.png --control-strength 0.8 -p "a character, side view" --cfg-scale 6 --steps 25 -W 1024 -H 1024 --diffusion-fa --vae-tiling
+```

examples/cli/main.cpp

@@ -817,6 +817,16 @@ int main(int argc, const char* argv[]) {
         }
     }
 
+    if (gen_params.ip_adapter_image_path.size() > 0) {
+        if (!load_sd_image_from_file(gen_params.ip_adapter_image.put(),
+                                     gen_params.ip_adapter_image_path.c_str(),
+                                     0,
+                                     0)) {
+            LOG_ERROR("load image from '%s' failed", gen_params.ip_adapter_image_path.c_str());
+            return 1;
+        }
+    }
+
     if (!gen_params.control_video_path.empty()) {
         gen_params.control_frames.clear();
         if (!load_images_from_dir(gen_params.control_video_path,

examples/common/common.cpp

@@ -427,6 +427,11 @@ ArgOptions SDContextParams::get_options() {
          "path to control net model",
          0,
          &control_net_path},
+        {"",
+         "--ip-adapter",
+         "path to IP-Adapter model (requires --clip_vision)",
+         0,
+         &ip_adapter_path},
         {"",
          "--motion-module",
          "path to AnimateDiff motion module (SD 1.5); enables video generation on --video-frames > 1",
@@ -876,6 +881,7 @@ sd_ctx_params_t SDContextParams::to_sd_ctx_params_t(bool taesd_preview) {
     sd_ctx_params.audio_vae_path                  = audio_vae_path.c_str();
     sd_ctx_params.taesd_path                      = taesd_path.c_str();
     sd_ctx_params.control_net_path                = control_net_path.c_str();
+    sd_ctx_params.ip_adapter_path                 = ip_adapter_path.c_str();
     sd_ctx_params.motion_module_path              = motion_module_path.c_str();
     sd_ctx_params.embeddings                      = embedding_vec.data();
     sd_ctx_params.embedding_count                 = static_cast<uint32_t>(embedding_vec.size());
@@ -966,6 +972,11 @@ ArgOptions SDGenerationParams::get_options() {
          "path to control image, control net",
          0,
          &control_image_path},
+        {"",
+         "--ip-adapter-image",
+         "path to the IP-Adapter reference image",
+         0,
+         &ip_adapter_image_path},
         {"",
          "--control-video",
          "path to control video frames, It must be a directory path. The video frames inside should be stored as images in "
@@ -1158,6 +1169,10 @@ ArgOptions SDGenerationParams::get_options() {
          "--control-strength",
          "strength to apply Control Net (default: 0.9). 1.0 corresponds to full destruction of information in init image",
          &control_strength},
+        {"",
+         "--ip-adapter-strength",
+         "strength to apply IP-Adapter (default: 1.0)",
+         &ip_adapter_strength},
         {"",
          "--moe-boundary",
          "timestep boundary for Wan2.2 MoE model. (default: 0.875). Only enabled if `--high-noise-steps` is set to -1",
@@ -2498,6 +2513,8 @@ sd_img_gen_params_t SDGenerationParams::to_sd_img_gen_params_t() {
     params.qwen_image_layers = qwen_image_layers;
     params.control_image     = control_image.get();
     params.control_strength  = control_strength;
+    params.ip_adapter_image  = ip_adapter_image.get();
+    params.ip_adapter_strength = ip_adapter_strength;
     params.pm_params         = pm_params;
     params.pulid_params      = pulid_params;
     params.vae_tiling_params = vae_tiling_params;

examples/common/common.h

@@ -133,6 +133,7 @@ struct SDContextParams {
     std::string taesd_path;
     std::string esrgan_path;
     std::string control_net_path;
+    std::string ip_adapter_path;
     std::string motion_module_path;
     std::string embedding_dir;
     std::string photo_maker_path;
@@ -200,6 +201,7 @@ struct SDGenerationParams {
     int64_t seed               = 42;
     float strength             = 0.75f;
     float control_strength     = 0.9f;
+    float ip_adapter_strength  = 1.0f;
     bool auto_resize_ref_image = true;
     bool increase_ref_index    = false;
     bool embed_image_metadata  = true;
@@ -208,6 +210,7 @@ struct SDGenerationParams {
     std::string end_image_path;
     std::string mask_image_path;
     std::string control_image_path;
+    std::string ip_adapter_image_path;
     std::vector<std::string> ref_image_paths;
     std::string control_video_path;
 
@@ -274,6 +277,7 @@ struct SDGenerationParams {
     std::vector<SDImageOwner> ref_images;
     SDImageOwner mask_image;
     SDImageOwner control_image;
+    SDImageOwner ip_adapter_image;
     std::vector<SDImageOwner> pm_id_images;
     std::vector<SDImageOwner> control_frames;
 

include/stable-diffusion.h

@@ -200,6 +200,7 @@ typedef struct {
     const char* audio_vae_path;
     const char* taesd_path;
     const char* control_net_path;
+    const char* ip_adapter_path;
     const char* motion_module_path;
     const sd_embedding_t* embeddings;
     uint32_t embedding_count;
@@ -375,6 +376,8 @@ typedef struct {
     int batch_count;
     sd_image_t control_image;
     float control_strength;
+    sd_image_t ip_adapter_image;
+    float ip_adapter_strength;
     sd_pm_params_t pm_params;
     sd_pulid_params_t pulid_params;
     sd_tiling_params_t vae_tiling_params;

src/core/ggml_extend.hpp

@@ -1689,6 +1689,8 @@ struct GGMLRunnerContext {
     bool conv2d_direct_enabled                                       = false;
     bool circular_x_enabled                                          = false;
     bool circular_y_enabled                                          = false;
+    ggml_tensor* ip_context                                          = nullptr;
+    float ip_scale                                                   = 1.0f;
     std::shared_ptr<WeightAdapter> weight_adapter                    = nullptr;
     std::vector<std::pair<ggml_tensor*, std::string>>* debug_tensors = nullptr;
     std::function<ggml_tensor*(const std::string&)> get_cache_tensor;

src/model/adapter/ip_adapter.hpp

@@ -0,0 +1,88 @@
+#ifndef __SD_MODEL_ADAPTER_IP_ADAPTER_HPP__
+#define __SD_MODEL_ADAPTER_IP_ADAPTER_HPP__
+
+#include "core/ggml_extend.hpp"
+#include "model/common/block.hpp"
+#include "model_loader.h"
+
+namespace IPAdapter {
+
+    struct ImageProjModel : public GGMLBlock {
+        int64_t num_tokens = 4;
+        int64_t ctx_dim    = 768;
+        int64_t clip_dim   = 1024;
+
+        ImageProjModel() {}
+        ImageProjModel(int64_t num_tokens, int64_t ctx_dim, int64_t clip_dim)
+            : num_tokens(num_tokens), ctx_dim(ctx_dim), clip_dim(clip_dim) {
+            blocks["proj"] = std::shared_ptr<GGMLBlock>(new Linear(clip_dim, num_tokens * ctx_dim, true));
+            blocks["norm"] = std::shared_ptr<GGMLBlock>(new LayerNorm(ctx_dim));
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* image_embeds) {
+            auto proj = std::dynamic_pointer_cast<Linear>(blocks["proj"]);
+            auto norm = std::dynamic_pointer_cast<LayerNorm>(blocks["norm"]);
+
+            int64_t n = image_embeds->ne[1];
+            auto x    = proj->forward(ctx, image_embeds);
+            x         = ggml_reshape_3d(ctx->ggml_ctx, x, ctx_dim, num_tokens, n);
+            x         = norm->forward(ctx, x);
+            return x;
+        }
+    };
+
+    struct IPAdapterRunner : public GGMLRunner {
+        ImageProjModel image_proj;
+        int64_t num_tokens = 4;
+        std::string prefix;
+
+        IPAdapterRunner(ggml_backend_t backend,
+                        const String2TensorStorage& tensor_storage_map,
+                        const std::string prefix,
+                        std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+            : GGMLRunner(backend, weight_manager), prefix(prefix) {
+            int64_t ctx_dim  = 768;
+            int64_t clip_dim = 1024;
+            int64_t out_dim  = 3072;
+            auto norm_iter   = tensor_storage_map.find(prefix + ".image_proj.norm.weight");
+            if (norm_iter != tensor_storage_map.end()) {
+                ctx_dim = norm_iter->second.ne[0];
+            }
+            auto proj_iter = tensor_storage_map.find(prefix + ".image_proj.proj.weight");
+            if (proj_iter != tensor_storage_map.end()) {
+                clip_dim = proj_iter->second.ne[0];
+                out_dim  = proj_iter->second.ne[1];
+            }
+            num_tokens = out_dim / ctx_dim;
+            image_proj = ImageProjModel(num_tokens, ctx_dim, clip_dim);
+            image_proj.init(params_ctx, tensor_storage_map, prefix + ".image_proj");
+        }
+
+        std::string get_desc() override {
+            return "ip_adapter";
+        }
+
+        void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string = "") {
+            image_proj.get_param_tensors(tensors, prefix + ".image_proj");
+        }
+
+        ggml_cgraph* build_graph(const sd::Tensor<float>& image_embeds_tensor) {
+            ggml_cgraph* gf     = new_graph_custom(1024);
+            ggml_tensor* embeds = make_input(image_embeds_tensor);
+            auto runner_ctx     = get_context();
+            ggml_tensor* out    = image_proj.forward(&runner_ctx, embeds);
+            ggml_build_forward_expand(gf, out);
+            return gf;
+        }
+
+        sd::Tensor<float> compute(int n_threads, const sd::Tensor<float>& image_embeds) {
+            auto get_graph = [&]() -> ggml_cgraph* {
+                return build_graph(image_embeds);
+            };
+            return take_or_empty(GGMLRunner::compute<float>(get_graph, n_threads, true, true, true));
+        }
+    };
+
+}  // namespace IPAdapter
+
+#endif  // __SD_MODEL_ADAPTER_IP_ADAPTER_HPP__

src/model/common/block.hpp

@@ -310,17 +310,33 @@ class CrossAttention : public GGMLBlock {
     int64_t context_dim;
     int64_t n_head;
     int64_t d_head;
-    bool xtra_dim = false;
+    bool xtra_dim  = false;
+    bool enable_ip = false;
+    bool has_ip    = false;
+
+    void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
+        GGMLBlock::init_params(ctx, tensor_storage_map, prefix);
+        if (enable_ip &&
+            tensor_storage_map.find(prefix + "to_k_ip.weight") != tensor_storage_map.end()) {
+            has_ip            = true;
+            int64_t inner_dim = d_head * n_head;
+            int64_t ip_dim    = tensor_storage_map.at(prefix + "to_k_ip.weight").ne[0];
+            blocks["to_k_ip"] = std::shared_ptr<GGMLBlock>(new Linear(ip_dim, inner_dim, false));
+            blocks["to_v_ip"] = std::shared_ptr<GGMLBlock>(new Linear(ip_dim, inner_dim, false));
+        }
+    }
 
 public:
     CrossAttention(int64_t query_dim,
                    int64_t context_dim,
                    int64_t n_head,
-                   int64_t d_head)
+                   int64_t d_head,
+                   bool enable_ip = false)
         : n_head(n_head),
           d_head(d_head),
           query_dim(query_dim),
-          context_dim(context_dim) {
+          context_dim(context_dim),
+          enable_ip(enable_ip) {
         int64_t inner_dim = d_head * n_head;
         if (context_dim == 320 && d_head == 320) {
             // LOG_DEBUG("CrossAttention: temp set dim to 1024 for sdxs_09");
@@ -363,6 +379,15 @@ class CrossAttention : public GGMLBlock {
         }
         x = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, n_head, nullptr, false, ctx->flash_attn_enabled);  // [N, n_token, inner_dim]
 
+        if (has_ip && ctx->ip_context != nullptr && ctx->ip_scale != 0.0f) {
+            auto to_k_ip = std::dynamic_pointer_cast<Linear>(blocks["to_k_ip"]);
+            auto to_v_ip = std::dynamic_pointer_cast<Linear>(blocks["to_v_ip"]);
+            auto k_ip    = to_k_ip->forward(ctx, ctx->ip_context);
+            auto v_ip    = to_v_ip->forward(ctx, ctx->ip_context);
+            auto x_ip    = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k_ip, v_ip, n_head, nullptr, false, ctx->flash_attn_enabled);
+            x            = ggml_add(ctx->ggml_ctx, x, ggml_scale(ctx->ggml_ctx, x_ip, ctx->ip_scale));
+        }
+
         x = to_out_0->forward(ctx, x);  // [N, n_token, query_dim]
         return x;
     }
@@ -387,7 +412,7 @@ class BasicTransformerBlock : public GGMLBlock {
         // inner_dim is always None or equal to dim
         // gated_ff is always True
         blocks["attn1"] = std::shared_ptr<GGMLBlock>(new CrossAttention(dim, dim, n_head, d_head));
-        blocks["attn2"] = std::shared_ptr<GGMLBlock>(new CrossAttention(dim, context_dim, n_head, d_head));
+        blocks["attn2"] = std::shared_ptr<GGMLBlock>(new CrossAttention(dim, context_dim, n_head, d_head, true));
         blocks["ff"]    = std::shared_ptr<GGMLBlock>(new FeedForward(dim, dim));
         blocks["norm1"] = std::shared_ptr<GGMLBlock>(new LayerNorm(dim));
         blocks["norm2"] = std::shared_ptr<GGMLBlock>(new LayerNorm(dim));

src/model/diffusion/model.hpp

@@ -44,6 +44,8 @@ struct UNetDiffusionExtra {
     int num_video_frames                           = -1;
     const std::vector<sd::Tensor<float>>* controls = nullptr;
     float control_strength                         = 0.f;
+    const sd::Tensor<float>* ip_context            = nullptr;
+    float ip_scale                                 = 1.f;
 };
 
 struct SkipLayerDiffusionExtra {