Repository: huggingface/diffusers

Issue:
[IP-Adapter] support face model
We recently added IP-adapter support to many of our pipelines in diffusers! You can now very easily load your IP-Adapter into a diffusers pipeline with `pipe.load_ip_adapter(...)`; 

However right now we do not support [`IPAdapterFull`](https://github.com/tencent-ailab/IP-Adapter/blob/e317f750e1829e846c944b9fdd7a2f2842c46501/ip_adapter/ip_adapter.py#L314) model checkpoints. Anyone interested to add that support for this? It will allow us to use the very cool face models with diffusers! 

the goal would be to make [this example](https://github.com/tencent-ailab/IP-Adapter/blob/main/ip_adapter-full-face_demo.ipynb) work with diffusers API. e.g.
```python
from diffusers import StableDiffusionPipeline
import torch
from diffusers.utils import load_image

base_model_path = "dreamlike-art/dreamlike-anime-1.0/"
pipeline = StableDiffusionPipeline.from_pretrained(
    base_model_path, torch_dtype=torch.float16)
pipeline.to("cuda")

image = load_image("https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/ai_face2.png")

pipeline.load_ip_adapter("h94/IP-Adapter", subfolder="models", weight_name="ip-adapter-full-face_sd15.bin")
pipeline.set_ip_adapter_scale(0.6)
generator = torch.Generator(device="cpu").manual_seed(33)
images = pipeline(
    prompt="A photo of a girl wearing a black dress, holding red roses in hand, upper body, behind is the Eiffel Tower",
    ip_adapter_image=image,
    negative_prompt="monochrome, lowres, bad anatomy, worst quality, low quality", 
    num_inference_steps=50,
    generator=generator,
).images
images[0]
```

 You will need to look into [this PR](https://github.com/huggingface/diffusers/pull/5713/files#diff-272077719e5ba4349d4f0ee80bdc959ef954934af1df782deac4d297e359c08a) to understand how `load_ip_adapter` works and then make it work for [`IPAdapterFull`](https://github.com/tencent-ailab/IP-Adapter/blob/e317f750e1829e846c944b9fdd7a2f2842c46501/ip_adapter/ip_adapter.py#L314) model from the original repo.





Discussion:
@yiyixuxu Can I work on this?
@hackpk sure!
@yiyixuxu Can i get started on this?
@yiyixuxu @hackpk do you mind if I try it too?

Resolving patch:
diff --git a/src/diffusers/loaders/unet.py b/src/diffusers/loaders/unet.py
--- a/src/diffusers/loaders/unet.py
+++ b/src/diffusers/loaders/unet.py
@@ -21,7 +21,7 @@
 import torch.nn.functional as F
 from torch import nn
 
-from ..models.embeddings import ImageProjection, Resampler
+from ..models.embeddings import ImageProjection, MLPProjection, Resampler
 from ..models.modeling_utils import _LOW_CPU_MEM_USAGE_DEFAULT, load_model_dict_into_meta
 from ..utils import (
     DIFFUSERS_CACHE,
@@ -675,6 +675,9 @@ def _load_ip_adapter_weights(self, state_dict):
         if "proj.weight" in state_dict["image_proj"]:
             # IP-Adapter
             num_image_text_embeds = 4
+        elif "proj.3.weight" in state_dict["image_proj"]:
+            # IP-Adapter Full Face
+            num_image_text_embeds = 257  # 256 CLIP tokens + 1 CLS token
         else:
             # IP-Adapter Plus
             num_image_text_embeds = state_dict["image_proj"]["latents"].shape[1]
@@ -744,8 +747,32 @@ def _load_ip_adapter_weights(self, state_dict):
                     "norm.bias": state_dict["image_proj"]["norm.bias"],
                 }
             )
+            image_projection.load_state_dict(image_proj_state_dict)
+            del image_proj_state_dict
 
+        elif "proj.3.weight" in state_dict["image_proj"]:
+            clip_embeddings_dim = state_dict["image_proj"]["proj.0.weight"].shape[0]
+            cross_attention_dim = state_dict["image_proj"]["proj.3.weight"].shape[0]
+
+            image_projection = MLPProjection(
+                cross_attention_dim=cross_attention_dim, image_embed_dim=clip_embeddings_dim
+            )
+            image_projection.to(dtype=self.dtype, device=self.device)
+
+            # load image projection layer weights
+            image_proj_state_dict = {}
+            image_proj_state_dict.update(
+                {
+                    "ff.net.0.proj.weight": state_dict["image_proj"]["proj.0.weight"],
+                    "ff.net.0.proj.bias": state_dict["image_proj"]["proj.0.bias"],
+                    "ff.net.2.weight": state_dict["image_proj"]["proj.2.weight"],
+                    "ff.net.2.bias": state_dict["image_proj"]["proj.2.bias"],
+                    "norm.weight": state_dict["image_proj"]["proj.3.weight"],
+                    "norm.bias": state_dict["image_proj"]["proj.3.bias"],
+                }
+            )
             image_projection.load_state_dict(image_proj_state_dict)
+            del image_proj_state_dict
 
         else:
             # IP-Adapter Plus
diff --git a/src/diffusers/models/__init__.py b/src/diffusers/models/__init__.py
--- a/src/diffusers/models/__init__.py
+++ b/src/diffusers/models/__init__.py
@@ -33,8 +33,8 @@
     _import_structure["consistency_decoder_vae"] = ["ConsistencyDecoderVAE"]
     _import_structure["controlnet"] = ["ControlNetModel"]
     _import_structure["dual_transformer_2d"] = ["DualTransformer2DModel"]
-    _import_structure["modeling_utils"] = ["ModelMixin"]
     _import_structure["embeddings"] = ["ImageProjection"]
+    _import_structure["modeling_utils"] = ["ModelMixin"]
     _import_structure["prior_transformer"] = ["PriorTransformer"]
     _import_structure["t5_film_transformer"] = ["T5FilmDecoder"]
     _import_structure["transformer_2d"] = ["Transformer2DModel"]
diff --git a/src/diffusers/models/embeddings.py b/src/diffusers/models/embeddings.py
--- a/src/diffusers/models/embeddings.py
+++ b/src/diffusers/models/embeddings.py
@@ -461,6 +461,18 @@ def forward(self, image_embeds: torch.FloatTensor):
         return image_embeds
 
 
+class MLPProjection(nn.Module):
+    def __init__(self, image_embed_dim=1024, cross_attention_dim=1024):
+        super().__init__()
+        from .attention import FeedForward
+
+        self.ff = FeedForward(image_embed_dim, cross_attention_dim, mult=1, activation_fn="gelu")
+        self.norm = nn.LayerNorm(cross_attention_dim)
+
+    def forward(self, image_embeds: torch.FloatTensor):
+        return self.norm(self.ff(image_embeds))
+
+
 class CombinedTimestepLabelEmbeddings(nn.Module):
     def __init__(self, num_classes, embedding_dim, class_dropout_prob=0.1):
         super().__init__()


Test patch:
diff --git a/tests/pipelines/ip_adapters/test_ip_adapter_stable_diffusion.py b/tests/pipelines/ip_adapters/test_ip_adapter_stable_diffusion.py
--- a/tests/pipelines/ip_adapters/test_ip_adapter_stable_diffusion.py
+++ b/tests/pipelines/ip_adapters/test_ip_adapter_stable_diffusion.py
@@ -182,6 +182,25 @@ def test_inpainting(self):
 
         assert np.allclose(image_slice, expected_slice, atol=1e-4, rtol=1e-4)
 
+    def test_text_to_image_full_face(self):
+        image_encoder = self.get_image_encoder(repo_id="h94/IP-Adapter", subfolder="models/image_encoder")
+        pipeline = StableDiffusionPipeline.from_pretrained(
+            "runwayml/stable-diffusion-v1-5", image_encoder=image_encoder, safety_checker=None, torch_dtype=self.dtype
+        )
+        pipeline.to(torch_device)
+        pipeline.load_ip_adapter("h94/IP-Adapter", subfolder="models", weight_name="ip-adapter-full-face_sd15.bin")
+        pipeline.set_ip_adapter_scale(0.7)
+
+        inputs = self.get_dummy_inputs()
+        images = pipeline(**inputs).images
+        image_slice = images[0, :3, :3, -1].flatten()
+
+        expected_slice = np.array(
+            [0.1706543, 0.1303711, 0.12573242, 0.21777344, 0.14550781, 0.14038086, 0.40820312, 0.41455078, 0.42529297]
+        )
+
+        assert np.allclose(image_slice, expected_slice, atol=1e-4, rtol=1e-4)
+
 
 @slow
 @require_torch_gpu