Daniel Regado commited on
Commit
f1f54ca
1 Parent(s): 1f5bb12

Upload 2 files

Browse files

As part of the integration to diffusers (see [PR](https://github.com/huggingface/diffusers/pull/9987)), we can simplify things changing the ip-adapter state dict keys here, instead of adding extra conversion code. The "ip-adapter.bin" file is updated to conform with the integration, and I also uploaded the equivalent in parameters in safetensors. The PR is nearly finished, so shouldn't be long until the integration is merged. The new checkpoints were obtained with the following code:

```python
import torch
from safetensors.torch import save_file
from diffusers.utils import _get_model_file
from diffusers.models.modeling_utils import load_state_dict

# Get weights from the hub
model_file = _get_model_file(
"InstantX/SD3.5-Large-IP-Adapter",
weights_name="ip-adapter.bin",
)
state_dict = load_state_dict(model_file)

# ip_adapter stays the same
safetensors_dict = {f"ip_adapter.{key}": value for key, value in state_dict["ip_adapter"].items()}

# image_proj to diffusers
for key, value in state_dict["image_proj"].items():
if key.startswith("layers."):
idx = key.split(".")[1]
key = key.replace(f"layers.{idx}.0.norm1", f"layers.{idx}.ln0")
key = key.replace(f"layers.{idx}.0.norm2", f"layers.{idx}.ln1")
key = key.replace(f"layers.{idx}.0.to_q", f"layers.{idx}.attn.to_q")
key = key.replace(f"layers.{idx}.0.to_kv", f"layers.{idx}.attn.to_kv")
key = key.replace(f"layers.{idx}.0.to_out", f"layers.{idx}.attn.to_out.0")
key = key.replace(f"layers.{idx}.1.0", f"layers.{idx}.adaln_norm")
key = key.replace(f"layers.{idx}.1.1", f"layers.{idx}.ff.net.0.proj")
key = key.replace(f"layers.{idx}.1.3", f"layers.{idx}.ff.net.2")
key = key.replace(f"layers.{idx}.2.1", f"layers.{idx}.adaln_proj")
safetensors_dict[f"image_proj.{key}"] = value

# Save safetensors
save_file(safetensors_dict, "ip-adapter.safetensors")


# ip_adapter stays the same
torch_new_dict = {"ip_adapter": state_dict["ip_adapter"], "image_proj": {}}

# image_proj to diffusers
for key, value in state_dict["image_proj"].items():
if key.startswith("layers."):
idx = key.split(".")[1]
key = key.replace(f"layers.{idx}.0.norm1", f"layers.{idx}.ln0")
key = key.replace(f"layers.{idx}.0.norm2", f"layers.{idx}.ln1")
key = key.replace(f"layers.{idx}.0.to_q", f"layers.{idx}.attn.to_q")
key = key.replace(f"layers.{idx}.0.to_kv", f"layers.{idx}.attn.to_kv")
key = key.replace(f"layers.{idx}.0.to_out", f"layers.{idx}.attn.to_out.0")
key = key.replace(f"layers.{idx}.1.0", f"layers.{idx}.adaln_norm")
key = key.replace(f"layers.{idx}.1.1", f"layers.{idx}.ff.net.0.proj")
key = key.replace(f"layers.{idx}.1.3", f"layers.{idx}.ff.net.2")
key = key.replace(f"layers.{idx}.2.1", f"layers.{idx}.adaln_proj")
torch_new_dict["image_proj"][key] = value

# Save torch pickle
torch.save(torch_new_dict, "ip-adapter.bin")
```

Files changed (2) hide show
  1. ip-adapter.bin +2 -2
  2. ip-adapter.safetensors +3 -0
ip-adapter.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9fe54774aa528e712d9145ff6a59dd93b1fcf1d5935304feffd980ae6d42ae03
3
- size 1595970439
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a12562389f6b4ed56e58077c21aff7c8298401d4de376204bfe04c5d8e2f6aa2
3
+ size 1595963463
ip-adapter.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7605332eeba0194f62bbe46deb62e5f1ee71a522cd38bba543d907a677d94949
3
+ size 1595885744