guoyww

/

animatediff-motion-adapter-v1-5-2

like 18

Hi,

I was trying to use another text to image model and ran into this issue. Are all models not compatible?

Tried- playgroundai/playground-v2-1024px-aesthetic
Code:

import torch
from diffusers import MotionAdapter, AnimateDiffPipeline, DDIMScheduler
from diffusers.utils import export_to_gif,export_to_video

# Load the motion adapter
adapter = MotionAdapter.from_pretrained("guoyww/animatediff-motion-adapter-v1-5-2")

model_id="playgroundai/playground-v2-1024px-aesthetic"
pipe = AnimateDiffPipeline.from_pretrained(model_id, motion_adapter=adapter)

Error log:

RuntimeError                              Traceback (most recent call last)
<ipython-input-11-b426c2fc29e0> in <cell line: 1>()
----> 1 pipe = AnimateDiffPipeline.from_pretrained(model_id, motion_adapter=adapter)
      2 scheduler = DDIMScheduler.from_pretrained(
      3     model_id, subfolder="scheduler", clip_sample=False, timestep_spacing="linspace", steps_offset=1
      4 )
      5 pipe.scheduler = scheduler

3 frames
/usr/local/lib/python3.10/dist-packages/diffusers/pipelines/pipeline_utils.py in from_pretrained(cls, pretrained_model_name_or_path, **kwargs)
   1355 
   1356         # 8. Instantiate the pipeline
-> 1357         model = pipeline_class(**init_kwargs)
   1358 
   1359         # 9. Save where the model was instantiated from

/usr/local/lib/python3.10/dist-packages/diffusers/pipelines/animatediff/pipeline_animatediff.py in __init__(self, vae, text_encoder, tokenizer, unet, motion_adapter, scheduler, feature_extractor, image_encoder)
    123     ):
    124         super().__init__()
--> 125         unet = UNetMotionModel.from_unet2d(unet, motion_adapter)
    126 
    127         self.register_modules(

/usr/local/lib/python3.10/dist-packages/diffusers/models/unet_motion_model.py in from_unet2d(cls, unet, motion_adapter, load_weights)
    427             model.down_blocks[i].resnets.load_state_dict(down_block.resnets.state_dict())
    428             if hasattr(model.down_blocks[i], "attentions"):
--> 429                 model.down_blocks[i].attentions.load_state_dict(down_block.attentions.state_dict())
    430             if model.down_blocks[i].downsamplers:
    431                 model.down_blocks[i].downsamplers.load_state_dict(down_block.downsamplers.state_dict())

/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py in load_state_dict(self, state_dict, strict, assign)
   2150 
   2151         if len(error_msgs) > 0:
-> 2152             raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
   2153                                self.__class__.__name__, "\n\t".join(error_msgs)))
   2154         return _IncompatibleKeys(missing_keys, unexpected_keys)

RuntimeError: Error(s) in loading state_dict for ModuleList:
    Unexpected key(s) in state_dict: "0.transformer_blocks.1.norm1.weight", "0.transformer_blocks.1.norm1.bias", "0.transformer_blocks.1.attn1.to_q.weight", "0.transformer_blocks.1.attn1.to_k.weight", "0.transformer_blocks.1.attn1.to_v.weight", "0.transformer_blocks.1.attn1.to_out.0.weight", "0.transformer_blocks.1.attn1.to_out.0.bias", "0.transformer_blocks.1.norm2.weight", "0.transformer_blocks.1.norm2.bias", "0.transformer_blocks.1.attn2.to_q.weight", "0.transformer_blocks.1.attn2.to_k.weight", "0.transformer_blocks.1.attn2.to_v.weight", "0.transformer_blocks.1.attn2.to_out.0.weight", "0.transformer_blocks.1.attn2.to_out.0.bias", "0.transformer_blocks.1.norm3.weight", "0.transformer_blocks.1.norm3.bias", "0.transformer_blocks.1.ff.net.0.proj.weight", "0.transformer_blocks.1.ff.net.0.proj.bias", "0.transformer_blocks.1.ff.net.2.weight", "0.transformer_blocks.1.ff.net.2.bias", "1.transformer_blocks.1.norm1.weight", "1.transformer_blocks.1.norm1.bias", "1.transformer_blocks.1.attn1.to_q.weight", "1.transformer_blocks.1.attn1.to_k.weight", "1.transformer_blocks.1.attn1.to_v.weight", "1.transformer_blocks.1.attn1.to_out.0.weight", "1.transformer_blocks.1.attn1.to_out.0.bias", "1.transformer_blocks.1.norm2.weight", "1.transformer_blocks.1.norm2.bias", "1.transformer_blocks.1.attn2.to_q.weight", "1.transformer_blocks.1.attn2.to_k.weight", "1.transformer_blocks.1.attn2.to_v.weight", "1.transformer_blocks.1.attn2.to_out.0.weight", "1.transformer_blocks.1.attn2.to_out.0.bias", "1.tr...