init

configs/inference/head_pose_temp/pose_temp.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed399e8a6dc5faab4eb98a676f211dbc108bddad5ec27164540cf25ad6d96818
+size 8048

configs/inference/inference_audio.yaml

@@ -0,0 +1,17 @@
+a2m_model:
+  out_dim: 1404
+  latent_dim: 512
+  model_path: ./pretrained_model/wav2vec2-base-960h
+  only_last_fetures: True
+  from_pretrained: True
+
+a2p_model:
+  out_dim: 6 
+  latent_dim: 512
+  model_path: ./pretrained_model/wav2vec2-base-960h
+  only_last_fetures: True
+  from_pretrained: True
+
+pretrained_model:
+  a2m_ckpt: ./pretrained_model/audio2mesh.pt
+  a2p_ckpt: ./pretrained_model/audio2pose.pt

configs/inference/inference_v2.yaml

@@ -0,0 +1,35 @@
+unet_additional_kwargs:
+  use_inflated_groupnorm: true
+  unet_use_cross_frame_attention: false 
+  unet_use_temporal_attention: false
+  use_motion_module: true
+  motion_module_resolutions:
+  - 1
+  - 2
+  - 4
+  - 8
+  motion_module_mid_block: true 
+  motion_module_decoder_only: false
+  motion_module_type: Vanilla
+  motion_module_kwargs:
+    num_attention_heads: 8
+    num_transformer_block: 1
+    attention_block_types:
+    - Temporal_Self
+    - Temporal_Self
+    temporal_position_encoding: true
+    temporal_position_encoding_max_len: 32
+    temporal_attention_dim_div: 1
+
+noise_scheduler_kwargs:
+  beta_start: 0.00085
+  beta_end: 0.012
+  beta_schedule: "linear"
+  clip_sample: false
+  steps_offset: 1
+  ### Zero-SNR params
+  prediction_type: "v_prediction"
+  rescale_betas_zero_snr: True
+  timestep_spacing: "trailing"
+
+sampler: DDIM 

configs/prompts/animation_audio.yaml

@@ -0,0 +1,14 @@
+pretrained_base_model_path: './pretrained_model/stable-diffusion-v1-5'
+pretrained_vae_path: './pretrained_model/sd-vae-ft-mse'
+image_encoder_path: './pretrained_model/image_encoder'
+
+denoising_unet_path: "./pretrained_model/denoising_unet.pth"
+reference_unet_path: "./pretrained_model/reference_unet.pth"
+pose_guider_path: "./pretrained_model/pose_guider.pth"
+motion_module_path: "./pretrained_model/motion_module.pth"
+
+audio_inference_config: "./configs/inference/inference_audio.yaml"
+inference_config: "./configs/inference/inference_v2.yaml"
+weight_dtype: 'fp16'
+
+pose_temp: "./configs/inference/head_pose_temp/pose_temp.npy"

configs/prompts/animation_facereenac.yaml

@@ -0,0 +1,11 @@
+pretrained_base_model_path: './pretrained_model/stable-diffusion-v1-5'
+pretrained_vae_path: './pretrained_model/sd-vae-ft-mse'
+image_encoder_path: './pretrained_model/image_encoder'
+
+denoising_unet_path: "./pretrained_model/denoising_unet.pth"
+reference_unet_path: "./pretrained_model/reference_unet.pth"
+pose_guider_path: "./pretrained_model/pose_guider.pth"
+motion_module_path: "./pretrained_model/motion_module.pth"
+
+inference_config: "./configs/inference/inference_v2.yaml"
+weight_dtype: 'fp16'