v0.0.2

.gitignore

@@ -152,4 +152,8 @@ dmypy.json
 # Cython debug symbols
 cython_debug/
 
-results/
+results/
+checkpoints/
+gradio_cached_examples/
+gfpgan/
+start.sh

app.py

@@ -3,15 +3,20 @@ import tempfile
 import gradio as gr
 from src.gradio_demo import SadTalker  
 from src.utils.text2speech import TTSTalker
+from huggingface_hub import snapshot_download
 
 def get_source_image(image):   
         return image
 
-
+def download_model():
+    REPO_ID = 'vinthony/SadTalker'
+    snapshot_download(repo_id=REPO_ID, local_dir='./checkpoints', local_dir_use_symlinks=True)
 
 def sadtalker_demo():
 
-    sad_talker = SadTalker()
+    download_model()
+
+    sad_talker = SadTalker(lazy_load=True)
     tts_talker = TTSTalker()
 
     with gr.Blocks(analytics_enabled=False) as sadtalker_interface:
@@ -42,7 +47,8 @@ def sadtalker_demo():
                 with gr.Tabs(elem_id="sadtalker_checkbox"):
                     with gr.TabItem('Settings'):
                         with gr.Column(variant='panel'):
-                            is_still_mode = gr.Checkbox(label="w/ Still Mode (fewer hand motion, works on full body)")
+                            preprocess_type = gr.Radio(['crop','resize','full'], value='crop', label='preprocess', info="How to handle input image?")
+                            is_still_mode = gr.Checkbox(label="w/ Still Mode (fewer hand motion, works with preprocess `full`)")
                             enhancer = gr.Checkbox(label="w/ GFPGAN as Face enhancer")
                             submit = gr.Button('Generate', elem_id="sadtalker_generate", variant='primary')
 
@@ -54,42 +60,83 @@ def sadtalker_demo():
                 [
                     'examples/source_image/full_body_1.png',
                     'examples/driven_audio/bus_chinese.wav',
+                    'crop',
                     True,
                     False
                 ],
                 [
                     'examples/source_image/full_body_2.png',
-                    'examples/driven_audio/itosinger1.wav',
+                    'examples/driven_audio/japanese.wav',
+                    'crop',
+                    False,
+                    False
+                ],
+                [
+                    'examples/source_image/full3.png',
+                    'examples/driven_audio/deyu.wav',
+                    'crop',
+                    False,
+                    True
+                ],
+                [
+                    'examples/source_image/full4.jpeg',
+                    'examples/driven_audio/eluosi.wav',
+                    'full',
+                    False,
+                    True
+                ],
+                [
+                    'examples/source_image/full4.jpeg',
+                    'examples/driven_audio/imagine.wav',
+                    'full',
+                    True,
+                    True
+                ],
+                [
+                    'examples/source_image/full_body_1.png',
+                    'examples/driven_audio/bus_chinese.wav',
+                    'full',
                     True,
                     False
                 ],
                 [
                     'examples/source_image/art_13.png',
                     'examples/driven_audio/fayu.wav',
+                    'resize',
                     True,
                     False
                 ],
                 [
                     'examples/source_image/art_5.png',
                     'examples/driven_audio/chinese_news.wav',
-                    True,
+                    'resize',
+                    False,
                     False
                 ],
+                [
+                    'examples/source_image/art_5.png',
+                    'examples/driven_audio/RD_Radio31_000.wav',
+                    'resize',
+                    True,
+                    True
+                ],
             ]
             gr.Examples(examples=examples,
                         inputs=[
                             source_image,
                             driven_audio,
+                            preprocess_type,
                             is_still_mode,
                             enhancer], 
                         outputs=[gen_video],
                         fn=sad_talker.test,
-                        cache_examples=os.getenv('SYSTEM') == 'spaces')
+                        cache_examples=True) # os.getenv('SYSTEM') == 'spaces')
 
         submit.click(
                     fn=sad_talker.test, 
                     inputs=[source_image,
                             driven_audio,
+                            preprocess_type,
                             is_still_mode,
                             enhancer], 
                     outputs=[gen_video]

checkpoints/BFM_Fitting.zip

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:785f77f3de288568e76666cd419dcf40646d3f74eae6d4fa3b766c933087a9d8
-size 404051745

checkpoints/BFM_Fitting/01_MorphableModel.mat

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:37b1f0742db356a3b1568a8365a06f5b0fe0ab687ac1c3068c803666cbd4d8e2
-size 240875364

checkpoints/BFM_Fitting/01_MorphableModel.mat

@@ -0,0 +1 @@
+../../../../../root/.cache/huggingface/hub/models--vinthony--SadTalker/blobs/37b1f0742db356a3b1568a8365a06f5b0fe0ab687ac1c3068c803666cbd4d8e2

checkpoints/BFM_Fitting/BFM09_model_info.mat

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:db8d00544f0b0182f1b8430a3bb87662b3ff674eb33c84e6f52dbe2971adb81b
-size 127170280

checkpoints/BFM_Fitting/BFM09_model_info.mat

@@ -0,0 +1 @@
+../../../../../root/.cache/huggingface/hub/models--vinthony--SadTalker/blobs/db8d00544f0b0182f1b8430a3bb87662b3ff674eb33c84e6f52dbe2971adb81b

checkpoints/BFM_Fitting/BFM_exp_idx.mat

@@ -0,0 +1 @@
+../../../../../root/.cache/huggingface/hub/models--vinthony--SadTalker/blobs/1146e4e9c3bef303a497383aa7974c014fe945c7

checkpoints/BFM_Fitting/BFM_front_idx.mat

@@ -0,0 +1 @@
+../../../../../root/.cache/huggingface/hub/models--vinthony--SadTalker/blobs/b9d7b0953dd1dc5b1e28144610485409ac321f9b

checkpoints/BFM_Fitting/Exp_Pca.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e7f31380e6cbdaf2aeec698db220bac4f221946e4d551d88c092d47ec49b1726
-size 51086404

checkpoints/BFM_Fitting/Exp_Pca.bin

@@ -0,0 +1 @@
+../../../../../root/.cache/huggingface/hub/models--vinthony--SadTalker/blobs/e7f31380e6cbdaf2aeec698db220bac4f221946e4d551d88c092d47ec49b1726

checkpoints/BFM_Fitting/facemodel_info.mat

@@ -0,0 +1 @@
+../../../../../root/.cache/huggingface/hub/models--vinthony--SadTalker/blobs/3e516ec7297fa3248098f49ecea10579f4831c0a

checkpoints/BFM_Fitting/select_vertex_id.mat

@@ -0,0 +1 @@
+../../../../../root/.cache/huggingface/hub/models--vinthony--SadTalker/blobs/5b8b220093d93b133acc94ffed159f31a74854cd

checkpoints/BFM_Fitting/similarity_Lm3D_all.mat

@@ -0,0 +1 @@
+../../../../../root/.cache/huggingface/hub/models--vinthony--SadTalker/blobs/a0e23588302bc71fc899eef53ff06df5f4df4c1d

checkpoints/BFM_Fitting/std_exp.txt

@@ -1 +0,0 @@
-453980 257264 263068 211890 135873 184721 47055.6 72732 62787.4 106226 56708.5 51439.8 34887.1 44378.7 51813.4 31030.7 23354.9 23128.1 19400 21827.6 22767.7 22057.4 19894.3 16172.8 17142.7 10035.3 14727.5 12972.5 10763.8 8953.93 8682.62 8941.81 6342.3 5205.3 7065.65 6083.35 6678.88 4666.63 5082.89 5134.76 4908.16 3964.93 3739.95 3180.09 2470.45 1866.62 1624.71 2423.74 1668.53 1471.65 1194.52 782.102 815.044 835.782 834.937 744.496 575.146 633.76 705.685 753.409 620.306 673.326 766.189 619.866 559.93 357.264 396.472 556.849 455.048 460.592 400.735 326.702 279.428 291.535 326.584 305.664 287.816 283.642 276.19

checkpoints/BFM_Fitting/std_exp.txt

@@ -0,0 +1 @@
+../../../../../root/.cache/huggingface/hub/models--vinthony--SadTalker/blobs/767b8de4ea1ca78b6f22b98ff2dee4fa345500bb

checkpoints/auido2exp_00300-model.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b7608f0e6b477e50e03ca569ac5b04a841b9217f89d502862fc78fda4e46dec4
-size 34278319

checkpoints/auido2exp_00300-model.pth

@@ -0,0 +1 @@
+../../../../root/.cache/huggingface/hub/models--vinthony--SadTalker/blobs/b7608f0e6b477e50e03ca569ac5b04a841b9217f89d502862fc78fda4e46dec4

checkpoints/auido2pose_00140-model.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:4fba6701852dc57efbed25b1e4276e4ff752941860d69fc4429f08a02326ebce
-size 95916155

checkpoints/auido2pose_00140-model.pth

@@ -0,0 +1 @@
+../../../../root/.cache/huggingface/hub/models--vinthony--SadTalker/blobs/4fba6701852dc57efbed25b1e4276e4ff752941860d69fc4429f08a02326ebce

checkpoints/epoch_20.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:6d17a6b23457b521801baae583cb6a58f7238fe6721fc3d65d76407460e9149b
-size 288860037

checkpoints/epoch_20.pth

@@ -0,0 +1 @@
+../../../../root/.cache/huggingface/hub/models--vinthony--SadTalker/blobs/6d17a6b23457b521801baae583cb6a58f7238fe6721fc3d65d76407460e9149b

checkpoints/facevid2vid_00189-model.pth.tar

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:fbad01d46f0510276dc4521322dde6824a873a4222cd0740c85762e7067ea71d
-size 2112619148

checkpoints/facevid2vid_00189-model.pth.tar

@@ -0,0 +1 @@
+../../../../root/.cache/huggingface/hub/models--vinthony--SadTalker/blobs/fbad01d46f0510276dc4521322dde6824a873a4222cd0740c85762e7067ea71d

checkpoints/hub/checkpoints/2DFAN4-cd938726ad.zip

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:cd938726adb1f15f361263cce2db9cb820c42585fa8796ec72ce19107f369a46
-size 96316515

checkpoints/hub/checkpoints/2DFAN4-cd938726ad.zip

@@ -0,0 +1 @@
+../../../../../../root/.cache/huggingface/hub/models--vinthony--SadTalker/blobs/cd938726adb1f15f361263cce2db9cb820c42585fa8796ec72ce19107f369a46

checkpoints/hub/checkpoints/s3fd-619a316812.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:619a31681264d3f7f7fc7a16a42cbbe8b23f31a256f75a366e5a1bcd59b33543
-size 89843225

checkpoints/hub/checkpoints/s3fd-619a316812.pth

@@ -0,0 +1 @@
+../../../../../../root/.cache/huggingface/hub/models--vinthony--SadTalker/blobs/619a31681264d3f7f7fc7a16a42cbbe8b23f31a256f75a366e5a1bcd59b33543

checkpoints/mapping_00229-model.pth.tar

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:62a1e06006cc963220f6477438518ed86e9788226c62ae382ddc42fbcefb83f1
-size 155521183

checkpoints/mapping_00229-model.pth.tar

@@ -0,0 +1 @@
+../../../../root/.cache/huggingface/hub/models--vinthony--SadTalker/blobs/62a1e06006cc963220f6477438518ed86e9788226c62ae382ddc42fbcefb83f1

checkpoints/shape_predictor_68_face_landmarks.dat

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:fbdc2cb80eb9aa7a758672cbfdda32ba6300efe9b6e6c7a299ff7e736b11b92f
-size 99693937

checkpoints/shape_predictor_68_face_landmarks.dat

@@ -0,0 +1 @@
+../../../../root/.cache/huggingface/hub/models--vinthony--SadTalker/blobs/fbdc2cb80eb9aa7a758672cbfdda32ba6300efe9b6e6c7a299ff7e736b11b92f

checkpoints/wav2lip.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b78b681b68ad9fe6c6fb1debc6ff43ad05834a8af8a62ffc4167b7b34ef63c37
-size 435807851

checkpoints/wav2lip.pth

@@ -0,0 +1 @@
+../../../../root/.cache/huggingface/hub/models--vinthony--SadTalker/blobs/b78b681b68ad9fe6c6fb1debc6ff43ad05834a8af8a62ffc4167b7b34ef63c37

inference.py

@@ -1,134 +0,0 @@
-import torch
-from time import  strftime
-import os, sys, time
-from argparse import ArgumentParser
-
-from src.utils.preprocess import CropAndExtract
-from src.test_audio2coeff import Audio2Coeff  
-from src.facerender.animate import AnimateFromCoeff
-from src.generate_batch import get_data
-from src.generate_facerender_batch import get_facerender_data
-
-def main(args):
-    #torch.backends.cudnn.enabled = False
-
-    pic_path = args.source_image
-    audio_path = args.driven_audio
-    save_dir = os.path.join(args.result_dir, strftime("%Y_%m_%d_%H.%M.%S"))
-    os.makedirs(save_dir, exist_ok=True)
-    pose_style = args.pose_style
-    device = args.device
-    batch_size = args.batch_size
-    camera_yaw_list = args.camera_yaw
-    camera_pitch_list = args.camera_pitch
-    camera_roll_list = args.camera_roll
-
-    current_code_path = sys.argv[0]
-    current_root_path = os.path.split(current_code_path)[0]
-
-    os.environ['TORCH_HOME']=os.path.join(current_root_path, args.checkpoint_dir)
-
-    path_of_lm_croper = os.path.join(current_root_path, args.checkpoint_dir, 'shape_predictor_68_face_landmarks.dat')
-    path_of_net_recon_model = os.path.join(current_root_path, args.checkpoint_dir, 'epoch_20.pth')
-    dir_of_BFM_fitting = os.path.join(current_root_path, args.checkpoint_dir, 'BFM_Fitting')
-    wav2lip_checkpoint = os.path.join(current_root_path, args.checkpoint_dir, 'wav2lip.pth')
-
-    audio2pose_checkpoint = os.path.join(current_root_path, args.checkpoint_dir, 'auido2pose_00140-model.pth')
-    audio2pose_yaml_path = os.path.join(current_root_path, 'src', 'config', 'auido2pose.yaml')
-    
-    audio2exp_checkpoint = os.path.join(current_root_path, args.checkpoint_dir, 'auido2exp_00300-model.pth')
-    audio2exp_yaml_path = os.path.join(current_root_path, 'src', 'config', 'auido2exp.yaml')
-
-    free_view_checkpoint = os.path.join(current_root_path, args.checkpoint_dir, 'facevid2vid_00189-model.pth.tar')
-    mapping_checkpoint = os.path.join(current_root_path, args.checkpoint_dir, 'mapping_00229-model.pth.tar')
-    facerender_yaml_path = os.path.join(current_root_path, 'src', 'config', 'facerender.yaml')
-
-    #init model
-    print(path_of_net_recon_model)
-    preprocess_model = CropAndExtract(path_of_lm_croper, path_of_net_recon_model, dir_of_BFM_fitting, device)
-
-    print(audio2pose_checkpoint)
-    print(audio2exp_checkpoint)
-    audio_to_coeff = Audio2Coeff(audio2pose_checkpoint, audio2pose_yaml_path, 
-                                audio2exp_checkpoint, audio2exp_yaml_path, 
-                                wav2lip_checkpoint, device)
-    
-    print(free_view_checkpoint)
-    print(mapping_checkpoint)
-    animate_from_coeff = AnimateFromCoeff(free_view_checkpoint, mapping_checkpoint, 
-                                            facerender_yaml_path, device)
-
-    #crop image and extract 3dmm from image
-    first_frame_dir = os.path.join(save_dir, 'first_frame_dir')
-    os.makedirs(first_frame_dir, exist_ok=True)
-    first_coeff_path, crop_pic_path =  preprocess_model.generate(pic_path, first_frame_dir)
-    if first_coeff_path is None:
-        print("Can't get the coeffs of the input")
-        return
-
-    #audio2ceoff
-    batch = get_data(first_coeff_path, audio_path, device)
-    coeff_path = audio_to_coeff.generate(batch, save_dir, pose_style)
-
-    # 3dface render
-    if args.face3dvis:
-        from src.face3d.visualize import gen_composed_video
-        gen_composed_video(args, device, first_coeff_path, coeff_path, audio_path, os.path.join(save_dir, '3dface.mp4'))
-    
-    #coeff2video
-    data = get_facerender_data(coeff_path, crop_pic_path, first_coeff_path, audio_path, 
-                                batch_size, camera_yaw_list, camera_pitch_list, camera_roll_list,
-                                expression_scale=args.expression_scale, still_mode=args.still)
-    
-    animate_from_coeff.generate(data, save_dir,  enhancer=args.enhancer)
-    video_name = data['video_name']
-
-    if args.enhancer is not None:
-        print(f'The generated video is named {video_name}_enhanced in {save_dir}')
-    else:
-        print(f'The generated video is named {video_name} in {save_dir}')
-    
-    return os.path.join(save_dir, video_name+'.mp4'), os.path.join(save_dir, video_name+'.mp4')
-
-    
-if __name__ == '__main__':
-
-    parser = ArgumentParser()  
-    parser.add_argument("--driven_audio", default='./examples/driven_audio/japanese.wav', help="path to driven audio")
-    parser.add_argument("--source_image", default='./examples/source_image/art_0.png', help="path to source image")
-    parser.add_argument("--checkpoint_dir", default='./checkpoints', help="path to output")
-    parser.add_argument("--result_dir", default='./results', help="path to output")
-    parser.add_argument("--pose_style", type=int, default=0,  help="input pose style from [0, 46)")
-    parser.add_argument("--batch_size", type=int, default=2,  help="the batch size of facerender")
-    parser.add_argument("--expression_scale", type=float, default=1.,  help="the batch size of facerender")
-    parser.add_argument('--camera_yaw', nargs='+', type=int, default=[0], help="the camera yaw degree")
-    parser.add_argument('--camera_pitch', nargs='+', type=int, default=[0], help="the camera pitch degree")
-    parser.add_argument('--camera_roll', nargs='+', type=int, default=[0], help="the camera roll degree")
-    parser.add_argument('--enhancer',  type=str, default=None, help="Face enhancer, [GFPGAN]")
-    parser.add_argument("--cpu", dest="cpu", action="store_true") 
-    parser.add_argument("--face3dvis", action="store_true", help="generate 3d face and 3d landmarks") 
-    parser.add_argument("--still", action="store_true") 
-
-    # net structure and parameters
-    parser.add_argument('--net_recon', type=str, default='resnet50', choices=['resnet18', 'resnet34', 'resnet50'], help='not use')
-    parser.add_argument('--init_path', type=str, default=None, help='not Use')
-    parser.add_argument('--use_last_fc',default=False, help='zero initialize the last fc')
-    parser.add_argument('--bfm_folder', type=str, default='./checkpoints/BFM_Fitting/')
-    parser.add_argument('--bfm_model', type=str, default='BFM_model_front.mat', help='bfm model')
-
-    # default renderer parameters
-    parser.add_argument('--focal', type=float, default=1015.)
-    parser.add_argument('--center', type=float, default=112.)
-    parser.add_argument('--camera_d', type=float, default=10.)
-    parser.add_argument('--z_near', type=float, default=5.)
-    parser.add_argument('--z_far', type=float, default=15.)
-
-    args = parser.parse_args()
-
-    if torch.cuda.is_available() and not args.cpu:
-        args.device = "cuda"
-    else:
-        args.device = "cpu"
-
-    main(args)
-

modules/gfpgan_inference.py

@@ -1,36 +0,0 @@
-import os,sys
-
-def gfpgan(scale, origin_mp4_path):
-    current_code_path = sys.argv[0]
-    current_root_path = os.path.split(current_code_path)[0]
-    print(current_root_path)
-    gfpgan_code_path = current_root_path+'/repositories/GFPGAN/inference_gfpgan.py'
-    print(gfpgan_code_path)
-    
-    #video2pic
-    result_dir = os.path.split(origin_mp4_path)[0]
-    video_name = os.path.split(origin_mp4_path)[1]
-    video_name = video_name.split('.')[0]
-    print(video_name)
-    str_scale = str(scale).replace('.', '_')
-    output_mp4_path = os.path.join(result_dir, video_name+'##'+str_scale+'.mp4')
-    temp_output_mp4_path = os.path.join(result_dir, 'temp_'+video_name+'##'+str_scale+'.mp4')
-
-    audio_name = video_name.split('##')[-1]
-    audio_path = os.path.join(result_dir, audio_name+'.wav')
-    temp_pic_dir1 = os.path.join(result_dir, video_name)
-    temp_pic_dir2 = os.path.join(result_dir, video_name+'##'+str_scale)
-    os.makedirs(temp_pic_dir1, exist_ok=True)
-    os.makedirs(temp_pic_dir2, exist_ok=True)
-    cmd1 = 'ffmpeg -i \"{}\" -start_number 0  \"{}\"/%06d.png -loglevel error -y'.format(origin_mp4_path, temp_pic_dir1)
-    os.system(cmd1)
-    cmd2 = f'python {gfpgan_code_path} -i {temp_pic_dir1} -o {temp_pic_dir2} -s {scale}'
-    os.system(cmd2)
-    cmd3 = f'ffmpeg -r 25 -f image2 -i {temp_pic_dir2}/%06d.png  -vcodec libx264 -crf 25  -pix_fmt yuv420p {temp_output_mp4_path}'
-    os.system(cmd3)
-    cmd4 = f'ffmpeg -y -i {temp_output_mp4_path}  -i {audio_path}  -vcodec copy {output_mp4_path}'
-    os.system(cmd4)
-    #shutil.rmtree(temp_pic_dir1)
-    #shutil.rmtree(temp_pic_dir2)
-
-    return output_mp4_path