fix: FPS and image shape of retarget

app.py

@@ -116,6 +116,7 @@ with gr.Blocks(theme=gr.themes.Soft()) as demo:
                 flag_relative_input = gr.Checkbox(value=True, label="relative motion")
                 flag_do_crop_input = gr.Checkbox(value=True, label="do crop")
                 flag_remap_input = gr.Checkbox(value=True, label="paste-back")
+    gr.Markdown(load_description("assets/gradio_description_animate_clear.md"))
     with gr.Row():
         with gr.Column():
             process_button_animation = gr.Button("🚀 Animate", variant="primary")

assets/gradio_description_animate_clear.md

@@ -0,0 +1,3 @@
+<div style="font-size: 1.2em; text-align: center;">
+    Step 3: click the <strong>🚀 Animate</strong> button below to generate, or click 🧹 Clear to erase the results
+</div>

assets/gradio_description_upload.md

@@ -1,8 +1,24 @@
-## 🤗 This is the official gradio demo for LivePortrait.
-<div style="font-size: 1.0em;">
+<!-- ## 🤗 This is the official gradio demo for LivePortrait. -->
+<!-- <div style="font-size: 1.0em;">
     If you find LivePortrait fun 🤪 or useful, please consider starring 🌟 our <a href="https://github.com/KwaiVGI/LivePortrait">GitHub Repo</a> to discover more features!
 </div>
+
+
+<!-- <div style="font-size: 1.2em;">
+    Step1: upload or use a webcam to get a <strong>Source Portrait</strong> (any aspect ratio) to left side.<br>
+    Step2: upload a <strong>Driving Video</strong> (1:1 aspect ratio) to right side.
+</div> -->
+
 <br>
-<div style="font-size: 1.2em;">
-    Please upload or use a webcam to get a <strong>Source Portrait</strong> (any aspect ratio) and upload a <strong>Driving Video</strong> (1:1 aspect ratio).
+<div style="font-size: 1.2em; display: flex; justify-content: space-between;">
+    <div style="flex: 1; text-align: center; margin-right: 20px;">
+        <div style="display: inline-block;">
+            Step 1: upload a <strong>Source Portrait</strong> (any aspect ratio) ⬇️
+        </div>
+    </div>
+    <div style="flex: 1; text-align: center; margin-left: 20px;">
+        <div style="display: inline-block;">
+            Step 2: upload a <strong>Driving Video</strong> (1:1 aspect ratio) ⬇️
+        </div>
+    </div>
 </div>

assets/gradio_title.md

@@ -9,6 +9,9 @@
       <a href="https://liveportrait.github.io"><img src="https://img.shields.io/badge/Project_Page-LivePortrait-green" alt="Project Page"></a>
       &nbsp;
       <a href="https://github.com/KwaiVGI/LivePortrait"><img src="https://img.shields.io/badge/Github-Code-blue"></a>
+      &nbsp;
+      <a href="https://github.com/KwaiVGI/LivePortrait"><img src="https://img.shields.io/github/stars/KwaiVGI/LivePortrait
+      "></a>
     </div>
   </div>
 </div>

src/gradio_pipeline.py

@@ -9,7 +9,7 @@ from .live_portrait_pipeline import LivePortraitPipeline
 from .utils.io import load_img_online
 from .utils.rprint import rlog as log
 from .utils.crop import prepare_paste_back, paste_back
-from .utils.camera import get_rotation_matrix
+# from .utils.camera import get_rotation_matrix
 
 def update_args(args, user_args):
     """update the args according to user inputs
@@ -33,7 +33,7 @@ class GradioPipeline(LivePortraitPipeline):
         flag_relative_input,
         flag_do_crop_input,
         flag_remap_input,
-        ):
+    ):
         """ for video driven potrait animation
         """
         if input_image_path is not None and input_video_path is not None:
@@ -91,7 +91,7 @@ class GradioPipeline(LivePortraitPipeline):
             # gr.Info("Upload successfully!", duration=2)
             inference_cfg = self.live_portrait_wrapper.cfg
             ######## process source portrait ########
-            img_rgb = load_img_online(input_image, mode='rgb', max_dim=1280, n=16)
+            img_rgb = load_img_online(input_image, mode='rgb', max_dim=1280, n=1) # n=1 means do not trim the pixels
             log(f"Load source image from {input_image}.")
             crop_info = self.cropper.crop_single_image(img_rgb)
             if flag_do_crop:
@@ -99,7 +99,7 @@ class GradioPipeline(LivePortraitPipeline):
             else:
                 I_s = self.live_portrait_wrapper.prepare_source(img_rgb)
             x_s_info = self.live_portrait_wrapper.get_kp_info(I_s)
-            R_s = get_rotation_matrix(x_s_info['pitch'], x_s_info['yaw'], x_s_info['roll'])
+            # R_s = get_rotation_matrix(x_s_info['pitch'], x_s_info['yaw'], x_s_info['roll'])
             ############################################
             f_s_user = self.live_portrait_wrapper.extract_feature_3d(I_s)
             x_s_user = self.live_portrait_wrapper.transform_keypoint(x_s_info)

src/live_portrait_pipeline.py

@@ -18,7 +18,7 @@ from .config.inference_config import InferenceConfig
 from .config.crop_config import CropConfig
 from .utils.cropper import Cropper
 from .utils.camera import get_rotation_matrix
-from .utils.video import images2video, concat_frames
+from .utils.video import images2video, concat_frames, get_fps
 from .utils.crop import _transform_img, prepare_paste_back, paste_back
 from .utils.retargeting_utils import calc_lip_close_ratio
 from .utils.io import load_image_rgb, load_driving_info, resize_to_limit
@@ -67,8 +67,12 @@ class LivePortraitPipeline(object):
         ############################################
 
         ######## process driving info ########
+        output_fps = 30 # default fps
         if is_video(args.driving_info):
             log(f"Load from video file (mp4 mov avi etc...): {args.driving_info}")
+            output_fps = int(get_fps(args.driving_info))
+            log(f'The FPS of {args.driving_info} is: {output_fps}')
+
             # TODO: 这里track一下驱动视频 -> 构建模板
             driving_rgb_lst = load_driving_info(args.driving_info)
             driving_rgb_lst_256 = [cv2.resize(_, (256, 256)) for _ in driving_rgb_lst]
@@ -177,13 +181,13 @@ class LivePortraitPipeline(object):
             frames_concatenated = concat_frames(I_p_lst, driving_rgb_lst, img_crop_256x256)
             # save (driving frames, source image, drived frames) result
             wfp_concat = osp.join(args.output_dir, f'{basename(args.source_image)}--{basename(args.driving_info)}_concat.mp4')
-            images2video(frames_concatenated, wfp=wfp_concat)
+            images2video(frames_concatenated, wfp=wfp_concat, fps=output_fps)
 
         # save drived result
         wfp = osp.join(args.output_dir, f'{basename(args.source_image)}--{basename(args.driving_info)}.mp4')
         if inference_cfg.flag_pasteback:
-            images2video(I_p_paste_lst, wfp=wfp)
+            images2video(I_p_paste_lst, wfp=wfp, fps=output_fps)
         else:
-            images2video(I_p_lst, wfp=wfp)
+            images2video(I_p_lst, wfp=wfp, fps=output_fps)
 
         return wfp, wfp_concat

src/utils/video.py

@@ -12,6 +12,7 @@ import cv2
 
 from rich.progress import track
 from .helper import prefix
+from .rprint import rlog as log
 from .rprint import rprint as print
 
 
@@ -131,9 +132,14 @@ def change_video_fps(input_file, output_file, fps=20, codec='libx264', crf=5):
     exec_cmd(cmd)
 
 
-def get_fps(filepath):
-    import ffmpeg
-    probe = ffmpeg.probe(filepath)
-    video_stream = next((stream for stream in probe['streams'] if stream['codec_type'] == 'video'), None)
-    fps = eval(video_stream['avg_frame_rate'])
+def get_fps(filepath, default_fps=25):
+    try:
+        fps = cv2.VideoCapture(filepath).get(cv2.CAP_PROP_FPS)
+
+        if fps in (0, None):
+            fps = default_fps
+    except Exception as e:
+        log(e)
+        fps = default_fps
+
     return fps