add mmpose

app.py

@@ -171,4 +171,4 @@ with gr.Blocks(css='style.css') as demo:
     run_button.click(fn=pipe.run, inputs=inputs, outputs=result)
 
 demo.queue().launch()
-# demo.queue().launch(share=False, server_name='0.0.0.0', server_port=7890)
+# demo.queue().launch(share=False, server_name='0.0.0.0', server_port=80)

inference_followyourpose.py

@@ -5,6 +5,7 @@ import copy
 import gradio as gr
 from transformers import AutoTokenizer, CLIPTextModel
 from huggingface_hub import snapshot_download
+from inference_mmpose import *
 import sys
 sys.path.append('FollowYourPose')
 
@@ -21,10 +22,11 @@ class merge_config_then_run():
         self.vae = None
         self.unet = None
         self.download_model()
+        self.mmpose = gr.Interface.load(name="spaces/fffiloni/mmpose-estimation")
     
     def download_model(self):
         REPO_ID = 'YueMafighting/FollowYourPose_v1'
-        snapshot_download(repo_id=REPO_ID, local_dir='./FollowYourPose/checkpoints/', local_dir_use_symlinks=False)    
+        snapshot_download(repo_id=REPO_ID, local_dir='./FollowYourPose/checkpoints/', local_dir_use_symlinks=False)     
  
             
     def run(
@@ -42,7 +44,7 @@ class merge_config_then_run():
         top_crop=0,
         bottom_crop=0,
     ):
-        
+        infer_skeleton(self.mmpose,data_path)
         default_edit_config='./FollowYourPose/configs/pose_sample.yaml'
         Omegadict_default_edit_config = OmegaConf.load(default_edit_config)
         
@@ -73,7 +75,7 @@ class merge_config_then_run():
         # ddim config
         config_now['validation_data']['guidance_scale'] = guidance_scale
         config_now['validation_data']['num_inference_steps'] = num_steps
-        config_now['skeleton_path'] = data_path
+        config_now['skeleton_path'] = './mmpose_result.mp4'
         
         save_path = test(**config_now)
         mp4_path = save_path.replace('_0.gif', '_0_0_0.mp4')

inference_mmpose.py

@@ -0,0 +1,104 @@
+import gradio as gr
+
+import os
+import cv2
+import numpy as np
+from PIL import Image
+from moviepy.editor import *
+
+import sys
+sys.path.append('FollowYourPose')
+
+def get_frames(video_in):
+    frames = []
+    #resize the video
+    clip = VideoFileClip(video_in)
+    start_frame = 0  # 起始帧数
+    end_frame = 50  # 结束帧数
+    
+    
+    #check fps
+    if clip.fps > 30:
+        print("vide rate is over 30, resetting to 30")
+        clip_resized = clip.resize(height=512)
+        clip_resized = clip_resized.subclip(start_frame / clip_resized.fps, end_frame / clip_resized.fps) # subclip 2 seconds
+        clip_resized.write_videofile("./video_resized.mp4", fps=30)
+    else:
+        print("video rate is OK")
+        clip_resized = clip.resize(height=512)
+        clip_resized = clip_resized.subclip(start_frame / clip.fps, end_frame / clip.fps) # subclip 5 seconds
+        clip_resized.write_videofile("./video_resized.mp4", fps=clip.fps)
+    
+    print("video resized to 512 height")
+    
+    # Opens the Video file with CV2
+    cap= cv2.VideoCapture("./video_resized.mp4")
+    
+    fps = cap.get(cv2.CAP_PROP_FPS)
+    print("video fps: " + str(fps))
+    i=0
+    while(cap.isOpened()):
+        ret, frame = cap.read()
+        if ret == False:
+            break
+        cv2.imwrite('./raw_frames/kang'+str(i)+'.jpg',frame)
+        frames.append('./raw_frames/kang'+str(i)+'.jpg')
+        i+=1
+    
+    cap.release()
+    cv2.destroyAllWindows()
+    print("broke the video into frames")
+    
+    return frames, fps
+
+def get_mmpose_filter(mmpose, i):
+    #image = Image.open(i)
+    
+    #image = np.array(image)
+    image = mmpose(i, fn_index=0)[1]
+    image = Image.open(image)
+    #image = Image.fromarray(image)
+    image.save("./mmpose_frames/mmpose_frame_" + str(i).split('/')[-1][:-4] + ".jpeg")
+    return "./mmpose_frames/mmpose_frame_" + str(i).split('/')[-1][:-4] + ".jpeg"
+
+def create_video(frames, fps, type):
+    print("building video result")
+    clip = ImageSequenceClip(frames, fps=fps)
+    clip.write_videofile(type + "_result.mp4", fps=fps)
+    
+    return type + "_result.mp4"
+
+def convertG2V(imported_gif):
+    clip = VideoFileClip(imported_gif.name)
+    clip.write_videofile("my_gif_video.mp4")
+    return "my_gif_video.mp4"
+    
+def infer_skeleton(mmpose, video_in):
+    
+    
+    # 1. break video into frames and get FPS
+    
+    break_vid = get_frames(video_in)
+    frames_list= break_vid[0]
+    fps = break_vid[1]
+    #n_frame = int(trim_value*fps)
+    n_frame = len(frames_list)
+    
+    if n_frame >= len(frames_list):
+        print("video is shorter than the cut value")
+        n_frame = len(frames_list)
+    
+    # 2. prepare frames result arrays
+    result_frames = []
+    print("set stop frames to: " + str(n_frame))
+    
+    for i in frames_list[0:int(n_frame)]:
+        mmpose_frame = get_mmpose_filter(mmpose, i)
+        result_frames.append(mmpose_frame)
+        print("frame " + i + "/" + str(n_frame) + ": done;")
+
+    
+    final_vid = create_video(result_frames, fps, "mmpose")
+    files = [final_vid]
+    
+    return final_vid, files

requirements.txt

@@ -110,3 +110,6 @@ Werkzeug==2.2.3
 xformers==0.0.16
 yarl==1.8.2
 zipp==3.14.0
+opencv-python
+moviepy
+controlnet_aux