init

.gitignore

@@ -159,4 +159,5 @@ cython_debug/
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
 src/Ev2Hands/outputs
-src/HandSimulator/logs
+src/HandSimulator/logs
+gradio_cached_examples/

app.py

@@ -1,25 +1,160 @@
 import gradio as gr
-import requests
+import os
+import cv2
+import numpy as np
 
+import esim_py
+from infererence import process_events, Ev2Hands
+from settings import OUTPUT_HEIGHT, OUTPUT_WIDTH, REF_PERIOD
 
 
-import gradio as gr
-import os
+os.makedirs("temp", exist_ok=True)
+ev2hands = Ev2Hands()
+
+
+def get_frames(video_in, trim_in):            
+    cap = cv2.VideoCapture(video_in)
+    
+    fps = cap.get(cv2.CAP_PROP_FPS)
+    stop_frame = int(trim_in * fps)
+
+    print("video fps: " + str(fps))
+    
+    frames = []
+    i = 0
+    while(cap.isOpened()):
+        ret, frame = cap.read()
+        if not ret:
+            break
+
+        frame = cv2.resize(frame, (OUTPUT_WIDTH, OUTPUT_HEIGHT))
+        frames.append(frame)
+
+        if i > stop_frame:
+            break
+             
+        i += 1
+    
+    
+    cap.release()
+    
+    return frames, fps
+
+
+
+def infer(video_inp, trim_in, threshold):
+    frames, fps = get_frames(video_inp, trim_in)
+    ts_s = 1 / fps
+    ts_ns = ts_s * 1e9 # convert s to ns
+
+    POS_THRESHOLD = NEG_THRESHOLD = threshold
+
+    esim = esim_py.EventSimulator(POS_THRESHOLD, NEG_THRESHOLD, REF_PERIOD, 1e-4, True)  
+    is_init = False
+
+    event_frame_vid_path = 'temp/event_video.mp4'
+    prediction_vid_path = 'temp/prediction_video.mp4'
+
+    height, width, _ = frames[0].shape
+    event_video = cv2.VideoWriter(event_frame_vid_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (width, height))
+    prediction_video = cv2.VideoWriter(prediction_vid_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (width, height))
+
+    for idx, frame in enumerate(frames):
+        frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+        frame_log = np.log(frame_gray.astype("float32") / 255 + 1e-4)
+    
+        current_ts_ns = idx * ts_ns
+
+        if not is_init:
+            esim.init(frame_log, current_ts_ns)
+            is_init = True
+            continue
+
+        events = esim.generateEventFromCVImage(frame_log, current_ts_ns)
+        data = process_events(events)
+
+        prediction_frame = ev2hands(data)
+        event_frame = data['event_frame'].cpu().numpy().astype(dtype=np.uint8)
+
+
+        event_video.write(event_frame)
+        prediction_video.write(prediction_frame)
+
+    event_video.release()
+    prediction_video.release()
+
+    return event_frame_vid_path, prediction_vid_path
+    
 
+title = """
+    <div style="text-align: center; max-width: 700px; margin: 0 auto;">
+        <div
+        style="
+            display: inline-flex;
+            align-items: center;
+            gap: 0.8rem;
+            font-size: 1.75rem;
+        "
+        >
+        <h1 style="font-weight: 900; margin-bottom: 7px; margin-top: 5px;">
+            Pix2Pix Video
+        </h1>
+        </div>
+        <p style="margin-bottom: 10px; font-size: 94%">
+        Apply Instruct Pix2Pix Diffusion to a video 
+        </p>
+    </div>
+"""
 
-def video_identity(video):
-    print(video)
-    return video
+article = """
+    
+    <div class="footer">
+        <p>
+        Examples by <a href="https://twitter.com/CitizenPlain" target="_blank">Nathan Shipley</a> •&nbsp;
+        Follow <a href="https://twitter.com/fffiloni" target="_blank">Sylvain Filoni</a> for future updates 🤗
+        </p>
+    </div>
+    <div id="may-like-container" style="display: flex;justify-content: center;flex-direction: column;align-items: center;margin-bottom: 30px;">
+        <p>You may also like: </p>
+        <div id="may-like-content" style="display:flex;flex-wrap: wrap;align-items:center;height:20px;">
+            
+            <svg height="20" width="162" style="margin-left:4px;margin-bottom: 6px;">       
+                 <a href="https://huggingface.co/spaces/timbrooks/instruct-pix2pix" target="_blank">
+                    <image href="https://img.shields.io/badge/🤗 Spaces-Instruct_Pix2Pix-blue" src="https://img.shields.io/badge/🤗 Spaces-Instruct_Pix2Pix-blue.png" height="20"/>
+                 </a>
+            </svg>
+            
+        </div>
+    
+    </div>
+    
+"""
 
+with gr.Blocks(css='style.css') as demo:
+    with gr.Column(elem_id="col-container"):
+        gr.HTML(title)
+        with gr.Row():
+            with gr.Column():
+                video_inp = gr.Video(label="Video source", elem_id="input-vid")
+                with gr.Row():
+                    trim_in = gr.Slider(label="Cut video at (s)", minimum=1, maximum=5, step=1, value=1)
+                    threshold = gr.Slider(label="Event Threshold", minimum=0.1, maximum=1, step=0.05, value=0.5)
 
-demo = gr.Interface(video_identity, 
-                    gr.Video(), 
-                    "playable_video", 
-                    examples=[
-                        os.path.join(os.path.dirname(__file__), 
-                                     "example/video.mp4")], 
-                    cache_examples=True)
+            with gr.Column():
+                event_frame_out = gr.Video(label="Event Frame", elem_id="video-output")
+                prediction_out = gr.Video(label="Ev2Hands result", elem_id="video-output")
 
-if __name__ == "__main__":
-    demo.launch(server_name="0.0.0.0", server_port=7860)
+                gr.HTML("""
+                <a style="display:inline-block" href="https://huggingface.co/spaces/fffiloni/Pix2Pix-Video?duplicate=true"><img src="https://img.shields.io/badge/-Duplicate%20Space-blue?labelColor=white&style=flat&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAP5JREFUOE+lk7FqAkEURY+ltunEgFXS2sZGIbXfEPdLlnxJyDdYB62sbbUKpLbVNhyYFzbrrA74YJlh9r079973psed0cvUD4A+4HoCjsA85X0Dfn/RBLBgBDxnQPfAEJgBY+A9gALA4tcbamSzS4xq4FOQAJgCDwV2CPKV8tZAJcAjMMkUe1vX+U+SMhfAJEHasQIWmXNN3abzDwHUrgcRGmYcgKe0bxrblHEB4E/pndMazNpSZGcsZdBlYJcEL9Afo75molJyM2FxmPgmgPqlWNLGfwZGG6UiyEvLzHYDmoPkDDiNm9JR9uboiONcBXrpY1qmgs21x1QwyZcpvxt9NS09PlsPAAAAAElFTkSuQmCC&logoWidth=14" alt="Duplicate Space"></a> 
+                work with longer videos / skip the queue: 
+                """, elem_id="duplicate-container")
 
+                submit_btn = gr.Button("Run Ev2Hands")
+        
+        inputs = [video_inp, trim_in, threshold]
+        outputs = [event_frame_out, prediction_out]        
+        gr.HTML(article)
+    
+    submit_btn.click(infer, inputs, outputs)
+        
+demo.queue(max_size=12).launch(server_name="0.0.0.0", server_port=7860)

demo.py → infererence.py

@@ -138,84 +138,52 @@ def demo(net, device, data):
     return frames
 
 
+class Ev2Hands:
+    def __init__(self) -> None:
+        arg_parser.demo()
+        device = torch.device('cpu')
+        net = TEHNetWrapper(device=device)
+
+        save_path = os.environ['CHECKPOINT_PATH']     
+
+        checkpoint = torch.load(save_path, map_location=device)
+        net.load_state_dict(checkpoint['state_dict'], strict=True)
+
+        renderer = pyrender.OffscreenRenderer(viewport_width=OUTPUT_WIDTH, viewport_height=OUTPUT_HEIGHT)
+
+        scene = pyrender.Scene(ambient_light=(0.3, 0.3, 0.3))
+        light = pyrender.DirectionalLight(color=[1.0, 1.0, 1.0], intensity=0.8)
+        light_pose = np.eye(4)
+        light_pose[:3, 3] = np.array([0, -1, 1])
+        scene.add(light, pose=light_pose)
+        light_pose[:3, 3] = np.array([0, 1, 1])
+        scene.add(light, pose=light_pose)
+        light_pose[:3, 3] = np.array([1, 1, 2])
+        scene.add(light, pose=light_pose)
+        
+        camera = MAIN_CAMERA
+        nc = pyrender.Node(camera=camera, matrix=np.eye(4))
+        scene.add_node(nc)
 
-def main():    
-    arg_parser.demo()
-    os.makedirs('outputs', exist_ok=True)
-
-    device = torch.device('cpu')
-
-    net = TEHNetWrapper(device=device)
-
-    save_path = os.environ['CHECKPOINT_PATH']     
-    batch_size = int(os.environ['BATCH_SIZE'])
-
-    checkpoint = torch.load(save_path, map_location=device)
-    net.load_state_dict(checkpoint['state_dict'], strict=True)
-    
-    renderer = pyrender.OffscreenRenderer(viewport_width=OUTPUT_WIDTH, viewport_height=OUTPUT_HEIGHT)
-
-    scene = pyrender.Scene(ambient_light=(0.3, 0.3, 0.3))
-    light = pyrender.DirectionalLight(color=[1.0, 1.0, 1.0], intensity=0.8)
-    light_pose = np.eye(4)
-    light_pose[:3, 3] = np.array([0, -1, 1])
-    scene.add(light, pose=light_pose)
-    light_pose[:3, 3] = np.array([0, 1, 1])
-    scene.add(light, pose=light_pose)
-    light_pose[:3, 3] = np.array([1, 1, 2])
-    scene.add(light, pose=light_pose)
-    
-    rot = trimesh.transformations.rotation_matrix(np.radians(180), [1, 0, 0])
-    
-    mano_hands = net.hands
-
-    # camera = cv2.VideoCapture(0)
-    input_video_stream = cv2.VideoCapture('video.mp4')  
-
-
-
-    video_fps = 25
-    video = cv2.VideoWriter('outputs/video.mp4', cv2.VideoWriter_fourcc(*'mp4v'), video_fps, (3 * OUTPUT_WIDTH, OUTPUT_HEIGHT))
-    
-
-    POS_THRESHOLD = 0.5
-    NEG_THRESHOLD = 0.5
-    REF_PERIOD = 0.000
-
-    esim = esim_py.EventSimulator(POS_THRESHOLD, NEG_THRESHOLD, REF_PERIOD, 1e-4, True)  
-
-
-    fps = cv2.CAP_PROP_FPS
-    ts_s = 1 / fps
-    ts_ns = ts_s * 1e9 # convert s to ns
-
-    is_init = False
-    idx = 0
-    while True:
-        _, frame_bgr = input_video_stream.read()
-        frame_bgr = cv2.resize(frame_bgr, (OUTPUT_WIDTH, OUTPUT_HEIGHT))
-        frame_gray = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2GRAY)
-        frame_log = np.log(frame_gray.astype("float32") / 255 + 1e-4)
-        height, width = frame_log.shape[:2]
-
-        current_ts_ns = idx * ts_ns
-
-        if not is_init:
-            esim.init(frame_log, current_ts_ns)
-            is_init = True
-            idx += 1
-            
-            continue
-        idx += 1
-
-        events = esim.generateEventFromCVImage(frame_log, current_ts_ns)
-        data = process_events(events)
-
-        event_frame = data['event_frame'].cpu().numpy().astype(dtype=np.uint8)
-
-        cv2.imwrite(f"outputs/event_frame_{idx}.png", event_frame)
 
-        print(idx, event_frame.shape)
+        rot = trimesh.transformations.rotation_matrix(np.radians(180), [1, 0, 0])
+        
+        mano_hands = net.hands
+
+        self.net = net
+        self.device = device
+        self.mano_hands = mano_hands
+        self.rot = rot  
+        self.renderer = renderer
+        self.scene = scene
+
+    def __call__(self, data):
+        net = self.net
+        device = self.device
+        mano_hands = self.mano_hands
+        rot = self.rot
+        renderer = self.renderer
+        scene = self.scene
 
         frame = demo(net=net, device=device, data=data)[0]
         seg_mask = frame['seg_mask']
@@ -231,33 +199,14 @@ def main():
         pred_meshes = trimesh.util.concatenate(pred_meshes)
         pred_meshes.apply_transform(rot)
 
-        camera = MAIN_CAMERA
-
-        nc = pyrender.Node(camera=camera, matrix=np.eye(4))
-        scene.add_node(nc)
 
         mesh_node = pyrender.Node(mesh=pyrender.Mesh.from_trimesh(pred_meshes))
         scene.add_node(mesh_node)            
         pred_rgb, depth = renderer.render(scene)           
         scene.remove_node(mesh_node)
-        scene.remove_node(nc)
 
         pred_rgb = cv2.cvtColor(pred_rgb, cv2.COLOR_RGB2BGR)           
         pred_rgb[pred_rgb == 255] = 0
 
-        img_stack = np.hstack([event_frame, seg_mask, pred_rgb])
-        video.write(img_stack)
-
-        cv2.imshow('image', img_stack)
-        c = cv2.waitKey(1)
-
-        if c == ord('q'):
-            video.release()
-            exit(0)
-        
-    video.release()
-
-
-if __name__ == '__main__':
-    main()
+        return pred_rgb