Initial commit

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.pdf filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,164 @@
+import gradio as gr
+import cv2
+import numpy as np
+import os
+import torch
+import torch.nn.functional as F
+from torchvision.transforms import Compose
+import tempfile
+
+from depth_anything.dpt import DepthAnything
+from depth_anything.util.transform import Resize, NormalizeImage, PrepareForNet
+
+def make_video(video_path, outdir='./vis_video_depth',encoder='vitl'):
+    # Define path for temporary processed frames
+    temp_frame_dir = tempfile.mkdtemp()
+    
+    margin_width = 50
+
+    DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
+    
+    depth_anything = DepthAnything.from_pretrained('LiheYoung/depth_anything_{}14'.format(encoder)).to(DEVICE).eval()
+    
+    total_params = sum(param.numel() for param in depth_anything.parameters())
+    print('Total parameters: {:.2f}M'.format(total_params / 1e6))
+    
+    transform = Compose([
+        Resize(
+            width=518,
+            height=518,
+            resize_target=False,
+            keep_aspect_ratio=True,
+            ensure_multiple_of=14,
+            resize_method='lower_bound',
+            image_interpolation_method=cv2.INTER_CUBIC,
+        ),
+        NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+        PrepareForNet(),
+    ])
+
+    if os.path.isfile(video_path):
+        if video_path.endswith('txt'):
+            with open(video_path, 'r') as f:
+                lines = f.read().splitlines()
+        else:
+            filenames = [video_path]
+    else:
+        filenames = os.listdir(video_path)
+        filenames = [os.path.join(video_path, filename) for filename in filenames if not filename.startswith('.')]
+        filenames.sort()
+    
+    # os.makedirs(outdir, exist_ok=True)
+    
+    for k, filename in enumerate(filenames):
+        print('Progress {:}/{:},'.format(k+1, len(filenames)), 'Processing', filename)
+        
+        raw_video = cv2.VideoCapture(filename)
+        frame_width, frame_height = int(raw_video.get(cv2.CAP_PROP_FRAME_WIDTH)), int(raw_video.get(cv2.CAP_PROP_FRAME_HEIGHT))
+        frame_rate = int(raw_video.get(cv2.CAP_PROP_FPS))
+        output_width = frame_width * 2 + margin_width
+        
+        filename = os.path.basename(filename)
+        # output_path = os.path.join(outdir, filename[:filename.rfind('.')] + '_video_depth.mp4')
+        with tempfile.NamedTemporaryFile(delete=False, suffix='.mp4') as tmpfile:
+            output_path = tmpfile.name
+        #out = cv2.VideoWriter(output_path, cv2.VideoWriter_fourcc(*"avc1"), frame_rate, (output_width, frame_height))
+        fourcc = cv2.VideoWriter_fourcc(*'mp4v')
+        out = cv2.VideoWriter(output_path, fourcc, frame_rate, (output_width, frame_height))
+        # count=0
+        while raw_video.isOpened():
+            ret, raw_frame = raw_video.read()
+            if not ret:
+                break
+            
+            frame = cv2.cvtColor(raw_frame, cv2.COLOR_BGR2RGB) / 255.0
+            
+            frame = transform({'image': frame})['image']
+            frame = torch.from_numpy(frame).unsqueeze(0).to(DEVICE)
+            
+            with torch.no_grad():
+                depth = depth_anything(frame)
+
+            depth = F.interpolate(depth[None], (frame_height, frame_width), mode='bilinear', align_corners=False)[0, 0]
+            depth = (depth - depth.min()) / (depth.max() - depth.min()) * 255.0
+            
+            depth = depth.cpu().numpy().astype(np.uint8)
+            depth_color = cv2.applyColorMap(depth, cv2.COLORMAP_INFERNO)
+            
+            split_region = np.ones((frame_height, margin_width, 3), dtype=np.uint8) * 255
+            combined_frame = cv2.hconcat([raw_frame, split_region, depth_color])
+            
+            # out.write(combined_frame)
+            # frame_path = os.path.join(temp_frame_dir, f"frame_{count:05d}.png")
+            # cv2.imwrite(frame_path, combined_frame)
+            out.write(combined_frame)
+            # count += 1
+        
+        raw_video.release()
+        out.release()
+        return output_path
+
+css = """
+#img-display-container {
+    max-height: 100vh;
+    }
+#img-display-input {
+    max-height: 80vh;
+    }
+#img-display-output {
+    max-height: 80vh;
+    }
+"""
+DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
+model = DepthAnything.from_pretrained('LiheYoung/depth_anything_vitl14').to(DEVICE).eval()
+
+title = "# Depth Anything Video Demo"
+description = """Depth Anything on full video files.
+
+Please refer to our [paper](https://arxiv.org/abs/2401.10891), [project page](https://depth-anything.github.io), or [github](https://github.com/LiheYoung/Depth-Anything) for more details."""
+
+transform = Compose([
+        Resize(
+            width=518,
+            height=518,
+            resize_target=False,
+            keep_aspect_ratio=True,
+            ensure_multiple_of=14,
+            resize_method='lower_bound',
+            image_interpolation_method=cv2.INTER_CUBIC,
+        ),
+        NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+        PrepareForNet(),
+])
+
+@torch.no_grad()
+def predict_depth(model, image):
+    return model(image)
+
+with gr.Blocks(css=css) as demo:
+    gr.Markdown(title)
+    gr.Markdown(description)
+    gr.Markdown("### Video Depth Prediction demo")
+
+    with gr.Row():
+        input_video = gr.Video(label="Input Video")
+    submit = gr.Button("Submit")
+    processed_video = gr.Video(label="Processed Video")
+
+    def on_submit(uploaded_video):
+                
+        # Process the video and get the path of the output video
+        output_video_path = make_video(uploaded_video)
+
+        return output_video_path
+
+    submit.click(on_submit, inputs=[input_video], outputs=processed_video)
+
+    example_files = os.listdir('assets/examples_video')
+    example_files.sort()
+    example_files = [os.path.join('assets/examples_video', filename) for filename in example_files]
+    examples = gr.Examples(examples=example_files, inputs=[input_video], outputs=processed_video, fn=on_submit, cache_examples=False)
+    
+
+if __name__ == '__main__':
+    demo.queue().launch()

assets/examples_video/davis_dolphins.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da2bdf883de86f3ad1f7ec58e34f50cd8dc1bbde8288e23a635a7396ba1af13d
+size 468524

assets/examples_video/davis_rollercoaster.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:291361c800b83ead49f50302ffc82f6ecd5205391934cc8354946b4b93e8cbb4
+size 596021

assets/examples_video/davis_seasnake.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:247f92487bc7a14bf2364847f83e23c7c99addf28abaa043bb353edb6531cead
+size 4010306

assets/paper.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e554e412ffc6e9e6edddc963baa2211692860ab0efa92d74bf7d09b18d2c597
+size 4549655

assets/video_edit/demo1_midas.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11c668214e74ff067cadd9f5beaa8b103360398f5c689c4a4db6c74b451a6963
+size 187513

assets/video_edit/demo1_ours.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57e88c7610076ee422f53c112907339d5a87338cc9186453eb5bfaf6ed3a9257
+size 431370

assets/video_edit/demo1_video.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dfcb14cb7cec14c6b8198a6a455609dc5fac26c42628867e0d4412d53bfa0af7
+size 174791

assets/video_edit/demo2_midas.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:833fd0e7b41f712073ff48373139b1433a0e61b0221a44971a25d57b34a92078
+size 145623

assets/video_edit/demo2_ours.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c49d36aab8eeac806613a80c870fa6d3b62694ad08da0550f2e7d6d1b29553fd
+size 222693

assets/video_edit/demo2_video.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb31e593a2a0973e37a5e004a25e7b5fde0e6e9234e283cc404ae7b9805b45ce
+size 112277

requirements.txt

@@ -0,0 +1,6 @@
+gradio_imageslider
+gradio==4.14.0
+torch
+torchvision
+opencv-python
+huggingface_hub