init

app.py

@@ -0,0 +1,171 @@
+"""
+Thanks to nateraw for making this scape happen! 
+This code has been mostly taken from https://huggingface.co/spaces/nateraw/animegan-v2-for-videos/tree/main
+"""
+import os
+os.system("wget https://github.com/Sxela/ArcaneGAN/releases/download/v0.3/ArcaneGANv0.3.jit")
+
+import sys
+from subprocess import call
+def run_cmd(command):
+    try:
+        print(command)
+        call(command, shell=True)
+    except KeyboardInterrupt:
+        print("Process interrupted")
+        sys.exit(1)
+
+print("⬇️ Installing latest gradio==2.4.7b9")
+run_cmd("pip install --upgrade pip")
+run_cmd('pip install gradio==2.4.7b9')
+
+import gc
+import math
+
+
+import gradio as gr
+import numpy as np
+import torch
+from encoded_video import EncodedVideo, write_video
+from PIL import Image
+from torchvision.transforms.functional import center_crop, to_tensor
+
+
+
+
+print("🧠 Loading Model...")
+model = torch.jit.load('ArcaneGANv0.3.jit').cuda().eval().half()
+
+# This function is taken from pytorchvideo!
+def uniform_temporal_subsample(x: torch.Tensor, num_samples: int, temporal_dim: int = -3) -> torch.Tensor:
+    """
+    Uniformly subsamples num_samples indices from the temporal dimension of the video.
+    When num_samples is larger than the size of temporal dimension of the video, it
+    will sample frames based on nearest neighbor interpolation.
+    Args:
+        x (torch.Tensor): A video tensor with dimension larger than one with torch
+            tensor type includes int, long, float, complex, etc.
+        num_samples (int): The number of equispaced samples to be selected
+        temporal_dim (int): dimension of temporal to perform temporal subsample.
+    Returns:
+        An x-like Tensor with subsampled temporal dimension.
+    """
+    t = x.shape[temporal_dim]
+    assert num_samples > 0 and t > 0
+    # Sample by nearest neighbor interpolation if num_samples > t.
+    indices = torch.linspace(0, t - 1, num_samples)
+    indices = torch.clamp(indices, 0, t - 1).long()
+    return torch.index_select(x, temporal_dim, indices)
+
+
+# This function is taken from pytorchvideo!
+def short_side_scale(
+    x: torch.Tensor,
+    size: int,
+    interpolation: str = "bilinear",
+) -> torch.Tensor:
+    """
+    Determines the shorter spatial dim of the video (i.e. width or height) and scales
+    it to the given size. To maintain aspect ratio, the longer side is then scaled
+    accordingly.
+    Args:
+        x (torch.Tensor): A video tensor of shape (C, T, H, W) and type torch.float32.
+        size (int): The size the shorter side is scaled to.
+        interpolation (str): Algorithm used for upsampling,
+            options: nearest' | 'linear' | 'bilinear' | 'bicubic' | 'trilinear' | 'area'
+    Returns:
+        An x-like Tensor with scaled spatial dims.
+    """
+    assert len(x.shape) == 4
+    assert x.dtype == torch.float32
+    c, t, h, w = x.shape
+    if w < h:
+        new_h = int(math.floor((float(h) / w) * size))
+        new_w = size
+    else:
+        new_h = size
+        new_w = int(math.floor((float(w) / h) * size))
+
+    return torch.nn.functional.interpolate(x, size=(new_h, new_w), mode=interpolation, align_corners=False)
+
+means = [0.485, 0.456, 0.406]
+stds = [0.229, 0.224, 0.225]
+
+from torchvision import transforms
+norm = transforms.Normalize(means,stds)
+
+norms = torch.tensor(means)[None,:,None,None].cuda()
+stds = torch.tensor(stds)[None,:,None,None].cuda()
+
+def inference_step(vid, start_sec, duration, out_fps):
+    clip = vid.get_clip(start_sec, start_sec + duration)
+    video_arr = torch.from_numpy(clip['video']).permute(3, 0, 1, 2)
+    audio_arr = np.expand_dims(clip['audio'], 0)
+    audio_fps = None if not vid._has_audio else vid._container.streams.audio[0].sample_rate
+
+    x = uniform_temporal_subsample(video_arr,  duration * out_fps)
+    x = center_crop(short_side_scale(x, 512), 512)
+    x /= 255.
+    x = x.permute(1, 0, 2, 3)
+    x = norm(x)
+
+    with torch.no_grad():
+        output = model(x.to('cuda').half())
+        output = (output * stds + norms).clip(0, 1) * 255.
+
+        output_video = output.permute(0, 2, 3, 1).float().detach().cpu().numpy()
+    
+    return output_video, audio_arr, out_fps, audio_fps
+
+
+def predict_fn(filepath, start_sec, duration, out_fps):
+    # out_fps=12
+    vid = EncodedVideo.from_path(filepath)
+    for i in range(duration):
+        video, audio, fps, audio_fps = inference_step(
+            vid = vid,
+            start_sec = i + start_sec,
+            duration = 1,
+            out_fps = out_fps
+        )
+        gc.collect()
+        if i == 0:
+            video_all = video
+            audio_all = audio
+        else:
+            video_all = np.concatenate((video_all, video))
+            audio_all = np.hstack((audio_all, audio))
+
+    write_video(
+        'out.mp4',
+        video_all,
+        fps=fps,
+        audio_array=audio_all,
+        audio_fps=audio_fps,
+        audio_codec='aac'
+    )
+
+    del video_all
+    del audio_all
+    
+    return 'out.mp4'
+
+
+title = "ArcaneGAN"
+description = "Gradio demo for ArcaneGAN, video to Arcane style. To use it, simply upload your video, or click one of the examples to load them. Follow <a href='https://twitter.com/devdef' target='_blank'>Alex Spirin</a> for more info and updates."
+article = "<div style='text-align: center;'>ArcaneGan by <a href='https://twitter.com/devdef' target='_blank'>Alex Spirin</a> | <a href='https://github.com/Sxela/ArcaneGAN' target='_blank'>Github Repo</a> | <center><img src='https://visitor-badge.glitch.me/badge?page_id=sxela_arcanegan_video_hf' alt='visitor badge'></center></div>"
+
+
+gr.Interface(
+    predict_fn,
+    inputs=[gr.inputs.Video(), gr.inputs.Slider(minimum=0, maximum=300, step=1, default=0), gr.inputs.Slider(minimum=1, maximum=10, step=1, default=2), gr.inputs.Slider(minimum=12, maximum=30, step=6, default=24)],
+    outputs=gr.outputs.Video(),
+    title='ArcaneGAN On Videos',
+    description="Applying ArcaneGAN to frame from video clips",
+    article = article,
+    enable_queue=True,
+    examples=[
+        ['obama.webm', 23, 10, 30],
+    ],
+    allow_flagging=False
+).launch()

requirements.txt

@@ -0,0 +1,9 @@
+gradio
+torch
+torchvision
+Pillow
+gdown
+numpy
+scipy
+opencv-python-headless
+encoded-video