Spaces:
Paused
Paused
File size: 7,980 Bytes
f9da83e 1dabd9c f9da83e 0beb2f8 f9da83e 0beb2f8 9fae970 f9da83e bda1475 9fae970 f9da83e 5605108 f9da83e 34a8e55 f9da83e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 |
import gradio as gr
import os
import cv2
import numpy as np
from moviepy.editor import *
#from share_btn import community_icon_html, loading_icon_html, share_js
os.system("python -m pip install git+https://github.com/MaureenZOU/detectron2-xyz.git")
import torch
import argparse
from xdecoder.BaseModel import BaseModel
from xdecoder import build_model
from utils.distributed import init_distributed
from utils.arguments import load_opt_from_config_files
from tasks import *
def parse_option():
parser = argparse.ArgumentParser('X-Decoder All-in-One Demo', add_help=False)
parser.add_argument('--conf_files', default="configs/xdecoder/svlp_focalt_lang.yaml", metavar="FILE", help='path to config file', )
args = parser.parse_args()
return args
'''
build args
'''
args = parse_option()
opt = load_opt_from_config_files(args.conf_files)
opt = init_distributed(opt)
# META DATA
pretrained_pth_last = os.path.join("xdecoder_focalt_last.pt")
pretrained_pth_novg = os.path.join("xdecoder_focalt_last_novg.pt")
if not os.path.exists(pretrained_pth_last):
os.system("wget {}".format("https://projects4jw.blob.core.windows.net/x-decoder/release/xdecoder_focalt_last.pt"))
if not os.path.exists(pretrained_pth_novg):
os.system("wget {}".format("https://projects4jw.blob.core.windows.net/x-decoder/release/xdecoder_focalt_last_novg.pt"))
'''
build model
'''
model_last = BaseModel(opt, build_model(opt)).from_pretrained(pretrained_pth_last).eval().cuda()
with torch.no_grad():
model_last.model.sem_seg_head.predictor.lang_encoder.get_text_embeddings(["background", "background"], is_eval=True)
'''
inference model
'''
@torch.no_grad()
def xdecoder(image, instruction, *args, **kwargs):
image = image.convert("RGB")
with torch.autocast(device_type='cuda', dtype=torch.float16):
return referring_inpainting_gpt3(model_last, image, instruction, *args, **kwargs)
#xdecoder = gr.Interface.load(name="spaces/xdecoder/Instruct-X-Decoder")
def get_frames(video_in):
frames = []
#resize the video
clip = VideoFileClip(video_in)
#check fps
if clip.fps > 30:
print("vide rate is over 30, resetting to 30")
clip_resized = clip.resize(height=512)
clip_resized.write_videofile("video_resized.mp4", fps=30)
else:
print("video rate is OK")
clip_resized = clip.resize(height=512)
clip_resized.write_videofile("video_resized.mp4", fps=clip.fps)
print("video resized to 512 height")
# Opens the Video file with CV2
cap= cv2.VideoCapture("video_resized.mp4")
fps = cap.get(cv2.CAP_PROP_FPS)
print("video fps: " + str(fps))
i=0
while(cap.isOpened()):
ret, frame = cap.read()
if ret == False:
break
cv2.imwrite('kang'+str(i)+'.jpg',frame)
frames.append('kang'+str(i)+'.jpg')
i+=1
cap.release()
cv2.destroyAllWindows()
print("broke the video into frames")
return frames, fps
def create_video(frames, fps):
print("building video result")
clip = ImageSequenceClip(frames, fps=fps)
clip.write_videofile("movie.mp4", fps=fps)
return 'movie.mp4'
def infer(prompt,video_in, trim_value):
print(prompt)
break_vid = get_frames(video_in)
frames_list= break_vid[0]
fps = break_vid[1]
n_frame = int(trim_value*fps)
if n_frame >= len(frames_list):
print("video is shorter than the cut value")
n_frame = len(frames_list)
result_frames = []
print("set stop frames to: " + str(n_frame))
for i in frames_list[0:int(n_frame)]:
#xdecoder_img = xdecoder(i, prompt, fn_index=0)
xdecoder_img = xdecoder(i, prompt)
#res_image = xdecoder_img[0]
#rgb_im = images[0].convert("RGB")
# exporting the image
#res_image.save(f"result_img-{i}.jpg")
result_frames.append(xdecoder_img)
print("frame " + i + "/" + str(n_frame) + ": done;")
print(result_frames)
final_vid = create_video(result_frames, fps)
print("finished !")
#return final_vid, gr.Group.update(visible=True)
return final_vid
title = """
<div style="text-align: center; max-width: 700px; margin: 0 auto;">
<div
style="
display: inline-flex;
align-items: center;
gap: 0.8rem;
font-size: 1.75rem;
"
>
<h1 style="font-weight: 900; margin-bottom: 7px; margin-top: 5px;">
Instruct X-Decoder Video
</h1>
</div>
<p style="margin-bottom: 10px; font-size: 94%">
Apply Instruct X-Decoder Diffusion to a video
</p>
</div>
"""
article = """
<div class="footer">
<p>
Examples by <a href="https://twitter.com/CitizenPlain" target="_blank">Nathan Shipley</a> •
Follow <a href="https://twitter.com/fffiloni" target="_blank">Sylvain Filoni</a> for future updates 🤗
</p>
</div>
<div id="may-like-container" style="display: flex;justify-content: center;flex-direction: column;align-items: center;margin-bottom: 30px;">
<p>You may also like: </p>
<div id="may-like-content" style="display:flex;flex-wrap: wrap;align-items:center;height:20px;">
<svg height="20" width="162" style="margin-left:4px;margin-bottom: 6px;">
<a href="https://huggingface.co/spaces/timbrooks/instruct-pix2pix" target="_blank">
<image href="https://img.shields.io/badge/🤗 Spaces-Instruct_Pix2Pix-blue" src="https://img.shields.io/badge/🤗 Spaces-Instruct_Pix2Pix-blue.png" height="20"/>
</a>
</svg>
</div>
</div>
"""
with gr.Blocks(css='style.css') as demo:
with gr.Column(elem_id="col-container"):
gr.HTML(title)
with gr.Row():
with gr.Column():
video_inp = gr.Video(label="Video source", source="upload", type="filepath", elem_id="input-vid")
prompt = gr.Textbox(label="Prompt", placeholder="enter prompt", show_label=False, elem_id="prompt-in")
with gr.Row():
trim_in = gr.Slider(label="Cut video at (s)", minimun=1, maximum=3, step=1, value=1)
with gr.Column():
video_out = gr.Video(label="Pix2pix video result", elem_id="video-output")
gr.HTML("""
<a style="display:inline-block" href="https://huggingface.co/spaces/fffiloni/Pix2Pix-Video?duplicate=true"><img src="https://img.shields.io/badge/-Duplicate%20Space-blue?labelColor=white&style=flat&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAP5JREFUOE+lk7FqAkEURY+ltunEgFXS2sZGIbXfEPdLlnxJyDdYB62sbbUKpLbVNhyYFzbrrA74YJlh9r079973psed0cvUD4A+4HoCjsA85X0Dfn/RBLBgBDxnQPfAEJgBY+A9gALA4tcbamSzS4xq4FOQAJgCDwV2CPKV8tZAJcAjMMkUe1vX+U+SMhfAJEHasQIWmXNN3abzDwHUrgcRGmYcgKe0bxrblHEB4E/pndMazNpSZGcsZdBlYJcEL9Afo75molJyM2FxmPgmgPqlWNLGfwZGG6UiyEvLzHYDmoPkDDiNm9JR9uboiONcBXrpY1qmgs21x1QwyZcpvxt9NS09PlsPAAAAAElFTkSuQmCC&logoWidth=14" alt="Duplicate Space"></a>
work with longer videos / skip the queue:
""", elem_id="duplicate-container")
submit_btn = gr.Button("Generate X-Decoder video")
#with gr.Group(elem_id="share-btn-container", visible=False) as share_group:
# community_icon = gr.HTML(community_icon_html)
# loading_icon = gr.HTML(loading_icon_html)
# share_button = gr.Button("Share to community", elem_id="share-btn")
inputs = [prompt, video_inp, trim_in]
#outputs = [video_out, share_group]
outputs = [video_out]
gr.HTML(article)
submit_btn.click(infer, inputs, outputs)
#share_button.click(None, [], [], _js=share_js)
demo.launch().queue(max_size=12) |