Spaces:
Runtime error
Runtime error
| from diffusers import StableDiffusionPipeline, DPMSolverMultistepScheduler | |
| import gradio as gr | |
| from PIL import Image | |
| import cv2 | |
| import os, random, gc | |
| import numpy as np | |
| from transformers import pipeline | |
| import PIL.Image | |
| from diffusers.utils import load_image, export_to_video | |
| from accelerate import Accelerator | |
| from diffusers import StableDiffusionXLControlNetImg2ImgPipeline, ControlNetModel, EulerDiscreteScheduler | |
| import torch | |
| from moviepy.video.fx.all import crop | |
| from diffusers.utils import export_to_gif | |
| import mediapy | |
| from image_tools.sizes import resize_and_crop | |
| from moviepy.editor import * | |
| from pathlib import Path | |
| from typing import Optional, List | |
| from tqdm import tqdm | |
| import supervision as sv | |
| accelerator = Accelerator(cpu=True) | |
| controlnet = accelerator.prepare(ControlNetModel.from_pretrained("diffusers/controlnet-canny-sdxl-1.0", torch_dtype=torch.bfloat16, use_safetensors=True, variant="fp16", safety_checker=None)) | |
| pipe = accelerator.prepare(StableDiffusionXLControlNetImg2ImgPipeline.from_pretrained("stabilityai/sdxl-turbo", controlnet=controlnet, torch_dtype=torch.bfloat16, use_safetensors=True, variant="fp16", safety_checker=None)) | |
| pipe.unet.to(memory_format=torch.channels_last) | |
| pipe.scheduler = accelerator.prepare(EulerDiscreteScheduler.from_config(pipe.scheduler.config)) | |
| pipe = accelerator.prepare(pipe.to("cpu")) | |
| def plex(fpath, text, neg_prompt, one, two, three, four, five): | |
| gc.collect() | |
| prompt = text | |
| video = './video.mp4' | |
| orvid = './orvid.mp4' | |
| canvid = './canvid.mp4' | |
| frames = [] | |
| canframes = [] | |
| orframes = [] | |
| fin_frames = [] | |
| max_frames=0 | |
| cap = cv2.VideoCapture(fpath) | |
| clip = VideoFileClip(fpath) | |
| height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) | |
| width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) | |
| fps = cap.get(cv2.CAP_PROP_FPS) | |
| aspect = width / height | |
| if aspect == 1 and height >= 512: | |
| nwidth = 512 | |
| nheight = 512 | |
| prep = clip.resize(height=nheight) | |
| left = 0 | |
| top = 0 | |
| right = 512 | |
| bottom = 512 | |
| if aspect > 1 and height >= 512: | |
| nheight = 512 | |
| nwidth = int(nheight * aspect) | |
| prep = clip.resize(height=nheight) | |
| left = (nwidth - width) / 2 | |
| top = 0 | |
| right = (nwidth + width) / 2 | |
| bottom = nheight | |
| if aspect < 1 and width >= 512: | |
| nwidth = 512 | |
| nheight = int(nwidth / aspect) | |
| prep = clip.resize(height=nheight) | |
| left = 0 | |
| top = (height - nheight) / 2 | |
| right = nwidth | |
| bottom = (height + nheight) / 2 | |
| if aspect < 1 and width < 512: | |
| return None | |
| if aspect > 1 and height < 512: | |
| return None | |
| closer = crop(clip, x1=left, y1=top, x2=right, y2=bottom) | |
| if fps > 10: | |
| closer.write_videofile('./video.mp4', fps=10) | |
| fps = 10 | |
| else: | |
| closer.write_videofile('./video.mp4', fps=fps) | |
| fps = fps | |
| max_frames = int(fps * 2) | |
| for frame in tqdm(sv.get_video_frames_generator(source_path=video,)): | |
| frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) | |
| cap.release() | |
| cv2.destroyAllWindows() | |
| ncap = cv2.VideoCapture(video) | |
| total_frames = int(ncap.get(cv2.CAP_PROP_FRAME_COUNT)) | |
| if total_frames <= 0: | |
| return None | |
| b = 0 | |
| if total_frames > max_frames: | |
| max_frames = int(max_frames) | |
| if total_frames < max_frames: | |
| max_frames = int(total_frames) | |
| for b in range(int(max_frames)): | |
| frame = frames[b] | |
| original = load_image(Image.fromarray(frame)) | |
| original.save('./image.png', 'PNG') | |
| original = original.resize((512, 512)) | |
| original = original.convert("RGB") | |
| original.save('./image.png', 'PNG') | |
| orframes.append(original) | |
| cannyimage = np.array(original) | |
| cannyimage = cv2.Canny(cannyimage, 100, 200) | |
| cannyimage = cannyimage[:, :, None] | |
| cannyimage = np.concatenate([cannyimage, cannyimage, cannyimage], axis=2) | |
| cannyimage = Image.fromarray(cannyimage) | |
| canframes.append(cannyimage) | |
| generator = torch.Generator(device="cpu").manual_seed(five) | |
| imoge = pipe(prompt=prompt,image=[original],control_image=[cannyimage],guidance_scale=four,num_inference_steps=one,generator=generator,strength=two,negative_prompt=neg_prompt,controlnet_conditioning_scale=three,width=512,height=512) | |
| fin_frames.append(imoge.images[0]) | |
| b += 1 | |
| ncap.release() | |
| cv2.destroyAllWindows() | |
| export_to_video(fin_frames, video, fps=fps) | |
| export_to_video(orframes, orvid, fps=fps) | |
| export_to_video(canframes, canvid, fps=fps) | |
| return video, canvid, orvid | |
| iface = gr.Interface(fn=plex, inputs=[gr.File(label="Your video",interactive=True),gr.Textbox(label="prompt"),gr.Textbox(label="neg prompt"), gr.Slider(label="num inference steps", minimum=2, step=1, maximum=5, value=2), gr.Slider(label="prompt strength", minimum=0.5, step=0.5, maximum=1.0, value=0.5), gr.Slider(label="controlnet scale", minimum=0.01, step=0.01, maximum=0.99, value=0.80), gr.Slider(label="Guidance scale", minimum=0.0, step=0.1, maximum=9.9, value=0.0), gr.Slider(label="Manual seed", minimum=0, step=32, maximum=4836928, value=0)], outputs=[gr.Video(label="final"), gr.Video(label="canny vid"), gr.Video(label="orig")],description="Running on cpu, very slow! by JoPmt.") | |
| iface.queue(max_size=1,api_open=False) | |
| iface.launch(max_threads=1) |