Spaces:
Runtime error
Runtime error
File size: 5,599 Bytes
c19ca42 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 |
"""
Additional params for Text-to-Video
<https://huggingface.co/docs/diffusers/api/pipelines/text_to_video>
TODO text2video items:
- Video-to-Video upscaling: <https://huggingface.co/cerspense/zeroscope_v2_XL>, <https://huggingface.co/damo-vilab/MS-Vid2Vid-XL>
"""
import gradio as gr
from modules import scripts, processing, shared, images, sd_models, modelloader
MODELS = [
{'name': 'None'},
{'name': 'ModelScope v1.7b', 'path': 'damo-vilab/text-to-video-ms-1.7b', 'params': [16,320,320]},
{'name': 'ZeroScope v1', 'path': 'cerspense/zeroscope_v1_320s', 'params': [16,320,320]},
{'name': 'ZeroScope v1.1', 'path': 'cerspense/zeroscope_v1-1_320s', 'params': [16,320,320]},
{'name': 'ZeroScope v2', 'path': 'cerspense/zeroscope_v2_576w', 'params': [24,576,320]},
{'name': 'ZeroScope v2 Dark', 'path': 'cerspense/zeroscope_v2_dark_30x448x256', 'params': [24,448,256]},
{'name': 'Potat v1', 'path': 'camenduru/potat1', 'params': [24,1024,576]},
]
class Script(scripts.Script):
def title(self):
return 'Text-to-Video'
def show(self, is_img2img):
return not is_img2img if shared.backend == shared.Backend.DIFFUSERS else False
# return signature is array of gradio components
def ui(self, _is_img2img):
def video_type_change(video_type):
return [
gr.update(visible=video_type != 'None'),
gr.update(visible=video_type == 'GIF' or video_type == 'PNG'),
gr.update(visible=video_type == 'MP4'),
gr.update(visible=video_type == 'MP4'),
]
def model_info_change(model_name):
if model_name == 'None':
return gr.update(value='')
else:
model = next(m for m in MODELS if m['name'] == model_name)
return gr.update(value=f'<span>   frames: {model["params"][0]} size: {model["params"][1]}x{model["params"][2]}</span> <a href="https://huggingface.co/{model["path"]}" target="_blank">link</a>')
with gr.Row():
gr.HTML('<span>  Text to video</span><br>')
with gr.Row():
model_name = gr.Dropdown(label='Model', value='None', choices=[m['name'] for m in MODELS])
with gr.Row():
model_info = gr.HTML()
model_name.change(fn=model_info_change, inputs=[model_name], outputs=[model_info])
with gr.Row():
use_default = gr.Checkbox(label='Use defaults', value=True)
num_frames = gr.Slider(label='Frames', minimum=1, maximum=50, step=1, value=0)
with gr.Row():
video_type = gr.Dropdown(label='Video file', choices=['None', 'GIF', 'PNG', 'MP4'], value='None')
duration = gr.Slider(label='Duration', minimum=0.25, maximum=10, step=0.25, value=2, visible=False)
with gr.Row():
gif_loop = gr.Checkbox(label='Loop', value=True, visible=False)
mp4_pad = gr.Slider(label='Pad frames', minimum=0, maximum=24, step=1, value=1, visible=False)
mp4_interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False)
video_type.change(fn=video_type_change, inputs=[video_type], outputs=[duration, gif_loop, mp4_pad, mp4_interpolate])
return [model_name, use_default, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate]
def run(self, p: processing.StableDiffusionProcessing, model_name, use_default, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument
if model_name == 'None':
return
model = [m for m in MODELS if m['name'] == model_name][0]
shared.log.debug(f'Text2Video: model={model} defaults={use_default} frames={num_frames}, video={video_type} duration={duration} loop={gif_loop} pad={mp4_pad} interpolate={mp4_interpolate}')
if model['path'] in shared.opts.sd_model_checkpoint:
shared.log.debug(f'Text2Video cached: model={shared.opts.sd_model_checkpoint}')
else:
checkpoint = sd_models.get_closet_checkpoint_match(model['path'])
if checkpoint is None:
shared.log.debug(f'Text2Video downloading: model={model["path"]}')
checkpoint = modelloader.download_diffusers_model(hub_id=model['path'])
sd_models.list_models()
if checkpoint is None:
shared.log.error(f'Text2Video: failed to find model={model["path"]}')
return
shared.log.debug(f'Text2Video loading: model={checkpoint}')
shared.opts.sd_model_checkpoint = checkpoint
sd_models.reload_model_weights(op='model')
p.ops.append('text2video')
p.do_not_save_grid = True
if use_default:
p.task_args['num_frames'] = model['params'][0]
p.width = model['params'][1]
p.height = model['params'][2]
elif num_frames > 0:
p.task_args['num_frames'] = num_frames
else:
shared.log.error('Text2Video: invalid number of frames')
return
shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE)
shared.log.debug(f'Text2Video: args={p.task_args}')
processed = processing.process_images(p)
if video_type != 'None':
images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate)
return processed
|