Doubiiu commited on
Commit
4afdd4d
1 Parent(s): 5a70a51

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +155 -154
app.py CHANGED
@@ -1,155 +1,156 @@
1
- import gradio as gr
2
- import os
3
- import sys
4
- import argparse
5
- import random
6
- import time
7
- from omegaconf import OmegaConf
8
- import torch
9
- import torchvision
10
- from pytorch_lightning import seed_everything
11
- from huggingface_hub import hf_hub_download
12
- from einops import repeat
13
- import torchvision.transforms as transforms
14
- from utils.utils import instantiate_from_config
15
- sys.path.insert(0, "scripts/evaluation")
16
- from funcs import (
17
- batch_ddim_sampling,
18
- load_model_checkpoint,
19
- get_latent_z,
20
- save_videos
21
- )
22
-
23
- def download_model():
24
- REPO_ID = 'Doubiiu/DynamiCrafter'
25
- filename_list = ['model.ckpt']
26
- if not os.path.exists('./checkpoints/dynamicrafter_256_v1/'):
27
- os.makedirs('./checkpoints/dynamicrafter_256_v1/')
28
- for filename in filename_list:
29
- local_file = os.path.join('./checkpoints/dynamicrafter_256_v1/', filename)
30
- if not os.path.exists(local_file):
31
- hf_hub_download(repo_id=REPO_ID, filename=filename, local_dir='./checkpoints/dynamicrafter_256_v1/', force_download=True)
32
-
33
-
34
- def infer(image, prompt, steps=50, cfg_scale=7.5, eta=1.0, fs=3, seed=123):
35
- download_model()
36
- ckpt_path='checkpoints/dynamicrafter_256_v1/model.ckpt'
37
- config_file='configs/inference_256_v1.0.yaml'
38
- config = OmegaConf.load(config_file)
39
- model_config = config.pop("model", OmegaConf.create())
40
- model_config['params']['unet_config']['params']['use_checkpoint']=False
41
- model = instantiate_from_config(model_config)
42
- assert os.path.exists(ckpt_path), "Error: checkpoint Not Found!"
43
- model = load_model_checkpoint(model, ckpt_path)
44
- model.eval()
45
- model = model.cuda()
46
- save_fps = 8
47
-
48
- seed_everything(seed)
49
- transform = transforms.Compose([
50
- transforms.Resize(256),
51
- transforms.CenterCrop(256),
52
- ])
53
- torch.cuda.empty_cache()
54
- print('start:', prompt, time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())))
55
- start = time.time()
56
- if steps > 60:
57
- steps = 60
58
-
59
- batch_size=1
60
- channels = model.model.diffusion_model.out_channels
61
- frames = model.temporal_length
62
- h, w = 256 // 8, 256 // 8
63
- noise_shape = [batch_size, channels, frames, h, w]
64
-
65
- # text cond
66
- text_emb = model.get_learned_conditioning([prompt])
67
-
68
- # img cond
69
- img_tensor = torch.from_numpy(image).permute(2, 0, 1).float().to(model.device)
70
- img_tensor = (img_tensor / 255. - 0.5) * 2
71
-
72
- image_tensor_resized = transform(img_tensor) #3,256,256
73
- videos = image_tensor_resized.unsqueeze(0) # bchw
74
-
75
- z = get_latent_z(model, videos.unsqueeze(2)) #bc,1,hw
76
-
77
- img_tensor_repeat = repeat(z, 'b c t h w -> b c (repeat t) h w', repeat=frames)
78
-
79
- cond_images = model.embedder(img_tensor.unsqueeze(0)) ## blc
80
- img_emb = model.image_proj_model(cond_images)
81
-
82
- imtext_cond = torch.cat([text_emb, img_emb], dim=1)
83
-
84
- fs = torch.tensor([fs], dtype=torch.long, device=model.device)
85
- cond = {"c_crossattn": [imtext_cond], "fs": fs, "c_concat": [img_tensor_repeat]}
86
-
87
- ## inference
88
- batch_samples = batch_ddim_sampling(model, cond, noise_shape, n_samples=1, ddim_steps=steps, ddim_eta=eta, cfg_scale=cfg_scale)
89
- ## b,samples,c,t,h,w
90
-
91
- video_path = './output.mp4'
92
- save_videos(batch_samples, './', filenames=['output'], fps=save_fps)
93
- model = model.cpu()
94
- return video_path
95
-
96
-
97
-
98
-
99
-
100
-
101
- i2v_examples = [
102
- ['prompts/art.png', 'man fishing in a boat at sunset', 50, 7.5, 1.0, 3, 234],
103
- ['prompts/boy.png', 'boy walking on the street', 50, 7.5, 1.0, 3, 125],
104
- ['prompts/dance1.jpeg', 'two people dancing', 50, 7.5, 1.0, 3, 116],
105
- ['prompts/fire_and_beach.jpg', 'a campfire on the beach and the ocean waves in the background', 50, 7.5, 1.0, 3, 111],
106
- ['prompts/girl3.jpeg', 'girl talking and blinking', 50, 7.5, 1.0, 3, 111],
107
- ['prompts/guitar0.jpeg', 'bear playing guitar happily, snowing', 50, 7.5, 1.0, 3, 111],
108
- ]
109
- css = """#input_img {max-width: 256px !important} #output_vid {max-width: 256px; max-height: 256px}"""
110
-
111
- with gr.Blocks(analytics_enabled=False, css=css) as dynamicrafter_iface:
112
- gr.Markdown("<div align='center'> <h1> DynamiCrafter: Animating Open-domain Images with Video Diffusion Priors </span> </h1> \
113
- <h2 style='font-weight: 450; font-size: 1rem; margin: 0rem'>\
114
- <a href='https://doubiiu.github.io/'>Jinbo Xing</a>, \
115
- <a href='https://menghanxia.github.io/'>Menghan Xia</a>, <a href='https://yzhang2016.github.io/'>Yong Zhang</a>, \
116
- <a href=''>Haoxin Chen</a>, <a href=''> Wangbo Yu</a>,\
117
- <a href='https://github.com/hyliu'>Hanyuan Liu</a>, <a href='https://xinntao.github.io/'>Xintao Wang</a>,\
118
- <a href='https://www.cse.cuhk.edu.hk/~ttwong/myself.html'>Tien-Tsin Wong</a>,\
119
- <a href='https://scholar.google.com/citations?user=4oXBp9UAAAAJ&hl=zh-CN'>Ying Shan</a>\
120
- </h2> \
121
- <a style='font-size:18px;color: #000000' href='https://arxiv.org/abs/2310.12190'> [ArXiv] </a>\
122
- <a style='font-size:18px;color: #000000' href='https://doubiiu.github.io/projects/DynamiCrafter/'> [Project Page] </a> \
123
- <a style='font-size:18px;color: #000000' href='https://github.com/Doubiiu/DynamiCrafter'> [Github] </a> </div>")
124
-
125
- with gr.Tab(label='ImageAnimation'):
126
- with gr.Column():
127
- with gr.Row():
128
- with gr.Column():
129
- with gr.Row():
130
- i2v_input_image = gr.Image(label="Input Image",elem_id="input_img")
131
- with gr.Row():
132
- i2v_input_text = gr.Text(label='Prompts')
133
- with gr.Row():
134
- i2v_seed = gr.Slider(label='Random Seed', minimum=0, maximum=10000, step=1, value=123)
135
- i2v_eta = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, label='ETA', value=1.0, elem_id="i2v_eta")
136
- i2v_cfg_scale = gr.Slider(minimum=1.0, maximum=15.0, step=0.5, label='CFG Scale', value=7.5, elem_id="i2v_cfg_scale")
137
- with gr.Row():
138
- i2v_steps = gr.Slider(minimum=1, maximum=60, step=1, elem_id="i2v_steps", label="Sampling steps", value=50)
139
- i2v_motion = gr.Slider(minimum=1, maximum=4, step=1, elem_id="i2v_motion", label="Motion magnitude", value=3)
140
- i2v_end_btn = gr.Button("Generate")
141
- # with gr.Tab(label='Result'):
142
- with gr.Row():
143
- i2v_output_video = gr.Video(label="Generated Video",elem_id="output_vid",autoplay=True,show_share_button=True)
144
-
145
- gr.Examples(examples=i2v_examples,
146
- inputs=[i2v_input_image, i2v_input_text, i2v_steps, i2v_cfg_scale, i2v_eta, i2v_motion, i2v_seed],
147
- outputs=[i2v_output_video],
148
- fn = infer,
149
- )
150
- i2v_end_btn.click(inputs=[i2v_input_image, i2v_input_text, i2v_steps, i2v_cfg_scale, i2v_eta, i2v_motion, i2v_seed],
151
- outputs=[i2v_output_video],
152
- fn = infer
153
- )
154
-
 
155
  dynamicrafter_iface.queue(max_size=12).launch(show_api=True)
 
1
+ import gradio as gr
2
+ import os
3
+ import sys
4
+ import argparse
5
+ import random
6
+ import time
7
+ from omegaconf import OmegaConf
8
+ import torch
9
+ import torchvision
10
+ from pytorch_lightning import seed_everything
11
+ from huggingface_hub import hf_hub_download
12
+ from einops import repeat
13
+ import torchvision.transforms as transforms
14
+ from utils.utils import instantiate_from_config
15
+ sys.path.insert(0, "scripts/evaluation")
16
+ from funcs import (
17
+ batch_ddim_sampling,
18
+ load_model_checkpoint,
19
+ get_latent_z,
20
+ save_videos
21
+ )
22
+
23
+ def download_model():
24
+ REPO_ID = 'Doubiiu/DynamiCrafter'
25
+ filename_list = ['model.ckpt']
26
+ if not os.path.exists('./checkpoints/dynamicrafter_256_v1/'):
27
+ os.makedirs('./checkpoints/dynamicrafter_256_v1/')
28
+ for filename in filename_list:
29
+ local_file = os.path.join('./checkpoints/dynamicrafter_256_v1/', filename)
30
+ if not os.path.exists(local_file):
31
+ hf_hub_download(repo_id=REPO_ID, filename=filename, local_dir='./checkpoints/dynamicrafter_256_v1/', force_download=True)
32
+
33
+
34
+ def infer(image, prompt, steps=50, cfg_scale=7.5, eta=1.0, fs=3, seed=123):
35
+ download_model()
36
+ ckpt_path='checkpoints/dynamicrafter_256_v1/model.ckpt'
37
+ config_file='configs/inference_256_v1.0.yaml'
38
+ config = OmegaConf.load(config_file)
39
+ model_config = config.pop("model", OmegaConf.create())
40
+ model_config['params']['unet_config']['params']['use_checkpoint']=False
41
+ model = instantiate_from_config(model_config)
42
+ assert os.path.exists(ckpt_path), "Error: checkpoint Not Found!"
43
+ model = load_model_checkpoint(model, ckpt_path)
44
+ model.eval()
45
+ model = model.cuda()
46
+ save_fps = 8
47
+
48
+ seed_everything(seed)
49
+ transform = transforms.Compose([
50
+ transforms.Resize(256),
51
+ transforms.CenterCrop(256),
52
+ ])
53
+ torch.cuda.empty_cache()
54
+ print('start:', prompt, time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())))
55
+ start = time.time()
56
+ if steps > 60:
57
+ steps = 60
58
+
59
+ batch_size=1
60
+ channels = model.model.diffusion_model.out_channels
61
+ frames = model.temporal_length
62
+ h, w = 256 // 8, 256 // 8
63
+ noise_shape = [batch_size, channels, frames, h, w]
64
+
65
+ # text cond
66
+ text_emb = model.get_learned_conditioning([prompt])
67
+
68
+ # img cond
69
+ img_tensor = torch.from_numpy(image).permute(2, 0, 1).float().to(model.device)
70
+ img_tensor = (img_tensor / 255. - 0.5) * 2
71
+
72
+ image_tensor_resized = transform(img_tensor) #3,256,256
73
+ videos = image_tensor_resized.unsqueeze(0) # bchw
74
+
75
+ z = get_latent_z(model, videos.unsqueeze(2)) #bc,1,hw
76
+
77
+ img_tensor_repeat = repeat(z, 'b c t h w -> b c (repeat t) h w', repeat=frames)
78
+
79
+ cond_images = model.embedder(img_tensor.unsqueeze(0)) ## blc
80
+ img_emb = model.image_proj_model(cond_images)
81
+
82
+ imtext_cond = torch.cat([text_emb, img_emb], dim=1)
83
+
84
+ fs = torch.tensor([fs], dtype=torch.long, device=model.device)
85
+ cond = {"c_crossattn": [imtext_cond], "fs": fs, "c_concat": [img_tensor_repeat]}
86
+
87
+ ## inference
88
+ batch_samples = batch_ddim_sampling(model, cond, noise_shape, n_samples=1, ddim_steps=steps, ddim_eta=eta, cfg_scale=cfg_scale)
89
+ ## b,samples,c,t,h,w
90
+
91
+ video_path = './output.mp4'
92
+ save_videos(batch_samples, './', filenames=['output'], fps=save_fps)
93
+ model = model.cpu()
94
+ return video_path
95
+
96
+
97
+
98
+
99
+
100
+
101
+ i2v_examples = [
102
+ ['prompts/art.png', 'man fishing in a boat at sunset', 50, 7.5, 1.0, 3, 234],
103
+ ['prompts/boy.png', 'boy walking on the street', 50, 7.5, 1.0, 3, 125],
104
+ ['prompts/dance1.jpeg', 'two people dancing', 50, 7.5, 1.0, 3, 116],
105
+ ['prompts/fire_and_beach.jpg', 'a campfire on the beach and the ocean waves in the background', 50, 7.5, 1.0, 3, 111],
106
+ ['prompts/girl3.jpeg', 'girl talking and blinking', 50, 7.5, 1.0, 3, 111],
107
+ ['prompts/guitar0.jpeg', 'bear playing guitar happily, snowing', 50, 7.5, 1.0, 3, 122],
108
+ ['prompts/surf.png', 'a man is surfing', 50, 7.5, 1.0, 3, 123],
109
+ ]
110
+ css = """#input_img {max-width: 256px !important} #output_vid {max-width: 256px; max-height: 256px}"""
111
+
112
+ with gr.Blocks(analytics_enabled=False, css=css) as dynamicrafter_iface:
113
+ gr.Markdown("<div align='center'> <h1> DynamiCrafter: Animating Open-domain Images with Video Diffusion Priors </span> </h1> \
114
+ <h2 style='font-weight: 450; font-size: 1rem; margin: 0rem'>\
115
+ <a href='https://doubiiu.github.io/'>Jinbo Xing</a>, \
116
+ <a href='https://menghanxia.github.io/'>Menghan Xia</a>, <a href='https://yzhang2016.github.io/'>Yong Zhang</a>, \
117
+ <a href=''>Haoxin Chen</a>, <a href=''> Wangbo Yu</a>,\
118
+ <a href='https://github.com/hyliu'>Hanyuan Liu</a>, <a href='https://xinntao.github.io/'>Xintao Wang</a>,\
119
+ <a href='https://www.cse.cuhk.edu.hk/~ttwong/myself.html'>Tien-Tsin Wong</a>,\
120
+ <a href='https://scholar.google.com/citations?user=4oXBp9UAAAAJ&hl=zh-CN'>Ying Shan</a>\
121
+ </h2> \
122
+ <a style='font-size:18px;color: #000000' href='https://arxiv.org/abs/2310.12190'> [ArXiv] </a>\
123
+ <a style='font-size:18px;color: #000000' href='https://doubiiu.github.io/projects/DynamiCrafter/'> [Project Page] </a> \
124
+ <a style='font-size:18px;color: #000000' href='https://github.com/Doubiiu/DynamiCrafter'> [Github] </a> </div>")
125
+
126
+ with gr.Tab(label='ImageAnimation'):
127
+ with gr.Column():
128
+ with gr.Row():
129
+ with gr.Column():
130
+ with gr.Row():
131
+ i2v_input_image = gr.Image(label="Input Image",elem_id="input_img")
132
+ with gr.Row():
133
+ i2v_input_text = gr.Text(label='Prompts')
134
+ with gr.Row():
135
+ i2v_seed = gr.Slider(label='Random Seed', minimum=0, maximum=10000, step=1, value=123)
136
+ i2v_eta = gr.Slider(minimum=0.0, maximum=1.0, step=0.1, label='ETA', value=1.0, elem_id="i2v_eta")
137
+ i2v_cfg_scale = gr.Slider(minimum=1.0, maximum=15.0, step=0.5, label='CFG Scale', value=7.5, elem_id="i2v_cfg_scale")
138
+ with gr.Row():
139
+ i2v_steps = gr.Slider(minimum=1, maximum=60, step=1, elem_id="i2v_steps", label="Sampling steps", value=50)
140
+ i2v_motion = gr.Slider(minimum=1, maximum=4, step=1, elem_id="i2v_motion", label="Motion magnitude", value=3)
141
+ i2v_end_btn = gr.Button("Generate")
142
+ # with gr.Tab(label='Result'):
143
+ with gr.Row():
144
+ i2v_output_video = gr.Video(label="Generated Video",elem_id="output_vid",autoplay=True,show_share_button=True)
145
+
146
+ gr.Examples(examples=i2v_examples,
147
+ inputs=[i2v_input_image, i2v_input_text, i2v_steps, i2v_cfg_scale, i2v_eta, i2v_motion, i2v_seed],
148
+ outputs=[i2v_output_video],
149
+ fn = infer,
150
+ )
151
+ i2v_end_btn.click(inputs=[i2v_input_image, i2v_input_text, i2v_steps, i2v_cfg_scale, i2v_eta, i2v_motion, i2v_seed],
152
+ outputs=[i2v_output_video],
153
+ fn = infer
154
+ )
155
+
156
  dynamicrafter_iface.queue(max_size=12).launch(show_api=True)