AlexKM commited on
Commit
5992276
1 Parent(s): e637b17

Create new file

Browse files
Files changed (1) hide show
  1. app.py +233 -0
app.py ADDED
@@ -0,0 +1,233 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+
3
+ os.system("git clone --recursive https://github.com/JD-P/cloob-latent-diffusion")
4
+ os.system("cd cloob-latent-diffusion;pip install omegaconf pillow pytorch-lightning einops wandb ftfy regex ./CLIP")
5
+
6
+ # setup some example images
7
+ examples = []
8
+ EXAMPLES_DIR = 'examples'
9
+ DEFAULT_PROMPT = "<image>"
10
+ if os.path.isdir(EXAMPLES_DIR):
11
+ for file in os.listdir(EXAMPLES_DIR):
12
+ path = EXAMPLES_DIR + "/" + file
13
+ examples.append([path, DEFAULT_PROMPT])
14
+
15
+ import argparse
16
+ from functools import partial
17
+ from pathlib import Path
18
+ import sys
19
+ sys.path.append('./cloob-latent-diffusion')
20
+ sys.path.append('./cloob-latent-diffusion/cloob-training')
21
+ sys.path.append('./cloob-latent-diffusion/latent-diffusion')
22
+ sys.path.append('./cloob-latent-diffusion/taming-transformers')
23
+ sys.path.append('./cloob-latent-diffusion/v-diffusion-pytorch')
24
+ from omegaconf import OmegaConf
25
+ from PIL import Image
26
+ import torch
27
+ from torch import nn
28
+ from torch.nn import functional as F
29
+ from torchvision import transforms
30
+ from torchvision.transforms import functional as TF
31
+ from tqdm import trange
32
+ from CLIP import clip
33
+ from cloob_training import model_pt, pretrained
34
+ import ldm.models.autoencoder
35
+ from diffusion import sampling, utils
36
+ import train_latent_diffusion as train
37
+ from huggingface_hub import hf_hub_url, cached_download
38
+ import random
39
+
40
+ # Download the model files
41
+ checkpoint = cached_download(hf_hub_url("huggan/distill-ccld-wa", filename="model_student.ckpt"))
42
+ ae_model_path = cached_download(hf_hub_url("huggan/ccld_wa", filename="ae_model.ckpt"))
43
+ ae_config_path = cached_download(hf_hub_url("huggan/ccld_wa", filename="ae_model.yaml"))
44
+
45
+ # Define a few utility functions
46
+
47
+
48
+ def parse_prompt(prompt, default_weight=3.):
49
+ if prompt.startswith('http://') or prompt.startswith('https://'):
50
+ vals = prompt.rsplit(':', 2)
51
+ vals = [vals[0] + ':' + vals[1], *vals[2:]]
52
+ else:
53
+ vals = prompt.rsplit(':', 1)
54
+ vals = vals + ['', default_weight][len(vals):]
55
+ return vals[0], float(vals[1])
56
+
57
+
58
+ def resize_and_center_crop(image, size):
59
+ fac = max(size[0] / image.size[0], size[1] / image.size[1])
60
+ image = image.resize((int(fac * image.size[0]), int(fac * image.size[1])), Image.LANCZOS)
61
+ return TF.center_crop(image, size[::-1])
62
+
63
+
64
+ # Load the models
65
+ device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
66
+ print('Using device:', device)
67
+ print('loading models')
68
+
69
+ # autoencoder
70
+ ae_config = OmegaConf.load(ae_config_path)
71
+ ae_model = ldm.models.autoencoder.AutoencoderKL(**ae_config.model.params)
72
+ ae_model.eval().requires_grad_(False).to(device)
73
+ ae_model.load_state_dict(torch.load(ae_model_path))
74
+ n_ch, side_y, side_x = 4, 32, 32
75
+
76
+ # diffusion model
77
+ model = train.DiffusionModel(192, [1,1,2,2], autoencoder_scale=torch.tensor(4.3084))
78
+ model.load_state_dict(torch.load(checkpoint, map_location='cpu'))
79
+ model = model.to(device).eval().requires_grad_(False)
80
+
81
+ # CLOOB
82
+ cloob_config = pretrained.get_config('cloob_laion_400m_vit_b_16_16_epochs')
83
+ cloob = model_pt.get_pt_model(cloob_config)
84
+ checkpoint = pretrained.download_checkpoint(cloob_config)
85
+ cloob.load_state_dict(model_pt.get_pt_params(cloob_config, checkpoint))
86
+ cloob.eval().requires_grad_(False).to(device)
87
+
88
+
89
+ # The key function: returns a list of n PIL images
90
+ def generate(n=1, prompts=['a red circle'], images=[], seed=42, steps=15,
91
+ method='plms', eta=None):
92
+ zero_embed = torch.zeros([1, cloob.config['d_embed']], device=device)
93
+ target_embeds, weights = [zero_embed], []
94
+
95
+ for prompt in prompts:
96
+ txt, weight = parse_prompt(prompt)
97
+ target_embeds.append(cloob.text_encoder(cloob.tokenize(txt).to(device)).float())
98
+ weights.append(weight)
99
+
100
+ for prompt in images:
101
+ path, weight = parse_prompt(prompt)
102
+ img = Image.open(utils.fetch(path)).convert('RGB')
103
+ clip_size = cloob.config['image_encoder']['image_size']
104
+ img = resize_and_center_crop(img, (clip_size, clip_size))
105
+ batch = TF.to_tensor(img)[None].to(device)
106
+ embed = F.normalize(cloob.image_encoder(cloob.normalize(batch)).float(), dim=-1)
107
+ target_embeds.append(embed)
108
+ weights.append(weight)
109
+
110
+ weights = torch.tensor([1 - sum(weights), *weights], device=device)
111
+
112
+ torch.manual_seed(seed)
113
+
114
+ def cfg_model_fn(x, t):
115
+ n = x.shape[0]
116
+ n_conds = len(target_embeds)
117
+ x_in = x.repeat([n_conds, 1, 1, 1])
118
+ t_in = t.repeat([n_conds])
119
+ clip_embed_in = torch.cat([*target_embeds]).repeat_interleave(n, 0)
120
+ vs = model(x_in, t_in, clip_embed_in).view([n_conds, n, *x.shape[1:]])
121
+ v = vs.mul(weights[:, None, None, None, None]).sum(0)
122
+ return v
123
+
124
+ def run(x, steps):
125
+ if method == 'ddpm':
126
+ return sampling.sample(cfg_model_fn, x, steps, 1., {})
127
+ if method == 'ddim':
128
+ return sampling.sample(cfg_model_fn, x, steps, eta, {})
129
+ if method == 'prk':
130
+ return sampling.prk_sample(cfg_model_fn, x, steps, {})
131
+ if method == 'plms':
132
+ return sampling.plms_sample(cfg_model_fn, x, steps, {})
133
+ if method == 'pie':
134
+ return sampling.pie_sample(cfg_model_fn, x, steps, {})
135
+ if method == 'plms2':
136
+ return sampling.plms2_sample(cfg_model_fn, x, steps, {})
137
+ assert False
138
+
139
+ batch_size = n
140
+ x = torch.randn([n, n_ch, side_y, side_x], device=device)
141
+ t = torch.linspace(1, 0, steps + 1, device=device)[:-1]
142
+ steps = utils.get_spliced_ddpm_cosine_schedule(t)
143
+ pil_ims = []
144
+ for i in trange(0, n, batch_size):
145
+ cur_batch_size = min(n - i, batch_size)
146
+ out_latents = run(x[i:i+cur_batch_size], steps)
147
+ outs = ae_model.decode(out_latents * torch.tensor(2.55).to(device))
148
+ for j, out in enumerate(outs):
149
+ pil_ims.append(utils.to_pil_image(out))
150
+
151
+ return pil_ims
152
+
153
+
154
+ import gradio as gr
155
+
156
+ def gen_ims(prompt, im_prompt=None, seed=None, n_steps=10, method='plms'):
157
+ if seed == None :
158
+ seed = random.randint(0, 10000)
159
+ print( prompt, im_prompt, seed, n_steps)
160
+ prompts = [prompt]
161
+ im_prompts = []
162
+ if im_prompt != None:
163
+ im_prompts = [im_prompt]
164
+ pil_ims = generate(n=1, prompts=prompts, images=im_prompts, seed=seed, steps=n_steps, method=method)
165
+ return pil_ims[0]
166
+
167
+ iface = gr.Interface(fn=gen_ims,
168
+ inputs=[#gr.inputs.Slider(minimum=1, maximum=1, step=1, default=1,label="Number of images"),
169
+ #gr.inputs.Slider(minimum=0, maximum=200, step=1, label='Random seed', default=0),
170
+ gr.inputs.Textbox(value=DEFAULT_PROMPT, label="Text prompt"),
171
+ gr.inputs.Image(optional=True, label="Image prompt", type='filepath'),
172
+ #gr.inputs.Slider(minimum=10, maximum=35, step=1, default=15,label="Number of steps")
173
+ ],
174
+ outputs=[gr.outputs.Image(type="pil", label="Generated Image")],
175
+ examples=[
176
+ ["Futurism, in the style of Wassily Kandinsky"],
177
+ ["Art Nouveau, in the style of John Singer Sargent"],
178
+ ["Surrealism, in the style of Edgar Degas"],
179
+ ["Expressionism, in the style of Wassily Kandinsky"],
180
+ ["Futurism, in the style of Egon Schiele"],
181
+ ["Neoclassicism, in the style of Gustav Klimt"],
182
+ ["Cubism, in the style of Gustav Klimt"],
183
+ ["Op Art, in the style of Marc Chagall"],
184
+ ["Romanticism, in the style of M.C. Escher"],
185
+ ["Futurism, in the style of M.C. Escher"],
186
+ ["Abstract Art, in the style of M.C. Escher"],
187
+ ["Mannerism, in the style of Paul Klee"],
188
+ ["Romanesque Art, in the style of Leonardo da Vinci"],
189
+ ["High Renaissance, in the style of Rembrandt"],
190
+ ["Magic Realism, in the style of Gustave Dore"],
191
+ ["Realism, in the style of Jean-Michel Basquiat"],
192
+ ["Art Nouveau, in the style of Paul Gauguin"],
193
+ ["Avant-garde, in the style of Pierre-Auguste Renoir"],
194
+ ["Baroque, in the style of Edward Hopper"],
195
+ ["Post-Impressionism, in the style of Wassily Kandinsky"],
196
+ ["Naturalism, in the style of Rene Magritte"],
197
+ ["Constructivism, in the style of Paul Cezanne"],
198
+ ["Abstract Expressionism, in the style of Henri Matisse"],
199
+ ["Pop Art, in the style of Vincent van Gogh"],
200
+ ["Futurism, in the style of Wassily Kandinsky"],
201
+ ["Futurism, in the style of Zdzislaw Beksinski"],
202
+ ['Surrealism, in the style of Salvador Dali'],
203
+ ["Aaron Wacker, oil on canvas"],
204
+ ["abstract"],
205
+ ["landscape"],
206
+ ["portrait"],
207
+ ["sculpture"],
208
+ ["genre painting"],
209
+ ["installation"],
210
+ ["photo"],
211
+ ["figurative"],
212
+ ["illustration"],
213
+ ["still life"],
214
+ ["history painting"],
215
+ ["cityscape"],
216
+ ["marina"],
217
+ ["animal painting"],
218
+ ["design"],
219
+ ["calligraphy"],
220
+ ["symbolic painting"],
221
+ ["graffiti"],
222
+ ["performance"],
223
+ ["mythological painting"],
224
+ ["battle painting"],
225
+ ["self-portrait"],
226
+ ["Impressionism, oil on canvas"]
227
+ ],
228
+ title='Art Generator and Style Mixer from 🧠 Cloob and 🎨 WikiArt - Visual Art Encyclopedia:',
229
+ description="Trained on images from the [WikiArt](https://www.wikiart.org/) dataset, comprised of visual arts",
230
+ article = 'Model used is: [model card](https://huggingface.co/huggan/distill-ccld-wa)..'
231
+
232
+ )
233
+ iface.launch(enable_queue=True) # , debug=True for colab debugging