sanjanatule commited on
Commit
db3176e
1 Parent(s): 75c3dd8

Upload 3 files

Browse files
Files changed (3) hide show
  1. app.py +48 -0
  2. requirements.txt +9 -0
  3. utils.py +266 -0
app.py ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ from utils import *
3
+
4
+
5
+ with gr.Blocks() as interface:
6
+ gr.HTML(value=HTML_TEMPLATE, show_label=False)
7
+ with gr.Row():
8
+ text_input = gr.Textbox(
9
+ label="Enter your prompt",
10
+ placeholder="A powerful mysterious sorceress..........",
11
+ )
12
+ concept_dropdown = gr.Dropdown(
13
+ label="Select a Concept",
14
+ choices=["Midjourney", "Dream", "Moebius", "Marc Allante", "Wlop"],
15
+ value='Dream'
16
+ )
17
+
18
+ method_dropdown = gr.Dropdown(
19
+ label="Select Guidance Method",
20
+ choices=["Elastic", "Symmetry", "Saturation", "Blue"],
21
+ value='Elastic'
22
+ )
23
+
24
+ seed_slider = gr.Slider(
25
+ label="Random Seed",
26
+ minimum=0,
27
+ maximum=1000,
28
+ step=1,
29
+ value=42
30
+ )
31
+ inputs = [text_input, concept_dropdown, method_dropdown, seed_slider]
32
+
33
+ with gr.Row():
34
+ outputs = gr.Gallery(
35
+ label="Generated Art", show_label=True,
36
+ columns=[2], rows=[1], object_fit="contain"
37
+ )
38
+
39
+ with gr.Row():
40
+ button = gr.Button("Generate Art")
41
+ button.click(generate_art, inputs=inputs, outputs=outputs)
42
+
43
+ with gr.Row():
44
+ gr.Examples(examples=get_examples(), inputs=inputs, outputs=outputs, fn=generate_art, cache_examples=True)
45
+
46
+
47
+ if __name__ == "__main__":
48
+ interface.launch(enable_queue=True)
requirements.txt ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ torch
2
+ torchvision
3
+ pillow
4
+ numpy
5
+ pandas
6
+ transformers
7
+ diffusers
8
+ scipy
9
+ accelerate
utils.py ADDED
@@ -0,0 +1,266 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import PIL
2
+ import torch
3
+ import numpy as np
4
+ from PIL import Image
5
+ from tqdm import tqdm
6
+ import torch.nn.functional as F
7
+ import torchvision.transforms as T
8
+ from diffusers import LMSDiscreteScheduler, DiffusionPipeline
9
+
10
+ # configurations
11
+ torch_device = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu"
12
+ height, width = 512, 512
13
+ guidance_scale = 8
14
+ blue_loss_scale = 200
15
+ num_inference_steps = 50
16
+
17
+ elastic_transformer = T.ElasticTransform(alpha=550.0, sigma=5.0)
18
+
19
+
20
+
21
+ pretrained_model_name_or_path = "segmind/tiny-sd"
22
+ pipe = DiffusionPipeline.from_pretrained(
23
+ pretrained_model_name_or_path,
24
+ low_cpu_mem_usage = True,
25
+ torch_dtype=torch.float16
26
+ ).to(torch_device)
27
+
28
+
29
+ pipe.load_textual_inversion("sd-concepts-library/dreams")
30
+ pipe.load_textual_inversion("sd-concepts-library/midjourney-style")
31
+ pipe.load_textual_inversion("sd-concepts-library/moebius")
32
+ pipe.load_textual_inversion("sd-concepts-library/style-of-marc-allante")
33
+ pipe.load_textual_inversion("sd-concepts-library/wlop-style")
34
+
35
+
36
+ concepts_mapping = {
37
+ "Dream": '<meeg>', "Midjourney":'<midjourney-style>',
38
+ "Marc Allante": '<Marc_Allante>', "Moebius": '<moebius>',
39
+ "Wlop": '<wlop-style>'
40
+ }
41
+
42
+
43
+ def image_loss(images, method='elastic'):
44
+
45
+ # elastic loss
46
+ if method == 'elastic':
47
+ transformed_imgs = elastic_transformer(images)
48
+ error = torch.abs(transformed_imgs - images).mean()
49
+
50
+ # symmetry loss - Flip the image along the width
51
+ elif method == "symmetry":
52
+ flipped_image = torch.flip(images, [3])
53
+ error = F.mse_loss(images, flipped_image)
54
+
55
+ # saturation loss
56
+ elif method == 'saturation':
57
+ transformed_imgs = T.functional.adjust_saturation(images,saturation_factor = 10)
58
+ error = torch.abs(transformed_imgs - images).mean()
59
+
60
+ # blue loss
61
+ elif method == 'blue':
62
+ error = torch.abs(images[:,2] - 0.9).mean() # [:,2] -> all images in batch, only the blue channel
63
+
64
+ return error
65
+
66
+
67
+ HTML_TEMPLATE = """
68
+ <style>
69
+ body {
70
+ background: linear-gradient(135deg, #f5f7fa, #c3cfe2);
71
+ }
72
+ #app-header {
73
+ text-align: center;
74
+ background: rgba(255, 255, 255, 0.8); /* Semi-transparent white */
75
+ padding: 20px;
76
+ border-radius: 10px;
77
+ box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
78
+ position: relative; /* To position the artifacts */
79
+ }
80
+ #app-header h1 {
81
+ color: #4CAF50;
82
+ font-size: 2em;
83
+ margin-bottom: 10px;
84
+ }
85
+ .concept {
86
+ position: relative;
87
+ transition: transform 0.3s;
88
+ }
89
+ .concept:hover {
90
+ transform: scale(1.1);
91
+ }
92
+ .concept img {
93
+ width: 100px;
94
+ border-radius: 10px;
95
+ box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
96
+ }
97
+ .concept-description {
98
+ position: absolute;
99
+ bottom: -30px;
100
+ left: 50%;
101
+ transform: translateX(-50%);
102
+ background-color: #4CAF50;
103
+ color: white;
104
+ padding: 5px 10px;
105
+ border-radius: 5px;
106
+ opacity: 0;
107
+ transition: opacity 0.3s;
108
+ }
109
+ .concept:hover .concept-description {
110
+ opacity: 1;
111
+ }
112
+ /* Artifacts */
113
+ .artifact {
114
+ position: absolute;
115
+ background: rgba(76, 175, 80, 0.1); /* Semi-transparent green */
116
+ border-radius: 50%; /* Make it circular */
117
+ }
118
+ .artifact.large {
119
+ width: 300px;
120
+ height: 300px;
121
+ top: -50px;
122
+ left: -150px;
123
+ }
124
+ .artifact.medium {
125
+ width: 200px;
126
+ height: 200px;
127
+ bottom: -50px;
128
+ right: -100px;
129
+ }
130
+ .artifact.small {
131
+ width: 100px;
132
+ height: 100px;
133
+ top: 50%;
134
+ left: 50%;
135
+ transform: translate(-50%, -50%);
136
+ }
137
+ </style>
138
+ <div id="app-header">
139
+ <!-- Artifacts -->
140
+ <div class="artifact large"></div>
141
+ <div class="artifact medium"></div>
142
+ <div class="artifact small"></div>
143
+ <!-- Content -->
144
+ <h1>Art Generator</h1>
145
+ <p>Generate new art in five different styles by providing a prompt.</p>
146
+ <div style="display: flex; justify-content: center; gap: 20px; margin-top: 20px;">
147
+ <div class="concept">
148
+ <img src="https://github.com/Delve-ERAV1/S20/assets/11761529/30ac92f8-fc62-4aab-9221-043865c6fe7c" alt="Midjourney">
149
+ <div class="concept-description">Midjourney Style</div>
150
+ </div>
151
+ <div class="concept">
152
+ <img src="https://github.com/Delve-ERAV1/S20/assets/11761529/54c9a61e-df9f-4054-835b-ec2c6ba5916c" alt="Dreams">
153
+ <div class="concept-description">Dreams Style</div>
154
+ </div>
155
+ <div class="concept">
156
+ <img src="https://github.com/Delve-ERAV1/S20/assets/11761529/2f37e402-15d1-4a74-ba85-bb1566da930e" alt="Moebius">
157
+ <div class="concept-description">Moebius Style</div>
158
+ </div>
159
+ <div class="concept">
160
+ <img src="https://github.com/Delve-ERAV1/S20/assets/11761529/f838e767-ac20-4996-b5be-65c61b365ce0" alt="Allante">
161
+ <div class="concept-description">Hong Kong born artist inspired by western and eastern influences</div>
162
+ </div>
163
+ <div class="concept">
164
+ <img src="https://github.com/Delve-ERAV1/S20/assets/11761529/9958140a-1b62-4972-83ca-85b023e3863f" alt="Wlop">
165
+ <div class="concept-description">WLOP (Born 1987) is known for Digital Art (NFTs)</div>
166
+ </div>
167
+ </div>
168
+ </div>
169
+ """
170
+
171
+
172
+ def get_examples():
173
+ examples = [
174
+ ['A powerful man in dreadlocks', 'Dream', 'Symmetry', 45],
175
+ ['World Peace', 'Marc Allante', 'Saturation', 147],
176
+ ['Storm trooper in the desert, dramatic lighting, high-detail', 'Moebius', 'Elastic', 28],
177
+ ['Delicious Italian pizza on a table, a window in the background overlooking a city skyline', 'Wlop', 'Blue', 50],
178
+ ]
179
+ return(examples)
180
+
181
+
182
+ def latents_to_pil(latents):
183
+ # bath of latents -> list of images
184
+ latents = (1 / 0.18215) * latents
185
+ with torch.no_grad():
186
+ image = pipe.vae.decode(latents).sample
187
+ image = (image / 2 + 0.5).clamp(0, 1) # 0 to 1
188
+ image = image.detach().cpu().permute(0, 2, 3, 1).numpy()
189
+ image = (image * 255).round().astype("uint8")
190
+ return Image.fromarray(image[0])
191
+
192
+
193
+ def generate_art(prompt, concept, method, seed):
194
+
195
+ prompt = f"{prompt} in the style of {concepts_mapping[concept]}"
196
+ img_no_loss = latents_to_pil(generate_image(prompt, method, seed))
197
+ img_loss = latents_to_pil(generate_image(prompt, method, seed, loss_apply=True))
198
+ return([img_no_loss, img_loss])
199
+
200
+
201
+ def generate_image(prompt, method, seed, loss_apply=False):
202
+
203
+ generator = torch.manual_seed(seed)
204
+ batch_size = 1
205
+ method = method.lower()
206
+
207
+ # scheduler
208
+ scheduler = LMSDiscreteScheduler(beta_start = 0.00085, beta_end = 0.012, beta_schedule = "scaled_linear", num_train_timesteps = 1000)
209
+ scheduler.set_timesteps(50)
210
+ scheduler.timesteps = scheduler.timesteps.to(torch.float32)
211
+
212
+ # text embeddings of the prompt
213
+ text_input = pipe.tokenizer([prompt], padding='max_length', max_length = pipe.tokenizer.model_max_length, truncation= True, return_tensors="pt")
214
+ input_ids = text_input.input_ids.to(torch_device)
215
+
216
+ with torch.no_grad():
217
+ text_embeddings = pipe.text_encoder(text_input.input_ids.to(torch_device))[0]
218
+
219
+ max_length = text_input.input_ids.shape[-1]
220
+ uncond_input = pipe.tokenizer(
221
+ [""] * 1, padding="max_length", max_length= max_length, return_tensors="pt"
222
+ )
223
+
224
+ with torch.no_grad():
225
+ uncond_embeddings = pipe.text_encoder(uncond_input.input_ids.to(torch_device))[0]
226
+
227
+ text_embeddings = torch.cat([uncond_embeddings,text_embeddings])
228
+
229
+ # random latent
230
+ latents = torch.randn(
231
+ (batch_size, pipe.unet.config.in_channels, height// 8, width //8),
232
+ generator = generator,
233
+ ).to(torch.float16)
234
+
235
+
236
+ latents = latents.to(torch_device)
237
+ latents = latents * scheduler.init_noise_sigma
238
+
239
+ for i, t in tqdm(enumerate(scheduler.timesteps), total = len(scheduler.timesteps)):
240
+
241
+ latent_model_input = torch.cat([latents] * 2)
242
+ sigma = scheduler.sigmas[i]
243
+ latent_model_input = scheduler.scale_model_input(latent_model_input, t)
244
+
245
+ with torch.no_grad():
246
+ noise_pred = pipe.unet(latent_model_input.to(torch.float16), t, encoder_hidden_states=text_embeddings)["sample"]
247
+
248
+ noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
249
+ noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
250
+
251
+ if loss_apply and i%5 == 0:
252
+
253
+ latents = latents.detach().requires_grad_()
254
+ latents_x0 = latents - sigma * noise_pred
255
+
256
+ # use vae to decode the image
257
+ denoised_images = pipe.vae.decode((1/ 0.18215) * latents_x0).sample / 2 + 0.5 # range(0,1)
258
+
259
+ loss = image_loss(denoised_images, method) * blue_loss_scale
260
+
261
+ cond_grad = torch.autograd.grad(loss, latents)[0]
262
+ latents = latents.detach() - cond_grad * sigma**2
263
+
264
+ latents = scheduler.step(noise_pred,t, latents).prev_sample
265
+
266
+ return latents