File size: 27,052 Bytes
0f61c4d
 
 
 
bf48e0b
0c133ce
 
 
 
0f61c4d
 
966167e
0f61c4d
 
 
966167e
93f708e
bf48e0b
0f61c4d
 
 
 
 
bf48e0b
94ba4d8
 
 
 
 
78551e3
f64d69b
0f61c4d
aea387e
 
cde0b82
2ade25e
cde0b82
 
 
 
 
 
 
 
 
 
 
 
 
0a8222c
cde0b82
0f61c4d
 
 
1781379
0a8222c
0f61c4d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
0a8222c
0f61c4d
 
0a8222c
0f61c4d
 
 
93f708e
 
 
 
 
 
 
08f7d1b
9dc9163
8487f87
9dc9163
 
3a5dd2f
0f61c4d
9dc9163
 
 
 
 
0f61c4d
9dc9163
0f61c4d
9dc9163
 
 
 
 
 
 
fddbcc2
9dc9163
8487f87
0f61c4d
 
9dc9163
 
 
 
0f61c4d
9dc9163
 
fddbcc2
0f61c4d
 
9dc9163
0f61c4d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9dc9163
8487f87
0f61c4d
9dc9163
 
0f61c4d
d643014
 
7d4f1d1
d643014
 
 
 
3c847e7
7d4f1d1
 
d643014
0f61c4d
 
9dc9163
 
0f61c4d
 
9dc9163
 
0f61c4d
 
9dc9163
0f61c4d
9dc9163
0f61c4d
 
9dc9163
0f61c4d
 
fddbcc2
0f61c4d
 
9dc9163
0f61c4d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9dc9163
0f61c4d
d643014
0f61c4d
 
78551e3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
0f61c4d
 
9dc9163
0f61c4d
15488e5
 
9dc9163
 
0f61c4d
f9238f7
 
 
0f61c4d
 
 
 
2af8ebe
b3b16a3
5d34271
08f7d1b
78551e3
2af8ebe
3326f03
2af8ebe
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f8b7086
 
2af8ebe
 
 
 
 
 
0f61c4d
78551e3
2af8ebe
 
 
 
0f61c4d
2af8ebe
 
0f61c4d
2af8ebe
 
0f61c4d
2af8ebe
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f7ea271
2af8ebe
0f61c4d
2af8ebe
b81bda4
78551e3
2af8ebe
9dc9163
2af8ebe
 
 
 
 
 
 
 
b81bda4
2af8ebe
 
 
 
 
 
 
b81bda4
2af8ebe
f7ea271
2af8ebe
f7ea271
 
9e4dc36
 
 
0f61c4d
a1ce13a
08f7d1b
a1ce13a
 
 
 
 
 
 
 
 
 
 
0a8222c
8b08967
a1ce13a
0a8222c
 
a1ce13a
 
 
 
 
 
 
ea0bceb
 
 
 
586de43
a1ce13a
 
 
 
 
 
 
586de43
a1ce13a
 
 
 
 
 
 
78551e3
ea0bceb
78551e3
 
a1ce13a
 
 
 
 
 
 
401dece
a1ce13a
401dece
a1ce13a
78551e3
ea0bceb
78551e3
 
a1ce13a
0f61c4d
 
9dc9163
d643014
9dc9163
d643014
9dc9163
0f61c4d
 
 
4373d44
0f61c4d
 
08f7d1b
 
 
 
 
 
0f61c4d
 
 
215a958
a0ee6a6
0f61c4d
 
 
 
 
8f438bd
a5e3b1c
215a958
0f61c4d
 
 
 
 
 
7585c5c
60ba9a9
a6470eb
0f61c4d
9dc9163
 
 
 
 
 
 
 
 
 
 
0f61c4d
5695ac1
a6470eb
57a3c2d
4373d44
8785c9a
78551e3
5d34271
0f61c4d
9dc9163
d643014
 
a5e3b1c
d643014
9dc9163
 
 
 
0f61c4d
 
d147789
cbddee5
 
 
 
46ad52c
c59049f
78551e3
0f61c4d
80238a0
cbddee5
 
 
 
057c5bb
4373d44
5d04da1
4373d44
 
 
 
 
 
 
 
 
 
 
f9238f7
4373d44
 
 
 
 
 
 
cbddee5
f9238f7
 
 
 
 
333a06a
cbddee5
0f61c4d
 
 
d147789
0f61c4d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
import os
import re
import zipfile
import torch
import gradio as gr

print('hello', gr.__version__)


import time
from transformers import CLIPTextModel, CLIPTokenizer
from diffusers import AutoencoderKL, DDPMScheduler, StableDiffusionPipeline, UNet2DConditionModel, DiffusionPipeline, LCMScheduler
from tqdm import tqdm
from PIL import Image
from PIL import Image, ImageDraw, ImageFont
import random
import copy

import string
alphabet = string.digits + string.ascii_lowercase + string.ascii_uppercase + string.punctuation + ' '  # len(aphabet) = 95
'''alphabet
0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ 
'''

if not os.path.exists('images2'):
    os.system('wget https://huggingface.co/datasets/JingyeChen22/TextDiffuser/resolve/main/images2.zip')
    with zipfile.ZipFile('images2.zip', 'r') as zip_ref:
        zip_ref.extractall('.')

# os.system('nvidia-smi')
os.system('ls')

#### import m1
from fastchat.model import load_model, get_conversation_template
from transformers import AutoTokenizer, AutoModelForCausalLM
m1_model_path = 'JingyeChen22/textdiffuser2_layout_planner'
# m1_model, m1_tokenizer = load_model(
#     m1_model_path,
#     'cuda',
#     1,
#     None,
#     False,
#     False,
#     revision="main",
#     debug=False,
# )

m1_tokenizer = AutoTokenizer.from_pretrained(m1_model_path, use_fast=False)
m1_model = AutoModelForCausalLM.from_pretrained(
    m1_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True
).cuda()

#### import diffusion models
text_encoder = CLIPTextModel.from_pretrained(
    'JingyeChen22/textdiffuser2-full-ft', subfolder="text_encoder"
).cuda().half()
tokenizer = CLIPTokenizer.from_pretrained(
    'runwayml/stable-diffusion-v1-5', subfolder="tokenizer"
)

#### additional tokens are introduced, including coordinate tokens and character tokens
print('***************')
print(len(tokenizer))
for i in range(520):
    tokenizer.add_tokens(['l' + str(i) ]) # left
    tokenizer.add_tokens(['t' + str(i) ]) # top
    tokenizer.add_tokens(['r' + str(i) ]) # width
    tokenizer.add_tokens(['b' + str(i) ]) # height    
for c in alphabet:
    tokenizer.add_tokens([f'[{c}]']) 
print(len(tokenizer))
print('***************')

vae = AutoencoderKL.from_pretrained('runwayml/stable-diffusion-v1-5', subfolder="vae").half().cuda()
unet = UNet2DConditionModel.from_pretrained(
    'JingyeChen22/textdiffuser2-full-ft', subfolder="unet"
).half().cuda()
text_encoder.resize_token_embeddings(len(tokenizer))


#### load lcm components
model_id = "lambdalabs/sd-pokemon-diffusers"
lcm_lora_id = "latent-consistency/lcm-lora-sdv1-5"
pipe = DiffusionPipeline.from_pretrained(model_id, unet=copy.deepcopy(unet), tokenizer=tokenizer, text_encoder=copy.deepcopy(text_encoder), torch_dtype=torch.float16)
pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config)
pipe.load_lora_weights(lcm_lora_id)
pipe.to(device="cuda")

global_dict = {}
#### for interactive
# stack = []
# state = 0   
font = ImageFont.truetype("./Arial.ttf", 32)

def skip_fun(i, t, guest_id):
    global_dict[guest_id]['state'] = 0
    # global state
    # state = 0


def exe_undo(i, t, guest_id):

    global_dict[guest_id]['stack'] = []
    global_dict[guest_id]['state'] = 0

    # global stack
    # global state
    # state = 0
    # stack = []
    image = Image.open(f'./gray256.jpg')
    # print('stack', stack)
    return image


def exe_redo(i, t, guest_id):
    # global state 
    # state = 0
    global_dict[guest_id]['state'] = 0

    if len(global_dict[guest_id]['stack']) > 0:
        global_dict[guest_id]['stack'].pop()
    image = Image.open(f'./gray256.jpg')
    draw = ImageDraw.Draw(image)

    for items in global_dict[guest_id]['stack']:
        # print('now', items)
        text_position, t = items
        if len(text_position) == 2:
            x, y = text_position
            text_color = (255, 0, 0)  
            draw.text((x+2, y), t, font=font, fill=text_color)
            r = 4
            leftUpPoint = (x-r, y-r)
            rightDownPoint = (x+r, y+r)
            draw.ellipse((leftUpPoint,rightDownPoint), fill='red')
        elif len(text_position) == 4:
            x0, y0, x1, y1 = text_position
            text_color = (255, 0, 0)  
            draw.text((x0+2, y0), t, font=font, fill=text_color)
            r = 4
            leftUpPoint = (x0-r, y0-r)
            rightDownPoint = (x0+r, y0+r)
            draw.ellipse((leftUpPoint,rightDownPoint), fill='red')
            draw.rectangle((x0,y0,x1,y1), outline=(255, 0, 0) )

    print('stack', global_dict[guest_id]['stack'])
    return image

def get_pixels(i, t, guest_id, evt: gr.SelectData):
    # global state

    # register
    if guest_id == '-1':
        seed = str(int(time.time()))
        global_dict[str(seed)] = {
            'state': 0,
            'stack': []
        }
        guest_id = str(seed)
    else:
        seed = guest_id

    text_position = evt.index

    if global_dict[guest_id]['state'] == 0:
        global_dict[guest_id]['stack'].append(
            (text_position, t)
        )
        print(text_position, global_dict[guest_id]['stack'])
        global_dict[guest_id]['state'] = 1
    else:
        
        (_, t) = global_dict[guest_id]['stack'].pop()
        x, y = _
        global_dict[guest_id]['stack'].append(
            ((x,y,text_position[0],text_position[1]), t)
        )
        global_dict[guest_id]['state'] = 0


    image = Image.open(f'./gray256.jpg')
    draw = ImageDraw.Draw(image)

    for items in global_dict[guest_id]['stack']:
        # print('now', items)
        text_position, t = items
        if len(text_position) == 2:
            x, y = text_position
            text_color = (255, 0, 0)  
            draw.text((x+2, y), t, font=font, fill=text_color)
            r = 4
            leftUpPoint = (x-r, y-r)
            rightDownPoint = (x+r, y+r)
            draw.ellipse((leftUpPoint,rightDownPoint), fill='red')
        elif len(text_position) == 4:
            x0, y0, x1, y1 = text_position
            text_color = (255, 0, 0)  
            draw.text((x0+2, y0), t, font=font, fill=text_color)
            r = 4
            leftUpPoint = (x0-r, y0-r)
            rightDownPoint = (x0+r, y0+r)
            draw.ellipse((leftUpPoint,rightDownPoint), fill='red')
            draw.rectangle((x0,y0,x1,y1), outline=(255, 0, 0) )

    print('stack', global_dict[guest_id]['stack'])

    return image, seed


font_layout = ImageFont.truetype('./Arial.ttf', 16)

def get_layout_image(ocrs):

    blank = Image.new('RGB', (256,256), (0,0,0))
    draw = ImageDraw.ImageDraw(blank)

    for line in ocrs.split('\n'):
        line = line.strip()

        if len(line) == 0:
            break

        pred = ' '.join(line.split()[:-1])
        box = line.split()[-1]
        l, t, r, b = [int(i)*2 for i in box.split(',')] # the size of canvas is 256x256
        draw.rectangle([(l, t), (r, b)], outline ="red")
        draw.text((l, t), pred, font=font_layout)
    
    return blank



def text_to_image(guest_id, prompt,keywords,positive_prompt,radio,slider_step,slider_guidance,slider_batch,slider_temperature,slider_natural):

    print(f'[info] Prompt: {prompt} | Keywords: {keywords} | Radio: {radio} | Steps: {slider_step} | Guidance: {slider_guidance} | Natural: {slider_natural}')

    # global stack
    # global state

    if len(positive_prompt.strip()) != 0:
        prompt += positive_prompt

    with torch.no_grad():
        time1 = time.time()
        user_prompt = prompt

        if slider_natural:
            user_prompt = f'{user_prompt}'
            composed_prompt = user_prompt
            prompt = tokenizer.encode(user_prompt)
            layout_image = None
        else:
            if guest_id not in global_dict or len(global_dict[guest_id]['stack']) == 0:

                if len(keywords.strip()) == 0:
                    template = f'Given a prompt that will be used to generate an image, plan the layout of visual text for the image. The size of the image is 128x128. Therefore, all properties of the positions should not exceed 128, including the coordinates of top, left, right, and bottom. All keywords are included in the caption. You dont need to specify the details of font styles. At each line, the format should be keyword left, top, right, bottom. So let us begin. Prompt: {user_prompt}'
                else:
                    keywords = keywords.split('/')
                    keywords = [i.strip() for i in keywords]
                    template = f'Given a prompt that will be used to generate an image, plan the layout of visual text for the image. The size of the image is 128x128. Therefore, all properties of the positions should not exceed 128, including the coordinates of top, left, right, and bottom. In addition, we also provide all keywords at random order for reference. You dont need to specify the details of font styles. At each line, the format should be keyword left, top, right, bottom. So let us begin. Prompt: {prompt}. Keywords: {str(keywords)}'

                msg = template
                conv = get_conversation_template(m1_model_path)
                conv.append_message(conv.roles[0], msg)
                conv.append_message(conv.roles[1], None)
                prompt = conv.get_prompt()
                inputs = m1_tokenizer([prompt], return_token_type_ids=False)
                inputs = {k: torch.tensor(v).to('cuda') for k, v in inputs.items()}
                output_ids = m1_model.generate(
                    **inputs,
                    do_sample=True,
                    temperature=slider_temperature,
                    repetition_penalty=1.0,
                    max_new_tokens=512,
                )

                if m1_model.config.is_encoder_decoder:
                    output_ids = output_ids[0]
                else:
                    output_ids = output_ids[0][len(inputs["input_ids"][0]) :]
                outputs = m1_tokenizer.decode(
                    output_ids, skip_special_tokens=True, spaces_between_special_tokens=False
                )
                print(f"[{conv.roles[0]}]\n{msg}")
                print(f"[{conv.roles[1]}]\n{outputs}")
                layout_image = get_layout_image(outputs)

                ocrs = outputs.split('\n')
                time2 = time.time()
                print(time2-time1)
                
                # user_prompt = prompt
                current_ocr = ocrs


                ocr_ids = [] 
                print('user_prompt', user_prompt)
                print('current_ocr', current_ocr)
                

                for ocr in current_ocr:
                    ocr = ocr.strip()

                    if len(ocr) == 0 or '###' in ocr or '.com' in ocr:
                        continue

                    items = ocr.split()
                    pred = ' '.join(items[:-1])
                    box = items[-1]
                
                    l,t,r,b = box.split(',')
                    l,t,r,b = int(l), int(t), int(r), int(b)
                    ocr_ids.extend(['l'+str(l), 't'+str(t), 'r'+str(r), 'b'+str(b)])

                    char_list = list(pred)
                    char_list = [f'[{i}]' for i in char_list]
                    ocr_ids.extend(char_list)
                    ocr_ids.append(tokenizer.eos_token_id)     

                caption_ids = tokenizer(
                    user_prompt, truncation=True, return_tensors="pt"
                ).input_ids[0].tolist() 

                try:
                    ocr_ids = tokenizer.encode(ocr_ids)
                    prompt = caption_ids + ocr_ids
                except:
                    prompt = caption_ids

                user_prompt = tokenizer.decode(prompt)
                composed_prompt = tokenizer.decode(prompt)
            
            else:
                user_prompt += ' <|endoftext|><|startoftext|>'
                layout_image = None
                
                for items in global_dict[guest_id]['stack']:
                    position, text = items

                    
                    if len(position) == 2:
                        x, y = position
                        x = x // 4
                        y = y // 4
                        text_str = ' '.join([f'[{c}]' for c in list(text)])
                        user_prompt += f' l{x} t{y} {text_str} <|endoftext|>'
                    elif len(position) == 4:
                        x0, y0, x1, y1 = position
                        x0 = x0 // 4
                        y0 = y0 // 4
                        x1 = x1 // 4
                        y1 = y1 // 4
                        text_str = ' '.join([f'[{c}]' for c in list(text)])
                        user_prompt += f' l{x0} t{y0} r{x1} b{y1} {text_str} <|endoftext|>'

                    # composed_prompt = user_prompt
                    prompt = tokenizer.encode(user_prompt)
                    composed_prompt = tokenizer.decode(prompt)

        prompt = prompt[:77]
        while len(prompt) < 77: 
            prompt.append(tokenizer.pad_token_id) 

        if radio == 'TextDiffuser-2':
            
            prompts_cond = prompt
            prompts_nocond = [tokenizer.pad_token_id]*77

            prompts_cond = [prompts_cond] * slider_batch
            prompts_nocond = [prompts_nocond] * slider_batch

            prompts_cond = torch.Tensor(prompts_cond).long().cuda()
            prompts_nocond = torch.Tensor(prompts_nocond).long().cuda()

            scheduler = DDPMScheduler.from_pretrained('runwayml/stable-diffusion-v1-5', subfolder="scheduler") 
            scheduler.set_timesteps(slider_step) 
            noise = torch.randn((slider_batch, 4, 64, 64)).to("cuda").half()
            input = noise

            encoder_hidden_states_cond = text_encoder(prompts_cond)[0].half()
            encoder_hidden_states_nocond = text_encoder(prompts_nocond)[0].half()


            for t in tqdm(scheduler.timesteps):
                with torch.no_grad():  # classifier free guidance
                    noise_pred_cond = unet(sample=input, timestep=t, encoder_hidden_states=encoder_hidden_states_cond[:slider_batch]).sample # b, 4, 64, 64
                    noise_pred_uncond = unet(sample=input, timestep=t, encoder_hidden_states=encoder_hidden_states_nocond[:slider_batch]).sample # b, 4, 64, 64
                    noisy_residual = noise_pred_uncond + slider_guidance * (noise_pred_cond - noise_pred_uncond) # b, 4, 64, 64     
                    input = scheduler.step(noisy_residual, t, input).prev_sample
                    del noise_pred_cond
                    del noise_pred_uncond

                    torch.cuda.empty_cache()

            # decode
            input = 1 / vae.config.scaling_factor * input 
            images = vae.decode(input, return_dict=False)[0] 
            width, height = 512, 512
            results = []
            new_image = Image.new('RGB', (2*width, 2*height))
            for index, image in enumerate(images.cpu().float()):
                image = (image / 2 + 0.5).clamp(0, 1).unsqueeze(0)
                image = image.cpu().permute(0, 2, 3, 1).numpy()[0]
                image = Image.fromarray((image * 255).round().astype("uint8")).convert('RGB')
                results.append(image)
                row = index // 2
                col = index % 2
                new_image.paste(image, (col*width, row*height))
            # os.system('nvidia-smi')
            torch.cuda.empty_cache()
            # os.system('nvidia-smi')
            return tuple(results),  composed_prompt, layout_image
        
        elif radio == 'TextDiffuser-2-LCM':
            generator = torch.Generator(device=pipe.device).manual_seed(random.randint(0,1000))
            image = pipe(
                prompt=user_prompt,
                generator=generator,
                # negative_prompt=negative_prompt,
                num_inference_steps=slider_step,
                guidance_scale=1,
                # num_images_per_prompt=slider_batch,
            ).images
            # os.system('nvidia-smi')
            torch.cuda.empty_cache()
            # os.system('nvidia-smi')
            return tuple(image), composed_prompt, layout_image
        
with gr.Blocks() as demo:


    # guest_id = random.randint(0,100000000)
    # register


    gr.HTML(
        """
        <div style="text-align: center; max-width: 1600px; margin: 20px auto;">
        <h2 style="font-weight: 900; font-size: 2.3rem; margin: 0rem">
            TextDiffuser-2: Unleashing the Power of Language Models for Text Rendering
        </h2>
        <h2 style="font-weight: 460; font-size: 1.1rem; margin: 0rem">
            <a href="https://jingyechen.github.io/">Jingye Chen</a>, <a href="https://hypjudy.github.io/website/">Yupan Huang</a>, <a href="https://scholar.google.com/citations?user=0LTZGhUAAAAJ&hl=en">Tengchao Lv</a>, <a href="https://www.microsoft.com/en-us/research/people/lecu/">Lei Cui</a>, <a href="https://cqf.io/">Qifeng Chen</a>, <a href="https://thegenerality.com/">Furu Wei</a>
        </h2>      
        <h2 style="font-weight: 460; font-size: 1.1rem; margin: 0rem">
            HKUST, Sun Yat-sen University, Microsoft Research
        </h2>  
        <h3 style="font-weight: 450; font-size: 1rem; margin: 0rem"> 
        [<a href="https://arxiv.org/abs/2311.16465" style="color:blue;">arXiv</a>] 
        [<a href="https://github.com/microsoft/unilm/tree/master/textdiffuser-2" style="color:blue;">Code</a>]
        [<a href="https://jingyechen.github.io/textdiffuser2/" style="color:blue;">Project Page</a>]
        [<a href="https://discord.gg/q7eHPupu" style="color:purple;">Discord</a>]
        </h3> 
        <h2 style="text-align: left; font-weight: 450; font-size: 1rem; margin-top: 0.5rem; margin-bottom: 0.5rem">
        We propose <b>TextDiffuser-2</b>, aiming at unleashing the power of language models for text rendering. Specifically, we <b>tame a language model into a layout planner</b> to transform user prompt into a layout using the caption-OCR pairs. The language model demonstrates flexibility and automation by inferring keywords from user prompts or incorporating user-specified keywords to determine their positions. Secondly, we <b>leverage the language model in the diffusion model as the layout encoder</b> to represent the position and content of text at the line level. This approach enables diffusion models to generate text images with broader diversity.
        </h2>
        <h2 style="text-align: left; font-weight: 450; font-size: 1rem; margin-top: 0.5rem; margin-bottom: 0.5rem">
        πŸ‘€ <b>Tips for using this demo</b>: <b>(1)</b> Please carefully read the disclaimer in the below. Current verison can only support English. <b>(2)</b> The specification of keywords is optional. If provided, the language model will do its best to plan layouts using the given keywords. <b>(3)</b> If a template is given, the layout planner (M1) is not used. <b>(4)</b> Three operations, including redo, undo, and skip are provided. When using skip, only the left-top point of a keyword will be recorded, resulting in more diversity but sometimes decreasing the accuracy. <b>(5)</b> The layout planner can produce different layouts. You can increase the temperature to enhance the diversity. ✨ <b>(6)</b> We also provide the experimental demo combining <b>TextDiffuser-2</b> and <b>LCM</b>. The inference is fast using less sampling steps, although the precision in text rendering might decrease. πŸ”₯ Demo for text inpainting is released at <a href="https://huggingface.co/spaces/JingyeChen22/TextDiffuser-2-Text-Inpainting">Link</a>. Welcome to play with it!
        </h2>
        <img src="https://raw.githubusercontent.com/JingyeChen/jingyechen.github.io/master/textdiffuser2/static/images/architecture_blank.jpg" alt="textdiffuser-2">
        </div>
        """)

    with gr.Tab("Text-to-Image"):
        with gr.Row():
            with gr.Column(scale=1):
                prompt = gr.Textbox(label="Prompt. You can let language model automatically identify keywords, or provide them below", placeholder="A beautiful city skyline stamp of Shanghai")
                keywords = gr.Textbox(label="(Optional) Keywords. Should be seperated by / (e.g., keyword1/keyword2/...)", placeholder="keyword1/keyword2")
                positive_prompt = gr.Textbox(label="(Optional) Positive prompt", value=", digital art, very detailed, fantasy, high definition, cinematic light, dnd, trending on artstation")

                # many encounter concurrent problem
                with gr.Accordion("(Optional) Template - Click to paint", open=False):
                    with gr.Row():
                        with gr.Column(scale=1):
                            i = gr.Image(label="Canvas", type='filepath', value=f'./gray256.jpg', height=256, width=256)
                        with gr.Column(scale=1):
                            t = gr.Textbox(label="Keyword", value='input_keyword')
                            redo = gr.Button(value='Redo - Cancel the last keyword') 
                            undo = gr.Button(value='Undo - Clear the canvas') 
                            skip_button = gr.Button(value='Skip - Operate the next keyword') 


                radio = gr.Radio(["TextDiffuser-2", "TextDiffuser-2-LCM"], label="Choice of models", value="TextDiffuser-2")
                slider_natural = gr.Checkbox(label="Natural image generation", value=False, info="The text position and content info will not be incorporated.")
                slider_step = gr.Slider(minimum=1, maximum=50, value=20, step=1, label="Sampling step", info="The sampling step for TextDiffuser-2. You may decease the step to 4 when using LCM.")
                slider_guidance = gr.Slider(minimum=1, maximum=13, value=7.5, step=0.5, label="Scale of classifier-free guidance", info="The scale of cfg and is set to 7.5 in default. When using LCM, cfg is set to 1.")
                slider_batch = gr.Slider(minimum=1, maximum=6, value=4, step=1, label="Batch size", info="The number of images to be sampled.")
                slider_temperature = gr.Slider(minimum=0.1, maximum=2, value=1.4, step=0.1, label="Temperature", info="Control the diversity of layout planner. Higher value indicates more diversity.")
                # slider_seed = gr.Slider(minimum=1, maximum=10000, label="Seed", randomize=True)
                button = gr.Button("Generate")



                guest_id_box = gr.Textbox(label="guest_id", value=f"-1", visible=False)
                i.select(get_pixels,[i,t,guest_id_box],[i,guest_id_box])
                redo.click(exe_redo, [i,t,guest_id_box],[i])
                undo.click(exe_undo, [i,t,guest_id_box],[i])
                skip_button.click(skip_fun, [i,t,guest_id_box])

                            
            with gr.Column(scale=1):
                output = gr.Gallery(label='Generated image')

                with gr.Accordion("Intermediate results", open=False):
                    gr.Markdown("Composed prompt")
                    composed_prompt = gr.Textbox(label='')
                    gr.Markdown("Layout visualization")
                    layout = gr.Image(height=256, width=256)


        button.click(text_to_image, inputs=[guest_id_box, prompt,keywords,positive_prompt, radio,slider_step,slider_guidance,slider_batch,slider_temperature,slider_natural], outputs=[output, composed_prompt, layout])

        gr.Markdown("## Prompt Examples")
        gr.Examples(
            [
                ["A new year greeting card of happy 2024, surrounded by ballons", "", False],
                ["A beautiful city skyline stamp of Shanghai", "", False],
                ["The words 'KFC VIVO50' are inscribed upon the wall in a neon light effect", "KFC/VIVO50", False],
                ["A logo of superman", "", False],
                ["A pencil sketch of a tree with the title nothing to tree here", "", False],
                ["handwritten signature of peter", "", False],
                ["Delicate greeting card of happy birthday to xyz", "", False],
                ["Book cover of good morning baby ", "", False],
                ["The handwritten words Hello World displayed on a wall in a neon light effect", "", False],
                ["Logo of winter in artistic font, made by snowflake", "", False],
                ["A book cover named summer vibe", "", False],
                ["Newspaper with the title Love Story", "", False],
                ["A logo for the company EcoGrow, where the letters look like plants", "EcoGrow", False],
                ["A poster titled 'Quails of North America', showing different kinds of quails.", "Quails/of/North/America", False],
                ["A detailed portrait of a fox guardian with a shield with Kung Fu written on it, by victo ngai and justin gerard, digital art, realistic painting", "kung/fu", False],
                ["A stamp of breath of the wild", "breath/of/the/wild", False],
                ["Poster of the incoming movie Transformers", "Transformers", False],
                ["Some apples are on a table", "", True],
                ["a hotdog with mustard and other toppings on it", "", True],
                ["a bathroom that has a slanted ceiling and a large bath tub", "", True],
                ["a man holding a tennis racquet on a tennis court", "", True],
                ["hamburger with bacon, lettuce, tomato and cheese| promotional image| hyperquality| products shot| full - color| extreme render| mouthwatering", "", True],
            ],
            [
                prompt,
                keywords,
                slider_natural
            ],
            examples_per_page=25
        )

    gr.HTML(
        """
        <div style="text-align: justify; max-width: 1100px; margin: 20px auto;">
        <h3 style="font-weight: 450; font-size: 0.8rem; margin: 0rem">
        <b>Version</b>: 1.0
        </h3>
        <h3 style="font-weight: 450; font-size: 0.8rem; margin: 0rem">
        <b>Contact</b>: 
        For help or issues using TextDiffuser-2, please email Jingye Chen <a href="mailto:qwerty.chen@connect.ust.hk">(qwerty.chen@connect.ust.hk)</a>, Yupan Huang <a href="mailto:huangyp28@mail2.sysu.edu.cn">(huangyp28@mail2.sysu.edu.cn)</a> or submit a GitHub issue. For other communications related to TextDiffuser-2, please contact Lei Cui <a href="mailto:lecu@microsoft.com">(lecu@microsoft.com)</a> or Furu Wei <a href="mailto:fuwei@microsoft.com">(fuwei@microsoft.com)</a>.
        </h3>
        <h3 style="font-weight: 450; font-size: 0.8rem; margin: 0rem">
        <b>Disclaimer</b>: 
        Please note that the demo is intended for academic and research purposes <b>ONLY</b>. Any use of the demo for generating inappropriate content is strictly prohibited. The responsibility for any misuse or inappropriate use of the demo lies solely with the users who generated such content, and this demo shall not be held liable for any such use.
        </h3>
        </div>
        """
    )


demo.launch()