Spaces:
Running
on
Zero
Running
on
Zero
Update app.py
Browse files
app.py
CHANGED
@@ -1,123 +1,87 @@
|
|
1 |
import spaces
|
2 |
import gradio as gr
|
3 |
import torch
|
4 |
-
from
|
5 |
-
from
|
6 |
-
import
|
7 |
import random
|
8 |
import numpy as np
|
9 |
-
import os
|
10 |
-
from huggingface_hub import snapshot_download
|
11 |
|
12 |
# Initialize models
|
13 |
device = "cuda" if torch.cuda.is_available() else "cpu"
|
14 |
-
dtype = torch.
|
15 |
|
16 |
-
|
|
|
17 |
|
18 |
-
|
19 |
-
|
20 |
-
|
21 |
-
repo_type="model",
|
22 |
-
ignore_patterns=["*.md", "*..gitattributes"],
|
23 |
-
local_dir="SD3",
|
24 |
-
token=huggingface_token, # type a new token-id.
|
25 |
-
)
|
26 |
-
|
27 |
-
# VLM Captioner
|
28 |
-
vlm_model = PaliGemmaForConditionalGeneration.from_pretrained("gokaygokay/sd3-long-captioner-v2").to(device).eval()
|
29 |
-
vlm_processor = PaliGemmaProcessor.from_pretrained("gokaygokay/sd3-long-captioner-v2")
|
30 |
|
31 |
# Prompt Enhancer
|
32 |
-
enhancer_medium = pipeline("summarization", model="gokaygokay/Lamini-Prompt-Enchance", device=device)
|
33 |
enhancer_long = pipeline("summarization", model="gokaygokay/Lamini-Prompt-Enchance-Long", device=device)
|
34 |
|
35 |
-
# SD3
|
36 |
-
sd3_pipe = StableDiffusion3Pipeline.from_pretrained(model_path, torch_dtype=dtype).to(device)
|
37 |
-
|
38 |
MAX_SEED = np.iinfo(np.int32).max
|
39 |
-
MAX_IMAGE_SIZE =
|
40 |
-
|
41 |
-
# VLM Captioner function
|
42 |
-
def create_captions_rich(image):
|
43 |
-
prompt = "caption en"
|
44 |
-
model_inputs = vlm_processor(text=prompt, images=image, return_tensors="pt").to(device)
|
45 |
-
input_len = model_inputs["input_ids"].shape[-1]
|
46 |
|
47 |
-
|
48 |
-
|
49 |
-
|
50 |
-
|
51 |
-
|
52 |
-
return modify_caption(decoded)
|
53 |
-
|
54 |
-
# Helper function for caption modification
|
55 |
-
def modify_caption(caption: str) -> str:
|
56 |
-
prefix_substrings = [
|
57 |
-
('captured from ', ''),
|
58 |
-
('captured at ', '')
|
59 |
-
]
|
60 |
-
pattern = '|'.join([re.escape(opening) for opening, _ in prefix_substrings])
|
61 |
-
replacers = {opening: replacer for opening, replacer in prefix_substrings}
|
62 |
|
63 |
-
|
64 |
-
|
65 |
-
|
66 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
67 |
|
68 |
# Prompt Enhancer function
|
69 |
-
def enhance_prompt(input_prompt
|
70 |
-
|
71 |
-
|
72 |
-
enhanced_text = result[0]['summary_text']
|
73 |
-
|
74 |
-
pattern = r'^.*?of\s+(.*?(?:\.|$))'
|
75 |
-
match = re.match(pattern, enhanced_text, re.IGNORECASE | re.DOTALL)
|
76 |
-
|
77 |
-
if match:
|
78 |
-
remaining_text = enhanced_text[match.end():].strip()
|
79 |
-
modified_sentence = match.group(1).capitalize()
|
80 |
-
enhanced_text = modified_sentence + ' ' + remaining_text
|
81 |
-
else: # Long
|
82 |
-
result = enhancer_long("Enhance the description: " + input_prompt)
|
83 |
-
enhanced_text = result[0]['summary_text']
|
84 |
-
|
85 |
return enhanced_text
|
86 |
|
87 |
-
|
88 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
89 |
if randomize_seed:
|
90 |
seed = random.randint(0, MAX_SEED)
|
91 |
|
92 |
-
generator = torch.Generator().manual_seed(seed)
|
93 |
|
94 |
-
image =
|
95 |
-
prompt=prompt,
|
96 |
-
|
97 |
-
|
98 |
-
|
99 |
-
width=width,
|
100 |
height=height,
|
101 |
-
|
102 |
).images[0]
|
103 |
|
104 |
-
return image, seed
|
105 |
-
|
106 |
-
# Gradio Interface
|
107 |
-
@spaces.GPU
|
108 |
-
def process_workflow(image, text_prompt, use_vlm, use_enhancer, model_choice, negative_prompt, seed, randomize_seed, width, height, guidance_scale, num_inference_steps):
|
109 |
-
if use_vlm and image is not None:
|
110 |
-
prompt = create_captions_rich(image)
|
111 |
-
else:
|
112 |
-
prompt = text_prompt
|
113 |
-
|
114 |
-
if use_enhancer:
|
115 |
-
prompt = enhance_prompt(prompt, model_choice)
|
116 |
-
|
117 |
-
generated_image, used_seed = generate_image(prompt, negative_prompt, seed, randomize_seed, width, height, guidance_scale, num_inference_steps)
|
118 |
-
|
119 |
-
return generated_image, prompt, used_seed
|
120 |
-
|
121 |
|
122 |
custom_css = """
|
123 |
.input-group, .output-group {
|
@@ -136,55 +100,46 @@ custom_css = """
|
|
136 |
}
|
137 |
"""
|
138 |
|
139 |
-
title = """<h1 align="center">
|
140 |
<p><center>
|
141 |
-
<a href="https://huggingface.co/
|
|
|
142 |
<a href="https://huggingface.co/gokaygokay/Lamini-Prompt-Enchance-Long" target="_blank">[Prompt Enhancer Long]</a>
|
143 |
-
<
|
144 |
-
<a href="https://github.com/gokayfem" target="_blank">[Github]</a>
|
145 |
-
<a href="https://x.com/NONDA30" target="_blank">[X/twitter]</a>
|
146 |
-
<p align="center">Dont forget to click <b>Use VLM Captioner</b> or <b>Use Prompt Enhancer</b> Buttons!</p>
|
147 |
</center></p>
|
148 |
"""
|
149 |
|
150 |
-
# Gradio Interface
|
151 |
with gr.Blocks(css=custom_css, theme=gr.themes.Soft(primary_hue="blue", secondary_hue="gray")) as demo:
|
152 |
-
|
153 |
gr.HTML(title)
|
154 |
|
155 |
with gr.Row():
|
156 |
with gr.Column(scale=1):
|
157 |
with gr.Group(elem_classes="input-group"):
|
158 |
-
input_image = gr.Image(label="Input Image
|
159 |
-
use_vlm = gr.Checkbox(label="Use VLM Captioner", value=False)
|
160 |
-
|
161 |
-
with gr.Group(elem_classes="input-group"):
|
162 |
-
text_prompt = gr.Textbox(label="Text Prompt")
|
163 |
-
use_enhancer = gr.Checkbox(label="Use Prompt Enhancer", value=False)
|
164 |
-
model_choice = gr.Radio(["Medium", "Long"], label="Enhancer Model", value="Long")
|
165 |
|
166 |
with gr.Accordion("Advanced Settings", open=False):
|
167 |
-
|
|
|
168 |
seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
|
169 |
randomize_seed = gr.Checkbox(label="Randomize Seed", value=True)
|
170 |
-
width = gr.Slider(label="Width", minimum=256, maximum=MAX_IMAGE_SIZE, step=
|
171 |
-
height = gr.Slider(label="Height", minimum=256, maximum=MAX_IMAGE_SIZE, step=
|
172 |
-
guidance_scale = gr.Slider(label="Guidance Scale", minimum=
|
173 |
num_inference_steps = gr.Slider(label="Inference Steps", minimum=1, maximum=50, step=1, value=28)
|
174 |
|
175 |
generate_btn = gr.Button("Generate Image", elem_classes="submit-btn")
|
176 |
|
177 |
with gr.Column(scale=1):
|
178 |
with gr.Group(elem_classes="output-group"):
|
179 |
-
output_image = gr.Image(label="
|
180 |
final_prompt = gr.Textbox(label="Final Prompt Used")
|
181 |
used_seed = gr.Number(label="Seed Used")
|
182 |
|
183 |
generate_btn.click(
|
184 |
fn=process_workflow,
|
185 |
inputs=[
|
186 |
-
input_image, text_prompt,
|
187 |
-
|
188 |
],
|
189 |
outputs=[output_image, final_prompt, used_seed]
|
190 |
)
|
|
|
1 |
import spaces
|
2 |
import gradio as gr
|
3 |
import torch
|
4 |
+
from PIL import Image
|
5 |
+
from transformers import AutoProcessor, AutoModelForCausalLM, pipeline
|
6 |
+
from diffusers import DiffusionPipeline
|
7 |
import random
|
8 |
import numpy as np
|
|
|
|
|
9 |
|
10 |
# Initialize models
|
11 |
device = "cuda" if torch.cuda.is_available() else "cpu"
|
12 |
+
dtype = torch.bfloat16
|
13 |
|
14 |
+
# FLUX.1-dev model
|
15 |
+
pipe = DiffusionPipeline.from_pretrained("black-forest-labs/FLUX.1-dev", torch_dtype=dtype).to(device)
|
16 |
|
17 |
+
# Initialize Florence model
|
18 |
+
florence_model = AutoModelForCausalLM.from_pretrained('microsoft/Florence-2-base', trust_remote_code=True).to(device).eval()
|
19 |
+
florence_processor = AutoProcessor.from_pretrained('microsoft/Florence-2-base', trust_remote_code=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
20 |
|
21 |
# Prompt Enhancer
|
|
|
22 |
enhancer_long = pipeline("summarization", model="gokaygokay/Lamini-Prompt-Enchance-Long", device=device)
|
23 |
|
|
|
|
|
|
|
24 |
MAX_SEED = np.iinfo(np.int32).max
|
25 |
+
MAX_IMAGE_SIZE = 2048
|
|
|
|
|
|
|
|
|
|
|
|
|
26 |
|
27 |
+
# Florence caption function
|
28 |
+
def florence_caption(image):
|
29 |
+
# Convert image to PIL if it's not already
|
30 |
+
if not isinstance(image, Image.Image):
|
31 |
+
image = Image.fromarray(image)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
32 |
|
33 |
+
inputs = florence_processor(text="<MORE_DETAILED_CAPTION>", images=image, return_tensors="pt").to(device)
|
34 |
+
generated_ids = florence_model.generate(
|
35 |
+
input_ids=inputs["input_ids"],
|
36 |
+
pixel_values=inputs["pixel_values"],
|
37 |
+
max_new_tokens=1024,
|
38 |
+
early_stopping=False,
|
39 |
+
do_sample=False,
|
40 |
+
num_beams=3,
|
41 |
+
)
|
42 |
+
generated_text = florence_processor.batch_decode(generated_ids, skip_special_tokens=False)[0]
|
43 |
+
parsed_answer = florence_processor.post_process_generation(
|
44 |
+
generated_text,
|
45 |
+
task="<MORE_DETAILED_CAPTION>",
|
46 |
+
image_size=(image.width, image.height)
|
47 |
+
)
|
48 |
+
return parsed_answer["<MORE_DETAILED_CAPTION>"]
|
49 |
|
50 |
# Prompt Enhancer function
|
51 |
+
def enhance_prompt(input_prompt):
|
52 |
+
result = enhancer_long("Enhance the description: " + input_prompt)
|
53 |
+
enhanced_text = result[0]['summary_text']
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
54 |
return enhanced_text
|
55 |
|
56 |
+
@spaces.GPU(duration=190)
|
57 |
+
def process_workflow(image, text_prompt, use_enhancer, seed, randomize_seed, width, height, guidance_scale, num_inference_steps, progress=gr.Progress(track_tqdm=True)):
|
58 |
+
if image is not None:
|
59 |
+
# Convert image to PIL if it's not already
|
60 |
+
if not isinstance(image, Image.Image):
|
61 |
+
image = Image.fromarray(image)
|
62 |
+
|
63 |
+
prompt = florence_caption(image)
|
64 |
+
else:
|
65 |
+
prompt = text_prompt
|
66 |
+
|
67 |
+
if use_enhancer:
|
68 |
+
prompt = enhance_prompt(prompt)
|
69 |
+
|
70 |
if randomize_seed:
|
71 |
seed = random.randint(0, MAX_SEED)
|
72 |
|
73 |
+
generator = torch.Generator(device=device).manual_seed(seed)
|
74 |
|
75 |
+
image = pipe(
|
76 |
+
prompt=prompt,
|
77 |
+
generator=generator,
|
78 |
+
num_inference_steps=num_inference_steps,
|
79 |
+
width=width,
|
|
|
80 |
height=height,
|
81 |
+
guidance_scale=guidance_scale
|
82 |
).images[0]
|
83 |
|
84 |
+
return image, prompt, seed
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
85 |
|
86 |
custom_css = """
|
87 |
.input-group, .output-group {
|
|
|
100 |
}
|
101 |
"""
|
102 |
|
103 |
+
title = """<h1 align="center">FLUX.1-dev with Florence-2 Captioner and Prompt Enhancer</h1>
|
104 |
<p><center>
|
105 |
+
<a href="https://huggingface.co/black-forest-labs/FLUX.1-dev" target="_blank">[FLUX.1-dev Model]</a>
|
106 |
+
<a href="https://huggingface.co/microsoft/Florence-2-base" target="_blank">[Florence-2 Model]</a>
|
107 |
<a href="https://huggingface.co/gokaygokay/Lamini-Prompt-Enchance-Long" target="_blank">[Prompt Enhancer Long]</a>
|
108 |
+
<p align="center">Create long prompts from images or enhance your short prompts with prompt enhancer</p>
|
|
|
|
|
|
|
109 |
</center></p>
|
110 |
"""
|
111 |
|
|
|
112 |
with gr.Blocks(css=custom_css, theme=gr.themes.Soft(primary_hue="blue", secondary_hue="gray")) as demo:
|
|
|
113 |
gr.HTML(title)
|
114 |
|
115 |
with gr.Row():
|
116 |
with gr.Column(scale=1):
|
117 |
with gr.Group(elem_classes="input-group"):
|
118 |
+
input_image = gr.Image(label="Input Image (Florence-2 Captioner)")
|
|
|
|
|
|
|
|
|
|
|
|
|
119 |
|
120 |
with gr.Accordion("Advanced Settings", open=False):
|
121 |
+
text_prompt = gr.Textbox(label="Text Prompt (optional, used if no image is uploaded)")
|
122 |
+
use_enhancer = gr.Checkbox(label="Use Prompt Enhancer", value=False)
|
123 |
seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0)
|
124 |
randomize_seed = gr.Checkbox(label="Randomize Seed", value=True)
|
125 |
+
width = gr.Slider(label="Width", minimum=256, maximum=MAX_IMAGE_SIZE, step=32, value=1024)
|
126 |
+
height = gr.Slider(label="Height", minimum=256, maximum=MAX_IMAGE_SIZE, step=32, value=1024)
|
127 |
+
guidance_scale = gr.Slider(label="Guidance Scale", minimum=1, maximum=15, step=0.1, value=3.5)
|
128 |
num_inference_steps = gr.Slider(label="Inference Steps", minimum=1, maximum=50, step=1, value=28)
|
129 |
|
130 |
generate_btn = gr.Button("Generate Image", elem_classes="submit-btn")
|
131 |
|
132 |
with gr.Column(scale=1):
|
133 |
with gr.Group(elem_classes="output-group"):
|
134 |
+
output_image = gr.Image(label="Result", elem_id="gallery", show_label=False)
|
135 |
final_prompt = gr.Textbox(label="Final Prompt Used")
|
136 |
used_seed = gr.Number(label="Seed Used")
|
137 |
|
138 |
generate_btn.click(
|
139 |
fn=process_workflow,
|
140 |
inputs=[
|
141 |
+
input_image, text_prompt, use_enhancer, seed, randomize_seed,
|
142 |
+
width, height, guidance_scale, num_inference_steps
|
143 |
],
|
144 |
outputs=[output_image, final_prompt, used_seed]
|
145 |
)
|