hotbiz commited on
Commit
3b8ee7b
1 Parent(s): 35073bc

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +350 -1
app.py CHANGED
@@ -1 +1,350 @@
1
- git commit -am 'Update space' && git push
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ import torch
3
+ from diffusers import StableDiffusionXLPipeline, AutoencoderKL
4
+ from huggingface_hub import hf_hub_download
5
+ from safetensors.torch import load_file
6
+ from share_btn import community_icon_html, loading_icon_html, share_js
7
+ from cog_sdxl_dataset_and_utils import TokenEmbeddingsHandler
8
+ import lora
9
+ import copy
10
+ import json
11
+ import gc
12
+ import random
13
+ from urllib.parse import quote
14
+ with open("sdxl_loras.json", "r") as file:
15
+ data = json.load(file)
16
+ sdxl_loras_raw = [
17
+ {
18
+ "image": item["image"],
19
+ "title": item["title"],
20
+ "repo": item["repo"],
21
+ "trigger_word": item["trigger_word"],
22
+ "weights": item["weights"],
23
+ "is_compatible": item["is_compatible"],
24
+ "is_pivotal": item.get("is_pivotal", False),
25
+ "text_embedding_weights": item.get("text_embedding_weights", None),
26
+ "likes": item.get("likes", 0),
27
+ "downloads": item.get("downloads", 0),
28
+ "is_nc": item.get("is_nc", False),
29
+ "new": item.get("new", False),
30
+ }
31
+ for item in data
32
+ ]
33
+
34
+ device = "cuda"
35
+
36
+ state_dicts = {}
37
+
38
+ for item in sdxl_loras_raw:
39
+ saved_name = hf_hub_download(item["repo"], item["weights"])
40
+
41
+ if not saved_name.endswith('.safetensors'):
42
+ state_dict = torch.load(saved_name)
43
+ else:
44
+ state_dict = load_file(saved_name)
45
+
46
+ state_dicts[item["repo"]] = {
47
+ "saved_name": saved_name,
48
+ "state_dict": state_dict
49
+ }
50
+
51
+ sdxl_loras_raw_new = [item for item in sdxl_loras_raw if item.get("new") == True]
52
+
53
+ sdxl_loras_raw = [item for item in sdxl_loras_raw if item.get("new") != True]
54
+
55
+ lcm_lora_id = "lcm-sd/lcm-sdxl-base-1.0-lora"
56
+
57
+ vae = AutoencoderKL.from_pretrained(
58
+ "madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16
59
+ )
60
+ pipe = StableDiffusionXLPipeline.from_pretrained(
61
+ "stabilityai/stable-diffusion-xl-base-1.0",
62
+ vae=vae,
63
+ torch_dtype=torch.float16,
64
+ )
65
+ original_pipe = copy.deepcopy(pipe)
66
+ pipe.to(device)
67
+
68
+ last_lora = ""
69
+ last_merged = False
70
+ last_fused = False
71
+ js = '''
72
+ var button = document.getElementById('button');
73
+ // Add a click event listener to the button
74
+ button.addEventListener('click', function() {
75
+ element.classList.add('selected');
76
+ });
77
+ '''
78
+ def update_selection(selected_state: gr.SelectData, sdxl_loras, is_new=False):
79
+ lora_repo = sdxl_loras[selected_state.index]["repo"]
80
+ instance_prompt = sdxl_loras[selected_state.index]["trigger_word"]
81
+ new_placeholder = "Type a prompt. This LoRA applies for all prompts, no need for a trigger word" if instance_prompt == "" else "Type a prompt to use your selected LoRA"
82
+ weight_name = sdxl_loras[selected_state.index]["weights"]
83
+ updated_text = f"### Selected: [{lora_repo}](https://huggingface.co/{lora_repo}) ✨ {'(non-commercial LoRA, `cc-by-nc`)' if sdxl_loras[selected_state.index]['is_nc'] else '' }"
84
+ is_compatible = sdxl_loras[selected_state.index]["is_compatible"]
85
+ is_pivotal = sdxl_loras[selected_state.index]["is_pivotal"]
86
+
87
+ use_with_diffusers = f'''
88
+ ## Using [`{lora_repo}`](https://huggingface.co/{lora_repo})
89
+
90
+ ## Use it with diffusers:
91
+ '''
92
+ if is_compatible:
93
+ use_with_diffusers += f'''
94
+ from diffusers import StableDiffusionXLPipeline
95
+ import torch
96
+
97
+ model_path = "stabilityai/stable-diffusion-xl-base-1.0"
98
+ pipe = StableDiffusionPipeline.from_pretrained(model_path, torch_dtype=torch.float16)
99
+ pipe.to("cuda")
100
+ pipe.load_lora_weights("{lora_repo}", weight_name="{weight_name}")
101
+
102
+ prompt = "{instance_prompt}..."
103
+ lora_scale= 0.9
104
+ image = pipe(prompt, num_inference_steps=30, guidance_scale=7.5, cross_attention_kwargs={{"scale": lora_scale}}).images[0]
105
+ image.save("image.png")
106
+ '''
107
+ elif not is_pivotal:
108
+ use_with_diffusers += "This LoRA is not compatible with diffusers natively yet. But you can still use it on diffusers with `bmaltais/kohya_ss` LoRA class, check out this [Google Colab](https://colab.research.google.com/drive/14aEJsKdEQ9_kyfsiV6JDok799kxPul0j )"
109
+ else:
110
+ use_with_diffusers += f"This LoRA is not compatible with diffusers natively yet. But you can still use it on diffusers with sdxl-cog `TokenEmbeddingsHandler` class, check out the [model repo](https://huggingface.co/{lora_repo}#inference-with-🧨-diffusers)"
111
+ use_with_uis = f'''
112
+ ## Use it with Comfy UI, Invoke AI, SD.Next, AUTO1111:
113
+
114
+ ### Download the `*.safetensors` weights of [here](https://huggingface.co/{lora_repo}/resolve/main/{weight_name})
115
+
116
+ - [ComfyUI guide](https://comfyanonymous.github.io/ComfyUI_examples/lora/)
117
+ - [Invoke AI guide](https://invoke-ai.github.io/InvokeAI/features/CONCEPTS/?h=lora#using-loras)
118
+ - [SD.Next guide](https://github.com/vladmandic/automatic)
119
+ - [AUTOMATIC1111 guide](https://stable-diffusion-art.com/lora/)
120
+ '''
121
+ if(is_new):
122
+ if(selected_state.index == 0):
123
+ selected_state.index = -9999
124
+ else:
125
+ selected_state.index *= -1
126
+
127
+ return (
128
+ updated_text,
129
+ instance_prompt,
130
+ gr.update(placeholder=new_placeholder),
131
+ selected_state,
132
+ use_with_diffusers,
133
+ use_with_uis,
134
+ gr.Gallery(selected_index=None)
135
+ )
136
+
137
+
138
+ def check_selected(selected_state):
139
+ if not selected_state:
140
+ raise gr.Error("You must select a LoRA")
141
+
142
+ def merge_incompatible_lora(full_path_lora, lora_scale):
143
+ for weights_file in [full_path_lora]:
144
+ if ";" in weights_file:
145
+ weights_file, multiplier = weights_file.split(";")
146
+ multiplier = float(multiplier)
147
+ else:
148
+ multiplier = lora_scale
149
+
150
+ lora_model, weights_sd = lora.create_network_from_weights(
151
+ multiplier,
152
+ full_path_lora,
153
+ pipe.vae,
154
+ pipe.text_encoder,
155
+ pipe.unet,
156
+ for_inference=True,
157
+ )
158
+ lora_model.merge_to(
159
+ pipe.text_encoder, pipe.unet, weights_sd, torch.float16, "cuda"
160
+ )
161
+ del weights_sd
162
+ del lora_model
163
+ gc.collect()
164
+
165
+ def run_lora(prompt, negative, lora_scale, selected_state, sdxl_loras, sdxl_loras_new, progress=gr.Progress(track_tqdm=True)):
166
+ global last_lora, last_merged, last_fused, pipe
167
+ print("Index when running ", selected_state.index)
168
+ if(selected_state.index < 0):
169
+ if(selected_state.index == -9999):
170
+ selected_state.index = 0
171
+ else:
172
+ selected_state.index *= -1
173
+ sdxl_loras = sdxl_loras_new
174
+ print("Selected State: ", selected_state.index)
175
+ print(sdxl_loras[selected_state.index]["repo"])
176
+ if negative == "":
177
+ negative = None
178
+
179
+ if not selected_state:
180
+ raise gr.Error("You must select a LoRA")
181
+ repo_name = sdxl_loras[selected_state.index]["repo"]
182
+ weight_name = sdxl_loras[selected_state.index]["weights"]
183
+
184
+ full_path_lora = state_dicts[repo_name]["saved_name"]
185
+ loaded_state_dict = copy.deepcopy(state_dicts[repo_name]["state_dict"])
186
+ cross_attention_kwargs = None
187
+ if last_lora != repo_name:
188
+ if(last_fused):
189
+ pipe.unfuse_lora()
190
+ pipe.load_lora_weights(loaded_state_dict)
191
+ pipe.fuse_lora()
192
+ last_fused = True
193
+ is_pivotal = sdxl_loras[selected_state.index]["is_pivotal"]
194
+ if(is_pivotal):
195
+ #Add the textual inversion embeddings from pivotal tuning models
196
+ text_embedding_name = sdxl_loras[selected_state.index]["text_embedding_weights"]
197
+ text_encoders = [pipe.text_encoder, pipe.text_encoder_2]
198
+ tokenizers = [pipe.tokenizer, pipe.tokenizer_2]
199
+ embedding_path = hf_hub_download(repo_id=repo_name, filename=text_embedding_name, repo_type="model")
200
+ embhandler = TokenEmbeddingsHandler(text_encoders, tokenizers)
201
+ embhandler.load_embeddings(embedding_path)
202
+
203
+ image = pipe(
204
+ prompt=prompt,
205
+ negative_prompt=negative,
206
+ width=1024,
207
+ height=1024,
208
+ num_inference_steps=20,
209
+ guidance_scale=7.5,
210
+ ).images[0]
211
+ last_lora = repo_name
212
+ gc.collect()
213
+ return image, gr.update(visible=True)
214
+
215
+ def shuffle_gallery(sdxl_loras):
216
+ random.shuffle(sdxl_loras)
217
+ return [(item["image"], item["title"]) for item in sdxl_loras], sdxl_loras
218
+
219
+ def swap_gallery(order, sdxl_loras):
220
+ if(order == "random"):
221
+ return shuffle_gallery(sdxl_loras)
222
+ else:
223
+ sorted_gallery = sorted(sdxl_loras, key=lambda x: x.get(order, 0), reverse=True)
224
+ return [(item["image"], item["title"]) for item in sorted_gallery], sorted_gallery
225
+
226
+ def deselect():
227
+ return gr.Gallery(selected_index=None)
228
+
229
+ with gr.Blocks(css="custom.css") as demo:
230
+ gr_sdxl_loras = gr.State(value=sdxl_loras_raw)
231
+ gr_sdxl_loras_new = gr.State(value=sdxl_loras_raw_new)
232
+ title = gr.HTML(
233
+ """<h1><img src="https://i.imgur.com/vT48NAO.png" alt="LoRA"> LoRA the Explorer</h1>""",
234
+ elem_id="title",
235
+ )
236
+ selected_state = gr.State()
237
+ with gr.Row(elem_id="main_app"):
238
+ with gr.Group(elem_id="gallery_box"):
239
+ selected_loras = gr.Gallery(label="Selected LoRAs", height=80, show_share_button=False, visible=False, elem_id="gallery_selected", )
240
+ order_gallery = gr.Radio(choices=["random", "likes"], value="random", label="Order by", elem_id="order_radio")
241
+ new_gallery = gr.Gallery(label="New LoRAs", elem_id="gallery_new", columns=3, value=[(item["image"], item["title"]) for item in sdxl_loras_raw_new], allow_preview=False, show_share_button=False)
242
+ gallery = gr.Gallery(
243
+ #value=[(item["image"], item["title"]) for item in sdxl_loras],
244
+ label="SDXL LoRA Gallery",
245
+ allow_preview=False,
246
+ columns=3,
247
+ elem_id="gallery",
248
+ show_share_button=False,
249
+ height=784
250
+ )
251
+ with gr.Column():
252
+ prompt_title = gr.Markdown(
253
+ value="### Click on a LoRA in the gallery to select it",
254
+ visible=True,
255
+ elem_id="selected_lora",
256
+ )
257
+ with gr.Row():
258
+ prompt = gr.Textbox(label="Prompt", show_label=False, lines=1, max_lines=1, placeholder="Type a prompt after selecting a LoRA", elem_id="prompt")
259
+ button = gr.Button("Run", elem_id="run_button")
260
+ with gr.Group(elem_id="share-btn-container", visible=False) as share_group:
261
+ community_icon = gr.HTML(community_icon_html)
262
+ loading_icon = gr.HTML(loading_icon_html)
263
+ share_button = gr.Button("Share to community", elem_id="share-btn")
264
+ result = gr.Image(
265
+ interactive=False, label="Generated Image", elem_id="result-image"
266
+ )
267
+ with gr.Accordion("Advanced options", open=False):
268
+ negative = gr.Textbox(label="Negative Prompt")
269
+ weight = gr.Slider(0, 10, value=0.8, step=0.1, label="LoRA weight")
270
+ with gr.Column(elem_id="extra_info"):
271
+ with gr.Accordion(
272
+ "Use it with: 🧨 diffusers, ComfyUI, Invoke AI, SD.Next, AUTO1111",
273
+ open=False,
274
+ elem_id="accordion",
275
+ ):
276
+ with gr.Row():
277
+ use_diffusers = gr.Markdown("""## Select a LoRA first 🤗""")
278
+ use_uis = gr.Markdown()
279
+ with gr.Accordion("Submit a LoRA! 📥", open=False):
280
+ submit_title = gr.Markdown(
281
+ "### Streamlined submission coming soon! Until then [suggest your LoRA in the community tab](https://huggingface.co/spaces/multimodalart/LoraTheExplorer/discussions) 🤗"
282
+ )
283
+ with gr.Group(elem_id="soon"):
284
+ submit_source = gr.Radio(
285
+ ["Hugging Face", "CivitAI"],
286
+ label="LoRA source",
287
+ value="Hugging Face",
288
+ )
289
+ with gr.Row():
290
+ submit_source_hf = gr.Textbox(
291
+ label="Hugging Face Model Repo",
292
+ info="In the format `username/model_id`",
293
+ )
294
+ submit_safetensors_hf = gr.Textbox(
295
+ label="Safetensors filename",
296
+ info="The filename `*.safetensors` in the model repo",
297
+ )
298
+ with gr.Row():
299
+ submit_trigger_word_hf = gr.Textbox(label="Trigger word")
300
+ submit_image = gr.Image(
301
+ label="Example image (optional if the repo already contains images)"
302
+ )
303
+ submit_button = gr.Button("Submit!")
304
+ submit_disclaimer = gr.Markdown(
305
+ "This is a curated gallery by me, [apolinário (multimodal.art)](https://twitter.com/multimodalart). I'll try to include as many cool LoRAs as they are submitted! You can [duplicate this Space](https://huggingface.co/spaces/multimodalart/LoraTheExplorer?duplicate=true) to use it privately, and add your own LoRAs by editing `sdxl_loras.json` in the Files tab of your private space."
306
+ )
307
+ order_gallery.change(
308
+ fn=swap_gallery,
309
+ inputs=[order_gallery, gr_sdxl_loras],
310
+ outputs=[gallery, gr_sdxl_loras],
311
+ queue=False
312
+ )
313
+ gallery.select(
314
+ fn=update_selection,
315
+ inputs=[gr_sdxl_loras],
316
+ outputs=[prompt_title, prompt, prompt, selected_state, use_diffusers, use_uis, new_gallery],
317
+ queue=False,
318
+ show_progress=False
319
+ )
320
+ new_gallery.select(
321
+ fn=update_selection,
322
+ inputs=[gr_sdxl_loras_new, gr.State(True)],
323
+ outputs=[prompt_title, prompt, prompt, selected_state, use_diffusers, use_uis, gallery],
324
+ queue=False,
325
+ show_progress=False
326
+ )
327
+ prompt.submit(
328
+ fn=check_selected,
329
+ inputs=[selected_state],
330
+ queue=False,
331
+ show_progress=False
332
+ ).success(
333
+ fn=run_lora,
334
+ inputs=[prompt, negative, weight, selected_state, gr_sdxl_loras, gr_sdxl_loras_new],
335
+ outputs=[result, share_group],
336
+ )
337
+ button.click(
338
+ fn=check_selected,
339
+ inputs=[selected_state],
340
+ queue=False,
341
+ show_progress=False
342
+ ).success(
343
+ fn=run_lora,
344
+ inputs=[prompt, negative, weight, selected_state, gr_sdxl_loras, gr_sdxl_loras_new],
345
+ outputs=[result, share_group],
346
+ )
347
+ share_button.click(None, [], [], js=share_js)
348
+ demo.load(fn=shuffle_gallery, inputs=[gr_sdxl_loras], outputs=[gallery, gr_sdxl_loras], queue=False, js=js)
349
+ demo.queue(max_size=20)
350
+ demo.launch(share=True)