Rename evoukiyoe_v1.py and update app.py

README.md

@@ -1,12 +1,12 @@
 ---
-title: Evo-Ukiyoe
-emoji: 🐠
+title: Evo Ukiyoe
+emoji: 🏆
 colorFrom: purple
 colorTo: blue
 sdk: gradio
-sdk_version: 4.38.1
+sdk_version: 4.37.2
 app_file: app.py
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,21 +1,16 @@
 import random
 
 from PIL import Image
-from diffusers import EulerDiscreteScheduler
 import gradio as gr
 import numpy as np
 import spaces
 import torch
 
-# torch._inductor.config.conv_1x1_as_mm = True
-# torch._inductor.config.coordinate_descent_tuning = True
-# torch._inductor.config.epilogue_fusion = False
-# torch._inductor.config.coordinate_descent_check_all_directions = True
 from evo_ukiyoe_v1 import load_evo_ukiyoe
 
 
 DESCRIPTION = """# 🐟 Evo-Ukiyoe
-🤗 [モデル一覧](https://huggingface.co/SakanaAI) | 📝 [ブログ](https://sakana.ai/evo-ukiyoe/) | 🐦 [Twitter](https://twitter.com/SakanaAILabs)
+🤗 [モデル一覧](https://huggingface.co/SakanaAI) | 📚 [技術レポート](https://arxiv.org/abs/2403.13187) | 📝 [ブログ](https://sakana.ai/evosdxl-jp/) | 🐦 [Twitter](https://twitter.com/SakanaAILabs)
 
 [Evo-Ukiyoe](https://huggingface.co/SakanaAI/Evo-Ukiyoe-v1)は[Sakana AI](https://sakana.ai/)が教育目的で開発した浮世絵に特化した画像生成モデルです。
 入力した日本語プロンプトに沿った浮世絵風の画像を生成することができます。より詳しくは、上記のブログをご参照ください。
@@ -51,16 +46,7 @@ if SAFETY_CHECKER:
         return images, has_nsfw_concepts
 
 
-pipe = load_evo_ukiyoe(device)
-pipe.scheduler = EulerDiscreteScheduler.from_config(
-    pipe.scheduler.config, use_karras_sigmas=True,
-)
-pipe.to(device=device, dtype=torch.float16)
-# pipe.unet.to(memory_format=torch.channels_last)
-# pipe.vae.to(memory_format=torch.channels_last)
-# # Compile the UNet and VAE.
-# pipe.unet = torch.compile(pipe.unet, mode="max-autotune", fullgraph=True)
-# pipe.vae.decode = torch.compile(pipe.vae.decode, mode="max-autotune", fullgraph=True)
+pipe = load_evo_ukiyoe("cpu").to(device)
 
 
 def randomize_seed_fn(seed: int, randomize_seed: bool) -> int:
@@ -73,21 +59,21 @@ def randomize_seed_fn(seed: int, randomize_seed: bool) -> int:
 @torch.inference_mode()
 def generate(
     prompt: str,
-    negative_prompt: str = "",
     seed: int = 0,
     randomize_seed: bool = False,
     progress=gr.Progress(track_tqdm=True),
 ):
+    pipe.to(device)
     seed = int(randomize_seed_fn(seed, randomize_seed))
     generator = torch.Generator().manual_seed(seed)
 
     images = pipe(
-        prompt=prompt + "最高品質の輻の浮世絵。超詳細。",
-        negative_prompt=negative_prompt,
+        prompt=prompt + "最高品質の輻の浮世絵。",
+        negative_prompt="",
         width=1024,
         height=1024,
         guidance_scale=8.0,
-        num_inference_steps=40,
+        num_inference_steps=50,
         generator=generator,
         num_images_per_prompt=NUM_IMAGES_PER_PROMPT,
         output_type="pil",
@@ -102,9 +88,9 @@ def generate(
 
 
 examples = [
-    "植物と花があります。蝶が飛んでいます。",
-    "鶴が庭に立っています。雪が降っています。",
-    "着物を着ている猫が庭でお茶を飲んでいます。",
+    ["鶴が庭に立っている。雪が降っている。"],
+    ["富士山、桜の木、川と人々の風景。"],
+    ["熊が本を読んでいる。"],
 ]
 
 css = """
@@ -119,10 +105,9 @@ with gr.Blocks(css=css) as demo:
             submit = gr.Button(scale=0)
         result = gr.Image(label="Evo-Ukiyoeからの生成結果", type="pil", show_label=False)
     with gr.Accordion("詳細設定", open=False):
-        negative_prompt = gr.Textbox(placeholder="日本語でネガティブプロンプトを入力してください。(空白可)", show_label=False)
         seed = gr.Slider(label="シード値", minimum=0, maximum=MAX_SEED, step=1, value=0)
         randomize_seed = gr.Checkbox(label="ランダムにシード値を決定", value=True)
-    gr.Examples(examples=examples, inputs=[prompt], outputs=[result, seed], fn=generate)
+    gr.Examples(examples=examples, inputs=prompt, outputs=[result, seed], fn=generate)
     gr.on(
         triggers=[
             prompt.submit,
@@ -131,7 +116,6 @@ with gr.Blocks(css=css) as demo:
         fn=generate,
         inputs=[
             prompt,
-            negative_prompt,
             seed,
             randomize_seed,
         ],

evo_ukiyoe_v1.py

@@ -12,8 +12,7 @@ import torch
 from tqdm import tqdm
 from transformers import AutoTokenizer, CLIPTextModelWithProjection
 
-
-# Base models
+# Base models (fine-tuned from SDXL-1.0)
 SDXL_REPO = "stabilityai/stable-diffusion-xl-base-1.0"
 DPO_REPO = "mhdang/dpo-sdxl-text2image-v1"
 JN_REPO = "RunDiffusion/Juggernaut-XL-v9"
@@ -117,7 +116,6 @@ def load_evo_ukiyoe(device="cuda") -> StableDiffusionXLPipeline:
     )
     jn_weights = split_conv_attn(load_from_pretrained(JN_REPO, device=device))
     jsdxl_weights = split_conv_attn(load_from_pretrained(JSDXL_REPO, device=device))
-
     # Merge base models
     tensors = [sdxl_weights, dpo_weights, jn_weights, jsdxl_weights]
     new_conv = merge_models(
@@ -138,14 +136,11 @@ def load_evo_ukiyoe(device="cuda") -> StableDiffusionXLPipeline:
             0.2198623756106564,
         ],
     )
-
-    # Delete no longer needed variables to free
     del sdxl_weights, dpo_weights, jn_weights, jsdxl_weights
     gc.collect()
     if "cuda" in device:
         torch.cuda.empty_cache()
 
-    # Instantiate UNet
     unet_config = UNet2DConditionModel.load_config(SDXL_REPO, subfolder="unet")
     unet = UNet2DConditionModel.from_config(unet_config).to(device=device)
     unet.load_state_dict({**new_conv, **new_attn})
@@ -167,23 +162,9 @@ def load_evo_ukiyoe(device="cuda") -> StableDiffusionXLPipeline:
         torch_dtype=torch.float16,
         variant="fp16",
     )
+    pipe = pipe.to(device, dtype=torch.float16)
+
     # Load Evo-Ukiyoe weights
     pipe.load_lora_weights(UKIYOE_REPO)
     pipe.fuse_lora(lora_scale=1.0)
-    pipe = pipe.to(device=torch.device(device), dtype=torch.float16)
-
     return pipe
-
-
-if __name__ == "__main__":
-    pipe = load_evo_ukiyoe()
-    images = pipe(
-        prompt="着物を着ている猫が庭でお茶を飲んでいる。最高品質の輻の浮世絵。超詳細。",
-        negative_prompt="",
-        guidance_scale=8.0,
-        num_inference_steps=50,
-        generator=torch.Generator().manual_seed(0),
-        num_images_per_prompt=1,
-        output_type="pil",
-    ).images
-    images[0].save("out.png")

requirements.txt

@@ -1,9 +1,10 @@
-torch==2.2.0
-torchvision==0.17.0
+--extra-index-url https://download.pytorch.org/whl/cu121
+torch==2.3.1+cu121
+torchvision==0.18.1+cu121
 
-accelerate==0.32.0
-diffusers==0.29.2
-gradio==4.38.1
+accelerate==0.31.0
+controlnet-aux==0.0.9
+diffusers==0.26.0
+gradio==4.37.2
 sentencepiece==0.2.0
-transformers==4.42.3
-peft==0.11.1
+transformers==4.42.3