first try

README.md

@@ -1,14 +1,13 @@
 ---
 title: Protogen Web-UI
-emoji: 🪄
-colorFrom: blue
-colorTo: pink
+emoji: ➕
+colorFrom: purple
+colorTo: black
 sdk: gradio
 sdk_version: 3.15.0
 app_file: app.py
 pinned: true
 license: mit
-duplicated_from: darkstorm2150/protogen-web-ui
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,7 +1,7 @@
 from diffusers import (
     StableDiffusionPipeline,
-    StableDiffusionImg2ImgPipeline,
     DPMSolverMultistepScheduler,
+    DiffusionPipeline,
 )
 import gradio as gr
 import torch
@@ -17,6 +17,10 @@ current_steps = 25
 
 SAFETY_CHECKER = StableDiffusionSafetyChecker.from_pretrained("CompVis/stable-diffusion-safety-checker", torch_dtype=torch.float16)
 
+UPSCALER = DiffusionPipeline.from_pretrained("stabilityai/sd-x2-latent-upscaler", torch_dtype=torch.float16)
+UPSCALER.to("cuda")
+UPSCALER.enable_xformers_memory_efficient_attention()
+
 
 class Model:
     def __init__(self, name, path=""):
@@ -30,21 +34,12 @@ class Model:
             self.pipe_t2i.scheduler = DPMSolverMultistepScheduler.from_config(
                 self.pipe_t2i.scheduler.config
             )
-            self.pipe_i2i = StableDiffusionImg2ImgPipeline(**self.pipe_t2i.components)
         else:
             self.pipe_t2i = None
-            self.pipe_i2i = None
 
 
 models = [
-    Model("Protogen v2.2 (Anime)", "darkstorm2150/Protogen_v2.2_Official_Release"),
-    Model("Protogen x3.4 (Photorealism)", "darkstorm2150/Protogen_x3.4_Official_Release"),
-    Model("Protogen x5.3 (Photorealism)", "darkstorm2150/Protogen_x5.3_Official_Release"),
-    Model("Protogen x5.8 Rebuilt (Scifi+Anime)", "darkstorm2150/Protogen_x5.8_Official_Release"),
-    Model("Protogen Dragon (RPG Model)", "darkstorm2150/Protogen_Dragon_Official_Release"),
-    Model("Protogen Nova", "darkstorm2150/Protogen_Nova_Official_Release"),
-    Model("Protogen Eclipse", "darkstorm2150/Protogen_Eclipse_Official_Release"),
-    Model("Protogen Infinity", "darkstorm2150/Protogen_Infinity_Official_Release"),
+    Model("Stable Diffusion v1-4", "CompVis/stable-diffusion-v1-4"),
 ]
 
 MODELS = {m.name: m for m in models}
@@ -66,12 +61,9 @@ def inference(
     prompt,
     guidance,
     steps,
-    n_images=1,
     width=512,
     height=512,
     seed=0,
-    img=None,
-    strength=0.5,
     neg_prompt="",
 ):
 
@@ -83,55 +75,30 @@ def inference(
     generator = torch.Generator("cuda").manual_seed(seed)
 
     try:
-        if img is not None:
-            return (
-                img_to_img(
-                    model_name,
-                    prompt,
-                    n_images,
-                    neg_prompt,
-                    img,
-                    strength,
-                    guidance,
-                    steps,
-                    width,
-                    height,
-                    generator,
-                    seed,
-                ),
-                f"Done. Seed: {seed}",
-            )
-        else:
-            return (
-                txt_to_img(
-                    model_name,
-                    prompt,
-                    n_images,
-                    neg_prompt,
-                    guidance,
-                    steps,
-                    width,
-                    height,
-                    generator,
-                    seed,
-                ),
-                f"Done. Seed: {seed}",
-            )
+        low_res_image, up_res_image = txt_to_img(
+            model_name,
+            prompt,
+            neg_prompt,
+            guidance,
+            steps,
+            width,
+            height,
+            generator,
+        )
+        return low_res_image, up_res_image, f"Done. Seed: {seed}",
     except Exception as e:
-        return None, error_str(e)
+        return None, None, error_str(e)
 
 
 def txt_to_img(
     model_name,
     prompt,
-    n_images,
     neg_prompt,
     guidance,
     steps,
     width,
     height,
     generator,
-    seed,
 ):
     pipe = MODELS[model_name].pipe_t2i
 
@@ -139,61 +106,34 @@ def txt_to_img(
         pipe = pipe.to("cuda")
         pipe.enable_xformers_memory_efficient_attention()
 
-    result = pipe(
+    low_res_latents = pipe(
         prompt,
         negative_prompt=neg_prompt,
-        num_images_per_prompt=n_images,
         num_inference_steps=int(steps),
         guidance_scale=guidance,
         width=width,
         height=height,
         generator=generator,
-    )
-
-    pipe.to("cpu")
-    torch.cuda.empty_cache()
-
-    return replace_nsfw_images(result)
-
+        output_type="latent",
+    ).images
 
-def img_to_img(
-    model_name,
-    prompt,
-    n_images,
-    neg_prompt,
-    img,
-    strength,
-    guidance,
-    steps,
-    width,
-    height,
-    generator,
-    seed,
-):
-    pipe = MODELS[model_name].pipe_i2i
+    with torch.no_grad():
+        low_res_image = pipe.decode_latents(low_res_latents)
+        low_res_image = pipe.numpy_to_pil(low_res_image)[0]
 
-    if torch.cuda.is_available():
-        pipe = pipe.to("cuda")
-        pipe.enable_xformers_memory_efficient_attention()
-
-    ratio = min(height / img.height, width / img.width)
-    img = img.resize((int(img.width * ratio), int(img.height * ratio)), Image.LANCZOS)
-
-    result = pipe(
-        prompt,
+    up_res_image = UPSCALER(
+        prompt=prompt,
         negative_prompt=neg_prompt,
-        num_images_per_prompt=n_images,
-        image=img,
-        num_inference_steps=int(steps),
-        strength=strength,
-        guidance_scale=guidance,
+        image=low_res_latents,
+        num_inference_steps=20,
+        guidance_scale=0,
         generator=generator,
-    )
+    ).images[0]
 
     pipe.to("cpu")
     torch.cuda.empty_cache()
 
-    return replace_nsfw_images(result)
+    return low_res_image, up_res_image
 
 
 def replace_nsfw_images(results):
@@ -252,8 +192,12 @@ with gr.Blocks(css="style.css") as demo:
                     )
 
                 # image_out = gr.Image(height=512)
-                gallery = gr.Gallery(
-                    label="Generated images", show_label=False, elem_id="gallery"
+                low_res_image = gr.Gallery(
+                    label="512-pix image", show_label=False, elem_id="gallery"
+                ).style(grid=[2], height="auto")
+
+                up_res_image = gr.Gallery(
+                    label="1024-pix image", show_label=False, elem_id="gallery"
                 ).style(grid=[2], height="auto")
 
             state_info = gr.Textbox(label="State", show_label=False, max_lines=2).style(
@@ -269,10 +213,6 @@ with gr.Blocks(css="style.css") as demo:
                         placeholder="What to exclude from the image",
                     )
 
-                    n_images = gr.Slider(
-                        label="Images", value=1, minimum=1, maximum=4, step=1
-                    )
-
                     with gr.Row():
                         guidance = gr.Slider(
                             label="Guidance scale", value=7.5, maximum=15
@@ -297,52 +237,29 @@ with gr.Blocks(css="style.css") as demo:
                         0, 2147483647, label="Seed (0 = random)", value=0, step=1
                     )
 
-            with gr.Tab("Image to image"):
-                with gr.Group():
-                    image = gr.Image(
-                        label="Image", height=256, tool="editor", type="pil"
-                    )
-                    strength = gr.Slider(
-                        label="Transformation strength",
-                        minimum=0,
-                        maximum=1,
-                        step=0.01,
-                        value=0.5,
-                    )
-
     inputs = [
         model_name,
         prompt,
         guidance,
         steps,
-        n_images,
         width,
         height,
         seed,
-        image,
-        strength,
         neg_prompt,
     ]
-    outputs = [gallery, error_output]
+    outputs = [low_res_image, up_res_image, error_output]
     prompt.submit(inference, inputs=inputs, outputs=outputs)
     generate.click(inference, inputs=inputs, outputs=outputs)
 
-    ex = gr.Examples(
-        [
-            [models[0].name, "portrait of a beautiful alyx vance half life", 10, 50, "canvas frame, ((disfigured)), ((bad art)), ((deformed)),((extra limbs)),((close up)),((b&w)), weird colors, blurry, (((duplicate))), ((morbid)), ((mutilated)), [out of frame], extra fingers, mutated hands, ((poorly drawn hands)), ((poorly drawn face)), (((mutation))), (((deformed))), blurry, ((bad anatomy)), (((bad proportions))), ((extra limbs)), cloned face, (((disfigured))), out of frame, ugly, extra limbs, (bad anatomy), gross proportions, (malformed limbs), ((missing arms)), ((missing legs)), (((extra arms))), (((extra legs))), mutated hands, (fused fingers), (too many fingers), (((long neck))), Photoshop, video game, ugly, tiling, poorly drawn hands, poorly drawn feet, poorly drawn face, out of frame, mutation, mutated, extra limbs, extra legs, extra arms, disfigured, deformed, cross-eye, body out of frame, blurry, bad art, bad anatomy"],
-            [models[1].name, "Brad Pitt with sunglasses, highly realistic", 7.5, 50, "canvas frame, ((disfigured)), ((bad art)), ((deformed)),((extra limbs)),((close up)),((b&w)), weird colors, blurry, (((duplicate))), ((morbid)), ((mutilated)), [out of frame], extra fingers, mutated hands, ((poorly drawn hands)), ((poorly drawn face)), (((mutation))), (((deformed))), blurry, ((bad anatomy)), (((bad proportions))), ((extra limbs)), cloned face, (((disfigured))), out of frame, ugly, extra limbs, (bad anatomy), gross proportions, (malformed limbs), ((missing arms)), ((missing legs)), (((extra arms))), (((extra legs))), mutated hands, (fused fingers), (too many fingers), (((long neck))), Photoshop, video game, ugly, tiling, poorly drawn hands, poorly drawn feet, poorly drawn face, out of frame, mutation, mutated, extra limbs, extra legs, extra arms, disfigured, deformed, cross-eye, body out of frame, blurry, bad art, bad anatomy"],
-            [models[2].name, "(extremely detailed CG unity 8k wallpaper), the most beautiful artwork in the world", 7.5, 50, "human, people, canvas frame, ((disfigured)), ((bad art)), ((deformed)),((extra limbs)),((close up)),((b&w)), weird colors, blurry, (((duplicate))), ((morbid)), ((mutilated)), [out of frame], extra fingers, mutated hands, ((poorly drawn hands)), ((poorly drawn face)), (((mutation))), (((deformed))), blurry, ((bad anatomy)), (((bad proportions))), ((extra limbs)), cloned face, (((disfigured))), out of frame, ugly, extra limbs, (bad anatomy), gross proportions, (malformed limbs), ((missing arms)), ((missing legs)), (((extra arms))), (((extra legs))), mutated hands, (fused fingers), (too many fingers), (((long neck))), Photoshop, video game, ugly, tiling, poorly drawn hands, poorly drawn feet, poorly drawn face, out of frame, mutation, mutated, extra limbs, extra legs, extra arms, disfigured, deformed, cross-eye, body out of frame, blurry, bad art, bad anatomy"],
-            [models[3].name, "(extremely detailed CG unity 8k wallpaper), full shot body photo star lord chris pratt posing in an outdoor spaceship, holding a gun, extremely detailed, trending on ArtStation, trending on CGSociety, Intricate, High Detail, dramatic, realism, beautiful and detailed lighting, shadows", 7.5, 50, "canvas frame, ((disfigured)), ((bad art)), ((deformed)),((extra limbs)),((close up)),((b&w)), weird colors, blurry, (((duplicate))), ((morbid)), ((mutilated)), [out of frame], extra fingers, mutated hands, ((poorly drawn hands)), ((poorly drawn face)), (((mutation))), (((deformed))), blurry, ((bad anatomy)), (((bad proportions))), ((extra limbs)), cloned face, (((disfigured))), out of frame, ugly, extra limbs, (bad anatomy), gross proportions, (malformed limbs), ((missing arms)), ((missing legs)), (((extra arms))), (((extra legs))), mutated hands, (fused fingers), (too many fingers), (((long neck))), Photoshop, video game, ugly, tiling, poorly drawn hands, poorly drawn feet, poorly drawn face, out of frame, mutation, mutated, extra limbs, extra legs, extra arms, disfigured, deformed, cross-eye, body out of frame, blurry, bad art, bad anatomy"],
-            [models[4].name, "(extremely detailed CG unity 8k wallpaper), full body portrait of (david:1.1), staring at us with a mysterious gaze, realistic, masterpiece, highest quality, ((scifi)), lens flare, ((light sparkles)), unreal engine, digital painting, trending on ArtStation, trending on CGSociety, Intricate, High Detail, dramatic, realism, beautiful and detailed lighting, shadows", 7.5, 50, "canvas frame, ((disfigured)), ((bad art)), ((deformed)),((extra limbs)),((close up)),((b&w)), weird colors, blurry, (((duplicate))), ((morbid)), ((mutilated)), [out of frame], extra fingers, mutated hands, ((poorly drawn hands)), ((poorly drawn face)), (((mutation))), (((deformed))), blurry, ((bad anatomy)), (((bad proportions))), ((extra limbs)), cloned face, (((disfigured))), out of frame, ugly, extra limbs, (bad anatomy), gross proportions, (malformed limbs), ((missing arms)), ((missing legs)), (((extra arms))), (((extra legs))), mutated hands, (fused fingers), (too many fingers), (((long neck))), Photoshop, video game, ugly, tiling, poorly drawn hands, poorly drawn feet, poorly drawn face, out of frame, mutation, mutated, extra limbs, extra legs, extra arms, disfigured, deformed, cross-eye, body out of frame, blurry, bad art, bad anatomy"],
-            [models[5].name, "(extremely detailed CG unity 8k wallpaper), full body portrait of (david:1.1), staring at us with a mysterious gaze, realistic, masterpiece, highest quality, ((scifi)), lens flare, ((light sparkles)), unreal engine, digital painting, trending on ArtStation, trending on CGSociety, Intricate, High Detail, dramatic, realism, beautiful and detailed lighting, shadows", 7.5, 50, "canvas frame, ((disfigured)), ((bad art)), ((deformed)),((extra limbs)),((close up)),((b&w)), weird colors, blurry, (((duplicate))), ((morbid)), ((mutilated)), [out of frame], extra fingers, mutated hands, ((poorly drawn hands)), ((poorly drawn face)), (((mutation))), (((deformed))), blurry, ((bad anatomy)), (((bad proportions))), ((extra limbs)), cloned face, (((disfigured))), out of frame, ugly, extra limbs, (bad anatomy), gross proportions, (malformed limbs), ((missing arms)), ((missing legs)), (((extra arms))), (((extra legs))), mutated hands, (fused fingers), (too many fingers), (((long neck))), Photoshop, video game, ugly, tiling, poorly drawn hands, poorly drawn feet, poorly drawn face, out of frame, mutation, mutated, extra limbs, extra legs, extra arms, disfigured, deformed, cross-eye, body out of frame, blurry, bad art, bad anatomy"],
-            [models[6].name, "(extremely detailed CG unity 8k wallpaper), full body portrait of (david:1.1), staring at us with a mysterious gaze, realistic, masterpiece, highest quality, ((scifi)), lens flare, ((light sparkles)), unreal engine, digital painting, trending on ArtStation, trending on CGSociety, Intricate, High Detail, dramatic, realism, beautiful and detailed lighting, shadows", 7.5, 50, "canvas frame, ((disfigured)), ((bad art)), ((deformed)),((extra limbs)),((close up)),((b&w)), weird colors, blurry, (((duplicate))), ((morbid)), ((mutilated)), [out of frame], extra fingers, mutated hands, ((poorly drawn hands)), ((poorly drawn face)), (((mutation))), (((deformed))), blurry, ((bad anatomy)), (((bad proportions))), ((extra limbs)), cloned face, (((disfigured))), out of frame, ugly, extra limbs, (bad anatomy), gross proportions, (malformed limbs), ((missing arms)), ((missing legs)), (((extra arms))), (((extra legs))), mutated hands, (fused fingers), (too many fingers), (((long neck))), Photoshop, video game, ugly, tiling, poorly drawn hands, poorly drawn feet, poorly drawn face, out of frame, mutation, mutated, extra limbs, extra legs, extra arms, disfigured, deformed, cross-eye, body out of frame, blurry, bad art, bad anatomy"],
-            [models[7].name, "(extremely detailed CG unity 8k wallpaper), full body portrait of (david:1.1), staring at us with a mysterious gaze, realistic, masterpiece, highest quality, ((scifi)), lens flare, ((light sparkles)), unreal engine, digital painting, trending on ArtStation, trending on CGSociety, Intricate, High Detail, dramatic, realism, beautiful and detailed lighting, shadows", 7.5, 50, "canvas frame, ((disfigured)), ((bad art)), ((deformed)),((extra limbs)),((close up)),((b&w)), weird colors, blurry, (((duplicate))), ((morbid)), ((mutilated)), [out of frame], extra fingers, mutated hands, ((poorly drawn hands)), ((poorly drawn face)), (((mutation))), (((deformed))), blurry, ((bad anatomy)), (((bad proportions))), ((extra limbs)), cloned face, (((disfigured))), out of frame, ugly, extra limbs, (bad anatomy), gross proportions, (malformed limbs), ((missing arms)), ((missing legs)), (((extra arms))), (((extra legs))), mutated hands, (fused fingers), (too many fingers), (((long neck))), Photoshop, video game, ugly, tiling, poorly drawn hands, poorly drawn feet, poorly drawn face, out of frame, mutation, mutated, extra limbs, extra legs, extra arms, disfigured, deformed, cross-eye, body out of frame, blurry, bad art, bad anatomy"],
-        ],
-        inputs=[model_name, prompt, guidance, steps, neg_prompt],
-        outputs=outputs,
-        fn=inference,
-        cache_examples=False,
-    )
+#    ex = gr.Examples(
+#        [
+#            [models[0].name, "portrait of a beautiful alyx vance half life", 10, 50, "canvas frame, ((disfigured)), ((bad art)), ((deformed)),((extra limbs)),((close up)),((b&w)), weird colors, blurry, (((duplicate))), ((morbid)), ((mutilated)), [out of frame], extra fingers, mutated hands, ((poorly drawn hands)), ((poorly drawn face)), (((mutation))), (((deformed))), blurry, ((bad anatomy)), (((bad proportions))), ((extra limbs)), cloned face, (((disfigured))), out of frame, ugly, extra limbs, (bad anatomy), gross proportions, (malformed limbs), ((missing arms)), ((missing legs)), (((extra arms))), (((extra legs))), mutated hands, (fused fingers), (too many fingers), (((long neck))), Photoshop, video game, ugly, tiling, poorly drawn hands, poorly drawn feet, poorly drawn face, out of frame, mutation, mutated, extra limbs, extra legs, extra arms, disfigured, deformed, cross-eye, body out of frame, blurry, bad art, bad anatomy"],
+#        ],
+#        inputs=[model_name, prompt, guidance, steps, neg_prompt],
+#        outputs=outputs,
+#        fn=inference,
+#        cache_examples=False,
+#    )
 
     gr.HTML(
         """

requirements.txt

@@ -1,16 +1,11 @@
 --extra-index-url https://download.pytorch.org/whl/cu113
 torch
 torchvision==0.13.1+cu113
-#diffusers
 git+https://github.com/huggingface/diffusers.git
-#transformers
 git+https://github.com/huggingface/transformers
 scipy
 ftfy
 psutil
-accelerate==0.12.0
-#OmegaConf
-#pytorch_lightning
+accelerate
 triton==2.0.0.dev20220701
-#https://github.com/apolinario/xformers/releases/download/0.0.3/xformers-0.0.14.dev0-cp38-cp38-linux_x86_64.whl
-https://github.com/camenduru/stable-diffusion-webui-colab/releases/download/0.0.15/xformers-0.0.15.dev0+4c06c79.d20221205-cp38-cp38-linux_x86_64.whl
+https://github.com/camenduru/stable-diffusion-webui-colab/releases/download/0.0.15/xformers-0.0.15.dev0+4c06c79.d20221205-cp38-cp38-linux_x86_64.whl