Add controlnet_v2

README.md

@@ -6,7 +6,7 @@ colorTo: red
 sdk: gradio
 sdk_version: 3.19
 app_file: app.py
-pinned: false
+pinned: true
 license: openrail
 tags:
   - making-demos

diffusion_webui/controlnet/controlnet_canny.py

@@ -10,12 +10,13 @@ import cv2
 
 stable_model_list = [
     "runwayml/stable-diffusion-v1-5",
-    "stabilityai/stable-diffusion-2",
-    "stabilityai/stable-diffusion-2-base",
     "stabilityai/stable-diffusion-2-1",
-    "stabilityai/stable-diffusion-2-1-base"
 ]
 
+controlnet_canny_model_list = [
+    "lllyasviel/sd-controlnet-canny",
+    "thibaud/controlnet-sd21-canny-diffusers"
+]
 
 
 stable_prompt_list = [
@@ -30,6 +31,7 @@ stable_negative_prompt_list = [
 
 def controlnet_canny(
     image_path:str,
+    controlnet_model_path:str,
 ):
     image = Image.open(image_path)
     image = np.array(image)
@@ -40,7 +42,7 @@ def controlnet_canny(
     image = Image.fromarray(image)
 
     controlnet = ControlNetModel.from_pretrained(
-        "lllyasviel/sd-controlnet-canny", 
+        controlnet_model_path,
         torch_dtype=torch.float16
     )
     return controlnet, image
@@ -48,17 +50,18 @@ def controlnet_canny(
 
 def stable_diffusion_controlnet_canny(
     image_path:str,
-    model_path:str,
+    stable_model_path:str,
+    controlnet_model_path:str,
     prompt:str,
     negative_prompt:str,
     guidance_scale:int,
     num_inference_step:int,
     ):
 
-    controlnet, image = controlnet_canny(image_path)
+    controlnet, image = controlnet_canny(image_path, controlnet_model_path)
     
     pipe = StableDiffusionControlNetPipeline.from_pretrained(
-        pretrained_model_name_or_path=model_path, 
+        pretrained_model_name_or_path=stable_model_path, 
         controlnet=controlnet, 
         safety_checker=None, 
         torch_dtype=torch.float16,
@@ -87,12 +90,18 @@ def stable_diffusion_controlnet_canny_app():
                     label='Image'
                 )
 
-                controlnet_canny_model_id = gr.Dropdown(
+                controlnet_canny_stable_model_id = gr.Dropdown(
                     choices=stable_model_list, 
                     value=stable_model_list[0], 
                     label='Stable Model Id'
                 )
-
+                
+                controlnet_canny_model_id = gr.Dropdown(
+                    choices=controlnet_canny_model_list,
+                    value=controlnet_canny_model_list[0],
+                    label='Controlnet Model Id'
+                )
+                    
                 controlnet_canny_prompt = gr.Textbox(
                     lines=1, 
                     value=stable_prompt_list[0], 
@@ -131,6 +140,7 @@ def stable_diffusion_controlnet_canny_app():
             fn=stable_diffusion_controlnet_canny,
             inputs=[
                 controlnet_canny_image_file,
+                controlnet_canny_stable_model_id,
                 controlnet_canny_model_id,
                 controlnet_canny_prompt,
                 controlnet_canny_negative_prompt,

diffusion_webui/controlnet/controlnet_depth.py

@@ -9,10 +9,12 @@ import torch
 
 stable_model_list = [
     "runwayml/stable-diffusion-v1-5",
-    "stabilityai/stable-diffusion-2",
-    "stabilityai/stable-diffusion-2-base",
     "stabilityai/stable-diffusion-2-1",
-    "stabilityai/stable-diffusion-2-1-base"
+]
+
+controlnet_depth_model_list = [
+    "fusing/stable-diffusion-v1-5-controlnet-depth",
+    "thibaud/controlnet-sd21-depth-diffusers"
 ]
 
 
@@ -27,7 +29,7 @@ stable_negative_prompt_list = [
     ]
 
 
-def controlnet_depth(image_path:str):
+def controlnet_depth(image_path:str, depth_model_path:str):
     depth_estimator = pipeline('depth-estimation')
 
     image = Image.open(image_path)
@@ -38,24 +40,25 @@ def controlnet_depth(image_path:str):
     image = Image.fromarray(image)
 
     controlnet = ControlNetModel.from_pretrained(
-        "fusing/stable-diffusion-v1-5-controlnet-depth", torch_dtype=torch.float16
+        depth_model_path, torch_dtype=torch.float16
     )
 
     return controlnet, image
 
 def stable_diffusion_controlnet_depth(
     image_path:str,
-    model_path:str,
+    stable_model_path:str,
+    depth_model_path:str,
     prompt:str,
     negative_prompt:str,
     guidance_scale:int,
     num_inference_step:int,
     ):
 
-    controlnet, image = controlnet_depth(image_path=image_path)
+    controlnet, image = controlnet_depth(image_path=image_path, depth_model_path=depth_model_path)
 
     pipe = StableDiffusionControlNetPipeline.from_pretrained(
-        pretrained_model_name_or_path=model_path, 
+        pretrained_model_name_or_path=stable_model_path, 
         controlnet=controlnet, 
         safety_checker=None, 
         torch_dtype=torch.float16
@@ -85,12 +88,18 @@ def stable_diffusion_controlnet_depth_app():
                     label='Image'
                 )
 
-                controlnet_depth_model_id = gr.Dropdown(
+                controlnet_depth_stable_model_id = gr.Dropdown(
                     choices=stable_model_list, 
                     value=stable_model_list[0], 
                     label='Stable Model Id'
                 )
 
+                controlnet_depth_model_id = gr.Dropdown(
+                    choices=controlnet_depth_model_list,
+                    value=controlnet_depth_model_list[0],
+                    label='ControlNet Model Id'
+                )
+                
                 controlnet_depth_prompt = gr.Textbox(
                     lines=1, 
                     value=stable_prompt_list[0], 
@@ -129,6 +138,7 @@ def stable_diffusion_controlnet_depth_app():
             fn=stable_diffusion_controlnet_depth,
             inputs=[
                 controlnet_depth_image_file,
+                controlnet_depth_stable_model_id,
                 controlnet_depth_model_id,
                 controlnet_depth_prompt,
                 controlnet_depth_negative_prompt,

diffusion_webui/controlnet/controlnet_hed.py

@@ -8,10 +8,12 @@ import torch
 
 stable_model_list = [
     "runwayml/stable-diffusion-v1-5",
-    "stabilityai/stable-diffusion-2",
-    "stabilityai/stable-diffusion-2-base",
     "stabilityai/stable-diffusion-2-1",
-    "stabilityai/stable-diffusion-2-1-base"
+]
+
+controlnet_hed_model_list = [
+    "fusing/stable-diffusion-v1-5-controlnet-hed",
+    "thibaud/controlnet-sd21-hed-diffusers"
 ]
 
 stable_prompt_list = [
@@ -25,14 +27,14 @@ stable_negative_prompt_list = [
     ]
 
 
-def controlnet_hed(image_path:str):
+def controlnet_hed(image_path:str, controlnet_hed_model_path:str):
     hed = HEDdetector.from_pretrained('lllyasviel/ControlNet')
 
     image = Image.open(image_path)
     image = hed(image)
 
     controlnet = ControlNetModel.from_pretrained(
-        "fusing/stable-diffusion-v1-5-controlnet-hed", 
+        controlnet_hed_model_path,
         torch_dtype=torch.float16
     )
     return controlnet, image
@@ -40,17 +42,18 @@ def controlnet_hed(image_path:str):
 
 def stable_diffusion_controlnet_hed(
     image_path:str,
-    model_path:str,
+    stable_model_path:str,
+    controlnet_hed_model_path:str,
     prompt:str,
     negative_prompt:str,
     guidance_scale:int,
     num_inference_step:int,
     ):
 
-    controlnet, image = controlnet_hed(image_path=image_path)
+    controlnet, image = controlnet_hed(image_path=image_path, controlnet_hed_model_path=controlnet_hed_model_path)
 
     pipe = StableDiffusionControlNetPipeline.from_pretrained(
-        pretrained_model_name_or_path=model_path, 
+        pretrained_model_name_or_path=stable_model_path, 
         controlnet=controlnet, 
         safety_checker=None, 
         torch_dtype=torch.float16
@@ -79,11 +82,17 @@ def stable_diffusion_controlnet_hed_app():
                     label='Image'
                 )
 
-                controlnet_hed_model_id = gr.Dropdown(
+                controlnet_hed_stable_model_id = gr.Dropdown(
                     choices=stable_model_list, 
                     value=stable_model_list[0], 
                     label='Stable Model Id'
                 )
+                
+                controlnet_hed_model_id = gr.Dropdown(
+                    choices=stable_model_list,
+                    value=stable_model_list[1],
+                    label='ControlNet Model Id'
+                )
 
                 controlnet_hed_prompt = gr.Textbox(
                     lines=1, 
@@ -124,6 +133,7 @@ def stable_diffusion_controlnet_hed_app():
             fn=stable_diffusion_controlnet_hed,
             inputs=[
                 controlnet_hed_image_file,
+                controlnet_hed_stable_model_id,
                 controlnet_hed_model_id,
                 controlnet_hed_prompt,
                 controlnet_hed_negative_prompt,

diffusion_webui/controlnet/controlnet_pose.py

@@ -9,10 +9,12 @@ import torch
 
 stable_model_list = [
     "runwayml/stable-diffusion-v1-5",
-    "stabilityai/stable-diffusion-2",
-    "stabilityai/stable-diffusion-2-base",
     "stabilityai/stable-diffusion-2-1",
-    "stabilityai/stable-diffusion-2-1-base"
+]
+
+controlnet_pose_model_list = [
+    "fusing/stable-diffusion-v1-5-controlnet-openpose",
+    "thibaud/controlnet-sd21-openpose-diffusers"
 ]
 
 stable_prompt_list = [
@@ -26,14 +28,14 @@ stable_negative_prompt_list = [
     ]
 
 
-def controlnet_pose(image_path:str):
+def controlnet_pose(image_path:str, controlnet_pose_model_path:str):
     openpose = OpenposeDetector.from_pretrained('lllyasviel/ControlNet')
 
     image = Image.open(image_path)
     image = openpose(image)
 
     controlnet = ControlNetModel.from_pretrained(
-        "fusing/stable-diffusion-v1-5-controlnet-openpose", 
+        controlnet_pose_model_path,
         torch_dtype=torch.float16
     )
 
@@ -41,17 +43,18 @@ def controlnet_pose(image_path:str):
 
 def stable_diffusion_controlnet_pose(
     image_path:str,
-    model_path:str,
+    stable_model_path:str,
+    controlnet_pose_model_path:str,
     prompt:str,
     negative_prompt:str,
     guidance_scale:int,
     num_inference_step:int,
     ):
 
-    controlnet, image = controlnet_pose(image_path=image_path)
+    controlnet, image = controlnet_pose(image_path=image_path, controlnet_pose_model_path=controlnet_pose_model_path)
 
     pipe = StableDiffusionControlNetPipeline.from_pretrained(
-        pretrained_model_name_or_path=model_path, 
+        pretrained_model_name_or_path=-stable_model_path, 
         controlnet=controlnet, 
         safety_checker=None, 
         torch_dtype=torch.float16
@@ -81,11 +84,18 @@ def stable_diffusion_controlnet_pose_app():
                     label='Image'
                 )
 
-                controlnet_pose_model_id = gr.Dropdown(
+                controlnet_pose_stable_model_id = gr.Dropdown(
                     choices=stable_model_list, 
                     value=stable_model_list[0], 
                     label='Stable Model Id'
                 )
+                
+                controlnet_pose_model_id = gr.Dropdown(
+                    choices=stable_model_list,
+                    value=stable_model_list[1],
+                    label='ControlNet Model Id'
+                )
+                       
 
                 controlnet_pose_prompt = gr.Textbox(
                     lines=1, 
@@ -125,6 +135,7 @@ def stable_diffusion_controlnet_pose_app():
             fn=stable_diffusion_controlnet_pose,
             inputs=[
                 controlnet_pose_image_file,
+                controlnet_pose_stable_model_id,
                 controlnet_pose_model_id,
                 controlnet_pose_prompt,
                 controlnet_pose_negative_prompt,

diffusion_webui/controlnet/controlnet_scribble.py

@@ -9,10 +9,12 @@ import torch
 
 stable_model_list = [
     "runwayml/stable-diffusion-v1-5",
-    "stabilityai/stable-diffusion-2",
-    "stabilityai/stable-diffusion-2-base",
     "stabilityai/stable-diffusion-2-1",
-    "stabilityai/stable-diffusion-2-1-base"
+]
+
+controlnet_hed_model_list = [
+    "fusing/stable-diffusion-v1-5-controlnet-hed",
+    "thibaud/controlnet-sd21-scribble-diffusers"
 ]
 
 stable_prompt_list = [
@@ -26,31 +28,32 @@ stable_negative_prompt_list = [
     ]
 
 
-def controlnet_scribble(image_path:str):
+def controlnet_scribble(image_path:str, controlnet_hed_model_path:str):
     hed = HEDdetector.from_pretrained('lllyasviel/ControlNet')
 
     image = Image.open(image_path)
     image = hed(image, scribble=True)
 
     controlnet = ControlNetModel.from_pretrained(
-        "fusing/stable-diffusion-v1-5-controlnet-scribble", torch_dtype=torch.float16
+        controlnet_hed_model_path, torch_dtype=torch.float16
     )
 
     return controlnet, image
 
 def stable_diffusion_controlnet_scribble(
     image_path:str,
-    model_path:str,
+    stable_model_path:str,
+    controlnet_hed_model_path:str,
     prompt:str,
     negative_prompt:str,
     guidance_scale:int,
     num_inference_step:int,
     ):
 
-    controlnet, image = controlnet_scribble(image_path=image_path)
+    controlnet, image = controlnet_scribble(image_path=image_path, controlnet_hed_model_path=controlnet_hed_model_path)
 
     pipe = StableDiffusionControlNetPipeline.from_pretrained(
-        pretrained_model_name_or_path=model_path, 
+        pretrained_model_name_or_path=stable_model_path, 
         controlnet=controlnet, 
         safety_checker=None, 
         torch_dtype=torch.float16
@@ -79,10 +82,16 @@ def stable_diffusion_controlnet_scribble_app():
                     label='Image'
                 )
 
-                controlnet_scribble_model_id = gr.Dropdown(
+                controlnet_scribble_stablev1_model_id = gr.Dropdown(
                     choices=stable_model_list, 
                     value=stable_model_list[0], 
-                    label='Stable Model Id'
+                    label='Stable v1.5 Model Id'
+                )
+                
+                controlnet_scribble_stablev2_model_id = gr.Dropdown(
+                    choices=stable_model_list,
+                    value=stable_model_list[1],
+                    label='Stable v2.1 Model Id'
                 )
 
                 controlnet_scribble_prompt = gr.Textbox(
@@ -123,7 +132,8 @@ def stable_diffusion_controlnet_scribble_app():
             fn=stable_diffusion_controlnet_scribble,
             inputs=[
                 controlnet_scribble_image_file,
-                controlnet_scribble_model_id,
+                controlnet_scribble_stablev1_model_id,
+                controlnet_scribble_stablev2_model_id,
                 controlnet_scribble_prompt,
                 controlnet_scribble_negative_prompt,
                 controlnet_scribble_guidance_scale,

diffusion_webui/stable_diffusion/img2img_app.py

@@ -12,11 +12,6 @@ stable_model_list = [
     "stabilityai/stable-diffusion-2-1-base"
 ]
 
-stable_inpiant_model_list = [
-    "stabilityai/stable-diffusion-2-inpainting",
-    "runwayml/stable-diffusion-inpainting"
-]
-
 stable_prompt_list = [
         "a photo of a man.",
         "a photo of a girl."

diffusion_webui/stable_diffusion/inpaint_app.py

@@ -1,18 +1,8 @@
 from diffusers import DiffusionPipeline, DDIMScheduler
-from PIL import Image
-import imageio
 import torch
 
 import gradio as gr
 
-stable_model_list = [
-    "runwayml/stable-diffusion-v1-5",
-    "stabilityai/stable-diffusion-2",
-    "stabilityai/stable-diffusion-2-base",
-    "stabilityai/stable-diffusion-2-1",
-    "stabilityai/stable-diffusion-2-1-base"
-]
-
 stable_inpiant_model_list = [
     "stabilityai/stable-diffusion-2-inpainting",
     "runwayml/stable-diffusion-inpainting"

diffusion_webui/stable_diffusion/keras_txt2img.py

@@ -5,8 +5,8 @@ import tensorflow as tf
 import gradio as gr
 
 keras_model_list = [
-    "kadirnar/dreambooth_diffusion_model_v1",
-    "keras-dreambooth/pink-floyd-division-bell"
+    "keras-dreambooth/keras_diffusion_lowpoly_world",
+    "keras-dreambooth/pink-floyd-division-bell",
     "keras-dreambooth/dreambooth_diffusion_model",
 ]
 

diffusion_webui/stable_diffusion/text2img_app.py

@@ -4,10 +4,14 @@ import torch
 
 stable_model_list = [
     "runwayml/stable-diffusion-v1-5",
-    "stabilityai/stable-diffusion-2",
-    "stabilityai/stable-diffusion-2-base",
     "stabilityai/stable-diffusion-2-1",
-    "stabilityai/stable-diffusion-2-1-base"
+    "sd-dreambooth-library/disco-diffusion-style",
+    "prompthero/openjourney-v2",
+    "andite/anything-v4.0",
+    "Lykon/DreamShaper",
+    "nitrosocke/Nitro-Diffusion",
+    "dreamlike-art/dreamlike-diffusion-1.0"
+    
 ]
 
 stable_prompt_list = [