xl

app.py

@@ -4,6 +4,7 @@ import torch
 from PIL import Image, ImageDraw, ImageFont, ImageOps, ImageEnhance
 from quanto import qfloat8, quantize, freeze
 from diffusers import StableDiffusionControlNetPipeline, ControlNetModel
+from diffusers import StableDiffusionXLControlNetPipeline, UNet2DConditionModel, DiffusionPipeline, LCMScheduler
 from diffusers.utils import make_image_grid
 
 atkbold = ImageFont.truetype("Atkinson-Hyperlegible-Bold-102.otf",50)
@@ -45,12 +46,12 @@ def mask_image_factory(mask_text="ASK FOR\nA SNACK", width=default_width, height
 
 preferred_device = "cuda" if torch.cuda.is_available() else ("mps" if torch.backends.mps.is_available() else "cpu")
 # preferred_device = "cpu"
-preferred_dtype = torch.float32
+preferred_dtype = torch.float32 if preferred_device == "cpu" else torch.float16
 
 controlnet = ControlNetModel.from_pretrained(
-    "monster-labs/control_v1p_sd15_qrcode_monster",
-#    "monster-labs/control_v1p_sdxl_qrcode_monster",
-    subfolder="v2",
+#    "monster-labs/control_v1p_sd15_qrcode_monster",
+    "monster-labs/control_v1p_sdxl_qrcode_monster",
+#    subfolder="v2",
     torch_dtype=preferred_dtype,
     #torch_dtype=unet_preferred_dtype
 ).to(preferred_device)
@@ -58,13 +59,22 @@ controlnet = ControlNetModel.from_pretrained(
 #quantize(controlnet, weights=qfloat8)
 #freeze(controlnet)
 
-ctlpipe = StableDiffusionControlNetPipeline.from_pretrained(
-    "SimianLuo/LCM_Dreamshaper_v7",
+unet = UNet2DConditionModel.from_pretrained(
+    "latent-consistency/lcm-sdxl",
+    torch_dtype=preferred_dtype,
+    variant="fp16",
+).to(preferred_device)
+
+ctlpipe = StableDiffusionXLControlNetPipeline.from_pretrained(
+    "stabilityai/stable-diffusion-xl-base-1.0",
+    unet=unet,
     controlnet=controlnet,
     torch_dtype=preferred_dtype,
     safety_checker=None,
 ).to(preferred_device)
 
+ctlpipe.scheduler = LCMScheduler.from_config(ctlpipe.scheduler.config)
+
 #quantize(ctlpipe.unet, weights=qfloat8)
 #freeze(ctlpipe.unet)
 #quantize(ctlpipe.text_encoder, weights=qfloat8)