demo_webcam.py

import random
import numpy as np
from PIL import Image
import base64
from io import BytesIO

import torch
import torchvision.transforms.functional as F
from diffusers import ControlNetModel, StableDiffusionControlNetPipeline
import gradio as gr

device = "mps"   # Linux & Windows
weight_type = torch.float16  # torch.float16 works as well, but pictures seem to be a bit worse

controlnet = ControlNetModel.from_pretrained(
    "IDKiro/sdxs-512-dreamshaper-sketch", torch_dtype=weight_type
).to(device)
pipe = StableDiffusionControlNetPipeline.from_pretrained(
    "IDKiro/sdxs-512-dreamshaper", controlnet=controlnet, torch_dtype=weight_type
)
pipe.to(device)

style_list = [
    {
        "name": "No Style",
        "prompt": "{prompt}",
    },
    {
        "name": "Cinematic",
        "prompt": "cinematic still {prompt} . emotional, harmonious, vignette, highly detailed, high budget, bokeh, cinemascope, moody, epic, gorgeous, film grain, grainy",
    },
    # Additional styles omitted for brevity
]

styles = {k["name"]: k["prompt"] for k in style_list}
STYLE_NAMES = list(styles.keys())
DEFAULT_STYLE_NAME = "No Style"
MAX_SEED = np.iinfo(np.int32).max


def pil_image_to_data_url(img, format="PNG"):
    buffered = BytesIO()
    img.save(buffered, format=format)
    img_str = base64.b64encode(buffered.getvalue()).decode()
    return f"data:image/{format.lower()};base64,{img_str}"


def run(
    image, 
    prompt, 
    prompt_template, 
    style_name, 
    controlnet_conditioning_scale,
    device_type="GPU",
    param_dtype='torch.float16',
):
    if device_type == "CPU":
        device = "cpu" 
        param_dtype = 'torch.float32'
    else:
        device = "cuda"
    
    pipe.to(torch_device=device, torch_dtype=torch.float16 if param_dtype == 'torch.float16' else torch.float32)

    print(f"prompt: {prompt}")
    if image is None:
        ones = Image.new("L", (512, 512), 255)
        temp_url = pil_image_to_data_url(ones)
        return ones, gr.update(link=temp_url), gr.update(link=temp_url)
    prompt = prompt_template.replace("{prompt}", prompt)
    control_image = image.convert("RGB")
    control_image = Image.fromarray(255 - np.array(control_image))

    output_pil = pipe(
        prompt=prompt,
        image=control_image,
        width=512,
        height=512,
        guidance_scale=0.0,
        num_inference_steps=1,
        num_images_per_prompt=1,
        output_type="pil",
        controlnet_conditioning_scale=controlnet_conditioning_scale,
    ).images[0]

    input_image_url = pil_image_to_data_url(control_image)
    output_image_url = pil_image_to_data_url(output_pil)
    return (
        output_pil,
        gr.update(link=input_image_url),
        gr.update(link=output_image_url),
    )


with gr.Blocks(css="style.css") as demo:
    gr.Markdown("# SDXS-512-DreamShaper-Webcam")
    with gr.Row():
        with gr.Column():
            gr.Markdown("## INPUT")
            # Replace canvas with webcam image
            image = gr.Image(
                source="webcam", type="pil", label="Webcam Image", interactive=True
            )

            prompt = gr.Textbox(label="Prompt", value="", show_label=True)
            style = gr.Dropdown(label="Style", choices=STYLE_NAMES, value=DEFAULT_STYLE_NAME)
            prompt_template = gr.Textbox(label="Prompt Style Template", value=styles[DEFAULT_STYLE_NAME])

            controlnet_conditioning_scale = gr.Slider(label="Control Strength", minimum=0, maximum=1, step=0.01, value=0.8)

            device_choices = ['GPU','CPU']
            device_type = gr.Radio(device_choices, label='Device', value=device_choices[0], interactive=True)

            dtype_choices = ['torch.float16','torch.float32']
            param_dtype = gr.Radio(dtype_choices, label='torch.weight_type', value=dtype_choices[0], interactive=True)

        with gr.Column():
            gr.Markdown("## OUTPUT")
            result = gr.Image(label="Result", show_label=False, show_download_button=True)

    inputs = [image, prompt, prompt_template, style, controlnet_conditioning_scale, device_type, param_dtype]
    outputs = [result]
    prompt.submit(fn=run, inputs=inputs, outputs=outputs)
    style.change(lambda x: styles[x], inputs=[style], outputs=[prompt_template])
    image.change(run, inputs=inputs, outputs=outputs)

if __name__ == "__main__":
    demo.queue().launch(debug=True)