!pip install opencv-python transformers accelerate

from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, UniPCMultistepScheduler
from diffusers.utils import load_image
import numpy as np
import torch

import cv2
from PIL import Image

download an image

image = load_image(
"https://wb-web.de/_Resources/Persistent/f/e/b/7/feb7a0d8ee35621aa5ac515bd019ed1e9f59746b/QRCode1.jpg"
)
image = np.array(image)

get canny image

image = cv2.Canny(image, 100, 200)
image = image[:, :, None]
image = np.concatenate([image, image, image], axis=2)
canny_image = Image.fromarray(image)

load control net and stable diffusion v1-5

controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny", torch_dtype=torch.float16)
pipe = StableDiffusionControlNetPipeline.from_pretrained(
"runwayml/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=torch.float16
)

speed up diffusion process with faster scheduler and memory optimization

pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)

remove following line if xformers is not installed

pipe.enable_xformers_memory_efficient_attention()

pipe.enable_model_cpu_offload()

generate image

generator = torch.manual_seed(0)
image = pipe(
"Harry Potter", num_inference_steps=20, generator=generator, image=canny_image
).images[0]

Save the image to a file

image.save("output.png")

Am I using the model right? How can I put in your model files in the pipeline and controlnet parameters to fully function? Sorry for the dumb question im still learning.