SD3 Controlnet
raw | control image | output |
---|---|---|
Install Diffusers-SD3-Controlnet
The current diffusers have not been merged into the official code yet.
git clone -b sd3_control https://github.com/instantX-research/diffusers_sd3_control.git
cd diffusers_sd3_control
pip install -e .
Demo
import torch
from diffusers.utils import load_image
import sys, os
sys.path.append('/path/diffusers/examples/community')
from pipeline_stable_diffusion_3_controlnet import StableDiffusion3CommonPipeline
# load pipeline
base_model = 'stabilityai/stable-diffusion-3-medium-diffusers'
pipe = StableDiffusion3CommonPipeline.from_pretrained(
base_model,
controlnet_list=['InstantX/SD3-Controlnet-Pose']
)
pipe.to('cuda:0', torch.float16)
prompt = 'Anime style illustration of a girl wearing a suit. A moon in sky. In the background we see a big rain approaching. text "InstantX" on image'
n_prompt = 'NSFW, nude, naked, porn, ugly'
# controlnet config
controlnet_conditioning = [
dict(
control_index=0,
control_image=load_image('https://huggingface.co/InstantX/SD3-Controlnet-Pose/resolve/main/pose.jpg'),
control_weight=0.7,
control_pooled_projections='zeros'
)
]
# infer
image = pipe(
prompt=prompt,
negative_prompt=n_prompt,
controlnet_conditioning=controlnet_conditioning,
num_inference_steps=28,
guidance_scale=7.0,
height=1024,
width=1024,
).images[0]