import spaces from diffusers import ControlNetModel from diffusers import StableDiffusionXLControlNetPipeline from diffusers import EulerAncestralDiscreteScheduler from PIL import Image import torch import numpy as np import cv2 import gradio as gr from torchvision import transforms from controlnet_aux import OpenposeDetector ratios_map = { 0.5:{"width":704,"height":1408}, 0.57:{"width":768,"height":1344}, 0.68:{"width":832,"height":1216}, 0.72:{"width":832,"height":1152}, 0.78:{"width":896,"height":1152}, 0.82:{"width":896,"height":1088}, 0.88:{"width":960,"height":1088}, 0.94:{"width":960,"height":1024}, 1.00:{"width":1024,"height":1024}, 1.13:{"width":1088,"height":960}, 1.21:{"width":1088,"height":896}, 1.29:{"width":1152,"height":896}, 1.38:{"width":1152,"height":832}, 1.46:{"width":1216,"height":832}, 1.67:{"width":1280,"height":768}, 1.75:{"width":1344,"height":768}, 2.00:{"width":1408,"height":704} } ratios = np.array(list(ratios_map.keys())) openpose = OpenposeDetector.from_pretrained('lllyasviel/ControlNet') controlnet = ControlNetModel.from_pretrained( "briaai/BRIA-2.3-ControlNet-Pose", torch_dtype=torch.float16 ).to('cuda') pipe = StableDiffusionXLControlNetPipeline.from_pretrained( "briaai/BRIA-2.3", controlnet=controlnet, torch_dtype=torch.float16, low_cpu_mem_usage=True, offload_state_dict=True, ).to('cuda').to(torch.float16) pipe.scheduler = EulerAncestralDiscreteScheduler( beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", num_train_timesteps=1000, steps_offset=1 ) # pipe.enable_freeu(b1=1.1, b2=1.1, s1=0.5, s2=0.7) # pipe.enable_xformers_memory_efficient_attention() pipe.force_zeros_for_empty_prompt = False def get_size(init_image): w,h=init_image.size curr_ratio = w/h ind = np.argmin(np.abs(curr_ratio-ratios)) ratio = ratios[ind] chosen_ratio = ratios_map[ratio] w,h = chosen_ratio['width'], chosen_ratio['height'] return w,h def resize_image(image): image = image.convert('RGB') w,h = get_size(image) resized_image = image.resize((w, h)) return resized_image def resize_image_old(image): image = image.convert('RGB') current_size = image.size if current_size[0] > current_size[1]: center_cropped_image = transforms.functional.center_crop(image, (current_size[1], current_size[1])) else: center_cropped_image = transforms.functional.center_crop(image, (current_size[0], current_size[0])) resized_image = transforms.functional.resize(center_cropped_image, (1024, 1024)) return resized_image @spaces.GPU def generate_(prompt, negative_prompt, pose_image, input_image, num_steps, controlnet_conditioning_scale, seed): generator = torch.Generator("cuda").manual_seed(seed) images = pipe( prompt, negative_prompt=negative_prompt, image=pose_image, num_inference_steps=num_steps, controlnet_conditioning_scale=float(controlnet_conditioning_scale), generator=generator, height=input_image.size[1], width=input_image.size[0], ).images return images @spaces.GPU def process(input_image, prompt, negative_prompt, num_steps, controlnet_conditioning_scale, seed): # resize input_image to 1024x1024 input_image = resize_image(input_image) pose_image = openpose(input_image, include_body=True, include_hand=True, include_face=True) images = generate_(prompt, negative_prompt, pose_image, input_image, num_steps, controlnet_conditioning_scale, seed) return [pose_image,images[0]] block = gr.Blocks().queue() with block: gr.Markdown("## BRIA 2.3 ControlNet Pose") gr.HTML('''
This is a demo for ControlNet Pose that using BRIA 2.3 text-to-image model as backbone. Trained on licensed data, BRIA 2.3 provide full legal liability coverage for copyright and privacy infringement.
''') with gr.Row(): with gr.Column(): input_image = gr.Image(sources=None, type="pil") # None for upload, ctrl+v and webcam prompt = gr.Textbox(label="Prompt") negative_prompt = gr.Textbox(label="Negative prompt", value="Logo,Watermark,Text,Ugly,Morbid,Extra fingers,Poorly drawn hands,Mutation,Blurry,Extra limbs,Gross proportions,Missing arms,Mutated hands,Long neck,Duplicate,Mutilated,Mutilated hands,Poorly drawn face,Deformed,Bad anatomy,Cloned face,Malformed limbs,Missing legs,Too many fingers") num_steps = gr.Slider(label="Number of steps", minimum=25, maximum=100, value=50, step=1) controlnet_conditioning_scale = gr.Slider(label="ControlNet conditioning scale", minimum=0.1, maximum=2.0, value=1.0, step=0.05) seed = gr.Slider(label="Seed", minimum=0, maximum=2147483647, step=1, randomize=True,) run_button = gr.Button(value="Run") with gr.Column(): with gr.Row(): pose_image_output = gr.Image(label="Pose Image", type="pil", interactive=False) generated_image_output = gr.Image(label="Generated Image", type="pil", interactive=False) ips = [input_image, prompt, negative_prompt, num_steps, controlnet_conditioning_scale, seed] run_button.click(fn=process, inputs=ips, outputs=[pose_image_output, generated_image_output]) block.launch(debug = True)