import spaces import random import torch from huggingface_hub import snapshot_download from kolors.pipelines.pipeline_stable_diffusion_xl_chatglm_256_inpainting import StableDiffusionXLInpaintPipeline from kolors.models.modeling_chatglm import ChatGLMModel from kolors.models.tokenization_chatglm import ChatGLMTokenizer from diffusers import AutoencoderKL, EulerDiscreteScheduler, UNet2DConditionModel import gradio as gr import numpy as np from PIL import Image device = "cuda" ckpt_dir = snapshot_download(repo_id="Kwai-Kolors/Kolors-Inpainting") text_encoder = ChatGLMModel.from_pretrained(f'{ckpt_dir}/text_encoder',torch_dtype=torch.float16).half().to(device) tokenizer = ChatGLMTokenizer.from_pretrained(f'{ckpt_dir}/text_encoder') vae = AutoencoderKL.from_pretrained(f"{ckpt_dir}/vae", revision=None).half().to(device) scheduler = EulerDiscreteScheduler.from_pretrained(f"{ckpt_dir}/scheduler") unet = UNet2DConditionModel.from_pretrained(f"{ckpt_dir}/unet", revision=None).half().to(device) pipe = StableDiffusionXLInpaintPipeline( vae=vae, text_encoder=text_encoder, tokenizer=tokenizer, unet=unet, scheduler=scheduler ) pipe.to(device) pipe.enable_attention_slicing() MAX_SEED = np.iinfo(np.int32).max MAX_IMAGE_SIZE = 1024 @spaces.GPU def infer(prompt, image, negative_prompt = "", seed = 0, randomize_seed = False, guidance_scale = 5.0, num_inference_steps = 25 ): if randomize_seed: seed = random.randint(0, MAX_SEED) generator = torch.Generator().manual_seed(seed) width, height = image['background'].size result = pipe( prompt = prompt, image = image['background'], mask_image = image['layers'][0], height=height, width=width, guidance_scale = guidance_scale, generator= generator, num_inference_steps= num_inference_steps, negative_prompt = negative_prompt, num_images_per_prompt = 1, strength = 0.999 ).images[0] return result examples = [ ] css=""" #col-left { margin: 0 auto; max-width: 600px; } #col-right { margin: 0 auto; max-width: 750px; } """ def load_description(fp): with open(fp, 'r', encoding='utf-8') as f: content = f.read() return content with gr.Blocks(css=css) as Kolors: gr.HTML(load_description("assets/title.md")) with gr.Row(): with gr.Column(elem_id="col-left"): with gr.Row(): prompt = gr.Textbox( label="Prompt", placeholder="Enter your prompt", lines=2 ) with gr.Row(): image = gr.ImageEditor(type='pil', image_mode='RGB', layers=False, brush=gr.Brush(colors=["#AAAAAA"], color_mode="fixed")) with gr.Accordion("Advanced Settings", open=False): negative_prompt = gr.Textbox( label="Negative prompt", placeholder="Enter a negative prompt", visible=True, value='残缺的手指,畸形的手指,畸形的手,残肢,模糊,低质量' ) seed = gr.Slider( label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0, ) randomize_seed = gr.Checkbox(label="Randomize seed", value=True) with gr.Row(): guidance_scale = gr.Slider( label="Guidance scale", minimum=0.0, maximum=10.0, step=0.1, value=5.0, ) num_inference_steps = gr.Slider( label="Number of inference steps", minimum=10, maximum=50, step=1, value=25, ) with gr.Row(): run_button = gr.Button("Run") with gr.Column(elem_id="col-right"): result = gr.Image(label="Result", show_label=False) # with gr.Row(): # gr.Examples( # fn = infer, # examples = examples, # inputs = [prompt, ip_adapter_image, ip_adapter_scale], # outputs = [result] # ) run_button.click( fn = infer, inputs = [prompt, image, negative_prompt, seed, randomize_seed, guidance_scale, num_inference_steps], outputs = [result] ) Kolors.queue().launch(debug=True)