gen_compare_v11/gen_diffusers_image.py

# Diffusers' ControlNet Implementation Subjective Evaluation

import einops
import numpy as np
import torch
import sys
import os
import yaml

from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, DDIMScheduler

from PIL import Image

test_prompt = "best quality, extremely detailed"
test_negative_prompt = "lowres, bad anatomy, worst quality, low quality"

def generate_image(seed, prompt, negative_prompt, control, guess_mode=False):
    latent = torch.randn((1,4,64,64), device="cpu", generator=torch.Generator(device="cpu").manual_seed(seed)).cuda()
    image = pipe(
        prompt=prompt,
        negative_prompt=negative_prompt,
        guidance_scale=4.0 if guess_mode else 9.0,
        num_inference_steps=50 if guess_mode else 20,
        latents=latent,
        image=control,
        # guess_mode=guess_mode,
    ).images[0]
    return image

def control_images(control_image_folder, model_name):
    with open('./control_images.yaml', 'r') as f:
        d = yaml.safe_load(f)
    filenames = d[model_name]
    return [Image.open(f'{control_image_folder}/{fn}').convert("RGB") for fn in filenames]

if __name__ == '__main__':
    model_name = sys.argv[1]
    control_image_folder = './control_images/converted/'
    output_image_folder = './output_images/diffusers/'
    os.makedirs(output_image_folder, exist_ok=True)

    model_id = f"lllyasviel/control_v11{model_name}"

    controlnet = ControlNetModel.from_pretrained(model_id)
    if model_name == 'p_sd15s2_lineart_anime':
        base_model_id = 'Linaqruf/anything-v3.0'
        base_model_revision = None
    else:   
        base_model_id = "runwayml/stable-diffusion-v1-5"
        base_model_revision = 'non-ema'

    pipe = StableDiffusionControlNetPipeline.from_pretrained(base_model_id, 
                                                             revision=base_model_revision,
                                                             controlnet=controlnet, 
                                                             safety_checker=None).to("cuda")
    pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config)

    for i, control_image in enumerate(control_images(control_image_folder, model_name)):
        control = np.array(control_image)[:,:,::-1].copy()
        control = torch.from_numpy(control).float().cuda() / 255.0
        control = torch.stack([control for _ in range(1)], dim=0)
        control = einops.rearrange(control, 'b h w c -> b c h w').clone()

        # if model_name == 'p_sd15_normalbae': # workaround, this should not be necessary
        #     control = torch.flip(control, dims=[1]) # RGB -> BGR

        for seed in range(4):
            image = generate_image(seed=seed, 
                                   prompt=test_prompt, 
                                   negative_prompt=test_negative_prompt, 
                                   control=control)
            image.save(f'{output_image_folder}output_{model_name}_{i}_{seed}.png')