File size: 3,029 Bytes
28c838b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 |
# Diffusers' ControlNet Implementation Subjective Evaluation
import einops
import numpy as np
import torch
import sys
import os
import yaml
from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, DDIMScheduler
from PIL import Image
test_prompt = "best quality, extremely detailed"
test_negative_prompt = "lowres, bad anatomy, worst quality, low quality"
def generate_image(seed, prompt, negative_prompt, control, guess_mode=False):
latent = torch.randn((1,4,64,64), device="cpu", generator=torch.Generator(device="cpu").manual_seed(seed)).cuda()
image = pipe(
prompt=prompt,
negative_prompt=negative_prompt,
guidance_scale=4.0 if guess_mode else 9.0,
num_inference_steps=50 if guess_mode else 20,
latents=latent,
image=control,
# guess_mode=guess_mode,
).images[0]
return image
def control_images(control_image_folder, model_name):
with open('./control_images.yaml', 'r') as f:
d = yaml.safe_load(f)
filenames = d[model_name]
return [Image.open(f'{control_image_folder}/{fn}').convert("RGB") for fn in filenames]
if __name__ == '__main__':
model_name = sys.argv[1]
control_image_folder = './control_images/converted/'
output_image_folder = './output_images/diffusers/'
os.makedirs(output_image_folder, exist_ok=True)
model_id = 'takuma104/control_v11'
subfolder = f'control_v11{model_name}'
controlnet = ControlNetModel.from_pretrained(model_id, subfolder=subfolder)
if model_name == 'p_sd15s2_lineart_anime':
base_model_id = 'Linaqruf/anything-v3.0'
base_model_revision = None
else:
base_model_id = "runwayml/stable-diffusion-v1-5"
base_model_revision = 'non-ema'
pipe = StableDiffusionControlNetPipeline.from_pretrained(base_model_id,
revision=base_model_revision,
controlnet=controlnet,
safety_checker=None).to("cuda")
pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config)
for i, control_image in enumerate(control_images(control_image_folder, model_name)):
control = np.array(control_image)[:,:,::-1].copy()
control = torch.from_numpy(control).float().cuda() / 255.0
control = torch.stack([control for _ in range(1)], dim=0)
control = einops.rearrange(control, 'b h w c -> b c h w').clone()
# if model_name == 'p_sd15_normalbae': # workaround, this should not be necessary
# control = torch.flip(control, dims=[1]) # RGB -> BGR
for seed in range(4):
image = generate_image(seed=seed,
prompt=test_prompt,
negative_prompt=test_negative_prompt,
control=control)
image.save(f'{output_image_folder}output_{model_name}_{i}_{seed}.png')
|