import torch from diffusers.utils.torch_utils import randn_tensor import json, os, cv2 from PIL import Image import numpy as np def parse_config(config): with open(config, 'r') as f: config = json.load(f) return config def load_config(config): activate_layer_indices_list = config['inference_info']['activate_layer_indices_list'] activate_step_indices_list = config['inference_info']['activate_step_indices_list'] ref_seeds = config['reference_info']['ref_seeds'] inf_seeds = config['inference_info']['inf_seeds'] attn_map_save_steps = config['inference_info']['attn_map_save_steps'] precomputed_path = config['precomputed_path'] guidance_scale = config['guidance_scale'] use_inf_negative_prompt = config['inference_info']['use_negative_prompt'] style_name_list = config["style_name_list"] ref_object_list = config["reference_info"]["ref_object_list"] inf_object_list = config["inference_info"]["inf_object_list"] ref_with_style_description = config['reference_info']['with_style_description'] inf_with_style_description = config['inference_info']['with_style_description'] use_shared_attention = config['inference_info']['use_shared_attention'] adain_queries = config['inference_info']['adain_queries'] adain_keys = config['inference_info']['adain_keys'] adain_values = config['inference_info']['adain_values'] use_advanced_sampling = config['inference_info']['use_advanced_sampling'] out = [ activate_layer_indices_list, activate_step_indices_list, ref_seeds, inf_seeds, attn_map_save_steps, precomputed_path, guidance_scale, use_inf_negative_prompt, style_name_list, ref_object_list, inf_object_list, ref_with_style_description, inf_with_style_description, use_shared_attention, adain_queries, adain_keys, adain_values, use_advanced_sampling ] return out def memory_efficient(model, device): try: model.to(device) except Exception as e: print("Error moving model to device:", e) try: model.enable_model_cpu_offload() except AttributeError: print("enable_model_cpu_offload is not supported.") try: model.enable_vae_slicing() except AttributeError: print("enable_vae_slicing is not supported.") try: model.enable_vae_tiling() except AttributeError: print("enable_vae_tiling is not supported.") try: model.enable_xformers_memory_efficient_attention() except AttributeError: print("enable_xformers_memory_efficient_attention is not supported.") def init_latent(model, device_name='cuda', dtype=torch.float16, seed=None): scale_factor = model.vae_scale_factor sample_size = model.default_sample_size latent_dim = model.unet.config.in_channels height = sample_size * scale_factor width = sample_size * scale_factor shape = (1, latent_dim, height // scale_factor, width // scale_factor) device = torch.device(device_name) generator = torch.Generator(device).manual_seed(seed) if seed is not None else None latent = randn_tensor(shape, generator=generator, dtype=dtype, device=device) return latent def get_canny_edge_array(canny_img_path, threshold1=100,threshold2=200): canny_image_list = [] # check if canny_img_path is a directory if os.path.isdir(canny_img_path): canny_img_list = os.listdir(canny_img_path) for canny_img in canny_img_list: canny_image_tmp = Image.open(os.path.join(canny_img_path, canny_img)) #resize image into1024x1024 canny_image_tmp = canny_image_tmp.resize((1024,1024)) canny_image_tmp = np.array(canny_image_tmp) canny_image_tmp = cv2.Canny(canny_image_tmp, threshold1, threshold2) canny_image_tmp = canny_image_tmp[:, :, None] canny_image_tmp = np.concatenate([canny_image_tmp, canny_image_tmp, canny_image_tmp], axis=2) canny_image = Image.fromarray(canny_image_tmp) canny_image_list.append(canny_image) return canny_image_list def get_depth_map(image, feature_extractor, depth_estimator, device='cuda'): image = feature_extractor(images=image, return_tensors="pt").pixel_values.to(device) with torch.no_grad(), torch.autocast(device): depth_map = depth_estimator(image).predicted_depth depth_map = torch.nn.functional.interpolate( depth_map.unsqueeze(1), size=(1024, 1024), mode="bicubic", align_corners=False, ) depth_min = torch.amin(depth_map, dim=[1, 2, 3], keepdim=True) depth_max = torch.amax(depth_map, dim=[1, 2, 3], keepdim=True) depth_map = (depth_map - depth_min) / (depth_max - depth_min) image = torch.cat([depth_map] * 3, dim=1) image = image.permute(0, 2, 3, 1).cpu().numpy()[0] image = Image.fromarray((image * 255.0).clip(0, 255).astype(np.uint8)) return image def get_depth_edge_array(depth_img_path, feature_extractor, depth_estimator, device='cuda'): depth_image_list = [] # check if canny_img_path is a directory if os.path.isdir(depth_img_path): depth_img_list = os.listdir(depth_img_path) for depth_img in depth_img_list: depth_image_tmp = Image.open(os.path.join(depth_img_path, depth_img)).convert('RGB') # get depth map depth_map = get_depth_map(depth_image_tmp, feature_extractor, depth_estimator, device) depth_image_list.append(depth_map) return depth_image_list