import numpy as np import cv2 from PIL import Image from .prompt import split_weighted_subprompts from .load_images import load_img, prepare_mask, check_mask_for_errors from .webui_sd_pipeline import get_webui_sd_pipeline from .animation import sample_from_cv2, sample_to_cv2 from .rich import console #Webui import cv2 from .animation import sample_from_cv2, sample_to_cv2 from modules import processing, sd_models from modules.shared import opts, sd_model from modules.processing import process_images, StableDiffusionProcessingTxt2Img from .deforum_controlnet import is_controlnet_enabled, process_txt2img_with_controlnet, process_img2img_with_controlnet import math, json, itertools import requests def load_mask_latent(mask_input, shape): # mask_input (str or PIL Image.Image): Path to the mask image or a PIL Image object # shape (list-like len(4)): shape of the image to match, usually latent_image.shape if isinstance(mask_input, str): # mask input is probably a file name if mask_input.startswith('http://') or mask_input.startswith('https://'): mask_image = Image.open(requests.get(mask_input, stream=True).raw).convert('RGBA') else: mask_image = Image.open(mask_input).convert('RGBA') elif isinstance(mask_input, Image.Image): mask_image = mask_input else: raise Exception("mask_input must be a PIL image or a file name") mask_w_h = (shape[-1], shape[-2]) mask = mask_image.resize(mask_w_h, resample=Image.LANCZOS) mask = mask.convert("L") return mask def isJson(myjson): try: json.loads(myjson) except ValueError as e: return False return True # Add pairwise implementation here not to upgrade # the whole python to 3.10 just for one function def pairwise_repl(iterable): a, b = itertools.tee(iterable) next(b, None) return zip(a, b) def generate(args, anim_args, loop_args, controlnet_args, root, frame = 0, return_sample=False, sampler_name=None): assert args.prompt is not None # Setup the pipeline p = get_webui_sd_pipeline(args, root, frame) p.prompt, p.negative_prompt = split_weighted_subprompts(args.prompt, frame) if not args.use_init and args.strength > 0 and args.strength_0_no_init: print("\nNo init image, but strength > 0. Strength has been auto set to 0, since use_init is False.") print("If you want to force strength > 0 with no init, please set strength_0_no_init to False.\n") args.strength = 0 processed = None mask_image = None init_image = None image_init0 = None if loop_args.use_looper: # TODO find out why we need to set this in the init tab if args.strength == 0: raise RuntimeError("Strength needs to be greater than 0 in Init tab and strength_0_no_init should *not* be checked") if args.seed_behavior != "schedule": raise RuntimeError("seed_behavior needs to be set to schedule in under 'Keyframes' tab --> 'Seed scheduling'") if not isJson(loop_args.imagesToKeyframe): raise RuntimeError("The images set for use with keyframe-guidance are not in a proper JSON format") args.strength = loop_args.imageStrength tweeningFrames = loop_args.tweeningFrameSchedule blendFactor = .07 colorCorrectionFactor = loop_args.colorCorrectionFactor jsonImages = json.loads(loop_args.imagesToKeyframe) framesToImageSwapOn = list(map(int, list(jsonImages.keys()))) # find which image to show frameToChoose = 0 for swappingFrame in framesToImageSwapOn[1:]: frameToChoose += (frame >= int(swappingFrame)) #find which frame to do our swapping on for tweening skipFrame = 25 for fs, fe in pairwise_repl(framesToImageSwapOn): if fs <= frame <= fe: skipFrame = fe - fs if frame % skipFrame <= tweeningFrames: # number of tweening frames blendFactor = loop_args.blendFactorMax - loop_args.blendFactorSlope*math.cos((frame % tweeningFrames) / (tweeningFrames / 2)) init_image2, _ = load_img(list(jsonImages.values())[frameToChoose], shape=(args.W, args.H), use_alpha_as_mask=args.use_alpha_as_mask) image_init0 = list(jsonImages.values())[0] else: # they passed in a single init image image_init0 = args.init_image available_samplers = { 'euler a':'Euler a', 'euler':'Euler', 'lms':'LMS', 'heun':'Heun', 'dpm2':'DPM2', 'dpm2 a':'DPM2 a', 'dpm++ 2s a':'DPM++ 2S a', 'dpm++ 2m':'DPM++ 2M', 'dpm++ sde':'DPM++ SDE', 'dpm fast':'DPM fast', 'dpm adaptive':'DPM adaptive', 'lms karras':'LMS Karras' , 'dpm2 karras':'DPM2 Karras', 'dpm2 a karras':'DPM2 a Karras', 'dpm++ 2s a karras':'DPM++ 2S a Karras', 'dpm++ 2m karras':'DPM++ 2M Karras', 'dpm++ sde karras':'DPM++ SDE Karras' } if sampler_name is not None: if sampler_name in available_samplers.keys(): args.sampler = available_samplers[sampler_name] if args.checkpoint is not None: info = sd_models.get_closet_checkpoint_match(args.checkpoint) if info is None: raise RuntimeError(f"Unknown checkpoint: {args.checkpoint}") sd_models.reload_model_weights(info=info) if args.init_sample is not None: # TODO: cleanup init_sample remains later img = args.init_sample init_image = img image_init0 = img if loop_args.use_looper and isJson(loop_args.imagesToKeyframe): init_image = Image.blend(init_image, init_image2, blendFactor) correction_colors = Image.blend(init_image, init_image2, colorCorrectionFactor) p.color_corrections = [processing.setup_color_correction(correction_colors)] # this is the first pass elif loop_args.use_looper or (args.use_init and ((args.init_image != None and args.init_image != ''))): init_image, mask_image = load_img(image_init0, # initial init image shape=(args.W, args.H), use_alpha_as_mask=args.use_alpha_as_mask) else: if anim_args.animation_mode != 'Interpolation': print(f"Not using an init image (doing pure txt2img)") p_txt = StableDiffusionProcessingTxt2Img( sd_model=sd_model, outpath_samples=root.tmp_deforum_run_duplicated_folder, outpath_grids=root.tmp_deforum_run_duplicated_folder, prompt=p.prompt, styles=p.styles, negative_prompt=p.negative_prompt, seed=p.seed, subseed=p.subseed, subseed_strength=p.subseed_strength, seed_resize_from_h=p.seed_resize_from_h, seed_resize_from_w=p.seed_resize_from_w, sampler_name=p.sampler_name, batch_size=p.batch_size, n_iter=p.n_iter, steps=p.steps, cfg_scale=p.cfg_scale, width=p.width, height=p.height, restore_faces=p.restore_faces, tiling=p.tiling, enable_hr=None, denoising_strength=None, ) # print dynamic table to cli print_generate_table(args, anim_args, p_txt) if is_controlnet_enabled(controlnet_args): processed = process_txt2img_with_controlnet(p, args, anim_args, loop_args, controlnet_args, root, frame) else: processed = processing.process_images(p_txt) if processed is None: # Mask functions if args.use_mask: mask = args.mask_image #assign masking options to pipeline if mask is not None: p.inpainting_mask_invert = args.invert_mask p.inpainting_fill = args.fill p.inpaint_full_res= args.full_res_mask p.inpaint_full_res_padding = args.full_res_mask_padding else: mask = None assert not ( (mask is not None and args.use_mask and args.overlay_mask) and (args.init_sample is None and init_image is None)), "Need an init image when use_mask == True and overlay_mask == True" p.init_images = [init_image] p.image_mask = mask p.image_cfg_scale = args.pix2pix_img_cfg_scale # print dynamic table to cli print_generate_table(args, anim_args, p) if is_controlnet_enabled(controlnet_args): processed = process_img2img_with_controlnet(p, args, anim_args, loop_args, controlnet_args, root, frame) else: processed = processing.process_images(p) if root.initial_info == None: root.initial_seed = processed.seed root.initial_info = processed.info if root.first_frame == None: root.first_frame = processed.images[0] results = processed.images[0] return results def print_generate_table(args, anim_args, p): from rich.table import Table from rich import box table = Table(padding=0, box=box.ROUNDED) field_names = ["Steps", "CFG"] if anim_args.animation_mode != 'Interpolation': field_names.append("Denoise") field_names += ["Subseed", "Subs. str"] * (anim_args.enable_subseed_scheduling) field_names += ["Sampler"] * anim_args.enable_sampler_scheduling field_names += ["Checkpoint"] * anim_args.enable_checkpoint_scheduling for field_name in field_names: table.add_column(field_name, justify="center") rows = [str(p.steps), str(p.cfg_scale)] if anim_args.animation_mode != 'Interpolation': rows.append(str(p.denoising_strength)) rows += [str(p.subseed), str(p.subseed_strength)] * (anim_args.enable_subseed_scheduling) rows += [p.sampler_name] * anim_args.enable_sampler_scheduling rows += [str(args.checkpoint)] * anim_args.enable_checkpoint_scheduling table.add_row(*rows) console.print(table)