Spaces:

multimodalart
/

cosxl

Running on Zero

App Files Files Community

cosxl / custom_pipeline.py

multimodalart HF staff

Upload custom_pipeline.py

517af83 verified about 2 months ago

raw history blame contribute delete

No virus

52.8 kB

	# Copyright 2024 Harutatsu Akiyama and The HuggingFace Team. All rights reserved.
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	import inspect
	from typing import Any, Callable, Dict, List, Optional, Tuple, Union

	import PIL.Image
	import torch
	from transformers import CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer

	from diffusers.image_processor import PipelineImageInput, VaeImageProcessor
	from diffusers.loaders import FromSingleFileMixin, StableDiffusionXLLoraLoaderMixin, TextualInversionLoaderMixin
	from diffusers.models import AutoencoderKL, UNet2DConditionModel
	from diffusers.models.attention_processor import (
	AttnProcessor2_0,
	FusedAttnProcessor2_0,
	LoRAAttnProcessor2_0,
	LoRAXFormersAttnProcessor,
	XFormersAttnProcessor,
	)
	from diffusers.models.lora import adjust_lora_scale_text_encoder
	from diffusers.schedulers import KarrasDiffusionSchedulers
	from diffusers.utils import (
	USE_PEFT_BACKEND,
	deprecate,
	is_invisible_watermark_available,
	is_torch_xla_available,
	logging,
	replace_example_docstring,
	scale_lora_layers,
	)
	from diffusers.utils.torch_utils import randn_tensor
	from diffusers.pipelines.pipeline_utils import DiffusionPipeline, StableDiffusionMixin
	from diffusers.pipelines.stable_diffusion_xl.pipeline_output import StableDiffusionXLPipelineOutput


	if is_invisible_watermark_available():
	from diffusers.pipelines.stable_diffusion_xl.watermark import StableDiffusionXLWatermarker

	if is_torch_xla_available():
	import torch_xla.core.xla_model as xm

	XLA_AVAILABLE = True
	else:
	XLA_AVAILABLE = False


	logger = logging.get_logger(__name__) # pylint: disable=invalid-name

	EXAMPLE_DOC_STRING = """
	Examples:
	```py
	>>> import torch
	>>> from diffusers import StableDiffusionXLInstructPix2PixPipeline
	>>> from diffusers.utils import load_image

	>>> resolution = 768
	>>> image = load_image(
	... "https://hf.co/datasets/diffusers/diffusers-images-docs/resolve/main/mountain.png"
	... ).resize((resolution, resolution))
	>>> edit_instruction = "Turn sky into a cloudy one"

	>>> pipe = StableDiffusionXLInstructPix2PixPipeline.from_pretrained(
	... "diffusers/sdxl-instructpix2pix-768", torch_dtype=torch.float16
	... ).to("cuda")

	>>> edited_image = pipe(
	... prompt=edit_instruction,
	... image=image,
	... height=resolution,
	... width=resolution,
	... guidance_scale=3.0,
	... image_guidance_scale=1.5,
	... num_inference_steps=30,
	... ).images[0]
	>>> edited_image
	```
	"""


	# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents
	def retrieve_latents(
	encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample"
	):
	if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
	return encoder_output.latent_dist.sample(generator)
	elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
	return encoder_output.latent_dist.mode()
	elif hasattr(encoder_output, "latents"):
	return encoder_output.latents
	else:
	raise AttributeError("Could not access latents of provided encoder_output")


	def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0):
	"""
	Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and
	Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4
	"""
	std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True)
	std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True)
	# rescale the results from guidance (fixes overexposure)
	noise_pred_rescaled = noise_cfg * (std_text / std_cfg)
	# mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images
	noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg
	return noise_cfg


	class CosStableDiffusionXLInstructPix2PixPipeline(
	DiffusionPipeline,
	StableDiffusionMixin,
	TextualInversionLoaderMixin,
	FromSingleFileMixin,
	StableDiffusionXLLoraLoaderMixin,
	):
	r"""
	Pipeline for pixel-level image editing by following text instructions. Based on Stable Diffusion XL.

	This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the
	library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)

	The pipeline also inherits the following loading methods:
	- [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings
	- [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files
	- [`~loaders.StableDiffusionXLLoraLoaderMixin.load_lora_weights`] for loading LoRA weights
	- [`~loaders.StableDiffusionXLLoraLoaderMixin.save_lora_weights`] for saving LoRA weights

	Args:
	vae ([`AutoencoderKL`]):
	Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
	text_encoder ([`CLIPTextModel`]):
	Frozen text-encoder. Stable Diffusion XL uses the text portion of
	[CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically
	the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.
	text_encoder_2 ([` CLIPTextModelWithProjection`]):
	Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of
	[CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection),
	specifically the
	[laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k)
	variant.
	tokenizer (`CLIPTokenizer`):
	Tokenizer of class
	[CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
	tokenizer_2 (`CLIPTokenizer`):
	Second Tokenizer of class
	[CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
	unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.
	scheduler ([`SchedulerMixin`]):
	A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of
	[`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].
	requires_aesthetics_score (`bool`, optional, defaults to `"False"`):
	Whether the `unet` requires a aesthetic_score condition to be passed during inference. Also see the config
	of `stabilityai/stable-diffusion-xl-refiner-1-0`.
	force_zeros_for_empty_prompt (`bool`, optional, defaults to `"True"`):
	Whether the negative prompt embeddings shall be forced to always be set to 0. Also see the config of
	`stabilityai/stable-diffusion-xl-base-1-0`.
	add_watermarker (`bool`, optional):
	Whether to use the [invisible_watermark library](https://github.com/ShieldMnt/invisible-watermark/) to
	watermark output images. If not defined, it will default to True if the package is installed, otherwise no
	watermarker will be used.
	"""

	model_cpu_offload_seq = "text_encoder->text_encoder_2->unet->vae"
	_optional_components = ["tokenizer", "tokenizer_2", "text_encoder", "text_encoder_2"]

	def __init__(
	self,
	vae: AutoencoderKL,
	text_encoder: CLIPTextModel,
	text_encoder_2: CLIPTextModelWithProjection,
	tokenizer: CLIPTokenizer,
	tokenizer_2: CLIPTokenizer,
	unet: UNet2DConditionModel,
	scheduler: KarrasDiffusionSchedulers,
	force_zeros_for_empty_prompt: bool = True,
	add_watermarker: Optional[bool] = None,
	):
	super().__init__()

	self.register_modules(
	vae=vae,
	text_encoder=text_encoder,
	text_encoder_2=text_encoder_2,
	tokenizer=tokenizer,
	tokenizer_2=tokenizer_2,
	unet=unet,
	scheduler=scheduler,
	)
	self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt)
	self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
	self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
	self.default_sample_size = self.unet.config.sample_size

	add_watermarker = add_watermarker if add_watermarker is not None else is_invisible_watermark_available()

	if add_watermarker:
	self.watermark = StableDiffusionXLWatermarker()
	else:
	self.watermark = None

	def encode_prompt(
	self,
	prompt: str,
	prompt_2: Optional[str] = None,
	device: Optional[torch.device] = None,
	num_images_per_prompt: int = 1,
	do_classifier_free_guidance: bool = True,
	negative_prompt: Optional[str] = None,
	negative_prompt_2: Optional[str] = None,
	prompt_embeds: Optional[torch.FloatTensor] = None,
	negative_prompt_embeds: Optional[torch.FloatTensor] = None,
	pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
	negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
	lora_scale: Optional[float] = None,
	):
	r"""
	Encodes the prompt into text encoder hidden states.

	Args:
	prompt (`str` or `List[str]`, optional):
	prompt to be encoded
	prompt_2 (`str` or `List[str]`, optional):
	The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
	used in both text-encoders
	device: (`torch.device`):
	torch device
	num_images_per_prompt (`int`):
	number of images that should be generated per prompt
	do_classifier_free_guidance (`bool`):
	whether to use classifier free guidance or not
	negative_prompt (`str` or `List[str]`, optional):
	The prompt or prompts not to guide the image generation. If not defined, one has to pass
	`negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
	less than `1`).
	negative_prompt_2 (`str` or `List[str]`, optional):
	The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
	`text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders
	prompt_embeds (`torch.FloatTensor`, optional):
	Pre-generated text embeddings. Can be used to easily tweak text inputs, e.g. prompt weighting. If not
	provided, text embeddings will be generated from `prompt` input argument.
	negative_prompt_embeds (`torch.FloatTensor`, optional):
	Pre-generated negative text embeddings. Can be used to easily tweak text inputs, e.g. prompt
	weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
	argument.
	pooled_prompt_embeds (`torch.FloatTensor`, optional):
	Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, e.g. prompt weighting.
	If not provided, pooled text embeddings will be generated from `prompt` input argument.
	negative_pooled_prompt_embeds (`torch.FloatTensor`, optional):
	Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, e.g. prompt
	weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
	input argument.
	lora_scale (`float`, optional):
	A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.
	"""
	device = device or self._execution_device

	# set lora scale so that monkey patched LoRA
	# function of text encoder can correctly access it
	if lora_scale is not None and isinstance(self, StableDiffusionXLLoraLoaderMixin):
	self._lora_scale = lora_scale

	# dynamically adjust the LoRA scale
	if self.text_encoder is not None:
	if not USE_PEFT_BACKEND:
	adjust_lora_scale_text_encoder(self.text_encoder, lora_scale)
	else:
	scale_lora_layers(self.text_encoder, lora_scale)

	if self.text_encoder_2 is not None:
	if not USE_PEFT_BACKEND:
	adjust_lora_scale_text_encoder(self.text_encoder_2, lora_scale)
	else:
	scale_lora_layers(self.text_encoder_2, lora_scale)

	if prompt is not None and isinstance(prompt, str):
	batch_size = 1
	elif prompt is not None and isinstance(prompt, list):
	batch_size = len(prompt)
	else:
	batch_size = prompt_embeds.shape[0]

	# Define tokenizers and text encoders
	tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2]
	text_encoders = (
	[self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2]
	)

	if prompt_embeds is None:
	prompt_2 = prompt_2 or prompt
	# textual inversion: process multi-vector tokens if necessary
	prompt_embeds_list = []
	prompts = [prompt, prompt_2]
	for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders):
	if isinstance(self, TextualInversionLoaderMixin):
	prompt = self.maybe_convert_prompt(prompt, tokenizer)

	text_inputs = tokenizer(
	prompt,
	padding="max_length",
	max_length=tokenizer.model_max_length,
	truncation=True,
	return_tensors="pt",
	)

	text_input_ids = text_inputs.input_ids
	untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids

	if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(
	text_input_ids, untruncated_ids
	):
	removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1])
	logger.warning(
	"The following part of your input was truncated because CLIP can only handle sequences up to"
	f" {tokenizer.model_max_length} tokens: {removed_text}"
	)

	prompt_embeds = text_encoder(
	text_input_ids.to(device),
	output_hidden_states=True,
	)

	# We are only ALWAYS interested in the pooled output of the final text encoder
	pooled_prompt_embeds = prompt_embeds[0]
	prompt_embeds = prompt_embeds.hidden_states[-2]

	prompt_embeds_list.append(prompt_embeds)

	prompt_embeds = torch.concat(prompt_embeds_list, dim=-1)

	# get unconditional embeddings for classifier free guidance
	zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt
	if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt:
	negative_prompt_embeds = torch.zeros_like(prompt_embeds)
	negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds)
	elif do_classifier_free_guidance and negative_prompt_embeds is None:
	negative_prompt = negative_prompt or ""
	negative_prompt_2 = negative_prompt_2 or negative_prompt

	uncond_tokens: List[str]
	if prompt is not None and type(prompt) is not type(negative_prompt):
	raise TypeError(
	f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
	f" {type(prompt)}."
	)
	elif isinstance(negative_prompt, str):
	uncond_tokens = [negative_prompt, negative_prompt_2]
	elif batch_size != len(negative_prompt):
	raise ValueError(
	f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
	f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
	" the batch size of `prompt`."
	)
	else:
	uncond_tokens = [negative_prompt, negative_prompt_2]

	negative_prompt_embeds_list = []
	for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, text_encoders):
	if isinstance(self, TextualInversionLoaderMixin):
	negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer)

	max_length = prompt_embeds.shape[1]
	uncond_input = tokenizer(
	negative_prompt,
	padding="max_length",
	max_length=max_length,
	truncation=True,
	return_tensors="pt",
	)

	negative_prompt_embeds = text_encoder(
	uncond_input.input_ids.to(device),
	output_hidden_states=True,
	)
	# We are only ALWAYS interested in the pooled output of the final text encoder
	negative_pooled_prompt_embeds = negative_prompt_embeds[0]
	negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2]

	negative_prompt_embeds_list.append(negative_prompt_embeds)

	negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1)

	prompt_embeds_dtype = self.text_encoder_2.dtype if self.text_encoder_2 is not None else self.unet.dtype
	prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
	bs_embed, seq_len, _ = prompt_embeds.shape
	# duplicate text embeddings for each generation per prompt, using mps friendly method
	prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
	prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)

	if do_classifier_free_guidance:
	# duplicate unconditional embeddings for each generation per prompt, using mps friendly method
	seq_len = negative_prompt_embeds.shape[1]
	negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device)
	negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
	negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)

	pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(
	bs_embed * num_images_per_prompt, -1
	)
	if do_classifier_free_guidance:
	negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(
	bs_embed * num_images_per_prompt, -1
	)

	return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds

	# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs
	def prepare_extra_step_kwargs(self, generator, eta):
	# prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
	# eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
	# eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502
	# and should be between [0, 1]

	accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())
	extra_step_kwargs = {}
	if accepts_eta:
	extra_step_kwargs["eta"] = eta

	# check if the scheduler accepts generator
	accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())
	if accepts_generator:
	extra_step_kwargs["generator"] = generator
	return extra_step_kwargs

	# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_instruct_pix2pix.StableDiffusionInstructPix2PixPipeline.check_inputs
	def check_inputs(
	self,
	prompt,
	callback_steps,
	negative_prompt=None,
	prompt_embeds=None,
	negative_prompt_embeds=None,
	callback_on_step_end_tensor_inputs=None,
	):
	if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0):
	raise ValueError(
	f"`callback_steps` has to be a positive integer but is {callback_steps} of type"
	f" {type(callback_steps)}."
	)

	if callback_on_step_end_tensor_inputs is not None and not all(
	k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
	):
	raise ValueError(
	f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
	)

	if prompt is not None and prompt_embeds is not None:
	raise ValueError(
	f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
	" only forward one of the two."
	)
	elif prompt is None and prompt_embeds is None:
	raise ValueError(
	"Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
	)
	elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
	raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")

	if negative_prompt is not None and negative_prompt_embeds is not None:
	raise ValueError(
	f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"
	f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
	)

	if prompt_embeds is not None and negative_prompt_embeds is not None:
	if prompt_embeds.shape != negative_prompt_embeds.shape:
	raise ValueError(
	"`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"
	f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"
	f" {negative_prompt_embeds.shape}."
	)

	# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents
	def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None):
	shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor)
	if isinstance(generator, list) and len(generator) != batch_size:
	raise ValueError(
	f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
	f" size of {batch_size}. Make sure the batch size matches the length of the generators."
	)

	if latents is None:
	latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
	else:
	latents = latents.to(device)

	# scale the initial noise by the standard deviation required by the scheduler
	latents = latents * self.scheduler.init_noise_sigma
	return latents

	def prepare_image_latents(
	self, image, batch_size, num_images_per_prompt, dtype, device, do_classifier_free_guidance, generator=None
	):
	if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)):
	raise ValueError(
	f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}"
	)

	image = image.to(device=device, dtype=dtype)

	batch_size = batch_size * num_images_per_prompt

	if image.shape[1] == 4:
	image_latents = image
	else:
	# make sure the VAE is in float32 mode, as it overflows in float16
	needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast
	if needs_upcasting:
	self.upcast_vae()
	image = image.to(next(iter(self.vae.post_quant_conv.parameters())).dtype)

	image_latents = retrieve_latents(self.vae.encode(image), sample_mode="argmax")

	# cast back to fp16 if needed
	if needs_upcasting:
	self.vae.to(dtype=torch.float16)

	if batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] == 0:
	# expand image_latents for batch_size
	deprecation_message = (
	f"You have passed {batch_size} text prompts (`prompt`), but only {image_latents.shape[0]} initial"
	" images (`image`). Initial images are now duplicating to match the number of text prompts. Note"
	" that this behavior is deprecated and will be removed in a version 1.0.0. Please make sure to update"
	" your script to pass as many initial images as text prompts to suppress this warning."
	)
	deprecate("len(prompt) != len(image)", "1.0.0", deprecation_message, standard_warn=False)
	additional_image_per_prompt = batch_size // image_latents.shape[0]
	image_latents = torch.cat([image_latents] * additional_image_per_prompt, dim=0)
	elif batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] != 0:
	raise ValueError(
	f"Cannot duplicate `image` of batch size {image_latents.shape[0]} to {batch_size} text prompts."
	)
	else:
	image_latents = torch.cat([image_latents], dim=0)

	if do_classifier_free_guidance:
	uncond_image_latents = torch.zeros_like(image_latents)
	image_latents = torch.cat([image_latents, image_latents, uncond_image_latents], dim=0)

	if image_latents.dtype != self.vae.dtype:
	image_latents = image_latents.to(dtype=self.vae.dtype)

	return image_latents

	# Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline._get_add_time_ids
	def _get_add_time_ids(
	self, original_size, crops_coords_top_left, target_size, dtype, text_encoder_projection_dim=None
	):
	add_time_ids = list(original_size + crops_coords_top_left + target_size)

	passed_add_embed_dim = (
	self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim
	)
	expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features

	if expected_add_embed_dim != passed_add_embed_dim:
	raise ValueError(
	f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`."
	)

	add_time_ids = torch.tensor([add_time_ids], dtype=dtype)
	return add_time_ids

	# Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.upcast_vae
	def upcast_vae(self):
	dtype = self.vae.dtype
	self.vae.to(dtype=torch.float32)
	use_torch_2_0_or_xformers = isinstance(
	self.vae.decoder.mid_block.attentions[0].processor,
	(
	AttnProcessor2_0,
	XFormersAttnProcessor,
	LoRAXFormersAttnProcessor,
	LoRAAttnProcessor2_0,
	FusedAttnProcessor2_0,
	),
	)
	# if xformers or torch_2_0 is used attention block does not need
	# to be in float32 which can save lots of memory
	if use_torch_2_0_or_xformers:
	self.vae.post_quant_conv.to(dtype)
	self.vae.decoder.conv_in.to(dtype)
	self.vae.decoder.mid_block.to(dtype)

	@torch.no_grad()
	@replace_example_docstring(EXAMPLE_DOC_STRING)
	def __call__(
	self,
	prompt: Union[str, List[str]] = None,
	prompt_2: Optional[Union[str, List[str]]] = None,
	image: PipelineImageInput = None,
	height: Optional[int] = None,
	width: Optional[int] = None,
	num_inference_steps: int = 100,
	denoising_end: Optional[float] = None,
	guidance_scale: float = 5.0,
	image_guidance_scale: float = 1.5,
	negative_prompt: Optional[Union[str, List[str]]] = None,
	negative_prompt_2: Optional[Union[str, List[str]]] = None,
	num_images_per_prompt: Optional[int] = 1,
	eta: float = 0.0,
	generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
	latents: Optional[torch.FloatTensor] = None,
	prompt_embeds: Optional[torch.FloatTensor] = None,
	negative_prompt_embeds: Optional[torch.FloatTensor] = None,
	pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
	negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
	output_type: Optional[str] = "pil",
	return_dict: bool = True,
	callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
	callback_steps: int = 1,
	cross_attention_kwargs: Optional[Dict[str, Any]] = None,
	guidance_rescale: float = 0.0,
	original_size: Tuple[int, int] = None,
	crops_coords_top_left: Tuple[int, int] = (0, 0),
	target_size: Tuple[int, int] = None,
	):
	r"""
	Function invoked when calling the pipeline for generation.

	Args:
	prompt (`str` or `List[str]`, optional):
	The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
	instead.
	prompt_2 (`str` or `List[str]`, optional):
	The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
	used in both text-encoders
	image (`torch.FloatTensor` or `PIL.Image.Image` or `np.ndarray` or `List[torch.FloatTensor]` or `List[PIL.Image.Image]` or `List[np.ndarray]`):
	The image(s) to modify with the pipeline.
	height (`int`, optional, defaults to self.unet.config.sample_size * self.vae_scale_factor):
	The height in pixels of the generated image.
	width (`int`, optional, defaults to self.unet.config.sample_size * self.vae_scale_factor):
	The width in pixels of the generated image.
	num_inference_steps (`int`, optional, defaults to 50):
	The number of denoising steps. More denoising steps usually lead to a higher quality image at the
	expense of slower inference.
	denoising_end (`float`, optional):
	When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be
	completed before it is intentionally prematurely terminated. As a result, the returned sample will
	still retain a substantial amount of noise as determined by the discrete timesteps selected by the
	scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a
	"Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image
	Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output)
	guidance_scale (`float`, optional, defaults to 5.0):
	Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
	`guidance_scale` is defined as `w` of equation 2. of [Imagen
	Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
	1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
	usually at the expense of lower image quality.
	image_guidance_scale (`float`, optional, defaults to 1.5):
	Image guidance scale is to push the generated image towards the initial image `image`. Image guidance
	scale is enabled by setting `image_guidance_scale > 1`. Higher image guidance scale encourages to
	generate images that are closely linked to the source image `image`, usually at the expense of lower
	image quality. This pipeline requires a value of at least `1`.
	negative_prompt (`str` or `List[str]`, optional):
	The prompt or prompts not to guide the image generation. If not defined, one has to pass
	`negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
	less than `1`).
	negative_prompt_2 (`str` or `List[str]`, optional):
	The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
	`text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders.
	num_images_per_prompt (`int`, optional, defaults to 1):
	The number of images to generate per prompt.
	eta (`float`, optional, defaults to 0.0):
	Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
	[`schedulers.DDIMScheduler`], will be ignored for others.
	generator (`torch.Generator` or `List[torch.Generator]`, optional):
	One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
	to make generation deterministic.
	latents (`torch.FloatTensor`, optional):
	Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
	generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
	tensor will ge generated by sampling using the supplied random `generator`.
	prompt_embeds (`torch.FloatTensor`, optional):
	Pre-generated text embeddings. Can be used to easily tweak text inputs, e.g. prompt weighting. If not
	provided, text embeddings will be generated from `prompt` input argument.
	negative_prompt_embeds (`torch.FloatTensor`, optional):
	Pre-generated negative text embeddings. Can be used to easily tweak text inputs, e.g. prompt
	weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
	argument.
	pooled_prompt_embeds (`torch.FloatTensor`, optional):
	Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, e.g. prompt weighting.
	If not provided, pooled text embeddings will be generated from `prompt` input argument.
	negative_pooled_prompt_embeds (`torch.FloatTensor`, optional):
	Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, e.g. prompt
	weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
	input argument.
	output_type (`str`, optional, defaults to `"pil"`):
	The output format of the generate image. Choose between
	[PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
	return_dict (`bool`, optional, defaults to `True`):
	Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionXLPipelineOutput`] instead of a
	plain tuple.
	callback (`Callable`, optional):
	A function that will be called every `callback_steps` steps during inference. The function will be
	called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
	callback_steps (`int`, optional, defaults to 1):
	The frequency at which the `callback` function will be called. If not specified, the callback will be
	called at every step.
	cross_attention_kwargs (`dict`, optional):
	A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
	`self.processor` in
	[diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
	guidance_rescale (`float`, optional, defaults to 0.0):
	Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are
	Flawed](https://arxiv.org/pdf/2305.08891.pdf) `guidance_scale` is defined as `φ` in equation 16. of
	[Common Diffusion Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf).
	Guidance rescale factor should fix overexposure when using zero terminal SNR.
	original_size (`Tuple[int]`, optional, defaults to (1024, 1024)):
	If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled.
	`original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as
	explained in section 2.2 of
	[https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
	crops_coords_top_left (`Tuple[int]`, optional, defaults to (0, 0)):
	`crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position
	`crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting
	`crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of
	[https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
	target_size (`Tuple[int]`, optional, defaults to (1024, 1024)):
	For most cases, `target_size` should be set to the desired height and width of the generated image. If
	not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in
	section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
	aesthetic_score (`float`, optional, defaults to 6.0):
	Used to simulate an aesthetic score of the generated image by influencing the positive text condition.
	Part of SDXL's micro-conditioning as explained in section 2.2 of
	[https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
	negative_aesthetic_score (`float`, optional, defaults to 2.5):
	Part of SDXL's micro-conditioning as explained in section 2.2 of
	[https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). Can be used to
	simulate an aesthetic score of the generated image by influencing the negative text condition.

	Examples:

	Returns:
	[`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] or `tuple`:
	[`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a
	`tuple`. When returning a tuple, the first element is a list with the generated images.
	"""
	# 0. Default height and width to unet
	height = height or self.default_sample_size * self.vae_scale_factor
	width = width or self.default_sample_size * self.vae_scale_factor

	original_size = original_size or (height, width)
	target_size = target_size or (height, width)

	# 1. Check inputs. Raise error if not correct
	self.check_inputs(prompt, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds)

	if image is None:
	raise ValueError("`image` input cannot be undefined.")

	# 2. Define call parameters
	if prompt is not None and isinstance(prompt, str):
	batch_size = 1
	elif prompt is not None and isinstance(prompt, list):
	batch_size = len(prompt)
	else:
	batch_size = prompt_embeds.shape[0]

	device = self._execution_device

	# here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
	# of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
	# corresponds to doing no classifier free guidance.
	do_classifier_free_guidance = guidance_scale > 1.0 and image_guidance_scale >= 1.0

	# 3. Encode input prompt
	text_encoder_lora_scale = (
	cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None
	)
	(
	prompt_embeds,
	negative_prompt_embeds,
	pooled_prompt_embeds,
	negative_pooled_prompt_embeds,
	) = self.encode_prompt(
	prompt=prompt,
	prompt_2=prompt_2,
	device=device,
	num_images_per_prompt=num_images_per_prompt,
	do_classifier_free_guidance=do_classifier_free_guidance,
	negative_prompt=negative_prompt,
	negative_prompt_2=negative_prompt_2,
	prompt_embeds=prompt_embeds,
	negative_prompt_embeds=negative_prompt_embeds,
	pooled_prompt_embeds=pooled_prompt_embeds,
	negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
	lora_scale=text_encoder_lora_scale,
	)

	# 4. Preprocess image
	image = self.image_processor.preprocess(image, height=height, width=width).to(device)

	# 5. Prepare timesteps
	self.scheduler.set_timesteps(num_inference_steps, device=device)
	timesteps = self.scheduler.timesteps

	# 6. Prepare Image latents
	image_latents = self.prepare_image_latents(
	image,
	batch_size,
	num_images_per_prompt,
	prompt_embeds.dtype,
	device,
	do_classifier_free_guidance,
	)

	image_latents = image_latents * self.vae.config.scaling_factor

	# 7. Prepare latent variables
	num_channels_latents = self.vae.config.latent_channels
	latents = self.prepare_latents(
	batch_size * num_images_per_prompt,
	num_channels_latents,
	height,
	width,
	prompt_embeds.dtype,
	device,
	generator,
	latents,
	)

	# 8. Check that shapes of latents and image match the UNet channels
	num_channels_image = image_latents.shape[1]
	if num_channels_latents + num_channels_image != self.unet.config.in_channels:
	raise ValueError(
	f"Incorrect configuration settings! The config of `pipeline.unet`: {self.unet.config} expects"
	f" {self.unet.config.in_channels} but received `num_channels_latents`: {num_channels_latents} +"
	f" `num_channels_image`: {num_channels_image} "
	f" = {num_channels_latents + num_channels_image}. Please verify the config of"
	" `pipeline.unet` or your `image` input."
	)

	# 9. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
	extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)

	# 10. Prepare added time ids & embeddings
	add_text_embeds = pooled_prompt_embeds
	if self.text_encoder_2 is None:
	text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1])
	else:
	text_encoder_projection_dim = self.text_encoder_2.config.projection_dim

	add_time_ids = self._get_add_time_ids(
	original_size,
	crops_coords_top_left,
	target_size,
	dtype=prompt_embeds.dtype,
	text_encoder_projection_dim=text_encoder_projection_dim,
	)

	if do_classifier_free_guidance:
	# The extra concat similar to how it's done in SD InstructPix2Pix.
	prompt_embeds = torch.cat([prompt_embeds, negative_prompt_embeds, negative_prompt_embeds], dim=0)
	add_text_embeds = torch.cat(
	[add_text_embeds, negative_pooled_prompt_embeds, negative_pooled_prompt_embeds], dim=0
	)
	add_time_ids = torch.cat([add_time_ids, add_time_ids, add_time_ids], dim=0)

	prompt_embeds = prompt_embeds.to(device)
	add_text_embeds = add_text_embeds.to(device)
	add_time_ids = add_time_ids.to(device).repeat(batch_size * num_images_per_prompt, 1)

	# 11. Denoising loop
	num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
	if denoising_end is not None and isinstance(denoising_end, float) and denoising_end > 0 and denoising_end < 1:
	discrete_timestep_cutoff = int(
	round(
	self.scheduler.config.num_train_timesteps
	- (denoising_end * self.scheduler.config.num_train_timesteps)
	)
	)
	num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps)))
	timesteps = timesteps[:num_inference_steps]

	with self.progress_bar(total=num_inference_steps) as progress_bar:
	for i, t in enumerate(timesteps):
	# Expand the latents if we are doing classifier free guidance.
	# The latents are expanded 3 times because for pix2pix the guidance
	# is applied for both the text and the input image.
	latent_model_input = torch.cat([latents] * 3) if do_classifier_free_guidance else latents

	# concat latents, image_latents in the channel dimension
	scaled_latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
	scaled_latent_model_input = torch.cat([scaled_latent_model_input, image_latents], dim=1)

	# predict the noise residual
	added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids}
	noise_pred = self.unet(
	scaled_latent_model_input,
	t,
	encoder_hidden_states=prompt_embeds,
	cross_attention_kwargs=cross_attention_kwargs,
	added_cond_kwargs=added_cond_kwargs,
	return_dict=False,
	)[0]

	# perform guidance
	if do_classifier_free_guidance:
	noise_pred_text, noise_pred_image, noise_pred_uncond = noise_pred.chunk(3)
	noise_pred = (
	noise_pred_uncond
	+ guidance_scale * (noise_pred_text - noise_pred_image)
	+ image_guidance_scale * (noise_pred_image - noise_pred_uncond)
	)

	if do_classifier_free_guidance and guidance_rescale > 0.0:
	# Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf
	noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=guidance_rescale)

	# compute the previous noisy sample x_t -> x_t-1
	latents_dtype = latents.dtype
	latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
	if latents.dtype != latents_dtype:
	if torch.backends.mps.is_available():
	# some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
	latents = latents.to(latents_dtype)

	# call the callback, if provided
	if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
	progress_bar.update()
	if callback is not None and i % callback_steps == 0:
	step_idx = i // getattr(self.scheduler, "order", 1)
	callback(step_idx, t, latents)

	if XLA_AVAILABLE:
	xm.mark_step()

	if not output_type == "latent":
	# make sure the VAE is in float32 mode, as it overflows in float16
	needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast

	if needs_upcasting:
	self.upcast_vae()
	latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype)
	elif latents.dtype != self.vae.dtype:
	if torch.backends.mps.is_available():
	# some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
	self.vae = self.vae.to(latents.dtype)

	# unscale/denormalize the latents
	# denormalize with the mean and std if available and not None
	has_latents_mean = hasattr(self.vae.config, "latents_mean") and self.vae.config.latents_mean is not None
	has_latents_std = hasattr(self.vae.config, "latents_std") and self.vae.config.latents_std is not None
	if has_latents_mean and has_latents_std:
	latents_mean = (
	torch.tensor(self.vae.config.latents_mean).view(1, 4, 1, 1).to(latents.device, latents.dtype)
	)
	latents_std = (
	torch.tensor(self.vae.config.latents_std).view(1, 4, 1, 1).to(latents.device, latents.dtype)
	)
	latents = latents * latents_std / self.vae.config.scaling_factor + latents_mean
	else:
	latents = latents / self.vae.config.scaling_factor

	image = self.vae.decode(latents, return_dict=False)[0]

	# cast back to fp16 if needed
	if needs_upcasting:
	self.vae.to(dtype=torch.float16)
	else:
	return StableDiffusionXLPipelineOutput(images=latents)

	# apply watermark if available
	if self.watermark is not None:
	image = self.watermark.apply_watermark(image)

	image = self.image_processor.postprocess(image, output_type=output_type)

	# Offload all models
	self.maybe_free_model_hooks()

	if not return_dict:
	return (image,)

	return StableDiffusionXLPipelineOutput(images=image)