Update pipeline.py (#1)

5c21c5c verified about 2 months ago

No virus

87.3 kB

	# Implementation of StableDiffusionXLPAGPipeline

	import inspect
	from typing import Any, Callable, Dict, List, Optional, Tuple, Union

	import torch
	import torch.nn.functional as F
	from packaging import version

	from transformers import (
	CLIPImageProcessor,
	CLIPTextModel,
	CLIPTextModelWithProjection,
	CLIPTokenizer,
	CLIPVisionModelWithProjection,
	)

	from diffusers.image_processor import PipelineImageInput, VaeImageProcessor
	from diffusers.loaders import (
	FromSingleFileMixin,
	IPAdapterMixin,
	StableDiffusionXLLoraLoaderMixin,
	TextualInversionLoaderMixin,
	)
	from diffusers.models import AutoencoderKL, ImageProjection, UNet2DConditionModel
	from diffusers.models.attention_processor import (
	AttnProcessor2_0,
	FusedAttnProcessor2_0,
	LoRAAttnProcessor2_0,
	LoRAXFormersAttnProcessor,
	XFormersAttnProcessor,
	)
	from diffusers.models.lora import adjust_lora_scale_text_encoder
	from diffusers.schedulers import KarrasDiffusionSchedulers
	from diffusers.utils import (
	USE_PEFT_BACKEND,
	deprecate,
	is_invisible_watermark_available,
	is_torch_xla_available,
	logging,
	replace_example_docstring,
	scale_lora_layers,
	unscale_lora_layers,
	)
	from diffusers.utils.torch_utils import randn_tensor
	from diffusers.pipelines.pipeline_utils import DiffusionPipeline, StableDiffusionMixin
	from diffusers.pipelines.stable_diffusion_xl.pipeline_output import StableDiffusionXLPipelineOutput

	from diffusers.models.attention_processor import Attention, AttnProcessor2_0

	logger = logging.get_logger(__name__) # pylint: disable=invalid-name

	EXAMPLE_DOC_STRING = """
	Examples:
	```py
	>>> import torch
	>>> from diffusers import StableDiffusionXLPipeline

	>>> pipe = StableDiffusionXLPipeline.from_pretrained(
	... "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16
	... )
	>>> pipe = pipe.to("cuda")

	>>> prompt = "a photo of an astronaut riding a horse on mars"
	>>> image = pipe(prompt).images[0]
	```
	"""


	class PAGIdentitySelfAttnProcessor:
	r"""
	Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0).
	"""

	def __init__(self):
	if not hasattr(F, "scaled_dot_product_attention"):
	raise ImportError("AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.")

	def __call__(
	self,
	attn: Attention,
	hidden_states: torch.FloatTensor,
	encoder_hidden_states: Optional[torch.FloatTensor] = None,
	attention_mask: Optional[torch.FloatTensor] = None,
	temb: Optional[torch.FloatTensor] = None,
	*args,
	**kwargs,
	) -> torch.FloatTensor:
	if len(args) > 0 or kwargs.get("scale", None) is not None:
	deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`."
	deprecate("scale", "1.0.0", deprecation_message)

	residual = hidden_states
	if attn.spatial_norm is not None:
	hidden_states = attn.spatial_norm(hidden_states, temb)

	input_ndim = hidden_states.ndim
	if input_ndim == 4:
	batch_size, channel, height, width = hidden_states.shape
	hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)

	# chunk
	hidden_states_org, hidden_states_ptb = hidden_states.chunk(2)

	# original path
	batch_size, sequence_length, _ = hidden_states_org.shape

	if attention_mask is not None:
	attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
	# scaled_dot_product_attention expects attention_mask shape to be
	# (batch, heads, source_length, target_length)
	attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1])

	if attn.group_norm is not None:
	hidden_states_org = attn.group_norm(hidden_states_org.transpose(1, 2)).transpose(1, 2)

	query = attn.to_q(hidden_states_org)
	key = attn.to_k(hidden_states_org)
	value = attn.to_v(hidden_states_org)

	inner_dim = key.shape[-1]
	head_dim = inner_dim // attn.heads

	query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)

	key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
	value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)

	# the output of sdp = (batch, num_heads, seq_len, head_dim)
	# TODO: add support for attn.scale when we move to Torch 2.1
	hidden_states_org = F.scaled_dot_product_attention(
	query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
	)

	hidden_states_org = hidden_states_org.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
	hidden_states_org = hidden_states_org.to(query.dtype)

	# linear proj
	hidden_states_org = attn.to_out[0](hidden_states_org)
	# dropout
	hidden_states_org = attn.to_out[1](hidden_states_org)

	if input_ndim == 4:
	hidden_states_org = hidden_states_org.transpose(-1, -2).reshape(batch_size, channel, height, width)

	# perturbed path (identity attention)
	batch_size, sequence_length, _ = hidden_states_ptb.shape

	if attention_mask is not None:
	attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
	# scaled_dot_product_attention expects attention_mask shape to be
	# (batch, heads, source_length, target_length)
	attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1])

	if attn.group_norm is not None:
	hidden_states_ptb = attn.group_norm(hidden_states_ptb.transpose(1, 2)).transpose(1, 2)

	value = attn.to_v(hidden_states_ptb)

	# hidden_states_ptb = torch.zeros(value.shape).to(value.get_device())
	hidden_states_ptb = value

	hidden_states_ptb = hidden_states_ptb.to(query.dtype)

	# linear proj
	hidden_states_ptb = attn.to_out[0](hidden_states_ptb)
	# dropout
	hidden_states_ptb = attn.to_out[1](hidden_states_ptb)

	if input_ndim == 4:
	hidden_states_ptb = hidden_states_ptb.transpose(-1, -2).reshape(batch_size, channel, height, width)

	# cat
	hidden_states = torch.cat([hidden_states_org, hidden_states_ptb])

	if attn.residual_connection:
	hidden_states = hidden_states + residual

	hidden_states = hidden_states / attn.rescale_output_factor

	return hidden_states


	class PAGCFGIdentitySelfAttnProcessor:
	r"""
	Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0).
	"""

	def __init__(self):
	if not hasattr(F, "scaled_dot_product_attention"):
	raise ImportError("AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.")

	def __call__(
	self,
	attn: Attention,
	hidden_states: torch.FloatTensor,
	encoder_hidden_states: Optional[torch.FloatTensor] = None,
	attention_mask: Optional[torch.FloatTensor] = None,
	temb: Optional[torch.FloatTensor] = None,
	*args,
	**kwargs,
	) -> torch.FloatTensor:
	if len(args) > 0 or kwargs.get("scale", None) is not None:
	deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`."
	deprecate("scale", "1.0.0", deprecation_message)

	residual = hidden_states
	if attn.spatial_norm is not None:
	hidden_states = attn.spatial_norm(hidden_states, temb)

	input_ndim = hidden_states.ndim
	if input_ndim == 4:
	batch_size, channel, height, width = hidden_states.shape
	hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2)

	# chunk
	hidden_states_uncond, hidden_states_org, hidden_states_ptb = hidden_states.chunk(3)
	hidden_states_org = torch.cat([hidden_states_uncond, hidden_states_org])

	# original path
	batch_size, sequence_length, _ = hidden_states_org.shape

	if attention_mask is not None:
	attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
	# scaled_dot_product_attention expects attention_mask shape to be
	# (batch, heads, source_length, target_length)
	attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1])

	if attn.group_norm is not None:
	hidden_states_org = attn.group_norm(hidden_states_org.transpose(1, 2)).transpose(1, 2)

	query = attn.to_q(hidden_states_org)
	key = attn.to_k(hidden_states_org)
	value = attn.to_v(hidden_states_org)

	inner_dim = key.shape[-1]
	head_dim = inner_dim // attn.heads

	query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)

	key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)
	value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2)

	# the output of sdp = (batch, num_heads, seq_len, head_dim)
	# TODO: add support for attn.scale when we move to Torch 2.1
	hidden_states_org = F.scaled_dot_product_attention(
	query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
	)

	hidden_states_org = hidden_states_org.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)
	hidden_states_org = hidden_states_org.to(query.dtype)

	# linear proj
	hidden_states_org = attn.to_out[0](hidden_states_org)
	# dropout
	hidden_states_org = attn.to_out[1](hidden_states_org)

	if input_ndim == 4:
	hidden_states_org = hidden_states_org.transpose(-1, -2).reshape(batch_size, channel, height, width)

	# perturbed path (identity attention)
	batch_size, sequence_length, _ = hidden_states_ptb.shape

	if attention_mask is not None:
	attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size)
	# scaled_dot_product_attention expects attention_mask shape to be
	# (batch, heads, source_length, target_length)
	attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1])

	if attn.group_norm is not None:
	hidden_states_ptb = attn.group_norm(hidden_states_ptb.transpose(1, 2)).transpose(1, 2)

	value = attn.to_v(hidden_states_ptb)
	hidden_states_ptb = value
	hidden_states_ptb = hidden_states_ptb.to(query.dtype)

	# linear proj
	hidden_states_ptb = attn.to_out[0](hidden_states_ptb)
	# dropout
	hidden_states_ptb = attn.to_out[1](hidden_states_ptb)

	if input_ndim == 4:
	hidden_states_ptb = hidden_states_ptb.transpose(-1, -2).reshape(batch_size, channel, height, width)

	# cat
	hidden_states = torch.cat([hidden_states_org, hidden_states_ptb])

	if attn.residual_connection:
	hidden_states = hidden_states + residual

	hidden_states = hidden_states / attn.rescale_output_factor

	return hidden_states

	if is_invisible_watermark_available():
	from diffusers.pipelines.stable_diffusion_xl.watermark import StableDiffusionXLWatermarker

	if is_torch_xla_available():
	import torch_xla.core.xla_model as xm

	XLA_AVAILABLE = True
	else:
	XLA_AVAILABLE = False

	logger = logging.get_logger(__name__) # pylint: disable=invalid-name

	def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0):
	"""
	Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and
	Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4
	"""
	std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True)
	std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True)
	# rescale the results from guidance (fixes overexposure)
	noise_pred_rescaled = noise_cfg * (std_text / std_cfg)
	# mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images
	noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg
	return noise_cfg


	def retrieve_timesteps(
	scheduler,
	num_inference_steps: Optional[int] = None,
	device: Optional[Union[str, torch.device]] = None,
	timesteps: Optional[List[int]] = None,
	**kwargs,
	):
	"""
	Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
	custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.

	Args:
	scheduler (`SchedulerMixin`):
	The scheduler to get timesteps from.
	num_inference_steps (`int`):
	The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
	must be `None`.
	device (`str` or `torch.device`, optional):
	The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
	timesteps (`List[int]`, optional):
	Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default
	timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps`
	must be `None`.

	Returns:
	`Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
	second element is the number of inference steps.
	"""
	if timesteps is not None:
	accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
	if not accepts_timesteps:
	raise ValueError(
	f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
	f" timestep schedules. Please check whether you are using the correct scheduler."
	)
	scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
	timesteps = scheduler.timesteps
	num_inference_steps = len(timesteps)
	else:
	scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
	timesteps = scheduler.timesteps
	return timesteps, num_inference_steps

	class StableDiffusionXLPipeline(
	DiffusionPipeline,
	StableDiffusionMixin,
	FromSingleFileMixin,
	StableDiffusionXLLoraLoaderMixin,
	TextualInversionLoaderMixin,
	IPAdapterMixin,
	):
	r"""
	Pipeline for text-to-image generation using Stable Diffusion XL.

	This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the
	library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)

	The pipeline also inherits the following loading methods:
	- [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings
	- [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files
	- [`~loaders.StableDiffusionXLLoraLoaderMixin.load_lora_weights`] for loading LoRA weights
	- [`~loaders.StableDiffusionXLLoraLoaderMixin.save_lora_weights`] for saving LoRA weights
	- [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters

	Args:
	vae ([`AutoencoderKL`]):
	Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
	text_encoder ([`CLIPTextModel`]):
	Frozen text-encoder. Stable Diffusion XL uses the text portion of
	[CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically
	the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.
	text_encoder_2 ([` CLIPTextModelWithProjection`]):
	Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of
	[CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection),
	specifically the
	[laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k)
	variant.
	tokenizer (`CLIPTokenizer`):
	Tokenizer of class
	[CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
	tokenizer_2 (`CLIPTokenizer`):
	Second Tokenizer of class
	[CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
	unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.
	scheduler ([`SchedulerMixin`]):
	A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of
	[`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].
	force_zeros_for_empty_prompt (`bool`, optional, defaults to `"True"`):
	Whether the negative prompt embeddings shall be forced to always be set to 0. Also see the config of
	`stabilityai/stable-diffusion-xl-base-1-0`.
	add_watermarker (`bool`, optional):
	Whether to use the [invisible_watermark library](https://github.com/ShieldMnt/invisible-watermark/) to
	watermark output images. If not defined, it will default to True if the package is installed, otherwise no
	watermarker will be used.
	"""

	model_cpu_offload_seq = "text_encoder->text_encoder_2->image_encoder->unet->vae"
	_optional_components = [
	"tokenizer",
	"tokenizer_2",
	"text_encoder",
	"text_encoder_2",
	"image_encoder",
	"feature_extractor",
	]
	_callback_tensor_inputs = [
	"latents",
	"prompt_embeds",
	"negative_prompt_embeds",
	"add_text_embeds",
	"add_time_ids",
	"negative_pooled_prompt_embeds",
	"negative_add_time_ids",
	]

	def __init__(
	self,
	vae: AutoencoderKL,
	text_encoder: CLIPTextModel,
	text_encoder_2: CLIPTextModelWithProjection,
	tokenizer: CLIPTokenizer,
	tokenizer_2: CLIPTokenizer,
	unet: UNet2DConditionModel,
	scheduler: KarrasDiffusionSchedulers,
	image_encoder: CLIPVisionModelWithProjection = None,
	feature_extractor: CLIPImageProcessor = None,
	force_zeros_for_empty_prompt: bool = True,
	add_watermarker: Optional[bool] = None,
	):
	super().__init__()

	self.register_modules(
	vae=vae,
	text_encoder=text_encoder,
	text_encoder_2=text_encoder_2,
	tokenizer=tokenizer,
	tokenizer_2=tokenizer_2,
	unet=unet,
	scheduler=scheduler,
	image_encoder=image_encoder,
	feature_extractor=feature_extractor,
	)
	self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt)
	self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
	self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)

	self.default_sample_size = self.unet.config.sample_size

	add_watermarker = add_watermarker if add_watermarker is not None else is_invisible_watermark_available()

	if add_watermarker:
	self.watermark = StableDiffusionXLWatermarker()
	else:
	self.watermark = None

	def encode_prompt(
	self,
	prompt: str,
	prompt_2: Optional[str] = None,
	device: Optional[torch.device] = None,
	num_images_per_prompt: int = 1,
	do_classifier_free_guidance: bool = True,
	negative_prompt: Optional[str] = None,
	negative_prompt_2: Optional[str] = None,
	prompt_embeds: Optional[torch.FloatTensor] = None,
	negative_prompt_embeds: Optional[torch.FloatTensor] = None,
	pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
	negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
	lora_scale: Optional[float] = None,
	clip_skip: Optional[int] = None,
	):
	r"""
	Encodes the prompt into text encoder hidden states.

	Args:
	prompt (`str` or `List[str]`, optional):
	prompt to be encoded
	prompt_2 (`str` or `List[str]`, optional):
	The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
	used in both text-encoders
	device: (`torch.device`):
	torch device
	num_images_per_prompt (`int`):
	number of images that should be generated per prompt
	do_classifier_free_guidance (`bool`):
	whether to use classifier free guidance or not
	negative_prompt (`str` or `List[str]`, optional):
	The prompt or prompts not to guide the image generation. If not defined, one has to pass
	`negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
	less than `1`).
	negative_prompt_2 (`str` or `List[str]`, optional):
	The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
	`text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders
	prompt_embeds (`torch.FloatTensor`, optional):
	Pre-generated text embeddings. Can be used to easily tweak text inputs, e.g. prompt weighting. If not
	provided, text embeddings will be generated from `prompt` input argument.
	negative_prompt_embeds (`torch.FloatTensor`, optional):
	Pre-generated negative text embeddings. Can be used to easily tweak text inputs, e.g. prompt
	weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
	argument.
	pooled_prompt_embeds (`torch.FloatTensor`, optional):
	Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, e.g. prompt weighting.
	If not provided, pooled text embeddings will be generated from `prompt` input argument.
	negative_pooled_prompt_embeds (`torch.FloatTensor`, optional):
	Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, e.g. prompt
	weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
	input argument.
	lora_scale (`float`, optional):
	A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.
	clip_skip (`int`, optional):
	Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
	the output of the pre-final layer will be used for computing the prompt embeddings.
	"""
	device = device or self._execution_device

	# set lora scale so that monkey patched LoRA
	# function of text encoder can correctly access it
	if lora_scale is not None and isinstance(self, StableDiffusionXLLoraLoaderMixin):
	self._lora_scale = lora_scale

	# dynamically adjust the LoRA scale
	if self.text_encoder is not None:
	if not USE_PEFT_BACKEND:
	adjust_lora_scale_text_encoder(self.text_encoder, lora_scale)
	else:
	scale_lora_layers(self.text_encoder, lora_scale)

	if self.text_encoder_2 is not None:
	if not USE_PEFT_BACKEND:
	adjust_lora_scale_text_encoder(self.text_encoder_2, lora_scale)
	else:
	scale_lora_layers(self.text_encoder_2, lora_scale)

	prompt = [prompt] if isinstance(prompt, str) else prompt

	if prompt is not None:
	batch_size = len(prompt)
	else:
	batch_size = prompt_embeds.shape[0]

	# Define tokenizers and text encoders
	tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2]
	text_encoders = (
	[self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2]
	)

	if prompt_embeds is None:
	prompt_2 = prompt_2 or prompt
	prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2

	# textual inversion: process multi-vector tokens if necessary
	prompt_embeds_list = []
	prompts = [prompt, prompt_2]
	for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders):
	if isinstance(self, TextualInversionLoaderMixin):
	prompt = self.maybe_convert_prompt(prompt, tokenizer)

	text_inputs = tokenizer(
	prompt,
	padding="max_length",
	max_length=tokenizer.model_max_length,
	truncation=True,
	return_tensors="pt",
	)

	text_input_ids = text_inputs.input_ids
	untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids

	if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(
	text_input_ids, untruncated_ids
	):
	removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1])
	logger.warning(
	"The following part of your input was truncated because CLIP can only handle sequences up to"
	f" {tokenizer.model_max_length} tokens: {removed_text}"
	)

	prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True)

	# We are only ALWAYS interested in the pooled output of the final text encoder
	pooled_prompt_embeds = prompt_embeds[0]
	if clip_skip is None:
	prompt_embeds = prompt_embeds.hidden_states[-2]
	else:
	# "2" because SDXL always indexes from the penultimate layer.
	prompt_embeds = prompt_embeds.hidden_states[-(clip_skip + 2)]

	prompt_embeds_list.append(prompt_embeds)

	prompt_embeds = torch.concat(prompt_embeds_list, dim=-1)

	# get unconditional embeddings for classifier free guidance
	zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt
	if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt:
	negative_prompt_embeds = torch.zeros_like(prompt_embeds)
	negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds)
	elif do_classifier_free_guidance and negative_prompt_embeds is None:
	negative_prompt = negative_prompt or ""
	negative_prompt_2 = negative_prompt_2 or negative_prompt

	# normalize str to list
	negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
	negative_prompt_2 = (
	batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2
	)

	uncond_tokens: List[str]
	if prompt is not None and type(prompt) is not type(negative_prompt):
	raise TypeError(
	f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
	f" {type(prompt)}."
	)
	elif batch_size != len(negative_prompt):
	raise ValueError(
	f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
	f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
	" the batch size of `prompt`."
	)
	else:
	uncond_tokens = [negative_prompt, negative_prompt_2]

	negative_prompt_embeds_list = []
	for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, text_encoders):
	if isinstance(self, TextualInversionLoaderMixin):
	negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer)

	max_length = prompt_embeds.shape[1]
	uncond_input = tokenizer(
	negative_prompt,
	padding="max_length",
	max_length=max_length,
	truncation=True,
	return_tensors="pt",
	)

	negative_prompt_embeds = text_encoder(
	uncond_input.input_ids.to(device),
	output_hidden_states=True,
	)
	# We are only ALWAYS interested in the pooled output of the final text encoder
	negative_pooled_prompt_embeds = negative_prompt_embeds[0]
	negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2]

	negative_prompt_embeds_list.append(negative_prompt_embeds)

	negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1)

	if self.text_encoder_2 is not None:
	prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device)
	else:
	prompt_embeds = prompt_embeds.to(dtype=self.unet.dtype, device=device)

	bs_embed, seq_len, _ = prompt_embeds.shape
	# duplicate text embeddings for each generation per prompt, using mps friendly method
	prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
	prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)

	if do_classifier_free_guidance:
	# duplicate unconditional embeddings for each generation per prompt, using mps friendly method
	seq_len = negative_prompt_embeds.shape[1]

	if self.text_encoder_2 is not None:
	negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device)
	else:
	negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.unet.dtype, device=device)

	negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
	negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)

	pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(
	bs_embed * num_images_per_prompt, -1
	)
	if do_classifier_free_guidance:
	negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view(
	bs_embed * num_images_per_prompt, -1
	)

	if self.text_encoder is not None:
	if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND:
	# Retrieve the original scale by scaling back the LoRA layers
	unscale_lora_layers(self.text_encoder, lora_scale)

	if self.text_encoder_2 is not None:
	if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND:
	# Retrieve the original scale by scaling back the LoRA layers
	unscale_lora_layers(self.text_encoder_2, lora_scale)

	return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds

	# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image
	def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None):
	dtype = next(self.image_encoder.parameters()).dtype

	if not isinstance(image, torch.Tensor):
	image = self.feature_extractor(image, return_tensors="pt").pixel_values

	image = image.to(device=device, dtype=dtype)
	if output_hidden_states:
	image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2]
	image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0)
	uncond_image_enc_hidden_states = self.image_encoder(
	torch.zeros_like(image), output_hidden_states=True
	).hidden_states[-2]
	uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave(
	num_images_per_prompt, dim=0
	)
	return image_enc_hidden_states, uncond_image_enc_hidden_states
	else:
	image_embeds = self.image_encoder(image).image_embeds
	image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0)
	uncond_image_embeds = torch.zeros_like(image_embeds)

	return image_embeds, uncond_image_embeds

	# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds
	def prepare_ip_adapter_image_embeds(
	self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance
	):
	if ip_adapter_image_embeds is None:
	if not isinstance(ip_adapter_image, list):
	ip_adapter_image = [ip_adapter_image]

	if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers):
	raise ValueError(
	f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters."
	)

	image_embeds = []
	for single_ip_adapter_image, image_proj_layer in zip(
	ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers
	):
	output_hidden_state = not isinstance(image_proj_layer, ImageProjection)
	single_image_embeds, single_negative_image_embeds = self.encode_image(
	single_ip_adapter_image, device, 1, output_hidden_state
	)
	single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0)
	single_negative_image_embeds = torch.stack(
	[single_negative_image_embeds] * num_images_per_prompt, dim=0
	)

	if do_classifier_free_guidance:
	single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds])
	single_image_embeds = single_image_embeds.to(device)

	image_embeds.append(single_image_embeds)
	else:
	repeat_dims = [1]
	image_embeds = []
	for single_image_embeds in ip_adapter_image_embeds:
	if do_classifier_free_guidance:
	single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2)
	single_image_embeds = single_image_embeds.repeat(
	num_images_per_prompt, (repeat_dims len(single_image_embeds.shape[1:]))
	)
	single_negative_image_embeds = single_negative_image_embeds.repeat(
	num_images_per_prompt, (repeat_dims len(single_negative_image_embeds.shape[1:]))
	)
	single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds])
	else:
	single_image_embeds = single_image_embeds.repeat(
	num_images_per_prompt, (repeat_dims len(single_image_embeds.shape[1:]))
	)
	image_embeds.append(single_image_embeds)

	return image_embeds

	# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs
	def prepare_extra_step_kwargs(self, generator, eta):
	# prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
	# eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
	# eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502
	# and should be between [0, 1]

	accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())
	extra_step_kwargs = {}
	if accepts_eta:
	extra_step_kwargs["eta"] = eta

	# check if the scheduler accepts generator
	accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())
	if accepts_generator:
	extra_step_kwargs["generator"] = generator
	return extra_step_kwargs

	def check_inputs(
	self,
	prompt,
	prompt_2,
	height,
	width,
	callback_steps,
	negative_prompt=None,
	negative_prompt_2=None,
	prompt_embeds=None,
	negative_prompt_embeds=None,
	pooled_prompt_embeds=None,
	negative_pooled_prompt_embeds=None,
	ip_adapter_image=None,
	ip_adapter_image_embeds=None,
	callback_on_step_end_tensor_inputs=None,
	):
	if height % 8 != 0 or width % 8 != 0:
	raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")

	if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0):
	raise ValueError(
	f"`callback_steps` has to be a positive integer but is {callback_steps} of type"
	f" {type(callback_steps)}."
	)

	if callback_on_step_end_tensor_inputs is not None and not all(
	k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
	):
	raise ValueError(
	f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
	)

	if prompt is not None and prompt_embeds is not None:
	raise ValueError(
	f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
	" only forward one of the two."
	)
	elif prompt_2 is not None and prompt_embeds is not None:
	raise ValueError(
	f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
	" only forward one of the two."
	)
	elif prompt is None and prompt_embeds is None:
	raise ValueError(
	"Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
	)
	elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
	raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
	elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)):
	raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}")

	if negative_prompt is not None and negative_prompt_embeds is not None:
	raise ValueError(
	f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"
	f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
	)
	elif negative_prompt_2 is not None and negative_prompt_embeds is not None:
	raise ValueError(
	f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:"
	f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
	)

	if prompt_embeds is not None and negative_prompt_embeds is not None:
	if prompt_embeds.shape != negative_prompt_embeds.shape:
	raise ValueError(
	"`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"
	f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"
	f" {negative_prompt_embeds.shape}."
	)

	if prompt_embeds is not None and pooled_prompt_embeds is None:
	raise ValueError(
	"If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`."
	)

	if negative_prompt_embeds is not None and negative_pooled_prompt_embeds is None:
	raise ValueError(
	"If `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`."
	)

	if ip_adapter_image is not None and ip_adapter_image_embeds is not None:
	raise ValueError(
	"Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined."
	)

	if ip_adapter_image_embeds is not None:
	if not isinstance(ip_adapter_image_embeds, list):
	raise ValueError(
	f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}"
	)
	elif ip_adapter_image_embeds[0].ndim not in [3, 4]:
	raise ValueError(
	f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D"
	)

	# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents
	def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None):
	shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor)
	if isinstance(generator, list) and len(generator) != batch_size:
	raise ValueError(
	f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
	f" size of {batch_size}. Make sure the batch size matches the length of the generators."
	)

	if latents is None:
	latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
	else:
	latents = latents.to(device)

	# scale the initial noise by the standard deviation required by the scheduler
	latents = latents * self.scheduler.init_noise_sigma
	return latents

	def _get_add_time_ids(
	self, original_size, crops_coords_top_left, target_size, dtype, text_encoder_projection_dim=None
	):
	add_time_ids = list(original_size + crops_coords_top_left + target_size)

	passed_add_embed_dim = (
	self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim
	)
	expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features

	if expected_add_embed_dim != passed_add_embed_dim:
	raise ValueError(
	f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`."
	)

	add_time_ids = torch.tensor([add_time_ids], dtype=dtype)
	return add_time_ids

	def upcast_vae(self):
	dtype = self.vae.dtype
	self.vae.to(dtype=torch.float32)
	use_torch_2_0_or_xformers = isinstance(
	self.vae.decoder.mid_block.attentions[0].processor,
	(
	AttnProcessor2_0,
	XFormersAttnProcessor,
	LoRAXFormersAttnProcessor,
	LoRAAttnProcessor2_0,
	FusedAttnProcessor2_0,
	),
	)
	# if xformers or torch_2_0 is used attention block does not need
	# to be in float32 which can save lots of memory
	if use_torch_2_0_or_xformers:
	self.vae.post_quant_conv.to(dtype)
	self.vae.decoder.conv_in.to(dtype)
	self.vae.decoder.mid_block.to(dtype)

	# Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding
	def get_guidance_scale_embedding(
	self, w: torch.Tensor, embedding_dim: int = 512, dtype: torch.dtype = torch.float32
	) -> torch.FloatTensor:
	"""
	See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298

	Args:
	w (`torch.Tensor`):
	Generate embedding vectors with a specified guidance scale to subsequently enrich timestep embeddings.
	embedding_dim (`int`, optional, defaults to 512):
	Dimension of the embeddings to generate.
	dtype (`torch.dtype`, optional, defaults to `torch.float32`):
	Data type of the generated embeddings.

	Returns:
	`torch.FloatTensor`: Embedding vectors with shape `(len(w), embedding_dim)`.
	"""
	assert len(w.shape) == 1
	w = w * 1000.0

	half_dim = embedding_dim // 2
	emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1)
	emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb)
	emb = w.to(dtype)[:, None] * emb[None, :]
	emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1)
	if embedding_dim % 2 == 1: # zero pad
	emb = torch.nn.functional.pad(emb, (0, 1))
	assert emb.shape == (w.shape[0], embedding_dim)
	return emb

	def pred_z0(self, sample, model_output, timestep):
	alpha_prod_t = self.scheduler.alphas_cumprod[timestep].to(sample.device)

	beta_prod_t = 1 - alpha_prod_t
	if self.scheduler.config.prediction_type == "epsilon":
	pred_original_sample = (sample - beta_prod_t ** (0.5) * model_output) / alpha_prod_t ** (0.5)
	elif self.scheduler.config.prediction_type == "sample":
	pred_original_sample = model_output
	elif self.scheduler.config.prediction_type == "v_prediction":
	pred_original_sample = (alpha_prod_t*0.5) sample - (beta_prod_t*0.5) model_output
	# predict V
	model_output = (alpha_prod_t*0.5) model_output + (beta_prod_t*0.5) sample
	else:
	raise ValueError(
	f"prediction_type given as {self.scheduler.config.prediction_type} must be one of `epsilon`, `sample`,"
	" or `v_prediction`"
	)

	return pred_original_sample

	def pred_x0(self, latents, noise_pred, t, generator, device, prompt_embeds, output_type):
	pred_z0 = self.pred_z0(latents, noise_pred, t)
	pred_x0 = self.vae.decode(
	pred_z0 / self.vae.config.scaling_factor,
	return_dict=False,
	generator=generator
	)[0]
	#pred_x0, ____ = self.run_safety_checker(pred_x0, device, prompt_embeds.dtype)
	do_denormalize = [True] * pred_x0.shape[0]
	pred_x0 = self.image_processor.postprocess(pred_x0, output_type=output_type, do_denormalize=do_denormalize)

	return pred_x0

	@property
	def guidance_scale(self):
	return self._guidance_scale

	@property
	def guidance_rescale(self):
	return self._guidance_rescale

	@property
	def clip_skip(self):
	return self._clip_skip

	# here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
	# of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
	# corresponds to doing no classifier free guidance.
	@property
	def do_classifier_free_guidance(self):
	return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None

	@property
	def cross_attention_kwargs(self):
	return self._cross_attention_kwargs

	@property
	def denoising_end(self):
	return self._denoising_end

	@property
	def num_timesteps(self):
	return self._num_timesteps

	@property
	def interrupt(self):
	return self._interrupt

	@property
	def pag_scale(self):
	return self._pag_scale

	@property
	def do_adversarial_guidance(self):
	return self._pag_scale > 0

	@property
	def pag_adaptive_scaling(self):
	return self._pag_adaptive_scaling

	@property
	def do_pag_adaptive_scaling(self):
	return self._pag_adaptive_scaling > 0

	@property
	def pag_drop_rate(self):
	return self._pag_drop_rate

	@property
	def pag_applied_layers(self):
	return self._pag_applied_layers

	@property
	def pag_applied_layers_index(self):
	return self._pag_applied_layers_index

	@torch.no_grad()
	@replace_example_docstring(EXAMPLE_DOC_STRING)
	def __call__(
	self,
	prompt: Union[str, List[str]] = None,
	prompt_2: Optional[Union[str, List[str]]] = None,
	height: Optional[int] = None,
	width: Optional[int] = None,
	num_inference_steps: int = 50,
	timesteps: List[int] = None,
	denoising_end: Optional[float] = None,
	guidance_scale: float = 5.0,
	pag_scale: float = 0.0,
	pag_adaptive_scaling: float = 0.0,
	pag_drop_rate: float = 0.5,
	pag_applied_layers: List[str] = ['mid'], #['down', 'mid', 'up']
	pag_applied_layers_index: List[str] = None, #['d4', 'd5', 'm0']
	negative_prompt: Optional[Union[str, List[str]]] = None,
	negative_prompt_2: Optional[Union[str, List[str]]] = None,
	num_images_per_prompt: Optional[int] = 1,
	eta: float = 0.0,
	generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
	latents: Optional[torch.FloatTensor] = None,
	prompt_embeds: Optional[torch.FloatTensor] = None,
	negative_prompt_embeds: Optional[torch.FloatTensor] = None,
	pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
	negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
	ip_adapter_image: Optional[PipelineImageInput] = None,
	ip_adapter_image_embeds: Optional[List[torch.FloatTensor]] = None,
	output_type: Optional[str] = "pil",
	return_dict: bool = True,
	cross_attention_kwargs: Optional[Dict[str, Any]] = None,
	guidance_rescale: float = 0.0,
	original_size: Optional[Tuple[int, int]] = None,
	crops_coords_top_left: Tuple[int, int] = (0, 0),
	target_size: Optional[Tuple[int, int]] = None,
	negative_original_size: Optional[Tuple[int, int]] = None,
	negative_crops_coords_top_left: Tuple[int, int] = (0, 0),
	negative_target_size: Optional[Tuple[int, int]] = None,
	clip_skip: Optional[int] = None,
	callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None,
	callback_on_step_end_tensor_inputs: List[str] = ["latents"],
	**kwargs,
	):
	r"""
	Function invoked when calling the pipeline for generation.

	Args:
	prompt (`str` or `List[str]`, optional):
	The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
	instead.
	prompt_2 (`str` or `List[str]`, optional):
	The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
	used in both text-encoders
	height (`int`, optional, defaults to self.unet.config.sample_size * self.vae_scale_factor):
	The height in pixels of the generated image. This is set to 1024 by default for the best results.
	Anything below 512 pixels won't work well for
	[stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)
	and checkpoints that are not specifically fine-tuned on low resolutions.
	width (`int`, optional, defaults to self.unet.config.sample_size * self.vae_scale_factor):
	The width in pixels of the generated image. This is set to 1024 by default for the best results.
	Anything below 512 pixels won't work well for
	[stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0)
	and checkpoints that are not specifically fine-tuned on low resolutions.
	num_inference_steps (`int`, optional, defaults to 50):
	The number of denoising steps. More denoising steps usually lead to a higher quality image at the
	expense of slower inference.
	timesteps (`List[int]`, optional):
	Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument
	in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is
	passed will be used. Must be in descending order.
	denoising_end (`float`, optional):
	When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be
	completed before it is intentionally prematurely terminated. As a result, the returned sample will
	still retain a substantial amount of noise as determined by the discrete timesteps selected by the
	scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a
	"Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image
	Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output)
	guidance_scale (`float`, optional, defaults to 5.0):
	Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
	`guidance_scale` is defined as `w` of equation 2. of [Imagen
	Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
	1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
	usually at the expense of lower image quality.
	negative_prompt (`str` or `List[str]`, optional):
	The prompt or prompts not to guide the image generation. If not defined, one has to pass
	`negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
	less than `1`).
	negative_prompt_2 (`str` or `List[str]`, optional):
	The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and
	`text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders
	num_images_per_prompt (`int`, optional, defaults to 1):
	The number of images to generate per prompt.
	eta (`float`, optional, defaults to 0.0):
	Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
	[`schedulers.DDIMScheduler`], will be ignored for others.
	generator (`torch.Generator` or `List[torch.Generator]`, optional):
	One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
	to make generation deterministic.
	latents (`torch.FloatTensor`, optional):
	Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
	generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
	tensor will ge generated by sampling using the supplied random `generator`.
	prompt_embeds (`torch.FloatTensor`, optional):
	Pre-generated text embeddings. Can be used to easily tweak text inputs, e.g. prompt weighting. If not
	provided, text embeddings will be generated from `prompt` input argument.
	negative_prompt_embeds (`torch.FloatTensor`, optional):
	Pre-generated negative text embeddings. Can be used to easily tweak text inputs, e.g. prompt
	weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
	argument.
	pooled_prompt_embeds (`torch.FloatTensor`, optional):
	Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, e.g. prompt weighting.
	If not provided, pooled text embeddings will be generated from `prompt` input argument.
	negative_pooled_prompt_embeds (`torch.FloatTensor`, optional):
	Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, e.g. prompt
	weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
	input argument.
	ip_adapter_image: (`PipelineImageInput`, optional): Optional image input to work with IP Adapters.
	ip_adapter_image_embeds (`List[torch.FloatTensor]`, optional):
	Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of
	IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should
	contain the negative image embedding if `do_classifier_free_guidance` is set to `True`. If not
	provided, embeddings are computed from the `ip_adapter_image` input argument.
	output_type (`str`, optional, defaults to `"pil"`):
	The output format of the generate image. Choose between
	[PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
	return_dict (`bool`, optional, defaults to `True`):
	Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] instead
	of a plain tuple.
	cross_attention_kwargs (`dict`, optional):
	A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
	`self.processor` in
	[diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
	guidance_rescale (`float`, optional, defaults to 0.0):
	Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are
	Flawed](https://arxiv.org/pdf/2305.08891.pdf) `guidance_scale` is defined as `φ` in equation 16. of
	[Common Diffusion Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf).
	Guidance rescale factor should fix overexposure when using zero terminal SNR.
	original_size (`Tuple[int]`, optional, defaults to (1024, 1024)):
	If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled.
	`original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as
	explained in section 2.2 of
	[https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
	crops_coords_top_left (`Tuple[int]`, optional, defaults to (0, 0)):
	`crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position
	`crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting
	`crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of
	[https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
	target_size (`Tuple[int]`, optional, defaults to (1024, 1024)):
	For most cases, `target_size` should be set to the desired height and width of the generated image. If
	not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in
	section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952).
	negative_original_size (`Tuple[int]`, optional, defaults to (1024, 1024)):
	To negatively condition the generation process based on a specific image resolution. Part of SDXL's
	micro-conditioning as explained in section 2.2 of
	[https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
	information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
	negative_crops_coords_top_left (`Tuple[int]`, optional, defaults to (0, 0)):
	To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's
	micro-conditioning as explained in section 2.2 of
	[https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
	information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
	negative_target_size (`Tuple[int]`, optional, defaults to (1024, 1024)):
	To negatively condition the generation process based on a target image resolution. It should be as same
	as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of
	[https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more
	information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208.
	callback_on_step_end (`Callable`, optional):
	A function that calls at the end of each denoising steps during the inference. The function is called
	with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,
	callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by
	`callback_on_step_end_tensor_inputs`.
	callback_on_step_end_tensor_inputs (`List`, optional):
	The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
	will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
	`._callback_tensor_inputs` attribute of your pipeline class.

	Examples:

	Returns:
	[`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] or `tuple`:
	[`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a
	`tuple`. When returning a tuple, the first element is a list with the generated images.
	"""

	callback = kwargs.pop("callback", None)
	callback_steps = kwargs.pop("callback_steps", None)

	if callback is not None:
	deprecate(
	"callback",
	"1.0.0",
	"Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",
	)
	if callback_steps is not None:
	deprecate(
	"callback_steps",
	"1.0.0",
	"Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`",
	)

	# 0. Default height and width to unet
	height = height or self.default_sample_size * self.vae_scale_factor
	width = width or self.default_sample_size * self.vae_scale_factor

	original_size = original_size or (height, width)
	target_size = target_size or (height, width)

	# 1. Check inputs. Raise error if not correct
	self.check_inputs(
	prompt,
	prompt_2,
	height,
	width,
	callback_steps,
	negative_prompt,
	negative_prompt_2,
	prompt_embeds,
	negative_prompt_embeds,
	pooled_prompt_embeds,
	negative_pooled_prompt_embeds,
	ip_adapter_image,
	ip_adapter_image_embeds,
	callback_on_step_end_tensor_inputs,
	)

	self._guidance_scale = guidance_scale
	self._guidance_rescale = guidance_rescale
	self._clip_skip = clip_skip
	self._cross_attention_kwargs = cross_attention_kwargs
	self._denoising_end = denoising_end
	self._interrupt = False

	self._pag_scale = pag_scale
	self._pag_adaptive_scaling = pag_adaptive_scaling
	self._pag_drop_rate = pag_drop_rate
	self._pag_applied_layers = pag_applied_layers
	self._pag_applied_layers_index = pag_applied_layers_index

	# 2. Define call parameters
	if prompt is not None and isinstance(prompt, str):
	batch_size = 1
	elif prompt is not None and isinstance(prompt, list):
	batch_size = len(prompt)
	else:
	batch_size = prompt_embeds.shape[0]

	device = self._execution_device

	# 3. Encode input prompt
	lora_scale = (
	self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None
	)

	(
	prompt_embeds,
	negative_prompt_embeds,
	pooled_prompt_embeds,
	negative_pooled_prompt_embeds,
	) = self.encode_prompt(
	prompt=prompt,
	prompt_2=prompt_2,
	device=device,
	num_images_per_prompt=num_images_per_prompt,
	do_classifier_free_guidance=self.do_classifier_free_guidance,
	negative_prompt=negative_prompt,
	negative_prompt_2=negative_prompt_2,
	prompt_embeds=prompt_embeds,
	negative_prompt_embeds=negative_prompt_embeds,
	pooled_prompt_embeds=pooled_prompt_embeds,
	negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
	lora_scale=lora_scale,
	clip_skip=self.clip_skip,
	)

	# 4. Prepare timesteps
	timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps)

	# 5. Prepare latent variables
	num_channels_latents = self.unet.config.in_channels
	latents = self.prepare_latents(
	batch_size * num_images_per_prompt,
	num_channels_latents,
	height,
	width,
	prompt_embeds.dtype,
	device,
	generator,
	latents,
	)

	# 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
	extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)

	# 7. Prepare added time ids & embeddings
	add_text_embeds = pooled_prompt_embeds
	if self.text_encoder_2 is None:
	text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1])
	else:
	text_encoder_projection_dim = self.text_encoder_2.config.projection_dim

	add_time_ids = self._get_add_time_ids(
	original_size,
	crops_coords_top_left,
	target_size,
	dtype=prompt_embeds.dtype,
	text_encoder_projection_dim=text_encoder_projection_dim,
	)
	if negative_original_size is not None and negative_target_size is not None:
	negative_add_time_ids = self._get_add_time_ids(
	negative_original_size,
	negative_crops_coords_top_left,
	negative_target_size,
	dtype=prompt_embeds.dtype,
	text_encoder_projection_dim=text_encoder_projection_dim,
	)
	else:
	negative_add_time_ids = add_time_ids

	#cfg
	if self.do_classifier_free_guidance and not self.do_adversarial_guidance:
	prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)
	add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0)
	add_time_ids = torch.cat([negative_add_time_ids, add_time_ids], dim=0)
	#pag
	elif not self.do_classifier_free_guidance and self.do_adversarial_guidance:
	prompt_embeds = torch.cat([prompt_embeds, prompt_embeds], dim=0)
	add_text_embeds = torch.cat([add_text_embeds, add_text_embeds], dim=0)
	add_time_ids = torch.cat([add_time_ids, add_time_ids], dim=0)
	#both
	elif self.do_classifier_free_guidance and self.do_adversarial_guidance:
	prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds, prompt_embeds], dim=0)
	add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds, add_text_embeds], dim=0)
	add_time_ids = torch.cat([negative_add_time_ids, add_time_ids, add_time_ids], dim=0)

	prompt_embeds = prompt_embeds.to(device)
	add_text_embeds = add_text_embeds.to(device)
	add_time_ids = add_time_ids.to(device).repeat(batch_size * num_images_per_prompt, 1)

	if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
	image_embeds = self.prepare_ip_adapter_image_embeds(
	ip_adapter_image,
	ip_adapter_image_embeds,
	device,
	batch_size * num_images_per_prompt,
	self.do_classifier_free_guidance,
	)

	# 8. Denoising loop
	num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)

	# 8.1 Apply denoising_end
	if (
	self.denoising_end is not None
	and isinstance(self.denoising_end, float)
	and self.denoising_end > 0
	and self.denoising_end < 1
	):
	discrete_timestep_cutoff = int(
	round(
	self.scheduler.config.num_train_timesteps
	- (self.denoising_end * self.scheduler.config.num_train_timesteps)
	)
	)
	num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps)))
	timesteps = timesteps[:num_inference_steps]

	# 9. Optionally get Guidance Scale Embedding
	timestep_cond = None
	if self.unet.config.time_cond_proj_dim is not None:
	guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt)
	timestep_cond = self.get_guidance_scale_embedding(
	guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim
	).to(device=device, dtype=latents.dtype)

	# 10. Create down mid and up layer lists
	if self.do_adversarial_guidance:
	down_layers = []
	mid_layers = []
	up_layers = []
	for name, module in self.unet.named_modules():
	if 'attn1' in name and 'to' not in name:
	layer_type = name.split('.')[0].split('_')[0]
	if layer_type == 'down':
	down_layers.append(module)
	elif layer_type == 'mid':
	mid_layers.append(module)
	elif layer_type == 'up':
	up_layers.append(module)
	else:
	raise ValueError(f"Invalid layer type: {layer_type}")

	self._num_timesteps = len(timesteps)
	with self.progress_bar(total=num_inference_steps) as progress_bar:
	for i, t in enumerate(timesteps):
	if self.interrupt:
	continue

	#cfg
	if self.do_classifier_free_guidance and not self.do_adversarial_guidance:
	latent_model_input = torch.cat([latents] * 2)
	#pag
	elif not self.do_classifier_free_guidance and self.do_adversarial_guidance:
	latent_model_input = torch.cat([latents] * 2)
	#both
	elif self.do_classifier_free_guidance and self.do_adversarial_guidance:
	latent_model_input = torch.cat([latents] * 3)
	#no
	else:
	latent_model_input = latents

	# change attention layer in UNet if use PAG
	if self.do_adversarial_guidance:

	if self.do_classifier_free_guidance:
	replace_processor = PAGCFGIdentitySelfAttnProcessor()
	else:
	replace_processor = PAGIdentitySelfAttnProcessor()

	if(self.pag_applied_layers_index):
	drop_layers = self.pag_applied_layers_index
	for drop_layer in drop_layers:
	layer_number = int(drop_layer[1:])
	try:
	if drop_layer[0] == 'd':
	down_layers[layer_number].processor = replace_processor
	elif drop_layer[0] == 'm':
	mid_layers[layer_number].processor = replace_processor
	elif drop_layer[0] == 'u':
	up_layers[layer_number].processor = replace_processor
	else:
	raise ValueError(f"Invalid layer type: {drop_layer[0]}")
	except IndexError:
	raise ValueError(
	f"Invalid layer index: {drop_layer}. Available layers: {len(down_layers)} down layers, {len(mid_layers)} mid layers, {len(up_layers)} up layers."
	)
	elif(self.pag_applied_layers):
	drop_full_layers = self.pag_applied_layers
	for drop_full_layer in drop_full_layers:
	try:
	if drop_full_layer == "down":
	for down_layer in down_layers:
	down_layer.processor = replace_processor
	elif drop_full_layer == "mid":
	for mid_layer in mid_layers:
	mid_layer.processor = replace_processor
	elif drop_full_layer == "up":
	for up_layer in up_layers:
	up_layer.processor = replace_processor
	else:
	raise ValueError(f"Invalid layer type: {drop_full_layer}")
	except IndexError:
	raise ValueError(
	f"Invalid layer index: {drop_full_layer}. Available layers are: down, mid and up. If you need to specify each layer index, you can use `pag_applied_layers_index`"
	)

	latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)

	# predict the noise residual
	added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids}
	if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
	added_cond_kwargs["image_embeds"] = image_embeds

	noise_pred = self.unet(
	latent_model_input,
	t,
	encoder_hidden_states=prompt_embeds,
	timestep_cond=timestep_cond,
	cross_attention_kwargs=self.cross_attention_kwargs,
	added_cond_kwargs=added_cond_kwargs,
	return_dict=False,
	)[0]

	# perform guidance
	if self.do_classifier_free_guidance and not self.do_adversarial_guidance:
	noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
	noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond)
	# pag
	elif not self.do_classifier_free_guidance and self.do_adversarial_guidance:
	noise_pred_original, noise_pred_perturb = noise_pred.chunk(2)

	signal_scale = self.pag_scale
	if self.do_pag_adaptive_scaling:
	signal_scale = self.pag_scale - self.pag_adaptive_scaling * (1000-t)
	if signal_scale<0:
	signal_scale = 0

	noise_pred = noise_pred_original + signal_scale * (noise_pred_original - noise_pred_perturb)

	# both
	elif self.do_classifier_free_guidance and self.do_adversarial_guidance:

	noise_pred_uncond, noise_pred_text, noise_pred_text_perturb = noise_pred.chunk(3)

	signal_scale = self.pag_scale
	if self.do_pag_adaptive_scaling:
	signal_scale = self.pag_scale - self.pag_adaptive_scaling * (1000-t)
	if signal_scale<0:
	signal_scale = 0

	noise_pred = noise_pred_text + (self.guidance_scale-1.0) * (noise_pred_text - noise_pred_uncond) + signal_scale * (noise_pred_text - noise_pred_text_perturb)

	if self.do_classifier_free_guidance and self.guidance_rescale > 0.0:
	# Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf
	noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=self.guidance_rescale)

	# compute the previous noisy sample x_t -> x_t-1
	latents_dtype = latents.dtype
	latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
	if latents.dtype != latents_dtype:
	if torch.backends.mps.is_available():
	# some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
	latents = latents.to(latents_dtype)

	if callback_on_step_end is not None:
	callback_kwargs = {}
	for k in callback_on_step_end_tensor_inputs:
	callback_kwargs[k] = locals()[k]
	callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)

	latents = callback_outputs.pop("latents", latents)
	prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
	negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
	add_text_embeds = callback_outputs.pop("add_text_embeds", add_text_embeds)
	negative_pooled_prompt_embeds = callback_outputs.pop(
	"negative_pooled_prompt_embeds", negative_pooled_prompt_embeds
	)
	add_time_ids = callback_outputs.pop("add_time_ids", add_time_ids)
	negative_add_time_ids = callback_outputs.pop("negative_add_time_ids", negative_add_time_ids)

	# call the callback, if provided
	if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
	progress_bar.update()
	if callback is not None and i % callback_steps == 0:
	step_idx = i // getattr(self.scheduler, "order", 1)
	callback(step_idx, t, latents)

	if XLA_AVAILABLE:
	xm.mark_step()

	if not output_type == "latent":
	# make sure the VAE is in float32 mode, as it overflows in float16
	needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast

	if needs_upcasting:
	self.upcast_vae()
	latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype)
	elif latents.dtype != self.vae.dtype:
	if torch.backends.mps.is_available():
	# some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
	self.vae = self.vae.to(latents.dtype)

	# unscale/denormalize the latents
	# denormalize with the mean and std if available and not None
	has_latents_mean = hasattr(self.vae.config, "latents_mean") and self.vae.config.latents_mean is not None
	has_latents_std = hasattr(self.vae.config, "latents_std") and self.vae.config.latents_std is not None
	if has_latents_mean and has_latents_std:
	latents_mean = (
	torch.tensor(self.vae.config.latents_mean).view(1, 4, 1, 1).to(latents.device, latents.dtype)
	)
	latents_std = (
	torch.tensor(self.vae.config.latents_std).view(1, 4, 1, 1).to(latents.device, latents.dtype)
	)
	latents = latents * latents_std / self.vae.config.scaling_factor + latents_mean
	else:
	latents = latents / self.vae.config.scaling_factor

	image = self.vae.decode(latents, return_dict=False)[0]

	# cast back to fp16 if needed
	if needs_upcasting:
	self.vae.to(dtype=torch.float16)
	else:
	image = latents

	if not output_type == "latent":
	# apply watermark if available
	if self.watermark is not None:
	image = self.watermark.apply_watermark(image)

	image = self.image_processor.postprocess(image, output_type=output_type)

	# Offload all models
	self.maybe_free_model_hooks()

	if not return_dict:
	return (image,)

	#Change the attention layers back to original ones after PAG was applied
	if self.do_adversarial_guidance:
	if(self.pag_applied_layers_index):
	drop_layers = self.pag_applied_layers_index
	for drop_layer in drop_layers:
	layer_number = int(drop_layer[1:])
	try:
	if drop_layer[0] == 'd':
	down_layers[layer_number].processor = AttnProcessor2_0()
	elif drop_layer[0] == 'm':
	mid_layers[layer_number].processor = AttnProcessor2_0()
	elif drop_layer[0] == 'u':
	up_layers[layer_number].processor = AttnProcessor2_0()
	else:
	raise ValueError(f"Invalid layer type: {drop_layer[0]}")
	except IndexError:
	raise ValueError(
	f"Invalid layer index: {drop_layer}. Available layers: {len(down_layers)} down layers, {len(mid_layers)} mid layers, {len(up_layers)} up layers."
	)
	elif(self.pag_applied_layers):
	drop_full_layers = self.pag_applied_layers
	for drop_full_layer in drop_full_layers:
	try:
	if drop_full_layer == "down":
	for down_layer in down_layers:
	down_layer.processor = AttnProcessor2_0()
	elif drop_full_layer == "mid":
	for mid_layer in mid_layers:
	mid_layer.processor = AttnProcessor2_0()
	elif drop_full_layer == "up":
	for up_layer in up_layers:
	up_layer.processor = AttnProcessor2_0()
	else:
	raise ValueError(f"Invalid layer type: {drop_full_layer}")
	except IndexError:
	raise ValueError(
	f"Invalid layer index: {drop_full_layer}. Available layers are: down, mid and up. If you need to specify each layer index, you can use `pag_applied_layers_index`"
	)
	return StableDiffusionXLPipelineOutput(images=image)