show-1-base-with-code / pipeline_t2v_base_pixel.py

Upload pipeline_t2v_base_pixel.py

860a8cc 6 months ago

No virus

37.4 kB

	# Copyright 2023 Show Labs, Alibaba DAMO-VILAB, and The HuggingFace Team. All rights reserved.
	# Copyright 2023 The ModelScope Team.
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	import html
	import inspect
	import re
	import urllib.parse as ul
	from dataclasses import dataclass
	from typing import Any, Callable, Dict, List, Optional, Union

	import numpy as np
	import torch
	import torch.utils.checkpoint
	from transformers import CLIPImageProcessor, T5EncoderModel, T5Tokenizer

	from diffusers import UNet3DConditionModel
	from diffusers.loaders import LoraLoaderMixin
	from diffusers.pipelines.pipeline_utils import DiffusionPipeline
	from diffusers.schedulers import DDPMScheduler
	from diffusers.utils import (
	BACKENDS_MAPPING,
	BaseOutput,
	is_accelerate_available,
	is_accelerate_version,
	is_bs4_available,
	is_ftfy_available,
	logging,
	)
	from diffusers.utils.torch_utils import randn_tensor


	logger = logging.get_logger(__name__) # pylint: disable=invalid-name

	if is_bs4_available():
	from bs4 import BeautifulSoup

	if is_ftfy_available():
	import ftfy


	@dataclass
	class TextToVideoPipelineOutput(BaseOutput):
	"""
	Output class for text to video pipelines.

	Args:
	frames (`List[np.ndarray]` or `torch.FloatTensor`)
	List of denoised frames (essentially images) as NumPy arrays of shape `(height, width, num_channels)` or as
	a `torch` tensor. NumPy array present the denoised images of the diffusion pipeline. The length of the list
	denotes the video length i.e., the number of frames.
	"""

	frames: Union[List[np.ndarray], torch.FloatTensor]


	def tensor2vid(video: torch.Tensor, mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) -> List[np.ndarray]:
	# This code is copied from https://github.com/modelscope/modelscope/blob/1509fdb973e5871f37148a4b5e5964cafd43e64d/modelscope/pipelines/multi_modal/text_to_video_synthesis_pipeline.py#L78
	# reshape to ncfhw
	mean = torch.tensor(mean, device=video.device).reshape(1, -1, 1, 1, 1)
	std = torch.tensor(std, device=video.device).reshape(1, -1, 1, 1, 1)
	# unnormalize back to [0,1]
	video = video.mul_(std).add_(mean)
	video.clamp_(0, 1)
	# prepare the final outputs
	i, c, f, h, w = video.shape
	images = video.permute(2, 3, 0, 4, 1).reshape(
	f, h, i * w, c
	) # 1st (frames, h, batch_size, w, c) 2nd (frames, h, batch_size * w, c)
	images = images.unbind(dim=0) # prepare a list of indvidual (consecutive frames)
	images = [(image.cpu().numpy() * 255).astype("uint8") for image in images] # f h w c
	return images


	class TextToVideoIFPipeline(DiffusionPipeline, LoraLoaderMixin):
	tokenizer: T5Tokenizer
	text_encoder: T5EncoderModel

	unet: UNet3DConditionModel
	scheduler: DDPMScheduler

	feature_extractor: Optional[CLIPImageProcessor]
	# safety_checker: Optional[IFSafetyChecker]

	# watermarker: Optional[IFWatermarker]

	bad_punct_regex = re.compile(
	r"[" + "#®•©™&@·º½¾¿¡§~" + "\)" + "\(" + "\]" + "\[" + "\}" + "\{" + "\\|" + "\\" + "\/" + "\*" + r"]{1,}"
	) # noqa

	_optional_components = [
	"tokenizer",
	"text_encoder",
	"safety_checker",
	"feature_extractor",
	"watermarker",
	]

	def __init__(
	self,
	tokenizer: T5Tokenizer,
	text_encoder: T5EncoderModel,
	unet: UNet3DConditionModel,
	scheduler: DDPMScheduler,
	feature_extractor: Optional[CLIPImageProcessor],
	):
	super().__init__()

	self.register_modules(
	tokenizer=tokenizer,
	text_encoder=text_encoder,
	unet=unet,
	scheduler=scheduler,
	feature_extractor=feature_extractor,
	)
	self.safety_checker = None

	def enable_sequential_cpu_offload(self, gpu_id=0):
	r"""
	Offloads all models to CPU using accelerate, significantly reducing memory usage. When called, the pipeline's
	models have their state dicts saved to CPU and then are moved to a `torch.device('meta') and loaded to GPU only
	when their specific submodule has its `forward` method called.
	"""
	if is_accelerate_available():
	from accelerate import cpu_offload
	else:
	raise ImportError("Please install accelerate via `pip install accelerate`")

	device = torch.device(f"cuda:{gpu_id}")

	models = [
	self.text_encoder,
	self.unet,
	]
	for cpu_offloaded_model in models:
	if cpu_offloaded_model is not None:
	cpu_offload(cpu_offloaded_model, device)

	if self.safety_checker is not None:
	cpu_offload(self.safety_checker, execution_device=device, offload_buffers=True)

	def enable_model_cpu_offload(self, gpu_id=0):
	r"""
	Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared
	to `enable_sequential_cpu_offload`, this method moves one whole model at a time to the GPU when its `forward`
	method is called, and the model remains in GPU until the next model runs. Memory savings are lower than with
	`enable_sequential_cpu_offload`, but performance is much better due to the iterative execution of the `unet`.
	"""
	if is_accelerate_available() and is_accelerate_version(">=", "0.17.0.dev0"):
	from accelerate import cpu_offload_with_hook
	else:
	raise ImportError("`enable_model_cpu_offload` requires `accelerate v0.17.0` or higher.")

	device = torch.device(f"cuda:{gpu_id}")

	self.unet.train()

	if self.device.type != "cpu":
	self.to("cpu", silence_dtype_warnings=True)
	torch.cuda.empty_cache() # otherwise we don't see the memory savings (but they probably exist)

	hook = None

	if self.text_encoder is not None:
	_, hook = cpu_offload_with_hook(self.text_encoder, device, prev_module_hook=hook)

	# Accelerate will move the next model to the device _before_ calling the offload hook of the
	# previous model. This will cause both models to be present on the device at the same time.
	# IF uses T5 for its text encoder which is really large. We can manually call the offload
	# hook for the text encoder to ensure it's moved to the cpu before the unet is moved to
	# the GPU.
	self.text_encoder_offload_hook = hook

	_, hook = cpu_offload_with_hook(self.unet, device, prev_module_hook=hook)

	# if the safety checker isn't called, `unet_offload_hook` will have to be called to manually offload the unet
	self.unet_offload_hook = hook

	if self.safety_checker is not None:
	_, hook = cpu_offload_with_hook(self.safety_checker, device, prev_module_hook=hook)

	# We'll offload the last model manually.
	self.final_offload_hook = hook

	def remove_all_hooks(self):
	if is_accelerate_available():
	from accelerate.hooks import remove_hook_from_module
	else:
	raise ImportError("Please install accelerate via `pip install accelerate`")

	for model in [self.text_encoder, self.unet, self.safety_checker]:
	if model is not None:
	remove_hook_from_module(model, recurse=True)

	self.unet_offload_hook = None
	self.text_encoder_offload_hook = None
	self.final_offload_hook = None

	@property
	# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._execution_device
	def _execution_device(self):
	r"""
	Returns the device on which the pipeline's models will be executed. After calling
	`pipeline.enable_sequential_cpu_offload()` the execution device can only be inferred from Accelerate's module
	hooks.
	"""
	if not hasattr(self.unet, "_hf_hook"):
	return self.device
	for module in self.unet.modules():
	if (
	hasattr(module, "_hf_hook")
	and hasattr(module._hf_hook, "execution_device")
	and module._hf_hook.execution_device is not None
	):
	return torch.device(module._hf_hook.execution_device)
	return self.device

	@torch.no_grad()
	def encode_prompt(
	self,
	prompt,
	do_classifier_free_guidance=True,
	num_images_per_prompt=1,
	device=None,
	negative_prompt=None,
	prompt_embeds: Optional[torch.FloatTensor] = None,
	negative_prompt_embeds: Optional[torch.FloatTensor] = None,
	clean_caption: bool = False,
	):
	r"""
	Encodes the prompt into text encoder hidden states.

	Args:
	prompt (`str` or `List[str]`, optional):
	prompt to be encoded
	device: (`torch.device`, optional):
	torch device to place the resulting embeddings on
	num_images_per_prompt (`int`, optional, defaults to 1):
	number of images that should be generated per prompt
	do_classifier_free_guidance (`bool`, optional, defaults to `True`):
	whether to use classifier free guidance or not
	negative_prompt (`str` or `List[str]`, optional):
	The prompt or prompts not to guide the image generation. If not defined, one has to pass
	`negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead.
	Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`).
	prompt_embeds (`torch.FloatTensor`, optional):
	Pre-generated text embeddings. Can be used to easily tweak text inputs, e.g. prompt weighting. If not
	provided, text embeddings will be generated from `prompt` input argument.
	negative_prompt_embeds (`torch.FloatTensor`, optional):
	Pre-generated negative text embeddings. Can be used to easily tweak text inputs, e.g. prompt
	weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
	argument.
	"""
	if prompt is not None and negative_prompt is not None:
	if type(prompt) is not type(negative_prompt):
	raise TypeError(
	f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
	f" {type(prompt)}."
	)

	if device is None:
	device = self._execution_device

	if prompt is not None and isinstance(prompt, str):
	batch_size = 1
	elif prompt is not None and isinstance(prompt, list):
	batch_size = len(prompt)
	else:
	batch_size = prompt_embeds.shape[0]

	# while T5 can handle much longer input sequences than 77, the text encoder was trained with a max length of 77 for IF
	max_length = 77

	if prompt_embeds is None:
	prompt = self._text_preprocessing(prompt, clean_caption=clean_caption)
	text_inputs = self.tokenizer(
	prompt,
	padding="max_length",
	max_length=max_length,
	truncation=True,
	add_special_tokens=True,
	return_tensors="pt",
	)
	text_input_ids = text_inputs.input_ids
	untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids

	if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(
	text_input_ids, untruncated_ids
	):
	removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_length - 1 : -1])
	logger.warning(
	"The following part of your input was truncated because CLIP can only handle sequences up to"
	f" {max_length} tokens: {removed_text}"
	)

	attention_mask = text_inputs.attention_mask.to(device)

	prompt_embeds = self.text_encoder(
	text_input_ids.to(device),
	attention_mask=attention_mask,
	)
	prompt_embeds = prompt_embeds[0]

	if self.text_encoder is not None:
	dtype = self.text_encoder.dtype
	elif self.unet is not None:
	dtype = self.unet.dtype
	else:
	dtype = None

	prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)

	bs_embed, seq_len, _ = prompt_embeds.shape
	# duplicate text embeddings for each generation per prompt, using mps friendly method
	prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
	prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)

	# get unconditional embeddings for classifier free guidance
	if do_classifier_free_guidance and negative_prompt_embeds is None:
	uncond_tokens: List[str]
	if negative_prompt is None:
	uncond_tokens = [""] * batch_size
	elif isinstance(negative_prompt, str):
	uncond_tokens = [negative_prompt]
	elif batch_size != len(negative_prompt):
	raise ValueError(
	f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
	f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
	" the batch size of `prompt`."
	)
	else:
	uncond_tokens = negative_prompt

	uncond_tokens = self._text_preprocessing(uncond_tokens, clean_caption=clean_caption)
	max_length = prompt_embeds.shape[1]
	uncond_input = self.tokenizer(
	uncond_tokens,
	padding="max_length",
	max_length=max_length,
	truncation=True,
	return_attention_mask=True,
	add_special_tokens=True,
	return_tensors="pt",
	)
	attention_mask = uncond_input.attention_mask.to(device)

	negative_prompt_embeds = self.text_encoder(
	uncond_input.input_ids.to(device),
	attention_mask=attention_mask,
	)
	negative_prompt_embeds = negative_prompt_embeds[0]

	if do_classifier_free_guidance:
	# duplicate unconditional embeddings for each generation per prompt, using mps friendly method
	seq_len = negative_prompt_embeds.shape[1]

	negative_prompt_embeds = negative_prompt_embeds.to(dtype=dtype, device=device)

	negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
	negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)

	# For classifier free guidance, we need to do two forward passes.
	# Here we concatenate the unconditional and text embeddings into a single batch
	# to avoid doing two forward passes
	else:
	negative_prompt_embeds = None

	return prompt_embeds, negative_prompt_embeds

	# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs
	def prepare_extra_step_kwargs(self, generator, eta):
	# prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
	# eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
	# eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502
	# and should be between [0, 1]

	accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())
	extra_step_kwargs = {}
	if accepts_eta:
	extra_step_kwargs["eta"] = eta

	# check if the scheduler accepts generator
	accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())
	if accepts_generator:
	extra_step_kwargs["generator"] = generator
	return extra_step_kwargs

	def check_inputs(
	self,
	prompt,
	callback_steps,
	negative_prompt=None,
	prompt_embeds=None,
	negative_prompt_embeds=None,
	):
	if (callback_steps is None) or (
	callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)
	):
	raise ValueError(
	f"`callback_steps` has to be a positive integer but is {callback_steps} of type"
	f" {type(callback_steps)}."
	)

	if prompt is not None and prompt_embeds is not None:
	raise ValueError(
	f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
	" only forward one of the two."
	)
	elif prompt is None and prompt_embeds is None:
	raise ValueError(
	"Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
	)
	elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
	raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")

	if negative_prompt is not None and negative_prompt_embeds is not None:
	raise ValueError(
	f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"
	f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
	)

	if prompt_embeds is not None and negative_prompt_embeds is not None:
	if prompt_embeds.shape != negative_prompt_embeds.shape:
	raise ValueError(
	"`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"
	f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"
	f" {negative_prompt_embeds.shape}."
	)

	def prepare_intermediate_images(
	self,
	batch_size,
	num_channels,
	num_frames,
	height,
	width,
	dtype,
	device,
	generator,
	):
	shape = (batch_size, num_channels, num_frames, height, width)
	if isinstance(generator, list) and len(generator) != batch_size:
	raise ValueError(
	f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
	f" size of {batch_size}. Make sure the batch size matches the length of the generators."
	)

	intermediate_images = randn_tensor(shape, generator=generator, device=device, dtype=dtype)

	# scale the initial noise by the standard deviation required by the scheduler
	intermediate_images = intermediate_images * self.scheduler.init_noise_sigma
	return intermediate_images

	def _text_preprocessing(self, text, clean_caption=False):
	if clean_caption and not is_bs4_available():
	logger.warn(BACKENDS_MAPPING["bs4"][-1].format("Setting `clean_caption=True`"))
	logger.warn("Setting `clean_caption` to False...")
	clean_caption = False

	if clean_caption and not is_ftfy_available():
	logger.warn(BACKENDS_MAPPING["ftfy"][-1].format("Setting `clean_caption=True`"))
	logger.warn("Setting `clean_caption` to False...")
	clean_caption = False

	if not isinstance(text, (tuple, list)):
	text = [text]

	def process(text: str):
	if clean_caption:
	text = self._clean_caption(text)
	text = self._clean_caption(text)
	else:
	text = text.lower().strip()
	return text

	return [process(t) for t in text]

	def _clean_caption(self, caption):
	caption = str(caption)
	caption = ul.unquote_plus(caption)
	caption = caption.strip().lower()
	caption = re.sub("<person>", "person", caption)
	# urls:
	caption = re.sub(
	r"\b((?:https?:(?:\/{1,3}\|[a-zA-Z0-9%])\|[a-zA-Z0-9.\-]+[.](?:com\|co\|ru\|net\|org\|edu\|gov\|it)[\w/-]*\b\/?(?!@)))", # noqa
	"",
	caption,
	) # regex for urls
	caption = re.sub(
	r"\b((?:www:(?:\/{1,3}\|[a-zA-Z0-9%])\|[a-zA-Z0-9.\-]+[.](?:com\|co\|ru\|net\|org\|edu\|gov\|it)[\w/-]*\b\/?(?!@)))", # noqa
	"",
	caption,
	) # regex for urls
	# html:
	caption = BeautifulSoup(caption, features="html.parser").text

	# @<nickname>
	caption = re.sub(r"@[\w\d]+\b", "", caption)

	# 31C0—31EF CJK Strokes
	# 31F0—31FF Katakana Phonetic Extensions
	# 3200—32FF Enclosed CJK Letters and Months
	# 3300—33FF CJK Compatibility
	# 3400—4DBF CJK Unified Ideographs Extension A
	# 4DC0—4DFF Yijing Hexagram Symbols
	# 4E00—9FFF CJK Unified Ideographs
	caption = re.sub(r"[\u31c0-\u31ef]+", "", caption)
	caption = re.sub(r"[\u31f0-\u31ff]+", "", caption)
	caption = re.sub(r"[\u3200-\u32ff]+", "", caption)
	caption = re.sub(r"[\u3300-\u33ff]+", "", caption)
	caption = re.sub(r"[\u3400-\u4dbf]+", "", caption)
	caption = re.sub(r"[\u4dc0-\u4dff]+", "", caption)
	caption = re.sub(r"[\u4e00-\u9fff]+", "", caption)
	#######################################################

	# все виды тире / all types of dash --> "-"
	caption = re.sub(
	r"[\u002D\u058A\u05BE\u1400\u1806\u2010-\u2015\u2E17\u2E1A\u2E3A\u2E3B\u2E40\u301C\u3030\u30A0\uFE31\uFE32\uFE58\uFE63\uFF0D]+", # noqa
	"-",
	caption,
	)

	# кавычки к одному стандарту
	caption = re.sub(r"[`´«»“”¨]", '"', caption)
	caption = re.sub(r"[‘’]", "'", caption)

	# "
	caption = re.sub(r""?", "", caption)
	# &amp
	caption = re.sub(r"&amp", "", caption)

	# ip adresses:
	caption = re.sub(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", " ", caption)

	# article ids:
	caption = re.sub(r"\d:\d\d\s+$", "", caption)

	# \n
	caption = re.sub(r"\\n", " ", caption)

	# "#123"
	caption = re.sub(r"#\d{1,3}\b", "", caption)
	# "#12345.."
	caption = re.sub(r"#\d{5,}\b", "", caption)
	# "123456.."
	caption = re.sub(r"\b\d{6,}\b", "", caption)
	# filenames:
	caption = re.sub(r"[\S]+\.(?:png\|jpg\|jpeg\|bmp\|webp\|eps\|pdf\|apk\|mp4)", "", caption)

	#
	caption = re.sub(r"[\"\']{2,}", r'"', caption) # """AUSVERKAUFT"""
	caption = re.sub(r"[\.]{2,}", r" ", caption) # """AUSVERKAUFT"""

	caption = re.sub(self.bad_punct_regex, r" ", caption) # *AUSVERKAUFT*, #AUSVERKAUFT
	caption = re.sub(r"\s+\.\s+", r" ", caption) # " . "

	# this-is-my-cute-cat / this_is_my_cute_cat
	regex2 = re.compile(r"(?:\-\|\_)")
	if len(re.findall(regex2, caption)) > 3:
	caption = re.sub(regex2, " ", caption)

	caption = ftfy.fix_text(caption)
	caption = html.unescape(html.unescape(caption))

	caption = re.sub(r"\b[a-zA-Z]{1,3}\d{3,15}\b", "", caption) # jc6640
	caption = re.sub(r"\b[a-zA-Z]+\d+[a-zA-Z]+\b", "", caption) # jc6640vc
	caption = re.sub(r"\b\d+[a-zA-Z]+\d+\b", "", caption) # 6640vc231

	caption = re.sub(r"(worldwide\s+)?(free\s+)?shipping", "", caption)
	caption = re.sub(r"(free\s)?download(\sfree)?", "", caption)
	caption = re.sub(r"\bclick\b\s(?:for\|on)\s\w+", "", caption)
	caption = re.sub(r"\b(?:png\|jpg\|jpeg\|bmp\|webp\|eps\|pdf\|apk\|mp4)(\simage[s]?)?", "", caption)
	caption = re.sub(r"\bpage\s+\d+\b", "", caption)

	caption = re.sub(r"\b\d[a-zA-Z]+\d+[a-zA-Z]+\d+[a-zA-Z\d]\b", r" ", caption) # j2d1a2a...

	caption = re.sub(r"\b\d+\.?\d[xх×]\d+\.?\d\b", "", caption)

	caption = re.sub(r"\b\s+\:\s+", r": ", caption)
	caption = re.sub(r"(\D[,\./])\b", r"\1 ", caption)
	caption = re.sub(r"\s+", " ", caption)

	caption.strip()

	caption = re.sub(r"^[\"\']([\w\W]+)[\"\']$", r"\1", caption)
	caption = re.sub(r"^[\'\_,\-\:;]", r"", caption)
	caption = re.sub(r"[\'\_,\-\:\-\+]$", r"", caption)
	caption = re.sub(r"^\.\S+$", "", caption)

	return caption.strip()

	@torch.no_grad()
	def __call__(
	self,
	prompt: Union[str, List[str]] = None,
	num_inference_steps: int = 100,
	timesteps: List[int] = None,
	guidance_scale: float = 7.0,
	negative_prompt: Optional[Union[str, List[str]]] = None,
	num_images_per_prompt: Optional[int] = 1,
	height: Optional[int] = None,
	width: Optional[int] = None,
	num_frames: int = 16,
	eta: float = 0.0,
	generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
	prompt_embeds: Optional[torch.FloatTensor] = None,
	negative_prompt_embeds: Optional[torch.FloatTensor] = None,
	output_type: Optional[str] = "np",
	return_dict: bool = True,
	callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
	callback_steps: int = 1,
	clean_caption: bool = True,
	cross_attention_kwargs: Optional[Dict[str, Any]] = None,
	):
	"""
	Function invoked when calling the pipeline for generation.

	Args:
	prompt (`str` or `List[str]`, optional):
	The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
	instead.
	num_inference_steps (`int`, optional, defaults to 50):
	The number of denoising steps. More denoising steps usually lead to a higher quality image at the
	expense of slower inference.
	timesteps (`List[int]`, optional):
	Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps`
	timesteps are used. Must be in descending order.
	guidance_scale (`float`, optional, defaults to 7.5):
	Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
	`guidance_scale` is defined as `w` of equation 2. of [Imagen
	Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
	1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
	usually at the expense of lower image quality.
	negative_prompt (`str` or `List[str]`, optional):
	The prompt or prompts not to guide the image generation. If not defined, one has to pass
	`negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
	less than `1`).
	num_images_per_prompt (`int`, optional, defaults to 1):
	The number of images to generate per prompt.
	height (`int`, optional, defaults to self.unet.config.sample_size):
	The height in pixels of the generated image.
	width (`int`, optional, defaults to self.unet.config.sample_size):
	The width in pixels of the generated image.
	eta (`float`, optional, defaults to 0.0):
	Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
	[`schedulers.DDIMScheduler`], will be ignored for others.
	generator (`torch.Generator` or `List[torch.Generator]`, optional):
	One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
	to make generation deterministic.
	prompt_embeds (`torch.FloatTensor`, optional):
	Pre-generated text embeddings. Can be used to easily tweak text inputs, e.g. prompt weighting. If not
	provided, text embeddings will be generated from `prompt` input argument.
	negative_prompt_embeds (`torch.FloatTensor`, optional):
	Pre-generated negative text embeddings. Can be used to easily tweak text inputs, e.g. prompt
	weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
	argument.
	output_type (`str`, optional, defaults to `"pil"`):
	The output format of the generate image. Choose between
	[PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
	return_dict (`bool`, optional, defaults to `True`):
	Whether or not to return a [`~pipelines.stable_diffusion.IFPipelineOutput`] instead of a plain tuple.
	callback (`Callable`, optional):
	A function that will be called every `callback_steps` steps during inference. The function will be
	called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
	callback_steps (`int`, optional, defaults to 1):
	The frequency at which the `callback` function will be called. If not specified, the callback will be
	called at every step.
	clean_caption (`bool`, optional, defaults to `True`):
	Whether or not to clean the caption before creating embeddings. Requires `beautifulsoup4` and `ftfy` to
	be installed. If the dependencies are not installed, the embeddings will be created from the raw
	prompt.
	cross_attention_kwargs (`dict`, optional):
	A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
	`self.processor` in
	[diffusers.cross_attention](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/cross_attention.py).

	Examples:

	Returns:
	[`~pipelines.stable_diffusion.IFPipelineOutput`] or `tuple`:
	[`~pipelines.stable_diffusion.IFPipelineOutput`] if `return_dict` is True, otherwise a `tuple. When
	returning a tuple, the first element is a list with the generated images, and the second element is a list
	of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work" (nsfw)
	or watermarked content, according to the `safety_checker`.
	"""
	# 1. Check inputs. Raise error if not correct
	self.check_inputs(
	prompt,
	callback_steps,
	negative_prompt,
	prompt_embeds,
	negative_prompt_embeds,
	)

	# 2. Define call parameters
	height = height or self.unet.config.sample_size
	width = width or self.unet.config.sample_size

	if prompt is not None and isinstance(prompt, str):
	batch_size = 1
	elif prompt is not None and isinstance(prompt, list):
	batch_size = len(prompt)
	else:
	batch_size = prompt_embeds.shape[0]

	device = self._execution_device

	# here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
	# of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
	# corresponds to doing no classifier free guidance.
	do_classifier_free_guidance = guidance_scale > 1.0

	# 3. Encode input prompt
	prompt_embeds, negative_prompt_embeds = self.encode_prompt(
	prompt,
	do_classifier_free_guidance,
	num_images_per_prompt=num_images_per_prompt,
	device=device,
	negative_prompt=negative_prompt,
	prompt_embeds=prompt_embeds,
	negative_prompt_embeds=negative_prompt_embeds,
	clean_caption=clean_caption,
	)

	if do_classifier_free_guidance:
	prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])

	# 4. Prepare timesteps
	if timesteps is not None:
	self.scheduler.set_timesteps(timesteps=timesteps, device=device)
	timesteps = self.scheduler.timesteps
	num_inference_steps = len(timesteps)
	else:
	self.scheduler.set_timesteps(num_inference_steps, device=device)
	timesteps = self.scheduler.timesteps

	# 5. Prepare intermediate images
	intermediate_images = self.prepare_intermediate_images(
	batch_size * num_images_per_prompt,
	self.unet.config.in_channels,
	num_frames,
	height,
	width,
	prompt_embeds.dtype,
	device,
	generator,
	)

	# 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
	extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)

	# HACK: see comment in `enable_model_cpu_offload`
	if hasattr(self, "text_encoder_offload_hook") and self.text_encoder_offload_hook is not None:
	self.text_encoder_offload_hook.offload()

	# 7. Denoising loop
	num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order
	with self.progress_bar(total=num_inference_steps) as progress_bar:
	for i, t in enumerate(timesteps):
	model_input = (
	torch.cat([intermediate_images] * 2) if do_classifier_free_guidance else intermediate_images
	)
	model_input = self.scheduler.scale_model_input(model_input, t)

	# predict the noise residual
	noise_pred = self.unet(
	model_input,
	t,
	encoder_hidden_states=prompt_embeds,
	cross_attention_kwargs=cross_attention_kwargs,
	).sample

	# perform guidance
	if do_classifier_free_guidance:
	noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
	noise_pred_uncond, _ = noise_pred_uncond.split(model_input.shape[1], dim=1)
	noise_pred_text, predicted_variance = noise_pred_text.split(model_input.shape[1], dim=1)
	noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
	noise_pred = torch.cat([noise_pred, predicted_variance], dim=1)

	if self.scheduler.config.variance_type not in [
	"learned",
	"learned_range",
	]:
	noise_pred, _ = noise_pred.split(model_input.shape[1], dim=1)

	# reshape latents
	bsz, channel, frames, height, width = intermediate_images.shape
	intermediate_images = intermediate_images.permute(0, 2, 1, 3, 4).reshape(
	bsz * frames, channel, height, width
	)
	noise_pred = noise_pred.permute(0, 2, 1, 3, 4).reshape(bsz * frames, -1, height, width)

	# compute the previous noisy sample x_t -> x_t-1
	intermediate_images = self.scheduler.step(
	noise_pred, t, intermediate_images, **extra_step_kwargs
	).prev_sample

	# reshape latents back
	intermediate_images = (
	intermediate_images[None, :].reshape(bsz, frames, channel, height, width).permute(0, 2, 1, 3, 4)
	)

	# call the callback, if provided
	if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
	progress_bar.update()
	if callback is not None and i % callback_steps == 0:
	callback(i, t, intermediate_images)

	video_tensor = intermediate_images

	if output_type == "pt":
	video = video_tensor
	else:
	video = tensor2vid(video_tensor)

	# Offload last model to CPU
	if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None:
	self.final_offload_hook.offload()

	if not return_dict:
	return (video,)

	return TextToVideoPipelineOutput(frames=video)