Spaces:
Running
on
Zero
Running
on
Zero
| from typing import Dict, List, Optional, Tuple, Union | |
| import torch | |
| import torch.nn as nn | |
| from einops import rearrange, repeat | |
| from ...modules.autoencoding.lpips.loss.lpips import LPIPS | |
| from ...modules.encoders.modules import GeneralConditioner | |
| from ...util import append_dims, instantiate_from_config | |
| from .denoiser import Denoiser | |
| class StandardDiffusionLoss(nn.Module): | |
| def __init__( | |
| self, | |
| sigma_sampler_config: dict, | |
| loss_weighting_config: dict, | |
| loss_type: str = "l2", | |
| offset_noise_level: float = 0.0, | |
| batch2model_keys: Optional[Union[str, List[str]]] = None, | |
| ): | |
| super().__init__() | |
| assert loss_type in ["l2", "l1", "lpips"] | |
| self.sigma_sampler = instantiate_from_config(sigma_sampler_config) | |
| self.loss_weighting = instantiate_from_config(loss_weighting_config) | |
| self.loss_type = loss_type | |
| self.offset_noise_level = offset_noise_level | |
| if loss_type == "lpips": | |
| self.lpips = LPIPS().eval() | |
| if not batch2model_keys: | |
| batch2model_keys = [] | |
| if isinstance(batch2model_keys, str): | |
| batch2model_keys = [batch2model_keys] | |
| self.batch2model_keys = set(batch2model_keys) | |
| def get_noised_input( | |
| self, sigmas_bc: torch.Tensor, noise: torch.Tensor, input: torch.Tensor | |
| ) -> torch.Tensor: | |
| noised_input = input + noise * sigmas_bc | |
| return noised_input | |
| def forward( | |
| self, | |
| network: nn.Module, | |
| denoiser: Denoiser, | |
| conditioner: GeneralConditioner, | |
| input: torch.Tensor, | |
| batch: Dict, | |
| return_model_output: bool = False, | |
| ) -> torch.Tensor: | |
| cond = conditioner(batch) | |
| # for video diffusion | |
| if "num_video_frames" in batch: | |
| num_frames = batch["num_video_frames"] | |
| for k in ["crossattn", "concat"]: | |
| cond[k] = repeat(cond[k], "b ... -> b t ...", t=num_frames) | |
| cond[k] = rearrange(cond[k], "b t ... -> (b t) ...", t=num_frames) | |
| return self._forward(network, denoiser, cond, input, batch, return_model_output) | |
| def _forward( | |
| self, | |
| network: nn.Module, | |
| denoiser: Denoiser, | |
| cond: Dict, | |
| input: torch.Tensor, | |
| batch: Dict, | |
| return_model_output: bool = False, | |
| ) -> Tuple[torch.Tensor, Dict]: | |
| additional_model_inputs = { | |
| key: batch[key] for key in self.batch2model_keys.intersection(batch) | |
| } | |
| sigmas = self.sigma_sampler(input.shape[0]).to(input) | |
| noise = torch.randn_like(input) | |
| if self.offset_noise_level > 0.0: | |
| offset_shape = ( | |
| (input.shape[0], 1, input.shape[2]) | |
| if self.n_frames is not None | |
| else (input.shape[0], input.shape[1]) | |
| ) | |
| noise = noise + self.offset_noise_level * append_dims( | |
| torch.randn(offset_shape, device=input.device), | |
| input.ndim, | |
| ) | |
| sigmas_bc = append_dims(sigmas, input.ndim) | |
| noised_input = self.get_noised_input(sigmas_bc, noise, input) | |
| model_output = denoiser( | |
| network, noised_input, sigmas, cond, **additional_model_inputs | |
| ) | |
| w = append_dims(self.loss_weighting(sigmas), input.ndim) | |
| if not return_model_output: | |
| return self.get_loss(model_output, input, w) | |
| else: | |
| return self.get_loss(model_output, input, w), model_output | |
| def get_loss(self, model_output, target, w): | |
| if self.loss_type == "l2": | |
| return torch.mean( | |
| (w * (model_output - target) ** 2).reshape(target.shape[0], -1), 1 | |
| ) | |
| elif self.loss_type == "l1": | |
| return torch.mean( | |
| (w * (model_output - target).abs()).reshape(target.shape[0], -1), 1 | |
| ) | |
| elif self.loss_type == "lpips": | |
| loss = self.lpips(model_output, target).reshape(-1) | |
| return loss | |
| else: | |
| raise NotImplementedError(f"Unknown loss type {self.loss_type}") | |
| class StandardDiffusionLossWithPixelNeRFLoss(StandardDiffusionLoss): | |
| def __init__( | |
| self, | |
| sigma_sampler_config: Dict, | |
| loss_weighting_config: Dict, | |
| loss_type: str = "l2", | |
| offset_noise_level: float = 0, | |
| batch2model_keys: str | List[str] | None = None, | |
| pixelnerf_loss_weight: float = 1.0, | |
| pixelnerf_loss_type: str = "l2", | |
| ): | |
| super().__init__( | |
| sigma_sampler_config, | |
| loss_weighting_config, | |
| loss_type, | |
| offset_noise_level, | |
| batch2model_keys, | |
| ) | |
| self.pixelnerf_loss_weight = pixelnerf_loss_weight | |
| self.pixelnerf_loss_type = pixelnerf_loss_type | |
| def get_pixelnerf_loss(self, model_output, target): | |
| if self.pixelnerf_loss_type == "l2": | |
| return torch.mean( | |
| ((model_output - target) ** 2).reshape(target.shape[0], -1), 1 | |
| ) | |
| elif self.pixelnerf_loss_type == "l1": | |
| return torch.mean( | |
| ((model_output - target).abs()).reshape(target.shape[0], -1), 1 | |
| ) | |
| elif self.pixelnerf_loss_type == "lpips": | |
| loss = self.lpips(model_output, target).reshape(-1) | |
| return loss | |
| else: | |
| raise NotImplementedError(f"Unknown loss type {self.loss_type}") | |
| def forward( | |
| self, | |
| network: nn.Module, | |
| denoiser: Denoiser, | |
| conditioner: GeneralConditioner, | |
| input: torch.Tensor, | |
| batch: Dict, | |
| return_model_output: bool = False, | |
| ) -> torch.Tensor: | |
| cond = conditioner(batch) | |
| return self._forward(network, denoiser, cond, input, batch, return_model_output) | |
| def _forward( | |
| self, | |
| network: nn.Module, | |
| denoiser: Denoiser, | |
| cond: Dict, | |
| input: torch.Tensor, | |
| batch: Dict, | |
| return_model_output: bool = False, | |
| ) -> Tuple[torch.Tensor | Dict]: | |
| loss = super()._forward( | |
| network, denoiser, cond, input, batch, return_model_output | |
| ) | |
| pixelnerf_loss = self.get_pixelnerf_loss( | |
| cond["rgb"], batch["pixelnerf_input"]["rgb"] | |
| ) | |
| if not return_model_output: | |
| return loss + self.pixelnerf_loss_weight * pixelnerf_loss | |
| else: | |
| return loss[0] + self.pixelnerf_loss_weight * pixelnerf_loss, loss[1] | |