Spaces:
Running
on
A10G
Running
on
A10G
| import math | |
| import random | |
| import torch | |
| from basicsr.utils.registry import ARCH_REGISTRY | |
| from torch import nn | |
| from torch.nn import functional as F | |
| from stylegan2_clean_arch import StyleGAN2GeneratorClean | |
| class StyleGAN2GeneratorCSFT(StyleGAN2GeneratorClean): | |
| """StyleGAN2 Generator with SFT modulation (Spatial Feature Transform). | |
| It is the clean version without custom compiled CUDA extensions used in StyleGAN2. | |
| Args: | |
| out_size (int): The spatial size of outputs. | |
| num_style_feat (int): Channel number of style features. Default: 512. | |
| num_mlp (int): Layer number of MLP style layers. Default: 8. | |
| channel_multiplier (int): Channel multiplier for large networks of StyleGAN2. Default: 2. | |
| narrow (float): The narrow ratio for channels. Default: 1. | |
| sft_half (bool): Whether to apply SFT on half of the input channels. Default: False. | |
| """ | |
| def __init__(self, out_size, num_style_feat=512, num_mlp=8, channel_multiplier=2, narrow=1, sft_half=False): | |
| super(StyleGAN2GeneratorCSFT, self).__init__( | |
| out_size, | |
| num_style_feat=num_style_feat, | |
| num_mlp=num_mlp, | |
| channel_multiplier=channel_multiplier, | |
| narrow=narrow) | |
| self.sft_half = sft_half | |
| def forward(self, | |
| styles, | |
| conditions, | |
| input_is_latent=False, | |
| noise=None, | |
| randomize_noise=True, | |
| truncation=1, | |
| truncation_latent=None, | |
| inject_index=None, | |
| return_latents=False): | |
| """Forward function for StyleGAN2GeneratorCSFT. | |
| Args: | |
| styles (list[Tensor]): Sample codes of styles. | |
| conditions (list[Tensor]): SFT conditions to generators. | |
| input_is_latent (bool): Whether input is latent style. Default: False. | |
| noise (Tensor | None): Input noise or None. Default: None. | |
| randomize_noise (bool): Randomize noise, used when 'noise' is False. Default: True. | |
| truncation (float): The truncation ratio. Default: 1. | |
| truncation_latent (Tensor | None): The truncation latent tensor. Default: None. | |
| inject_index (int | None): The injection index for mixing noise. Default: None. | |
| return_latents (bool): Whether to return style latents. Default: False. | |
| """ | |
| # style codes -> latents with Style MLP layer | |
| if not input_is_latent: | |
| styles = [self.style_mlp(s) for s in styles] | |
| # noises | |
| if noise is None: | |
| if randomize_noise: | |
| noise = [None] * self.num_layers # for each style conv layer | |
| else: # use the stored noise | |
| noise = [getattr(self.noises, f'noise{i}') for i in range(self.num_layers)] | |
| # style truncation | |
| if truncation < 1: | |
| style_truncation = [] | |
| for style in styles: | |
| style_truncation.append(truncation_latent + truncation * (style - truncation_latent)) | |
| styles = style_truncation | |
| # get style latents with injection | |
| if len(styles) == 1: | |
| inject_index = self.num_latent | |
| if styles[0].ndim < 3: | |
| # repeat latent code for all the layers | |
| latent = styles[0].unsqueeze(1).repeat(1, inject_index, 1) | |
| else: # used for encoder with different latent code for each layer | |
| latent = styles[0] | |
| elif len(styles) == 2: # mixing noises | |
| if inject_index is None: | |
| inject_index = random.randint(1, self.num_latent - 1) | |
| latent1 = styles[0].unsqueeze(1).repeat(1, inject_index, 1) | |
| latent2 = styles[1].unsqueeze(1).repeat(1, self.num_latent - inject_index, 1) | |
| latent = torch.cat([latent1, latent2], 1) | |
| # main generation | |
| out = self.constant_input(latent.shape[0]) | |
| out = self.style_conv1(out, latent[:, 0], noise=noise[0]) | |
| skip = self.to_rgb1(out, latent[:, 1]) | |
| i = 1 | |
| for conv1, conv2, noise1, noise2, to_rgb in zip(self.style_convs[::2], self.style_convs[1::2], noise[1::2], | |
| noise[2::2], self.to_rgbs): | |
| out = conv1(out, latent[:, i], noise=noise1) | |
| # the conditions may have fewer levels | |
| if i < len(conditions): | |
| # SFT part to combine the conditions | |
| if self.sft_half: # only apply SFT to half of the channels | |
| out_same, out_sft = torch.split(out, int(out.size(1) // 2), dim=1) | |
| out_sft = out_sft * conditions[i - 1] + conditions[i] | |
| out = torch.cat([out_same, out_sft], dim=1) | |
| else: # apply SFT to all the channels | |
| out = out * conditions[i - 1] + conditions[i] | |
| out = conv2(out, latent[:, i + 1], noise=noise2) | |
| skip = to_rgb(out, latent[:, i + 2], skip) # feature back to the rgb space | |
| i += 2 | |
| image = skip | |
| if return_latents: | |
| return image, latent | |
| else: | |
| return image, None | |
| class ResBlock(nn.Module): | |
| """Residual block with bilinear upsampling/downsampling. | |
| Args: | |
| in_channels (int): Channel number of the input. | |
| out_channels (int): Channel number of the output. | |
| mode (str): Upsampling/downsampling mode. Options: down | up. Default: down. | |
| """ | |
| def __init__(self, in_channels, out_channels, mode='down'): | |
| super(ResBlock, self).__init__() | |
| self.conv1 = nn.Conv2d(in_channels, in_channels, 3, 1, 1) | |
| self.conv2 = nn.Conv2d(in_channels, out_channels, 3, 1, 1) | |
| self.skip = nn.Conv2d(in_channels, out_channels, 1, bias=False) | |
| if mode == 'down': | |
| self.scale_factor = 0.5 | |
| elif mode == 'up': | |
| self.scale_factor = 2 | |
| def forward(self, x): | |
| out = F.leaky_relu_(self.conv1(x), negative_slope=0.2) | |
| # upsample/downsample | |
| out = F.interpolate(out, scale_factor=self.scale_factor, mode='bilinear', align_corners=False) | |
| out = F.leaky_relu_(self.conv2(out), negative_slope=0.2) | |
| # skip | |
| x = F.interpolate(x, scale_factor=self.scale_factor, mode='bilinear', align_corners=False) | |
| skip = self.skip(x) | |
| out = out + skip | |
| return out | |
| class GFPGANv1Clean(nn.Module): | |
| """The GFPGAN architecture: Unet + StyleGAN2 decoder with SFT. | |
| It is the clean version without custom compiled CUDA extensions used in StyleGAN2. | |
| Ref: GFP-GAN: Towards Real-World Blind Face Restoration with Generative Facial Prior. | |
| Args: | |
| out_size (int): The spatial size of outputs. | |
| num_style_feat (int): Channel number of style features. Default: 512. | |
| channel_multiplier (int): Channel multiplier for large networks of StyleGAN2. Default: 2. | |
| decoder_load_path (str): The path to the pre-trained decoder model (usually, the StyleGAN2). Default: None. | |
| fix_decoder (bool): Whether to fix the decoder. Default: True. | |
| num_mlp (int): Layer number of MLP style layers. Default: 8. | |
| input_is_latent (bool): Whether input is latent style. Default: False. | |
| different_w (bool): Whether to use different latent w for different layers. Default: False. | |
| narrow (float): The narrow ratio for channels. Default: 1. | |
| sft_half (bool): Whether to apply SFT on half of the input channels. Default: False. | |
| """ | |
| def __init__( | |
| self, | |
| out_size, | |
| num_style_feat=512, | |
| channel_multiplier=1, | |
| decoder_load_path=None, | |
| fix_decoder=True, | |
| # for stylegan decoder | |
| num_mlp=8, | |
| input_is_latent=False, | |
| different_w=False, | |
| narrow=1, | |
| sft_half=False): | |
| super(GFPGANv1Clean, self).__init__() | |
| self.input_is_latent = input_is_latent | |
| self.different_w = different_w | |
| self.num_style_feat = num_style_feat | |
| unet_narrow = narrow * 0.5 # by default, use a half of input channels | |
| channels = { | |
| '4': int(512 * unet_narrow), | |
| '8': int(512 * unet_narrow), | |
| '16': int(512 * unet_narrow), | |
| '32': int(512 * unet_narrow), | |
| '64': int(256 * channel_multiplier * unet_narrow), | |
| '128': int(128 * channel_multiplier * unet_narrow), | |
| '256': int(64 * channel_multiplier * unet_narrow), | |
| '512': int(32 * channel_multiplier * unet_narrow), | |
| '1024': int(16 * channel_multiplier * unet_narrow) | |
| } | |
| self.log_size = int(math.log(out_size, 2)) | |
| first_out_size = 2**(int(math.log(out_size, 2))) | |
| self.conv_body_first = nn.Conv2d(3, channels[f'{first_out_size}'], 1) | |
| # downsample | |
| in_channels = channels[f'{first_out_size}'] | |
| self.conv_body_down = nn.ModuleList() | |
| for i in range(self.log_size, 2, -1): | |
| out_channels = channels[f'{2**(i - 1)}'] | |
| self.conv_body_down.append(ResBlock(in_channels, out_channels, mode='down')) | |
| in_channels = out_channels | |
| self.final_conv = nn.Conv2d(in_channels, channels['4'], 3, 1, 1) | |
| # upsample | |
| in_channels = channels['4'] | |
| self.conv_body_up = nn.ModuleList() | |
| for i in range(3, self.log_size + 1): | |
| out_channels = channels[f'{2**i}'] | |
| self.conv_body_up.append(ResBlock(in_channels, out_channels, mode='up')) | |
| in_channels = out_channels | |
| # to RGB | |
| self.toRGB = nn.ModuleList() | |
| for i in range(3, self.log_size + 1): | |
| self.toRGB.append(nn.Conv2d(channels[f'{2**i}'], 3, 1)) | |
| if different_w: | |
| linear_out_channel = (int(math.log(out_size, 2)) * 2 - 2) * num_style_feat | |
| else: | |
| linear_out_channel = num_style_feat | |
| self.final_linear = nn.Linear(channels['4'] * 4 * 4, linear_out_channel) | |
| # the decoder: stylegan2 generator with SFT modulations | |
| self.stylegan_decoder = StyleGAN2GeneratorCSFT( | |
| out_size=out_size, | |
| num_style_feat=num_style_feat, | |
| num_mlp=num_mlp, | |
| channel_multiplier=channel_multiplier, | |
| narrow=narrow, | |
| sft_half=sft_half) | |
| # load pre-trained stylegan2 model if necessary | |
| if decoder_load_path: | |
| self.stylegan_decoder.load_state_dict( | |
| torch.load(decoder_load_path, map_location=lambda storage, loc: storage)['params_ema']) | |
| # fix decoder without updating params | |
| if fix_decoder: | |
| for _, param in self.stylegan_decoder.named_parameters(): | |
| param.requires_grad = False | |
| # for SFT modulations (scale and shift) | |
| self.condition_scale = nn.ModuleList() | |
| self.condition_shift = nn.ModuleList() | |
| for i in range(3, self.log_size + 1): | |
| out_channels = channels[f'{2**i}'] | |
| if sft_half: | |
| sft_out_channels = out_channels | |
| else: | |
| sft_out_channels = out_channels * 2 | |
| self.condition_scale.append( | |
| nn.Sequential( | |
| nn.Conv2d(out_channels, out_channels, 3, 1, 1), nn.LeakyReLU(0.2, True), | |
| nn.Conv2d(out_channels, sft_out_channels, 3, 1, 1))) | |
| self.condition_shift.append( | |
| nn.Sequential( | |
| nn.Conv2d(out_channels, out_channels, 3, 1, 1), nn.LeakyReLU(0.2, True), | |
| nn.Conv2d(out_channels, sft_out_channels, 3, 1, 1))) | |
| def forward(self, x, return_latents=False, return_rgb=True, randomize_noise=True): | |
| """Forward function for GFPGANv1Clean. | |
| Args: | |
| x (Tensor): Input images. | |
| return_latents (bool): Whether to return style latents. Default: False. | |
| return_rgb (bool): Whether return intermediate rgb images. Default: True. | |
| randomize_noise (bool): Randomize noise, used when 'noise' is False. Default: True. | |
| """ | |
| conditions = [] | |
| unet_skips = [] | |
| out_rgbs = [] | |
| # encoder | |
| feat = F.leaky_relu_(self.conv_body_first(x), negative_slope=0.2) | |
| for i in range(self.log_size - 2): | |
| feat = self.conv_body_down[i](feat) | |
| unet_skips.insert(0, feat) | |
| feat = F.leaky_relu_(self.final_conv(feat), negative_slope=0.2) | |
| # style code | |
| style_code = self.final_linear(feat.view(feat.size(0), -1)) | |
| if self.different_w: | |
| style_code = style_code.view(style_code.size(0), -1, self.num_style_feat) | |
| # decode | |
| for i in range(self.log_size - 2): | |
| # add unet skip | |
| feat = feat + unet_skips[i] | |
| # ResUpLayer | |
| feat = self.conv_body_up[i](feat) | |
| # generate scale and shift for SFT layers | |
| scale = self.condition_scale[i](feat) | |
| conditions.append(scale.clone()) | |
| shift = self.condition_shift[i](feat) | |
| conditions.append(shift.clone()) | |
| # generate rgb images | |
| if return_rgb: | |
| out_rgbs.append(self.toRGB[i](feat)) | |
| # decoder | |
| image, _ = self.stylegan_decoder([style_code], | |
| conditions, | |
| return_latents=return_latents, | |
| input_is_latent=self.input_is_latent, | |
| randomize_noise=randomize_noise) | |
| return image, out_rgbs | |