# -------------------------------------------------------- # Reversible Column Networks # Copyright (c) 2022 Megvii Inc. # Licensed under The Apache License 2.0 [see LICENSE for details] # Written by Yuxuan Cai # -------------------------------------------------------- import imp import torch import torch.nn as nn import torch.nn.functional as F from timm.models.layers import DropPath class UpSampleConvnext(nn.Module): def __init__(self, ratio, inchannel, outchannel): super().__init__() self.ratio = ratio self.channel_reschedule = nn.Sequential( # LayerNorm(inchannel, eps=1e-6, data_format="channels_last"), nn.Linear(inchannel, outchannel), LayerNorm(outchannel, eps=1e-6, data_format="channels_last")) self.upsample = nn.Upsample(scale_factor=2**ratio, mode='nearest') def forward(self, x): x = x.permute(0, 2, 3, 1) x = self.channel_reschedule(x) x = x = x.permute(0, 3, 1, 2) return self.upsample(x) class LayerNorm(nn.Module): r""" LayerNorm that supports two data formats: channels_last (default) or channels_first. The ordering of the dimensions in the inputs. channels_last corresponds to inputs with shape (batch_size, height, width, channels) while channels_first corresponds to inputs with shape (batch_size, channels, height, width). """ def __init__(self, normalized_shape, eps=1e-6, data_format="channels_first", elementwise_affine = True): super().__init__() self.elementwise_affine = elementwise_affine if elementwise_affine: self.weight = nn.Parameter(torch.ones(normalized_shape)) self.bias = nn.Parameter(torch.zeros(normalized_shape)) self.eps = eps self.data_format = data_format if self.data_format not in ["channels_last", "channels_first"]: raise NotImplementedError self.normalized_shape = (normalized_shape, ) def forward(self, x): if self.data_format == "channels_last": return F.layer_norm(x, self.normalized_shape, self.weight, self.bias, self.eps) elif self.data_format == "channels_first": u = x.mean(1, keepdim=True) s = (x - u).pow(2).mean(1, keepdim=True) x = (x - u) / torch.sqrt(s + self.eps) if self.elementwise_affine: x = self.weight[:, None, None] * x + self.bias[:, None, None] return x class ConvNextBlock(nn.Module): r""" ConvNeXt Block. There are two equivalent implementations: (1) DwConv -> LayerNorm (channels_first) -> 1x1 Conv -> GELU -> 1x1 Conv; all in (N, C, H, W) (2) DwConv -> Permute to (N, H, W, C); LayerNorm (channels_last) -> Linear -> GELU -> Linear; Permute back We use (2) as we find it slightly faster in PyTorch Args: dim (int): Number of input channels. drop_path (float): Stochastic depth rate. Default: 0.0 layer_scale_init_value (float): Init value for Layer Scale. Default: 1e-6. """ def __init__(self, in_channel, hidden_dim, out_channel, kernel_size=3, layer_scale_init_value=1e-6, drop_path= 0.0): super().__init__() self.dwconv = nn.Conv2d(in_channel, in_channel, kernel_size=kernel_size, padding=(kernel_size - 1) // 2, groups=in_channel) # depthwise conv self.norm = nn.LayerNorm(in_channel, eps=1e-6) self.pwconv1 = nn.Linear(in_channel, hidden_dim) # pointwise/1x1 convs, implemented with linear layers self.act = nn.GELU() self.pwconv2 = nn.Linear(hidden_dim, out_channel) self.gamma = nn.Parameter(layer_scale_init_value * torch.ones((out_channel)), requires_grad=True) if layer_scale_init_value > 0 else None self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() def forward(self, x): input = x x = self.dwconv(x) x = x.permute(0, 2, 3, 1) # (N, C, H, W) -> (N, H, W, C) x = self.norm(x) x = self.pwconv1(x) x = self.act(x) # print(f"x min: {x.min()}, x max: {x.max()}, input min: {input.min()}, input max: {input.max()}, x mean: {x.mean()}, x var: {x.var()}, ratio: {torch.sum(x>8)/x.numel()}") x = self.pwconv2(x) if self.gamma is not None: x = self.gamma * x x = x.permute(0, 3, 1, 2) # (N, H, W, C) -> (N, C, H, W) x = input + self.drop_path(x) return x class Decoder(nn.Module): def __init__(self, depth=[2,2,2,2], dim=[112, 72, 40, 24], block_type = None, kernel_size = 3) -> None: super().__init__() self.depth = depth self.dim = dim self.block_type = block_type self._build_decode_layer(dim, depth, kernel_size) self.projback = nn.Sequential( nn.Conv2d( in_channels=dim[-1], out_channels=4 ** 2 * 3, kernel_size=1), nn.PixelShuffle(4), ) def _build_decode_layer(self, dim, depth, kernel_size): normal_layers = nn.ModuleList() upsample_layers = nn.ModuleList() proj_layers = nn.ModuleList() norm_layer = LayerNorm for i in range(1, len(dim)): module = [self.block_type(dim[i], dim[i], dim[i], kernel_size) for _ in range(depth[i])] normal_layers.append(nn.Sequential(*module)) upsample_layers.append(nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)) proj_layers.append(nn.Sequential( nn.Conv2d(dim[i-1], dim[i], 1, 1), norm_layer(dim[i]), nn.GELU() )) self.normal_layers = normal_layers self.upsample_layers = upsample_layers self.proj_layers = proj_layers def _forward_stage(self, stage, x): x = self.proj_layers[stage](x) x = self.upsample_layers[stage](x) return self.normal_layers[stage](x) def forward(self, c3): x = self._forward_stage(0, c3) #14 x = self._forward_stage(1, x) #28 x = self._forward_stage(2, x) #56 x = self.projback(x) return x class SimDecoder(nn.Module): def __init__(self, in_channel, encoder_stride) -> None: super().__init__() self.projback = nn.Sequential( LayerNorm(in_channel), nn.Conv2d( in_channels=in_channel, out_channels=encoder_stride ** 2 * 3, kernel_size=1), nn.PixelShuffle(encoder_stride), ) def forward(self, c3): return self.projback(c3)