jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def __init__(self, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=0, p_dropout=0):
        super(WN, self).__init__()
        assert (kernel_size % 2 == 1)
        self.hidden_channels = hidden_channels
        self.kernel_size = kernel_size,
        self.dilation_rate = dilation_rate
        self.n_layers = n_layers
        self.gin_channels = gin_channels
        self.p_dropout = p_dropout

        self.in_layers = torch.nn.ModuleList()
        self.res_skip_layers = torch.nn.ModuleList()
        self.drop = nn.Dropout(p_dropout)

        if gin_channels != 0:
            cond_layer = torch.nn.Conv1d(
                gin_channels, 2*hidden_channels*n_layers, 1)
            self.cond_layer = torch.nn.utils.parametrizations.weight_norm(
                cond_layer, name='weight')

        for i in range(n_layers):
            dilation = dilation_rate ** i
            padding = int((kernel_size * dilation - dilation) / 2)
            in_layer = torch.nn.Conv1d(hidden_channels, 2*hidden_channels, kernel_size,
                                       dilation=dilation, padding=padding)
            in_layer = torch.nn.utils.parametrizations.weight_norm(in_layer, name='weight')
            self.in_layers.append(in_layer)

            # last one is not necessary
            if i < n_layers - 1:
                res_skip_channels = 2 * hidden_channels
            else:
                res_skip_channels = hidden_channels

            res_skip_layer = torch.nn.Conv1d(
                hidden_channels, res_skip_channels, 1)
            res_skip_layer = torch.nn.utils.parametrizations.weight_norm(
                res_skip_layer, name='weight')
            self.res_skip_layers.append(res_skip_layer)
//  [python]
def __init__(
            self,
            in_channels,
            cond_channels,
            stride,
            dilations=[1, 3, 9, 27],
            lReLU_slope=0.2,
            conv_kernel_size=3,
            cond_hop_length=256,
            kpnet_hidden_channels=64,
            kpnet_conv_size=3,
            kpnet_dropout=0.0,
        ):
        super().__init__()

        self.cond_hop_length = cond_hop_length
        self.conv_layers = len(dilations)
        self.conv_kernel_size = conv_kernel_size

        self.kernel_predictor = KernelPredictor(
            cond_channels=cond_channels,
            conv_in_channels=in_channels,
            conv_out_channels=2 * in_channels, 
            conv_layers=len(dilations),
            conv_kernel_size=conv_kernel_size,
            kpnet_hidden_channels=kpnet_hidden_channels,
            kpnet_conv_size=kpnet_conv_size,
            kpnet_dropout=kpnet_dropout,
            kpnet_nonlinear_activation_params={"negative_slope":lReLU_slope}
        )
        
        self.convt_pre = nn.Sequential(
            nn.LeakyReLU(lReLU_slope),
            nn.utils.weight_norm(nn.ConvTranspose1d(in_channels, in_channels, 2 * stride, stride=stride, padding=stride // 2 + stride % 2, output_padding=stride % 2)),
        )
        
        self.conv_blocks = nn.ModuleList()
        for dilation in dilations:
            self.conv_blocks.append(
                nn.Sequential(
                    nn.LeakyReLU(lReLU_slope),
                    nn.utils.weight_norm(nn.Conv1d(in_channels, in_channels, conv_kernel_size, padding=dilation * (conv_kernel_size - 1) // 2, dilation=dilation)),
                    nn.LeakyReLU(lReLU_slope),
                )
            )
//  [python]
def __init__(
            self,
            in_channels,
            cond_channels,
            stride,
            dilations=[1, 3, 9, 27],
            lReLU_slope=0.2,
            conv_kernel_size=3,
            cond_hop_length=256,
            kpnet_hidden_channels=64,
            kpnet_conv_size=3,
            kpnet_dropout=0.0,
        ):
        super().__init__()

        self.cond_hop_length = cond_hop_length
        self.conv_layers = len(dilations)
        self.conv_kernel_size = conv_kernel_size

        self.kernel_predictor = KernelPredictor(
            cond_channels=cond_channels,
            conv_in_channels=in_channels,
            conv_out_channels=2 * in_channels, 
            conv_layers=len(dilations),
            conv_kernel_size=conv_kernel_size,
            kpnet_hidden_channels=kpnet_hidden_channels,
            kpnet_conv_size=kpnet_conv_size,
            kpnet_dropout=kpnet_dropout,
            kpnet_nonlinear_activation_params={"negative_slope":lReLU_slope}
        )
        
        self.convt_pre = nn.Sequential(
            nn.LeakyReLU(lReLU_slope),
            nn.utils.parametrizations.weight_norm(nn.ConvTranspose1d(in_channels, in_channels, 2 * stride, stride=stride, padding=stride // 2 + stride % 2, output_padding=stride % 2)),
        )
        
        self.conv_blocks = nn.ModuleList()
        for dilation in dilations:
            self.conv_blocks.append(
                nn.Sequential(
                    nn.LeakyReLU(lReLU_slope),
                    nn.utils.parametrizations.weight_norm(nn.Conv1d(in_channels, in_channels, conv_kernel_size, padding=dilation * (conv_kernel_size - 1) // 2, dilation=dilation)),
                    nn.LeakyReLU(lReLU_slope),
                )
            )
//  [python]
def __init__(self, inter_channel, gen, gin_channels=0):
        super(Generator, self).__init__()
        self.mel_channel = inter_channel
        self.noise_dim = gen.noise_dim
        self.hop_length = gen.hop_length
        channel_size = gen.channel_size
        kpnet_conv_size = gen.kpnet_conv_size
        gin_channels = gin_channels

        self.res_stack = nn.ModuleList()
        hop_length = 1
        for stride in gen.strides:
            hop_length = stride * hop_length
            self.res_stack.append(
                LVCBlock(
                    channel_size,
                    inter_channel,
                    stride=stride,
                    dilations=gen.dilations,
                    lReLU_slope=gen.lReLU_slope,
                    cond_hop_length=hop_length,
                    kpnet_conv_size=kpnet_conv_size
                )
            )
        
        self.conv_pre = \
            nn.utils.weight_norm(nn.Conv1d(gen.noise_dim, channel_size, 7, padding=3, padding_mode='reflect'))

        self.conv_post = nn.Sequential(
            nn.LeakyReLU(gen.lReLU_slope),
            nn.utils.weight_norm(nn.Conv1d(channel_size, 1, 7, padding=3, padding_mode='reflect')),
            nn.Tanh(),
        )

        if gin_channels != 0:
          self.cond = nn.Conv1d(gin_channels, self.mel_channel, 1)
//  [python]
def __init__(self, inter_channel, gen, gin_channels=0):
        super(Generator, self).__init__()
        self.mel_channel = inter_channel
        self.noise_dim = gen.noise_dim
        self.hop_length = gen.hop_length
        channel_size = gen.channel_size
        kpnet_conv_size = gen.kpnet_conv_size
        gin_channels = gin_channels

        self.res_stack = nn.ModuleList()
        hop_length = 1
        for stride in gen.strides:
            hop_length = stride * hop_length
            self.res_stack.append(
                LVCBlock(
                    channel_size,
                    inter_channel,
                    stride=stride,
                    dilations=gen.dilations,
                    lReLU_slope=gen.lReLU_slope,
                    cond_hop_length=hop_length,
                    kpnet_conv_size=kpnet_conv_size
                )
            )
        
        self.conv_pre = \
            nn.utils.parametrizations.weight_norm(nn.Conv1d(gen.noise_dim, channel_size, 7, padding=3, padding_mode='reflect'))

        self.conv_post = nn.Sequential(
            nn.LeakyReLU(gen.lReLU_slope),
            nn.utils.parametrizations.weight_norm(nn.Conv1d(channel_size, 1, 7, padding=3, padding_mode='reflect')),
            nn.Tanh(),
        )

        if gin_channels != 0:
          self.cond = nn.Conv1d(gin_channels, self.mel_channel, 1)
//  [python]
def test_weightnorm(self):
        for i in range(30):
            weight = np.random.uniform(0,1,(i+10,40))
            jm = jt_module(weight)
            tm = torch_module(weight)
            inp = np.random.uniform(0,1,(40,i+30))
            torch.nn.utils.weight_norm(tm, 'linear', -1)
            weight_norm(jm, 'linear', -1)
            jinp = jt.array(inp)
            tinp = Variable(torch.from_numpy(inp), requires_grad=True)
            joup = jm(jinp)
            toup = tm(tinp)
            np.testing.assert_allclose(joup.data, toup.detach().numpy(), rtol=1e-4, atol=1e-6)
            gq = jt.grad(joup, jinp).data
            tgq = torch.autograd.grad(toup, tinp, torch.ones_like(toup), retain_graph=True)
            np.testing.assert_allclose(gq, tgq[0].numpy(), rtol=1e-4, atol=1e-6)
//  [python]
def test_weightnorm(self):
        for i in range(30):
            weight = np.random.uniform(0,1,(i+10,40))
            jm = jt_module(weight)
            tm = torch_module(weight)
            inp = np.random.uniform(0,1,(40,i+30))
            torch.nn.utils.parametrizations.weight_norm(tm, 'linear', -1)
            weight_norm(jm, 'linear', -1)
            jinp = jt.array(inp)
            tinp = Variable(torch.from_numpy(inp), requires_grad=True)
            joup = jm(jinp)
            toup = tm(tinp)
            np.testing.assert_allclose(joup.data, toup.detach().numpy(), rtol=1e-4, atol=1e-6)
            gq = jt.grad(joup, jinp).data
            tgq = torch.autograd.grad(toup, tinp, torch.ones_like(toup), retain_graph=True)
            np.testing.assert_allclose(gq, tgq[0].numpy(), rtol=1e-4, atol=1e-6)
//  [python]
def __init__(self, h):
        super(Generator_hifi, self).__init__()
        self.h = h
        self.num_kernels = len(h.resblock_kernel_sizes)
        self.num_upsamples = len(h.upsample_rates)
        self.conv_pre = weight_norm(Conv1d(192, h.upsample_initial_channel, 7, 1, padding=3))
        if h.resblock == '1':
            resblock = ResBlock1
        elif h.resblock == '2':
            resblock = ResBlock2
        else:
            resblock = ResBlock3

        self.ups = nn.ModuleList()
        for i, (u, k) in enumerate(zip(h.upsample_rates, h.upsample_kernel_sizes)):
            self.ups.append(weight_norm(
                ConvTranspose1d(h.upsample_initial_channel//(2**i), h.upsample_initial_channel//(2**(i+1)),
                                k, u, padding=(u//2 + u%2), output_padding=u%2)))

        self.resblocks = nn.ModuleList()
        for i in range(len(self.ups)):
            ch = h.upsample_initial_channel//(2**(i+1))
            for j, (k, d) in enumerate(zip(h.resblock_kernel_sizes, h.resblock_dilation_sizes)):
                self.resblocks.append(resblock(h, ch, k, d))

        self.conv_post = weight_norm(Conv1d(ch, 4, 7, 1, padding=3))
        self.ups.apply(init_weights)
        self.conv_post.apply(init_weights)
        self.upsample_rates = h.upsample_rates
        self.pqmf = PQMF()
//  [python]
def __init__(self, h):
        super(Generator_hifi, self).__init__()
        self.h = h
        self.num_kernels = len(h.resblock_kernel_sizes)
        self.num_upsamples = len(h.upsample_rates)
        self.conv_pre = nn.utils.parametrizations.weight_norm(Conv1d(192, h.upsample_initial_channel, 7, 1, padding=3))
        if h.resblock == '1':
            resblock = ResBlock1
        elif h.resblock == '2':
            resblock = ResBlock2
        else:
            resblock = ResBlock3

        self.ups = nn.ModuleList()
        for i, (u, k) in enumerate(zip(h.upsample_rates, h.upsample_kernel_sizes)):
            self.ups.append(nn.utils.parametrizations.weight_norm(
                ConvTranspose1d(h.upsample_initial_channel//(2**i), h.upsample_initial_channel//(2**(i+1)),
                                k, u, padding=(u//2 + u%2), output_padding=u%2)))

        self.resblocks = nn.ModuleList()
        for i in range(len(self.ups)):
            ch = h.upsample_initial_channel//(2**(i+1))
            for j, (k, d) in enumerate(zip(h.resblock_kernel_sizes, h.resblock_dilation_sizes)):
                self.resblocks.append(resblock(h, ch, k, d))

        self.conv_post = nn.utils.parametrizations.weight_norm(Conv1d(ch, 4, 7, 1, padding=3))
        self.ups.apply(init_weights)
        self.conv_post.apply(init_weights)
        self.upsample_rates = h.upsample_rates
        self.pqmf = PQMF()
//  [python]
def __init__(self, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=0, p_dropout=0):
        super(WN, self).__init__()
        assert(kernel_size % 2 == 1)
        self.hidden_channels =hidden_channels
        self.kernel_size = kernel_size,
        self.dilation_rate = dilation_rate
        self.n_layers = n_layers
        self.gin_channels = gin_channels
        self.p_dropout = p_dropout

        self.in_layers = nn.ModuleList()
        self.res_skip_layers = nn.ModuleList()
        self.drop = nn.Dropout(p_dropout)

        if gin_channels != 0:
            cond_layer = nn.Conv1d(gin_channels, 2*hidden_channels*n_layers, 1)
            self.cond_layer = nn.utils.weight_norm(cond_layer, name='weight')

        for i in range(n_layers):
            dilation = dilation_rate ** i
            padding = int((kernel_size * dilation - dilation) / 2)
            in_layer = nn.Conv1d(hidden_channels, 2*hidden_channels, kernel_size,
                                     dilation=dilation, padding=padding)
            in_layer = nn.utils.weight_norm(in_layer, name='weight')
            self.in_layers.append(in_layer)

            # last one is not necessary
            if i < n_layers - 1:
                res_skip_channels = 2 * hidden_channels
            else:
                res_skip_channels = hidden_channels

            res_skip_layer = nn.Conv1d(hidden_channels, res_skip_channels, 1)
            res_skip_layer = nn.utils.weight_norm(res_skip_layer, name='weight')
            self.res_skip_layers.append(res_skip_layer)
//  [python]
def __init__(self, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=0, p_dropout=0):
        super(WN, self).__init__()
        assert(kernel_size % 2 == 1)
        self.hidden_channels =hidden_channels
        self.kernel_size = kernel_size,
        self.dilation_rate = dilation_rate
        self.n_layers = n_layers
        self.gin_channels = gin_channels
        self.p_dropout = p_dropout

        self.in_layers = nn.ModuleList()
        self.res_skip_layers = nn.ModuleList()
        self.drop = nn.Dropout(p_dropout)

        if gin_channels != 0:
            cond_layer = nn.Conv1d(gin_channels, 2*hidden_channels*n_layers, 1)
            self.cond_layer = nn.utils.parametrizations.weight_norm(cond_layer, name='weight')

        for i in range(n_layers):
            dilation = dilation_rate ** i
            padding = int((kernel_size * dilation - dilation) / 2)
            in_layer = nn.Conv1d(hidden_channels, 2*hidden_channels, kernel_size,
                                     dilation=dilation, padding=padding)
            in_layer = nn.utils.parametrizations.weight_norm(in_layer, name='weight')
            self.in_layers.append(in_layer)

            # last one is not necessary
            if i < n_layers - 1:
                res_skip_channels = 2 * hidden_channels
            else:
                res_skip_channels = hidden_channels

            res_skip_layer = nn.Conv1d(hidden_channels, res_skip_channels, 1)
            res_skip_layer = nn.utils.parametrizations.weight_norm(res_skip_layer, name='weight')
            self.res_skip_layers.append(res_skip_layer)
//  [python]
def __init__(self, cuda=True, scale=4, n_res=8, n_feats=64, block_feats=8,
                 res_scale=1, n_colors=3, kernel_size=3,
                 mean=(99.00332925, 124.7647323, 128.69159715)):
        super(MODEL, self).__init__()
        # hyper-params
        act = nn.ReLU(True)

        def wn(x):
            return torch.nn.utils.weight_norm(x)

        self.mean = torch.FloatTensor(mean).view([1, n_colors, 1, 1])
        if cuda:
            self.mean = self.mean.cuda()

        # define head module
        head = list()
        head.append(wn(nn.Conv2d(n_colors, n_feats, 3, padding=3 // 2)))

        # define body module
        body = list()
        for i in range(n_res):
            body.append(Block(n_feats, kernel_size, block_feats, wn, act, res_scale))

        # define tail module
        tail = list()
        out_feats = scale * scale * n_colors
        tail.append(wn(nn.Conv2d(n_feats, out_feats, 3, padding=3 // 2)))
        tail.append(nn.PixelShuffle(scale))

        # define skip module
        skip = list()
        skip.append(wn(nn.Conv2d(n_colors, out_feats, 5, padding=5 // 2)))
        skip.append(nn.PixelShuffle(scale))

        # make object members
        self.head = nn.Sequential(*head)
        self.body = nn.Sequential(*body)
        self.tail = nn.Sequential(*tail)
        self.skip = nn.Sequential(*skip)

        # 初始化权重，参考EDVR
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                init.xavier_normal_(m.weight, gain=init.calculate_gain('relu'))
                if m.bias is not None:
                    init.normal_(m.bias, 0.0001)
            elif isinstance(m, nn.Linear):
                init.xavier_normal_(m.weight, gain=init.calculate_gain('relu'))
                if m.bias is not None:
                    init.normal_(m.bias, 0.0001)
        return
//  [python]
def __init__(self, cuda=True, scale=4, n_res=8, n_feats=64, block_feats=8,
                 res_scale=1, n_colors=3, kernel_size=3,
                 mean=(99.00332925, 124.7647323, 128.69159715)):
        super(MODEL, self).__init__()
        # hyper-params
        act = nn.ReLU(True)

        def wn(x):
            return torch.nn.utils.parametrizations.weight_norm(x)

        self.mean = torch.FloatTensor(mean).view([1, n_colors, 1, 1])
        if cuda:
            self.mean = self.mean.cuda()

        # define head module
        head = list()
        head.append(wn(nn.Conv2d(n_colors, n_feats, 3, padding=3 // 2)))

        # define body module
        body = list()
        for i in range(n_res):
            body.append(Block(n_feats, kernel_size, block_feats, wn, act, res_scale))

        # define tail module
        tail = list()
        out_feats = scale * scale * n_colors
        tail.append(wn(nn.Conv2d(n_feats, out_feats, 3, padding=3 // 2)))
        tail.append(nn.PixelShuffle(scale))

        # define skip module
        skip = list()
        skip.append(wn(nn.Conv2d(n_colors, out_feats, 5, padding=5 // 2)))
        skip.append(nn.PixelShuffle(scale))

        # make object members
        self.head = nn.Sequential(*head)
        self.body = nn.Sequential(*body)
        self.tail = nn.Sequential(*tail)
        self.skip = nn.Sequential(*skip)

        # 初始化权重，参考EDVR
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                init.xavier_normal_(m.weight, gain=init.calculate_gain('relu'))
                if m.bias is not None:
                    init.normal_(m.bias, 0.0001)
            elif isinstance(m, nn.Linear):
                init.xavier_normal_(m.weight, gain=init.calculate_gain('relu'))
                if m.bias is not None:
                    init.normal_(m.bias, 0.0001)
        return
//  [python]
def __init__(self, kdim, qdim, adim, atype, n_heads, init_r, bias=True, param_init='', conv1d=False, conv_kernel_size=5):
        super().__init__()
        assert conv_kernel_size % 2 == 1, "Kernel size should be odd for 'same' conv."
        self.key = None
        self.mask = None
        self.atype = atype
        assert adim % n_heads == 0
        self.d_k = adim // n_heads
        self.n_heads = n_heads
        self.scale = math.sqrt(adim)
        if atype == 'add':
            self.w_key = nn.Linear(kdim, adim)
            self.v = nn.Linear(adim, n_heads, bias=False)
            self.w_query = nn.Linear(qdim, adim, bias=False)
        elif atype == 'scaled_dot':
            self.w_key = nn.Linear(kdim, adim, bias=bias)
            self.w_query = nn.Linear(qdim, adim, bias=bias)
        else:
            raise NotImplementedError(atype)
        self.r = nn.Parameter(torch.Tensor([init_r]))
        logger.info('init_r is initialized with %d' % init_r)
        self.conv1d = None
        if conv1d:
            self.conv1d = nn.Conv1d(kdim, kdim, conv_kernel_size, padding=(conv_kernel_size - 1) // 2)
            for n, p in self.conv1d.named_parameters():
                init_with_lecun_normal(n, p, 0.1)
        if atype == 'add':
            self.v = nn.utils.weight_norm(self.v, name='weight', dim=0)
            self.v.weight_g.data = torch.Tensor([1 / adim]).sqrt()
        elif atype == 'scaled_dot':
            if param_init == 'xavier_uniform':
                self.reset_parameters_xavier_uniform(bias)
//  [python]
def __init__(self, kdim, qdim, adim, atype, n_heads, init_r, bias=True, param_init='', conv1d=False, conv_kernel_size=5):
        super().__init__()
        assert conv_kernel_size % 2 == 1, "Kernel size should be odd for 'same' conv."
        self.key = None
        self.mask = None
        self.atype = atype
        assert adim % n_heads == 0
        self.d_k = adim // n_heads
        self.n_heads = n_heads
        self.scale = math.sqrt(adim)
        if atype == 'add':
            self.w_key = nn.Linear(kdim, adim)
            self.v = nn.utils.parametrizations.weight_norm(nn.Linear(adim, n_heads, bias=False), dim=0)
            self.w_query = nn.Linear(qdim, adim, bias=False)
        elif atype == 'scaled_dot':
            self.w_key = nn.Linear(kdim, adim, bias=bias)
            self.w_query = nn.Linear(qdim, adim, bias=bias)
        else:
            raise NotImplementedError(atype)
        self.r = nn.Parameter(torch.Tensor([init_r]))
        logger.info('init_r is initialized with %d' % init_r)
        self.conv1d = None
        if conv1d:
            self.conv1d = nn.Conv1d(kdim, kdim, conv_kernel_size, padding=(conv_kernel_size - 1) // 2)
            for n, p in self.conv1d.named_parameters():
                init_with_lecun_normal(n, p, 0.1)
        if atype == 'add':
            self.v.weight_g.data = torch.Tensor([1 / adim]).sqrt()
        elif atype == 'scaled_dot':
            if param_init == 'xavier_uniform':
                self.reset_parameters_xavier_uniform(bias)
//  [python]
def __init__(self, hps, initial_channel, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes, n_speakers=0, spk_channels=0):
        super(Generator, self).__init__()
        self.num_kernels = len(resblock_kernel_sizes)
        self.num_upsamples = len(upsample_rates)
        self.conv_pre = Conv1d(initial_channel, upsample_initial_channel, 7, 1, padding=3)
        self.upsample_rates = upsample_rates
        self.n_speakers = n_speakers

        resblock = modules.ResBlock1 if resblock == '1' else modules.R
        
        self.downs = nn.ModuleList()
        for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
            i = len(upsample_rates) - 1 - i
            u = upsample_rates[i]
            k = upsample_kernel_sizes[i]
            #print("down: ",upsample_initial_channel//(2**(i+1))," -> ", upsample_initial_channel//(2**i))
            self.downs.append(weight_norm(
                Conv1d(hps.model.n_harmonic + 2, hps.model.n_harmonic + 2,
                       k, u, padding=k//2)))
        
        self.resblocks_downs = nn.ModuleList()
        for i in range(len(self.downs)):
            j = len(upsample_rates) - 1 - i
            self.resblocks_downs.append(ResBlock3(hps.model.n_harmonic + 2, 3, (1, 3)))

        
        self.concat_pre = Conv1d(upsample_initial_channel + hps.model.n_harmonic + 2, upsample_initial_channel, 3, 1, padding=1)
        self.concat_conv = nn.ModuleList()
        for i in range(len(upsample_rates)):
          ch = upsample_initial_channel//(2**(i+1))
          self.concat_conv.append(Conv1d(ch + hps.model.n_harmonic + 2, ch, 3, 1, padding=1, bias=False))


        self.ups = nn.ModuleList()
        for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
            self.ups.append(weight_norm(
                ConvTranspose1d(upsample_initial_channel//(2**i), upsample_initial_channel//(2**(i+1)),
                                k, u, padding=(k-u)//2)))

        self.resblocks = nn.ModuleList()
        for i in range(len(self.ups)):
            ch = upsample_initial_channel//(2**(i+1))
            for j, (k, d) in enumerate(zip(resblock_kernel_sizes, resblock_dilation_sizes)):
                self.resblocks.append(resblock(ch, k, d))

        self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False)
        self.ups.apply(init_weights)

        if self.n_speakers != 0:
            self.cond = nn.Conv1d(spk_channels, upsample_initial_channel, 1)
//  [python]
def __init__(self, hps, initial_channel, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes, n_speakers=0, spk_channels=0):
        super(Generator, self).__init__()
        self.num_kernels = len(resblock_kernel_sizes)
        self.num_upsamples = len(upsample_rates)
        self.conv_pre = Conv1d(initial_channel, upsample_initial_channel, 7, 1, padding=3)
        self.upsample_rates = upsample_rates
        self.n_speakers = n_speakers

        resblock = modules.ResBlock1 if resblock == '1' else modules.R
        
        self.downs = nn.ModuleList()
        for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
            i = len(upsample_rates) - 1 - i
            u = upsample_rates[i]
            k = upsample_kernel_sizes[i]
            #print("down: ",upsample_initial_channel//(2**(i+1))," -> ", upsample_initial_channel//(2**i))
            self.downs.append(nn.utils.parametrizations.weight_norm(
                Conv1d(hps.model.n_harmonic + 2, hps.model.n_harmonic + 2,
                       k, u, padding=k//2)))
        
        self.resblocks_downs = nn.ModuleList()
        for i in range(len(self.downs)):
            j = len(upsample_rates) - 1 - i
            self.resblocks_downs.append(ResBlock3(hps.model.n_harmonic + 2, 3, (1, 3)))

        
        self.concat_pre = Conv1d(upsample_initial_channel + hps.model.n_harmonic + 2, upsample_initial_channel, 3, 1, padding=1)
        self.concat_conv = nn.ModuleList()
        for i in range(len(upsample_rates)):
          ch = upsample_initial_channel//(2**(i+1))
          self.concat_conv.append(Conv1d(ch + hps.model.n_harmonic + 2, ch, 3, 1, padding=1, bias=False))


        self.ups = nn.ModuleList()
        for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
            self.ups.append(nn.utils.parametrizations.weight_norm(
                ConvTranspose1d(upsample_initial_channel//(2**i), upsample_initial_channel//(2**(i+1)),
                                k, u, padding=(k-u)//2)))

        self.resblocks = nn.ModuleList()
        for i in range(len(self.ups)):
            ch = upsample_initial_channel//(2**(i+1))
            for j, (k, d) in enumerate(zip(resblock_kernel_sizes, resblock_dilation_sizes)):
                self.resblocks.append(resblock(ch, k, d))

        self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False)
        self.ups.apply(init_weights)

        if self.n_speakers != 0:
            self.cond = nn.Conv1d(spk_channels, upsample_initial_channel, 1)
//  [python]
def __init__(self, input_dim, in_channel, channels, kernel_sizes, dropout, last_proj_dim, param_init):
        super(GatedConvEncoder, self).__init__()
        (channels, kernel_sizes, _, _), _ = parse_cnn_config(channels, kernel_sizes, '', '')
        self.in_channel = in_channel
        assert input_dim % in_channel == 0
        self.input_freq = input_dim // in_channel
        self.bridge = None
        assert len(channels) > 0
        assert len(channels) == len(kernel_sizes)
        layers = OrderedDict()
        for lth in range(len(channels)):
            layers['conv%d' % lth] = ConvGLUBlock(kernel_sizes[lth][0], input_dim, channels[lth], weight_norm=True, dropout=0.2)
            input_dim = channels[lth]
        self.fc_glu = nn.utils.weight_norm(nn.Linear(input_dim, input_dim * 2), name='weight', dim=0)
        self._odim = int(input_dim)
        if last_proj_dim > 0:
            self.bridge = nn.Linear(self._odim, last_proj_dim)
            self._odim = last_proj_dim
        self.layers = nn.Sequential(layers)
        self._factor = 1
        self.reset_parameters(param_init)
//  [python]
def __init__(self, input_dim, in_channel, channels, kernel_sizes, dropout, last_proj_dim, param_init):
        super(GatedConvEncoder, self).__init__()
        (channels, kernel_sizes, _, _), _ = parse_cnn_config(channels, kernel_sizes, '', '')
        self.in_channel = in_channel
        assert input_dim % in_channel == 0
        self.input_freq = input_dim // in_channel
        self.bridge = None
        assert len(channels) > 0
        assert len(channels) == len(kernel_sizes)
        layers = OrderedDict()
        for lth in range(len(channels)):
            layers['conv%d' % lth] = ConvGLUBlock(kernel_sizes[lth][0], input_dim, channels[lth], weight_norm=True, dropout=0.2)
            input_dim = channels[lth]
        self.fc_glu = nn.utils.parametrizations.weight_norm(nn.Linear(input_dim, input_dim * 2), name='weight', dim=0)
        self._odim = int(input_dim)
        if last_proj_dim > 0:
            self.bridge = nn.Linear(self._odim, last_proj_dim)
            self._odim = last_proj_dim
        self.layers = nn.Sequential(layers)
        self._factor = 1
        self.reset_parameters(param_init)
//  [python]
def test_weight_norm():
    """Test for atten::_weight_norm"""
    in_channels = 32
    out_channels = 64
    input_data_conv = torch.rand((1, in_channels, 32, 32)).float()

    conv_wn = torch.nn.utils.weight_norm(torch.nn.Conv2d(in_channels, out_channels, kernel_size=3))
    verify_model(conv_wn.eval().float(), input_data_conv)

    conv_wn_groups = torch.nn.utils.weight_norm(
        torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, groups=2)
    )
    verify_model(conv_wn_groups.eval().float(), input_data_conv)

    conv_wn = torch.nn.utils.weight_norm(
        torch.nn.Conv2d(in_channels, out_channels, kernel_size=3), dim=1
    )
    verify_model(conv_wn.eval().float(), input_data_conv)

    linear_wn = torch.nn.utils.weight_norm(torch.nn.Linear(in_channels, out_channels))
    input_data_linear = torch.rand((128, in_channels)).float()
    verify_model(linear_wn.eval().float(), input_data_linear)
//  [python]
def test_weight_norm():
    """Test for atten::_weight_norm"""
    in_channels = 32
    out_channels = 64
    input_data_conv = torch.rand((1, in_channels, 32, 32)).float()

    conv_wn = torch.nn.utils.parametrizations.weight_norm(torch.nn.Conv2d(in_channels, out_channels, kernel_size=3))
    verify_model(conv_wn.eval().float(), input_data_conv)

    conv_wn_groups = torch.nn.utils.parametrizations.weight_norm(
        torch.nn.Conv2d(in_channels, out_channels, kernel_size=3, groups=2)
    )
    verify_model(conv_wn_groups.eval().float(), input_data_conv)

    conv_wn = torch.nn.utils.parametrizations.weight_norm(
        torch.nn.Conv2d(in_channels, out_channels, kernel_size=3), dim=1
    )
    verify_model(conv_wn.eval().float(), input_data_conv)

    linear_wn = torch.nn.utils.parametrizations.weight_norm(torch.nn.Linear(in_channels, out_channels))
    input_data_linear = torch.rand((128, in_channels)).float()
    verify_model(linear_wn.eval().float(), input_data_linear)
//  [python]
def test_weight_norm_nodim(self):
        model = torch.nn.utils.weight_norm(torch.nn.Linear(5, 10), dim=None)
        x = torch.randn(3, 4, 5, requires_grad=True)
        self.run_test(model, x)
//  [python]
def test_weight_norm_nodim(self):
        model = torch.nn.utils.parametrizations.weight_norm(torch.nn.Linear(5, 10), dim=None)
        x = torch.randn(3, 4, 5, requires_grad=True)
        self.run_test(model, x)
//  [python]
def __init__(self, kernel_size, in_ch, out_ch, bottlececk_dim=0, dropout=0.0):
        super().__init__()
        self.conv_residual = None
        if in_ch != out_ch:
            self.conv_residual = nn.utils.weight_norm(nn.Conv2d(in_channels=in_ch, out_channels=out_ch, kernel_size=(1, 1)), name='weight', dim=0)
            self.dropout_residual = nn.Dropout(p=dropout)
        self.pad_left = nn.ConstantPad2d((0, 0, kernel_size - 1, 0), 0)
        layers = OrderedDict()
        if bottlececk_dim == 0:
            layers['conv'] = nn.utils.weight_norm(nn.Conv2d(in_channels=in_ch, out_channels=out_ch * 2, kernel_size=(kernel_size, 1)), name='weight', dim=0)
            layers['dropout'] = nn.Dropout(p=dropout)
            layers['glu'] = nn.GLU()
        elif bottlececk_dim > 0:
            layers['conv_in'] = nn.utils.weight_norm(nn.Conv2d(in_channels=in_ch, out_channels=bottlececk_dim, kernel_size=(1, 1)), name='weight', dim=0)
            layers['dropout_in'] = nn.Dropout(p=dropout)
            layers['conv_bottleneck'] = nn.utils.weight_norm(nn.Conv2d(in_channels=bottlececk_dim, out_channels=bottlececk_dim, kernel_size=(kernel_size, 1)), name='weight', dim=0)
            layers['dropout'] = nn.Dropout(p=dropout)
            layers['glu'] = nn.GLU()
            layers['conv_out'] = nn.utils.weight_norm(nn.Conv2d(in_channels=bottlececk_dim, out_channels=out_ch * 2, kernel_size=(1, 1)), name='weight', dim=0)
            layers['dropout_out'] = nn.Dropout(p=dropout)
        self.layers = nn.Sequential(layers)
//  [python]
def __init__(self, kernel_size, in_ch, out_ch, bottlececk_dim=0, dropout=0.0):
        super().__init__()
        self.conv_residual = None
        if in_ch != out_ch:
            self.conv_residual = nn.utils.parametrizations.weight_norm(nn.Conv2d(in_channels=in_ch, out_channels=out_ch, kernel_size=(1, 1)), name='weight', dim=0)
            self.dropout_residual = nn.Dropout(p=dropout)
        self.pad_left = nn.ConstantPad2d((0, 0, kernel_size - 1, 0), 0)
        layers = OrderedDict()
        if bottlececk_dim == 0:
            layers['conv'] = nn.utils.parametrizations.weight_norm(nn.Conv2d(in_channels=in_ch, out_channels=out_ch * 2, kernel_size=(kernel_size, 1)), name='weight', dim=0)
            layers['dropout'] = nn.Dropout(p=dropout)
            layers['glu'] = nn.GLU()
        elif bottlececk_dim > 0:
            layers['conv_in'] = nn.utils.parametrizations.weight_norm(nn.Conv2d(in_channels=in_ch, out_channels=bottlececk_dim, kernel_size=(1, 1)), name='weight', dim=0)
            layers['dropout_in'] = nn.Dropout(p=dropout)
            layers['conv_bottleneck'] = nn.utils.parametrizations.weight_norm(nn.Conv2d(in_channels=bottlececk_dim, out_channels=bottlececk_dim, kernel_size=(kernel_size, 1)), name='weight', dim=0)
            layers['dropout'] = nn.Dropout(p=dropout)
            layers['glu'] = nn.GLU()
            layers['conv_out'] = nn.utils.parametrizations.weight_norm(nn.Conv2d(in_channels=bottlececk_dim, out_channels=out_ch * 2, kernel_size=(1, 1)), name='weight', dim=0)
            layers['dropout_out'] = nn.Dropout(p=dropout)
        self.layers = nn.Sequential(layers)
//  [python]
def __init__(self,channels):
        super(DenseGroup,self).__init__()
        wn = lambda x: torch.nn.utils.weight_norm(x)
        self.resblock1 = ResBlock(channels)
        self.conv1 = wn(nn.Conv2d(2*channels,channels,1))
        self.resblock2 = ResBlock(channels)
        self.conv2 = wn(nn.Conv2d(3*channels,channels,1))
        self.resblock3 = ResBlock(channels)
//  [python]
def __init__(self,channels):
        super(DenseGroup,self).__init__()
        wn = lambda x: torch.nn.utils.parametrizations.weight_norm(x)
        self.resblock1 = ResBlock(channels)
        self.conv1 = wn(nn.Conv2d(2*channels,channels,1))
        self.resblock2 = ResBlock(channels)
        self.conv2 = wn(nn.Conv2d(3*channels,channels,1))
        self.resblock3 = ResBlock(channels)
//  [python]
def __init__(self, n_in_channels, n_mel_channels, n_layers, n_channels, kernel_size):
        super(WN, self).__init__()
        assert kernel_size % 2 == 1
        assert n_channels % 2 == 0
        self.n_layers = n_layers
        self.n_channels = n_channels
        self.in_layers = torch.nn.ModuleList()
        self.res_skip_layers = torch.nn.ModuleList()
        self.cond_layers = torch.nn.ModuleList()
        start = torch.nn.Conv1d(n_in_channels, n_channels, 1)
        start = torch.nn.utils.weight_norm(start, name='weight')
        self.start = start
        end = torch.nn.Conv1d(n_channels, 2 * n_in_channels, 1)
        end.weight.data.zero_()
        end.bias.data.zero_()
        self.end = end
        for i in range(n_layers):
            dilation = 2 ** i
            padding = int((kernel_size * dilation - dilation) / 2)
            in_layer = torch.nn.Conv1d(n_channels, 2 * n_channels, kernel_size, dilation=dilation, padding=padding)
            in_layer = torch.nn.utils.weight_norm(in_layer, name='weight')
            self.in_layers.append(in_layer)
            cond_layer = torch.nn.Conv1d(n_mel_channels, 2 * n_channels, 1)
            cond_layer = torch.nn.utils.weight_norm(cond_layer, name='weight')
            self.cond_layers.append(cond_layer)
            if i < n_layers - 1:
                res_skip_channels = 2 * n_channels
            else:
                res_skip_channels = n_channels
            res_skip_layer = torch.nn.Conv1d(n_channels, res_skip_channels, 1)
            res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name='weight')
            self.res_skip_layers.append(res_skip_layer)
//  [python]
def __init__(self, n_in_channels, n_mel_channels, n_layers, n_channels, kernel_size):
        super(WN, self).__init__()
        assert kernel_size % 2 == 1
        assert n_channels % 2 == 0
        self.n_layers = n_layers
        self.n_channels = n_channels
        self.in_layers = torch.nn.ModuleList()
        self.res_skip_layers = torch.nn.ModuleList()
        self.cond_layers = torch.nn.ModuleList()
        start = torch.nn.Conv1d(n_in_channels, n_channels, 1)
        start = torch.nn.utils.parametrizations.weight_norm(start, name='weight')
        self.start = start
        end = torch.nn.Conv1d(n_channels, 2 * n_in_channels, 1)
        end.weight.data.zero_()
        end.bias.data.zero_()
        self.end = end
        for i in range(n_layers):
            dilation = 2 ** i
            padding = int((kernel_size * dilation - dilation) / 2)
            in_layer = torch.nn.Conv1d(n_channels, 2 * n_channels, kernel_size, dilation=dilation, padding=padding)
            in_layer = torch.nn.utils.parametrizations.weight_norm(in_layer, name='weight')
            self.in_layers.append(in_layer)
            cond_layer = torch.nn.Conv1d(n_mel_channels, 2 * n_channels, 1)
            cond_layer = torch.nn.utils.parametrizations.weight_norm(cond_layer, name='weight')
            self.cond_layers.append(cond_layer)
            if i < n_layers - 1:
                res_skip_channels = 2 * n_channels
            else:
                res_skip_channels = n_channels
            res_skip_layer = torch.nn.Conv1d(n_channels, res_skip_channels, 1)
            res_skip_layer = torch.nn.utils.parametrizations.weight_norm(res_skip_layer, name='weight')
            self.res_skip_layers.append(res_skip_layer)
//  [python]
def __init__(self,channels):
        super(ResBlock,self).__init__()
        wn = lambda x: torch.nn.utils.weight_norm(x)
        expand = 6
        linear = 0.8
        ##WideActivation
        self.WA=nn.Sequential(
            wn(nn.Conv2d(channels, channels*expand, 1)),
            nn.ReLU(True),
            wn(nn.Conv2d(channels*expand, int(channels*linear), 1)),
            wn(nn.Conv2d(int(channels*linear), channels, 3,padding=1))
            )
        ##Squeeze and Excitation
        self.se = SE_Block(channels)
        self.add=AdaptiveAdd()
//  [python]
def __init__(self,channels):
        super(ResBlock,self).__init__()
        wn = lambda x: torch.nn.utils.parametrizations.weight_norm(x)
        expand = 6
        linear = 0.8
        ##WideActivation
        self.WA=nn.Sequential(
            wn(nn.Conv2d(channels, channels*expand, 1)),
            nn.ReLU(True),
            wn(nn.Conv2d(channels*expand, int(channels*linear), 1)),
            wn(nn.Conv2d(int(channels*linear), channels, 3,padding=1))
            )
        ##Squeeze and Excitation
        self.se = SE_Block(channels)
        self.add=AdaptiveAdd()
//  [python]
def _update_model_cond(old_model, new_model):
    for idx in range(0, len(new_model.WN)):
        wavenet = new_model.WN[idx]
        n_channels = wavenet.n_channels
        n_layers = wavenet.n_layers
        n_mel_channels = wavenet.cond_layers[0].weight.shape[1]
        cond_layer = torch.nn.Conv1d(n_mel_channels, 2 * n_channels * n_layers, 1)
        cond_layer_weight = []
        cond_layer_bias = []
        for i in range(0, n_layers):
            _cond_layer = torch.nn.utils.remove_weight_norm(wavenet.cond_layers[i])
            cond_layer_weight.append(_cond_layer.weight)
            cond_layer_bias.append(_cond_layer.bias)
        cond_layer.weight = torch.nn.Parameter(torch.cat(cond_layer_weight))
        cond_layer.bias = torch.nn.Parameter(torch.cat(cond_layer_bias))
        cond_layer = torch.nn.utils.weight_norm(cond_layer, name='weight')
        wavenet.cond_layer = cond_layer
        del wavenet.cond_layers
//  [python]
def _update_model_cond(old_model, new_model):
    for idx in range(0, len(new_model.WN)):
        wavenet = new_model.WN[idx]
        n_channels = wavenet.n_channels
        n_layers = wavenet.n_layers
        n_mel_channels = wavenet.cond_layers[0].weight.shape[1]
        cond_layer = torch.nn.Conv1d(n_mel_channels, 2 * n_channels * n_layers, 1)
        cond_layer_weight = []
        cond_layer_bias = []
        for i in range(0, n_layers):
            _cond_layer = torch.nn.utils.remove_weight_norm(wavenet.cond_layers[i])
            cond_layer_weight.append(_cond_layer.weight)
            cond_layer_bias.append(_cond_layer.bias)
        cond_layer.weight = torch.nn.Parameter(torch.cat(cond_layer_weight))
        cond_layer.bias = torch.nn.Parameter(torch.cat(cond_layer_bias))
        cond_layer = torch.nn.utils.parametrizations.weight_norm(cond_layer, name='weight')
        wavenet.cond_layer = cond_layer
        del wavenet.cond_layers
//  [python]
def _update_model_res_skip(old_model, new_model):
    for idx in range(0, len(new_model.WN)):
        wavenet = new_model.WN[idx]
        n_channels = wavenet.n_channels
        n_layers = wavenet.n_layers
        wavenet.res_skip_layers = torch.nn.ModuleList()
        for i in range(0, n_layers):
            if i < n_layers - 1:
                res_skip_channels = 2 * n_channels
            else:
                res_skip_channels = n_channels
            res_skip_layer = torch.nn.Conv1d(n_channels, res_skip_channels, 1)
            skip_layer = torch.nn.utils.remove_weight_norm(wavenet.skip_layers[i])
            if i < n_layers - 1:
                res_layer = torch.nn.utils.remove_weight_norm(wavenet.res_layers[i])
                res_skip_layer.weight = torch.nn.Parameter(torch.cat([res_layer.weight, skip_layer.weight]))
                res_skip_layer.bias = torch.nn.Parameter(torch.cat([res_layer.bias, skip_layer.bias]))
            else:
                res_skip_layer.weight = torch.nn.Parameter(skip_layer.weight)
                res_skip_layer.bias = torch.nn.Parameter(skip_layer.bias)
            res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name='weight')
            wavenet.res_skip_layers.append(res_skip_layer)
        del wavenet.res_layers
        del wavenet.skip_layers
//  [python]
def _update_model_res_skip(old_model, new_model):
    for idx in range(0, len(new_model.WN)):
        wavenet = new_model.WN[idx]
        n_channels = wavenet.n_channels
        n_layers = wavenet.n_layers
        wavenet.res_skip_layers = torch.nn.ModuleList()
        for i in range(0, n_layers):
            if i < n_layers - 1:
                res_skip_channels = 2 * n_channels
            else:
                res_skip_channels = n_channels
            res_skip_layer = torch.nn.Conv1d(n_channels, res_skip_channels, 1)
            skip_layer = torch.nn.utils.remove_weight_norm(wavenet.skip_layers[i])
            if i < n_layers - 1:
                res_layer = torch.nn.utils.remove_weight_norm(wavenet.res_layers[i])
                res_skip_layer.weight = torch.nn.Parameter(torch.cat([res_layer.weight, skip_layer.weight]))
                res_skip_layer.bias = torch.nn.Parameter(torch.cat([res_layer.bias, skip_layer.bias]))
            else:
                res_skip_layer.weight = torch.nn.Parameter(skip_layer.weight)
                res_skip_layer.bias = torch.nn.Parameter(skip_layer.bias)
            res_skip_layer = torch.nn.utils.parametrizations.weight_norm(res_skip_layer, name='weight')
            wavenet.res_skip_layers.append(res_skip_layer)
        del wavenet.res_layers
        del wavenet.skip_layers
//  [python]
def __init__(self,channels=16):
        super(OverNet,self).__init__()
        wn = lambda x: torch.nn.utils.weight_norm(x)

        #NEW
        self.sub_mean = ops.MeanShift((0.4488, 0.4371, 0.4040), sub=True)
        self.add_mean = ops.MeanShift((0.4488, 0.4371, 0.4040), sub=False)


        self.conv3x3=wn(nn.Conv2d(3,channels,3,padding=1))
        self.DG1=DenseGroup(channels)
        self.conv1x1_1=wn(nn.Conv2d(2*channels,channels,1))
        self.DG2=DenseGroup(channels)
        self.conv1x1_2=wn(nn.Conv2d(3*channels,channels,1))
        self.DG3=DenseGroup(channels)
        self.conv1x1_3=wn(nn.Conv2d(3*channels,channels,1))
        self.longRangeSkip=nn.Sequential(
        nn.AdaptiveAvgPool2d(1),
        wn(nn.Conv2d(channels,channels,1)),
        nn.ReLU(True)
        )
        self.add=AdaptiveAdd()

        self.OSMBlock=nn.Sequential(
            wn(nn.Conv2d(channels, 1600,3,padding=1)),
            nn.PixelShuffle(5),
            wn(nn.Conv2d(64,3,3,padding=1))
            )
//  [python]
def __init__(self,channels=16):
        super(OverNet,self).__init__()
        wn = lambda x: torch.nn.utils.parametrizations.weight_norm(x)

        #NEW
        self.sub_mean = ops.MeanShift((0.4488, 0.4371, 0.4040), sub=True)
        self.add_mean = ops.MeanShift((0.4488, 0.4371, 0.4040), sub=False)


        self.conv3x3=wn(nn.Conv2d(3,channels,3,padding=1))
        self.DG1=DenseGroup(channels)
        self.conv1x1_1=wn(nn.Conv2d(2*channels,channels,1))
        self.DG2=DenseGroup(channels)
        self.conv1x1_2=wn(nn.Conv2d(3*channels,channels,1))
        self.DG3=DenseGroup(channels)
        self.conv1x1_3=wn(nn.Conv2d(3*channels,channels,1))
        self.longRangeSkip=nn.Sequential(
        nn.AdaptiveAvgPool2d(1),
        wn(nn.Conv2d(channels,channels,1)),
        nn.ReLU(True)
        )
        self.add=AdaptiveAdd()

        self.OSMBlock=nn.Sequential(
            wn(nn.Conv2d(channels, 1600,3,padding=1)),
            nn.PixelShuffle(5),
            wn(nn.Conv2d(64,3,3,padding=1))
            )
//  [python]
def __init__(self):
        super().__init__()
        self.conv = torch.nn.utils.weight_norm(torch.nn.Conv2d(3, 3, 3, stride=2, padding=1))
//  [python]
def __init__(self):
        super().__init__()
        self.conv = torch.nn.utils.parametrizations.weight_norm(torch.nn.Conv2d(3, 3, 3, stride=2, padding=1))