jiahuimbzuai/CoQuIR huggingface dataset:
//  [python]
def __init__(self, hp):
        super(Generator, self).__init__()
        self.hp = hp
        self.num_kernels = len(hp.gen.resblock_kernel_sizes)
        self.num_upsamples = len(hp.gen.upsample_rates)
        # speaker adaper, 256 should change by what speaker encoder you use
        self.adapter = nn.ModuleList()
        # 1024 should change by your whisper out
        self.cond_pit = nn.Linear(2048, hp.audio.n_mel_channels)
        self.cond_pre = nn.Linear(1024, hp.audio.n_mel_channels)
        self.cond_pos = nn.Embedding(3, hp.audio.n_mel_channels)
        # pre conv
        self.conv_pit = nn.utils.weight_norm(
            Conv1d(hp.audio.n_mel_channels, hp.gen.upsample_initial_channel, 7, 1, padding=3))
        self.conv_pre = nn.utils.weight_norm(
            Conv1d(hp.audio.n_mel_channels, hp.gen.upsample_initial_channel, 7, 1, padding=3))
        # transposed conv-based upsamplers. does not apply anti-aliasing
        self.ups = nn.ModuleList()
        self.upp = nn.ModuleList()
        for i, (u, k) in enumerate(zip(hp.gen.upsample_rates, hp.gen.upsample_kernel_sizes)):
            # spk
            self.adapter.append(SpeakerAdapter(
                256, hp.gen.upsample_initial_channel // (2 ** (i + 1))))
            # print(f'ups: {i} {k}, {u}, {(k - u) // 2}')
            # base
            self.ups.append(nn.ModuleList([
                weight_norm(ConvTranspose1d(hp.gen.upsample_initial_channel // (2 ** i),
                                            hp.gen.upsample_initial_channel // (
                                                2 ** (i + 1)),
                                            k, u, padding=(k - u) // 2))
            ]))
        for i, (u, k) in enumerate(zip(hp.gen.upsample_rates, hp.gen.upsample_kernel_sizes_p)):
            # print(f'upp: {i} {k}, {u}, {(k - u) // 2}')
            # pit
            self.upp.append(nn.ModuleList([
                weight_norm(ConvTranspose1d(hp.gen.upsample_initial_channel // (2 ** i),
                                            hp.gen.upsample_initial_channel // (
                                                2 ** (i + 1)),
                                            k, u, padding=(k - u) // 2))
            ]))

        # residual blocks using anti-aliased multi-periodicity composition modules (AMP)
        self.resblocks = nn.ModuleList()
        for i in range(len(self.ups)):
            ch = hp.gen.upsample_initial_channel // (2 ** (i + 1))
            for k, d in zip(hp.gen.resblock_kernel_sizes, hp.gen.resblock_dilation_sizes):
                self.resblocks.append(AMPBlock(hp, ch, k, d))

        # post conv
        activation_post = SnakeBeta(ch, alpha_logscale=True)
        self.activation_post = Activation1d(activation=activation_post)
        self.conv_post = weight_norm(Conv1d(ch, 1, 7, 1, padding=3))

        # weight initialization
        for i in range(len(self.ups)):
            self.ups[i].apply(init_weights)
        for i in range(len(self.upp)):
            self.upp[i].apply(init_weights)
        self.conv_post.apply(init_weights)
//  [python]
def __init__(self, hp):
        super(Generator, self).__init__()
        self.hp = hp
        self.num_kernels = len(hp.gen.resblock_kernel_sizes)
        self.num_upsamples = len(hp.gen.upsample_rates)
        # speaker adaper, 256 should change by what speaker encoder you use
        self.adapter = nn.ModuleList()
        # 1024 should change by your whisper out
        self.cond_pit = nn.Linear(2048, hp.audio.n_mel_channels)
        self.cond_pre = nn.Linear(1024, hp.audio.n_mel_channels)
        self.cond_pos = nn.Embedding(3, hp.audio.n_mel_channels)
        # pre conv
        self.conv_pit = nn.utils.parametrizations.weight_norm(
            Conv1d(hp.audio.n_mel_channels, hp.gen.upsample_initial_channel, 7, 1, padding=3))
        self.conv_pre = nn.utils.parametrizations.weight_norm(
            Conv1d(hp.audio.n_mel_channels, hp.gen.upsample_initial_channel, 7, 1, padding=3))
        # transposed conv-based upsamplers. does not apply anti-aliasing
        self.ups = nn.ModuleList()
        self.upp = nn.ModuleList()
        for i, (u, k) in enumerate(zip(hp.gen.upsample_rates, hp.gen.upsample_kernel_sizes)):
            # spk
            self.adapter.append(SpeakerAdapter(
                256, hp.gen.upsample_initial_channel // (2 ** (i + 1))))
            # print(f'ups: {i} {k}, {u}, {(k - u) // 2}')
            # base
            self.ups.append(nn.ModuleList([
                parametrizations.weight_norm(ConvTranspose1d(hp.gen.upsample_initial_channel // (2 ** i),
                                            hp.gen.upsample_initial_channel // (
                                                2 ** (i + 1)),
                                            k, u, padding=(k - u) // 2))
            ]))
        for i, (u, k) in enumerate(zip(hp.gen.upsample_rates, hp.gen.upsample_kernel_sizes_p)):
            # print(f'upp: {i} {k}, {u}, {(k - u) // 2}')
            # pit
            self.upp.append(nn.ModuleList([
                parametrizations.weight_norm(ConvTranspose1d(hp.gen.upsample_initial_channel // (2 ** i),
                                            hp.gen.upsample_initial_channel // (
                                                2 ** (i + 1)),
                                            k, u, padding=(k - u) // 2))
            ]))

        # residual blocks using anti-aliased multi-periodicity composition modules (AMP)
        self.resblocks = nn.ModuleList()
        for i in range(len(self.ups)):
            ch = hp.gen.upsample_initial_channel // (2 ** (i + 1))
            for k, d in zip(hp.gen.resblock_kernel_sizes, hp.gen.resblock_dilation_sizes):
                self.resblocks.append(AMPBlock(hp, ch, k, d))

        # post conv
        activation_post = SnakeBeta(ch, alpha_logscale=True)
        self.activation_post = Activation1d(activation=activation_post)
        self.conv_post = parametrizations.weight_norm(Conv1d(ch, 1, 7, 1, padding=3))

        # weight initialization
        for i in range(len(self.ups)):
            self.ups[i].apply(init_weights)
        for i in range(len(self.upp)):
            self.upp[i].apply(init_weights)
        self.conv_post.apply(init_weights)
//  [python]
def __init__(self, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=0, p_dropout=0, share_cond_layers=False):
        super(WN, self).__init__()
        assert kernel_size % 2 == 1
        assert hidden_channels % 2 == 0
        self.hidden_channels = hidden_channels
        self.kernel_size = kernel_size
        self.dilation_rate = dilation_rate
        self.n_layers = n_layers
        self.gin_channels = gin_channels
        self.p_dropout = p_dropout
        self.share_cond_layers = share_cond_layers
        self.in_layers = torch.nn.ModuleList()
        self.res_skip_layers = torch.nn.ModuleList()
        self.drop = nn.Dropout(p_dropout)
        if gin_channels != 0 and not share_cond_layers:
            cond_layer = torch.nn.Conv1d(gin_channels, 2 * hidden_channels * n_layers, 1)
            self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name='weight')
        for i in range(n_layers):
            dilation = dilation_rate ** i
            padding = int((kernel_size * dilation - dilation) / 2)
            in_layer = torch.nn.Conv1d(hidden_channels, 2 * hidden_channels, kernel_size, dilation=dilation, padding=padding)
            in_layer = torch.nn.utils.weight_norm(in_layer, name='weight')
            self.in_layers.append(in_layer)
            if i < n_layers - 1:
                res_skip_channels = 2 * hidden_channels
            else:
                res_skip_channels = hidden_channels
            res_skip_layer = torch.nn.Conv1d(hidden_channels, res_skip_channels, 1)
            res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name='weight')
            self.res_skip_layers.append(res_skip_layer)
//  [python]
def __init__(self, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=0, p_dropout=0, share_cond_layers=False):
        super(WN, self).__init__()
        assert kernel_size % 2 == 1
        assert hidden_channels % 2 == 0
        self.hidden_channels = hidden_channels
        self.kernel_size = kernel_size
        self.dilation_rate = dilation_rate
        self.n_layers = n_layers
        self.gin_channels = gin_channels
        self.p_dropout = p_dropout
        self.share_cond_layers = share_cond_layers
        self.in_layers = torch.nn.ModuleList()
        self.res_skip_layers = torch.nn.ModuleList()
        self.drop = nn.Dropout(p_dropout)
        if gin_channels != 0 and not share_cond_layers:
            cond_layer = torch.nn.Conv1d(gin_channels, 2 * hidden_channels * n_layers, 1)
            self.cond_layer = torch.nn.utils.parametrizations.weight_norm(cond_layer, name='weight')
        for i in range(n_layers):
            dilation = dilation_rate ** i
            padding = int((kernel_size * dilation - dilation) / 2)
            in_layer = torch.nn.Conv1d(hidden_channels, 2 * hidden_channels, kernel_size, dilation=dilation, padding=padding)
            in_layer = torch.nn.utils.parametrizations.weight_norm(in_layer, name='weight')
            self.in_layers.append(in_layer)
            if i < n_layers - 1:
                res_skip_channels = 2 * hidden_channels
            else:
                res_skip_channels = hidden_channels
            res_skip_layer = torch.nn.Conv1d(hidden_channels, res_skip_channels, 1)
            res_skip_layer = torch.nn.utils.parametrizations.weight_norm(res_skip_layer, name='weight')
            self.res_skip_layers.append(res_skip_layer)
//  [python]
def __init__(self, in_channels, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=0, p_dropout=0, sigmoid_scale=False, share_cond_layers=False, wn=None):
        super().__init__()
        self.in_channels = in_channels
        self.hidden_channels = hidden_channels
        self.kernel_size = kernel_size
        self.dilation_rate = dilation_rate
        self.n_layers = n_layers
        self.gin_channels = gin_channels
        self.p_dropout = p_dropout
        self.sigmoid_scale = sigmoid_scale
        start = torch.nn.Conv1d(in_channels // 2, hidden_channels, 1)
        start = torch.nn.utils.weight_norm(start)
        self.start = start
        end = torch.nn.Conv1d(hidden_channels, in_channels, 1)
        end.weight.data.zero_()
        end.bias.data.zero_()
        self.end = end
        self.wn = WN(hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels, p_dropout, share_cond_layers)
        if wn is not None:
            self.wn.in_layers = wn.in_layers
            self.wn.res_skip_layers = wn.res_skip_layers
//  [python]
def __init__(self, in_channels, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=0, p_dropout=0, sigmoid_scale=False, share_cond_layers=False, wn=None):
        super().__init__()
        self.in_channels = in_channels
        self.hidden_channels = hidden_channels
        self.kernel_size = kernel_size
        self.dilation_rate = dilation_rate
        self.n_layers = n_layers
        self.gin_channels = gin_channels
        self.p_dropout = p_dropout
        self.sigmoid_scale = sigmoid_scale
        start = torch.nn.Conv1d(in_channels // 2, hidden_channels, 1)
        start = torch.nn.utils.parametrizations.weight_norm(start)
        self.start = start
        end = torch.nn.Conv1d(hidden_channels, in_channels, 1)
        end.weight.data.zero_()
        end.bias.data.zero_()
        self.end = end
        self.wn = WN(hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels, p_dropout, share_cond_layers)
        if wn is not None:
            self.wn.in_layers = wn.in_layers
            self.wn.res_skip_layers = wn.res_skip_layers
//  [python]
def __init__(self, in_channels=256, hidden_channels=256, kernel_size=3, dilation_rate=1, n_blocks=8, n_layers=4, p_dropout=0.0, n_split=4, n_split_blocks=3, sigmoid_scale=False, gin_channels=0, share_cond_layers=True):
        super().__init__()
        self.in_channels = in_channels
        self.hidden_channels = hidden_channels
        self.kernel_size = kernel_size
        self.dilation_rate = dilation_rate
        self.n_blocks = n_blocks
        self.n_layers = n_layers
        self.p_dropout = p_dropout
        self.n_split = n_split
        self.n_split_blocks = n_split_blocks
        self.sigmoid_scale = sigmoid_scale
        self.gin_channels = gin_channels
        self.cond_layers = nn.ModuleList()
        self.share_cond_layers = share_cond_layers
        self.flows = nn.ModuleList()
        in_channels = in_channels * 2
        for l in range(n_split_blocks):
            blocks = nn.ModuleList()
            self.flows.append(blocks)
            gin_channels = gin_channels * 2
            if gin_channels != 0 and share_cond_layers:
                cond_layer = torch.nn.Conv1d(gin_channels, 2 * hidden_channels * n_layers, 1)
                self.cond_layers.append(torch.nn.utils.weight_norm(cond_layer, name='weight'))
            for b in range(n_blocks):
                blocks.append(ActNorm(channels=in_channels))
                blocks.append(InvConvNear(channels=in_channels, n_split=n_split))
                blocks.append(CouplingBlock(in_channels, hidden_channels, kernel_size=kernel_size, dilation_rate=dilation_rate, n_layers=n_layers, gin_channels=gin_channels, p_dropout=p_dropout, sigmoid_scale=sigmoid_scale, share_cond_layers=share_cond_layers))
//  [python]
def __init__(self, in_channels=256, hidden_channels=256, kernel_size=3, dilation_rate=1, n_blocks=8, n_layers=4, p_dropout=0.0, n_split=4, n_split_blocks=3, sigmoid_scale=False, gin_channels=0, share_cond_layers=True):
        super().__init__()
        self.in_channels = in_channels
        self.hidden_channels = hidden_channels
        self.kernel_size = kernel_size
        self.dilation_rate = dilation_rate
        self.n_blocks = n_blocks
        self.n_layers = n_layers
        self.p_dropout = p_dropout
        self.n_split = n_split
        self.n_split_blocks = n_split_blocks
        self.sigmoid_scale = sigmoid_scale
        self.gin_channels = gin_channels
        self.cond_layers = nn.ModuleList()
        self.share_cond_layers = share_cond_layers
        self.flows = nn.ModuleList()
        in_channels = in_channels * 2
        for l in range(n_split_blocks):
            blocks = nn.ModuleList()
            self.flows.append(blocks)
            gin_channels = gin_channels * 2
            if gin_channels != 0 and share_cond_layers:
                cond_layer = torch.nn.Conv1d(gin_channels, 2 * hidden_channels * n_layers, 1)
                self.cond_layers.append(torch.nn.utils.parametrizations.weight_norm(cond_layer, name='weight'))
            for b in range(n_blocks):
                blocks.append(ActNorm(channels=in_channels))
                blocks.append(InvConvNear(channels=in_channels, n_split=n_split))
                blocks.append(CouplingBlock(in_channels, hidden_channels, kernel_size=kernel_size, dilation_rate=dilation_rate, n_layers=n_layers, gin_channels=gin_channels, p_dropout=p_dropout, sigmoid_scale=sigmoid_scale, share_cond_layers=share_cond_layers))
//  [python]
def apply_weight_norm(self):
        def _apply_weight_norm(m):
            if isinstance(m, (torch.nn.Conv1d, torch.nn.Conv2d)):
                torch.nn.utils.weight_norm(m)

        self.apply(_apply_weight_norm)
//  [python]
def apply_weight_norm(self):
        def _apply_weight_norm(m):
            if isinstance(m, (torch.nn.Conv1d, torch.nn.Conv2d)):
                torch.nn.utils.parametrizations.weight_norm(m)

        self.apply(_apply_weight_norm)
//  [python]
def LinearActivation(
        d_input,
        d_output,
        bias=True,
        zero_bias_init=False,
        transposed=False,
        initializer=None,
        activation=None,
        activate=False,  # Apply activation as part of this module
        weight_norm=False,
        **kwargs,
):
    """Returns a linear nn.Module with control over axes order, initialization, and activation"""

    # Construct core module
    linear_cls = TransposedLinear if transposed else nn.Linear
    if activation == "glu":
        d_output *= 2
    linear = linear_cls(d_input, d_output, bias=bias, **kwargs)

    # Initialize weight
    if initializer is not None:
        get_initializer(initializer, activation)(linear.weight)

    # Initialize bias
    if bias and zero_bias_init:
        nn.init.zeros_(linear.bias)

    # Weight norm
    if weight_norm:
        linear = nn.utils.weight_norm(linear)

    if activate and activation is not None:
        activation = Activation(activation, dim=-2 if transposed else -1)
        linear = nn.Sequential(linear, activation)
    return linear
//  [python]
def LinearActivation(
        d_input,
        d_output,
        bias=True,
        zero_bias_init=False,
        transposed=False,
        initializer=None,
        activation=None,
        activate=False,  # Apply activation as part of this module
        weight_norm=False,
        **kwargs,
):
    """Returns a linear nn.Module with control over axes order, initialization, and activation"""

    # Construct core module
    linear_cls = TransposedLinear if transposed else nn.Linear
    if activation == "glu":
        d_output *= 2
    linear = linear_cls(d_input, d_output, bias=bias, **kwargs)

    # Initialize weight
    if initializer is not None:
        get_initializer(initializer, activation)(linear.weight)

    # Initialize bias
    if bias and zero_bias_init:
        nn.init.zeros_(linear.bias)

    # Weight norm
    if weight_norm:
        linear = nn.utils.parametrizations.weight_norm(linear)

    if activate and activation is not None:
        activation = Activation(activation, dim=-2 if transposed else -1)
        linear = nn.Sequential(linear, activation)
    return linear
//  [python]
def __init__(self, h, c_out=1):
        super(HifiGanGenerator, self).__init__()
        self.h = h
        self.num_kernels = len(h['resblock_kernel_sizes'])
        self.num_upsamples = len(h['upsample_rates'])
        if h['use_pitch_embed']:
            self.harmonic_num = 8
            self.f0_upsamp = torch.nn.Upsample(scale_factor=np.prod(h['upsample_rates']))
            self.m_source = SourceModuleHnNSF(sampling_rate=h['audio_sample_rate'], harmonic_num=self.harmonic_num)
            self.noise_convs = nn.ModuleList()
        self.conv_pre = weight_norm(Conv1d(80, h['upsample_initial_channel'], 7, 1, padding=3))
        resblock = ResBlock1 if h['resblock'] == '1' else ResBlock2
        self.ups = nn.ModuleList()
        for i, (u, k) in enumerate(zip(h['upsample_rates'], h['upsample_kernel_sizes'])):
            c_cur = h['upsample_initial_channel'] // 2 ** (i + 1)
            self.ups.append(weight_norm(ConvTranspose1d(c_cur * 2, c_cur, k, u, padding=(k - u) // 2)))
            if h['use_pitch_embed']:
                if i + 1 < len(h['upsample_rates']):
                    stride_f0 = np.prod(h['upsample_rates'][i + 1:])
                    self.noise_convs.append(Conv1d(1, c_cur, kernel_size=stride_f0 * 2, stride=stride_f0, padding=stride_f0 // 2))
                else:
                    self.noise_convs.append(Conv1d(1, c_cur, kernel_size=1))
        self.resblocks = nn.ModuleList()
        for i in range(len(self.ups)):
            ch = h['upsample_initial_channel'] // 2 ** (i + 1)
            for j, (k, d) in enumerate(zip(h['resblock_kernel_sizes'], h['resblock_dilation_sizes'])):
                self.resblocks.append(resblock(h, ch, k, d))
        self.conv_post = weight_norm(Conv1d(ch, c_out, 7, 1, padding=3))
        self.ups.apply(init_weights)
        self.conv_post.apply(init_weights)
//  [python]
def __init__(self, h, c_out=1):
        super(HifiGanGenerator, self).__init__()
        self.h = h
        self.num_kernels = len(h['resblock_kernel_sizes'])
        self.num_upsamples = len(h['upsample_rates'])
        if h['use_pitch_embed']:
            self.harmonic_num = 8
            self.f0_upsamp = torch.nn.Upsample(scale_factor=np.prod(h['upsample_rates']))
            self.m_source = SourceModuleHnNSF(sampling_rate=h['audio_sample_rate'], harmonic_num=self.harmonic_num)
            self.noise_convs = nn.ModuleList()
        self.conv_pre = torch.nn.utils.parametrizations.weight_norm(Conv1d(80, h['upsample_initial_channel'], 7, 1, padding=3))
        resblock = ResBlock1 if h['resblock'] == '1' else ResBlock2
        self.ups = nn.ModuleList()
        for i, (u, k) in enumerate(zip(h['upsample_rates'], h['upsample_kernel_sizes'])):
            c_cur = h['upsample_initial_channel'] // 2 ** (i + 1)
            self.ups.append(torch.nn.utils.parametrizations.weight_norm(ConvTranspose1d(c_cur * 2, c_cur, k, u, padding=(k - u) // 2)))
            if h['use_pitch_embed']:
                if i + 1 < len(h['upsample_rates']):
                    stride_f0 = np.prod(h['upsample_rates'][i + 1:])
                    self.noise_convs.append(Conv1d(1, c_cur, kernel_size=stride_f0 * 2, stride=stride_f0, padding=stride_f0 // 2))
                else:
                    self.noise_convs.append(Conv1d(1, c_cur, kernel_size=1))
        self.resblocks = nn.ModuleList()
        for i in range(len(self.ups)):
            ch = h['upsample_initial_channel'] // 2 ** (i + 1)
            for j, (k, d) in enumerate(zip(h['resblock_kernel_sizes'], h['resblock_dilation_sizes'])):
                self.resblocks.append(resblock(h, ch, k, d))
        self.conv_post = torch.nn.utils.parametrizations.weight_norm(Conv1d(ch, c_out, 7, 1, padding=3))
        self.ups.apply(init_weights)
        self.conv_post.apply(init_weights)
//  [python]
def __init__(self, kernel_sizes, strides, n_channels, group_sizes, act_func=nn.LeakyReLU(0.2, inplace=True)):
        super(DW_Module, self).__init__()
        assert n_channels[-1] == 1
        assert group_sizes[-1] == 1
        
        n_in_channel = 1
        self.convs = nn.ModuleList()
        for i, (kernel_size, stride, n_channel, group_size) in enumerate(zip(kernel_sizes, strides, n_channels, group_sizes)):
            conv = nn.Conv1d(n_in_channel, n_channel, kernel_size, stride=stride, groups=group_size)
            conv = nn.utils.weight_norm(conv)
            self.convs.append(conv)
            n_in_channel = n_channel
        
        self.act_func = act_func
        self.res_weights = nn.Parameter( torch.rand(len(kernel_sizes))*0.1+0.01 )
        self.layr_weights = nn.Parameter( torch.ones(len(kernel_sizes)) )
//  [python]
def __init__(self, kernel_sizes, strides, n_channels, group_sizes, act_func=nn.LeakyReLU(0.2, inplace=True)):
        super(DW_Module, self).__init__()
        assert n_channels[-1] == 1
        assert group_sizes[-1] == 1
        
        n_in_channel = 1
        self.convs = nn.ModuleList()
        for i, (kernel_size, stride, n_channel, group_size) in enumerate(zip(kernel_sizes, strides, n_channels, group_sizes)):
            conv = nn.Conv1d(n_in_channel, n_channel, kernel_size, stride=stride, groups=group_size)
            conv = nn.utils.parametrizations.weight_norm(conv)
            self.convs.append(conv)
            n_in_channel = n_channel
        
        self.act_func = act_func
        self.res_weights = nn.Parameter( torch.rand(len(kernel_sizes))*0.1+0.01 )
        self.layr_weights = nn.Parameter( torch.ones(len(kernel_sizes)) )
//  [python]
def __init__(self, in_channels, out_channels, hidden_channels, kernel_size=3, scales=[16, 16], use_last_layer_act_func=False, weightnorm=False, residual=False, residual_linear=False, rezero=False):
        super(TransposedUpsampleNet, self).__init__()
        self.residual = residual
        self.residual_linear = residual_linear
        self.res_weight = nn.Parameter(torch.rand(1)*0.02+0.01) if rezero else None
        self.scales = scales
        self.t_convs = nn.ModuleList()
        for i, scale in enumerate(scales):
            is_first_layer = bool(i == 0)
            is_last_layer = bool(i+1 == len(scales))
            in_dim = in_channels if is_first_layer else hidden_channels
            out_dim = out_channels if is_last_layer else hidden_channels
            k_size = kernel_size[i] if type(kernel_size) == list else kernel_size
            t_conv = nn.ConvTranspose1d(in_dim, out_dim, k_size, scale, padding=(k_size-scale)//2)
            if weightnorm:
                t_conv = torch.nn.utils.weight_norm(t_conv)
            self.t_convs.append(t_conv)
            if not is_last_layer or use_last_layer_act_func:
                self.t_convs.append( nn.LeakyReLU(negative_slope=0.4, inplace=True) )
        self.res_channels = min(in_channels, out_channels)
//  [python]
def __init__(self, in_channels, out_channels, hidden_channels, kernel_size=3, scales=[16, 16], use_last_layer_act_func=False, weightnorm=False, residual=False, residual_linear=False, rezero=False):
        super(TransposedUpsampleNet, self).__init__()
        self.residual = residual
        self.residual_linear = residual_linear
        self.res_weight = nn.Parameter(torch.rand(1)*0.02+0.01) if rezero else None
        self.scales = scales
        self.t_convs = nn.ModuleList()
        for i, scale in enumerate(scales):
            is_first_layer = bool(i == 0)
            is_last_layer = bool(i+1 == len(scales))
            in_dim = in_channels if is_first_layer else hidden_channels
            out_dim = out_channels if is_last_layer else hidden_channels
            k_size = kernel_size[i] if type(kernel_size) == list else kernel_size
            t_conv = nn.ConvTranspose1d(in_dim, out_dim, k_size, scale, padding=(k_size-scale)//2)
            if weightnorm:
                t_conv = torch.nn.utils.parametrizations.weight_norm(t_conv)
            self.t_convs.append(t_conv)
            if not is_last_layer or use_last_layer_act_func:
                self.t_convs.append( nn.LeakyReLU(negative_slope=0.4, inplace=True) )
        self.res_channels = min(in_channels, out_channels)
//  [python]
def test_weight_norm_nodim(self):
        # addmm for 3-d inputs converts to onnx::MatMul
        model = torch.nn.utils.weight_norm(torch.nn.Linear(5, 10), dim=None)
        x = torch.randn(3, 4, 5, requires_grad=True)
        self.run_test(model, x)

        # addmm for 2-d inputs converts to onnx::Gemm
        model = torch.nn.utils.weight_norm(torch.nn.Linear(5, 10), dim=None)
        x = torch.randn(4, 5, requires_grad=True)
        self.run_test(model, x)
//  [python]
def test_weight_norm_nodim(self):
        # addmm for 3-d inputs converts to onnx::MatMul
        model = torch.nn.utils.parametrizations.weight_norm(torch.nn.Linear(5, 10), dim=None)
        x = torch.randn(3, 4, 5, requires_grad=True)
        self.run_test(model, x)

        # addmm for 2-d inputs converts to onnx::Gemm
        model = torch.nn.utils.parametrizations.weight_norm(torch.nn.Linear(5, 10), dim=None)
        x = torch.randn(4, 5, requires_grad=True)
        self.run_test(model, x)
//  [python]
def __init__(self, args, favor=False, weight_drop=0.0, predict_language=0, n_languages=1, branchformer=False):
        """
        :param args:
        :param favor: Performer Attention
        """
        super().__init__()

        self.rotary_attention = False
        self.positional_encoder = None
        self.relative_attention = False
        self.dropout = args.dropout
        self.embedding_dim = args.encoder_embed_dim
        self.favor = favor
        self.weight_drop = weight_drop
        self.num_heads = args.encoder_attention_heads
        self.num_layers = args.encoder_layers
        self.attention_dropout = args.attention_dropout
        self.activation_dropout = args.activation_dropout
        self.deepspeed = False
        self.branchformer = branchformer

        self.pos_conv = nn.Conv1d(
            self.embedding_dim,
            self.embedding_dim,
            kernel_size=args.conv_pos,
            padding=args.conv_pos // 2,
            groups=args.conv_pos_groups,
        )
        dropout = 0
        std = math.sqrt((4 * (1.0 - dropout)) / (args.conv_pos * self.embedding_dim))
        nn.init.normal_(self.pos_conv.weight, mean=0, std=std)
        nn.init.constant_(self.pos_conv.bias, 0)

        self.pos_conv = nn.utils.weight_norm(self.pos_conv, name="weight", dim=2)
        self.pos_conv = nn.Sequential(self.pos_conv, SamePad(args.conv_pos), nn.GELU())

        self.layers = nn.ModuleList(
            [
                TransformerSentenceEncoderLayer(
                    embedding_dim=self.embedding_dim,
                    ffn_embedding_dim=args.encoder_ffn_embed_dim,
                    num_attention_heads=args.encoder_attention_heads,
                    dropout=self.dropout,
                    weight_drop=self.weight_drop,
                    attention_dropout=args.attention_dropout,
                    activation_dropout=args.activation_dropout,
                    activation_fn=args.activation_fn,
                    layer_norm_first=args.layer_norm_first,
                    favor=favor,
                    branchformer=self.branchformer
                )
                for _ in range(args.encoder_layers)
            ]
        )

        self.layer_norm_first = args.layer_norm_first
        self.layer_norm = LayerNorm(self.embedding_dim)
        self.layerdrop = args.encoder_layerdrop
        self.args = args

        self.apply(init_bert_params)
        self.predict_language = predict_language

        if self.predict_language > 0:
            self.layer_norm_cls = LayerNorm(self.embedding_dim)
            self.linear_cls = torch.nn.Linear(self.embedding_dim, n_languages)
        else:
            self.linear_cls = None
            self.layer_norm_cls = None

        # from onmt.modules.optimized.fast_mha import fast_bert_mha
        # self.fast_bert_mha = fast_bert_mha
        from onmt.modules.optimized.flash_mha import flash_bert_mha
        self.fast_bert_mha = flash_bert_mha
        self.using_s4 = False
//  [python]
def __init__(self, args, favor=False, weight_drop=0.0, predict_language=0, n_languages=1, branchformer=False):
        """
        :param args:
        :param favor: Performer Attention
        """
        super().__init__()

        self.rotary_attention = False
        self.positional_encoder = None
        self.relative_attention = False
        self.dropout = args.dropout
        self.embedding_dim = args.encoder_embed_dim
        self.favor = favor
        self.weight_drop = weight_drop
        self.num_heads = args.encoder_attention_heads
        self.num_layers = args.encoder_layers
        self.attention_dropout = args.attention_dropout
        self.activation_dropout = args.activation_dropout
        self.deepspeed = False
        self.branchformer = branchformer

        self.pos_conv = nn.Conv1d(
            self.embedding_dim,
            self.embedding_dim,
            kernel_size=args.conv_pos,
            padding=args.conv_pos // 2,
            groups=args.conv_pos_groups,
        )
        dropout = 0
        std = math.sqrt((4 * (1.0 - dropout)) / (args.conv_pos * self.embedding_dim))
        nn.init.normal_(self.pos_conv.weight, mean=0, std=std)
        nn.init.constant_(self.pos_conv.bias, 0)

        self.pos_conv = nn.utils.parametrizations.weight_norm(self.pos_conv, name="weight", dim=2)
        self.pos_conv = nn.Sequential(self.pos_conv, SamePad(args.conv_pos), nn.GELU())

        self.layers = nn.ModuleList(
            [
                TransformerSentenceEncoderLayer(
                    embedding_dim=self.embedding_dim,
                    ffn_embedding_dim=args.encoder_ffn_embed_dim,
                    num_attention_heads=args.encoder_attention_heads,
                    dropout=self.dropout,
                    weight_drop=self.weight_drop,
                    attention_dropout=args.attention_dropout,
                    activation_dropout=args.activation_dropout,
                    activation_fn=args.activation_fn,
                    layer_norm_first=args.layer_norm_first,
                    favor=favor,
                    branchformer=self.branchformer
                )
                for _ in range(args.encoder_layers)
            ]
        )

        self.layer_norm_first = args.layer_norm_first
        self.layer_norm = LayerNorm(self.embedding_dim)
        self.layerdrop = args.encoder_layerdrop
        self.args = args

        self.apply(init_bert_params)
        self.predict_language = predict_language

        if self.predict_language > 0:
            self.layer_norm_cls = LayerNorm(self.embedding_dim)
            self.linear_cls = torch.nn.Linear(self.embedding_dim, n_languages)
        else:
            self.linear_cls = None
            self.layer_norm_cls = None

        # from onmt.modules.optimized.fast_mha import fast_bert_mha
        # self.fast_bert_mha = fast_bert_mha
        from onmt.modules.optimized.flash_mha import flash_bert_mha
        self.fast_bert_mha = flash_bert_mha
        self.using_s4 = False
//  [python]
def test_weight_norm(self):
        # addmm for 3-d inputs converts to onnx::MatMul
        model = torch.nn.utils.weight_norm(torch.nn.Linear(5, 10), dim=1)
        x = torch.randn(3, 4, 5, requires_grad=True)
        self.run_test(model, x)

        # addmm for 2-d inputs converts to onnx::Gemm
        model = torch.nn.utils.weight_norm(torch.nn.Linear(5, 10), dim=1)
        x = torch.randn(4, 5, requires_grad=True)
        self.run_test(model, x)

        model = torch.nn.utils.weight_norm(torch.nn.Conv1d(1, 1, 3))
        x = torch.randn(1, 1, 5, requires_grad=True)
        self.run_test(model, x)

        model = torch.nn.utils.weight_norm(torch.nn.Conv1d(1, 1, 3), dim=-2)
        x = torch.randn(1, 1, 5, requires_grad=True)
        self.run_test(model, x)

        model = torch.nn.utils.weight_norm(torch.nn.Conv1d(3, 6, 3), name="weight")
        x = torch.randn(3, 3, 5, requires_grad=True)
        self.run_test(model, x)
//  [python]
def test_weight_norm(self):
        # addmm for 3-d inputs converts to onnx::MatMul
        model = torch.nn.utils.parametrizations.weight_norm(torch.nn.Linear(5, 10), dim=1)
        x = torch.randn(3, 4, 5, requires_grad=True)
        self.run_test(model, x)

        # addmm for 2-d inputs converts to onnx::Gemm
        model = torch.nn.utils.parametrizations.weight_norm(torch.nn.Linear(5, 10), dim=1)
        x = torch.randn(4, 5, requires_grad=True)
        self.run_test(model, x)

        model = torch.nn.utils.parametrizations.weight_norm(torch.nn.Conv1d(1, 1, 3))
        x = torch.randn(1, 1, 5, requires_grad=True)
        self.run_test(model, x)

        model = torch.nn.utils.parametrizations.weight_norm(torch.nn.Conv1d(1, 1, 3), dim=-2)
        x = torch.randn(1, 1, 5, requires_grad=True)
        self.run_test(model, x)

        model = torch.nn.utils.parametrizations.weight_norm(torch.nn.Conv1d(3, 6, 3), name="weight")
        x = torch.randn(3, 3, 5, requires_grad=True)
        self.run_test(model, x)
//  [python]
def __init__(self, n_in_channels, n_out_channels, n_layers, n_channels, kernel_size=3, end_kernel_size=5, dilations=None, seperable_conv=False, res_skip=True, merge_res_skip=False, gated_unit='GTU'):
        super(WN, self).__init__()
        assert(kernel_size % 2 == 1)
        assert(n_channels % 2 == 0)
        self.n_layers = n_layers
        self.n_channels = n_channels
        self.gated_unit = get_gate_func(gated_unit)
        self.merge_res_skip = merge_res_skip
        
        self.in_layers = nn.ModuleList()
        self.res_skip_layers = nn.ModuleList()
        
        start = nn.Conv1d(n_in_channels, n_channels, 1)
        start = nn.utils.weight_norm(start, name='weight')
        self.start = start
        
        end = nn.Conv1d(n_channels, n_out_channels, end_kernel_size, padding=(end_kernel_size-1)//2)
        end = nn.utils.weight_norm(end, name='weight')
        self.end = end
        
        if type(dilations) == int:
            dilations = [dilations,]*n_layers # constant dilation if using int
            print("WARNING!! Using constant dilation factor for WN in_layer dilation width.")
        for i in range(n_layers):
            dilation = 2 ** i if dilations is None else dilations[i]
            padding = int((kernel_size*dilation - dilation)/2)
            if (not seperable_conv) or (kernel_size == 1):
                in_layer = nn.Conv1d(n_channels, 2*n_channels, kernel_size,
                                           dilation=dilation, padding=padding, padding_mode='zeros')
                in_layer = nn.utils.weight_norm(in_layer, name='weight')
            else:
                depthwise = nn.Conv1d(n_channels, n_channels, kernel_size,
                                    dilation=dilation, padding=padding, padding_mode='zeros', groups=n_channels)
                depthwise = nn.utils.weight_norm(depthwise, name='weight')
                pointwise = nn.Conv1d(n_channels, 2*n_channels, 1,
                                    dilation=dilation, padding=0)
                pointwise = nn.utils.weight_norm(pointwise, name='weight')
                in_layer = torch.nn.Sequential(depthwise, pointwise)
            self.in_layers.append(in_layer)
            
            # last one is not necessary
            if i < n_layers - 1 and not self.merge_res_skip:
                res_skip_channels = 2*n_channels
            else:
                res_skip_channels = n_channels
            
            if res_skip:
                res_skip_layer = nn.Conv1d(n_channels, res_skip_channels, 1)
                res_skip_layer = nn.utils.weight_norm(res_skip_layer, name='weight')
                self.res_skip_layers.append(res_skip_layer)
//  [python]
def __init__(self, n_in_channels, n_out_channels, n_layers, n_channels, kernel_size=3, end_kernel_size=5, dilations=None, seperable_conv=False, res_skip=True, merge_res_skip=False, gated_unit='GTU'):
        super(WN, self).__init__()
        assert(kernel_size % 2 == 1)
        assert(n_channels % 2 == 0)
        self.n_layers = n_layers
        self.n_channels = n_channels
        self.gated_unit = get_gate_func(gated_unit)
        self.merge_res_skip = merge_res_skip
        
        self.in_layers = nn.ModuleList()
        self.res_skip_layers = nn.ModuleList()
        
        start = nn.Conv1d(n_in_channels, n_channels, 1)
        start = nn.utils.parametrizations.weight_norm(start, name='weight')
        self.start = start
        
        end = nn.Conv1d(n_channels, n_out_channels, end_kernel_size, padding=(end_kernel_size-1)//2)
        end = nn.utils.parametrizations.weight_norm(end, name='weight')
        self.end = end
        
        if type(dilations) == int:
            dilations = [dilations,]*n_layers # constant dilation if using int
            print("WARNING!! Using constant dilation factor for WN in_layer dilation width.")
        for i in range(n_layers):
            dilation = 2 ** i if dilations is None else dilations[i]
            padding = int((kernel_size*dilation - dilation)/2)
            if (not seperable_conv) or (kernel_size == 1):
                in_layer = nn.Conv1d(n_channels, 2*n_channels, kernel_size,
                                           dilation=dilation, padding=padding, padding_mode='zeros')
                in_layer = nn.utils.parametrizations.weight_norm(in_layer, name='weight')
            else:
                depthwise = nn.Conv1d(n_channels, n_channels, kernel_size,
                                    dilation=dilation, padding=padding, padding_mode='zeros', groups=n_channels)
                depthwise = nn.utils.parametrizations.weight_norm(depthwise, name='weight')
                pointwise = nn.Conv1d(n_channels, 2*n_channels, 1,
                                    dilation=dilation, padding=0)
                pointwise = nn.utils.parametrizations.weight_norm(pointwise, name='weight')
                in_layer = torch.nn.Sequential(depthwise, pointwise)
            self.in_layers.append(in_layer)
            
            # last one is not necessary
            if i < n_layers - 1 and not self.merge_res_skip:
                res_skip_channels = 2*n_channels
            else:
                res_skip_channels = n_channels
            
            if res_skip:
                res_skip_layer = nn.Conv1d(n_channels, res_skip_channels, 1)
                res_skip_layer = nn.utils.parametrizations.weight_norm(res_skip_layer, name='weight')
                self.res_skip_layers.append(res_skip_layer)
//  [python]
def __init__(self, ni, nf, kernel_size=None, ks=3, stride=1, padding='same', bias=None, bias_std=0.01, norm='Batch', zero_norm=False, bn_1st=True,
                 act=nn.ReLU, act_kwargs={}, init='auto', dropout=0., xtra=None, coord=False, separable=False,  **kwargs):
        kernel_size = kernel_size or ks
        ndim = 1
        layers = [AddCoords1d()] if coord else []
        norm_type = getattr(NormType,f"{snake2camel(norm)}{'Zero' if zero_norm else ''}") if norm is not None else None
        bn = norm_type in (NormType.Batch, NormType.BatchZero)
        inn = norm_type in (NormType.Instance, NormType.InstanceZero)
        if bias is None: bias = not (bn or inn)
        if separable: conv = SeparableConv1d(ni + coord, nf, ks=kernel_size, bias=bias, stride=stride, padding=padding, **kwargs)
        else: conv = Conv1d(ni + coord, nf, ks=kernel_size, bias=bias, stride=stride, padding=padding, **kwargs)
        act = None if act is None else act(**act_kwargs)
        if not separable: init_linear(conv, act, init=init, bias_std=bias_std)
        if   norm_type==NormType.Weight:   conv = weight_norm(conv)
        elif norm_type==NormType.Spectral: conv = spectral_norm(conv)
        layers += [conv]
        act_bn = []        
        if act is not None: act_bn.append(act)
        if bn: act_bn.append(BatchNorm(nf, norm_type=norm_type, ndim=ndim))
        if inn: act_bn.append(InstanceNorm(nf, norm_type=norm_type, ndim=ndim))
        if bn_1st: act_bn.reverse()
        if dropout: layers += [nn.Dropout(dropout)]
        layers += act_bn
        if xtra: layers.append(xtra)
        super().__init__(*layers)
//  [python]
def __init__(self, arch="resnet50", checkpoint_path=None, num_classes=126, bottleneck_dim=256):
        super().__init__()

        self.arch = arch
        self.bottleneck_dim = bottleneck_dim
        self.weight_norm_dim = 0

        # 1) ResNet backbone (up to penultimate layer)
        if not self.use_bottleneck:
            model = models.__dict__[self.arch](pretrained=True)
            modules = list(model.children())[:-1]
            self.encoder = torch.nn.Sequential(*modules)
            self._output_dim = model.fc.in_features
        # 2) ResNet backbone + bottlenck (last fc as bottleneck)
        else:
            model = models.__dict__[self.arch](pretrained=True)
            model.fc = torch.nn.Linear(model.fc.in_features, self.bottleneck_dim)
            bn = torch.nn.BatchNorm1d(self.bottleneck_dim)
            self.encoder = torch.nn.Sequential(model, bn)
            self._output_dim = self.bottleneck_dim

        self.fc = torch.nn.Linear(self.output_dim, num_classes)

        if self.use_weight_norm:
            self.fc = torch.nn.utils.weight_norm(self.fc, dim=self.weight_norm_dim)

        if checkpoint_path:
            self.load_from_checkpoint(checkpoint_path)
        else:
            logger.warning(f"No checkpoint path was specified. Continue with ImageNet pre-trained weights!")

        # add input normalization to the model
        self.encoder = nn.Sequential(ImageNormalizer((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)), self.encoder)
//  [python]
def __init__(self, arch="resnet50", checkpoint_path=None, num_classes=126, bottleneck_dim=256):
        super().__init__()

        self.arch = arch
        self.bottleneck_dim = bottleneck_dim
        self.weight_norm_dim = 0

        # 1) ResNet backbone (up to penultimate layer)
        if not self.use_bottleneck:
            model = models.__dict__[self.arch](pretrained=True)
            modules = list(model.children())[:-1]
            self.encoder = torch.nn.Sequential(*modules)
            self._output_dim = model.fc.in_features
        # 2) ResNet backbone + bottlenck (last fc as bottleneck)
        else:
            model = models.__dict__[self.arch](pretrained=True)
            model.fc = torch.nn.Linear(model.fc.in_features, self.bottleneck_dim)
            bn = torch.nn.BatchNorm1d(self.bottleneck_dim)
            self.encoder = torch.nn.Sequential(model, bn)
            self._output_dim = self.bottleneck_dim

        self.fc = torch.nn.Linear(self.output_dim, num_classes)

        if self.use_weight_norm:
            self.fc = torch.nn.utils.parametrizations.weight_norm(self.fc, dim=self.weight_norm_dim)

        if checkpoint_path:
            self.load_from_checkpoint(checkpoint_path)
        else:
            logger.warning(f"No checkpoint path was specified. Continue with ImageNet pre-trained weights!")

        # add input normalization to the model
        self.encoder = nn.Sequential(ImageNormalizer((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)), self.encoder)
//  [python]
def __init__(self, hidden_channels: int, kernel_size: int, dilation_rate: int, n_layers: int, gin_channels: int = 0, p_dropout: int = 0):
        super(WN, self).__init__()
        assert (kernel_size % 2 == 1)
        self.hidden_channels = hidden_channels
        self.kernel_size = kernel_size,
        self.dilation_rate = dilation_rate
        self.n_layers = n_layers
        self.gin_channels = gin_channels
        self.p_dropout = p_dropout

        self.in_layers = torch.nn.ModuleList()
        self.res_skip_layers = torch.nn.ModuleList()
        self.drop = nn.Dropout(p_dropout)

        if gin_channels != 0:
            cond_layer = torch.nn.Conv1d(gin_channels, 2*hidden_channels*n_layers, 1)
            self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name='weight')

        for i in range(n_layers):
            dilation = dilation_rate ** i
            padding = int((kernel_size * dilation - dilation) / 2)
            in_layer = torch.nn.Conv1d(hidden_channels, 2*hidden_channels, kernel_size, dilation=dilation, padding=padding)
            in_layer = torch.nn.utils.weight_norm(in_layer, name='weight')
            self.in_layers.append(in_layer)

            # last one is not necessary
            if i < n_layers - 1:
                res_skip_channels = 2 * hidden_channels
            else:
                res_skip_channels = hidden_channels

            res_skip_layer = torch.nn.Conv1d(
                hidden_channels, res_skip_channels, 1)
            res_skip_layer = torch.nn.utils.weight_norm(
                res_skip_layer, name='weight')
            self.res_skip_layers.append(res_skip_layer)
//  [python]
def __init__(self, hidden_channels: int, kernel_size: int, dilation_rate: int, n_layers: int, gin_channels: int = 0, p_dropout: int = 0):
        super(WN, self).__init__()
        assert (kernel_size % 2 == 1)
        self.hidden_channels = hidden_channels
        self.kernel_size = kernel_size,
        self.dilation_rate = dilation_rate
        self.n_layers = n_layers
        self.gin_channels = gin_channels
        self.p_dropout = p_dropout

        self.in_layers = torch.nn.ModuleList()
        self.res_skip_layers = torch.nn.ModuleList()
        self.drop = nn.Dropout(p_dropout)

        if gin_channels != 0:
            cond_layer = torch.nn.Conv1d(gin_channels, 2*hidden_channels*n_layers, 1)
            self.cond_layer = torch.nn.utils.parametrizations.weight_norm(cond_layer, name='weight')

        for i in range(n_layers):
            dilation = dilation_rate ** i
            padding = int((kernel_size * dilation - dilation) / 2)
            in_layer = torch.nn.Conv1d(hidden_channels, 2*hidden_channels, kernel_size, dilation=dilation, padding=padding)
            in_layer = torch.nn.utils.parametrizations.weight_norm(in_layer, name='weight')
            self.in_layers.append(in_layer)

            # last one is not necessary
            if i < n_layers - 1:
                res_skip_channels = 2 * hidden_channels
            else:
                res_skip_channels = hidden_channels

            res_skip_layer = torch.nn.Conv1d(
                hidden_channels, res_skip_channels, 1)
            res_skip_layer = torch.nn.utils.parametrizations.weight_norm(
                res_skip_layer, name='weight')
            self.res_skip_layers.append(res_skip_layer)