VoiceConversionWebUI / infer_pack /models.py

lj1995

Upload 67 files

1c75048 over 1 year ago

No virus

24.2 kB

	import math,pdb,os
	from time import time as ttime
	import torch
	from torch import nn
	from torch.nn import functional as F
	from infer_pack import modules
	from infer_pack import attentions
	from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d
	from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
	from infer_pack.commons import init_weights
	import numpy as np
	from infer_pack import commons
	class TextEncoder256(nn.Module):
	def __init__(
	self, out_channels, hidden_channels, filter_channels, n_heads, n_layers, kernel_size, p_dropout, f0=True ):
	super().__init__()
	self.out_channels = out_channels
	self.hidden_channels = hidden_channels
	self.filter_channels = filter_channels
	self.n_heads = n_heads
	self.n_layers = n_layers
	self.kernel_size = kernel_size
	self.p_dropout = p_dropout
	self.emb_phone = nn.Linear(256, hidden_channels)
	self.lrelu=nn.LeakyReLU(0.1,inplace=True)
	if(f0==True):
	self.emb_pitch = nn.Embedding(256, hidden_channels) # pitch 256
	self.encoder = attentions.Encoder(
	hidden_channels, filter_channels, n_heads, n_layers, kernel_size, p_dropout
	)
	self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)

	def forward(self, phone, pitch, lengths):
	if(pitch==None):
	x = self.emb_phone(phone)
	else:
	x = self.emb_phone(phone) + self.emb_pitch(pitch)
	x = x * math.sqrt(self.hidden_channels) # [b, t, h]
	x=self.lrelu(x)
	x = torch.transpose(x, 1, -1) # [b, h, t]
	x_mask = torch.unsqueeze(commons.sequence_mask(lengths, x.size(2)), 1).to(
	x.dtype
	)
	x = self.encoder(x * x_mask, x_mask)
	stats = self.proj(x) * x_mask

	m, logs = torch.split(stats, self.out_channels, dim=1)
	return m, logs, x_mask
	class TextEncoder256km(nn.Module):
	def __init__(
	self, out_channels, hidden_channels, filter_channels, n_heads, n_layers, kernel_size, p_dropout, f0=True ):
	super().__init__()
	self.out_channels = out_channels
	self.hidden_channels = hidden_channels
	self.filter_channels = filter_channels
	self.n_heads = n_heads
	self.n_layers = n_layers
	self.kernel_size = kernel_size
	self.p_dropout = p_dropout
	# self.emb_phone = nn.Linear(256, hidden_channels)
	self.emb_phone = nn.Embedding(500, hidden_channels)
	self.lrelu=nn.LeakyReLU(0.1,inplace=True)
	if(f0==True):
	self.emb_pitch = nn.Embedding(256, hidden_channels) # pitch 256
	self.encoder = attentions.Encoder(
	hidden_channels, filter_channels, n_heads, n_layers, kernel_size, p_dropout
	)
	self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)

	def forward(self, phone, pitch, lengths):
	if(pitch==None):
	x = self.emb_phone(phone)
	else:
	x = self.emb_phone(phone) + self.emb_pitch(pitch)
	x = x * math.sqrt(self.hidden_channels) # [b, t, h]
	x=self.lrelu(x)
	x = torch.transpose(x, 1, -1) # [b, h, t]
	x_mask = torch.unsqueeze(commons.sequence_mask(lengths, x.size(2)), 1).to(
	x.dtype
	)
	x = self.encoder(x * x_mask, x_mask)
	stats = self.proj(x) * x_mask

	m, logs = torch.split(stats, self.out_channels, dim=1)
	return m, logs, x_mask
	class ResidualCouplingBlock(nn.Module):
	def __init__(
	self,
	channels,
	hidden_channels,
	kernel_size,
	dilation_rate,
	n_layers,
	n_flows=4,
	gin_channels=0,
	):
	super().__init__()
	self.channels = channels
	self.hidden_channels = hidden_channels
	self.kernel_size = kernel_size
	self.dilation_rate = dilation_rate
	self.n_layers = n_layers
	self.n_flows = n_flows
	self.gin_channels = gin_channels

	self.flows = nn.ModuleList()
	for i in range(n_flows):
	self.flows.append(
	modules.ResidualCouplingLayer(
	channels,
	hidden_channels,
	kernel_size,
	dilation_rate,
	n_layers,
	gin_channels=gin_channels,
	mean_only=True,
	)
	)
	self.flows.append(modules.Flip())

	def forward(self, x, x_mask, g=None, reverse=False):
	if not reverse:
	for flow in self.flows:
	x, _ = flow(x, x_mask, g=g, reverse=reverse)
	else:
	for flow in reversed(self.flows):
	x = flow(x, x_mask, g=g, reverse=reverse)
	return x

	def remove_weight_norm(self):
	for i in range(self.n_flows):
	self.flows[i * 2].remove_weight_norm()
	class PosteriorEncoder(nn.Module):
	def __init__(
	self,
	in_channels,
	out_channels,
	hidden_channels,
	kernel_size,
	dilation_rate,
	n_layers,
	gin_channels=0,
	):
	super().__init__()
	self.in_channels = in_channels
	self.out_channels = out_channels
	self.hidden_channels = hidden_channels
	self.kernel_size = kernel_size
	self.dilation_rate = dilation_rate
	self.n_layers = n_layers
	self.gin_channels = gin_channels

	self.pre = nn.Conv1d(in_channels, hidden_channels, 1)
	self.enc = modules.WN(
	hidden_channels,
	kernel_size,
	dilation_rate,
	n_layers,
	gin_channels=gin_channels,
	)
	self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)

	def forward(self, x, x_lengths, g=None):
	x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(
	x.dtype
	)
	x = self.pre(x) * x_mask
	x = self.enc(x, x_mask, g=g)
	stats = self.proj(x) * x_mask
	m, logs = torch.split(stats, self.out_channels, dim=1)
	z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask
	return z, m, logs, x_mask

	def remove_weight_norm(self):
	self.enc.remove_weight_norm()
	class Generator(torch.nn.Module):
	def __init__(
	self,
	initial_channel,
	resblock,
	resblock_kernel_sizes,
	resblock_dilation_sizes,
	upsample_rates,
	upsample_initial_channel,
	upsample_kernel_sizes,
	gin_channels=0,
	):
	super(Generator, self).__init__()
	self.num_kernels = len(resblock_kernel_sizes)
	self.num_upsamples = len(upsample_rates)
	self.conv_pre = Conv1d(
	initial_channel, upsample_initial_channel, 7, 1, padding=3
	)
	resblock = modules.ResBlock1 if resblock == "1" else modules.ResBlock2

	self.ups = nn.ModuleList()
	for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
	self.ups.append(
	weight_norm(
	ConvTranspose1d(
	upsample_initial_channel // (2**i),
	upsample_initial_channel // (2 ** (i + 1)),
	k,
	u,
	padding=(k - u) // 2,
	)
	)
	)

	self.resblocks = nn.ModuleList()
	for i in range(len(self.ups)):
	ch = upsample_initial_channel // (2 ** (i + 1))
	for j, (k, d) in enumerate(
	zip(resblock_kernel_sizes, resblock_dilation_sizes)
	):
	self.resblocks.append(resblock(ch, k, d))

	self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False)
	self.ups.apply(init_weights)

	if gin_channels != 0:
	self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1)

	def forward(self, x, g=None):
	x = self.conv_pre(x)
	if g is not None:
	x = x + self.cond(g)

	for i in range(self.num_upsamples):
	x = F.leaky_relu(x, modules.LRELU_SLOPE)
	x = self.ups[i](x)
	xs = None
	for j in range(self.num_kernels):
	if xs is None:
	xs = self.resblocks[i * self.num_kernels + j](x)
	else:
	xs += self.resblocks[i * self.num_kernels + j](x)
	x = xs / self.num_kernels
	x = F.leaky_relu(x)
	x = self.conv_post(x)
	x = torch.tanh(x)

	return x

	def remove_weight_norm(self):
	for l in self.ups:
	remove_weight_norm(l)
	for l in self.resblocks:
	l.remove_weight_norm()
	class SineGen(torch.nn.Module):
	""" Definition of sine generator
	SineGen(samp_rate, harmonic_num = 0,
	sine_amp = 0.1, noise_std = 0.003,
	voiced_threshold = 0,
	flag_for_pulse=False)
	samp_rate: sampling rate in Hz
	harmonic_num: number of harmonic overtones (default 0)
	sine_amp: amplitude of sine-wavefrom (default 0.1)
	noise_std: std of Gaussian noise (default 0.003)
	voiced_thoreshold: F0 threshold for U/V classification (default 0)
	flag_for_pulse: this SinGen is used inside PulseGen (default False)
	Note: when flag_for_pulse is True, the first time step of a voiced
	segment is always sin(np.pi) or cos(0)
	"""

	def __init__(self, samp_rate, harmonic_num=0,
	sine_amp=0.1, noise_std=0.003,
	voiced_threshold=0,
	flag_for_pulse=False):
	super(SineGen, self).__init__()
	self.sine_amp = sine_amp
	self.noise_std = noise_std
	self.harmonic_num = harmonic_num
	self.dim = self.harmonic_num + 1
	self.sampling_rate = samp_rate
	self.voiced_threshold = voiced_threshold

	def _f02uv(self, f0):
	# generate uv signal
	uv = torch.ones_like(f0)
	uv = uv * (f0 > self.voiced_threshold)
	return uv

	def forward(self, f0,upp):
	""" sine_tensor, uv = forward(f0)
	input F0: tensor(batchsize=1, length, dim=1)
	f0 for unvoiced steps should be 0
	output sine_tensor: tensor(batchsize=1, length, dim)
	output uv: tensor(batchsize=1, length, 1)
	"""
	with torch.no_grad():
	f0 = f0[:, None].transpose(1, 2)
	f0_buf = torch.zeros(f0.shape[0], f0.shape[1], self.dim,device=f0.device)
	# fundamental component
	f0_buf[:, :, 0] = f0[:, :, 0]
	for idx in np.arange(self.harmonic_num):f0_buf[:, :, idx + 1] = f0_buf[:, :, 0] * (idx + 2)# idx + 2: the (idx+1)-th overtone, (idx+2)-th harmonic
	rad_values = (f0_buf / self.sampling_rate) % 1###%1意味着n_har的乘积无法后处理优化
	rand_ini = torch.rand(f0_buf.shape[0], f0_buf.shape[2], device=f0_buf.device)
	rand_ini[:, 0] = 0
	rad_values[:, 0, :] = rad_values[:, 0, :] + rand_ini
	tmp_over_one = torch.cumsum(rad_values, 1)# % 1 #####%1意味着后面的cumsum无法再优化
	tmp_over_one*=upp
	tmp_over_one=F.interpolate(tmp_over_one.transpose(2, 1), scale_factor=upp, mode='linear', align_corners=True).transpose(2, 1)
	rad_values=F.interpolate(rad_values.transpose(2, 1), scale_factor=upp, mode='nearest').transpose(2, 1)#######
	tmp_over_one%=1
	tmp_over_one_idx = (tmp_over_one[:, 1:, :] - tmp_over_one[:, :-1, :]) < 0
	cumsum_shift = torch.zeros_like(rad_values)
	cumsum_shift[:, 1:, :] = tmp_over_one_idx * -1.0
	sine_waves = torch.sin(torch.cumsum(rad_values + cumsum_shift, dim=1) * 2 * np.pi)
	sine_waves = sine_waves * self.sine_amp
	uv = self._f02uv(f0)
	uv = F.interpolate(uv.transpose(2, 1), scale_factor=upp, mode='nearest').transpose(2, 1)
	noise_amp = uv * self.noise_std + (1 - uv) * self.sine_amp / 3
	noise = noise_amp * torch.randn_like(sine_waves)
	sine_waves = sine_waves * uv + noise
	return sine_waves, uv, noise
	class SourceModuleHnNSF(torch.nn.Module):
	""" SourceModule for hn-nsf
	SourceModule(sampling_rate, harmonic_num=0, sine_amp=0.1,
	add_noise_std=0.003, voiced_threshod=0)
	sampling_rate: sampling_rate in Hz
	harmonic_num: number of harmonic above F0 (default: 0)
	sine_amp: amplitude of sine source signal (default: 0.1)
	add_noise_std: std of additive Gaussian noise (default: 0.003)
	note that amplitude of noise in unvoiced is decided
	by sine_amp
	voiced_threshold: threhold to set U/V given F0 (default: 0)
	Sine_source, noise_source = SourceModuleHnNSF(F0_sampled)
	F0_sampled (batchsize, length, 1)
	Sine_source (batchsize, length, 1)
	noise_source (batchsize, length 1)
	uv (batchsize, length, 1)
	"""

	def __init__(self, sampling_rate, harmonic_num=0, sine_amp=0.1,
	add_noise_std=0.003, voiced_threshod=0,is_half=True):
	super(SourceModuleHnNSF, self).__init__()

	self.sine_amp = sine_amp
	self.noise_std = add_noise_std
	self.is_half=is_half
	# to produce sine waveforms
	self.l_sin_gen = SineGen(sampling_rate, harmonic_num,
	sine_amp, add_noise_std, voiced_threshod)

	# to merge source harmonics into a single excitation
	self.l_linear = torch.nn.Linear(harmonic_num + 1, 1)
	self.l_tanh = torch.nn.Tanh()

	def forward(self, x,upp=None):
	sine_wavs, uv, _ = self.l_sin_gen(x,upp)
	if(self.is_half==True):sine_wavs=sine_wavs.half()
	sine_merge = self.l_tanh(self.l_linear(sine_wavs))
	return sine_merge,None,None# noise, uv
	class GeneratorNSF(torch.nn.Module):
	def __init__(
	self,
	initial_channel,
	resblock,
	resblock_kernel_sizes,
	resblock_dilation_sizes,
	upsample_rates,
	upsample_initial_channel,
	upsample_kernel_sizes,
	gin_channels=0,
	sr=40000,
	is_half=False
	):
	super(GeneratorNSF, self).__init__()
	self.num_kernels = len(resblock_kernel_sizes)
	self.num_upsamples = len(upsample_rates)

	self.f0_upsamp = torch.nn.Upsample(scale_factor=np.prod(upsample_rates))
	self.m_source = SourceModuleHnNSF(
	sampling_rate=sr,
	harmonic_num=0,
	is_half=is_half
	)
	self.noise_convs = nn.ModuleList()
	self.conv_pre = Conv1d(
	initial_channel, upsample_initial_channel, 7, 1, padding=3
	)
	resblock = modules.ResBlock1 if resblock == "1" else modules.ResBlock2

	self.ups = nn.ModuleList()
	for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
	c_cur = upsample_initial_channel // (2 ** (i + 1))
	self.ups.append(
	weight_norm(
	ConvTranspose1d(
	upsample_initial_channel // (2**i),
	upsample_initial_channel // (2 ** (i + 1)),
	k,
	u,
	padding=(k - u) // 2,
	)
	)
	)
	if i + 1 < len(upsample_rates):
	stride_f0 = np.prod(upsample_rates[i + 1:])
	self.noise_convs.append(Conv1d(
	1, c_cur, kernel_size=stride_f0 * 2, stride=stride_f0, padding=stride_f0 // 2))
	else:
	self.noise_convs.append(Conv1d(1, c_cur, kernel_size=1))

	self.resblocks = nn.ModuleList()
	for i in range(len(self.ups)):
	ch = upsample_initial_channel // (2 ** (i + 1))
	for j, (k, d) in enumerate(
	zip(resblock_kernel_sizes, resblock_dilation_sizes)
	):
	self.resblocks.append(resblock(ch, k, d))

	self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False)
	self.ups.apply(init_weights)

	if gin_channels != 0:
	self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1)

	self.upp=np.prod(upsample_rates)

	def forward(self, x, f0,g=None):
	har_source, noi_source, uv = self.m_source(f0,self.upp)
	har_source = har_source.transpose(1, 2)
	x = self.conv_pre(x)
	if g is not None:
	x = x + self.cond(g)

	for i in range(self.num_upsamples):
	x = F.leaky_relu(x, modules.LRELU_SLOPE)
	x = self.ups[i](x)
	x_source = self.noise_convs[i](har_source)
	x = x + x_source
	xs = None
	for j in range(self.num_kernels):
	if xs is None:
	xs = self.resblocks[i * self.num_kernels + j](x)
	else:
	xs += self.resblocks[i * self.num_kernels + j](x)
	x = xs / self.num_kernels
	x = F.leaky_relu(x)
	x = self.conv_post(x)
	x = torch.tanh(x)
	return x

	def remove_weight_norm(self):
	for l in self.ups:
	remove_weight_norm(l)
	for l in self.resblocks:
	l.remove_weight_norm()
	class SynthesizerTrnMs256NSF(nn.Module):
	"""
	Synthesizer for Training
	"""

	def __init__(
	self,
	spec_channels,
	segment_size,
	inter_channels,
	hidden_channels,
	filter_channels,
	n_heads,
	n_layers,
	kernel_size,
	p_dropout,
	resblock,
	resblock_kernel_sizes,
	resblock_dilation_sizes,
	upsample_rates,
	upsample_initial_channel,
	upsample_kernel_sizes,
	spk_embed_dim,
	gin_channels=0,
	sr=40000,
	**kwargs
	):

	super().__init__()
	self.spec_channels = spec_channels
	self.inter_channels = inter_channels
	self.hidden_channels = hidden_channels
	self.filter_channels = filter_channels
	self.n_heads = n_heads
	self.n_layers = n_layers
	self.kernel_size = kernel_size
	self.p_dropout = p_dropout
	self.resblock = resblock
	self.resblock_kernel_sizes = resblock_kernel_sizes
	self.resblock_dilation_sizes = resblock_dilation_sizes
	self.upsample_rates = upsample_rates
	self.upsample_initial_channel = upsample_initial_channel
	self.upsample_kernel_sizes = upsample_kernel_sizes
	self.segment_size = segment_size
	self.gin_channels = gin_channels
	self.spk_embed_dim=spk_embed_dim
	self.enc_p = TextEncoder256(
	inter_channels,
	hidden_channels,
	filter_channels,
	n_heads,
	n_layers,
	kernel_size,
	p_dropout,
	)
	self.dec = GeneratorNSF(
	inter_channels,
	resblock,
	resblock_kernel_sizes,
	resblock_dilation_sizes,
	upsample_rates,
	upsample_initial_channel,
	upsample_kernel_sizes,
	gin_channels=0,
	sr=sr,
	is_half=kwargs["is_half"]
	)
	self.enc_q = PosteriorEncoder(
	spec_channels,
	inter_channels,
	hidden_channels,
	5,
	1,
	16,
	gin_channels=gin_channels,
	)
	self.flow = ResidualCouplingBlock(
	inter_channels, hidden_channels, 5, 1, 3, gin_channels=gin_channels
	)
	self.emb_g = nn.Linear(self.spk_embed_dim, gin_channels)

	def remove_weight_norm(self):
	self.dec.remove_weight_norm()
	self.flow.remove_weight_norm()
	self.enc_q.remove_weight_norm()

	def infer(self, phone, phone_lengths, pitch,pitchf, ds,max_len=None):
	m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths)
	if("float16"in str(m_p.dtype)):ds=ds.half()
	ds=ds.to(m_p.device)
	g = self.emb_g(ds).unsqueeze(-1) # [b, h, 1]#
	z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66) * x_mask

	z = self.flow(z_p, x_mask, g=g, reverse=True)
	o = self.dec((z * x_mask)[:, :, :max_len],pitchf, g=None)
	return o, x_mask, (z, z_p, m_p, logs_p)
	class SynthesizerTrn256NSFkm(nn.Module):
	"""
	Synthesizer for Training
	"""

	def __init__(
	self,
	spec_channels,
	segment_size,
	inter_channels,
	hidden_channels,
	filter_channels,
	n_heads,
	n_layers,
	kernel_size,
	p_dropout,
	resblock,
	resblock_kernel_sizes,
	resblock_dilation_sizes,
	upsample_rates,
	upsample_initial_channel,
	upsample_kernel_sizes,
	spk_embed_dim,
	gin_channels=0,
	sr=40000,
	**kwargs
	):

	super().__init__()
	self.spec_channels = spec_channels
	self.inter_channels = inter_channels
	self.hidden_channels = hidden_channels
	self.filter_channels = filter_channels
	self.n_heads = n_heads
	self.n_layers = n_layers
	self.kernel_size = kernel_size
	self.p_dropout = p_dropout
	self.resblock = resblock
	self.resblock_kernel_sizes = resblock_kernel_sizes
	self.resblock_dilation_sizes = resblock_dilation_sizes
	self.upsample_rates = upsample_rates
	self.upsample_initial_channel = upsample_initial_channel
	self.upsample_kernel_sizes = upsample_kernel_sizes
	self.segment_size = segment_size
	self.gin_channels = gin_channels

	self.enc_p = TextEncoder256km(
	inter_channels,
	hidden_channels,
	filter_channels,
	n_heads,
	n_layers,
	kernel_size,
	p_dropout,
	)
	self.dec = GeneratorNSF(
	inter_channels,
	resblock,
	resblock_kernel_sizes,
	resblock_dilation_sizes,
	upsample_rates,
	upsample_initial_channel,
	upsample_kernel_sizes,
	gin_channels=0,
	sr=sr,
	is_half=kwargs["is_half"]
	)
	self.enc_q = PosteriorEncoder(
	spec_channels,
	inter_channels,
	hidden_channels,
	5,
	1,
	16,
	gin_channels=gin_channels,
	)
	self.flow = ResidualCouplingBlock(
	inter_channels, hidden_channels, 5, 1, 3, gin_channels=gin_channels
	)

	def remove_weight_norm(self):
	self.dec.remove_weight_norm()
	self.flow.remove_weight_norm()
	self.enc_q.remove_weight_norm()

	def forward(self, phone, phone_lengths, pitch, pitchf, y, y_lengths):
	m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths)

	z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=None)
	z_p = self.flow(z, y_mask, g=None)

	z_slice, ids_slice = commons.rand_slice_segments(
	z, y_lengths, self.segment_size
	)

	pitchf = commons.slice_segments2(
	pitchf, ids_slice, self.segment_size
	)
	o = self.dec(z_slice, pitchf,g=None)
	return o, ids_slice, x_mask, y_mask, (z, z_p, m_p, logs_p, m_q, logs_q)

	def infer(self, phone, phone_lengths, pitch, nsff0,max_len=None):
	# torch.cuda.synchronize()
	# t0=ttime()
	m_p, logs_p, x_mask = self.enc_p(phone, pitch, phone_lengths)
	# torch.cuda.synchronize()
	# t1=ttime()
	z_p = (m_p + torch.exp(logs_p) * torch.randn_like(m_p) * 0.66) * x_mask
	# torch.cuda.synchronize()
	# t2=ttime()
	z = self.flow(z_p, x_mask, g=None, reverse=True)
	# torch.cuda.synchronize()
	# t3=ttime()
	o = self.dec((z * x_mask)[:, :, :max_len], nsff0,g=None)
	# torch.cuda.synchronize()
	# t4=ttime()
	# print(1233333333333333333333333,t1-t0,t2-t1,t3-t2,t4-t3)
	return o, x_mask, (z, z_p, m_p, logs_p)