Upload /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.10/site-packages/TTS/tts/utils/helpers.py with huggingface_hub
Browse files
home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.10/site-packages/TTS/tts/utils/helpers.py
ADDED
@@ -0,0 +1,258 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import numpy as np
|
2 |
+
import torch
|
3 |
+
from scipy.stats import betabinom
|
4 |
+
from torch.nn import functional as F
|
5 |
+
|
6 |
+
try:
|
7 |
+
from TTS.tts.utils.monotonic_align.core import maximum_path_c
|
8 |
+
|
9 |
+
CYTHON = True
|
10 |
+
except ModuleNotFoundError:
|
11 |
+
CYTHON = False
|
12 |
+
|
13 |
+
|
14 |
+
class StandardScaler:
|
15 |
+
"""StandardScaler for mean-scale normalization with the given mean and scale values."""
|
16 |
+
|
17 |
+
def __init__(self, mean: np.ndarray = None, scale: np.ndarray = None) -> None:
|
18 |
+
self.mean_ = mean
|
19 |
+
self.scale_ = scale
|
20 |
+
|
21 |
+
def set_stats(self, mean, scale):
|
22 |
+
self.mean_ = mean
|
23 |
+
self.scale_ = scale
|
24 |
+
|
25 |
+
def reset_stats(self):
|
26 |
+
delattr(self, "mean_")
|
27 |
+
delattr(self, "scale_")
|
28 |
+
|
29 |
+
def transform(self, X):
|
30 |
+
X = np.asarray(X)
|
31 |
+
X -= self.mean_
|
32 |
+
X /= self.scale_
|
33 |
+
return X
|
34 |
+
|
35 |
+
def inverse_transform(self, X):
|
36 |
+
X = np.asarray(X)
|
37 |
+
X *= self.scale_
|
38 |
+
X += self.mean_
|
39 |
+
return X
|
40 |
+
|
41 |
+
|
42 |
+
# from https://gist.github.com/jihunchoi/f1434a77df9db1bb337417854b398df1
|
43 |
+
def sequence_mask(sequence_length, max_len=None):
|
44 |
+
"""Create a sequence mask for filtering padding in a sequence tensor.
|
45 |
+
|
46 |
+
Args:
|
47 |
+
sequence_length (torch.tensor): Sequence lengths.
|
48 |
+
max_len (int, Optional): Maximum sequence length. Defaults to None.
|
49 |
+
|
50 |
+
Shapes:
|
51 |
+
- mask: :math:`[B, T_max]`
|
52 |
+
"""
|
53 |
+
if max_len is None:
|
54 |
+
max_len = sequence_length.max()
|
55 |
+
seq_range = torch.arange(max_len, dtype=sequence_length.dtype, device=sequence_length.device)
|
56 |
+
# B x T_max
|
57 |
+
return seq_range.unsqueeze(0) < sequence_length.unsqueeze(1)
|
58 |
+
|
59 |
+
|
60 |
+
def segment(x: torch.tensor, segment_indices: torch.tensor, segment_size=4, pad_short=False):
|
61 |
+
"""Segment each sample in a batch based on the provided segment indices
|
62 |
+
|
63 |
+
Args:
|
64 |
+
x (torch.tensor): Input tensor.
|
65 |
+
segment_indices (torch.tensor): Segment indices.
|
66 |
+
segment_size (int): Expected output segment size.
|
67 |
+
pad_short (bool): Pad the end of input tensor with zeros if shorter than the segment size.
|
68 |
+
"""
|
69 |
+
# pad the input tensor if it is shorter than the segment size
|
70 |
+
if pad_short and x.shape[-1] < segment_size:
|
71 |
+
x = torch.nn.functional.pad(x, (0, segment_size - x.size(2)))
|
72 |
+
|
73 |
+
segments = torch.zeros_like(x[:, :, :segment_size])
|
74 |
+
|
75 |
+
for i in range(x.size(0)):
|
76 |
+
index_start = segment_indices[i]
|
77 |
+
index_end = index_start + segment_size
|
78 |
+
x_i = x[i]
|
79 |
+
if pad_short and index_end >= x.size(2):
|
80 |
+
# pad the sample if it is shorter than the segment size
|
81 |
+
x_i = torch.nn.functional.pad(x_i, (0, (index_end + 1) - x.size(2)))
|
82 |
+
segments[i] = x_i[:, index_start:index_end]
|
83 |
+
return segments
|
84 |
+
|
85 |
+
|
86 |
+
def rand_segments(
|
87 |
+
x: torch.tensor, x_lengths: torch.tensor = None, segment_size=4, let_short_samples=False, pad_short=False
|
88 |
+
):
|
89 |
+
"""Create random segments based on the input lengths.
|
90 |
+
|
91 |
+
Args:
|
92 |
+
x (torch.tensor): Input tensor.
|
93 |
+
x_lengths (torch.tensor): Input lengths.
|
94 |
+
segment_size (int): Expected output segment size.
|
95 |
+
let_short_samples (bool): Allow shorter samples than the segment size.
|
96 |
+
pad_short (bool): Pad the end of input tensor with zeros if shorter than the segment size.
|
97 |
+
|
98 |
+
Shapes:
|
99 |
+
- x: :math:`[B, C, T]`
|
100 |
+
- x_lengths: :math:`[B]`
|
101 |
+
"""
|
102 |
+
_x_lenghts = x_lengths.clone()
|
103 |
+
B, _, T = x.size()
|
104 |
+
if pad_short:
|
105 |
+
if T < segment_size:
|
106 |
+
x = torch.nn.functional.pad(x, (0, segment_size - T))
|
107 |
+
T = segment_size
|
108 |
+
if _x_lenghts is None:
|
109 |
+
_x_lenghts = T
|
110 |
+
len_diff = _x_lenghts - segment_size
|
111 |
+
if let_short_samples:
|
112 |
+
_x_lenghts[len_diff < 0] = segment_size
|
113 |
+
len_diff = _x_lenghts - segment_size
|
114 |
+
else:
|
115 |
+
assert all(
|
116 |
+
len_diff > 0
|
117 |
+
), f" [!] At least one sample is shorter than the segment size ({segment_size}). \n {_x_lenghts}"
|
118 |
+
segment_indices = (torch.rand([B]).type_as(x) * (len_diff + 1)).long()
|
119 |
+
ret = segment(x, segment_indices, segment_size, pad_short=pad_short)
|
120 |
+
return ret, segment_indices
|
121 |
+
|
122 |
+
|
123 |
+
def average_over_durations(values, durs):
|
124 |
+
"""Average values over durations.
|
125 |
+
|
126 |
+
Shapes:
|
127 |
+
- values: :math:`[B, 1, T_de]`
|
128 |
+
- durs: :math:`[B, T_en]`
|
129 |
+
- avg: :math:`[B, 1, T_en]`
|
130 |
+
"""
|
131 |
+
durs_cums_ends = torch.cumsum(durs, dim=1).long()
|
132 |
+
durs_cums_starts = torch.nn.functional.pad(durs_cums_ends[:, :-1], (1, 0))
|
133 |
+
values_nonzero_cums = torch.nn.functional.pad(torch.cumsum(values != 0.0, dim=2), (1, 0))
|
134 |
+
values_cums = torch.nn.functional.pad(torch.cumsum(values, dim=2), (1, 0))
|
135 |
+
|
136 |
+
bs, l = durs_cums_ends.size()
|
137 |
+
n_formants = values.size(1)
|
138 |
+
dcs = durs_cums_starts[:, None, :].expand(bs, n_formants, l)
|
139 |
+
dce = durs_cums_ends[:, None, :].expand(bs, n_formants, l)
|
140 |
+
|
141 |
+
values_sums = (torch.gather(values_cums, 2, dce) - torch.gather(values_cums, 2, dcs)).float()
|
142 |
+
values_nelems = (torch.gather(values_nonzero_cums, 2, dce) - torch.gather(values_nonzero_cums, 2, dcs)).float()
|
143 |
+
|
144 |
+
avg = torch.where(values_nelems == 0.0, values_nelems, values_sums / values_nelems)
|
145 |
+
return avg
|
146 |
+
|
147 |
+
|
148 |
+
def convert_pad_shape(pad_shape):
|
149 |
+
l = pad_shape[::-1]
|
150 |
+
pad_shape = [item for sublist in l for item in sublist]
|
151 |
+
return pad_shape
|
152 |
+
|
153 |
+
|
154 |
+
def generate_path(duration, mask):
|
155 |
+
"""
|
156 |
+
Shapes:
|
157 |
+
- duration: :math:`[B, T_en]`
|
158 |
+
- mask: :math:'[B, T_en, T_de]`
|
159 |
+
- path: :math:`[B, T_en, T_de]`
|
160 |
+
"""
|
161 |
+
b, t_x, t_y = mask.shape
|
162 |
+
cum_duration = torch.cumsum(duration, 1)
|
163 |
+
|
164 |
+
cum_duration_flat = cum_duration.view(b * t_x)
|
165 |
+
path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype)
|
166 |
+
path = path.view(b, t_x, t_y)
|
167 |
+
path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1]
|
168 |
+
path = path * mask
|
169 |
+
return path
|
170 |
+
|
171 |
+
|
172 |
+
def maximum_path(value, mask):
|
173 |
+
if CYTHON:
|
174 |
+
return maximum_path_cython(value, mask)
|
175 |
+
return maximum_path_numpy(value, mask)
|
176 |
+
|
177 |
+
|
178 |
+
def maximum_path_cython(value, mask):
|
179 |
+
"""Cython optimised version.
|
180 |
+
Shapes:
|
181 |
+
- value: :math:`[B, T_en, T_de]`
|
182 |
+
- mask: :math:`[B, T_en, T_de]`
|
183 |
+
"""
|
184 |
+
value = value * mask
|
185 |
+
device = value.device
|
186 |
+
dtype = value.dtype
|
187 |
+
value = value.data.cpu().numpy().astype(np.float32)
|
188 |
+
path = np.zeros_like(value).astype(np.int32)
|
189 |
+
mask = mask.data.cpu().numpy()
|
190 |
+
|
191 |
+
t_x_max = mask.sum(1)[:, 0].astype(np.int32)
|
192 |
+
t_y_max = mask.sum(2)[:, 0].astype(np.int32)
|
193 |
+
maximum_path_c(path, value, t_x_max, t_y_max)
|
194 |
+
return torch.from_numpy(path).to(device=device, dtype=dtype)
|
195 |
+
|
196 |
+
|
197 |
+
def maximum_path_numpy(value, mask, max_neg_val=None):
|
198 |
+
"""
|
199 |
+
Monotonic alignment search algorithm
|
200 |
+
Numpy-friendly version. It's about 4 times faster than torch version.
|
201 |
+
value: [b, t_x, t_y]
|
202 |
+
mask: [b, t_x, t_y]
|
203 |
+
"""
|
204 |
+
if max_neg_val is None:
|
205 |
+
max_neg_val = -np.inf # Patch for Sphinx complaint
|
206 |
+
value = value * mask
|
207 |
+
|
208 |
+
device = value.device
|
209 |
+
dtype = value.dtype
|
210 |
+
value = value.cpu().detach().numpy()
|
211 |
+
mask = mask.cpu().detach().numpy().astype(bool)
|
212 |
+
|
213 |
+
b, t_x, t_y = value.shape
|
214 |
+
direction = np.zeros(value.shape, dtype=np.int64)
|
215 |
+
v = np.zeros((b, t_x), dtype=np.float32)
|
216 |
+
x_range = np.arange(t_x, dtype=np.float32).reshape(1, -1)
|
217 |
+
for j in range(t_y):
|
218 |
+
v0 = np.pad(v, [[0, 0], [1, 0]], mode="constant", constant_values=max_neg_val)[:, :-1]
|
219 |
+
v1 = v
|
220 |
+
max_mask = v1 >= v0
|
221 |
+
v_max = np.where(max_mask, v1, v0)
|
222 |
+
direction[:, :, j] = max_mask
|
223 |
+
|
224 |
+
index_mask = x_range <= j
|
225 |
+
v = np.where(index_mask, v_max + value[:, :, j], max_neg_val)
|
226 |
+
direction = np.where(mask, direction, 1)
|
227 |
+
|
228 |
+
path = np.zeros(value.shape, dtype=np.float32)
|
229 |
+
index = mask[:, :, 0].sum(1).astype(np.int64) - 1
|
230 |
+
index_range = np.arange(b)
|
231 |
+
for j in reversed(range(t_y)):
|
232 |
+
path[index_range, index, j] = 1
|
233 |
+
index = index + direction[index_range, index, j] - 1
|
234 |
+
path = path * mask.astype(np.float32)
|
235 |
+
path = torch.from_numpy(path).to(device=device, dtype=dtype)
|
236 |
+
return path
|
237 |
+
|
238 |
+
|
239 |
+
def beta_binomial_prior_distribution(phoneme_count, mel_count, scaling_factor=1.0):
|
240 |
+
P, M = phoneme_count, mel_count
|
241 |
+
x = np.arange(0, P)
|
242 |
+
mel_text_probs = []
|
243 |
+
for i in range(1, M + 1):
|
244 |
+
a, b = scaling_factor * i, scaling_factor * (M + 1 - i)
|
245 |
+
rv = betabinom(P, a, b)
|
246 |
+
mel_i_prob = rv.pmf(x)
|
247 |
+
mel_text_probs.append(mel_i_prob)
|
248 |
+
return np.array(mel_text_probs)
|
249 |
+
|
250 |
+
|
251 |
+
def compute_attn_prior(x_len, y_len, scaling_factor=1.0):
|
252 |
+
"""Compute attention priors for the alignment network."""
|
253 |
+
attn_prior = beta_binomial_prior_distribution(
|
254 |
+
x_len,
|
255 |
+
y_len,
|
256 |
+
scaling_factor,
|
257 |
+
)
|
258 |
+
return attn_prior # [y_len, x_len]
|