declare-lab-sutd commited on
Commit
a120d23
1 Parent(s): 3a6eceb
tools/.ipynb_checkpoints/mix-checkpoint.py ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+
3
+
4
+ def a_weight(fs, n_fft, min_db=-80.0):
5
+ freq = np.linspace(0, fs // 2, n_fft // 2 + 1)
6
+ freq_sq = np.power(freq, 2)
7
+ freq_sq[0] = 1.0
8
+ weight = 2.0 + 20.0 * (2 * np.log10(12194) + 2 * np.log10(freq_sq)
9
+ - np.log10(freq_sq + 12194 ** 2)
10
+ - np.log10(freq_sq + 20.6 ** 2)
11
+ - 0.5 * np.log10(freq_sq + 107.7 ** 2)
12
+ - 0.5 * np.log10(freq_sq + 737.9 ** 2))
13
+ weight = np.maximum(weight, min_db)
14
+
15
+ return weight
16
+
17
+
18
+ def compute_gain(sound, fs, min_db=-80.0, mode="A_weighting"):
19
+ if fs == 16000:
20
+ n_fft = 2048
21
+ elif fs == 44100:
22
+ n_fft = 4096
23
+ else:
24
+ raise Exception("Invalid fs {}".format(fs))
25
+ stride = n_fft // 2
26
+
27
+ gain = []
28
+ for i in range(0, len(sound) - n_fft + 1, stride):
29
+ if mode == "RMSE":
30
+ g = np.mean(sound[i: i + n_fft] ** 2)
31
+ elif mode == "A_weighting":
32
+ spec = np.fft.rfft(np.hanning(n_fft + 1)[:-1] * sound[i: i + n_fft])
33
+ power_spec = np.abs(spec) ** 2
34
+ a_weighted_spec = power_spec * np.power(10, a_weight(fs, n_fft) / 10)
35
+ g = np.sum(a_weighted_spec)
36
+ else:
37
+ raise Exception("Invalid mode {}".format(mode))
38
+ gain.append(g)
39
+
40
+ gain = np.array(gain)
41
+ gain = np.maximum(gain, np.power(10, min_db / 10))
42
+ gain_db = 10 * np.log10(gain)
43
+ return gain_db
44
+
45
+
46
+ def mix(sound1, sound2, r, fs):
47
+ gain1 = np.max(compute_gain(sound1, fs)) # Decibel
48
+ gain2 = np.max(compute_gain(sound2, fs))
49
+ t = 1.0 / (1 + np.power(10, (gain1 - gain2) / 20.) * (1 - r) / r)
50
+ sound = ((sound1 * t + sound2 * (1 - t)) / np.sqrt(t ** 2 + (1 - t) ** 2))
51
+ return sound
tools/.ipynb_checkpoints/torch_tools-checkpoint.py ADDED
@@ -0,0 +1,133 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torchaudio
3
+ import random
4
+ import itertools
5
+ import numpy as np
6
+ from tools.mix import mix
7
+
8
+
9
+ def normalize_wav(waveform):
10
+ waveform = waveform - torch.mean(waveform)
11
+ waveform = waveform / (torch.max(torch.abs(waveform)) + 1e-8)
12
+ return waveform * 0.5
13
+
14
+
15
+ def pad_wav(waveform, segment_length):
16
+ waveform_length = len(waveform)
17
+
18
+ if segment_length is None or waveform_length == segment_length:
19
+ return waveform
20
+ elif waveform_length > segment_length:
21
+ return waveform[:segment_length]
22
+ else:
23
+ pad_wav = torch.zeros(segment_length - waveform_length).to(waveform.device)
24
+ waveform = torch.cat([waveform, pad_wav])
25
+ return waveform
26
+
27
+
28
+ def _pad_spec(fbank, target_length=1024):
29
+ batch, n_frames, channels = fbank.shape
30
+ p = target_length - n_frames
31
+ if p > 0:
32
+ pad = torch.zeros(batch, p, channels).to(fbank.device)
33
+ fbank = torch.cat([fbank, pad], 1)
34
+ elif p < 0:
35
+ fbank = fbank[:, :target_length, :]
36
+
37
+ if channels % 2 != 0:
38
+ fbank = fbank[:, :, :-1]
39
+
40
+ return fbank
41
+
42
+
43
+ def read_wav_file(filename, segment_length):
44
+ waveform, sr = torchaudio.load(filename) # Faster!!!
45
+ try:
46
+ waveform = torchaudio.functional.resample(waveform, orig_freq=sr, new_freq=16000)[0]
47
+ except:
48
+ print ("0 length wav encountered. Setting to random:", filename)
49
+ waveform = torch.rand(160000)
50
+
51
+ try:
52
+ waveform = normalize_wav(waveform)
53
+ except:
54
+ print ("Exception normalizing:", filename)
55
+ waveform = torch.ones(160000)
56
+ waveform = pad_wav(waveform, segment_length).unsqueeze(0)
57
+ waveform = waveform / torch.max(torch.abs(waveform))
58
+ waveform = 0.5 * waveform
59
+ return waveform
60
+
61
+
62
+ def get_mel_from_wav(audio, _stft):
63
+ audio = torch.nan_to_num(torch.clip(audio, -1, 1))
64
+ audio = torch.autograd.Variable(audio, requires_grad=False)
65
+ melspec, log_magnitudes_stft, energy = _stft.mel_spectrogram(audio)
66
+ return melspec, log_magnitudes_stft, energy
67
+
68
+
69
+ def wav_to_fbank(paths, target_length=1024, fn_STFT=None):
70
+ assert fn_STFT is not None
71
+
72
+ waveform = torch.cat([read_wav_file(path, target_length * 160) for path in paths], 0) # hop size is 160
73
+
74
+ fbank, log_magnitudes_stft, energy = get_mel_from_wav(waveform, fn_STFT)
75
+ fbank = fbank.transpose(1, 2)
76
+ log_magnitudes_stft = log_magnitudes_stft.transpose(1, 2)
77
+
78
+ fbank, log_magnitudes_stft = _pad_spec(fbank, target_length), _pad_spec(
79
+ log_magnitudes_stft, target_length
80
+ )
81
+
82
+ return fbank, log_magnitudes_stft, waveform
83
+
84
+
85
+ def uncapitalize(s):
86
+ if s:
87
+ return s[:1].lower() + s[1:]
88
+ else:
89
+ return ""
90
+
91
+
92
+ def mix_wavs_and_captions(path1, path2, caption1, caption2, target_length=1024):
93
+ sound1 = read_wav_file(path1, target_length * 160)[0].numpy()
94
+ sound2 = read_wav_file(path2, target_length * 160)[0].numpy()
95
+ mixed_sound = mix(sound1, sound2, 0.5, 16000).reshape(1, -1)
96
+ mixed_caption = "{} and {}".format(caption1, uncapitalize(caption2))
97
+ return mixed_sound, mixed_caption
98
+
99
+
100
+ def augment(paths, texts, num_items=4, target_length=1024):
101
+ mixed_sounds, mixed_captions = [], []
102
+ combinations = list(itertools.combinations(list(range(len(texts))), 2))
103
+ random.shuffle(combinations)
104
+ if len(combinations) < num_items:
105
+ selected_combinations = combinations
106
+ else:
107
+ selected_combinations = combinations[:num_items]
108
+
109
+ for (i, j) in selected_combinations:
110
+ new_sound, new_caption = mix_wavs_and_captions(paths[i], paths[j], texts[i], texts[j], target_length)
111
+ mixed_sounds.append(new_sound)
112
+ mixed_captions.append(new_caption)
113
+
114
+ waveform = torch.tensor(np.concatenate(mixed_sounds, 0))
115
+ waveform = waveform / torch.max(torch.abs(waveform))
116
+ waveform = 0.5 * waveform
117
+
118
+ return waveform, mixed_captions
119
+
120
+
121
+ def augment_wav_to_fbank(paths, texts, num_items=4, target_length=1024, fn_STFT=None):
122
+ assert fn_STFT is not None
123
+
124
+ waveform, captions = augment(paths, texts)
125
+ fbank, log_magnitudes_stft, energy = get_mel_from_wav(waveform, fn_STFT)
126
+ fbank = fbank.transpose(1, 2)
127
+ log_magnitudes_stft = log_magnitudes_stft.transpose(1, 2)
128
+
129
+ fbank, log_magnitudes_stft = _pad_spec(fbank, target_length), _pad_spec(
130
+ log_magnitudes_stft, target_length
131
+ )
132
+
133
+ return fbank, log_magnitudes_stft, waveform, captions
tools/__init__.py ADDED
File without changes
tools/__pycache__/__init__.cpython-39.pyc ADDED
Binary file (144 Bytes). View file
 
tools/__pycache__/mix.cpython-39.pyc ADDED
Binary file (1.7 kB). View file
 
tools/__pycache__/torch_tools.cpython-39.pyc ADDED
Binary file (3.87 kB). View file
 
tools/mix.py ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import numpy as np
2
+
3
+
4
+ def a_weight(fs, n_fft, min_db=-80.0):
5
+ freq = np.linspace(0, fs // 2, n_fft // 2 + 1)
6
+ freq_sq = np.power(freq, 2)
7
+ freq_sq[0] = 1.0
8
+ weight = 2.0 + 20.0 * (2 * np.log10(12194) + 2 * np.log10(freq_sq)
9
+ - np.log10(freq_sq + 12194 ** 2)
10
+ - np.log10(freq_sq + 20.6 ** 2)
11
+ - 0.5 * np.log10(freq_sq + 107.7 ** 2)
12
+ - 0.5 * np.log10(freq_sq + 737.9 ** 2))
13
+ weight = np.maximum(weight, min_db)
14
+
15
+ return weight
16
+
17
+
18
+ def compute_gain(sound, fs, min_db=-80.0, mode="A_weighting"):
19
+ if fs == 16000:
20
+ n_fft = 2048
21
+ elif fs == 44100:
22
+ n_fft = 4096
23
+ else:
24
+ raise Exception("Invalid fs {}".format(fs))
25
+ stride = n_fft // 2
26
+
27
+ gain = []
28
+ for i in range(0, len(sound) - n_fft + 1, stride):
29
+ if mode == "RMSE":
30
+ g = np.mean(sound[i: i + n_fft] ** 2)
31
+ elif mode == "A_weighting":
32
+ spec = np.fft.rfft(np.hanning(n_fft + 1)[:-1] * sound[i: i + n_fft])
33
+ power_spec = np.abs(spec) ** 2
34
+ a_weighted_spec = power_spec * np.power(10, a_weight(fs, n_fft) / 10)
35
+ g = np.sum(a_weighted_spec)
36
+ else:
37
+ raise Exception("Invalid mode {}".format(mode))
38
+ gain.append(g)
39
+
40
+ gain = np.array(gain)
41
+ gain = np.maximum(gain, np.power(10, min_db / 10))
42
+ gain_db = 10 * np.log10(gain)
43
+ return gain_db
44
+
45
+
46
+ def mix(sound1, sound2, r, fs):
47
+ gain1 = np.max(compute_gain(sound1, fs)) # Decibel
48
+ gain2 = np.max(compute_gain(sound2, fs))
49
+ t = 1.0 / (1 + np.power(10, (gain1 - gain2) / 20.) * (1 - r) / r)
50
+ sound = ((sound1 * t + sound2 * (1 - t)) / np.sqrt(t ** 2 + (1 - t) ** 2))
51
+ return sound
tools/torch_tools.py ADDED
@@ -0,0 +1,133 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torchaudio
3
+ import random
4
+ import itertools
5
+ import numpy as np
6
+ from tools.mix import mix
7
+
8
+
9
+ def normalize_wav(waveform):
10
+ waveform = waveform - torch.mean(waveform)
11
+ waveform = waveform / (torch.max(torch.abs(waveform)) + 1e-8)
12
+ return waveform * 0.5
13
+
14
+
15
+ def pad_wav(waveform, segment_length):
16
+ waveform_length = len(waveform)
17
+
18
+ if segment_length is None or waveform_length == segment_length:
19
+ return waveform
20
+ elif waveform_length > segment_length:
21
+ return waveform[:segment_length]
22
+ else:
23
+ pad_wav = torch.zeros(segment_length - waveform_length).to(waveform.device)
24
+ waveform = torch.cat([waveform, pad_wav])
25
+ return waveform
26
+
27
+
28
+ def _pad_spec(fbank, target_length=1024):
29
+ batch, n_frames, channels = fbank.shape
30
+ p = target_length - n_frames
31
+ if p > 0:
32
+ pad = torch.zeros(batch, p, channels).to(fbank.device)
33
+ fbank = torch.cat([fbank, pad], 1)
34
+ elif p < 0:
35
+ fbank = fbank[:, :target_length, :]
36
+
37
+ if channels % 2 != 0:
38
+ fbank = fbank[:, :, :-1]
39
+
40
+ return fbank
41
+
42
+
43
+ def read_wav_file(filename, segment_length):
44
+ waveform, sr = torchaudio.load(filename) # Faster!!!
45
+ try:
46
+ waveform = torchaudio.functional.resample(waveform, orig_freq=sr, new_freq=16000)[0]
47
+ except:
48
+ print ("0 length wav encountered. Setting to random:", filename)
49
+ waveform = torch.rand(160000)
50
+
51
+ try:
52
+ waveform = normalize_wav(waveform)
53
+ except:
54
+ print ("Exception normalizing:", filename)
55
+ waveform = torch.ones(160000)
56
+ waveform = pad_wav(waveform, segment_length).unsqueeze(0)
57
+ waveform = waveform / torch.max(torch.abs(waveform))
58
+ waveform = 0.5 * waveform
59
+ return waveform
60
+
61
+
62
+ def get_mel_from_wav(audio, _stft):
63
+ audio = torch.nan_to_num(torch.clip(audio, -1, 1))
64
+ audio = torch.autograd.Variable(audio, requires_grad=False)
65
+ melspec, log_magnitudes_stft, energy = _stft.mel_spectrogram(audio)
66
+ return melspec, log_magnitudes_stft, energy
67
+
68
+
69
+ def wav_to_fbank(paths, target_length=1024, fn_STFT=None):
70
+ assert fn_STFT is not None
71
+
72
+ waveform = torch.cat([read_wav_file(path, target_length * 160) for path in paths], 0) # hop size is 160
73
+
74
+ fbank, log_magnitudes_stft, energy = get_mel_from_wav(waveform, fn_STFT)
75
+ fbank = fbank.transpose(1, 2)
76
+ log_magnitudes_stft = log_magnitudes_stft.transpose(1, 2)
77
+
78
+ fbank, log_magnitudes_stft = _pad_spec(fbank, target_length), _pad_spec(
79
+ log_magnitudes_stft, target_length
80
+ )
81
+
82
+ return fbank, log_magnitudes_stft, waveform
83
+
84
+
85
+ def uncapitalize(s):
86
+ if s:
87
+ return s[:1].lower() + s[1:]
88
+ else:
89
+ return ""
90
+
91
+
92
+ def mix_wavs_and_captions(path1, path2, caption1, caption2, target_length=1024):
93
+ sound1 = read_wav_file(path1, target_length * 160)[0].numpy()
94
+ sound2 = read_wav_file(path2, target_length * 160)[0].numpy()
95
+ mixed_sound = mix(sound1, sound2, 0.5, 16000).reshape(1, -1)
96
+ mixed_caption = "{} and {}".format(caption1, uncapitalize(caption2))
97
+ return mixed_sound, mixed_caption
98
+
99
+
100
+ def augment(paths, texts, num_items=4, target_length=1024):
101
+ mixed_sounds, mixed_captions = [], []
102
+ combinations = list(itertools.combinations(list(range(len(texts))), 2))
103
+ random.shuffle(combinations)
104
+ if len(combinations) < num_items:
105
+ selected_combinations = combinations
106
+ else:
107
+ selected_combinations = combinations[:num_items]
108
+
109
+ for (i, j) in selected_combinations:
110
+ new_sound, new_caption = mix_wavs_and_captions(paths[i], paths[j], texts[i], texts[j], target_length)
111
+ mixed_sounds.append(new_sound)
112
+ mixed_captions.append(new_caption)
113
+
114
+ waveform = torch.tensor(np.concatenate(mixed_sounds, 0))
115
+ waveform = waveform / torch.max(torch.abs(waveform))
116
+ waveform = 0.5 * waveform
117
+
118
+ return waveform, mixed_captions
119
+
120
+
121
+ def augment_wav_to_fbank(paths, texts, num_items=4, target_length=1024, fn_STFT=None):
122
+ assert fn_STFT is not None
123
+
124
+ waveform, captions = augment(paths, texts)
125
+ fbank, log_magnitudes_stft, energy = get_mel_from_wav(waveform, fn_STFT)
126
+ fbank = fbank.transpose(1, 2)
127
+ log_magnitudes_stft = log_magnitudes_stft.transpose(1, 2)
128
+
129
+ fbank, log_magnitudes_stft = _pad_spec(fbank, target_length), _pad_spec(
130
+ log_magnitudes_stft, target_length
131
+ )
132
+
133
+ return fbank, log_magnitudes_stft, waveform, captions