Repository: speechbrain/speechbrain

Issue:
NaN when encountering silence files in speech_augmentation.AddNoise
So... I ran into another issue that is, when the noise file is all silence (Some noise files in the DNS dataset are all zeros, not sure why), the `noise_amplitude` [https://github.com/speechbrain/speechbrain/blob/1a558cd04dcc6002fecb2d9e400729e06cc3dcaa/speechbrain/processing/speech_augmentation.py#L135](url) will be zero, thus making the next line divided by zero and resulting in NaN noisy waveforms.

In the original DNS script they use an `activity_threshold` to filter out silence files, should we:
1. add some threshold detector too, or
2. just add a small number on the denominator to avoid INF

I can make a pull request to fix this, what do you think? @pplantinga 


Discussion:
Go ahead!

Resolving patch:
diff --git a/speechbrain/processing/speech_augmentation.py b/speechbrain/processing/speech_augmentation.py
--- a/speechbrain/processing/speech_augmentation.py
+++ b/speechbrain/processing/speech_augmentation.py
@@ -32,6 +32,9 @@ class AddNoise(torch.nn.Module):
     csv_file : str
         The name of a csv file containing the location of the
         noise audio files. If none is provided, white noise will be used.
+    csv_read : list, None, optional
+        Default: None . One data entry for the noise data should be specified.
+        If None, the csv file is expected to have only one data entry.
     order : str
         The order to iterate the csv file, from one of the
         following options: random, original, ascending, and descending.
@@ -67,6 +70,7 @@ class AddNoise(torch.nn.Module):
     def __init__(
         self,
         csv_file=None,
+        csv_read=None,
         order="random",
         do_cache=False,
         snr_low=0,
@@ -79,6 +83,7 @@ def __init__(
         super().__init__()
 
         self.csv_file = csv_file
+        self.csv_read = csv_read
         self.order = order
         self.do_cache = do_cache
         self.snr_low = snr_low
@@ -133,12 +138,13 @@ def forward(self, waveforms, lengths):
 
             # Rescale and add
             noise_amplitude = compute_amplitude(noise_waveform, noise_length)
-            noise_waveform *= new_noise_amplitude / noise_amplitude
+            noise_waveform *= new_noise_amplitude / (noise_amplitude + 1e-14)
             noisy_waveform += noise_waveform
 
         return noisy_waveform
 
     def _load_noise(self, lengths, max_length):
+        """Load a batch of noises"""
         lengths = lengths.long().squeeze(1)
         batch_size = len(lengths)
 
@@ -151,6 +157,7 @@ def _load_noise(self, lengths, max_length):
             if self.csv_file is not None:
                 self.data_loader = DataLoaderFactory(
                     csv_file=self.csv_file,
+                    csv_read=self.csv_read,
                     sentence_sorting=self.order,
                     batch_size=batch_size,
                     cache=self.do_cache,
@@ -158,21 +165,11 @@ def _load_noise(self, lengths, max_length):
                 )
                 self.noise_data = iter(self.data_loader())
 
-        # Ensure loaded noise batch has enough noises
-        noise_batch, noise_len = self._load_noise_batch()
-        while len(noise_batch) < batch_size:
-            added_noises, added_lens = self._load_noise_batch()
-            noise_batch = noise_batch.cat(added_noises)
-            noise_len = noise_len.cat(added_lens)
-
+        # Load noise to correct device
+        noise_batch, noise_len = self._load_noise_batch_of_size(batch_size)
         noise_batch = noise_batch.to(lengths.device)
         noise_len = noise_len.to(lengths.device)
 
-        # Chop or expand to correct size
-        if len(noise_batch) > batch_size:
-            noise_batch = noise_batch[:batch_size]
-            noise_len = noise_len[:batch_size]
-
         # Convert relative length to an index
         noise_len = (noise_len * noise_batch.shape[1]).long()
 
@@ -202,7 +199,46 @@ def _load_noise(self, lengths, max_length):
         noise_len = (noise_len - start_index).clamp(max=max_length).unsqueeze(1)
         return noise_batch, noise_len
 
+    def _load_noise_batch_of_size(self, batch_size):
+        """Concatenate noise batches, then chop to correct size"""
+        noise_batch, noise_lens = self._load_noise_batch()
+
+        # Expand
+        while len(noise_batch) < batch_size:
+            added_noise, added_lens = self._load_noise_batch()
+            noise_batch, noise_lens = AddNoise._concat_batch(
+                noise_batch, noise_lens, added_noise, added_lens
+            )
+
+        # Contract
+        if len(noise_batch) > batch_size:
+            noise_batch = noise_batch[:batch_size]
+            noise_lens = noise_lens[:batch_size]
+
+        return noise_batch, noise_lens
+
+    @staticmethod
+    def _concat_batch(noise_batch, noise_lens, added_noise, added_lens):
+        """Concatenate two noise batches of potentially different lengths"""
+
+        # pad shorter batch to correct length
+        noise_tensor_len = noise_batch.shape[1]
+        added_tensor_len = added_noise.shape[1]
+        pad = (0, abs(noise_tensor_len - added_tensor_len))
+        if noise_tensor_len > added_tensor_len:
+            added_noise = torch.nn.functional.pad(added_noise, pad)
+            added_lens = added_lens * added_tensor_len / noise_tensor_len
+        else:
+            noise_batch = torch.nn.functional.pad(noise_batch, pad)
+            noise_lens = noise_lens * noise_tensor_len / added_tensor_len
+
+        noise_batch = torch.cat((noise_batch, added_noise))
+        noise_lens = torch.cat((noise_lens, added_lens))
+
+        return noise_batch, noise_lens
+
     def _load_noise_batch(self):
+        """Load a batch of noises, restarting iteration if necessary."""
         try:
             wav_id, noise_batch, noise_len = next(self.noise_data)[0]
         except StopIteration:


Test patch:
diff --git a/tests/unittests/test_augment.py b/tests/unittests/test_augment.py
--- a/tests/unittests/test_augment.py
+++ b/tests/unittests/test_augment.py
@@ -6,6 +6,20 @@
 def test_add_noise(tmpdir):
     from speechbrain.processing.speech_augmentation import AddNoise
 
+    # Test concatenation of batches
+    wav_a = torch.sin(torch.arange(8000.0)).unsqueeze(0)
+    a_len = torch.ones(1)
+    wav_b = torch.cos(torch.arange(10000.0)).unsqueeze(0).repeat(2, 1)
+    b_len = torch.ones(2)
+    concat, lens = AddNoise._concat_batch(wav_a, a_len, wav_b, b_len)
+    assert concat.shape == (3, 10000)
+    assert lens.allclose(torch.Tensor([0.8, 1, 1]))
+    print(a_len)
+    print(b_len)
+    concat, lens = AddNoise._concat_batch(wav_b, b_len, wav_a, a_len)
+    assert concat.shape == (3, 10000)
+    assert lens.allclose(torch.Tensor([1, 1, 0.8]))
+
     test_waveform = torch.sin(torch.arange(16000.0)).unsqueeze(0)
     test_noise = torch.cos(torch.arange(16000.0)).unsqueeze(0)
     wav_lens = torch.ones(1)