Sizes of tensors no equal while using WaveRNN in torchaudio

yuam_lu: <p>While i’m trainning the WaveRNN, i found that the size of tensor has always different and there’s an error.<br>
I had checked the source code and print the tensor in the background, i found that the tensor’s size had becoem strange after it pass self.upsample().Size shoud be torch.Size([8, 1, 132300]) but it is torch.Size([8, 1, 132300]). I don’t know how and why.(Here’s a photo, and it’s a notebook on colab:<a href="https://colab.research.google.com/drive/1zACtWU0jf--qbMF3GCHnX3anrFufioIY?usp=sharing" class="inline-onebox" rel="noopener nofollow ugc">Google Colab</a>)</p>
<p><div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/3X/b/f/bf59e22fb10fbc2fc6f593c50d0afc9b6f53e193.png" data-download-href="https://discuss.pytorch.org/uploads/default/bf59e22fb10fbc2fc6f593c50d0afc9b6f53e193" title="image"><img src="https://discuss.pytorch.org/uploads/default/optimized/3X/b/f/bf59e22fb10fbc2fc6f593c50d0afc9b6f53e193_2_690x303.png" alt="image" data-base62-sha1="riLLPsJVkfo5RAgnfKGZaZO9kkP" width="690" height="303" srcset="https://discuss.pytorch.org/uploads/default/optimized/3X/b/f/bf59e22fb10fbc2fc6f593c50d0afc9b6f53e193_2_690x303.png, https://discuss.pytorch.org/uploads/default/optimized/3X/b/f/bf59e22fb10fbc2fc6f593c50d0afc9b6f53e193_2_1035x454.png 1.5x, https://discuss.pytorch.org/uploads/default/optimized/3X/b/f/bf59e22fb10fbc2fc6f593c50d0afc9b6f53e193_2_1380x606.png 2x" data-dominant-color="2A2A2A"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">image</span><span class="informations">2428×1067 365 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div></p>
<h2><a name="error-msg-1" class="anchor" href="#error-msg-1"></a>error msg:</h2>
<h2><a name="runtimeerror-sizes-of-tensors-must-match-except-in-dimension-2-expected-size-132300-but-got-size-131600-for-tensor-number-1-in-the-list-2" class="anchor" href="#runtimeerror-sizes-of-tensors-must-match-except-in-dimension-2-expected-size-132300-but-got-size-131600-for-tensor-number-1-in-the-list-2"></a>RuntimeError: Sizes of tensors must match except in dimension 2. Expected size 132300 but got size 131600 for tensor number 1 in the list.</h2>
<h2><a name="part-of-code-3" class="anchor" href="#part-of-code-3"></a>part of code:</h2>
<h1><a name="data-processing-4" class="anchor" href="#data-processing-4"></a>data processing</h1>
<p>import IPython.display as ipd<br>
import numpy as np</p>
<p>from tqdm import tqdm</p>
<h1><a name="make-all-of-the-audio-the-same-lengthmax-len-5" class="anchor" href="#make-all-of-the-audio-the-same-lengthmax-len-5"></a>make all of the audio the same length(max len)</h1>
<p>max_len = max([len(i[‘audio’]) for i in simple_data])<br>
for i in simple_data:<br>
i[‘audio’] = np.pad(i[‘audio’], (0, max_len - len(i[‘audio’])), ‘constant’)</p>
<h1><a name="turn-the-audio-into-mel-spectrograms-6" class="anchor" href="#turn-the-audio-into-mel-spectrograms-6"></a>turn the audio into mel spectrograms</h1>
<p>preprose = transforms.MelSpectrogram(sample_rate=16000, n_fft=1024, win_length=1024, hop_length=256, n_mels=128, f_min=0.0, f_max=8000.0, power=2.0)</p>
<p>import torchaudio, os<br>
waveforms = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
mel_spectrograms = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
labels = <span class="chcklst-box fa fa-square-o fa-fw"></span></p>
<h1><a name="save-to-wav-7" class="anchor" href="#save-to-wav-7"></a>save to wav</h1>
<p>for i in simple_data:<br>
torchaudio.save(f"audio.wav", torch.tensor(i[‘audio’]).float().unsqueeze(0), 16000)<br>
waveform, sample_rate = torchaudio.load(f"audio.wav")<br>
from torchaudio.transforms import MelSpectrogram<br>
mel_spectrogram = MelSpectrogram(sample_rate)(waveform)<br>
mel_spectrograms.append(mel_spectrogram)<br>
waveforms.append(waveform)<br>
labels.append(i[‘disease’])<br>
# delete the wav file<br>
os.remove(f"audio.wav")<br>
features = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
for i in tqdm(range(len(mel_spectrograms))):<br>
features.append([waveforms[i] ,mel_spectrograms[i]])</p>
<p>train_x, test_x, train_y, test_y = train_test_split(features, labels, test_size=0.2, random_state=random_state)<br>
train_audio = torch.stack([x[0] for x in train_x])<br>
train_mel = torch.stack([x[1] for x in train_x])<br>
train_dataset = torch.utils.data.TensorDataset(train_audio, train_mel, torch.tensor(train_y))</p>
<p>test_audio = torch.stack([x[0] for x in test_x])<br>
test_mel = torch.stack([x[1] for x in test_x])<br>
test_dataset = torch.utils.data.TensorDataset(test_audio, test_mel, torch.tensor(test_y))</p>
<p>train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)<br>
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=True)</p>
<h1><a name="model-8" class="anchor" href="#model-8"></a>model</h1>
<p>model = models.WaveRNN(upsample_scales=[5, 5, 8], n_classes=data_classes , n_output = 6,hop_length=200, kernel_size = 5)<br>
model = model.cuda()</p>
<p>optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)<br>
criterion = torch.nn.CrossEntropyLoss()</p>
<p>epochs = 10<br>
for epoch in range(epochs):<br>
model.train()<br>
for batch in train_loader:<br>
wave, mel, y = batch<br>
wave = wave.cuda()<br>
mel = mel.cuda()<br>
y = y.cuda()<br>
optimizer.zero_grad()<br>
print(wave.shape, mel.shape)<br>
y_pred = model(wave, mel)<br>
loss = criterion(y_pred, y)<br>
loss.backward()<br>
optimizer.step()<br>
f1, precision, recall, report = evaluate(model, test_loader)<br>
print(f"Epoch: {epoch}, F1: {f1}, Precision: {precision}, Recall: {recall}")<br>
print(report)</p>
<hr>