Fluctuating Loss in Torch Neural Network

Shivakumar_Kannan: <p>Here is my code for Music Genre Classification ( I cite Valerio Velardo’s Music Genre classification code and PyTorch website)<br>
import torch<br>
import torchaudio<br>
from torch.utils.data import DataLoader<br>
from torch import nn<br>
import numpy as np<br>
import matplotlib.pyplot as plt</p>
<h1><a name="global-variables-1" class="anchor" href="#global-variables-1"></a>Global Variables</h1>
<p>batch_size = 20<br>
n_mfcc = 40</p>
<h1><a name="gathering-loading-data-device-2" class="anchor" href="#gathering-loading-data-device-2"></a>Gathering &amp; Loading Data / Device</h1>
<p>train_data = torchaudio.datasets.GTZAN(<br>
root=“MUSIC_CLIPS”,<br>
url=“<a href="http://opihi.cs.uvic.ca/sound/genres.tar.gz%E2%80%9D" rel="noopener nofollow ugc">http://opihi.cs.uvic.ca/sound/genres.tar.gz”</a>,<br>
folder_in_archive=“/Users/shivakannan/PycharmProjects/MUSIC_GENRE_CLASSIFICATION_project/genres_original”,<br>
download=True,<br>
subset=“training”<br>
)</p>
<p>test_data = torchaudio.datasets.GTZAN(<br>
root=“MUSIC_CLIPS”,<br>
url=“<a href="http://opihi.cs.uvic.ca/sound/genres.tar.gz%E2%80%9D" rel="noopener nofollow ugc">http://opihi.cs.uvic.ca/sound/genres.tar.gz”</a>,<br>
folder_in_archive=“/Users/shivakannan/PycharmProjects/MUSIC_GENRE_CLASSIFICATION_project/genres_original”,<br>
download=True,<br>
subset=“testing”<br>
)</p>
<p>train_load = DataLoader(train_data, batch_size=batch_size, shuffle=True)<br>
test_load = DataLoader(test_data, batch_size=batch_size, shuffle=True)</p>
<p>device = torch.device(“mps”)</p>
<h1><a name="dictionary-for-mapping-3" class="anchor" href="#dictionary-for-mapping-3"></a>Dictionary for mapping</h1>
<p>labels = {</p>
<pre><code class="lang-auto">'blues': torch.tensor(0, dtype=torch.float32),
'classical': torch.tensor(1, dtype=torch.float32),
'country': torch.tensor(2, dtype=torch.float32),
'disco': torch.tensor(3, dtype=torch.float32),
'hiphop': torch.tensor(4, dtype=torch.float32),
'jazz': torch.tensor(5, dtype=torch.float32),
'metal': torch.tensor(6, dtype=torch.float32),
'pop': torch.tensor(7, dtype=torch.float32),
'reggae': torch.tensor(8, dtype=torch.float32),
'rock': torch.tensor(9, dtype=torch.float32)
</code></pre>
<p>}</p>
<h1><a name="extracting-waveforms-4" class="anchor" href="#extracting-waveforms-4"></a>Extracting waveforms</h1>
<p>items_train =<br>
X_train =<br>
y_train =<br>
for ii in range(443):<br>
items_train.append(torchaudio.datasets.GTZAN.<strong>getitem</strong>(train_data, n=ii))<br>
X_train.append(items_train[ii][0][0][0:660000])<br>
y_train.append(labels[items_train[ii][2]])</p>
<p>items_test =<br>
X_test =<br>
y_test =<br>
for ii in range(290):<br>
items_test.append(torchaudio.datasets.GTZAN.<strong>getitem</strong>(test_data, n=ii))<br>
X_test.append(items_test[ii][0][0][0:660000])<br>
y_test.append(labels[items_test[ii][2]])</p>
<p>X_train = torch.from_numpy(np.array(X_train))<br>
y_train = torch.from_numpy(np.array(y_train))<br>
X_test = torch.from_numpy(np.array(X_test))<br>
y_test = torch.from_numpy(np.array(y_test))</p>
<h1><a name="everything-works-till-now-5" class="anchor" href="#everything-works-till-now-5"></a>EVERYTHING WORKS TILL NOW</h1>
<p>num_segments = 100</p>
<h1><a name="extracting-mfccs-6" class="anchor" href="#extracting-mfccs-6"></a>Extracting MFCCs</h1>
<p>MFCC_transform = torchaudio.transforms.MFCC(<br>
sample_rate=22050,<br>
n_mfcc=n_mfcc,<br>
norm=“ortho”,<br>
log_mels=False,<br>
)</p>
<p>MFCC_train =</p>
<p>for ii in range(443):</p>
<pre><code class="lang-auto">temp = []
for jj in range(num_segments):

    start_idx = jj * 6600
    end_idx = start_idx + 6600

    signal = X_train[ii][start_idx:end_idx]
    temp.append(MFCC_transform(signal))

MFCC_train.append(temp)
</code></pre>
<p>MFCC_train = torch.from_numpy(np.array(MFCC_train))</p>
<h1><a name="everything-works-till-here-7" class="anchor" href="#everything-works-till-here-7"></a>Everything Works Till Here</h1>
<p>MFCC_test =</p>
<p>for ii in range(290):</p>
<pre><code class="lang-auto">temp = []
for jj in range(num_segments):

    start_idx = jj * 6600
    end_idx = start_idx + 6600

    signal = X_test[ii][start_idx:end_idx]
    temp.append(MFCC_transform(signal))

MFCC_test.append(temp)
</code></pre>
<p>MFCC_test = torch.from_numpy(np.array(MFCC_test))</p>
<h1><a name="mfccs-for-training-and-testing-have-been-obtained-8" class="anchor" href="#mfccs-for-training-and-testing-have-been-obtained-8"></a>MFCCs for Training and Testing have been Obtained</h1>
<h1><a name="expected-outputs-for-training-and-testing-have-been-obtained-9" class="anchor" href="#expected-outputs-for-training-and-testing-have-been-obtained-9"></a>Expected Outputs for Training and Testing have been Obtained</h1>
<p>y_train_use = torch.zeros((443, 100))<br>
y_test_use = torch.zeros((290, 100))</p>
<p>for ii in range(443):</p>
<pre><code class="lang-auto">for jj in range(100):

    y_train_use[ii][jj] = y_train[ii]
</code></pre>
<p>for ii in range(290):</p>
<pre><code class="lang-auto">for jj in range(100):

    y_test_use[ii][jj] = y_test[ii]
</code></pre>
<h1><a name="everything-works-till-here-10" class="anchor" href="#everything-works-till-here-10"></a>Everything Works Till Here</h1>
<h1><a name="neural-network-class-11" class="anchor" href="#neural-network-class-11"></a>Neural Network Class</h1>
<p>class NN(nn.Module):</p>
<pre><code class="lang-auto">def __init__(self):

    super().__init__()

    self.flatten = nn.Flatten()

    self.linear_relu_stack = nn.Sequential(

        nn.Linear(40*34, 512),
        nn.ReLU(),
        nn.Linear(512, 512),
        nn.ReLU(),
        nn.Linear(512, 10)

    )

def forward(self, x):

    x = self.flatten(x)
    logits = self.linear_relu_stack(x)
    return logits
</code></pre>
<p>model = NN().to(device)</p>
<p>loss_fn = nn.CrossEntropyLoss()<br>
optimizer = torch.optim.Adam(model.parameters(), lr=1e-6)</p>
<h1><a name="everything-works-till-here-12" class="anchor" href="#everything-works-till-here-12"></a>Everything Works Till Here</h1>
<p>loss_list =<br>
idx_list =<br>
def train(model, dataloader):</p>
<pre><code class="lang-auto">count = 0
size = len(dataloader.dataset)
model.train()

for ii in range(443):

    for jj in range(5):

        idx_list.append(count)
        X = MFCC_train[ii][jj * batch_size : jj * batch_size + batch_size]
        #print("XSIZE", X.size())
        y = np.array(y_train_use[ii][jj * batch_size : jj * batch_size + batch_size])
        X = X.to(device)
        y = torch.from_numpy(y).to(device)

        pred = model(X)

        loss = loss_fn(pred, y)
        #print("LOSS", type(loss))

        loss.backward()
        optimizer.step()
        optimizer.zero_grad()


        loss_list.append(loss.item())

        count += 1


plt.plot(np.array(idx_list), np.array(loss_list))
plt.show()
</code></pre>
<p>train(model, train_load)</p>
<p>The plot for loss for training fluctuates so much.<br>
Here is the plot:<br>
<img src="https://discuss.pytorch.org/uploads/default/original/3X/3/a/3a986f3753cb55933c9bf9607c5841ab58f32bb0.png" alt="Figure_1" data-base62-sha1="8mmhwaILTqZZdtKUn2wj0BarfMI" width="640" height="480"></p>
<p>Please explain me possible issues and ways to improve this plot.</p>