Training loop crashing due to loss.backwards function

deep_learning: <p>I have been trying to train a lstm model with lyrics data I found on Kaggle. When I first started with this architecture, I had glove embeddings which worked fine with the model and how I shaped it. I switched totokenization and then z scoring my input parameters and all of a sudden my program hangs and then closes out without any error, I only found out about the error when I checked my Event viewer and found this:</p>
<p>Faulting application name: python.exe, version: 3.12.2150.1013, time stamp: 0x65c2a4c7<br>
Faulting module name: c10.dll, version: 0.0.0.0, time stamp: 0x65a00f1b<br>
Exception code: 0xc0000005<br>
Fault offset: 0x0000000000060bb5<br>
Faulting process id: 0x0x35EC<br>
Faulting application start time: 0x0x1DA5E8859C9145E<br>
Faulting application path: C:\Users\rohun\AppData\Local\Programs\Python\Python312\python.exe<br>
Faulting module path: C:\Users\rohun\final_train\env\Lib\site-packages\torch\lib\c10.dll<br>
Report Id: 8d624ff4-59a7-448f-b320-6e7aeed0ecd4<br>
Faulting package full name:<br>
Faulting package-relative application ID:</p>
<p>I’ve tried a lot of different thigns to fix like changing model architecture, data preprocessing, hardware limitations, and such, but I haven’t found a single thing that could be casuing me this problem. ANy help much appreciated.</p>
<p>For reference this is my code:<br>
import pandas as pd<br>
import string<br>
import torch<br>
import torchtext<br>
import random<br>
import tracemalloc<br>
from torch.utils.data import Dataset, DataLoader, random_split<br>
from torch.nn.utils.rnn import pad_sequence<br>
import torch.nn as nn<br>
import torch.nn.functional as F<br>
from torch.optim.lr_scheduler import ReduceLROnPlateau<br>
from operator import itemgetter<br>
from memory_profiler import profile</p>
<p>if torch.cuda.is_available():<br>
device = torch.device(“cuda”)<br>
print(“CUDA is available.”)<br>
else:<br>
device = torch.device(“cpu”)<br>
print(“CUDA is not available. Running on CPU.”)</p>
<p>class EarlyStopping:<br>
def <strong>init</strong>(self, patience=5, min_delta=0.0):<br>
self.patience = patience<br>
self.min_delta = min_delta<br>
self.counter = 0<br>
self.best_loss = float(‘inf’)</p>
<pre><code>def __call__(self, val_loss):
    if val_loss &lt; self.best_loss - self.min_delta:
        self.best_loss = val_loss
        self.counter = 0
    else:
        self.counter += 1
        if self.counter &gt;= self.patience:
            return True  # Stop training
    return False
def reset(self):
    self.counter = 0
    self.best_loss = float('inf')
</code></pre>
<h1><a name="grid-search-1" class="anchor" href="#grid-search-1"></a>Grid Search</h1>
<p>batch_sizes = [32] # try with 64 and 128<br>
hidden_lstm_sizes = [25] # if it does better wiht the 512 lets use 1024 and a higher value as well, should give us 4 concrete valuesto work with.<br>
hidden_linear_sizes = [124] # 512<br>
epochs = 8 # max number of epochs before overfitting<br>
drop_percentages = [0] # better values with increased dropout percentage, 0.25 working well so lets keep it<br>
learning_rates = [.01]<br>
weight_decays = [0] # 0, 0.10, 0.25</p>
<p>class LSTM(nn.Module):</p>
<pre><code>def __init__(self, embedding_dim, hidden_dim, hidden1dim, output_dim, dropout_dim,sequence_len):
    super(LSTM, self).__init__()
    self.lstm = nn.LSTM(embedding_dim, hidden_dim, batch_first=True, bidirectional=False)
    self.hidden = nn.Linear(hidden_dim, hidden1dim)
    self.output = nn.Linear(hidden_dim, output_dim)
    self.drop = nn.Dropout(dropout_dim)

def forward(self, data):
    lstm_out, _ = self.lstm(data)
    #hidden_out = self.hidden(lstm_out)
    #drop_out = self.drop(hidden_out)
    logits = self.output(lstm_out)
    return logits
</code></pre>
<p>class data_set(Dataset):<br>
def <strong>init</strong>(self,X_data,Y_data):<br>
self.X_data = X_data<br>
self.Y_data = Y_data</p>
<pre><code>def __len__(self): 
    return len(self.X_data) 

def __getitem__(self, index):
    return self.X_data[index], self.Y_data[index]
</code></pre>
<p>early_stopper = EarlyStopping(patience=3, min_delta=0.0025)<br>
loss_fn = torch.nn.CrossEntropyLoss()</p>
<p>def loop(training_data_loader, validation_data_loader, testing_data_loader,optimizer,epochs,model):<br>
epoch_losses = {}<br>
early_stopper.reset()<br>
earlyStopping = False</p>
<pre><code>print(len(training_data_loader))

#c_s = len(training_data_loader) // 10
for epoch in range(epochs):
    if earlyStopping:
        break
    epoch_losses[epoch] = []
    print("Epoch: " + str(epoch))
    total_loss = 0
    running_loss = 0
    for i,batch in enumerate(training_data_loader):
        x_i, y_i = batch[0],batch[1]
        logits = model(x_i)
        loss = loss_fn(logits, y_i)
        optimizer.zero_grad()


        loss.backward()
        optimizer.step()
    


    model.eval()
    total_loss = 0
    running_loss = 0

    for i,batch in enumerate(validation_data_loader):
        x_i, y_i = batch[0],batch[1]

        # forward pass
        with torch.no_grad():
            logits = model(x_i)
        loss = loss_fn(logits, y_i)
        total_loss += loss
        running_loss += loss

        # check early stopping here?

    print("Total Validation Average Loss: " + str((total_loss/len(validation_data_loader)).item()))
    epoch_losses[epoch].append(total_loss/len(validation_data_loader))
    scheduler.step(total_loss/len(validation_data_loader))
    if early_stopper(total_loss/len(validation_data_loader)):
        print(f"Early stopping at epoch {epoch}")
        earlyStopping = True
    

    total_loss = 0
    running_loss = 0

    for i,batch in enumerate(testing_data_loader):
        x_i, y_i = batch[0],batch[1]

        # forward pass
        with torch.no_grad():
            logits = model(x_i)
        loss = loss_fn(logits, y_i)
        total_loss += loss
        running_loss += loss

        # check early stopping here?


    print("Total Testing Average Loss: " + str((total_loss/len(testing_data_loader)).item()))
    epoch_losses[epoch].append(total_loss/len(testing_data_loader))

return epoch_losses
</code></pre>
<h1><a name="read-data-pre-process-data-2" class="anchor" href="#read-data-pre-process-data-2"></a>read data, pre process data</h1>
<p>df = pd.read_csv(‘spotify_millsongdata.csv’)<br>
<span class="hashtag-raw">#df</span> = df.loc[(df[‘artist’] == ‘ABBA’) | (df[‘artist’] == ‘Donna Summer’) | (df[‘artist’] == ‘Bob Dylan’)]<br>
df = df.sample(n=1)<br>
df = df[‘text’]</p>
<p>print("Size of dataset: " + str(len(df)))</p>
<p>table = str.maketrans(‘’, ‘’, string.punctuation)</p>
<p>words_set = set()<br>
number_to_words_dictionary = {}<br>
words_to_numbers_dictionary = {}</p>
<p>def preProcess(x):<br>
x = x.strip()<br>
x = x.lower()<br>
x = x.splitlines()</p>
<pre><code>for i in range(len(x)):
    x[i] = x[i].lower()
    x[i] = x[i].translate(table)
    x[i] = x[i].split()
    x[i].insert(0,"&lt;BOS&gt;")
    x[i].append("&lt;EOS&gt;")
    words_set.update(x[i])

return x
</code></pre>
<p>print(“Preprocessing staring…”)<br>
data = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
for index, row in df.items():<br>
data.extend(preProcess(row))</p>
<p>words_set = list(sorted(words_set))<br>
for i,word in enumerate(words_set):<br>
number_to_words_dictionary[i] = word<br>
words_to_numbers_dictionary[word] = i</p>
<p>max_len = max(len(x) for x in data) - 1</p>
<h1><a name="load-glove-embeddings-3" class="anchor" href="#load-glove-embeddings-3"></a>Load GloVe embeddings</h1>
<p><span class="hashtag-raw">#glove</span> = torchtext.vocab.GloVe(name=“6B”, dim=50)</p>
<p><span class="hashtag-raw">#convert</span> data into X and y data, with X being embeddings and y as numbers<br>
X = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
Y = <span class="chcklst-box fa fa-square-o fa-fw"></span></p>
<h1><a name="convert-into-word-embeddings-4" class="anchor" href="#convert-into-word-embeddings-4"></a>convert into word embeddings</h1>
<h1><a name="torch-size-to-be-max_size-50-5" class="anchor" href="#torch-size-to-be-max_size-50-5"></a>torch size to be (max_size, 50)</h1>
<p>window_size = 3</p>
<p>print(“token embeddings starting …”)</p>
<p>for i in range(len(data)):<br>
update = torch.tensor(<span class="chcklst-box fa fa-square-o fa-fw"></span>)<br>
for j in range(len(data[i]) - window_size):<br>
keys = data[i][:j+window_size]<br>
X.append(torch.tensor(itemgetter(*keys)(words_to_numbers_dictionary)))<br>
Y.append(torch.tensor(words_to_numbers_dictionary[data[i][j+window_size]]))</p>
<p>print(“token embeddings completed…”)</p>
<h1><a name="pad-the-sequences-6" class="anchor" href="#pad-the-sequences-6"></a>Pad the sequences</h1>
<p>print(“padding sequences starting…”)<br>
padded_X = pad_sequence(X, batch_first=True).float()</p>
<h1><a name="z-score-7" class="anchor" href="#z-score-7"></a>Z score</h1>
<p>mean = padded_X.mean(dim=None)<br>
std = padded_X.std(dim=None)</p>
<h1><a name="z-score-normalization-8" class="anchor" href="#z-score-normalization-8"></a>Z-score normalization</h1>
<p>padded_X = (padded_X - mean) / std</p>
<p>print(padded_X.size())</p>
<p>Y = torch.tensor(Y).long()</p>
<p>print(“padding sequences ending…”)</p>
<h1><a name="split-training-data-into-training-validation-and-testing-9" class="anchor" href="#split-training-data-into-training-validation-and-testing-9"></a>split training data into training, validation and testing</h1>
<h1><a name="creates-dataset-10" class="anchor" href="#creates-dataset-10"></a>creates dataset</h1>
<p>dataset = data_set(padded_X,Y)</p>
<p>train_size = int(0.7 * len(dataset))<br>
test_size = int(0.2 * len(dataset))<br>
val_size = len(dataset) - train_size - test_size</p>
<h1><a name="split-the-dataset-11" class="anchor" href="#split-the-dataset-11"></a>Split the dataset</h1>
<p>train_dataset, test_dataset, val_dataset = random_split(dataset, [train_size, test_size, val_size])</p>
<p>print("Preprocessing Completed … ")</p>
<p>print(“GRID SEARCH STARTING …”)</p>
<p>classes_size = len(number_to_words_dictionary)</p>
<p><span class="hashtag-raw">#accuracy</span> = Accuracy(task=“multiclass”, num_classes=classes_size)</p>
<h1><a name="starts-the-whole-loop-12" class="anchor" href="#starts-the-whole-loop-12"></a>starts the whole loop</h1>
<p>sequence_length = padded_X.size()[1]<br>
input_dim = 1</p>
<p><span class="hashtag-raw">#tracemalloc</span>.start()<br>
with open(“readme.txt”, “w”) as f:<br>
for batch in batch_sizes:</p>
<pre><code># Create dataloaders
    train_loader = DataLoader(train_dataset, batch_size=batch, shuffle=True)
    test_loader = DataLoader(test_dataset, batch_size=batch, shuffle=True)
    val_loader = DataLoader(val_dataset, batch_size=batch, shuffle=True)

    for hidden_lstm in hidden_lstm_sizes:
        for hidden_linear in hidden_linear_sizes:
            for drop_p in drop_percentages:
                
                # creates model
                model = LSTM(input_dim, hidden_lstm, hidden_linear, classes_size, drop_p,sequence_len=sequence_length)

                model.to(device)

                for lr in learning_rates:
                    for wd in weight_decays:
                # runs training loop
                        optimizer = torch.optim.Adam(model.parameters(), lr=lr,weight_decay=wd)
                        scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=3)
                        name = f"batch: {batch}, hidden_lstm: {hidden_lstm}, hidden_linear: {hidden_linear}, drop_percentage: {drop_p}, learning_rate: {lr}, weight_decay: {wd}"
                        print(name)
                        print("----------------------------------------------------------------------------------------------------------------------------------------------")
                        val= loop(train_loader,val_loader,test_loader,optimizer,epochs=epochs,model=model)
                        f.write(name + "\n")
                        for key,value in val.items():
                            f.write(f"| At epoch: {key} training_loss = {value[0]} validation_loss = {value[1]} testing_loss = {value[2]}" + "\n")
                            print(f"|   At epoch: {key}     |   training_loss = {value[0]}        |   validation_loss = {value[1]}        |   testing_loss = {value[2]}        |")
                        f.write("\n")
                        val = None
                        del val
                        #print(tracemalloc.get_traced_memory())
                        del optimizer
                        del scheduler
                del model
    train_loader = None
    val_loader= None
    test_loader = None
    del train_loader
    del val_loader
    del test_loader
</code></pre>
<p><span class="hashtag-raw">#tracemalloc</span>.stop()<br>
print(“GRID SEARCH COMPLETED”)</p>