Getting "RuntimeError: CUDA error: device-side assert triggered" for an LSTM network

talhak: <p>I’m trying to replicate <a href="https://medium.com/@adam.wearne/lets-get-sentimental-with-pytorch-dcdd9e1ea4c9" rel="nofollow noopener">the tutorial posted on Medium</a> that demonstrates the utilization of an LSTM network for sentiment analysis. During the execution via PyTorch, getting this error: <code>RuntimeError: CUDA error: device-side assert triggered</code>. Could you please explain me what is the reason for this error?</p>
<p><strong>Full error stacktrace:</strong></p>
<pre><code class="lang-auto">---------------------------------------------------------------------------

RuntimeError Traceback (most recent call last)

[&lt;ipython-input-16-40911770614d&gt;](https://yordizmwag-496ff2e9c6d22116-0-colab.googleusercontent.com/outputframe.html?vrz=colab-20200723-085747-RC00_322789762#) in &lt;module&gt;() 197 198 if __name__ == '__main__': --&gt; 199 main()

4 frames

[/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py](https://yordizmwag-496ff2e9c6d22116-0-colab.googleusercontent.com/outputframe.html?vrz=colab-20200723-085747-RC00_322789762#) in convert(t) 439 if convert_to_format is not None and t.dim() == 4: 440 return t.to(device, dtype if t.is_floating_point() else None, non_blocking, memory_format=convert_to_format) --&gt; 441 return t.to(device, dtype if t.is_floating_point() else None, non_blocking) 442 443 return self._apply(convert)

RuntimeError: CUDA error: device-side assert triggered
</code></pre>
<p><strong>Here is my script:</strong></p>
<pre><code class="lang-auto">import random
import time

import torch
import torch.nn as nn
import torch.optim as optim
from torchtext import data
from torchtext.datasets import IMDB

CUDA_LAUNCH_BLOCKING=1

def epoch_time(start_time, end_time):
    elapsed_time = end_time - start_time
    elapsed_mins = int(elapsed_time / 60)
    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))
    return elapsed_mins, elapsed_secs


class LSTM_IMDB(nn.Module):
    def __init__(self, input_dim, embedding_dim, hidden_dim, output_dim, n_layers, dropout=0.0, padding_idx=None, is_bidirectional=False):
        super().__init__()
        self.hidden_dim = hidden_dim
        self.embedding = nn.Embedding(input_dim, embedding_dim)
        self.lstm = nn.LSTM(embedding_dim, hidden_dim, num_layers=n_layers, bidirectional=is_bidirectional, dropout=dropout)
        self.fc = nn.Linear(hidden_dim, output_dim)
        self.is_bidirectional = False

    def forward(self, text, sequence_length):
        embeddings = self.embedding(text)
    
        packed_embeddings = nn.utils.rnn.pack_padded_sequence(embeddings, 
                                                            sequence_length)
        
        packed_output, (hidden_state, cell_state) = self.lstm(packed_embeddings)
            
        if self.is_bidirectional:
          output = torch.cat((hidden_state[-2,:,:], hidden_state[-1,:,:]), dim = 1)
        else:
          output = hidden_state[-1,:,:]
          scores = self.fc(output)
        
        return scores


def binary_accuracy(preds, y):
    """
    Returns accuracy per batch, i.e. if you get 8/10 right, this returns 0.8, NOT 8
    """

    # round predictions to the closest integer
    rounded_preds = torch.round(torch.sigmoid(preds))
    correct = (rounded_preds == y).float()  # convert into float for division
    acc = correct.sum() / len(correct)
    return acc


def train(model, iterator, optimizer, criterion):
    epoch_loss = 0
    epoch_acc = 0

    model.train()

    for batch in iterator:
        optimizer.zero_grad()

        text, text_lengths = batch.text

        predictions = model(text, text_lengths).squeeze(1)

        loss = criterion(predictions, batch.label)

        acc = binary_accuracy(predictions, batch.label)

        loss.backward()

        optimizer.step()

        epoch_loss += loss.item()
        epoch_acc += acc.item()

    return epoch_loss / len(iterator), epoch_acc / len(iterator)


def evaluate(model, iterator, criterion):
    epoch_loss = 0
    epoch_acc = 0

    model.eval()

    with torch.no_grad():
        for batch in iterator:
            predictions = model(batch.text).squeeze(1)

            loss = criterion(predictions, batch.label)

            acc = binary_accuracy(predictions, batch.label)

            epoch_loss += loss.item()
            epoch_acc += acc.item()

    return epoch_loss / len(iterator), epoch_acc / len(iterator)


def main():
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    SEED = 407
    TEXT = data.Field(tokenize = 'spacy', lower=True, include_lengths = True)
    LABEL = data.LabelField(dtype = torch.float)

    train_data, test_data = IMDB.splits(TEXT, LABEL)
    train_data, valid_data = train_data.split(random_state = random.seed(SEED))

    print(f'Number of training examples: {len(train_data)}')
    print(f'Number of validation examples: {len(valid_data)}')
    print(f'Number of testing examples: {len(test_data)}')

    MAX_VOCAB_SIZE = 10_000

    TEXT.build_vocab(train_data, 
                    max_size=MAX_VOCAB_SIZE,
                    vectors='glove.6B.300d',
                    unk_init=torch.Tensor.normal_)
    LABEL.build_vocab(train_data)

    BATCH_SIZE = 64
    train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(
        (train_data, valid_data, test_data),
        batch_size=BATCH_SIZE,
        sort_within_batch = True,
        device=device)


    INPUT_DIM = len(TEXT.vocab)
    EMBEDDING_DIM = 300 # This needs to match the size of the pre-trained embeddings!
    HIDDEN_DIM = 256
    OUTPUT_DIM = 1
    num_layers = 3
    dropout = 0.5
    pad_idx = TEXT.vocab.stoi[TEXT.pad_token]

    model = LSTM_IMDB(INPUT_DIM, EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM, n_layers=num_layers,  dropout=dropout, padding_idx=pad_idx)
    model.to(device)

    # Initialize word embeddings
    glove_vectors = TEXT.vocab.vectors
    model.embedding.weight.data.copy_(glove_vectors)
    # Zero out &lt;unk&gt; and &lt;pad&gt; tokens
    unk_idx = TEXT.vocab.stoi[TEXT.unk_token]
    model.embedding.weight.data[unk_idx] = torch.zeros(EMBEDDING_DIM)
    model.embedding.weight.data[pad_idx] = torch.zeros(EMBEDDING_DIM)


    lr = 1e-2
    criterion = nn.BCEWithLogitsLoss()
    criterion = criterion.to(device)
    optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.9)

    N_EPOCHS = 5

    best_valid_loss = float('inf')

    for epoch in range(N_EPOCHS):

        start_time = time.time()

        train_loss, train_acc = train(model, train_iterator, optimizer, criterion)
        valid_loss, valid_acc = evaluate(model, valid_iterator, criterion)

        end_time = time.time()

        epoch_mins, epoch_secs = epoch_time(start_time, end_time)

        if valid_loss &lt; best_valid_loss:
            best_valid_loss = valid_loss
            torch.save(model.state_dict(), 'tut1-model.pt')

        print(f'Epoch: {epoch + 1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')
        print(f'\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc * 100:.2f}%')
        print(f'\t Val. Loss: {valid_loss:.3f} |  Val. Acc: {valid_acc * 100:.2f}%')


if __name__ == '__main__':
    main()
</code></pre>

ptrblck: <p>Is your code working fine on the CPU?<br>
If so, could you rerun the code with <code>CUDA_LAUNCH_BLOCKING=1 python script.py args</code> and post the stack trace here, please?</p>

talhak: <p>Thank you very much, now works like a charm! <img src="https://discuss.pytorch.org/images/emoji/apple/clap.png?v=9" title=":clap:" class="emoji" alt=":clap:"></p>

ptrblck: <p>My suggestion is not a fix, but a debugging step. <img src="https://discuss.pytorch.org/images/emoji/apple/wink.png?v=9" title=":wink:" class="emoji" alt=":wink:"><br>
If I understand the issue correctly, your code is now working on the CPU as well as with <code>CUDA_LAUNCH_BLOCKING=1</code> on the GPU, but raises the assert statement on a plain GPU run?</p>