CUDA device assert triggered in pack_padded_sequence

username: Adrian_Coutsoftides
cooked: <p>I move a pretrained embedding model onto the GPU in an effort to make my model faster and I keep getting this device assert trigger, im running the code with CUDA_LAUNCH_BLOCKING= 1 and this is the (rather unhelpful) stacktrace I received, any ideas?</p>
<pre><code class="lang-python">RuntimeError                              Traceback (most recent call last)
&lt;ipython-input-32-f94813b879a4&gt; in &lt;module&gt;
     11     start_time = time.time()
     12 
---&gt; 13     train_loss = train(model, train_iterator, optimizer, criterion, CLIP)
     14     valid_loss = evaluate(model, valid_iterator, criterion)
     15 

&lt;ipython-input-28-f807d4f617eb&gt; in train(model, iterator, optimizer, criterion, clip)
     11         optimizer.optimizer.zero_grad()
     12 
---&gt; 13         output = model(src, trg[:,:-1])
     14         #output = [batch size, trg sent len - 1, output dim]
     15         #trg = [batch size, trg sent len]

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--&gt; 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

&lt;ipython-input-19-ee8c0f2d26d1&gt; in forward(self, src, trg)
     31         #trg = [batch size, trg sent len]
     32         src_mask, trg_mask = self.make_masks(src, trg)
---&gt; 33         src_inject = self.embedder.get_embeddings(src,src.size()[1])
     34         src_inject = torch.stack(src_inject).cuda()
     35         enc_src = self.encoder(src, src_inject, src_mask)

~/project/Embedder.py in get_embeddings(self, primary_sequences, d_embed, alphabet, pack_only, compile_only)
     27         if pack_only == True:
     28             return fully_packed_sequences
---&gt; 29         compiled_sequences = self.encoder.embedding(fully_packed_sequences)
     30         if compile_only == True:
     31             return compiled_sequences

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--&gt; 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

~/project/src/models/embedding.py in forward(self, x)
    119 
    120         if self.lm:
--&gt; 121             h = self.embed(x)
    122         else:
    123             if type(x) is PackedSequence:

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--&gt; 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

~/project/src/models/embedding.py in forward(self, x)
     23     def forward(self, x):
     24         packed = type(x) is PackedSequence
---&gt; 25         h_lm = self.lm.encode(x)
     26 
     27         # embed and unpack if packed

~/project/src/models/sequence.py in encode(self, x)
    180 
    181     def encode(self, x):
--&gt; 182         z_fwd,z_rvs = self.embed_and_split(x, pad=True)
    183         h_fwd_layers,h_rvs_layers = self.transform(z_fwd, z_rvs)
    184 

~/project/src/models/sequence.py in embed_and_split(self, x, pad)
    156         if packed:
    157             lengths = [s-1 for s in batch_sizes]
--&gt; 158             z_fwd = pack_padded_sequence(z_fwd, lengths, batch_first=True)
    159             z_rvs = pack_padded_sequence(z_rvs, lengths, batch_first=True)
    160         # reverse z_rvs

/opt/conda/lib/python3.6/site-packages/torch/nn/utils/rnn.py in pack_padded_sequence(input, lengths, batch_first)
    146                       category=torch.jit.TracerWarning, stacklevel=2)
    147     lengths = torch.as_tensor(lengths, dtype=torch.int64)
--&gt; 148     return PackedSequence(torch._C._VariableFunctions._pack_padded_sequence(input, lengths, batch_first))
    149 
    150 

RuntimeError: CUDA error: device-side assert triggered
</code></pre>

username: ptrblck
cooked: <p>Is your code running fine on the CPU?<br>
Sometimes the error messages thrown on the CPU are clearer that the CUDA ones.</p>

username: Adrian_Coutsoftides
cooked: <p>No the code does not run onthe CPU,<br>
but this is because it is expecting a CPU too tensor and receives a GPU tensor instead:</p>
<pre><code class="lang-python">---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
&lt;ipython-input-32-f94813b879a4&gt; in &lt;module&gt;
     11     start_time = time.time()
     12 
---&gt; 13     train_loss = train(model, train_iterator, optimizer, criterion, CLIP)
     14     valid_loss = evaluate(model, valid_iterator, criterion)
     15 

&lt;ipython-input-28-f807d4f617eb&gt; in train(model, iterator, optimizer, criterion, clip)
     11         optimizer.optimizer.zero_grad()
     12 
---&gt; 13         output = model(src, trg[:,:-1])
     14         #output = [batch size, trg sent len - 1, output dim]
     15         #trg = [batch size, trg sent len]

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--&gt; 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

&lt;ipython-input-19-ee8c0f2d26d1&gt; in forward(self, src, trg)
     31         #trg = [batch size, trg sent len]
     32         src_mask, trg_mask = self.make_masks(src, trg)
---&gt; 33         src_inject = self.embedder.get_embeddings(src,src.size()[1])
     34         src_inject = torch.stack(src_inject).cuda()
     35         enc_src = self.encoder(src, src_inject, src_mask)

~/project/Embedder.py in get_embeddings(self, primary_sequences, d_embed, alphabet, pack_only, compile_only)
     27         if pack_only == True:
     28             return fully_packed_sequences
---&gt; 29         compiled_sequences = self.encoder.embedding(fully_packed_sequences)
     30         if compile_only == True:
     31             return compiled_sequences

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--&gt; 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

~/project/src/models/embedding.py in forward(self, x)
    119 
    120         if self.lm:
--&gt; 121             h = self.embed(x)
    122         else:
    123             if type(x) is PackedSequence:

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--&gt; 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

~/project/src/models/embedding.py in forward(self, x)
     23     def forward(self, x):
     24         packed = type(x) is PackedSequence
---&gt; 25         h_lm = self.lm.encode(x)
     26 
     27         # embed and unpack if packed

~/project/src/models/sequence.py in encode(self, x)
    180 
    181     def encode(self, x):
--&gt; 182         z_fwd,z_rvs = self.embed_and_split(x, pad=True)
    183         h_fwd_layers,h_rvs_layers = self.transform(z_fwd, z_rvs)
    184 

~/project/src/models/sequence.py in embed_and_split(self, x, pad)
    149 
    150         # now, encode x as distributed vectors
--&gt; 151         z = self.embed(x)
    152 
    153         # to pass to transform, we discard the last element for z_fwd and the first element for z_rvs

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--&gt; 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/sparse.py in forward(self, input)
    116         return F.embedding(
    117             input, self.weight, self.padding_idx, self.max_norm,
--&gt; 118             self.norm_type, self.scale_grad_by_freq, self.sparse)
    119 
    120     def extra_repr(self):

/opt/conda/lib/python3.6/site-packages/torch/nn/functional.py in embedding(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)
   1452         # remove once script supports set_grad_enabled
   1453         _no_grad_embedding_renorm_(weight, input, max_norm, norm_type)
-&gt; 1454     return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
   1455 
   1456 

RuntimeError: Expected object of backend CPU but got backend CUDA for argument #3 'index'
</code></pre>
<p>So I loaded the pre-trained model onto the GPU instead:</p>
<pre><code class="lang-python">from Embedder import Embedder
e = Embedder()
e.get_embedder().cuda() #pretrained model loaded onto the GPU here
pad_idx = SRC.vocab.stoi['&lt;pad&gt;']
model = Seq2Seq(enc, dec, pad_idx, device, e).to(device)
</code></pre>

username: ptrblck
cooked: <p>Try to remove the <code>.cuda()</code> calls and replace them with <code>.to(device)</code>, so that you can write device-agnistic code.<br>
Switching <code>device = 'cpu'</code> in your script should yield a CPU-only run.<br>
Once this is done, you might get a better error message for the initial issue.</p>

username: Adrian_Coutsoftides
cooked: <p>I get the same error when I replace cuda() with .to(‘cpu’)</p>

username: ptrblck
cooked: <p>Then other tensors are still pushed to the GPU.<br>
Check e.g. <code>index</code> and try to replace hardcoded <code>cuda()</code> calls with <code>to(device)</code> calls.</p>

username: Adrian_Coutsoftides
cooked: <p>I’ve tried hard coding cpu call throughout the process flow and still get the same error (even when the same tensor is moved to the cpu). Only a single tensor is involved in this process:</p>
<pre><code class="lang-python">---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
&lt;ipython-input-32-f94813b879a4&gt; in &lt;module&gt;
     11     start_time = time.time()
     12 
---&gt; 13     train_loss = train(model, train_iterator, optimizer, criterion, CLIP)
     14     valid_loss = evaluate(model, valid_iterator, criterion)
     15 

&lt;ipython-input-28-f807d4f617eb&gt; in train(model, iterator, optimizer, criterion, clip)
     11         optimizer.optimizer.zero_grad()
     12 
---&gt; 13         output = model(src, trg[:,:-1])
     14         #output = [batch size, trg sent len - 1, output dim]
     15         #trg = [batch size, trg sent len]

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--&gt; 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

&lt;ipython-input-19-15e3ff9eb197&gt; in forward(self, src, trg)
     32         src_mask, trg_mask = self.make_masks(src, trg)
     33 
---&gt; 34         src_inject = self.embedder.get_embeddings(src,src.size()[1])
     35         src_inject = torch.stack(src_inject).cuda()
     36         enc_src = self.encoder(src, src_inject, src_mask)

~/project/Embedder.py in get_embeddings(self, primary_sequences, d_embed, alphabet, pack_only, compile_only)
     28             return fully_packed_sequences
     29         fully_packed_sequences.to('cpu') #the input is moved to the CPU
---&gt; 30         compiled_sequences = self.encoder.embedding(fully_packed_sequences)
     31         if compile_only == True:
     32             return compiled_sequences

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--&gt; 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

~/project/src/models/embedding.py in forward(self, x)
    119 
    120         if self.lm:
--&gt; 121             h = self.embed(x)
    122         else:
    123             if type(x) is PackedSequence:

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--&gt; 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

~/project/src/models/embedding.py in forward(self, x)
     23     def forward(self, x):
     24         packed = type(x) is PackedSequence
---&gt; 25         h_lm = self.lm.encode(x)
     26 
     27         # embed and unpack if packed

~/project/src/models/sequence.py in encode(self, x)
    182     def encode(self, x):
    183         x.to('cpu') #again
--&gt; 184         z_fwd,z_rvs = self.embed_and_split(x, pad=True)
    185         h_fwd_layers,h_rvs_layers = self.transform(z_fwd, z_rvs)
    186 

~/project/src/models/sequence.py in embed_and_split(self, x, pad)
    150         # now, encode x as distributed vectors
    151         x.to('cpu') # a third time
--&gt; 152         z = self.embed(x)
    153 
    154         # to pass to transform, we discard the last element for z_fwd and the first element for z_rvs

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--&gt; 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/sparse.py in forward(self, input)
    116         return F.embedding(
    117             input, self.weight, self.padding_idx, self.max_norm,
--&gt; 118             self.norm_type, self.scale_grad_by_freq, self.sparse)
    119 
    120     def extra_repr(self):

/opt/conda/lib/python3.6/site-packages/torch/nn/functional.py in embedding(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)
   1452         # remove once script supports set_grad_enabled
   1453         _no_grad_embedding_renorm_(weight, input, max_norm, norm_type)
-&gt; 1454     return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
   1455 
   1456 

RuntimeError: Expected object of backend CPU but got backend CUDA for argument #3 'index'
</code></pre>