CUDA error: CUBLAS_STATUS_INTERNAL_ERROR when calling `cublasSgemm( handle, opa, opb, m, n, k, &alpha, a, lda, b, ldb, &beta, c, ldc)

username: Gorgen
cooked: <p>I don’t know i miss the problem “CUDA error: CUBLAS_STATUS_INTERNAL_ERROR when calling `cublasSgemm( handle, opa, opb, m, n, k, &amp;alpha, a, lda, b, ldb, &amp;beta, c, ldc)”. could you please give me some advice to solve it ? many thanks</p>
<p>hers is my code as follows， I add " self.linear_in_pro = torch.nn.Linear(feature_dim_audio, feature_dim_input_tokens)<br>
device = torch.device(“cuda:0”)<br>
linear_in_pro = self.linear_in_pro.to(device)<br>
print(‘linear_in_pro’, self.linear_in_pro)" in my model. I train on “import os<br>
os.environ[“CUDA_VISIBLE_DEVICES”]= ‘0’”:<br>
“</p>
<h1><a name="coding-utf-8-1" class="anchor" href="#coding-utf-8-1"></a>-<em>- coding: utf-8 -</em>-</h1>
<h1><a name="import-os-2" class="anchor" href="#import-os-2"></a>import os</h1>
<h1><a name="osenvironcuda_visible_devices-0-3" class="anchor" href="#osenvironcuda_visible_devices-0-3"></a>os.environ[“CUDA_VISIBLE_DEVICES”]= ‘0’</h1>
<p>import numpy as np<br>
import torch<br>
import torch.nn as nn<br>
import torch.nn.functional as F<br>
import os, sys<br>
import math<br>
import pandas as pd<br>
import pdb</p>
<p>from transformers import RobertaTokenizer, RobertaModel<br>
from transformers import BertTokenizer, BertModel<br>
from transformers import GPT2Tokenizer, GPT2Model</p>
<p>from transformers import RobertaConfig, BertConfig</p>
<p>class ERC_model(nn.Module):<br>
def <strong>init</strong>(self, model_type, clsNum, last, freeze, initial):<br>
super(ERC_model, self).<strong>init</strong>()<br>
self.gpu = True<br>
self.last = last</p>
<pre><code>    """Model Setting"""
    # model_path = '/data/project/rw/rung/model/'+model_type
    model_path = model_type
    if 'roberta' in model_type:
        self.context_model = RobertaModel.from_pretrained(model_path)
                
        if initial == 'scratch':
            config = RobertaConfig.from_pretrained(model_path)
            self.speaker_model = RobertaModel(config)
        else:
            self.speaker_model = RobertaModel.from_pretrained(model_path)
    elif model_type == 'bert-large-uncased':
        self.context_model = BertModel.from_pretrained(model_path)
        
        if initial == 'scratch':
            config = BertConfig.from_pretrained(model_path)
            self.speaker_model = BertModel(config)
        else:
            self.speaker_model = BertModel.from_pretrained(model_path)
    else:
        self.context_model = GPT2Model.from_pretrained(model_path)
        tokenizer = GPT2Tokenizer.from_pretrained(model_path)
        tokenizer.add_special_tokens({'cls_token': '[CLS]', 'pad_token': '[PAD]'})
        self.context_model.resize_token_embeddings(len(tokenizer))
        
        self.speaker_model = GPT2Model.from_pretrained(model_path)
        self.speaker_model.resize_token_embeddings(len(tokenizer))
    self.hiddenDim = self.context_model.config.hidden_size
    
    zero = torch.empty(2, 1, self.hiddenDim).cuda()
    self.h0 = torch.zeros_like(zero) # (num_layers * num_directions, batch, hidden_size)
    self.speakerGRU = nn.GRU(self.hiddenDim, self.hiddenDim, 2, dropout=0.3) # (input, hidden, num_layer) (BERT_emb, BERT_emb, num_layer)
    
    #audio projection

    """score"""
    # self.SC = nn.Linear(self.hiddenDim, self.hiddenDim)
    self.W = nn.Linear(self.hiddenDim, clsNum)
    
    """parameters"""
    self.train_params = list(self.context_model.parameters())+list(self.speakerGRU.parameters())+list(self.W.parameters()) # +list(self.SC.parameters())
    if not freeze:
        self.train_params += list(self.speaker_model.parameters())

def forward(self, batch_input_tokens, batch_speaker_tokens, batch_audio):
    """
        batch_input_tokens: (batch, len)
        batch_speaker_tokens: [(speaker_utt_num, len), ..., ]
    """
    # print('batch_input_tokens', batch_input_tokens.shape, batch_input_tokens.device, batch_input_tokens.dtype)
    batch_audio = torch.tensor(batch_audio, dtype=torch.int64)
    # print('batch_audio', batch_audio.shape)
    batch_size_input, feature_dim_input_tokens = batch_input_tokens.size()
    batch_size_input, feature_dim_audio = batch_audio.size()
      
    self.linear_in_pro = torch.nn.Linear(feature_dim_audio, feature_dim_input_tokens)
    device = torch.device("cuda:0")
    linear_in_pro = self.linear_in_pro.to(device)  
    print('linear_in_pro', self.linear_in_pro)
    batch_input_tokens = self.linear_in_pro(batch_audio.float())
    # batch_input_tokens = batch_input_tokens.clone().detach()
    batch_input_tokens = torch.tensor(batch_input_tokens, dtype=torch.int64)
    print('batch_input_tokens', batch_input_tokens.shape, batch_input_tokens.device, batch_input_tokens.dtype)

    if self.last:
        batch_context_output = self.context_model(batch_input_tokens).last_hidden_state[:,-1,:] # (batch, 1024)
    else:
        # self.context_model = self.context_model.cuda()
        batch_context_output = self.context_model(batch_input_tokens).last_hidden_state[:,0,:] # (batch, 1024)
        #audio
        # batch_context_output = batch_audio
    
    batch_speaker_output = []
    for speaker_tokens in batch_speaker_tokens:
        if speaker_tokens.shape[0] == 0:
            speaker_track_vector = torch.zeros(1, self.hiddenDim).cuda()
        else:
            if self.last:
                speaker_output = self.speaker_model(speaker_tokens.cuda()).last_hidden_state[:,-1,:] # (speaker_utt_num, 1024)
            else:
                speaker_output = self.speaker_model(speaker_tokens.cuda()).last_hidden_state[:,0,:] # (speaker_utt_num, 1024)
            speaker_output = speaker_output.unsqueeze(1) # (speaker_utt_num, 1, 1024)
            speaker_GRU_output, _ = self.speakerGRU(speaker_output, self.h0) # (speaker_utt_num, 1, 1024) &lt;- (seq_len, batch, output_size)
            speaker_track_vector = speaker_GRU_output[-1,:,:] # (1, 1024)
        batch_speaker_output.append(speaker_track_vector)
    batch_speaker_output = torch.cat(batch_speaker_output, 0) # (batch, 1024)
               
    final_output = batch_context_output + batch_speaker_output
    # print('findal_output', final_output.shape)
    # final_output = batch_context_output + self.SC(batch_speaker_output)        
    context_logit = self.W(final_output) # (batch, clsNum)
    return context_logit
</code></pre>
<p>”</p>
<p><div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/3X/b/e/be67780a2b6af2f01a494918904c1ccb404702e1.png" data-download-href="https://discuss.pytorch.org/uploads/default/be67780a2b6af2f01a494918904c1ccb404702e1" title="image"><img src="https://discuss.pytorch.org/uploads/default/original/3X/b/e/be67780a2b6af2f01a494918904c1ccb404702e1.png" alt="image" data-base62-sha1="raooPuwrOwV7vxocR9ltz74xjsR" width="690" height="395" data-dominant-color="2C2C2C"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">image</span><span class="informations">1161×665 52.8 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div></p>

username: ptrblck
cooked: <p>Could you try to narrow down the issue and post a minimal and executable code snippet please?<br>
Additionally, could you update to the latest PyTorch release in case you are using an older one and check if you would still see this error?</p>

username: Gorgen
cooked: <p>thank you for your detailed advice. the problem is input of pretrained Robert model. something wrong token.</p>
<p>solve it. many thanks</p>