I GET RuntimeError: CUDA error: an illegal memory access was encountered

tr_marwa: <p>hello everyone,<br>
if someone can help me<br>
my problem is that i’m using kfols method to do the cross validation, which means that i have 5 itération to fit my data into training and testing set<br>
i made a loop for around my data, when i do just a training it works normally but when i do the validation after that, i get this error RuntimeError: CUDA error: an illegal memory access was encountered<br>
knowing that i tryed</p>
<blockquote>
<p>device = torch.device(‘cuda’ if torch.cuda.is_available() else ‘cpu’)<br>
print(‘Using device:’, device)<br>
print()</p>
<p><span class="hashtag-raw">#Additional</span> Info when using cuda<br>
if device.type == ‘cuda’:<br>
print(torch.cuda.get_device_name(0))<br>
print(‘Memory Usage:’)<br>
print(‘Allocated:’, round(torch.cuda.memory_allocated(0)/1024<strong>3,1), ‘GB’)<br>
print('Cached:   ', round(torch.cuda.memory_cached(0)/1024</strong>3,1), ‘GB’)</p>
</blockquote>
<p>and i get</p>
<blockquote>
<p>Using device: cuda</p>
<p>GeForce GTX 1080 Ti<br>
Memory Usage:<br>
Allocated: 0.4 GB<br>
Cached:    1.9 GB</p>
</blockquote>
<p>here is my code :</p>
<blockquote>
<p>for l in range(0,len(XTrain)):</p>
<pre><code>loss=0.0
loss_test=0.0

Net=UNet(1,4)  
Net.apply(init_weights)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# torch.backends.cudnn.benchmark = True
# torch.cuda.set_device(0)
Net.to(device)
size=[]
size_test=[]
taille=[]
x={}
y={}
z={}
x_test={}
y_test={}
z_test={}
data_train_norm=[]
data_test_norm=[]
gt_train_norm=[]
gt_test_norm=[]

for k in range(0,int(0.8*len(data_ed))):
    data_train_norm.append(Preprocessing(XTrain[l][k]))
    gt_train_norm.append((YTrain[l][k]))
    size.append(data_train_norm[k].shape)
    x[k],y[k],z[k]=size[k]
    
for k in range(0,int(0.2*len(data_ed))):
    
    data_test_norm.append(Preprocessing(XTest[l][k]))
    gt_test_norm.append((YTest[l][k]))
    size_test.append(data_test_norm[k].shape)
    x_test[k],y_test[k],z_test[k]=size_test[k]

    
    
image_resized_train=[]
image_resized_test=[]
image_resized_train_gt=[]
image_resized_test_gt=[]


for i in range(0,int(0.8*len(data_ed))):
    for j in range(z[i]):
        
        image_resized_train.append(resize_image(data_train_norm[i][:,:,j],H_train[l][i][1],1.25))
        image_resized_train_gt.append(resize_gt(gt_train_norm[i][:,:,j],H_train[l][i][1],1.25))
      

for i in range(0,int(0.2*len(data_ed))):
    for j in range(z_test[i]):
        
        image_resized_test.append(resize_image(data_test_norm[i][:,:,j],H_test[l][i][1],1.25))
        image_resized_test_gt.append(resize_gt(gt_test_norm[i][:,:,j],H_test[l][i][1],1.25))
final_train=[] 
final_test=[]
final_train_gt=[] 
final_test_gt=[]
img_train=[]  
img_test=[]
img_train_gt=[]  
img_test_gt=[]
width=557
height=667
data_tensor_test=[]
data_tensor_train=[]
gt_tensor_test=[]
gt_tensor_train=[]

for i in range(0,len(image_resized_train)):  
    
    img_train.append(Resize(image_resized_train[i],width,height))
    
    final_train.append(crop(img_train[i],256,256))

    data_tensor_train.append(torch.from_numpy(np.array(final_train[i])).float())
    # groudtruth train 
    
    img_train_gt.append(Resize(image_resized_train_gt[i],width,height))
    
    final_train_gt.append(crop(img_train_gt[i],256,256))

    gt_tensor_train.append(torch.from_numpy(np.array(final_train_gt[i])).float())
    
      
for i in range(0,len(image_resized_test)):
    
    img_test.append(Resize(image_resized_test[i],width,height))
    
    final_test.append(crop(img_test[i],256,256))

    data_tensor_test.append(torch.from_numpy(np.array(final_test[i])).float())
    
    img_test_gt.append(Resize(image_resized_test_gt[i],width,height))
    
    final_test_gt.append(crop(img_test_gt[i],256,256))

    gt_tensor_test.append(torch.from_numpy(np.array(final_test_gt[i])).float())

    
X_train=torch.stack(data_tensor_train)
X_test=torch.stack(data_tensor_test)
Y_train=torch.stack(gt_tensor_train)
Y_test=torch.stack(gt_tensor_test)
print(l,X_train.shape,X_test.shape,Y_train.shape,Y_test.shape)

class LoadDataset_train(Dataset):
    

    def __init__(self):
        # load data
        self.X=X_train
        self.Y=Y_train
        self.len=X_train.shape[0]
        

    def __len__(self):
        
        # dataset's size
        return self.len
        

    def __getitem__(self, idx):
        X,Y = self.X[idx], self.Y[idx]
        # the position
        X = X.view(1,X.shape[0],X.shape[1])
        sample=X,Y
        return sample 
    
class LoadDataset_test(Dataset):
    

    def __init__(self):
        # load data
        self.X=X_test
        self.Y=Y_test
        self.len=X_test.shape[0]
        

    def __len__(self):
        
        # dataset's size
        return self.len
        

    def __getitem__(self, idx):
        X,Y = self.X[idx], self.Y[idx]
        # the position
        X = X.view(1,X.shape[0],X.shape[1])
        sample=X,Y
        return sample       
    
dataset=LoadDataset_train()   
trainloader = torch.utils.data.DataLoader(dataset=dataset, batch_size=4, shuffle=True, num_workers=1)
dataset_test=LoadDataset_test()       
trainloader_test = torch.utils.data.DataLoader(dataset=dataset_test, batch_size=1, shuffle=True, num_workers=1)


# print(trainloader)

learning_rate=10e-3
#define the weights using in the loss function 
weights = torch.tensor([0.07,0.31,0.31,0.31]).to(device)  ########
#define a loss function
criterion = nn.CrossEntropyLoss(weights)  
#define the optimizer aDAM 
optimizer=optim.Adam(Net.parameters(), lr=learning_rate , weight_decay=10e-8)



n_epochs=10
loss_values=[]
loss_values_test=[]

for epoch in range(n_epochs):
    Net.train()
    running_loss = 0.0
  
    
    
    
    for i, data in enumerate(trainloader):
        
            
        # get the inputs; data is a list of [inputs, labels]

        inputs, labels = data[0].to(device), data[1].to(device).long()
        
        
            
        optimizer.zero_grad()
        outputs=Net(inputs)
        loss=criterion(outputs,labels)
        running_loss+=loss.item()
        loss.backward()
        optimizer.step()
        
       
        print(i)
    
    loss_values.append(running_loss/len(trainloader)) 
     
    running_loss_test=0.0
    
    
    Net.eval()
    with torch.no_grad():
        
        for j, data_test in enumerate(trainloader_test,0):
        
    
            inputs_test, labels_test = data_test[0].to(device), data_test[1].to(device).long()
            outputs_test=Net(inputs_test)
            loss_test=criterion(outputs_test,labels_test)
            running_loss_test+=loss_test.item()
            

    loss_values_test.append(running_loss_test/len(trainloader_test))
</code></pre>
</blockquote>
<p>THANK YOU !!!</p>

ptrblck: <p>Could you check, if the labels contains only values in the range <code>[0, nb_classes-1]</code>?<br>
If you are using PyTorch <code>1.5.0</code>, could you update to <code>1.5.1</code>, as we had a bug in <code>1.5.0</code>, which skips the valid assertions inside some CUDA methods and instead crashes with these unfriendly CUDA errors?</p>

tr_marwa: <p>thank you very much for reply <img src="https://discuss.pytorch.org/images/emoji/apple/slight_smile.png?v=12" title=":slight_smile:" class="emoji" alt=":slight_smile:" loading="lazy" width="20" height="20"><br>
effectively, it’s because of the label values, I didn’t take the real labels … !<br>
after training and validation i had this losses using kfolds cross validation (5 iterations), the BLUE one is for training and the ORANGE is for testing</p>
<p><img src="https://discuss.pytorch.org/uploads/default/original/3X/3/2/32b1e3fd4fd0eaa1a58055a5d5c0507fbe2bd67a.png" alt="loss(groupe1)" data-base62-sha1="7et08ZTHoTfXclk5cyPAKzzh3jc" width="540" height="331"> <img src="https://discuss.pytorch.org/uploads/default/original/3X/a/4/a448fc19cea80e6d43bdd90cca28ffb76c5ddca1.png" alt="loss(groupe2)" data-base62-sha1="nrkFO0twGEzEBUMdBbhYlS6p9L3" width="423" height="319"> <img src="https://discuss.pytorch.org/uploads/default/original/3X/9/4/94dc2e106fcd8a9b5aac00a464fed076d2214352.png" alt="loss(groupe3)" data-base62-sha1="leSoJDp4pi6OjHxhN5w4xJi3M7U" width="411" height="324"> <img src="https://discuss.pytorch.org/uploads/default/original/3X/d/f/df162f9eacad8685dbdc79cea958efc6da00ef4d.png" alt="loss(groupe4)" data-base62-sha1="vPw0ztgEQITPgZj5yvRBHkWAPpP" width="403" height="280"> <img src="https://discuss.pytorch.org/uploads/default/original/3X/b/5/b5e5889007f45afeb265155742f432f580ce8787.png" alt="loss(groupe5)" data-base62-sha1="pX8c6IgWHdBO3Atzm2dFQniys9V" width="446" height="325"></p>
<p>Do you think that what i’m having is good ??<br>
Thank you</p>

ptrblck: <p>The loss curves look reasonable.<br>
If they are “good” depends a bit on your use case and if the validation and training accuracy are in the range you want them to be. <img src="https://discuss.pytorch.org/images/emoji/apple/wink.png?v=9" title=":wink:" class="emoji" alt=":wink:"></p>

tr_marwa: <p>THANK YOU VERY MUCH <a class="mention" href="/u/ptrblck">@ptrblck</a>, it’s so kind of you, hope you a great day !!! <img src="https://discuss.pytorch.org/images/emoji/apple/pray.png?v=9" title=":pray:" class="emoji" alt=":pray:"> <img src="https://discuss.pytorch.org/images/emoji/apple/smiley.png?v=9" title=":smiley:" class="emoji" alt=":smiley:"></p>