RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn (1D CNN error)

username: hfdp
cooked: <p>Hello,</p>
<p>I am running a CNN and trying to train it later on, however i get the error <strong>RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn</strong><br>
The error appears when I cal ‘loss.backward()’ in the validation step.<br>
Here is my code, help would be appreciated <img src="https://discuss.pytorch.org/images/emoji/apple/smiley.png?v=9" title=":smiley:" class="emoji" alt=":smiley:"></p>
<pre><code class="lang-auto">&gt; import torch.nn as nn
&gt; import torch.nn.functional as F
&gt; 
&gt; class MyModel(nn.Module):
&gt;     def __init__(self):
&gt;         super(MyModel, self).__init__()
&gt;         self.conv1 = nn.Conv1d(1, 3, kernel_size=3) 
&gt;         self.conv2 = nn.Conv1d(3, 6, kernel_size=6) 
&gt;         self.conv3 = nn.Conv1d(6, 12, kernel_size=12) 
&gt;         self.conv4 = nn.Conv1d(12,24, kernel_size=24) 
&gt;         self.conv5 = nn.Conv1d(24,48, kernel_size = 48)
&gt;         self.conv_drop = nn.Dropout2d()
&gt;         self.conv6_transpose = nn.ConvTranspose1d(48,24, kernel_size = 48)
&gt;         self.conv7_transpose = nn.ConvTranspose1d(24,12, kernel_size = 24)
&gt;         self.conv8_transpose = nn.ConvTranspose1d(12,6, kernel_size = 12)
&gt;         self.conv9_transpose=nn.ConvTranspose1d(6,3, kernel_size = 6)
&gt;         self.conv10_transpose = nn.ConvTranspose1d(3,1, kernel_size = 3)
&gt; 
&gt; 
&gt;         self.bn1 = nn.BatchNorm1d(3)
&gt;         self.bn2 = nn.BatchNorm1d(6)
&gt;         self.bn3 = nn.BatchNorm1d(12)
&gt;         self.bn4 = nn.BatchNorm1d(24)
&gt;         self.bn5 = nn.BatchNorm1d(48)
&gt;         self.bn6 = nn.BatchNorm1d(24)
&gt;         self.bn7 = nn.BatchNorm1d(12)
&gt;         self.bn8 = nn.BatchNorm1d(6)
&gt;         self.bn9 = nn.BatchNorm1d(3)
&gt;         self.bn10 = nn.BatchNorm1d(1)
&gt; 
&gt; 
&gt;     def forward(self, x):
&gt;         x = x.unsqueeze(0).permute(1, 0, 2)
&gt; #         print (x.shape)
&gt;         out = self.conv1(x)
&gt;         out = F.relu(out)
&gt;         out = self.bn1(out)
&gt; #         print (out.shape)
&gt;         #        out = out.view(out.shape[0], -1)
&gt; 
&gt;         out = self.conv2(out)
&gt;         out = F.relu(out)
&gt;         out = self.bn2(out)
&gt; #         print (out.shape)
&gt; 
&gt;         out = self.conv3(out)
&gt;         out = F.relu(out)
&gt;         out = self.bn3(out)
&gt; #         print (out.shape)
&gt; 
&gt;         out = self.conv4(out)
&gt;         out = F.relu(out)
&gt;         out = self.bn4(out)
&gt; #         print (out.shape)
&gt; 
&gt;         #bottleneck
&gt;         out = self.conv5(out)
&gt;         out = F.relu(out)
&gt;         out = self.bn5(out)
&gt;         out = self.conv_drop(out)
&gt; #         print (out.shape)
&gt; 
&gt; 
&gt;         #upsample
&gt;         out = self.conv6_transpose(out)
&gt;         out = self.conv_drop(out)
&gt;         out = F.relu(out)
&gt; #         print (out.shape)
&gt; #         out = out.unsqueeze(0)
&gt;         out = self.bn6(out)
&gt; 
&gt;         out = self.conv7_transpose(out)
&gt;         out = self.conv_drop(out)
&gt;         out = F.relu(out)
&gt; #         out = out.unsqueeze(0)
&gt;         out = self.bn7(out)
&gt; 
&gt;         out = self.conv8_transpose(out)
&gt;         out = self.conv_drop(out)
&gt;         out = F.relu(out)
&gt; #         out = out.unsqueeze(0)
&gt;         out = self.bn8(out)
&gt; 
&gt;         out = self.conv9_transpose(out)
&gt;         out = self.conv_drop(out)
&gt;         out = F.relu(out)
&gt; #         out = out.unsqueeze(0)
&gt;         out = self.bn9(out)
&gt; 
&gt;         out = self.conv10_transpose(out)
&gt;         out = self.conv_drop(out)
&gt;         out = F.relu(out)
&gt; #         out = out.unsqueeze(1)
&gt;         out = self.bn10(out)
&gt;         return out        
&gt;         
&gt; model = MyModel()
&gt; 
&gt; print(model)
&gt; 
&gt; #%% Here we train and evaluate the model
&gt;         
&gt; 
&gt; import torch
&gt; from torch.utils import data
&gt; import numpy as np
&gt; import torch.optim as optim
&gt; 
&gt; 
&gt; # Check if CUDA is available
&gt; use_cuda = torch.cuda.is_available()
&gt; device = torch.device("cuda:0" if use_cuda else "cpu")
&gt; # cudnn.benchmark = True
&gt; 
&gt; 
&gt; # Set training parameters
&gt; params = {'batch_size': 64,
&gt;           'shuffle': True,
&gt;           'num_workers': 6}
&gt; max_epochs = 100
&gt; 
&gt; # Load all the data from the txt file
&gt; file_IDs = open('ID_list.txt','r').read().split('\n')
&gt; file_IDs = file_IDs[:-1] # remove last line
&gt; complete_dataset = Dataset(file_IDs)
&gt; 
&gt; 
&gt; #%% Here we define a loss function and the optimizer
&gt; # create your loss function
&gt; def rmse(y, y_hat):
&gt;     """Compute root mean squared error"""
&gt;     return torch.sqrt(torch.mean((y - y_hat).pow(2)))
&gt; 
&gt; # create your optimizer
&gt; optimizer = optim.SGD(model.parameters(), lr=0.0003, momentum = 0.1)
&gt; 
&gt; #%% Here we train the network
&gt; 
&gt; # Divide the dataset into the training and validation set
&gt; lengths = [int(np.ceil(len(complete_dataset)*0.8)), int(np.floor(len(complete_dataset)*0.1)), int(np.floor(len(complete_dataset)*0.1))]
&gt; training_set, validation_set, evaluation_set = torch.utils.data.random_split(complete_dataset, lengths)
&gt; training_generator = data.DataLoader(training_set, **params)
&gt; validation_generator = data.DataLoader(validation_set, **params)
&gt; evaluation_generator = data.DataLoader(evaluation_set, **params)
&gt; 
&gt; 
&gt; # instantiate the model to make it a double tensor
&gt; forward_model = model.double()
&gt; 
&gt; # check if model works for random instance of the data 
&gt; t20, t100 ,t250 = next(iter(training_generator))
&gt; one_prediction = forward_model(t20)
&gt; 
&gt; 
&gt; # Loop over epochs
&gt; for epoch in range(max_epochs):
&gt;  for param in forward_model.parameters():
&gt;  	param.requires_grad = True
&gt;     
&gt;     
&gt;     # Training
&gt;  with torch.set_grad_enabled(True):
&gt;   for t20MHz, t100MHz, t250MHz in training_generator:
&gt;         # Transfer to GPU if available
&gt;             t20MHz, t100MHz, t250MHz = t20MHz.to(device), t100MHz.to(device), t250MHz.to(device)
&gt; 
&gt;         #zero the parameter gradients
&gt;             optimizer.zero_grad()
&gt; 
&gt; 
&gt;         # forward, backward and optimize
&gt;             prediction_training = forward_model(t20MHz)
&gt;             loss = rmse(prediction_training, t250MHz) #Here compute tha value that estimates how far the outut is from the target
&gt;             loss.backward() #The whole graph is differentiatted wrt the loss and all ensors in the graph that have 'required_gard = true' will have their .grad tensot accumulaed ith the gradient. 
&gt;             optimizer.step()
&gt;         
&gt;         
&gt;         
&gt;         
&gt;         # Validation (needs to be adapted)
&gt;  with torch.set_grad_enabled(False):
&gt;   for t20MHz, t100MHz, t250MHz in validation_generator:
&gt;             # Transfer to GPU if available
&gt;                 t20MHz, t100MHz, t250MHz = t20MHz.to(device), t100MHz.to(device), t250MHz.to(device)  
&gt; 
&gt;             # Model computations
&gt;                 prediction_validation = forward_model(t20MHz)
&gt;                 loss = rmse(prediction_validation, t250MHz)
&gt;                 loss.backward()
&gt;             
&gt;                 #torch.save(forward_model.state_dict(), data_path + 'conv_net_model.pth')
&gt;         
&gt; 
&gt;         
&gt;               
&gt; 
&gt;         
&gt;       
&gt; 
&gt; #%% Here we are going to evaluate the network
&gt; 
&gt; forward_model.eval()
&gt; with torch.set_grad_enabled(False):
&gt; # Loop over epochs
&gt;     for epoch in range(max_epochs):
&gt;     
&gt;     
&gt;     # Training
&gt;         for t20MHz, t100MHz, t250MHz in evaluation_generator:
&gt;         # Transfer to GPU if available
&gt;             t20MHz, t100MHz, t250MHz = t20MHz.to(device), t100MHz.to(device), t250MHz.to(device)
&gt; 
&gt;         #zero the parameter gradients
&gt;             optimizer.zero_grad()
&gt; 
&gt; 
&gt;         # forward, backward and optimize
&gt;             prediction_evaluation = forward_model(t20MHz)
&gt; #            loss = rmse(prediction, t250MHz) #Here compute tha value that estimates how far the outut is from the target
&gt; #            loss.backward() #The whole graph is differentiatted wrt the loss and all ensors in the graph that have 'required_gard = true' will have their .grad tensot accumulaed ith the gradient. 
&gt; #            optimizer.step()                    
&gt;         
&gt; 
&gt;             torch.save(forward_model.state_dict(), data_path + 'conv_net_model.pt')
</code></pre>

username: ptrblck
cooked: <p>You won’t be able to call <code>loss.backward()</code> inside a block, where gradient calculation is disabled via <code>with torch.set_grad_enabled(False)</code>.<br>
You could of course enable it, but calculating gradients in the validation set (and updating the model) is a bad idea, as you are leaking the validation dataset into the training and thus make the validation set unusable to validate the training.</p>