RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [800, 306]]

Isabella_Costa_Maia: <p>Hello,</p>
<p>I’m implementing the bahdanau’s attention for the task of regression and I’ m having this issue.</p>
<blockquote>
<p>RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [800, 306]] is at version 2; expected version 1 instead. Hint: the backtrace further above shows the operation that failed to compute its gradient. The variable in question was changed in there or anywhere later. Good luck!</p>
</blockquote>
<p>Apparently I’m doing some inplace operation in some tensor that requires gradient, as I read in some related posts. I already did plenty of copy() where I thought I could be changing some tensor, but it didn’t work. Can someone help find the mistake? Thanks,</p>
<pre data-code-wrap="python"><code class="lang-python">class LSTM(nn.Module):
   def __init__(self, input_size, hidden_size, num_layers=1, dropout = 0.2):
       super().__init__()
       self.hidden_size = hidden_size
       self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True, dropout=dropout)
       self.num_layers = num_layers
       
   def forward(self, x, hs):

       out, hs = self.lstm(x, hs)        

       return out, hs
   
   def init_hidden(self):
       return (torch.zeros(self.num_layers , 1, self.hidden_size),
               torch.zeros(self.num_layers , 1, self.hidden_size))
   

class BahdanauDecoder(nn.Module):
   def __init__(self, hidden_size, output_size, n_layers=1, drop_prob=0.1):
       super(BahdanauDecoder, self).__init__()
       self.hidden_size = hidden_size
       self.output_size = output_size
       self.n_layers = n_layers
       self.drop_prob = drop_prob
       self.input_size = input_size #number of machines 

       self.fc_hidden = nn.Linear(self.hidden_size, self.hidden_size, bias=False)
       self.fc_encoder = nn.Linear(self.hidden_size, self.hidden_size, bias=False)
       self.weight = nn.Parameter(torch.FloatTensor(1, hidden_size))
       self.attn_combine = nn.Linear(self.hidden_size * 2, self.hidden_size)
       self.dropout = nn.Dropout(self.drop_prob)
       self.lstm = nn.LSTM(self.hidden_size + self.input_size, self.hidden_size, batch_first=True)
       self.linear= nn.Linear(self.hidden_size, self.output_size)

   def forward(self, inputs, hidden, encoder__outputs):
       #inputs are actually the previous decoder outputs
       
       encoder_outputs = encoder__outputs.clone().squeeze()
       
       embedded = inputs[0].clone() #shape (#num_machines, 1)
       
       embedded = self.dropout(embedded)

       # Calculating Alignment Scores
       x = torch.tanh(self.fc_hidden(hidden[0].clone())+self.fc_encoder(encoder_outputs))
       a = self.weight.unsqueeze(2).detach()
       alignment_scores = x.clone().bmm(a)  

       # Softmaxing alignment scores to get Attention weights
       attn_weights = F.softmax(alignment_scores.clone().view(1,-1), dim=1)
       
       # Multiplying the Attention weights with encoder outputs to get the context vector
       b = attn_weights.clone().unsqueeze(0).detach()
       c = encoder_outputs.clone().unsqueeze(0).detach()
       context_vector = torch.bmm(b,
                                c)
       
       output = torch.cat((embedded.clone().view(1,-1), context_vector[0].clone()), 1).unsqueeze(0)
       
       # Passing the concatenated vector as input to the LSTM cell
       output, hidden = self.lstm(output, hidden)

       output = self.linear(output[0].clone())
       
       return output, hidden, attn_weights
   
   def init_in(self):
       return torch.zeros(1 , input_size, 1)
   
   def init_hidden(self):
       return (torch.zeros(1 , 1, hidden_size),  
                        torch.zeros(1 , 1, hidden_size)) 
</code></pre>
<p>and the training code</p>
<pre data-code-wrap="python"><code class="lang-python">def train(model_encoder, model_decoder, epochs, 
         train_set, train_window, device, 
         valid_data=None, lr=0.001, batch_size =1, 
         print_every=10, loss_fct = 'MSE'):

   if device == 'cuda':
       model_encoder.cuda()
       model_decoder.cuda()
       
   
   if loss_fct == "MSE":
       criterion = nn.MSELoss()

   torch.autograd.set_detect_anomaly(True)
   opt_encod = optim.Adam(model_encoder.parameters(), lr=lr)
   opt_decod  = optim.Adam(model_decoder.parameters(), lr=lr)
   
   train_loss = []
   valid_loss = []
   
   for e in range(epochs):
       
       #initial values
       hs_encoder = model_encoder.init_hidden()
       in_decoder = model_decoder.init_in()
       
       if device == 'cuda':
           hs_encoder = tuple([i.cuda().to(device) for i in hs_encoder]) 
           in_decoder = in_decoder.cuda().to(device)

       t_loss = []
       for x, y in get_batches_dataloader(train_set, train_window, batch_size = batch_size):
           
           if device == 'cuda':
               x = x.cuda().to(device)
               y = y.cuda().to(device)

           opt_encod.zero_grad()
           opt_decod.zero_grad()
           
           # Create batch_size dimension if it doesn't exists
           if (len(x.shape)==2):
               x = x.unsqueeze(0)
               
           out_encoder, hs_encoder = model_encoder(x, hs_encoder)
           hs_decoder = hs_encoder
           out_decoder, hs_decoder, aw = model_decoder(in_decoder, hs_decoder, out_encoder)
           out_decoder = out_decoder.unsqueeze(0)
           in_decoder = out_decoder.detach()


           loss = criterion(out_decoder.view(1,1,-1), y.unsqueeze(0))

           loss.backward(retain_graph=True)
           opt_encod.step()
           opt_decod.step()
           t_loss.append(loss.item())


</code></pre>

el_youssfi_azeddine: <p>If you have operation like a+=b replace it by a=a+b.</p>

ptrblck: <p>Besides the inplace op check, could you explain why and if <code>retain_graph=True</code> is needed, as it often points towards a wrong usage (in order to avoid another error)?</p>

Isabella_Costa_Maia: <p>You’re right, I had added it as a suggestion of the error I had before:</p>
<blockquote>
<p>RuntimeError: Trying to backward through the graph a second time, but the saved intermediate results have already been freed. Specify retain_graph=True when calling .backward() or autograd.grad() the first time.</p>
</blockquote>
<p>I could fix this inplace operation error that I described before by detaching the input of the last linear layer in the decoder, however I’m not sure about the meaning of dropping this tensor from the gradient computation graph…could you explain me if it makes sense not to propagate the gradient in this tensor?</p>
<pre data-code-wrap="python"><code class="lang-python">class BahdanauDecoder(nn.Module): 
   def __init__(self, hidden_size, output_size, n_layers=1, drop_prob=0.1):
       super(BahdanauDecoder, self).__init__()
       self.hidden_size = hidden_size
       self.output_size = output_size
       self.n_layers = n_layers
       self.drop_prob = drop_prob
       self.input_size = input_size #number of machines 

       self.fc_hidden = nn.Linear(self.hidden_size, self.hidden_size, bias=False)
       self.fc_encoder = nn.Linear(self.hidden_size, self.hidden_size, bias=False)
       self.weight = nn.Parameter(torch.FloatTensor(1, hidden_size))
       self.attn_combine = nn.Linear(self.hidden_size * 2, self.hidden_size)
       self.dropout = nn.Dropout(self.drop_prob)
       self.lstm = nn.LSTM(self.hidden_size + self.input_size, self.hidden_size, batch_first=True)
       self.classifier = nn.Linear(self.hidden_size, self.output_size)

   def forward(self, inputs, hidden, encoder_outputs):
       #inputs are actually the previous decoder outputs
       encoder_outputs = encoder_outputs.squeeze()
       
       embedded = inputs[0].clone() #shape (#num_machines, 1)
       
       embedded = self.dropout(embedded)

       # Calculating Alignment Scores
       x = torch.tanh(self.fc_hidden(hidden[0])+self.fc_encoder(encoder_outputs))
       alignment_scores = x.bmm(self.weight.unsqueeze(2))  

       # Softmaxing alignment scores to get Attention weights
       attn_weights = F.softmax(alignment_scores.view(1,-1), dim=1)

       # Multiplying the Attention weights with encoder outputs to get the context vector
       context_vector = torch.bmm(attn_weights.unsqueeze(0),
                                encoder_outputs.unsqueeze(0))

       # Concatenating context vector with embedded input word
       output = torch.cat((embedded.view(1,-1), context_vector[0]), 1).unsqueeze(0)
       
       # Passing the concatenated vector as input to the LSTM cell
       output, hidden = self.lstm(output, hidden)

       output = self.classifier(output[0].clone().detach()) #CHANGED HERE

       return output, hidden, attn_weights
   
   def init_in(self):
       return torch.zeros(1 , input_size, 1)
   
   def init_hidden(self):
       return (torch.zeros(1 , 1, hidden_size),  
                        torch.zeros(1 , 1, hidden_size))

</code></pre>
<p>The above works, even when taking out the option <code>retain_graph=True</code> from the backward propagation in the train, however I’m not sure why.</p>

ptrblck: <p>Thanks for the update.<br>
No, I don’t think you should <code>detach()</code> this tensor, since the computation graph will be cut at this place and no modules used before the <code>detach()</code> operation was used will get a valid gradient (<code>self.classifier</code> would be the only trained module).<br>
The initial error (without using <code>retain_graph=True</code>) is raised e.g. if you try to calculate the gradients via a second <code>backward()</code> call while the intermediate forward activations were already freed in the first <code>backward()</code> operation.<br>
This is often an issue in RNNs if you are e.g. reusing the hidden states without detaching these (the computation graph would grow in each iteration and the <code>backward()</code> operation would then try to calculate the gradients for the current and the previous iterations).<br>
Given that, I think you should check, if using <code>output, hidden = self.lstm(output, hidden.detach())</code> would work (and pass <code>output</code> directly to <code>self.classifier</code> again).</p>

Isabella_Costa_Maia: <p>Thank you for your answer.<br>
Just to confirm, you’re saying that I should detach the hidden state in the input of the lstm layer from the decoder class, right?<br>
And what is the proper way of detaching the hidden state if it is a tuple? I’ve tried to do it as you suggested but I’m getting the following:</p>
<pre><code class="lang-python">AttributeError: 'tuple' object has no attribute 'detach'
</code></pre>
<p>Thank you,</p>

ptrblck: <aside class="quote no-group" data-username="Isabella_Costa_Maia" data-post="6" data-topic="129109">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/isabella_costa_maia/48/39081_2.png" class="avatar"> Isabella_Costa_Maia:</div>
<blockquote>
<p>Just to confirm, you’re saying that I should detach the hidden state in the input of the lstm layer from the decoder class, right?</p>
</blockquote>
</aside>
<p>Yes, assuming you want to use the previous “values” of the hidden state, but do not want to extend the computation graph.</p>
<aside class="quote no-group" data-username="Isabella_Costa_Maia" data-post="6" data-topic="129109">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/isabella_costa_maia/48/39081_2.png" class="avatar"> Isabella_Costa_Maia:</div>
<blockquote>
<p>And what is the proper way of detaching the hidden state if it is a tuple? I’ve tried to do it as you suggested but I’m getting the following:</p>
</blockquote>
</aside>
<p>You can recreate the <code>tuple</code> via:</p>
<pre data-code-wrap="python"><code class="lang-python">hidden = tuple(h.detach() for h in hidden)
</code></pre>

Isabella_Costa_Maia: <p>Thank you for the answer.</p>
<p>I’ve included</p>
<aside class="quote no-group" data-username="ptrblck" data-post="7" data-topic="129109">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/ptrblck/48/1823_2.png" class="avatar"> ptrblck:</div>
<blockquote>
<pre><code class="lang-auto">hidden = tuple(h.detach() for h in hidden)
</code></pre>
</blockquote>
</aside>
<p>this line just before the use of the lstm layer as so the code becomes the following:</p>
<pre data-code-wrap="python"><code class="lang-python">
class BahdanauDecoder(nn.Module): 
   def __init__(self, hidden_size, output_size, n_layers=1, drop_prob=0.1):
       super(BahdanauDecoder, self).__init__()
       self.hidden_size = hidden_size
       self.output_size = output_size
       self.n_layers = n_layers
       self.drop_prob = drop_prob
       self.input_size = input_size #number of machines 

       self.fc_hidden = nn.Linear(self.hidden_size, self.hidden_size, bias=False)
       self.fc_encoder = nn.Linear(self.hidden_size, self.hidden_size, bias=False)
       self.weight = nn.Parameter(torch.FloatTensor(1, hidden_size))
       self.attn_combine = nn.Linear(self.hidden_size * 2, self.hidden_size)
       self.dropout = nn.Dropout(self.drop_prob)
       self.lstm = nn.LSTM(self.hidden_size + self.input_size, self.hidden_size, batch_first=True)
       self.classifier = nn.Linear(self.hidden_size, self.output_size)

   def forward(self, inputs, hidden, encoder_outputs):
       #inputs are actually the previous decoder outputs
       encoder_outputs = encoder_outputs.squeeze()
       
       embedded = inputs[0].clone() #shape (#num_machines, 1)
       
       embedded = self.dropout(embedded)

       # Calculating Alignment Scores
       x = torch.tanh(self.fc_hidden(hidden[0])+self.fc_encoder(encoder_outputs))
       alignment_scores = x.bmm(self.weight.unsqueeze(2))  

       # Softmaxing alignment scores to get Attention weights
       attn_weights = F.softmax(alignment_scores.view(1,-1), dim=1)

       # Multiplying the Attention weights with encoder outputs to get the context vector
       context_vector = torch.bmm(attn_weights.unsqueeze(0),
                                encoder_outputs.unsqueeze(0))

       # Concatenating context vector with embedded input word
       output = torch.cat((embedded.view(1,-1), context_vector[0]), 1).unsqueeze(0)
       
       # Passing the concatenated vector as input to the LSTM cell
       hidden = tuple(h.detach() for h in hidden) #HERE
       output, hidden = self.lstm(output, hidden)

       output = self.classifier(output[0])

       return output, hidden, attn_weights
   
   def init_in(self):
       return torch.zeros(1 , input_size, 1)
   
   def init_hidden(self):
       return (torch.zeros(1 , 1, hidden_size),  
                        torch.zeros(1 , 1, hidden_size))


</code></pre>
<p>However I’m getting the same sort of error as before:</p>
<blockquote>
<p>RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [500, 309]], which is output 0 of TBackward, is at version 2; expected version 1 instead. Hint: the backtrace further above shows the operation that failed to compute its gradient. The variable in question was changed in there or anywhere later. Good luck!</p>
</blockquote>