Trying to backward through the graph a second time, but the saved intermediate results have already been freed. Specify retain_graph=True when calling backward the first time

Ibrahim_Shaer: <p>I checked the previous solutions for the same problem, which advise to add detach to any intermediate result obtained from implementing the mode but to no avail.<br>
Here is a snipped of the code and some of the functions used.</p>
<p><div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/3X/f/a/faa1013ca54e2d66d5c1ef70b93d2deebf4b026f.png" data-download-href="https://discuss.pytorch.org/uploads/default/faa1013ca54e2d66d5c1ef70b93d2deebf4b026f" title="image_1_pytorch"><img src="https://discuss.pytorch.org/uploads/default/original/3X/f/a/faa1013ca54e2d66d5c1ef70b93d2deebf4b026f.png" alt="image_1_pytorch" data-base62-sha1="zLajLKoyoLTXdGtcWl3rfFaVg99" width="690" height="437" data-dominant-color="F1F1F1"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">image_1_pytorch</span><span class="informations">957×607 29.3 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div><br>
<div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/3X/e/9/e9a820ae635dc5082fd60caf39c06a4eb507a6a3.png" data-download-href="https://discuss.pytorch.org/uploads/default/e9a820ae635dc5082fd60caf39c06a4eb507a6a3" title="image_2_pytorch"><img src="https://discuss.pytorch.org/uploads/default/original/3X/e/9/e9a820ae635dc5082fd60caf39c06a4eb507a6a3.png" alt="image_2_pytorch" data-base62-sha1="xl1sFIOTbPZdo1lHSH7NJ2hB6FR" width="527" height="500" data-dominant-color="EFEFEF"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">image_2_pytorch</span><span class="informations">777×736 23.5 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div></p>

ptrblck: <p>Could you post a minimal, executable code snippet reproducing the error, please?<br>
You can wrap the code into three backticks ```, which makes debugging easier since we cannot copy the code from your screenshots.</p>

Ibrahim_Shaer: <p>Here is the full code.</p>
<pre data-code-wrap="from"><code class="lang-nohighlight">from scipy.stats import pearsonr

class ModelTrainer:
    
    def __init__(self, input_sizes, hidden_dim):
        self.input_sizes = input_sizes
        self.h_dim = hidden_dim
        
        model_nn = self.ae_model()
        self.net = model_nn
        self.optimizer = torch.optim.SGD(self.net.parameters(),lr=1e-5)
        self.train_dt_1, self.train_dt_2, self.train_dt_3 = self.generate_train_test_dts()
        self.log_interval = 100
        
    def generate_train_test_dts(self, batch_size=16):
        X_1, y_1 = torch.randn(10000, 64), torch.randn(10000, 1)
        X_2, y_2 = torch.randn(10000, 48), torch.randn(10000, 1)
        X_3, y_3 = torch.randn(10000, 64), torch.randn(10000, 1)
        
        train_dt_1 = TensorDataset(X_1, y_1)
        train_dt_2 = TensorDataset(X_2, y_2)
        train_dt_3 = TensorDataset(X_3, y_3)
        
        dt_1_loader = DataLoader(train_dt_1, batch_size=16)
        dt_2_loader = DataLoader(train_dt_2, batch_size=16)
        dt_3_loader = DataLoader(train_dt_3, batch_size=16)

        
        return dt_1_loader,dt_2_loader, dt_3_loader 

    
    def ae_model(self):
        model_nn = Corr_Net(self.input_sizes, self.h_dim)
        return model_nn
    
    def calculate_l2_loss(self, set_inputs, lambda_val =0.02):
        sum_combinations = 0.0
        input_names = list(set_inputs.keys())
        all_combinations = np.array(np.meshgrid(input_names, input_names)).T.reshape(-1, 2)
        
        for combination in all_combinations:
            input_1, input_2 = combination[0], combination[1]
            if input_1 == input_2:
                continue
            
            sum_combinations += self.calculate_correlation(set_inputs[input_1], set_inputs[input_2])
            
        return -lambda_val * sum_combinations
    
    def calculate_l1_loss(self, set_inputs, loss_fn):
        r_l1_loss, set_hidden_rep = 0.0, {}
        for i in range(len(set_inputs)):
            x_input = []
            for idx, input_batch in enumerate(set_inputs):
                if i == idx:
                    x_input.append(input_batch)
                else:
                    x_input.append(torch.zeros_like(input_batch))
                    

            temp_output = self.net(x_input)
            rel_output = temp_output[i]
            r_l1_loss += loss_fn(x_input[i], rel_output.detach())
            hidden_rep = self.net.out[i].detach()
            set_hidden_rep[f'input_{i}'] = hidden_rep
            
        return r_l1_loss, set_hidden_rep
        
    def calculate_correlation(self, h1, h2):
        h1_mean, h2_mean = torch.mean(h1), torch.mean(h2)
        h1_centered, h2_centered = torch.subtract(h1, h1_mean), torch.subtract(h2, h2_mean)
        corr_nr = torch.sum(torch.multiply(h1_centered, h2_centered))
        
        corr_dr1 = torch.sqrt(torch.square(h1_centered))
        corr_dr2 = torch.sqrt(torch.square(h2_centered))
      
        corr_dr = torch.add(torch.multiply(corr_dr1, corr_dr2), 1e-5)
        return corr
        
    def train(self, epoch):
        self.net.train()
        
        running_loss = 0.0
        
        itr_loader_2, itr_loader_3 = iter(self.train_dt_2), iter(self.train_dt_3)
        
        for i , (input_dt_1, label_dt_1) in enumerate(self.train_dt_1):
            (input_dt_2, labels_dt_2) = next(itr_loader_2)
            (input_dt_3, labels_dt_3) = next(itr_loader_3)
            
            set_inputs = [input_dt_1, input_dt_2, input_dt_3]
            
            self.optimizer.zero_grad()
            outputs = self.net(set_inputs)
            
            #l1_loss that minimizes the reconstruction error when other inputs are inexistent
            loss=nn.MSELoss()
            l1_loss, set_hidden_rep = self.calculate_l1_loss(set_inputs, loss)
            
            #l2_loss that calculates the correlation between hidden representations to encourage the hidden units 
            #of the representation to be shared between the representations
            l2_loss = self.calculate_l2_loss(set_hidden_rep, lambda_val =0.02)
            print(l1_loss, l2_loss)
            
            loss_values = l1_loss + l2_loss
            loss_values.backward()
            
            self.optimizer.step()
            
            running_loss += loss_values
            
            if i % self.log_interval == 0:
                print('[%d, %d/%d] Loss L1 %.3f, Loss L2 %.3f, Loss' %(epoch, i, len(self.train_dt_1)
                                                                       , l1_loss, l2_loss, (running_loss / ((i+1) * len(outputs[0])))))```
Thank you for your help!</code></pre>

ptrblck: <p>Thanks, but it’s not executable as e.g. <code>Corr_Net</code> is undefined. Make sure you can copy/paste the posted code into a new script and directly execute it to reproduce the issue.</p>

Ibrahim_Shaer: <p>Here is the code for the Corr_Net:</p>
<pre><code class="lang-auto">class Corr_Net(nn.Module):
    
    def __init__(self, input_sizes, hidden_dim):
        super(Corr_Net, self).__init__()
        self.encoders = nn.ModuleList()
    
        for input_size in input_sizes:
            self.encoders.append(
                nn.Sequential(
                    nn.Linear(input_size, hidden_dim),
                    nn.ReLU()
                ))
            
        self.decoders = nn.ModuleList()
        
        for input_size in input_sizes:
            self.decoders.append(nn.Sequential(
                    nn.Linear(hidden_dim, input_size),
                    nn.ReLU()
                ))
            
    
    def forward(self, inputs):
        out = []
        for idx, enc in enumerate(self.encoders):
            out.append(enc(inputs[idx]))
            
        self.out = out
        
        common_rep = out[0]
        for i in range(1, len(out)):
            common_rep = torch.add(common_rep, out[idx])
        
        self.common_rep = common_rep
        
        reconstructed_layer = []
        
        for i in range(len(out)):
            reconstructed_layer.append(self.decoders[i](out[i]))
        
        return reconstructed_layer

m_t = ModelTrainer([64, 48, 64], 10)
for epoch in range(0, 5):
    m_t.train(epoch)

</code></pre>

ptrblck: <p>Thanks for the update.<br>
Your losses do not depend on any parameters used in the model as they are calculated using the inputs:</p>
<pre><code class="lang-python">            set_inputs = [input_dt_1, input_dt_2, input_dt_3]
            
            self.optimizer.zero_grad()
            outputs = self.net(set_inputs)
            
            #l1_loss that minimizes the reconstruction error when other inputs are inexistent
            loss=nn.MSELoss()
            l1_loss, set_hidden_rep = self.calculate_l1_loss(set_inputs, loss)
            
            #l2_loss that calculates the correlation between hidden representations to encourage the hidden units 
            #of the representation to be shared between the representations
            l2_loss = self.calculate_l2_loss(set_hidden_rep, lambda_val =0.02)
</code></pre>
<p>As you can see, <code>l1_loss</code> is calculated using <code>set_inputs</code> and <code>l2_loss</code> using the output of <code>calculate_l1_loss</code>.</p>

Ibrahim_Shaer: <p>Thank you for your answer.<br>
In <code>calculate_l1_loss</code>, I am using the network to obtain <code>temp_output</code>.</p>

ptrblck: <p>Yes, but you are detaching it, so these tensors are constants:</p>
<pre><code class="lang-python">            r_l1_loss += loss_fn(x_input[i], rel_output.detach())
            hidden_rep = self.net.out[i].detach()
</code></pre>

Ibrahim_Shaer: <p>Thank you again for your answer. The issue is related to a type. The updated code for <code>calculate_l1_loss</code> and <code>calculate_l2_loss</code> is as follows:</p>
<pre><code class="lang-auto">def calculate_l2_loss(self, set_inputs, lambda_val =0.02):
        sum_combinations = 0.0
        input_names = list(set_inputs.keys())
        all_combinations = np.array(np.meshgrid(input_names, input_names)).T.reshape(-1, 2)
        
        for combination in all_combinations:
            input_1, input_2 = combination[0], combination[1]
            if input_1 == input_2:
                continue
            
            sum_combinations += self.calculate_correlation(set_inputs[input_1], set_inputs[input_2], lambda_val)
            
        return sum_combinations
    
    def calculate_l1_loss(self, set_inputs, loss_fn):
        r_l1_loss, set_hidden_rep = 0.0, {}
        for i in range(len(set_inputs)):
            x_input = []
            for idx, input_batch in enumerate(set_inputs):
                if i == idx:
                    x_input.append(input_batch)
                else:
                    x_input.append(torch.zeros_like(input_batch))
                    

            temp_output = self.net(x_input)
            rel_output = temp_output[i]
            r_l1_loss += loss_fn(x_input[i], rel_output)
            hidden_rep = self.net.out[i]
            set_hidden_rep[f'input_{i}'] = hidden_rep
            
        return r_l1_loss, set_hidden_rep
        
    def calculate_correlation(self, h1, h2, lambda_val):
        h1_mean, h2_mean = torch.mean(h1), torch.mean(h2)
        h1_centered, h2_centered = torch.subtract(h1, h1_mean), torch.subtract(h2, h2_mean)
        corr_nr = torch.sum(torch.multiply(h1_centered, h2_centered))
        
        corr_dr1 = torch.sqrt(torch.square(h1_centered))
        corr_dr2 = torch.sqrt(torch.square(h2_centered))
      
        corr_dr = torch.add(torch.multiply(corr_dr1, corr_dr2), 1e-5)
        corr = torch.multiply(torch.divide(corr_nr, corr_dr), -lambda_val)
        
        return torch.mean(corr)

</code></pre>
<p>Thank you for your help.</p>