Backward through the graph a second time

username: Mohan_Krishna_Shrest
cooked: <p>Hi ,<br>
In this code , i used backward on variable computed using different weights. One weight came from random and another came from correcting given weights through .grad . Then those weights were detached from prev graph and copied to another list and then used those for another epoches(say) .<br>
But an error occured :Traceback (most recent call last):<br>
File “C:\new_folder\computer\new_neurons.py”, line 191, in <br>
error_test , weights_test = train_function(new_weights)<br>
^^^^^^^^^^^^^^^^^^^^^^^^^^^<br>
File “C:\new_folder\computer\new_neurons.py”, line 170, in train_function<br>
err[epoch].backward()<br>
File “C:\new_folder\computer\venv\Lib\site-packages\torch_tensor.py”, line 653, in backward<br>
torch.autograd.backward(<br>
File “C:\new_folder\computer\venv\Lib\site-packages\torch\autograd_<em>init</em>_.py”, line 354, in backward<br>
_engine_run_backward(<br>
File “C:\new_folder\computer\venv\Lib\site-packages\torch\autograd\graph.py”, line 829, in _engine_run_backward<br>
return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass<br>
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^<br>
RuntimeError: Trying to backward through the graph a second time (or directly access saved tensors after they have already been freed). Saved intermediate values of the graph are freed when you call .backward() or autograd.grad(). Specify retain_graph=True if you need to backward through the graph a second time or if you need to access saved tensors after calling backward.<br>
Here is the code:</p>
<pre><code class="lang-auto">import random
import torch
lr = 0.01
layers = [3,2,5]
epoches = 10 
n = 100 # number of dimensions
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print("Using device:", device)

class Neuron:
    def __init__(self, weights, inputs):
        self.inputs = inputs
        self.weights = weights
        self.output, self.weights, self.unit_output = self.calculate_output()

    def calculate_output(self , inputs_training = []):
        some = []
        weights = self.weights
        # print("the current weight is :" , weights)
        if inputs_training != []:
            self.inputs = inputs_training
        
        
        for j in range(0 , len(self.inputs)):
              
            try: 
                some.append(weights[j] * self.inputs[j])
            except:
                break
        
        total_sum = torch.stack(some).sum()
        total_sum = total_sum.clone().requires_grad_(True)
        return total_sum, weights, some
    def replace_weights(self, new_weights , training = False):
        if training:
            for j in new_weights:
                if type(j) is torch.tensor:
                    self.weights.append(j.detach().clone().requires_grad_(True))

            # print(len(self.inputs))
            # print(type(self.weights))
        
def create_inputs(n):
    inputs = []
    weight = []
    expected_output = []
    for _ in range(n):
        temp_weights = torch.tensor(random.uniform(0,0.5) , requires_grad=True, device=device)
        weight.append(temp_weights)

        inputs.append(random.uniform(1, 10))
        expected_output.append(random.uniform(0, 5))
    
    
   
    
    return inputs, weight, expected_output

inputs, weights, z = create_inputs(n)
layer_neurons = [[None for _ in range(layers[i])] for i in range(len(layers))]

layer_outputs = [[None for _ in range(layers[i])] for i in range(len(layers))]
layer_unit_outputs = [[None for _ in range(layers[i])] for i in range(len(layers))]
layer_weights = [[[0 for _ in range(0,n)] for i in range(0,max(layers))] for _ in range(0,len(layers))]
for i in range(0,len(layers)):
    for j in range(0 , layers[i]):
        
      
        if i == 0:
            layer_weights[i][j] = weights
            layer_neurons[i][j] = Neuron(weights, inputs)
        else: 
            
            #print(layer_outputs)
            inputs = layer_outputs[i-1]
            #print(inputs)
            weights_1 = []
            for k in range(0,n):
                ran = torch.tensor(random.uniform(0,0.5) , requires_grad=True, device=device)
                weights_1.append(ran)
                
                layer_weights[i][j][k] = ran
            layer_neurons[i][j] = Neuron(weights_1, inputs)
        layer_outputs[i][j] = layer_neurons[i][j].output 
        layer_unit_outputs[i][j] = layer_neurons[i][j].unit_output 

            
      
        
#print(layer_outputs)
error = 0 
for i in range(0 , len(layer_outputs[-1])): 
    
    error += (layer_outputs[-1][i] - z[i]) ** 2
error = error / n

error.backward()
#print(error)

a = 0
i_num = 0
j_num = 0
k_num = 0
new_weights = [[[0 for _ in range(0,n)] for i in range(0,max(layers))] for _ in range(0,len(layers))]


for i in layer_weights:
    
    j_num = 0
    for j in i:
        k_num = 0
        #print("\n\nThis is layer weights : " , j)
        
        for k in j:
            
            if k == 0 :
                continue
            
            # print(len(j))
            # print(k.is_leaf)
            # print(k)
            
            if k_num &gt;= layers[i_num-1]:
                break
            
            var_temp = k - lr*k.grad
            print(var_temp)
            new_weights[i_num][j_num][k_num] = var_temp.detach().clone().requires_grad_(True)
            # print("is_grad")
            a+=1
            k_num+=1
        j_num = j_num+1
    i_num = i_num + 1     


      
# # print(layer_weights[0][0])
# # print(layer_weights[1][0])
# # print(layer_weights[2][0])
# # print(a)
err = [0 for _ in range(0,epoches)]
def train_function(weights , layer_neurons = layer_neurons ,inputs = inputs , z=z, layers=  layers , epoch = 1):
    

    layer_outputs = [[None for _ in range(layers[i])] for i in range(len(layers))]
    layer_unit_outputs = [[None for _ in range(layers[i])] for i in range(len(layers))]
 
    for i in range( 0, len(layers)):
        
        for j in range( 0 , layers[i]):
            #print("\n\nThis is replaced weights : " , weights[i][j])
            
            
            layer_neurons[i][j].replace_weights(weights[i][j] , training=True)
            layer_neurons[i][j].calculate_output(inputs) if i == 0  else layer_neurons[i][j].calculate_output(layer_outputs[i-1])
            
            layer_unit_outputs[i][j] = layer_neurons[i][j].unit_output
            layer_outputs[i][j] = layer_neurons[i][j].output
                
    
      
        
#print(layer_outputs)
    error_training = 0 
    for i in range(0 , len(layer_outputs[-1])): 
        
        error_training += (layer_outputs[-1][i] - z[i]) ** 2
    err[epoch] = error_training / n

    err[epoch].backward()
   
    
    a = 0
    i_num = 0
    j_num = 0
    k_num = 0
    new_weights = [[[0 for _ in range(0,n)] for i in range(0,max(layers))] for _ in range(0,len(layers))]
    for i in layer_weights:
        j_num = 0
        for j in i:
            k_num = 0
            for k in j:
           
                new_weights[i_num][j_num][k_num] = k - lr*k.grad
                a+=1
                k_num+=1
            j_num = j_num+1
        i_num = i_num + 1     
    return err[epoch] , new_weights
#print("\n\n\n new weights is : " , new_weights)
error_test , weights_test = train_function(new_weights)
#print("Error after training:", error_test)
#print("Updated weights:" , weights_test)
    
</code></pre>

username: Dhia-naouali
cooked: <p>hiii</p>
<p>I like how you’re trying to implement the back prop from scratch !</p>
<h1><a name="p-472819-the-thing-is-that-pytorch-doesnt-really-work-that-way-hear-me-out-1" class="anchor" href="#p-472819-the-thing-is-that-pytorch-doesnt-really-work-that-way-hear-me-out-1"></a>the thing is that, pytorch doesn’t really work that way, hear me out</h1>
<p>the basic trainign loop steps are:</p>
<ul>
<li>forward propagation (: as in using the current params, make your prediction)</li>
<li>compute loss</li>
<li>backward propagation (: computing the gradient of each param w.r.t the loss, as in, if nodge each param by a small value, how would that affect the loss)</li>
<li>update (going in the direction that minimizes the loss)</li>
</ul>
<p>to do so, pytorch track all operations on all used tensors (keeping track of intermediate values)<br>
from inputs to loss in a computational graph</p>
<p>using .backward(), iterate the graph in revers (loss to inputs), using the chain rule, computing the gradient of each intermediate value<br>
two things to mention tho, each intermediate tensor have two attributes:</p>
<ul>
<li>resuires_grad: telling pytorch to compute the gradient even if it’s a leaf node in the graph</li>
<li>retain_grad(): tells pytorch to keep that gradient after using it as an intermediate grad during the backward pass</li>
</ul>
<p>a slightly simpler version of your code that does the almost the same thing would look something like:</p>
<pre data-code-wrap="python"><code class="lang-python">import torch


# 3 =&gt; 4 =&gt; 8 =&gt; 1


layer_1_w = torch.randn(4, 3, requires_grad=True)
layer_1_b = torch.randn(4, 1, requires_grad=True)

layer_2_w = torch.randn(8, 4, requires_grad=True)
layer_2_b = torch.randn(8, 1, requires_grad=True)

layer_3_w = torch.randn(1, 8, requires_grad=True)
layer_3_b = torch.randn(1, 1, requires_grad=True)

layer_1_w.retain_grad()
layer_1_b.retain_grad()
layer_2_w.retain_grad()
layer_2_b.retain_grad()
layer_3_w.retain_grad()
layer_3_b.retain_grad()

# Z = W.X + B
# A = activation(Z)

inputs = torch.randn(16, 3) # 16 inputs, each is a vector of shape (3,)
y_true = torch.zeros(16, 1)

# (4, 3).(3, 16) =&gt; (4, 16) + (4, 1) =&gt; (4, 16)
Z_1 = torch.mm(layer_1_w, inputs.T) + layer_1_b
A_1 = torch.sigmoid(Z_1)


Z_2 = torch.mm(layer_2_w, A_1) + layer_2_b
A_2 = torch.sigmoid(Z_2)

Z_3 = torch.mm(layer_3_w, A_2) + layer_3_b
A_3 = torch.sigmoid(Z_3)


error = (y_true - A_3).pow(2).sum()

error.backward()

import IPython; IPython.embed()
</code></pre>
<h1><a name="p-472819-the-error-youre-facing-tho-2" class="anchor" href="#p-472819-the-error-youre-facing-tho-2"></a>the error you’re facing tho</h1>
<p>you’re running a forward pass (pytorch keep track of intermediate values),<br>
then you run backward() the first time (pytorch computes gradients and free intermediate values after using them, to free memory)<br>
then you run .backward() once again (pytorch already freed the intermediate values it would need to perform the backward pass)</p>
<p>the solution to this problem, and to ge accurate grads for your use case, for each set of weights you want to update:</p>
<ul>
<li>run a forward pass to obtain predictions</li>
<li>compute the loss using the obtained predictions</li>
<li>run a backward pass to obtain gradients</li>
<li>update using the obtained gradients</li>
</ul>
<p>doing so would be like:</p>
<pre data-code-wrap="python"><code class="lang-python">
import torch


def create_a_set_of_params():
    # making it produce a static shpae just to keep it as clear as possible
    layer_1_w = torch.randn(4, 3, requires_grad=True)
    layer_1_b = torch.randn(4, 1, requires_grad=True)

    layer_2_w = torch.randn(8, 4, requires_grad=True)
    layer_2_b = torch.randn(8, 1, requires_grad=True)

    layer_3_w = torch.randn(1, 8, requires_grad=True)
    layer_3_b = torch.randn(1, 1, requires_grad=True)

    set_of_params = [
        [layer_1_w, layer_1_b],
        [layer_2_w, layer_2_b],
        [layer_3_w, layer_3_b],
    ]

    return set_of_params


def forward_pass(set_of_params, inputs):
    a = inputs.T
    for layer in set_of_params:
        w, b = layer
        w.retain_grad()
        b.retain_grad()
        z = torch.mm(w, a) + b
        a = torch.sigmoid(z)

    return a


def compute_error(predictions, ground_truth):
    return (predictions - ground_truth).pow(2).sum()


inputs = torch.randn(16, 3) # 16 inputs, each is a vector of shape (3,)
y_true = torch.zeros(16, 1)


set_of_params1 = create_a_set_of_params()

preds1 = forward_pass(set_of_params1, inputs)
error1 = compute_error(preds1, y_true)
error1.backward()


print("using the first set of params")
print(f"gradient of the second layer's weight: {set_of_params1[1][0].grad}")


set_of_params2 = create_a_set_of_params()

preds2 = forward_pass(set_of_params2, inputs)
error2 = compute_error(preds2, y_true)
error2.backward()

print("using the second set of params")
print(f"gradient of the second layer's weight: {set_of_params2[1][0].grad}")

import IPython; IPython.embed()
</code></pre>

username: Mohan_Krishna_Shrest
cooked: <p>Dhia-naouali you are suggesting to use retain_grad() on each tensors or weights right , thats a good fix but I just want to compute loss based on current grad of new weights . I dont want to stretch epoch loss to be computed from prev epoch’s grads.</p>