Change the gradient during backward pass

username: cbd
cooked: <p>Below is the output and its code.  I want to change the gradient during the backward pass. For example out1.grad[0][0] is -22 which i want to modify it to 50 manually.</p>
<pre><code class="lang-auto">output and loss is  tensor([8., 8.], grad_fn=&lt;MulBackward0&gt;) tensor(386.5000, grad_fn=&lt;MseLossBackward&gt;)
tensor([-22., -17.])
output and loss is  tensor([289.6000, 225.6000], grad_fn=&lt;MulBackward0&gt;) tensor(53816.2656, grad_fn=&lt;MseLossBackward&gt;)
tensor([9397.5205, 5656.9204])
</code></pre>
<pre><code class="lang-auto">import torch.nn as nn
class Net(torch.nn.Module):
    def __init__(self):       
        super(Net, self).__init__()
        self.w=torch.nn.Parameter(torch.ones(2))
    def forward(self, x):
        out1= 2 * x
        out1.register_hook(lambda grad: print(grad))
        out = torch.mul(out1,self.w)
        return out

   
learning_rate = 0.2
model=Net()
criterion=torch.nn.MSELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

x=4*torch.ones((2),requires_grad=True)
target=torch.FloatTensor([30,25])

for i in range (0,2):
    optimizer.zero_grad()
    out=model(x)
    loss= criterion(target,out)
    print("output and loss is ",out,loss)
    loss.backward(retain_graph=True)
    optimizer.step()  ```</code></pre>

username: albanD
cooked: <p>Hi,</p>
<p>Your hook can return a new Tensor that will then be used instead of the given Tensor.<br>
Note that hooks should <em>never</em> modify their input inplace. So you can do:</p>
<pre><code class="lang-auto">def my_hook(grad):
    grad = grad.clone()
    grad[0][0] = 50
    print("new grad: ", grad)
    return grad
</code></pre>
<p>And register that as a hook for out1</p>

username: cbd
cooked: <p>I am calling it from forward as mentioned in code but it gives error “my_hook” is not defined. I make changes mentioned in code 2 but in that also it gives the same error.</p>
<pre><code class="lang-auto">import torch.nn as nn
class Net(torch.nn.Module):

    def __init__(self):       
        super(Net, self).__init__()
        self.w=torch.nn.Parameter(torch.ones(2))
       
    def my_hook(grad):
      grad = grad.clone()
      grad[0][0] = 50
      print("new grad: ", grad)
      return grad

    def forward(self, x):
        out1= 2 * x
        out1.register_hook(my_hook)
        out = torch.mul(out1,self.w)
        return out

   
learning_rate = 0.2
model=Net()
criterion=torch.nn.MSELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

x=4*torch.ones((2),requires_grad=True)
target=torch.FloatTensor([30,25])

for i in range (0,2):
    optimizer.zero_grad()
    out=model(x)
    loss= criterion(target,out)
    print("output and loss is ",out,loss)
    loss.backward(retain_graph=True)
    optimizer.step()
</code></pre>
<p>Code:2</p>
<pre><code class="lang-auto">import torch

import torch.nn as nn

class Net(torch.nn.Module):

    def __init__(self):       

        super(Net, self).__init__()

        self.w=torch.nn.Parameter(torch.ones(2))

        self.out1=torch.ones(2)

       

    def my_hook(self,grad):

      grad = grad.clone()

      grad[0][0] = 50

      print("new grad: ", grad)

      return grad

    def forward(self, x):

        self.out1= 2 * x

        # out1.register_hook(my_hook)

        out = torch.mul(self.out1,self.w)

        return out

   

learning_rate = 0.2

model=Net()

criterion=torch.nn.MSELoss()

optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

x=4*torch.ones((2),requires_grad=True)

target=torch.FloatTensor([30,25])

for i in range (0,2):

    optimizer.zero_grad()

    out=model(x)

    loss= criterion(target,out)

    print("output and loss is ",out,loss)

    loss.backward(retain_graph=True)

    model.out1.register_hook(my_hook)

    optimizer.step()</code></pre>

username: albanD
cooked: <p>Defining <code>my_hook</code> on the class will make it a class method. And so you need to specify it as <code>self.my_hook</code>.<br>
Otherwise you can put it inline in the function:</p>
<pre><code class="lang-python">import torch
import torch.nn as nn
class Net(torch.nn.Module):

    def __init__(self):       
        super(Net, self).__init__()
        self.w=torch.nn.Parameter(torch.ones(2))
       

    def forward(self, x):
        def my_hook(grad):
            grad = grad.clone()
            grad[0] = 50
            print("new grad: ", grad)
            return grad

        out1= 2 * x
        out1.register_hook(my_hook)
        out = torch.mul(out1,self.w)
        return out

   
learning_rate = 0.2
model=Net()
criterion=torch.nn.MSELoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

model(torch.rand(2, requires_grad=True)).sum().backward()
</code></pre>