Computing Gradients with respect to intermediate activations

teja5832: <p>Hi,</p>
<p>I am looking to do the following in C++:</p>
<pre><code class="lang-auto">auto x = torch::zeros({5}, torch::requires_grad());
auto out_1 = torch::relu( W_1.matmul(x) + b_1);
auto out_2 = torch::relu( W_1.matmul(out_2) + b_2); 
auto logits = torch::log_softmax(out_2, 1);
auto loss = logits[0] - logits[1];
</code></pre>
<p>I want to find the values of d(loss)/d(out_1) and  d(loss)/d(out_2).</p>
<p>How do I do this? Attempts to do</p>
<pre><code class="lang-auto">loss.backward();
</code></pre>
<p>and then accessing</p>
<pre><code class="lang-auto">out_2.grad()
</code></pre>
<p>have seemingly failed: there is a runtime error which says <code>libc++abi.dylib: terminating with uncaught exception of type c10::Error: tensor with backend UNKNOWN_BACKEND does not have a device</code></p>
<p>I can provide a minimal breaking example if need be.</p>

Geeks_Sid: <p>Make sure to push your tensors on a particular device!</p>

teja5832: <p>I think I have done that.<br>
Here’s my code:</p>
<pre><code class="lang-auto">#include &lt;iostream&gt;
#include&lt;vector&gt;
#include &lt;string&gt;
#include&lt;cassert&gt;

#include&lt;torch/torch.h&gt;

#include "TorchNeuralNet.hpp"

// std::pair&lt; std::vector&lt;torch::Tensor&gt;, std::vector&lt;torch::Tensor&gt; &gt;
void getActsAndGrads(TorchNeuralNetwork&amp; net, torch::Tensor&amp; x, torch::Device device) //, torch::nn::Functional&amp; L) 
{
    assert(x.numel() == net.getLayerInputSize(0));
    auto out = x;
    std::cout &lt;&lt; "output_init" &lt;&lt; std::endl;
    std::cout &lt;&lt; out &lt;&lt; std::endl;
    std::cout &lt;&lt; out.options() &lt;&lt; std::endl;
    std::cout &lt;&lt; out.sizes() &lt;&lt; std::endl;

    std::vector&lt;torch::Tensor&gt; activations;
    std::vector&lt;torch::Tensor&gt; gradients;
    // std::vector&lt;torch::Tensor&gt; act_gradients;

    //outputs
    for(unsigned i=0; i &lt; net.getNumLayers(); i++) {

        std::cout &lt;&lt; i &lt;&lt; std::endl;

        auto w = net.getWeightTensor(i, device);
        auto b = net.getBiasTensor(i, device);

        // std::cout &lt;&lt; "w" &lt;&lt; i &lt;&lt; std::endl;
        // std::cout &lt;&lt; w &lt;&lt; std::endl;
        // std::cout &lt;&lt; w.options() &lt;&lt; std::endl;

        // std::cout &lt;&lt; "b" &lt;&lt; i &lt;&lt; std::endl;
        // std::cout &lt;&lt; b &lt;&lt; std::endl;
        // std::cout &lt;&lt; b.options() &lt;&lt; std::endl;

        out = w.matmul(out) + b;
        out = torch::relu(out);
        activations.push_back(out);

        assert(out.dtype() == torch::kFloat64);
        assert(out.layout() == torch::kStrided);
        assert(out.device().type() == torch::kCPU);
        assert(out.requires_grad());
        std::cout &lt;&lt; "out_" &lt;&lt; i &lt;&lt; " asserts success!" &lt;&lt; std::endl;

        // std::cout &lt;&lt; "out" &lt;&lt; i &lt;&lt; std::endl;
        // std::cout &lt;&lt; out &lt;&lt; std::endl;
        // std::cout &lt;&lt; out.options() &lt;&lt; std::endl;
        // std::cout &lt;&lt; out.sizes() &lt;&lt; std::endl &lt;&lt; std::endl;
        
        // std::cout &lt;&lt; "\n\n\n";
    }

    out = out[0];
    std::cout &lt;&lt; "out_final" &lt;&lt; std::endl;
    std::cout &lt;&lt; out &lt;&lt; std::endl;
    std::cout &lt;&lt; out.options() &lt;&lt; std::endl;
    std::cout &lt;&lt; out.sizes() &lt;&lt; std::endl &lt;&lt; std::endl;

    assert(out.dtype() == torch::kFloat64);
    assert(out.layout() == torch::kStrided);
    assert(out.device().type() == torch::kCPU);
    assert(out.requires_grad());
    std::cout &lt;&lt; "out_final asserts success!" &lt;&lt; std::endl;

    int k = 0;
    out.backward();
    for(auto&amp; o: activations) {
        std::cout &lt;&lt; "act_grad_" &lt;&lt; k &lt;&lt; std::endl;
        auto g = o.grad();
        gradients.push_back(g);
        std::cout &lt;&lt; g &lt;&lt; std::endl;
        std::cout &lt;&lt; g.options() &lt;&lt; std::endl;
        std::cout &lt;&lt; g.sizes() &lt;&lt; std::endl &lt;&lt; std::endl;
        std::cout &lt;&lt; "\n\n\n";
        k++;
    }
}

int main()
{
    const std::string FILE_NAME = "../nnet/ACASXU_run2a_1_1_batch_2000.nnet";
    TorchNeuralNetwork net(FILE_NAME);

    torch::DeviceType device_type;
    if (torch::cuda::is_available()) {
        std::cout &lt;&lt; "CUDA available! Training on GPU." &lt;&lt; std::endl;
        device_type = torch::kCUDA;
    } else {
        std::cout &lt;&lt; "Training on CPU." &lt;&lt; std::endl;
        device_type = torch::kCPU;
    }
    torch::Device device(device_type);

    auto options = torch::TensorOptions()
                    .dtype(torch::kFloat64)
                    .layout(torch::kStrided)
                    .device(device)
                    .requires_grad(true);

    auto x = torch::zeros({5}, options);

    assert(x.dtype() == torch::kFloat64);
    assert(x.layout() == torch::kStrided);
    assert(x.device().type() == torch::kCPU);
    assert(x.requires_grad());
    std::cout &lt;&lt; "x asserts success!" &lt;&lt; std::endl;


    std::cout &lt;&lt; "x" &lt;&lt; std::endl;
    std::cout &lt;&lt; x &lt;&lt; std::endl;
    std::cout &lt;&lt; x.options() &lt;&lt; std::endl;
    std::cout &lt;&lt; x.sizes() &lt;&lt; std::endl;
    // auto grads = 
    getActsAndGrads(net, x, device);
    return 0;
}

</code></pre>
<p>All asserts are succesful; even the <code>gradientss.push_back(g)</code> is(I have verified this by separating the loops). Here’s the output:</p>
<pre><code class="lang-auto">Training on CPU.
x asserts success!
x
 0
 0
 0
 0
 0
[ Variable[CPUDoubleType]{5} ]
TensorOptions(dtype=double, device=cpu, layout=Strided, requires_grad=false)
[5]
output_init
 0
 0
 0
 0
 0
[ Variable[CPUDoubleType]{5} ]
TensorOptions(dtype=double, device=cpu, layout=Strided, requires_grad=false)
[5]
0
out_0 asserts success!
1
out_1 asserts success!
2
out_2 asserts success!
3
out_3 asserts success!
4
out_4 asserts success!
5
out_5 asserts success!
6
out_6 asserts success!
out_final
0
[ Variable[CPUDoubleType]{} ]
TensorOptions(dtype=double, device=cpu, layout=Strided, requires_grad=false)
[]

out_final asserts success!
act_grad_0
[ Tensor (undefined) ]
terminate called after throwing an instance of 'c10::Error'
  what():  tensor with backend UNKNOWN_BACKEND does not have a device (device at /nobackup/libtorch/include/c10/core/TensorImpl.h:442)
frame #0: std::function&lt;std::string ()&gt;::operator()() const + 0x11 (0x7f56da498b91 in /nobackup/libtorch/lib/libc10.so)
frame #1: c10::Error::Error(c10::SourceLocation, std::string const&amp;) + 0x2a (0x7f56da497d1a in /nobackup/libtorch/lib/libc10.so)
frame #2: c10::TensorImpl::device() const + 0x100 (0x559e4e318974 in ./int_grad)
frame #3: at::Tensor::device() const + 0x20 (0x559e4e319548 in ./int_grad)
frame #4: at::Tensor::options() const + 0x7d (0x559e4e31912b in ./int_grad)
frame #5: getActsAndGrads(TorchNeuralNetwork&amp;, at::Tensor&amp;, c10::Device) + 0x7a3 (0x559e4e3171ca in ./int_grad)
frame #6: main + 0x3d7 (0x559e4e317747 in ./int_grad)
frame #7: __libc_start_main + 0xe7 (0x7f56ce780b97 in /lib/x86_64-linux-gnu/libc.so.6)
frame #8: _start + 0x2a (0x559e4e31434a in ./int_grad)
</code></pre>

teja5832: <p><a class="mention" href="/u/cndn">@cndn</a> can you help? I see that <a href="https://discuss.pytorch.org/t/is-retain-grad-supported-in-new-c-api/24947/4" class="inline-onebox">Is retain_grad() supported in new C++ API?</a> is kinda close to my issue. Thanks a lot!</p>
<p>I have to accumulate these gradients in a list data structure, which I want to pass into the constructor of the hook:</p>
<pre><code class="lang-auto">struct GradSaveHook : public torch::autograd::FunctionPreHook {
  GradSaveHook(std::vector&lt;std::vector&lt;torch::autograd::Variable&gt; &gt;&amp; g, torch::autograd::Variable* x) : v(x), gradients(g) {};
  torch::autograd::variable_list operator()(const torch::autograd::variable_list&amp; grads) {
    std::cout &lt;&lt; "grads" &lt;&lt; grads &lt;&lt; "\n" &lt;&lt; std::endl;
    gradients.push_back(grads);
    return grads;
  }
  torch::autograd::Variable* v;
  std::vector&lt;std::vector&lt;torch::autograd::Variable&gt; &gt; gradients;
};
</code></pre>
<p>I then run the following loop:</p>
<pre><code class="lang-auto">for(unsigned i=0; i &lt; net.getNumLayers(); i++) {
        auto w = net.getWeightTensor(i, device);
        auto b = net.getBiasTensor(i, device);

        out = w.matmul(out) + b;
        out = torch::relu(out);
        activations.push_back(out);

        auto hook_ptr = std::make_shared&lt;GradSaveHook&gt;(new GradSaveHook(gradients, &amp;out));
        out.add_hook(hook_ptr);
    }
</code></pre>
<p>Then, I define the loss:</p>
<pre><code class="lang-auto">auto loss = out[0];
loss.backward();
</code></pre>

yf225: <p><a class="mention" href="/u/teja5832">@teja5832</a> Do you have an equivalent Python code that you successfully ran? Looking at the C++ code you shared there seems to be some issues (e.g. <code>out.backward</code> should be called with gradients). It would be best if we work out the correct logic in Python first, and then translate it to C++.</p>