How to calculate a loss twice with same input and same function

Ynjxsjmh: <p>I want to calculate loss from same input twice but feed the result to different optimizer</p>
<pre><code class="lang-python">def cal_loss(output, target):
    return F.nll_loss(output, target)

def train(args, model, device, train_loader, epoch):
    optimizer1 = optim.Adadelta(model.parameters(), lr=args.lr)
    optimizer2 = optim.Adadelta(model.parameters(), lr=args.lr)

    model.train()
    for batch_idx, (data, target) in enumerate(train_loader):
        data, target = data.to(device), target.to(device)
        output = model(data)

        optimizer1.zero_grad()
        loss1 = cal_loss(output, target)
        loss1.backward()
        optimizer1.step()

        optimizer2.zero_grad()
        loss2 = cal_loss(output, target)
        loss2.backward()
        optimizer2.step()
</code></pre>
<p>But I get the error on <code>loss2.backward()</code></p>
<pre><code class="lang-auto">RuntimeError: Trying to backward through the graph a second time (or directly access saved tensors after they have already been freed). Saved intermediate values of the graph are freed when you call .backward() or autograd.grad(). Specify retain_graph=True if you need to backward through the graph a second time or if you need to access saved tensors after calling backward.
</code></pre>
<p>I tried to use <code>torch.Tensor.clone</code> to clone the input, but the error still occurs.</p>
<pre><code class="lang-python">        loss1 = cal_loss(torch.Tensor.clone(output),
                         torch.Tensor.clone(target))
</code></pre>
<details>
<summary>
Click to view Mnist reproducable example
</summary>
<pre><code class="lang-python">from __future__ import print_function
import argparse
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torch.optim.lr_scheduler import StepLR


class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 32, 3, 1)
        self.conv2 = nn.Conv2d(32, 64, 3, 1)
        self.dropout1 = nn.Dropout(0.25)
        self.dropout2 = nn.Dropout(0.5)
        self.fc1 = nn.Linear(9216, 128)
        self.fc2 = nn.Linear(128, 10)

    def forward(self, x):
        x = self.conv1(x)
        x = F.relu(x)
        x = self.conv2(x)
        x = F.relu(x)
        x = F.max_pool2d(x, 2)
        x = self.dropout1(x)
        x = torch.flatten(x, 1)
        x = self.fc1(x)
        x = F.relu(x)
        x = self.dropout2(x)
        x = self.fc2(x)
        output = F.log_softmax(x, dim=1)
        return output


def cal_loss(output, target):
    return F.nll_loss(output, target)

def train(args, model, device, train_loader, epoch):
    optimizer1 = optim.Adadelta(model.parameters(), lr=args.lr)
    optimizer2 = optim.Adadelta(model.parameters(), lr=args.lr)

    model.train()
    for batch_idx, (data, target) in enumerate(train_loader):
        data, target = data.to(device), target.to(device)
        output = model(data)

        optimizer1.zero_grad()
        loss1 = cal_loss(output, target)
        loss1.backward()
        optimizer1.step()

        optimizer2.zero_grad()
        loss2 = cal_loss(output, target)
        loss2.backward()
        optimizer2.step()

        if batch_idx % args.log_interval == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader), loss.item()))
            if args.dry_run:
                break


def test(model, device, test_loader):
    model.eval()
    test_loss = 0
    correct = 0
    with torch.no_grad():
        for data, target in test_loader:
            data, target = data.to(device), target.to(device)
            output = model(data)
            test_loss += F.nll_loss(output, target, reduction='sum').item()  # sum up batch loss
            pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability
            correct += pred.eq(target.view_as(pred)).sum().item()

    test_loss /= len(test_loader.dataset)

    print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
        test_loss, correct, len(test_loader.dataset),
        100. * correct / len(test_loader.dataset)))


def main():
    # Training settings
    parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
    parser.add_argument('--batch-size', type=int, default=64, metavar='N',
                        help='input batch size for training (default: 64)')
    parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N',
                        help='input batch size for testing (default: 1000)')
    parser.add_argument('--epochs', type=int, default=14, metavar='N',
                        help='number of epochs to train (default: 14)')
    parser.add_argument('--lr', type=float, default=1.0, metavar='LR',
                        help='learning rate (default: 1.0)')
    parser.add_argument('--gamma', type=float, default=0.7, metavar='M',
                        help='Learning rate step gamma (default: 0.7)')
    parser.add_argument('--no-cuda', action='store_true', default=False,
                        help='disables CUDA training')
    parser.add_argument('--dry-run', action='store_true', default=False,
                        help='quickly check a single pass')
    parser.add_argument('--seed', type=int, default=1, metavar='S',
                        help='random seed (default: 1)')
    parser.add_argument('--log-interval', type=int, default=10, metavar='N',
                        help='how many batches to wait before logging training status')
    parser.add_argument('--save-model', action='store_true', default=False,
                        help='For Saving the current Model')
    args = parser.parse_args()
    use_cuda = not args.no_cuda and torch.cuda.is_available()

    torch.manual_seed(args.seed)

    device = torch.device("cuda" if use_cuda else "cpu")

    train_kwargs = {'batch_size': args.batch_size}
    test_kwargs = {'batch_size': args.test_batch_size}
    if use_cuda:
        cuda_kwargs = {'num_workers': 1,
                       'pin_memory': True,
                       'shuffle': True}
        train_kwargs.update(cuda_kwargs)
        test_kwargs.update(cuda_kwargs)

    transform=transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.1307,), (0.3081,))
        ])
    dataset1 = datasets.MNIST('./data', train=True, download=True,
                              transform=transform)
    dataset2 = datasets.MNIST('./data', train=False,
                              transform=transform)
    train_loader = torch.utils.data.DataLoader(dataset1,**train_kwargs)
    test_loader = torch.utils.data.DataLoader(dataset2, **test_kwargs)

    model = Net().to(device)

    #scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma)
    for epoch in range(1, args.epochs + 1):
        train(args, model, device, train_loader, epoch)
        test(model, device, test_loader)
        #scheduler.step()

    if args.save_model:
        torch.save(model.state_dict(), "mnist_cnn.pt")


if __name__ == '__main__':
    main()
</code></pre>
</details>

KFrank: <p>Hi Ynj!</p>
<aside class="quote no-group" data-username="Ynjxsjmh" data-post="1" data-topic="152237" data-full="true">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/letter_avatar_proxy/v4/letter/y/7c8e57/48.png" class="avatar"> Ynjxsjmh:</div>
<blockquote>
<pre data-code-wrap="python"><code class="lang-python">        optimizer1.zero_grad()
        loss1 = cal_loss(output, target)
        loss1.backward()
        optimizer1.step()

        optimizer2.zero_grad()
        loss2 = cal_loss(output, target)
        loss2.backward()
        optimizer2.step()
</code></pre>
</blockquote>
</aside>
<p>If I understand your use case, this should be as simple as:</p>
<pre data-code-wrap="python"><code class="lang-python">        optimizer1.zero_grad()
        loss1 = cal_loss(output, target)
        loss1.backward()
        # save gradients
        optimizer1.step()
        # restore gradients
        optimizer2.step()
</code></pre>
<p><code>loss1</code> and <code>loss2</code> are the same, so calculating <code>loss2</code> and calling<br>
<code>loss2.backward()</code> (after zeroing the gradients) just repopulates the<br>
gradients with what they were before.</p>
<p>Note, pytorch optimizers with momentum and weight decay implement<br>
these features by modifying the gradients before performing the<br>
parameter-update step.  This is why we “restore gradients” before<br>
calling <code>optimizer2.step()</code>.</p>
<p>(I don’t know the specifics of <code>Adadelta</code>, so maybe it doesn’t modify the<br>
gradients, but it might well, and if so, restoring the gradients would be<br>
necessary.)</p>
<p>Best.</p>
<p>K. Frank</p>

Ynjxsjmh: <p>Thanks for your reply, Frank! This would work in this case but I have simplified the problem, the real case I’m facing is that the first loss is a sum of <code>cal_loss</code> with the other loss and the second loss is just the <code>cal_loss</code>, so I think it couldn’t be simplified to do one <code>backward</code>. Also the parameters passed into <code>optimizer</code> are different.</p>
<pre><code class="lang-python">def cal_loss(output, target):
    return F.nll_loss(output, target)

def train(args, model, device, train_loader, epoch):
    optimizer1 = optim.Adadelta(model.parameters(), lr=args.lr)
    optimizer2 = optim.Adadelta(model.parameters(), lr=args.lr)

    model.train()
    for batch_idx, (data, target) in enumerate(train_loader):
        data, target = data.to(device), target.to(device)
        output = model(data)

        optimizer1.zero_grad()
        loss1 = cal_loss(output, target)
        loss2 = cal_loss2() # different loss calculation function
        loss3 = cal_loss3() # another different loss calculation function
        loss = loss1 + loss2 + loss3
        loss.backward()
        optimizer1.step()

        optimizer2.zero_grad()
        loss4 = cal_loss(output, target)
        loss4.backward()
        optimizer2.step()
</code></pre>
<p>What confuses me is why the error occurs when I <code>copy</code> the object with <code>torch.Tensor.clone</code>.</p>

Ynjxsjmh: <p>I try to fix it by adding <code>retain_graph=True</code> to the first backward and move <code>optimizer1.step()</code> after the second backward.</p>
<pre><code class="lang-python">    for batch_idx, (data, target) in enumerate(train_loader):
        data, target = data.to(device), target.to(device)
        output = model(data)

        optimizer1.zero_grad()
        loss1 = cal_loss(output,
                         target)
        loss1.backward(retain_graph=True)

        optimizer2.zero_grad()
        loss2 = cal_loss(output,
                          target)
        loss2.backward()

        optimizer1.step()
        optimizer2.step()
</code></pre>
<p>But I’m not sure if it is suitable since the input for <code>cal_loss</code> is the same and if <code>torch.Tensor.clone</code> is needed.</p>

KFrank: <p>Hi Ynj!</p>
<aside class="quote no-group" data-username="Ynjxsjmh" data-post="4" data-topic="152237" data-full="true">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/letter_avatar_proxy/v4/letter/y/7c8e57/48.png" class="avatar"> Ynjxsjmh:</div>
<blockquote>
<p>I try to fix it by adding <code>retain_graph=True</code> to the first backward</p>
</blockquote>
</aside>
<p>Using <code>retain_graph = True</code> should work for your use case by letting<br>
you backpropagate through the <code>output = model (data)</code> part of the<br>
graph twice as you backpropagate your two losses.</p>
<aside class="quote no-group">
<blockquote>
<p>and move <code>optimizer1.step()</code> after the second backward.</p>
</blockquote>
</aside>
<p>I would not recommend this.  There is no point in doing so, and if your<br>
two optimizers share any parameters (and modify the gradients, as I<br>
assume <code>Adadelta</code> does), the call to <code>optimizer1.step()</code> will corrupt<br>
the gradients used by <code>optimizer2.step()</code>.  Move <code>optimizer1.step()</code><br>
to just after <code>loss1.backward(retain_graph=True)</code> and before<br>
<code>optimizer2.zero_grad()</code>.</p>
<aside class="quote no-group">
<blockquote>
<p>But I’m not sure if …   <code>torch.Tensor.clone</code> is needed.</p>
</blockquote>
</aside>
<p>There is no need for <code>torch.Tensor.clone()</code></p>
<aside class="quote no-group">
<blockquote>
<pre data-code-wrap="python"><code class="lang-python">        optimizer1.zero_grad()
        loss1 = cal_loss(output,
                         target)
        loss1.backward(retain_graph=True)

        optimizer2.zero_grad()
        loss2 = cal_loss(output,
                          target)
        loss2.backward()
</code></pre>
</blockquote>
</aside>
<p>As an aside, it’s really unhelpful that you’re not posting your actual code.<br>
Here, <code>loss1</code> and <code>loss2</code> are the same, but in your previous post you<br>
emphasized that they are actually different (as is the case in the code in<br>
your previous post).</p>
<p>And then in your previous post you said</p>
<aside class="quote no-group">
<blockquote>
<p>Also the parameters passed into <code>optimizer</code> are different.</p>
</blockquote>
</aside>
<p>but posted this code:</p>
<aside class="quote no-group">
<blockquote>
<pre data-code-wrap="python"><code class="lang-python">    optimizer1 = optim.Adadelta(model.parameters(), lr=args.lr)
    optimizer2 = optim.Adadelta(model.parameters(), lr=args.lr)
</code></pre>
</blockquote>
</aside>
<p>in which the same parameters are passed into the two <code>Optimizer</code>s.</p>
<p>If you randomly modify your code before posting it, you are unlikely to<br>
get useful answers to your questions.</p>
<p>The <em>best</em> way to post code is to post fully-self-contained, runnable code,<br>
together with the actual output you get from running that code, followed<br>
by an explanation of why that output isn’t what you wanted or expected.</p>
<p>If you have to simply or trim down your code to make it fully self-contained,<br>
that’s actually a good thing, as doing so is the first step in your debugging<br>
process.</p>
<p>Best.</p>
<p>K. Frank</p>

Ynjxsjmh: <p>Thanks for your time and detailed explanation and sorry for not bring you good answering experience.</p>
<blockquote>
<p>Move <code>optimizer1.step()</code> to just after <code>loss1.backward(retain_graph=True)</code> and before <code>optimizer2.zero_grad()</code> .</p>
</blockquote>
<p>If doing so, I will get <code>modified by an inplace</code> error.</p>
<pre><code class="lang-auto">RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [64, 32, 3, 3]] is at version 2; expected version 1 instead. Hint: enable anomaly detection to find the operation that failed to compute its gradient, with torch.autograd.set_detect_anomaly(True).
</code></pre>
<blockquote>
<p>As an aside, it’s really unhelpful that you’re not posting your actual code.</p>
</blockquote>
<p>I’m sorry this causes inconvenience for you. If I post the actual code, it is not executable and quite long to read. So I decided to extract the key problem and create a minimal Mnist example and attach the reproducible code below the post.</p>
<hr>
<aside class="quote no-group">
<blockquote>
<aside class="quote no-group">
<blockquote>
<p>And then in your previous post you said</p>
<aside class="quote no-group">
<blockquote>
<p>Also the parameters passed into <code>optimizer</code> are different.</p>
</blockquote>
</aside>
<p>but posted this code:</p>
<pre><code class="lang-auto">optimizer1 = optim.Adadelta(model.parameters(), lr=args.lr)
optimizer2 = optim.Adadelta(model.parameters(), lr=args.lr)
</code></pre>
</blockquote>
</aside>
<p>in which the same parameters are passed into the two <code>Optimizer</code>s.</p>
</blockquote>
</aside>
<p>It’s my bad, I didn’t notice that optimizer parameters matter much in this case, so is the optimizer. Util you point out that <em>if your two optimizers share any parameters (and modify the gradients, as I assume <code>Adadelta</code> does), the call to <code>optimizer1.step()</code> will corrupt the gradients used by <code>optimizer2.step()</code></em>.</p>
<aside class="quote no-group">
<blockquote>
<p>If you randomly modify your code before posting it, you are unlikely to<br>
get useful answers to your questions.</p>
<p>The best way to post code is to post fully-self-contained, runnable code,<br>
together with the actual output you get from running that code, followed<br>
by an explanation of why that output isn’t what you wanted or expected.</p>
</blockquote>
</aside>
<p>You’re right, it’s my bad trimming my original code too much in this example[1]. And the simplicity seems to lead to a X-Y problem. Finally it turns out that the problem is not I want to calculate the same loss twice but the sequence of calling <code>optimizer.step()</code> and the twice <code>loss.backward()</code>.</p>
<p>[1] I don’t know except <code>F.nll_loss</code>, which loss is suitable for Mnist example. So I only create the same loss but not create other loss methods.</p>
<p>Anyway, if anyone is interested, in the following code I define the original try into <code>bad_train</code> and the runnable fixed one into <code>train</code>.</p>
<details>
<summary>
Click to view code
</summary>
<pre data-code-wrap="python"><code class="lang-python">from __future__ import print_function
import argparse
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torch.optim.lr_scheduler import StepLR


class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 32, 3, 1)
        self.conv2 = nn.Conv2d(32, 64, 3, 1)
        self.dropout1 = nn.Dropout(0.25)
        self.dropout2 = nn.Dropout(0.5)
        self.fc1 = nn.Linear(9216, 128)
        self.fc2 = nn.Linear(128, 10)

    def forward(self, x):
        x = self.conv1(x)
        x = F.relu(x)
        x = self.conv2(x)
        x = F.relu(x)
        x = F.max_pool2d(x, 2)
        x = self.dropout1(x)
        x = torch.flatten(x, 1)
        x = self.fc1(x)
        x = F.relu(x)
        x = self.dropout2(x)
        x = self.fc2(x)
        output = F.log_softmax(x, dim=1)
        return output


def cal_loss(output, target):
    return F.nll_loss(output, target)

def bad_train(args, model, device, train_loader, epoch):
    optimizer1 = optim.Adam(list(model.parameters())[:4], lr=args.lr)
    optimizer2 = optim.Adam(list(model.parameters())[4:], lr=args.lr)

    model.train()
    for batch_idx, (data, target) in enumerate(train_loader):
        data, target = data.to(device), target.to(device)
        output = model(data)

        optimizer1.zero_grad()
        loss1 = cal_loss(output,
                         target)
        loss1.backward()
        optimizer1.step()

        optimizer2.zero_grad()
        loss2 = cal_loss(output,
                         target)
        loss2.backward()
        optimizer2.step()

        if batch_idx % args.log_interval == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss1: {:.6f}\tLoss2: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader), loss1.item(), loss2.item()))
            if args.dry_run:
                break

def train(args, model, device, train_loader, epoch):
    optimizer1 = optim.Adam(list(model.parameters())[:4], lr=args.lr)
    optimizer2 = optim.Adam(list(model.parameters())[4:], lr=args.lr)

    model.train()
    for batch_idx, (data, target) in enumerate(train_loader):
        data, target = data.to(device), target.to(device)
        output = model(data)

        optimizer1.zero_grad()
        loss1 = cal_loss(output,
                         target)
        loss1.backward(retain_graph=True)

        optimizer2.zero_grad()
        loss2 = cal_loss(output,
                         target)
        loss2.backward()

        optimizer1.step()
        optimizer2.step()

        if batch_idx % args.log_interval == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss1: {:.6f}\tLoss2: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader), loss1.item(), loss2.item()))
            if args.dry_run:
                break

def test(model, device, test_loader):
    model.eval()
    test_loss = 0
    correct = 0
    with torch.no_grad():
        for data, target in test_loader:
            data, target = data.to(device), target.to(device)
            output = model(data)
            test_loss += F.nll_loss(output, target, reduction='sum').item()  # sum up batch loss
            pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability
            correct += pred.eq(target.view_as(pred)).sum().item()

    test_loss /= len(test_loader.dataset)

    print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
        test_loss, correct, len(test_loader.dataset),
        100. * correct / len(test_loader.dataset)))


def main():
    # Training settings
    parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
    parser.add_argument('--batch-size', type=int, default=64, metavar='N',
                        help='input batch size for training (default: 64)')
    parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N',
                        help='input batch size for testing (default: 1000)')
    parser.add_argument('--epochs', type=int, default=14, metavar='N',
                        help='number of epochs to train (default: 14)')
    parser.add_argument('--lr', type=float, default=1.0, metavar='LR',
                        help='learning rate (default: 1.0)')
    parser.add_argument('--gamma', type=float, default=0.7, metavar='M',
                        help='Learning rate step gamma (default: 0.7)')
    parser.add_argument('--no-cuda', action='store_true', default=False,
                        help='disables CUDA training')
    parser.add_argument('--dry-run', action='store_true', default=False,
                        help='quickly check a single pass')
    parser.add_argument('--seed', type=int, default=1, metavar='S',
                        help='random seed (default: 1)')
    parser.add_argument('--log-interval', type=int, default=10, metavar='N',
                        help='how many batches to wait before logging training status')
    parser.add_argument('--save-model', action='store_true', default=False,
                        help='For Saving the current Model')
    args = parser.parse_args()
    use_cuda = not args.no_cuda and torch.cuda.is_available()

    torch.manual_seed(args.seed)

    device = torch.device("cuda" if use_cuda else "cpu")

    train_kwargs = {'batch_size': args.batch_size}
    test_kwargs = {'batch_size': args.test_batch_size}
    if use_cuda:
        cuda_kwargs = {'num_workers': 1,
                       'pin_memory': True,
                       'shuffle': True}
        train_kwargs.update(cuda_kwargs)
        test_kwargs.update(cuda_kwargs)

    transform=transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.1307,), (0.3081,))
        ])
    dataset1 = datasets.MNIST('./data', train=True, download=True,
                              transform=transform)
    dataset2 = datasets.MNIST('./data', train=False,
                              transform=transform)
    train_loader = torch.utils.data.DataLoader(dataset1,**train_kwargs)
    test_loader = torch.utils.data.DataLoader(dataset2, **test_kwargs)

    model = Net().to(device)

    #scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma)
    for epoch in range(1, args.epochs + 1):
        train(args, model, device, train_loader, epoch)
        test(model, device, test_loader)
        #scheduler.step()

    if args.save_model:
        torch.save(model.state_dict(), "mnist_cnn.pt")


if __name__ == '__main__':
    main()
</code></pre>
</details>