Error-Pytorch-BiGAn

hajar: <p>Hi, I implement the BiGAN code and I face this error:<br>
RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [1, 1024, 1, 1]] is at version 2; expected version 1 instead. Hint: the backtrace further above shows the operation that failed to compute its gradient. The variable in question was changed in there or anywhere later. Good luck!</p>
<p>Here is the code:<br>
import torch<br>
import torch.nn as nn<br>
import torch.nn.functional as F<br>
class Discriminator(nn.Module):<br>
def <strong>init</strong>(self, z_dim=32, wasserstein=False):<br>
super(Discriminator, self).<strong>init</strong>()<br>
self.wass = wasserstein</p>
<pre><code>    # Inference over x
    self.conv1x = nn.Conv2d(3, 32, 5, stride=1, bias=False)
    self.conv2x = nn.Conv2d(32, 64, 4, stride=2, bias=False)
    self.bn2x = nn.BatchNorm2d(64)
    self.conv3x = nn.Conv2d(64, 128, 4, stride=1, bias=False)
    self.bn3x = nn.BatchNorm2d(128)
    self.conv4x = nn.Conv2d(128, 256, 4, stride=2, bias=False)
    self.bn4x = nn.BatchNorm2d(256)
    self.conv5x = nn.Conv2d(256, 512, 4, stride=1, bias=False)
    self.bn5x = nn.BatchNorm2d(512)

    # Inference over z
    self.conv1z = nn.Conv2d(z_dim, 512, 1, stride=1, bias=False)
    self.conv2z = nn.Conv2d(512, 512, 1, stride=1, bias=False)

    # Joint inference
    self.conv1xz = nn.Conv2d(1024, 1024, 1, stride=1, bias=False)
    self.conv2xz = nn.Conv2d(1024, 1024, 1, stride=1, bias=False)
    self.conv3xz = nn.Conv2d(1024, 1, 1, stride=1, bias=False)

def inf_x(self, x):
    x = F.dropout2d(F.leaky_relu(self.conv1x(x), negative_slope=0.1), 0.2, inplace=False)
    x = F.dropout2d(F.leaky_relu(self.bn2x(self.conv2x(x)), negative_slope=0.1), 0.2,inplace=False)
    x = F.dropout2d(F.leaky_relu(self.bn3x(self.conv3x(x)), negative_slope=0.1), 0.2,inplace=False)
    x = F.dropout2d(F.leaky_relu(self.bn4x(self.conv4x(x)), negative_slope=0.1), 0.2,inplace=False)
    x = F.dropout2d(F.leaky_relu(self.bn5x(self.conv5x(x)), negative_slope=0.1), 0.2,inplace=False)
    return x

def inf_z(self, z):
    z = F.dropout2d(F.leaky_relu(self.conv1z(z), negative_slope=0.1), 0.2, inplace=False)
    z = F.dropout2d(F.leaky_relu(self.conv2z(z), negative_slope=0.1), 0.2,inplace=False)
    return z

def inf_xz(self, xz):
    xz = F.dropout(F.leaky_relu(self.conv1xz(xz), negative_slope=0.1), 0.2,inplace=False)
    xz = F.dropout(F.leaky_relu(self.conv2xz(xz), negative_slope=0.1), 0.2,inplace=False)
    return self.conv3xz(xz)

def forward(self, x, z):
    x = self.inf_x(x)
    z = self.inf_z(z)
    xz = torch.cat((x,z), dim=1)
    out = self.inf_xz(xz)
    if self.wass:
        return out
    else:
        return torch.sigmoid(out)
</code></pre>
<p>class Generator(nn.Module):<br>
def <strong>init</strong>(self, z_dim=32):<br>
super(Generator, self).<strong>init</strong>()<br>
self.z_dim = z_dim</p>
<pre><code>    self.output_bias = nn.Parameter(torch.zeros(3, 32, 32), requires_grad=True)
    self.deconv1 = nn.ConvTranspose2d(z_dim, 256, 4, stride=1, bias=False)
    self.bn1 = nn.BatchNorm2d(256)
    self.deconv2 = nn.ConvTranspose2d(256, 128, 4, stride=2, bias=False)
    self.bn2 = nn.BatchNorm2d(128)
    self.deconv3 = nn.ConvTranspose2d(128, 64, 4, stride=1, bias=False)
    self.bn3 = nn.BatchNorm2d(64)
    self.deconv4 = nn.ConvTranspose2d(64, 32, 4, stride=2, bias=False)
    self.bn4 = nn.BatchNorm2d(32)
    self.deconv5 = nn.ConvTranspose2d(32, 32, 5, stride=1, bias=False)
    self.bn5 = nn.BatchNorm2d(32)
    self.deconv6 = nn.Conv2d(32, 3, 1, stride=1, bias=True)

def forward(self, z):
    z = F.leaky_relu(self.bn1(self.deconv1(z)), negative_slope=0.1,inplace=False)
    z = F.leaky_relu(self.bn2(self.deconv2(z)), negative_slope=0.1,inplace=False)
    z = F.leaky_relu(self.bn3(self.deconv3(z)), negative_slope=0.1,inplace=False)
    z = F.leaky_relu(self.bn4(self.deconv4(z)), negative_slope=0.1,inplace=False)
    z = F.leaky_relu(self.bn5(self.deconv5(z)), negative_slope=0.1,inplace=False)
    return torch.sigmoid(self.deconv6(z) + self.output_bias)
</code></pre>
<p>class Encoder(nn.Module):<br>
def <strong>init</strong>(self, z_dim=32):<br>
super(Encoder, self).<strong>init</strong>()<br>
self.z_dim = z_dim<br>
self.conv1 = nn.Conv2d(3, 32, 5, stride=1, bias=False)<br>
self.bn1 = nn.BatchNorm2d(32)<br>
self.conv2 = nn.Conv2d(32, 64, 4, stride=2, bias=False)<br>
self.bn2 = nn.BatchNorm2d(64)<br>
self.conv3 = nn.Conv2d(64, 128, 4, stride=1, bias=False)<br>
self.bn3 = nn.BatchNorm2d(128)<br>
self.conv4 = nn.Conv2d(128, 256, 4, stride=2, bias=False)<br>
self.bn4 = nn.BatchNorm2d(256)<br>
self.conv5 = nn.Conv2d(256, 512, 4, stride=1, bias=False)<br>
self.bn5 = nn.BatchNorm2d(512)<br>
self.conv6 = nn.Conv2d(512, 512, 1, stride=1, bias=False)<br>
self.bn6 = nn.BatchNorm2d(512)<br>
self.bn7 = nn.Conv2d(512, z_dim*2, 1, stride=1, bias=True)</p>
<pre><code>def reparameterize(self, z):
    z = z.view(z.size(0), -1)
    mu, log_sigma = z[:, :self.z_dim], z[:, self.z_dim:]
    std = torch.exp(log_sigma)
    eps = torch.randn_like(std)
    return mu + eps * std

def forward(self, x):
    x = F.leaky_relu(self.bn1(self.conv1(x)), negative_slope=0.1,inplace=False)
    x = F.leaky_relu(self.bn2(self.conv2(x)), negative_slope=0.1,inplace=False)
    x = F.leaky_relu(self.bn3(self.conv3(x)), negative_slope=0.1,inplace=False)
    x = F.leaky_relu(self.bn4(self.conv4(x)), negative_slope=0.1,inplace=False)
    x = F.leaky_relu(self.bn5(self.conv5(x)), negative_slope=0.1,inplace=False)
    x = F.leaky_relu(self.bn6(self.conv6(x)), negative_slope=0.1,inplace=False)
    z = self.reparameterize(self.conv6(x))
    return z.view(x.size(0), self.z_dim, 1, 1)
</code></pre>
<p>import torch<br>
import torch.nn as nn<br>
from torch import optim<br>
from torch.autograd import Variable<br>
import torch.nn.functional as F<br>
import torchvision.utils as vutils</p>
<p>import numpy as np<br>
from barbar import Bar</p>
<p>class TrainerBiGAN:<br>
def <strong>init</strong>(self, args, data, device):<br>
self.args = args<br>
self.train_loader = data<br>
self.device = device</p>
<pre><code>def train(self):
    """Training the BiGAN"""
    self.G = Generator(self.args.latent_dim).to(self.device)
    self.E = Encoder(self.args.latent_dim).to(self.device)
    self.D = Discriminator(self.args.latent_dim, self.args.wasserstein).to(self.device)

    self.G.apply(weights_init_normal)
    self.E.apply(weights_init_normal)
    self.D.apply(weights_init_normal)

    if self.args.wasserstein:
        optimizer_ge = optim.RMSprop(list(self.G.parameters()) +
                                     list(self.E.parameters()), lr=self.args.lr_rmsprop)
        optimizer_d = optim.RMSprop(self.D.parameters(), lr=self.args.lr_rmsprop)
    else:
        optimizer_ge = optim.Adam(list(self.G.parameters()) +
                                  list(self.E.parameters()), lr=self.args.lr_adam)
        optimizer_d = optim.Adam(self.D.parameters(), lr=self.args.lr_adam)

    fixed_z = Variable(torch.randn((16, self.args.latent_dim, 1, 1)),
                       requires_grad=False).to(self.device)
    criterion = nn.BCELoss()
    for epoch in range(self.args.num_epochs+1):
        ge_losses = 0
        d_losses = 0
        for x, _ in Bar(self.train_loader):
            #Defining labels
            y_true = Variable(torch.ones((x.size(0), 1)).to(self.device))
            y_fake = Variable(torch.zeros((x.size(0), 1)).to(self.device))

            #Noise for improving training.
            noise1 = Variable(torch.Tensor(x.size()).normal_(0, 
                              0.1 * (self.args.num_epochs - epoch) / self.args.num_epochs),
                              requires_grad=False).to(self.device)
            noise2 = Variable(torch.Tensor(x.size()).normal_(0, 
                              0.1 * (self.args.num_epochs - epoch) / self.args.num_epochs),
                              requires_grad=False).to(self.device)

            #Cleaning gradients.
            optimizer_d.zero_grad()
            optimizer_ge.zero_grad()

            #Generator:
            z_fake = Variable(torch.randn((x.size(0), self.args.latent_dim, 1, 1)).to(self.device),
                              requires_grad=False)
            x_fake = self.G(z_fake)

            #Encoder:
            x_true = x.float().to(self.device)
            z_true = self.E(x_true)

            #Discriminator
            out_true = self.D(x_true + noise1, z_true)
            out_fake = self.D(x_fake + noise2, z_fake)

            #Losses
            if self.args.wasserstein:
                loss_d = - torch.mean(out_true) + torch.mean(out_fake)
                loss_ge = - torch.mean(out_fake) + torch.mean(out_true)
            else:
                # 27/01/21
                y_true = y_true.unsqueeze(1)
                y_true = y_true.unsqueeze(1)
                y_fake=y_fake.unsqueeze(1)
                y_fake=y_fake.unsqueeze(1)
                loss_d = criterion(out_true, y_true) + criterion(out_fake, y_fake)
              
      
                loss_ge =(criterion(out_fake, y_true) + criterion(out_true, y_fake)).clone()

            #Computing gradients and backpropagate.
            loss_d.backward(retain_graph=True)
            optimizer_d.step()
            #with torch.autograd.set_detect_anomaly(True):
            torch.autograd.set_detect_anomaly(True)
            loss_ge.backward(retain_graph=True)
            optimizer_ge.step()


            if self.args.wasserstein:
                for p in self.D.parameters():
                    p.data.clamp_(-self.args.clamp, self.args.clamp)
            torch.autograd.backward(self, gradient, retain_graph, create_graph)
            ge_losses += loss_ge.item()
            d_losses += loss_d.item()

        if epoch % 50 == 0:
            vutils.save_image(self.G(fixed_z).data, './images/{}_fake.png'.format(epoch))

        print("Training... Epoch: {}, Discrimiantor Loss: {:.3f}, Generator Loss: {:.3f}".format(
            epoch, d_losses/len(self.train_loader), ge_losses/len(self.train_loader)
        ))
</code></pre>
<p>import torch</p>
<p>def weights_init_normal(m):<br>
classname = m.<strong>class</strong>.<strong>name</strong><br>
if classname.find(“Conv”) != -1 and classname != ‘Conv’:<br>
torch.nn.init.normal_(m.weight.data, 0.0, 0.02)<br>
if m.bias is not None:<br>
m.bias.data.fill_(0)<br>
elif classname.find(“Linear”) != -1:<br>
torch.nn.init.normal_(m.weight.data, 0.0, 0.02)<br>
if m.bias is not None:<br>
m.bias.data.fill_(0)<br>
elif classname.find(‘BatchNorm’) != -1:<br>
m.weight.data.normal_(1.0, 0.01)<br>
if m.bias is not None:<br>
m.bias.data.fill_(0)<br>
import torch<br>
import numpy as np<br>
from torchvision import datasets<br>
from torch.utils.data import DataLoader<br>
import torchvision.transforms as transforms</p>
<p>def get_cifar10(args, data_dir=’./data/cifar/’):<br>
“”“Returning cifar dataloder.”""<br>
transform = transforms.Compose([transforms.Resize(32), <span class="hashtag">#3x32x32</span> images.<br>
transforms.ToTensor()])<br>
data = datasets.CIFAR10(root=data_dir, train=True, download=True, transform=transform)<br>
dataloader = DataLoader(data, batch_size=args.batch_size, shuffle=True)<br>
return dataloader<br>
import numpy as np<br>
import argparse<br>
import torch</p>
<p>if <strong>name</strong> == ‘<strong>main</strong>’:<br>
parser = argparse.ArgumentParser()<br>
parser.add_argument("–num_epochs", type=int, default=200,<br>
help=“number of epochs”)<br>
parser.add_argument(’–lr_adam’, type=float, default=1e-4,<br>
help=‘learning rate’)<br>
parser.add_argument(’–lr_rmsprop’, type=float, default=1e-4,<br>
help=‘learning rate RMSprop if WGAN is True.’)<br>
parser.add_argument("–batch_size", type=int, default=128,<br>
help=“Batch size”)<br>
parser.add_argument(’–latent_dim’, type=int, default=256,<br>
help=‘Dimension of the latent variable z’)<br>
parser.add_argument(’–wasserstein’, type=bool, default=False,<br>
help=‘If WGAN.’)<br>
parser.add_argument(’–clamp’, type=float, default=1e-2,<br>
help=‘Clipping gradients for WGAN.’)<br>
<span class="hashtag">#parsing</span> arguments.<br>
args = parser.parse_args()</p>
<pre><code>#check if cuda is available.
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

data = get_cifar10(args)

bigan = TrainerBiGAN(args, data, device)
bigan.train()
</code></pre>
<p>Someone coul help me please?</p>

ptrblck: <p>Could you please check <a href="https://discuss.pytorch.org/t/struggling-with-a-runtime-error-related-to-in-place-operations/102222/6">this post</a>, which describes common errors in GAN training, where stale activations are used and which might raise this error?</p>
<p>PS: you can post code snippets by wrapping them into three backticks ```, which makes debugging easier. <img src="https://discuss.pytorch.org/images/emoji/apple/wink.png?v=9" title=":wink:" class="emoji" alt=":wink:"></p>

hajar: <aside class="quote no-group" data-username="ptrblck" data-post="2" data-topic="113626">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/ptrblck/48/1823_2.png" class="avatar"> ptrblck:</div>
<blockquote>
<p>this post</p>
</blockquote>
</aside>
<p>Hi, the problem comes from the computation of gradients and backpropagation:<br>
loss_d.backward(retain_graph=True)<br>
optimizer_d.step()</p>
<p>loss_ge.backward(retain_graph=True)<br>
optimizer_ge.step()</p>
<p>When I remove these two lines:<br>
loss_ge.backward(retain_graph=True)<br>
optimizer_ge.step()<br>
the code works but sure it is incorrect. So I understand That the problem come from here. Do you have any suggestions please?<br>
Thank you,</p>

ptrblck: <p>The link in my previous post explains this behavior in more detail.<br>
The short version is:</p>
<ul>
<li>
<code>loss_d</code> and <code>loss_ge</code> were calculated in some code before your posted code snippet</li>
<li>
<code>optimizer_d.step()</code> changes the parameters of the discriminator</li>
<li>
<code>loss_ge.backward()</code> calculates the gradients by backpropagating through the discriminator into the generator</li>
<li>since the discriminator was already updated (changed parameters), the backward pass fails</li>
</ul>
<p>To avoid these issues, you could e.g. update the discriminator before, perform the forward pass and loss calculation for the generator update, and calculate the gradients for the generator as the last step before calling <code>optimizer_ge.step()</code>.</p>

hajar: <aside class="quote no-group" data-username="ptrblck" data-post="4" data-topic="113626">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/ptrblck/48/1823_2.png" class="avatar"> ptrblck:</div>
<blockquote>
<p>The short version is:</p>
<ul>
<li><code>loss_d</code> and <code>loss_ge</code> were calculated in some code before your posted code snippet</li>
<li><code>optimizer_d.step()</code> changes the parameters of the discriminator</li>
<li><code>loss_ge.backward()</code> calculates the gradients by backpropagating through the discriminator into the generator</li>
<li>since the discriminator was already updated (changed parameters), the backward pass fails</li>
</ul>
<p>To avoid these issues, you could e.g. update the discriminator before, perform the forward pass and loss calculation for the generator update, and calculate the gradients for the generator as the last step before calling <code>optimizer_ge.step()</code>.</p>
</blockquote>
</aside>
<p>Thank you for you answer, I maid this change and it works. Could you tell me please if it’s correct?</p>
<p>““loss_d.backward(retain_graph=True)<br>
loss_ge.backward()<br>
optimizer_d.step()<br>
optimizer_ge.step()””</p>
<p>Thank you,</p>

ptrblck: <p>I don’t think this is correct, since the discriminator would now get the gradients for the generator as well.<br>
Have a look at the <a href="https://github.com/pytorch/examples/blob/master/dcgan/main.py">DCGAN example</a> to see how both models are updated.</p>