Pytorch RuntimeError in gradient computation

Sarmad_GTU: <p>Hi all<br>
i am working with the DRIT model code. I just copied it from GitHub and am trying to remove some unnecessary loss functions in the model part:<br>
The modified model.py below:</p>
<p>import networks<br>
import torch<br>
import torch.nn as nn</p>
<p>class DRIT(nn.Module):<br>
def <strong>init</strong>(self, opts):<br>
super(DRIT, self).<strong>init</strong>()</p>
<pre><code># parameters
lr = 0.0001
lr_dcontent = lr / 2.5
self.nz = 8  # No. of output channel in conv2d 




self.disA = networks.Dis(opts.input_dim_a, norm=opts.dis_norm, sn=opts.dis_spectral_norm)
self.disB = networks.Dis(opts.input_dim_b, norm=opts.dis_norm, sn=opts.dis_spectral_norm)


self.disContent = networks.Dis_content()

# encoders
self.enc_c = networks.E_content(opts.input_dim_a, opts.input_dim_b) # define the content encoder 

self.enc_a = networks.E_attr(opts.input_dim_a, opts.input_dim_b, self.nz)

# generator

self.gen = networks.G(opts.input_dim_a, opts.input_dim_b, nz=self.nz)

# optimizers
self.disA_opt = torch.optim.Adam(self.disA.parameters(), lr=lr, betas=(0.5, 0.999), weight_decay=0.0001)
self.disB_opt = torch.optim.Adam(self.disB.parameters(), lr=lr, betas=(0.5, 0.999), weight_decay=0.0001)
self.disContent_opt = torch.optim.Adam(self.disContent.parameters(), lr=lr_dcontent, betas=(0.5, 0.999), weight_decay=0.0001)
self.enc_c_opt = torch.optim.Adam(self.enc_c.parameters(), lr=lr, betas=(0.5, 0.999), weight_decay=0.0001)
self.enc_a_opt = torch.optim.Adam(self.enc_a.parameters(), lr=lr, betas=(0.5, 0.999), weight_decay=0.0001)
self.gen_opt = torch.optim.Adam(self.gen.parameters(), lr=lr, betas=(0.5, 0.999), weight_decay=0.0001)

# Setup the loss function for training
self.criterionL1 = torch.nn.L1Loss()
</code></pre>
<p>def initialize(self):<br>
self.disA.apply(networks.gaussian_weights_init)<br>
self.disB.apply(networks.gaussian_weights_init)</p>
<pre><code>self.disContent.apply(networks.gaussian_weights_init)
self.gen.apply(networks.gaussian_weights_init)
self.enc_c.apply(networks.gaussian_weights_init)
self.enc_a.apply(networks.gaussian_weights_init)
</code></pre>
<p>def set_scheduler(self, opts, last_ep=0):<br>
self.disA_sch = networks.get_scheduler(self.disA_opt, opts, last_ep)<br>
self.disB_sch = networks.get_scheduler(self.disB_opt, opts, last_ep)<br>
self.disContent_sch = networks.get_scheduler(self.disContent_opt, opts, last_ep)<br>
self.enc_c_sch = networks.get_scheduler(self.enc_c_opt, opts, last_ep)<br>
self.enc_a_sch = networks.get_scheduler(self.enc_a_opt, opts, last_ep)<br>
self.gen_sch = networks.get_scheduler(self.gen_opt, opts, last_ep)</p>
<p>def setgpu(self, gpu):<br>
self.gpu = gpu<br>
self.disA.cuda(self.gpu)<br>
self.disB.cuda(self.gpu)<br>
self.disContent.cuda(self.gpu)<br>
self.enc_c.cuda(self.gpu)<br>
self.enc_a.cuda(self.gpu)<br>
self.gen.cuda(self.gpu)</p>
<p>def test_forward(self, image, a2b=True):<br>
self.z_random = self.get_z_random(image.size(0), self.nz, ‘gauss’)<br>
if a2b:<br>
self.z_content = self.enc_c.forward_a(image)<br>
output = self.gen.forward_b(self.z_content, self.z_random)<br>
else:<br>
self.z_content = self.enc_c.forward_b(image)<br>
output = self.gen.forward_a(self.z_content, self.z_random)<br>
return output</p>
<p>def test_forward_transfer(self, image_a, image_b, a2b=True):<br>
self.z_content_a, self.z_content_b = self.enc_c.forward(image_a, image_b)<br>
if self.concat:<br>
self.mu_a, self.logvar_a, self.mu_b, self.logvar_b = self.enc_a.forward(image_a, image_b)<br>
std_a = self.logvar_a.mul(0.5).exp_()<br>
eps = self.get_z_random(std_a.size(0), std_a.size(1), ‘gauss’)<br>
self.z_attr_a = eps.mul(std_a).add_(self.mu_a)<br>
std_b = self.logvar_b.mul(0.5).exp_()<br>
eps = self.get_z_random(std_b.size(0), std_b.size(1), ‘gauss’)<br>
self.z_attr_b = eps.mul(std_b).add_(self.mu_b)<br>
else:<br>
self.z_attr_a, self.z_attr_b = self.enc_a.forward(image_a, image_b)<br>
if a2b:<br>
output = self.gen.forward_b(self.z_content_a, self.z_attr_b)<br>
else:<br>
output = self.gen.forward_a(self.z_content_b, self.z_attr_a)<br>
return output</p>
<p>def forward(self):<br>
# input images<br>
half_size = 1  # take one image only<br>
real_A = self.input_A<br>
real_B = self.input_B<br>
self.real_A_encoded = real_A[0:half_size] # batch 0<br>
self.real_B_encoded = real_B[0:half_size]# batch 0</p>
<pre><code># get encoded z_c
self.z_content_a, self.z_content_b = self.enc_c.forward(self.real_A_encoded, self.real_B_encoded)#ok



self.z_attr_a, self.z_attr_b = self.enc_a.forward(self.real_A_encoded, self.real_B_encoded)





# first cross translation

input_content_forA = torch.cat((self.z_content_b, self.z_content_a),0)
input_content_forB = torch.cat((self.z_content_a, self.z_content_b),0)
input_attr_forA = torch.cat((self.z_attr_a, self.z_attr_a),0)
input_attr_forB = torch.cat((self.z_attr_b, self.z_attr_b),0)
output_fakeA = self.gen.forward_a(input_content_forA, input_attr_forA)
output_fakeB = self.gen.forward_b(input_content_forB, input_attr_forB)
self.fake_A_encoded, self.fake_AA_encoded = torch.split(output_fakeA, self.z_content_a.size(0), dim=0)
self.fake_B_encoded, self.fake_BB_encoded = torch.split(output_fakeB, self.z_content_a.size(0), dim=0)

# second cross translation
# get reconstructed encoded z_c
self.z_content_recon_b, self.z_content_recon_a = self.enc_c.forward(self.fake_A_encoded, self.fake_B_encoded)
self.z_attr_recon_a, self.z_attr_recon_b = self.enc_a.forward(self.fake_A_encoded, self.fake_B_encoded)
# second cross translation
self.fake_A_recon = self.gen.forward_a(self.z_content_recon_a, self.z_attr_recon_a)
self.fake_B_recon = self.gen.forward_b(self.z_content_recon_b, self.z_attr_recon_b)
</code></pre>
<p>def forward_content(self):<br>
half_size = 1<br>
self.real_A_encoded = self.input_A[0:half_size]<br>
self.real_B_encoded = self.input_B[0:half_size]<br>
# get encoded z_c<br>
self.z_content_a, self.z_content_b = self.enc_c.forward(self.real_A_encoded, self.real_B_encoded)</p>
<p>def update_D_content(self, image_a, image_b):<br>
self.input_A = image_a<br>
self.input_B = image_b<br>
self.forward_content()<br>
self.disContent_opt.zero_grad()<br>
loss_D_Content = self.backward_contentD(self.z_content_a, self.z_content_b)<br>
self.disContent_loss = loss_D_Content.item()<br>
nn.utils.clip_grad_norm_(self.disContent.parameters(), 5)<br>
self.disContent_opt.step()</p>
<p>def update_D(self, image_a, image_b):<br>
self.input_A = image_a<br>
self.input_B = image_b<br>
self.forward()<br>
#########################################</p>
<pre><code># update disA
self.disA_opt.zero_grad()
loss_D1_A = self.backward_D(self.disA, self.real_A_encoded, self.fake_A_encoded)
self.disA_loss = loss_D1_A.item()
self.disA_opt.step()

# update disB
self.disB_opt.zero_grad()
loss_D1_B = self.backward_D(self.disB, self.real_B_encoded, self.fake_B_encoded)
self.disB_loss = loss_D1_B.item()
self.disB_opt.step()



# update disContent   # just to udpadte the disContent twise 
self.disContent_opt.zero_grad()
loss_D_Content = self.backward_contentD(self.z_content_a, self.z_content_b)
self.disContent_loss = loss_D_Content.item()
nn.utils.clip_grad_norm_(self.disContent.parameters(), 5)
self.disContent_opt.step()
</code></pre>
<p>def backward_D(self, netD, real, fake):<br>
pred_fake = netD.forward(fake.detach())<br>
pred_real = netD.forward(real)<br>
loss_D = 0<br>
for it, (out_a, out_b) in enumerate(zip(pred_fake, pred_real)):<br>
out_fake = torch.sigmoid(out_a)<br>
out_real = torch.sigmoid(out_b)<br>
all0 = torch.zeros_like(out_fake).cuda(self.gpu)<br>
all1 = torch.ones_like(out_real).cuda(self.gpu)<br>
ad_fake_loss = nn.functional.binary_cross_entropy(out_fake, all0)<br>
ad_true_loss = nn.functional.binary_cross_entropy(out_real, all1)<br>
loss_D = loss_D + (ad_true_loss + ad_fake_loss)<br>
loss_D.backward()<br>
return loss_D</p>
<p>def backward_contentD(self, imageA, imageB):<br>
pred_fake = self.disContent.forward(imageA.detach())<br>
pred_real = self.disContent.forward(imageB.detach())<br>
for it, (out_a, out_b) in enumerate(zip(pred_fake, pred_real)):<br>
out_fake = torch.sigmoid(out_a)<br>
out_real = torch.sigmoid(out_b)<br>
all1 = torch.ones((out_real.size(0))).cuda(self.gpu)<br>
all0 = torch.zeros((out_fake.size(0))).cuda(self.gpu)<br>
ad_true_loss = nn.functional.binary_cross_entropy(out_real, all1)<br>
ad_fake_loss = nn.functional.binary_cross_entropy(out_fake, all0)<br>
loss_D = ad_true_loss + ad_fake_loss<br>
loss_D.backward()<br>
return loss_D</p>
<p>def update_EG(self):<br>
# update G, Ec, Ea<br>
self.enc_c_opt.zero_grad()<br>
self.enc_a_opt.zero_grad()<br>
self.gen_opt.zero_grad()</p>
<pre><code># content Ladv for generator
loss_G_GAN_Acontent = self.backward_G_GAN_content(self.z_content_a)
loss_G_GAN_Bcontent = self.backward_G_GAN_content(self.z_content_b)

# Ladv for generator
loss_G_GAN_A = self.backward_G_GAN(self.fake_A_encoded, self.disA)
loss_G_GAN_B = self.backward_G_GAN(self.fake_B_encoded, self.disB)



# cross cycle consistency loss
loss_G_L1_A = self.criterionL1(self.fake_A_recon, self.real_A_encoded) * 10 # we can remove cyclGAN 
loss_G_L1_B = self.criterionL1(self.fake_B_recon, self.real_B_encoded) * 10 # we can remove cyclGAN
loss_G_L1_AA = self.criterionL1(self.fake_AA_encoded, self.real_A_encoded) * 10
loss_G_L1_BB = self.criterionL1(self.fake_BB_encoded, self.real_B_encoded) * 10

loss_G = loss_G_GAN_A + loss_G_GAN_B + loss_G_GAN_Acontent + loss_G_GAN_Bcontent + loss_G_L1_AA + loss_G_L1_BB + loss_G_L1_A + loss_G_L1_B 
         

#do backward()
loss_G.backward(retain_graph=True)
#self.backward_EG()


# do optimisation
self.enc_c_opt.step()
self.enc_a_opt.step()
self.gen_opt.step()


self.gan_loss_a = loss_G_GAN_A.item()
self.gan_loss_b = loss_G_GAN_B.item()
self.gan_loss_acontent = loss_G_GAN_Acontent.item()
self.gan_loss_bcontent = loss_G_GAN_Bcontent.item()

self.l1_recon_A_loss = loss_G_L1_A.item()
self.l1_recon_B_loss = loss_G_L1_B.item()
self.l1_recon_AA_loss = loss_G_L1_AA.item()
self.l1_recon_BB_loss = loss_G_L1_BB.item()
self.G_loss = loss_G.item()
</code></pre>
<p><span class="hashtag">#def</span> backward_EG(self):</p>
<p>def backward_G_GAN_content(self, data):<br>
outs = self.disContent.forward(data)<br>
for out in outs:<br>
outputs_fake = torch.sigmoid(out)<br>
all_half = 0.5*torch.ones((outputs_fake.size(0))).cuda(self.gpu)<br>
ad_loss = nn.functional.binary_cross_entropy(outputs_fake, all_half)<br>
return ad_loss</p>
<p>def backward_G_GAN(self, fake, netD=None):<br>
outs_fake = netD.forward(fake)<br>
loss_G = 0<br>
for out_a in outs_fake:<br>
outputs_fake = torch.sigmoid(out_a)<br>
all_ones = torch.ones_like(outputs_fake).cuda(self.gpu)<br>
loss_G = loss_G + (nn.functional.binary_cross_entropy(outputs_fake, all_ones))<br>
return loss_G</p>
<p>def update_lr(self):<br>
self.disA_sch.step()<br>
self.disB_sch.step()<br>
self.disContent_sch.step()<br>
self.enc_c_sch.step()<br>
self.enc_a_sch.step()<br>
self.gen_sch.step()</p>
<p>def save(self, filename, ep, total_it):<br>
state = {<br>
‘disA’: self.disA.state_dict(),<br>
‘disB’: self.disB.state_dict(),<br>
‘disContent’: self.disContent.state_dict(),<br>
‘enc_c’: self.enc_c.state_dict(),<br>
‘enc_a’: self.enc_a.state_dict(),<br>
‘gen’: self.gen.state_dict(),</p>
<pre><code>         'disA_opt': self.disA_opt.state_dict(),
         'disContent_opt': self.disContent_opt.state_dict(),
         'enc_c_opt': self.enc_c_opt.state_dict(),
         'enc_a_opt': self.enc_a_opt.state_dict(),
         'gen_opt': self.gen_opt.state_dict(),
         'ep': ep,
         'total_it': total_it
          }
torch.save(state, filename)
return
</code></pre>
<p>def assemble_outputs(self):<br>
images_a =self.real_A_encoded.detach()<br>
images_a1 =self.fake_A_encoded.detach()<br>
images_a2 = self.normalize_image(self.fake_AA_encoded).detach()<br>
images_a3 = self.normalize_image(self.fake_A_recon).detach()</p>
<pre><code>images_b = self.real_B_encoded.detach()
images_b1 = self.normalize_image(self.fake_B_encoded).detach()
images_b2 = self.normalize_image(self.fake_BB_encoded).detach()
images_b3 = self.normalize_image(self.fake_B_recon).detach()

row1 = torch.cat((images_a[0:1, ::], images_b1[0:1, ::], images_a2[0:1, ::], images_a3[0:1, ::]),3)
row2 = torch.cat((images_b[0:1, ::], images_a1[0:1, ::], images_b2[0:1, ::], images_b3[0:1, ::]),3)
return torch.cat((row1,row2),2)
</code></pre>
<p>and the train part is:</p>
<h1>
<a name="train-1" class="anchor" href="#train-1"></a>train</h1>
<p>print(‘\n— train —’)<br>
for ep in range(ep0, opts.n_ep):<br>
gc.collect()<br>
torch.cuda.empty_cache()<br>
for it,(images_a, images_b) in enumerate(tqdm(train_loader, desc=‘Epoch: {}/{}’.format(ep, opts.n_ep))):<br>
# input data<br>
images_a = images_a.cuda(opts.gpu).detach()<br>
images_b = images_b.cuda(opts.gpu).detach()</p>
<pre><code>    # update model
    if (it + 1) % opts.d_iter != 0 and it &lt; len(train_loader) - 2:
        model.update_D_content(images_a, images_b)
        continue
    else:
        model.update_D(images_a, images_b)
        model.update_EG()
</code></pre>
<h1>
<a name="printtotal_it-d-ep-d-it-d-lr-08f-total_it-ep-it-modelgen_optparam_groups0lr-2" class="anchor" href="#printtotal_it-d-ep-d-it-d-lr-08f-total_it-ep-it-modelgen_optparam_groups0lr-2"></a>print(‘total_it: %d (ep %d, it %d), lr %08f’ % (total_it, ep, it, model.gen_opt.param_groups[0][‘lr’]))</h1>
<pre><code>    total_it += 1


       

    model.update_lr()

# save result image
saver.write_img(ep, model)

# Save network weights
if (ep % 50 == 0):
    saver.write_model(ep, total_it, model)
    print('total_it: %d (ep %d, it %d), lr %08f' % (total_it, ep, it, model.gen_opt.param_groups[0]['lr']))
</code></pre>
<h2>
<a name="after-running-the-train-part-the-runtime-error-below-is-coming-3" class="anchor" href="#after-running-the-train-part-the-runtime-error-below-is-coming-3"></a>after running the train part the runtime error below is coming:</h2>
<p>RuntimeError                              Traceback (most recent call last)<br>
Input In [6], in &lt;cell line: 4&gt;()<br>
16         else:<br>
17             model.update_D(images_a, images_b)<br>
—&gt; 18             model.update_EG()<br>
20 #         print(‘total_it: %d (ep %d, it %d), lr %08f’ % (total_it, ep, it, model.gen_opt.param_groups[0][‘lr’]))<br>
21         total_it += 1</p>
<p>File ~\Project_All\DRIT-master\model.py:237, in DRIT.update_EG(self)<br>
233 loss_G = loss_G_GAN_A + loss_G_GAN_B + loss_G_GAN_Acontent + loss_G_GAN_Bcontent + loss_G_L1_AA + loss_G_L1_BB + loss_G_L1_A + loss_G_L1_B<br>
236 <span class="hashtag">#do</span> backward()<br>
 → 237 loss_G.backward(retain_graph=True)<br>
238 <span class="hashtag">#self</span>.backward_EG()<br>
239<br>
240<br>
241 # do optimisation<br>
242 self.enc_c_opt.step()</p>
<p>File ~\anaconda3\envs\Deeplearning\lib\site-packages\torch_tensor.py:363, in Tensor.backward(self, gradient, retain_graph, create_graph, inputs)<br>
354 if has_torch_function_unary(self):<br>
355     return handle_torch_function(<br>
356         Tensor.backward,<br>
357         (self,),<br>
(…)<br>
361         create_graph=create_graph,<br>
362         inputs=inputs)<br>
 → 363 torch.autograd.backward(self, gradient, retain_graph, create_graph, inputs=inputs)</p>
<p>File ~\anaconda3\envs\Deeplearning\lib\site-packages\torch\autograd_<em>init</em>_.py:173, in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)<br>
168     retain_graph = create_graph<br>
170 # The reason we repeat same the comment below is that<br>
171 # some Python versions print out the first line of a multi-line function<br>
172 # calls in the traceback and some print out the last line<br>
 → 173 Variable.<em>execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass<br>
174     tensors, grad_tensors</em>, retain_graph, create_graph, inputs,<br>
175     allow_unreachable=True, accumulate_grad=True)</p>
<p>RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [1, 256, 64, 64]], which is output 0 of ReluBackward0, is at version 1; expected version 0 instead. Hint: enable anomaly detection to find the operation that failed to compute its gradient, with torch.autograd.set_detect_anomaly(True).</p>
<p>I deleted the inplace operation in the model part and also i changed the inplace = false in Rule operation but i still have this error!</p>
<p>How can i solve this issue?</p>

AlphaBetaGamma96: <p>Hi <a class="mention" href="/u/sarmad_gtu">@Sarmad_GTU</a>,</p>
<aside class="quote no-group" data-username="Sarmad_GTU" data-post="1" data-topic="163088">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/sarmad_gtu/48/41574_2.png" class="avatar"> Sarmad_GTU:</div>
<blockquote>
<p>Hint: enable anomaly detection to find the operation that failed to compute its gradient, with torch.autograd.set_detect_anomaly(True).</p>
</blockquote>
</aside>
<p>Have you ran your code within a <code>torch.autograd.set_detect_anamoly</code> context manager? (And shared the entire stacktrace?)</p>
<p>Also, make sure to correct initalize your <code>nn.Module</code> objects.</p>
<aside class="quote no-group" data-username="Sarmad_GTU" data-post="1" data-topic="163088">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/sarmad_gtu/48/41574_2.png" class="avatar"> Sarmad_GTU:</div>
<blockquote>
<p>class DRIT(nn.Module):<br>
def <strong>init</strong>(self, opts):<br>
super(DRIT, self).<strong>init</strong>()</p>
</blockquote>
</aside>
<p>Should be,</p>
<pre><code class="lang-auto">class DRIT(nn.Module):
  def __init__(self, opts):
    super(DRIT, self).__init__()
</code></pre>
<p>(the underscores matter)</p>

Sarmad_GTU: <aside class="quote no-group" data-username="AlphaBetaGamma96" data-post="2" data-topic="163088">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/letter_avatar_proxy/v4/letter/a/3da27b/48.png" class="avatar"> AlphaBetaGamma96:</div>
<blockquote>
<pre><code class="lang-auto">class DRIT(nn.Module):
  def __init__(self, opts):
    super(DRIT, self).__init__()
</code></pre>
</blockquote>
</aside>
<p>Thank you for reply<br>
The nn.Module inilaize correct but when I posted the code some underscores  deleted.<br>
I rune the below code for  enable anomaly detection:</p>
<h1><a name="train-1" class="anchor" href="#train-1"></a>train</h1>
<p>with torch.autograd.set_detect_anomaly(True):<br>
print(‘\n— train —’)<br>
for ep in range(ep0, opts.n_ep):<br>
gc.collect()<br>
torch.cuda.empty_cache()<br>
for it,(images_a, images_b) in enumerate(tqdm(train_loader, desc=‘Epoch: {}/{}’.format(ep, opts.n_ep))):<br>
# input data<br>
images_a = images_a.cuda(opts.gpu).detach()<br>
images_b = images_b.cuda(opts.gpu).detach()</p>
<pre><code>        # update model
        if (it + 1) % opts.d_iter != 0 and it &lt; len(train_loader) - 2:
            model.update_D_content(images_a, images_b)
            continue
        else:
            model.update_D(images_a, images_b)
            model.update_EG()
</code></pre>
<h1><a name="printtotal_it-d-ep-d-it-d-lr-08f-total_it-ep-it-modelgen_optparam_groups0lr-2" class="anchor" href="#printtotal_it-d-ep-d-it-d-lr-08f-total_it-ep-it-modelgen_optparam_groups0lr-2"></a>print(‘total_it: %d (ep %d, it %d), lr %08f’ % (total_it, ep, it, model.gen_opt.param_groups[0][‘lr’]))</h1>
<pre><code>        total_it += 1


       

        model.update_lr()

# save result image
    saver.write_img(ep, model)

# Save network weights
    if (ep % 50 == 0):
        saver.write_model(ep, total_it, model)
        print('total_it: %d (ep %d, it %d), lr %08f' % (total_it, ep, it, model.gen_opt.param_groups[0]['lr']))
</code></pre>
<p>and I got:</p>
<h1><a name="train-3" class="anchor" href="#train-3"></a>train</h1>
<p>with torch.autograd.set_detect_anomaly(True):<br>
print(‘\n— train —’)<br>
for ep in range(ep0, opts.n_ep):<br>
gc.collect()<br>
torch.cuda.empty_cache()<br>
for it,(images_a, images_b) in enumerate(tqdm(train_loader, desc=‘Epoch: {}/{}’.format(ep, opts.n_ep))):<br>
# input data<br>
images_a = images_a.cuda(opts.gpu).detach()<br>
images_b = images_b.cuda(opts.gpu).detach()</p>
<pre><code>        # update model
        if (it + 1) % opts.d_iter != 0 and it &lt; len(train_loader) - 2:
            model.update_D_content(images_a, images_b)
            continue
        else:
            model.update_D(images_a, images_b)
            model.update_EG()
</code></pre>
<h1><a name="printtotal_it-d-ep-d-it-d-lr-08f-total_it-ep-it-modelgen_optparam_groups0lr-4" class="anchor" href="#printtotal_it-d-ep-d-it-d-lr-08f-total_it-ep-it-modelgen_optparam_groups0lr-4"></a>print(‘total_it: %d (ep %d, it %d), lr %08f’ % (total_it, ep, it, model.gen_opt.param_groups[0][‘lr’]))</h1>
<pre><code>        total_it += 1


       

        model.update_lr()

# save result image
    saver.write_img(ep, model)

# Save network weights
    if (ep % 50 == 0):
        saver.write_model(ep, total_it, model)
        print('total_it: %d (ep %d, it %d), lr %08f' % (total_it, ep, it, model.gen_opt.param_groups[0]['lr']))
</code></pre>

AlphaBetaGamma96: <p>Where is the stacktrace?</p>

Sarmad_GTU: <p>Epoch: 0/50:   0%|          | 1/1573 [00:08&lt;3:44:09,  8.56s/it]<br>
Epoch: 0/50:   0%|          | 2/1573 [00:08&lt;2:37:47,  6.03s/it]C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\torch\autograd_<em>init</em>_.py:173: UserWarning: Error detected in ReluBackward0. Traceback of forward call that caused the error:<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\runpy.py”, line 194, in _run_module_as_main<br>
return _run_code(code, main_globals, None,<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\runpy.py”, line 87, in _run_code<br>
exec(code, run_globals)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\ipykernel_launcher.py”, line 16, in <br>
app.launch_new_instance()<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\traitlets\config\application.py”, line 846, in launch_instance<br>
app.start()<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\ipykernel\kernelapp.py”, line 677, in start<br>
self.io_loop.start()<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\tornado\platform\asyncio.py”, line 199, in start<br>
self.asyncio_loop.run_forever()<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\asyncio\base_events.py”, line 570, in run_forever<br>
self._run_once()<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\asyncio\base_events.py”, line 1859, in _run_once<br>
handle._run()<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\asyncio\events.py”, line 81, in _run<br>
self._context.run(self._callback, *self._args)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\ipykernel\kernelbase.py”, line 471, in dispatch_queue<br>
await self.process_one()<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\ipykernel\kernelbase.py”, line 460, in process_one<br>
await dispatch(*args)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\ipykernel\kernelbase.py”, line 367, in dispatch_shell<br>
await result<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\ipykernel\kernelbase.py”, line 662, in execute_request<br>
reply_content = await reply_content<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\ipykernel\ipkernel.py”, line 360, in do_execute<br>
res = shell.run_cell(code, store_history=store_history, silent=silent)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\ipykernel\zmqshell.py”, line 532, in run_cell<br>
return super().run_cell(*args, **kwargs)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\IPython\core\interactiveshell.py”, line 2863, in run_cell<br>
result = self._run_cell(<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\IPython\core\interactiveshell.py”, line 2909, in _run_cell<br>
return runner(coro)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\IPython\core\async_helpers.py”, line 129, in <em>pseudo_sync_runner<br>
coro.send(None)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\IPython\core\interactiveshell.py”, line 3106, in run_cell_async<br>
has_raised = await self.run_ast_nodes(code_ast.body, cell_name,<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\IPython\core\interactiveshell.py”, line 3309, in run_ast_nodes<br>
if await self.run_code(code, result, async</em>=asy):<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\IPython\core\interactiveshell.py”, line 3369, in run_code<br>
exec(code_obj, self.user_global_ns, self.user_ns)<br>
File “C:\Users\gtu\AppData\Local\Temp\ipykernel_8216\2562760679.py”, line 18, in &lt;cell line: 2&gt;<br>
model.update_EG()<br>
File “C:\Users\gtu\Project_All\DRIT-master\model.py”, line 216, in update_EG<br>
File “C:\Users\gtu\Project_All\DRIT-master\model.py”, line 135, in forward<br>
self.fake_B_recon = self.gen.forward_b(self.z_content_recon_b, self.z_attr_recon_b)<br>
File “C:\Users\gtu\Project_All\DRIT-master\networks.py”, line 233, in forward_b<br>
out4 = self.decB4(out3, z4)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\torch\nn\modules\module.py”, line 1110, in _call_impl<br>
return forward_call(*input, **kwargs)<br>
File “C:\Users\gtu\Project_All\DRIT-master\networks.py”, line 424, in forward<br>
out = self.blk2(torch.cat([o3, z_expand], dim=1))<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\torch\nn\modules\module.py”, line 1110, in _call_impl<br>
return forward_call(*input, **kwargs)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\torch\nn\modules\container.py”, line 141, in forward<br>
input = module(input)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\torch\nn\modules\module.py”, line 1110, in _call_impl<br>
return forward_call(*input, **kwargs)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\torch\nn\modules\activation.py”, line 98, in forward<br>
return F.relu(input, inplace=self.inplace)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\torch\nn\functional.py”, line 1442, in relu<br>
result = torch.relu(input)<br>
(Triggered internally at  C:\cb\pytorch_1000000000000\work\torch\csrc\autograd\python_anomaly_mode.cpp:104.)<br>
Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass</p>

AlphaBetaGamma96: <aside class="quote no-group" data-username="Sarmad_GTU" data-post="5" data-topic="163088">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/sarmad_gtu/48/41574_2.png" class="avatar"> Sarmad_GTU:</div>
<blockquote>
<p>File “C:\Users\gtu\Project_All\DRIT-master\networks.py”, line 424, in forward<br>
out = self.blk2(torch.cat([o3, z_expand], dim=1))</p>
</blockquote>
</aside>
<p>Can you check how <code>blk2</code> is defined within <code>C:\Users\gtu\Project_All\DRIT-master\networks.py</code> at line 424? You seem to have an in-place operation there.</p>

Sarmad_GTU: <aside class="quote no-group" data-username="AlphaBetaGamma96" data-post="6" data-topic="163088">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/letter_avatar_proxy/v4/letter/a/3da27b/48.png" class="avatar"> AlphaBetaGamma96:</div>
<blockquote>
<p><code>blk2</code></p>
</blockquote>
</aside>
<p>No inPlace operation:<br>
the Network.py below:</p>
<p>import  torch<br>
import torch.nn as nn<br>
from torch.autograd import Variable<br>
import functools<br>
from torch.optim import lr_scheduler<br>
import torch.nn.functional as F</p>
<p>####################################################################<br>
<span class="hashtag-raw">#-------------------------</span> Discriminators --------------------------<br>
####################################################################<br>
class Dis_content(nn.Module):<br>
def <strong>init</strong>(self):<br>
super(Dis_content, self).<strong>init</strong>()<br>
model = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
model += [LeakyReLUConv2d(256, 256, kernel_size=7, stride=2, padding=1, norm=‘Instance’)]<br>
model += [LeakyReLUConv2d(256, 256, kernel_size=7, stride=2, padding=1, norm=‘Instance’)]<br>
model += [LeakyReLUConv2d(256, 256, kernel_size=7, stride=2, padding=1, norm=‘Instance’)]<br>
model += [LeakyReLUConv2d(256, 256, kernel_size=4, stride=1, padding=0)]<br>
model += [nn.Conv2d(256, 1, kernel_size=1, stride=1, padding=0)]<br>
self.model = nn.Sequential(*model)</p>
<p>def forward(self, x):<br>
out = self.model(x)<br>
out = out.view(-1)<br>
outs = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
outs.append(out)<br>
return outs</p>
<p>class MultiScaleDis(nn.Module):<br>
def <strong>init</strong>(self, input_dim, n_scale=3, n_layer=4, norm=‘None’, sn=False):<br>
super(MultiScaleDis, self).<strong>init</strong>()<br>
ch = 64<br>
self.downsample = nn.AvgPool2d(3, stride=2, padding=1, count_include_pad=False)<br>
self.Diss = nn.ModuleList()<br>
for _ in range(n_scale):<br>
self.Diss.append(self._make_net(ch, input_dim, n_layer, norm, sn))</p>
<p>def _make_net(self, ch, input_dim, n_layer, norm, sn):<br>
model = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
model += [LeakyReLUConv2d(input_dim, ch, 4, 2, 1, norm, sn)]<br>
tch = ch<br>
for _ in range(1, n_layer):<br>
model += [LeakyReLUConv2d(tch, tch * 2, 4, 2, 1, norm, sn)]<br>
tch *= 2<br>
if sn:<br>
model += [spectral_norm(nn.Conv2d(tch, 1, 1, 1, 0))]<br>
else:<br>
model += [nn.Conv2d(tch, 1, 1, 1, 0)]<br>
return nn.Sequential(*model)</p>
<p>def forward(self, x):<br>
outs = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
for Dis in self.Diss:<br>
outs.append(Dis(x))<br>
x = self.downsample(x)<br>
return outs</p>
<p>class Dis(nn.Module):<br>
def <strong>init</strong>(self, input_dim, norm=‘None’, sn=False):<br>
super(Dis, self).<strong>init</strong>()<br>
ch = 64<br>
n_layer = 6<br>
self.model = self._make_net(ch, input_dim, n_layer, norm, sn)</p>
<p>def _make_net(self, ch, input_dim, n_layer, norm, sn):<br>
model = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
model += [LeakyReLUConv2d(input_dim, ch, kernel_size=3, stride=2, padding=1, norm=norm, sn=sn)] <span class="hashtag-raw">#16</span><br>
tch = ch<br>
for i in range(1, n_layer-1):<br>
model += [LeakyReLUConv2d(tch, tch * 2, kernel_size=3, stride=2, padding=1, norm=norm, sn=sn)] # 8<br>
tch *= 2<br>
model += [LeakyReLUConv2d(tch, tch * 2, kernel_size=3, stride=2, padding=1, norm=‘None’, sn=sn)] # 2<br>
tch *= 2<br>
if sn:<br>
model += [spectral_norm(nn.Conv2d(tch, 1, kernel_size=1, stride=1, padding=0))]  # 1<br>
else:<br>
model += [nn.Conv2d(tch, 1, kernel_size=1, stride=1, padding=0)]  # 1<br>
return nn.Sequential(*model)</p>
<p>def cuda(self,gpu):<br>
self.model.cuda(gpu)</p>
<p>def forward(self, x_A):<br>
out_A = self.model(x_A)<br>
out_A = out_A.view(-1)<br>
outs_A = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
outs_A.append(out_A)<br>
return outs_A</p>
<p>####################################################################<br>
<span class="hashtag-raw">#----------------------------</span> Encoders -----------------------------<br>
####################################################################<br>
class E_content(nn.Module):<br>
def <strong>init</strong>(self, input_dim_a, input_dim_b):<br>
super(E_content, self).<strong>init</strong>()<br>
encA_c = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
tch = 64<br>
encA_c += [LeakyReLUConv2d(input_dim_a, tch, kernel_size=7, stride=1, padding=3)]<br>
for i in range(1, 3):<br>
encA_c += [ReLUINSConv2d(tch, tch * 2, kernel_size=3, stride=2, padding=1)]<br>
tch *= 2<br>
for i in range(0, 3):<br>
encA_c += [INSResBlock(tch, tch)]</p>
<pre><code>encB_c = []
tch = 64
encB_c += [LeakyReLUConv2d(input_dim_b, tch, kernel_size=7, stride=1, padding=3)]
for i in range(1, 3):
  encB_c += [ReLUINSConv2d(tch, tch * 2, kernel_size=3, stride=2, padding=1)]
  tch *= 2
for i in range(0, 3):
  encB_c += [INSResBlock(tch, tch)]

enc_share = []
for i in range(0, 1):
  enc_share += [INSResBlock(tch, tch)]
  enc_share += [GaussianNoiseLayer()]
  self.conv_share = nn.Sequential(*enc_share)

self.convA = nn.Sequential(*encA_c)
self.convB = nn.Sequential(*encB_c)
</code></pre>
<p>def forward(self, xa, xb):<br>
outputA = self.convA(xa)<br>
outputB = self.convB(xb)<br>
outputA = self.conv_share(outputA)<br>
outputB = self.conv_share(outputB)<br>
return outputA, outputB</p>
<p>def forward_a(self, xa):<br>
outputA = self.convA(xa)<br>
outputA = self.conv_share(outputA)<br>
return outputA</p>
<p>def forward_b(self, xb):<br>
outputB = self.convB(xb)<br>
outputB = self.conv_share(outputB)<br>
return outputB</p>
<p>class E_attr(nn.Module):<br>
def <strong>init</strong>(self, input_dim_a, input_dim_b, output_nc=8):<br>
super(E_attr, self).<strong>init</strong>()<br>
dim = 64<br>
self.model_a = nn.Sequential(<br>
nn.ReflectionPad2d(3),<br>
nn.Conv2d(input_dim_a, dim, 7, 1),<br>
nn.ReLU(inplace=False),<br>
nn.ReflectionPad2d(1),<br>
nn.Conv2d(dim, dim<em>2, 4, 2),<br>
nn.ReLU(inplace=False),<br>
nn.ReflectionPad2d(1),<br>
nn.Conv2d(dim</em>2, dim<em>4, 4, 2),<br>
nn.ReLU(inplace=False),<br>
nn.ReflectionPad2d(1),<br>
nn.Conv2d(dim</em>4, dim<em>4, 4, 2),<br>
nn.ReLU(inplace=False),<br>
nn.ReflectionPad2d(1),<br>
nn.Conv2d(dim</em>4, dim<em>4, 4, 2),<br>
nn.ReLU(inplace=False),<br>
nn.AdaptiveAvgPool2d(1),<br>
nn.Conv2d(dim</em>4, output_nc, 1, 1, 0))<br>
self.model_b = nn.Sequential(<br>
nn.ReflectionPad2d(3),<br>
nn.Conv2d(input_dim_b, dim, 7, 1),<br>
nn.ReLU(inplace=False),<br>
nn.ReflectionPad2d(1),<br>
nn.Conv2d(dim, dim<em>2, 4, 2),<br>
nn.ReLU(inplace=False),<br>
nn.ReflectionPad2d(1),<br>
nn.Conv2d(dim</em>2, dim<em>4, 4, 2),<br>
nn.ReLU(inplace=False),<br>
nn.ReflectionPad2d(1),<br>
nn.Conv2d(dim</em>4, dim<em>4, 4, 2),<br>
nn.ReLU(inplace=False),<br>
nn.ReflectionPad2d(1),<br>
nn.Conv2d(dim</em>4, dim<em>4, 4, 2),<br>
nn.ReLU(inplace=False),<br>
nn.AdaptiveAvgPool2d(1),<br>
nn.Conv2d(dim</em>4, output_nc, 1, 1, 0))<br>
return</p>
<p>def forward(self, xa, xb):<br>
xa = self.model_a(xa)<br>
xb = self.model_b(xb)<br>
output_A = xa.view(xa.size(0), -1)<br>
output_B = xb.view(xb.size(0), -1)<br>
return output_A, output_B</p>
<p>def forward_a(self, xa):<br>
xa = self.model_a(xa)<br>
output_A = xa.view(xa.size(0), -1)<br>
return output_A</p>
<p>def forward_b(self, xb):<br>
xb = self.model_b(xb)<br>
output_B = xb.view(xb.size(0), -1)<br>
return output_B</p>
<p>class E_attr_concat(nn.Module):<br>
def <strong>init</strong>(self, input_dim_a, input_dim_b, output_nc=8, norm_layer=None, nl_layer=None):<br>
super(E_attr_concat, self).<strong>init</strong>()</p>
<pre><code>ndf = 64
n_blocks=4
max_ndf = 4

conv_layers_A = [nn.ReflectionPad2d(1)]
conv_layers_A += [nn.Conv2d(input_dim_a, ndf, kernel_size=4, stride=2, padding=0, bias=True)]
for n in range(1, n_blocks):
  input_ndf = ndf * min(max_ndf, n)  # 2**(n-1)
  output_ndf = ndf * min(max_ndf, n+1)  # 2**n
  conv_layers_A += [BasicBlock(input_ndf, output_ndf, norm_layer, nl_layer)]
conv_layers_A += [nl_layer(), nn.AdaptiveAvgPool2d(1)] # AvgPool2d(13)
self.fc_A = nn.Sequential(*[nn.Linear(output_ndf, output_nc)])
self.fcVar_A = nn.Sequential(*[nn.Linear(output_ndf, output_nc)])
self.conv_A = nn.Sequential(*conv_layers_A)

conv_layers_B = [nn.ReflectionPad2d(1)]
conv_layers_B += [nn.Conv2d(input_dim_b, ndf, kernel_size=4, stride=2, padding=0, bias=True)]
for n in range(1, n_blocks):
  input_ndf = ndf * min(max_ndf, n)  # 2**(n-1)
  output_ndf = ndf * min(max_ndf, n+1)  # 2**n
  conv_layers_B += [BasicBlock(input_ndf, output_ndf, norm_layer, nl_layer)]
conv_layers_B += [nl_layer(), nn.AdaptiveAvgPool2d(1)] # AvgPool2d(13)
self.fc_B = nn.Sequential(*[nn.Linear(output_ndf, output_nc)])
self.fcVar_B = nn.Sequential(*[nn.Linear(output_ndf, output_nc)])
self.conv_B = nn.Sequential(*conv_layers_B)
</code></pre>
<p>def forward(self, xa, xb):<br>
x_conv_A = self.conv_A(xa)<br>
conv_flat_A = x_conv_A.view(xa.size(0), -1)<br>
output_A = self.fc_A(conv_flat_A)<br>
outputVar_A = self.fcVar_A(conv_flat_A)<br>
x_conv_B = self.conv_B(xb)<br>
conv_flat_B = x_conv_B.view(xb.size(0), -1)<br>
output_B = self.fc_B(conv_flat_B)<br>
outputVar_B = self.fcVar_B(conv_flat_B)<br>
return output_A, outputVar_A, output_B, outputVar_B</p>
<p>def forward_a(self, xa):<br>
x_conv_A = self.conv_A(xa)<br>
conv_flat_A = x_conv_A.view(xa.size(0), -1)<br>
output_A = self.fc_A(conv_flat_A)<br>
outputVar_A = self.fcVar_A(conv_flat_A)<br>
return output_A, outputVar_A</p>
<p>def forward_b(self, xb):<br>
x_conv_B = self.conv_B(xb)<br>
conv_flat_B = x_conv_B.view(xb.size(0), -1)<br>
output_B = self.fc_B(conv_flat_B)<br>
outputVar_B = self.fcVar_B(conv_flat_B)<br>
return output_B, outputVar_B</p>
<p>####################################################################<br>
<span class="hashtag-raw">#---------------------------</span> Generators ----------------------------<br>
####################################################################<br>
class G(nn.Module):<br>
def <strong>init</strong>(self, output_dim_a, output_dim_b, nz):<br>
super(G, self).<strong>init</strong>()<br>
self.nz = nz<br>
ini_tch = 256<br>
tch_add = ini_tch<br>
tch = ini_tch<br>
self.tch_add = tch_add<br>
self.decA1 = MisINSResBlock(tch, tch_add)<br>
self.decA2 = MisINSResBlock(tch, tch_add)<br>
self.decA3 = MisINSResBlock(tch, tch_add)<br>
self.decA4 = MisINSResBlock(tch, tch_add)</p>
<pre><code>decA5 = []
decA5 += [ReLUINSConvTranspose2d(tch, tch//2, kernel_size=3, stride=2, padding=1, output_padding=1)]
tch = tch//2
decA5 += [ReLUINSConvTranspose2d(tch, tch//2, kernel_size=3, stride=2, padding=1, output_padding=1)]
tch = tch//2
decA5 += [nn.ConvTranspose2d(tch, output_dim_a, kernel_size=1, stride=1, padding=0)]
decA5 += [nn.Tanh()]
self.decA5 = nn.Sequential(*decA5)

tch = ini_tch
self.decB1 = MisINSResBlock(tch, tch_add)
self.decB2 = MisINSResBlock(tch, tch_add)
self.decB3 = MisINSResBlock(tch, tch_add)
self.decB4 = MisINSResBlock(tch, tch_add)
decB5 = []
decB5 += [ReLUINSConvTranspose2d(tch, tch//2, kernel_size=3, stride=2, padding=1, output_padding=1)]
tch = tch//2
decB5 += [ReLUINSConvTranspose2d(tch, tch//2, kernel_size=3, stride=2, padding=1, output_padding=1)]
tch = tch//2
decB5 += [nn.ConvTranspose2d(tch, output_dim_b, kernel_size=1, stride=1, padding=0)]
decB5 += [nn.Tanh()]
self.decB5 = nn.Sequential(*decB5)

self.mlpA = nn.Sequential(
    nn.Linear(8, 256),
    nn.ReLU(inplace=False),
    nn.Linear(256, 256),
    nn.ReLU(inplace=False),
    nn.Linear(256, tch_add*4))
self.mlpB = nn.Sequential(
    nn.Linear(8, 256),
    nn.ReLU(inplace=False),
    nn.Linear(256, 256),
    nn.ReLU(inplace=False),
    nn.Linear(256, tch_add*4))
return
</code></pre>
<p>def forward_a(self, x, z):<br>
z = self.mlpA(z)<br>
z1, z2, z3, z4 = torch.split(z, self.tch_add, dim=1)<br>
z1, z2, z3, z4 = z1.contiguous(), z2.contiguous(), z3.contiguous(), z4.contiguous()<br>
out1 = self.decA1(x, z1)<br>
out2 = self.decA2(out1, z2)<br>
out3 = self.decA3(out2, z3)<br>
out4 = self.decA4(out3, z4)<br>
out = self.decA5(out4)<br>
return out</p>
<p>def forward_b(self, x, z):<br>
z = self.mlpB(z)<br>
z1, z2, z3, z4 = torch.split(z, self.tch_add, dim=1)<br>
z1, z2, z3, z4 = z1.contiguous(), z2.contiguous(), z3.contiguous(), z4.contiguous()<br>
out1 = self.decB1(x, z1)<br>
out2 = self.decB2(out1, z2)<br>
out3 = self.decB3(out2, z3)<br>
out4 = self.decB4(out3, z4)<br>
out = self.decB5(out4)<br>
return out</p>
<p>class G_concat(nn.Module):<br>
def <strong>init</strong>(self, output_dim_a, output_dim_b, nz):<br>
super(G_concat, self).<strong>init</strong>()<br>
self.nz = nz<br>
tch = 256<br>
dec_share = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
dec_share += [INSResBlock(tch, tch)]<br>
self.dec_share = nn.Sequential(<em>dec_share)<br>
tch = 256+self.nz<br>
decA1 = []<br>
for i in range(0, 3):<br>
decA1 += [INSResBlock(tch, tch)]<br>
tch = tch + self.nz<br>
decA2 = ReLUINSConvTranspose2d(tch, tch//2, kernel_size=3, stride=2, padding=1, output_padding=1)<br>
tch = tch//2<br>
tch = tch + self.nz<br>
decA3 = ReLUINSConvTranspose2d(tch, tch//2, kernel_size=3, stride=2, padding=1, output_padding=1)<br>
tch = tch//2<br>
tch = tch + self.nz<br>
decA4 = [nn.ConvTranspose2d(tch, output_dim_a, kernel_size=1, stride=1, padding=0)]+[nn.Tanh()]<br>
self.decA1 = nn.Sequential(<em>decA1)<br>
self.decA2 = nn.Sequential(</em>[decA2])<br>
self.decA3 = nn.Sequential(</em>[decA3])<br>
self.decA4 = nn.Sequential(*decA4)</p>
<pre><code>tch = 256+self.nz
decB1 = []
for i in range(0, 3):
  decB1 += [INSResBlock(tch, tch)]
tch = tch + self.nz
decB2 = ReLUINSConvTranspose2d(tch, tch//2, kernel_size=3, stride=2, padding=1, output_padding=1)
tch = tch//2
tch = tch + self.nz
decB3 = ReLUINSConvTranspose2d(tch, tch//2, kernel_size=3, stride=2, padding=1, output_padding=1)
tch = tch//2
tch = tch + self.nz
decB4 = [nn.ConvTranspose2d(tch, output_dim_b, kernel_size=1, stride=1, padding=0)]+[nn.Tanh()]
self.decB1 = nn.Sequential(*decB1)
self.decB2 = nn.Sequential(*[decB2])
self.decB3 = nn.Sequential(*[decB3])
self.decB4 = nn.Sequential(*decB4)
</code></pre>
<p>def forward_a(self, x, z):<br>
out0 = self.dec_share(x)<br>
z_img = z.view(z.size(0), z.size(1), 1, 1).expand(z.size(0), z.size(1), x.size(2), x.size(3))<br>
x_and_z = torch.cat([out0, z_img], 1)<br>
out1 = self.decA1(x_and_z)<br>
z_img2 = z.view(z.size(0), z.size(1), 1, 1).expand(z.size(0), z.size(1), out1.size(2), out1.size(3))<br>
x_and_z2 = torch.cat([out1, z_img2], 1)<br>
out2 = self.decA2(x_and_z2)<br>
z_img3 = z.view(z.size(0), z.size(1), 1, 1).expand(z.size(0), z.size(1), out2.size(2), out2.size(3))<br>
x_and_z3 = torch.cat([out2, z_img3], 1)<br>
out3 = self.decA3(x_and_z3)<br>
z_img4 = z.view(z.size(0), z.size(1), 1, 1).expand(z.size(0), z.size(1), out3.size(2), out3.size(3))<br>
x_and_z4 = torch.cat([out3, z_img4], 1)<br>
out4 = self.decA4(x_and_z4)<br>
return out4</p>
<p>def forward_b(self, x, z):<br>
out0 = self.dec_share(x)<br>
z_img = z.view(z.size(0), z.size(1), 1, 1).expand(z.size(0), z.size(1), x.size(2), x.size(3))<br>
x_and_z = torch.cat([out0,  z_img], 1)<br>
out1 = self.decB1(x_and_z)<br>
z_img2 = z.view(z.size(0), z.size(1), 1, 1).expand(z.size(0), z.size(1), out1.size(2), out1.size(3))<br>
x_and_z2 = torch.cat([out1, z_img2], 1)<br>
out2 = self.decB2(x_and_z2)<br>
z_img3 = z.view(z.size(0), z.size(1), 1, 1).expand(z.size(0), z.size(1), out2.size(2), out2.size(3))<br>
x_and_z3 = torch.cat([out2, z_img3], 1)<br>
out3 = self.decB3(x_and_z3)<br>
z_img4 = z.view(z.size(0), z.size(1), 1, 1).expand(z.size(0), z.size(1), out3.size(2), out3.size(3))<br>
x_and_z4 = torch.cat([out3, z_img4], 1)<br>
out4 = self.decB4(x_and_z4)<br>
return out4</p>
<p>####################################################################<br>
<span class="hashtag-raw">#-------------------------</span> Basic Functions -------------------------<br>
####################################################################<br>
def get_scheduler(optimizer, opts, cur_ep=-1):<br>
if opts.lr_policy == ‘lambda’:<br>
def lambda_rule(ep):<br>
lr_l = 1.0 - max(0, ep - opts.n_ep_decay) / float(opts.n_ep - opts.n_ep_decay + 1)<br>
return lr_l<br>
scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda_rule, last_epoch=cur_ep)<br>
elif opts.lr_policy == ‘step’:<br>
scheduler = lr_scheduler.StepLR(optimizer, step_size=opts.n_ep_decay, gamma=0.1, last_epoch=cur_ep)<br>
else:<br>
return NotImplementedError(‘no such learn rate policy’)<br>
return scheduler</p>
<p>def meanpoolConv(inplanes, outplanes):<br>
sequence = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
sequence += [nn.AvgPool2d(kernel_size=2, stride=2)]<br>
sequence += [nn.Conv2d(inplanes, outplanes, kernel_size=1, stride=1, padding=0, bias=True)]<br>
return nn.Sequential(*sequence)</p>
<p>def convMeanpool(inplanes, outplanes):<br>
sequence = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
sequence += conv3x3(inplanes, outplanes)<br>
sequence += [nn.AvgPool2d(kernel_size=2, stride=2)]<br>
return nn.Sequential(*sequence)</p>
<p>def get_norm_layer(layer_type=‘instance’):<br>
if layer_type == ‘batch’:<br>
norm_layer = functools.partial(nn.BatchNorm2d, affine=True)<br>
elif layer_type == ‘instance’:<br>
norm_layer = functools.partial(nn.InstanceNorm2d, affine=False)<br>
elif layer_type == ‘none’:<br>
norm_layer = None<br>
else:<br>
raise NotImplementedError(‘normalization layer [%s] is not found’ % layer_type)<br>
return norm_layer</p>
<p>def get_non_linearity(layer_type=‘relu’):<br>
if layer_type == ‘relu’:<br>
nl_layer = functools.partial(nn.ReLU, inplace=False)<br>
elif layer_type == ‘lrelu’:<br>
nl_layer = functools.partial(nn.LeakyReLU, negative_slope=0.2, inplace=False)<br>
elif layer_type == ‘elu’:<br>
nl_layer = functools.partial(nn.ELU, inplace=False)<br>
else:<br>
raise NotImplementedError(‘nonlinearity activitation [%s] is not found’ % layer_type)<br>
return nl_layer<br>
def conv3x3(in_planes, out_planes):<br>
return [nn.ReflectionPad2d(1), nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=1, padding=0, bias=True)]</p>
<p>def gaussian_weights_init(m):<br>
classname = m.<strong>class</strong>.<strong>name</strong><br>
if classname.find(‘Conv’) != -1 and classname.find(‘Conv’) == 0:<br>
m.weight.data.normal_(0.0, 0.02)</p>
<p>####################################################################<br>
<span class="hashtag-raw">#--------------------------</span> Basic Blocks --------------------------<br>
####################################################################</p>
<h2><a name="the-code-of-layernorm-is-modified-from-munit-httpsgithubcomnvlabsmunit-1" class="anchor" href="#the-code-of-layernorm-is-modified-from-munit-httpsgithubcomnvlabsmunit-1"></a>The code of LayerNorm is modified from MUNIT (<a href="https://github.com/NVlabs/MUNIT" class="inline-onebox" rel="noopener nofollow ugc">GitHub - NVlabs/MUNIT: Multimodal Unsupervised Image-to-Image Translation</a>)</h2>
<p>class LayerNorm(nn.Module):<br>
def <strong>init</strong>(self, n_out, eps=1e-5, affine=True):<br>
super(LayerNorm, self).<strong>init</strong>()<br>
self.n_out = n_out<br>
self.affine = affine<br>
if self.affine:<br>
self.weight = nn.Parameter(torch.ones(n_out, 1, 1))<br>
self.bias = nn.Parameter(torch.zeros(n_out, 1, 1))<br>
return<br>
def forward(self, x):<br>
normalized_shape = x.size()[1:]<br>
if self.affine:<br>
return F.layer_norm(x, normalized_shape, self.weight.expand(normalized_shape), self.bias.expand(normalized_shape))<br>
else:<br>
return F.layer_norm(x, normalized_shape)</p>
<p>class BasicBlock(nn.Module):<br>
def <strong>init</strong>(self, inplanes, outplanes, norm_layer=None, nl_layer=None):<br>
super(BasicBlock, self).<strong>init</strong>()<br>
layers = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
if norm_layer is not None:<br>
layers += [norm_layer(inplanes)]<br>
layers += [nl_layer()]<br>
layers += conv3x3(inplanes, inplanes)<br>
if norm_layer is not None:<br>
layers += [norm_layer(inplanes)]<br>
layers += [nl_layer()]<br>
layers += [convMeanpool(inplanes, outplanes)]<br>
self.conv = nn.Sequential(*layers)<br>
self.shortcut = meanpoolConv(inplanes, outplanes)<br>
def forward(self, x):<br>
out = self.conv(x) + self.shortcut(x)<br>
return out</p>
<p>class LeakyReLUConv2d(nn.Module):<br>
def <strong>init</strong>(self, n_in, n_out, kernel_size, stride, padding=0, norm=‘None’, sn=False):<br>
super(LeakyReLUConv2d, self).<strong>init</strong>()<br>
model = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
model += [nn.ReflectionPad2d(padding)]<br>
if sn:<br>
model += [spectral_norm(nn.Conv2d(n_in, n_out, kernel_size=kernel_size, stride=stride, padding=0, bias=True))]<br>
else:<br>
model += [nn.Conv2d(n_in, n_out, kernel_size=kernel_size, stride=stride, padding=0, bias=True)]<br>
if ‘norm’ == ‘Instance’:<br>
model += [nn.InstanceNorm2d(n_out, affine=False)]<br>
model += [nn.LeakyReLU(inplace=False)]<br>
self.model = nn.Sequential(*model)<br>
self.model.apply(gaussian_weights_init)<br>
<span class="hashtag-raw">#elif</span> == ‘Group’<br>
def forward(self, x):<br>
return self.model(x)</p>
<p>class ReLUINSConv2d(nn.Module):<br>
def <strong>init</strong>(self, n_in, n_out, kernel_size, stride, padding=0):<br>
super(ReLUINSConv2d, self).<strong>init</strong>()<br>
model = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
model += [nn.ReflectionPad2d(padding)]<br>
model += [nn.Conv2d(n_in, n_out, kernel_size=kernel_size, stride=stride, padding=0, bias=True)]<br>
model += [nn.InstanceNorm2d(n_out, affine=False)]<br>
model += [nn.ReLU(inplace=False)]<br>
self.model = nn.Sequential(*model)<br>
self.model.apply(gaussian_weights_init)<br>
def forward(self, x):<br>
return self.model(x)</p>
<p>class INSResBlock(nn.Module):<br>
def conv3x3(self, inplanes, out_planes, stride=1):<br>
return [nn.ReflectionPad2d(1), nn.Conv2d(inplanes, out_planes, kernel_size=3, stride=stride)]<br>
def <strong>init</strong>(self, inplanes, planes, stride=1, dropout=0.0):<br>
super(INSResBlock, self).<strong>init</strong>()<br>
model = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
model += self.conv3x3(inplanes, planes, stride)<br>
model += [nn.InstanceNorm2d(planes)]<br>
model += [nn.ReLU(inplace=False)]<br>
model += self.conv3x3(planes, planes)<br>
model += [nn.InstanceNorm2d(planes)]<br>
if dropout &gt; 0:<br>
model += [nn.Dropout(p=dropout)]<br>
self.model = nn.Sequential(*model)<br>
self.model.apply(gaussian_weights_init)<br>
def forward(self, x):<br>
residual = x<br>
out = self.model(x)<br>
out += residual<br>
return out</p>
<p>class MisINSResBlock(nn.Module):<br>
def conv3x3(self, dim_in, dim_out, stride=1):<br>
return nn.Sequential(nn.ReflectionPad2d(1), nn.Conv2d(dim_in, dim_out, kernel_size=3, stride=stride))<br>
def conv1x1(self, dim_in, dim_out):<br>
return nn.Conv2d(dim_in, dim_out, kernel_size=1, stride=1, padding=0)<br>
def <strong>init</strong>(self, dim, dim_extra, stride=1, dropout=0.0):<br>
super(MisINSResBlock, self).<strong>init</strong>()<br>
self.conv1 = nn.Sequential(<br>
self.conv3x3(dim, dim, stride),<br>
nn.InstanceNorm2d(dim))<br>
self.conv2 = nn.Sequential(<br>
self.conv3x3(dim, dim, stride),<br>
nn.InstanceNorm2d(dim))<br>
self.blk1 = nn.Sequential(<br>
self.conv1x1(dim + dim_extra, dim + dim_extra),<br>
nn.ReLU(inplace=False),<br>
self.conv1x1(dim + dim_extra, dim),<br>
nn.ReLU(inplace=False))<br>
self.blk2 = nn.Sequential(<br>
self.conv1x1(dim + dim_extra, dim + dim_extra),<br>
nn.ReLU(inplace=False),<br>
self.conv1x1(dim + dim_extra, dim),<br>
nn.ReLU(inplace=False))<br>
model = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
if dropout &gt; 0:<br>
model += [nn.Dropout(p=dropout)]<br>
self.model = nn.Sequential(*model)<br>
self.model.apply(gaussian_weights_init)<br>
self.conv1.apply(gaussian_weights_init)<br>
self.conv2.apply(gaussian_weights_init)<br>
self.blk1.apply(gaussian_weights_init)<br>
self.blk2.apply(gaussian_weights_init)<br>
def forward(self, x, z):<br>
residual = x<br>
z_expand = z.view(z.size(0), z.size(1), 1, 1).expand(z.size(0), z.size(1), x.size(2), x.size(3))<br>
o1 = self.conv1(x)<br>
o2 = self.blk1(torch.cat([o1, z_expand], dim=1))<br>
o3 = self.conv2(o2)<br>
out = self.blk2(torch.cat([o3, z_expand], dim=1))<br>
out += residual<br>
return out</p>
<p>class GaussianNoiseLayer(nn.Module):<br>
def <strong>init</strong>(self,):<br>
super(GaussianNoiseLayer, self).<strong>init</strong>()<br>
def forward(self, x):<br>
if self.training == False:<br>
return x<br>
noise = Variable(torch.randn(x.size()).cuda(x.get_device()))<br>
return x + noise</p>
<p>class ReLUINSConvTranspose2d(nn.Module):<br>
def <strong>init</strong>(self, n_in, n_out, kernel_size, stride, padding, output_padding):<br>
super(ReLUINSConvTranspose2d, self).<strong>init</strong>()<br>
model = <span class="chcklst-box fa fa-square-o fa-fw"></span><br>
model += [nn.ConvTranspose2d(n_in, n_out, kernel_size=kernel_size, stride=stride, padding=padding, output_padding=output_padding, bias=True)]<br>
model += [LayerNorm(n_out)]<br>
model += [nn.ReLU(inplace = False)]<br>
self.model = nn.Sequential(*model)<br>
self.model.apply(gaussian_weights_init)<br>
def forward(self, x):<br>
return self.model(x)</p>
<p>####################################################################<br>
<span class="hashtag-raw">#---------------------</span> Spectral Normalization ---------------------</p>
<h1><a name="this-part-of-code-is-copied-from-pytorch-master-branch-050-2" class="anchor" href="#this-part-of-code-is-copied-from-pytorch-master-branch-050-2"></a>This part of code is copied from pytorch master branch (0.5.0)</h1>
<p>####################################################################<br>
class SpectralNorm(object):<br>
def <strong>init</strong>(self, name=‘weight’, n_power_iterations=1, dim=0, eps=1e-12):<br>
self.name = name<br>
self.dim = dim<br>
if n_power_iterations &lt;= 0:<br>
raise ValueError('Expected n_power_iterations to be positive, but ’<br>
‘got n_power_iterations={}’.format(n_power_iterations))<br>
self.n_power_iterations = n_power_iterations<br>
self.eps = eps<br>
def compute_weight(self, module):<br>
weight = getattr(module, self.name + ‘_orig’)<br>
u = getattr(module, self.name + ‘_u’)<br>
weight_mat = weight<br>
if self.dim != 0:<br>
# permute dim to front<br>
weight_mat = weight_mat.permute(self.dim,<br>
*[d for d in range(weight_mat.dim()) if d != self.dim])<br>
height = weight_mat.size(0)<br>
weight_mat = weight_mat.reshape(height, -1)<br>
with torch.no_grad():<br>
for _ in range(self.n_power_iterations):<br>
v = F.normalize(torch.matmul(weight_mat.t(), u), dim=0, eps=self.eps)<br>
u = F.normalize(torch.matmul(weight_mat, v), dim=0, eps=self.eps)<br>
sigma = torch.dot(u, torch.matmul(weight_mat, v))<br>
weight = weight / sigma<br>
return weight, u<br>
def remove(self, module):<br>
weight = getattr(module, self.name)<br>
delattr(module, self.name)<br>
delattr(module, self.name + ‘_u’)<br>
delattr(module, self.name + ‘_orig’)<br>
module.register_parameter(self.name, torch.nn.Parameter(weight))<br>
def <strong>call</strong>(self, module, inputs):<br>
if module.training:<br>
weight, u = self.compute_weight(module)<br>
setattr(module, self.name, weight)<br>
setattr(module, self.name + '<em>u’, u)<br>
else:<br>
r_g = getattr(module, self.name + '<em>orig’).requires_grad<br>
getattr(module, self.name).detach</em>().requires_grad</em>(r_g)</p>
<p><span class="mention">@staticmethod</span><br>
def apply(module, name, n_power_iterations, dim, eps):<br>
fn = SpectralNorm(name, n_power_iterations, dim, eps)<br>
weight = module.<em>parameters[name]<br>
height = weight.size(dim)<br>
u = F.normalize(weight.new_empty(height).normal</em>(0, 1), dim=0, eps=fn.eps)<br>
delattr(module, fn.name)<br>
module.register_parameter(fn.name + “_orig”, weight)<br>
module.register_buffer(fn.name, weight.data)<br>
module.register_buffer(fn.name + “_u”, u)<br>
module.register_forward_pre_hook(fn)<br>
return fn</p>
<p>def spectral_norm(module, name=‘weight’, n_power_iterations=1, eps=1e-12, dim=None):<br>
if dim is None:<br>
if isinstance(module, (torch.nn.ConvTranspose1d,<br>
torch.nn.ConvTranspose2d,<br>
torch.nn.ConvTranspose3d)):<br>
dim = 1<br>
else:<br>
dim = 0<br>
SpectralNorm.apply(module, name, n_power_iterations, dim, eps)<br>
return module</p>
<p>def remove_spectral_norm(module, name=‘weight’):<br>
for k, hook in module._forward_pre_hooks.items():<br>
if isinstance(hook, SpectralNorm) and hook.name == name:<br>
hook.remove(module)<br>
del module._forward_pre_hooks[k]<br>
return module<br>
raise ValueError(“spectral_norm of ‘{}’ not found in {}”.format(name, module))</p>

AlphaBetaGamma96: <aside class="quote no-group" data-username="Sarmad_GTU" data-post="7" data-topic="163088">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/sarmad_gtu/48/41574_2.png" class="avatar"> Sarmad_GTU:</div>
<blockquote>
<p>out += residual</p>
</blockquote>
</aside>
<p>replace this with <code>out = out + residual</code>, this is an in-place operation.</p>

Sarmad_GTU: <p>I changed them and same issue:</p>
<p>Epoch: 0/50:   0%|          | 2/1573 [00:10&lt;3:09:45,  7.25s/it]C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\torch\autograd_<em>init</em>_.py:173: UserWarning: Error detected in ReluBackward0. Traceback of forward call that caused the error:<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\runpy.py”, line 194, in _run_module_as_main<br>
return _run_code(code, main_globals, None,<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\runpy.py”, line 87, in _run_code<br>
exec(code, run_globals)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\ipykernel_launcher.py”, line 16, in <br>
app.launch_new_instance()<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\traitlets\config\application.py”, line 846, in launch_instance<br>
app.start()<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\ipykernel\kernelapp.py”, line 677, in start<br>
self.io_loop.start()<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\tornado\platform\asyncio.py”, line 199, in start<br>
self.asyncio_loop.run_forever()<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\asyncio\base_events.py”, line 570, in run_forever<br>
self._run_once()<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\asyncio\base_events.py”, line 1859, in _run_once<br>
handle._run()<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\asyncio\events.py”, line 81, in _run<br>
self._context.run(self._callback, *self._args)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\ipykernel\kernelbase.py”, line 471, in dispatch_queue<br>
await self.process_one()<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\ipykernel\kernelbase.py”, line 460, in process_one<br>
await dispatch(*args)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\ipykernel\kernelbase.py”, line 367, in dispatch_shell<br>
await result<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\ipykernel\kernelbase.py”, line 662, in execute_request<br>
reply_content = await reply_content<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\ipykernel\ipkernel.py”, line 360, in do_execute<br>
res = shell.run_cell(code, store_history=store_history, silent=silent)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\ipykernel\zmqshell.py”, line 532, in run_cell<br>
return super().run_cell(*args, **kwargs)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\IPython\core\interactiveshell.py”, line 2863, in run_cell<br>
result = self._run_cell(<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\IPython\core\interactiveshell.py”, line 2909, in _run_cell<br>
return runner(coro)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\IPython\core\async_helpers.py”, line 129, in <em>pseudo_sync_runner<br>
coro.send(None)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\IPython\core\interactiveshell.py”, line 3106, in run_cell_async<br>
has_raised = await self.run_ast_nodes(code_ast.body, cell_name,<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\IPython\core\interactiveshell.py”, line 3309, in run_ast_nodes<br>
if await self.run_code(code, result, async</em>=asy):<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\IPython\core\interactiveshell.py”, line 3369, in run_code<br>
exec(code_obj, self.user_global_ns, self.user_ns)<br>
File “C:\Users\gtu\AppData\Local\Temp\ipykernel_9588\2562760679.py”, line 17, in &lt;cell line: 2&gt;<br>
model.update_D(images_a, images_b)<br>
File “C:\Users\gtu\Project_All\DRIT-master\model.py”, line 157, in update_D<br>
self.forward()<br>
File “C:\Users\gtu\Project_All\DRIT-master\model.py”, line 135, in forward<br>
self.fake_B_recon = self.gen.forward_b(self.z_content_recon_b, self.z_attr_recon_b)<br>
File “C:\Users\gtu\Project_All\DRIT-master\networks.py”, line 233, in forward_b<br>
out4 = self.decB4(out3, z4)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\torch\nn\modules\module.py”, line 1110, in _call_impl<br>
return forward_call(*input, **kwargs)<br>
File “C:\Users\gtu\Project_All\DRIT-master\networks.py”, line 424, in forward<br>
out = self.blk2(torch.cat([o3, z_expand], dim=1))<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\torch\nn\modules\module.py”, line 1110, in _call_impl<br>
return forward_call(*input, **kwargs)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\torch\nn\modules\container.py”, line 141, in forward<br>
input = module(input)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\torch\nn\modules\module.py”, line 1110, in _call_impl<br>
return forward_call(*input, **kwargs)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\torch\nn\modules\activation.py”, line 98, in forward<br>
return F.relu(input, inplace=self.inplace)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\torch\nn\functional.py”, line 1442, in relu<br>
result = torch.relu(input)<br>
(Triggered internally at  C:\cb\pytorch_1000000000000\work\torch\csrc\autograd\python_anomaly_mode.cpp:104.)<br>
Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass</p>

AlphaBetaGamma96: <p>Did you replace all the <code>+=</code> usage in the share code? So for example when placing <code>nn.Module</code> objects within a list?</p>
<aside class="quote no-group" data-username="Sarmad_GTU" data-post="7" data-topic="163088">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/sarmad_gtu/48/41574_2.png" class="avatar"> Sarmad_GTU:</div>
<blockquote>
<p>encA_c += [ReLUINSConv2d(tch, tch * 2, kernel_size=3, stride=2, padding=1)]</p>
</blockquote>
</aside>
<p>Also, I see you’re running this within ipython, make sure you re-run all previous cells otherwise your code won’t have updated.</p>
<aside class="quote no-group" data-username="Sarmad_GTU" data-post="9" data-topic="163088">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/sarmad_gtu/48/41574_2.png" class="avatar"> Sarmad_GTU:</div>
<blockquote>
<p>self.fake_B_recon = self.gen.forward_b(self.z_content_recon_b, self.z_attr_recon_b)</p>
</blockquote>
</aside>
<p>Whereabouts is defined, because it seems that this is the function triggers the in-place call somewhere. It’s defined within <code>File “C:\Users\gtu\Project_All\DRIT-master\model.py”, line 135, in forward</code> Can you share that code only?</p>

Sarmad_GTU: <p>Now the train has progressed for one epoch but still shows a new error:</p>
<h2>
<a name="epoch-050-87-13671573-11030113-279itscusersgtuanaconda3envsdeeplearninglibsite-packagestorchautograd__init__py173-userwarning-error-detected-in-convolutionbackward0-traceback-of-forward-call-that-caused-the-error-file-cusersgtuanaconda3envsdeeplearninglibrunpypy-line-194-in-_run_module_as_main-return-_run_codecode-main_globals-none-file-cusersgtuanaconda3envsdeeplearninglibrunpypy-line-87-in-_run_code-execcode-run_globals-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipykernel_launcherpy-line-16-in-module-applaunch_new_instance-file-cusersgtuanaconda3envsdeeplearninglibsite-packagestraitletsconfigapplicationpy-line-846-in-launch_instance-appstart-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipykernelkernelapppy-line-677-in-start-selfio_loopstart-file-cusersgtuanaconda3envsdeeplearninglibsite-packagestornadoplatformasynciopy-line-199-in-start-selfasyncio_looprun_forever-file-cusersgtuanaconda3envsdeeplearninglibasynciobase_eventspy-line-570-in-run_forever-self_run_once-file-cusersgtuanaconda3envsdeeplearninglibasynciobase_eventspy-line-1859-in-_run_once-handle_run-file-cusersgtuanaconda3envsdeeplearninglibasyncioeventspy-line-81-in-_run-self_contextrunself_callback-self_args-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipykernelkernelbasepy-line-471-in-dispatch_queue-await-selfprocess_one-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipykernelkernelbasepy-line-460-in-process_one-await-dispatchargs-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipykernelkernelbasepy-line-367-in-dispatch_shell-await-result-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipykernelkernelbasepy-line-662-in-execute_request-reply_content-await-reply_content-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipykernelipkernelpy-line-360-in-do_execute-res-shellrun_cellcode-store_historystore_history-silentsilent-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipykernelzmqshellpy-line-532-in-run_cell-return-superrun_cellargs-kwargs-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipythoncoreinteractiveshellpy-line-2863-in-run_cell-result-self_run_cell-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipythoncoreinteractiveshellpy-line-2909-in-_run_cell-return-runnercoro-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipythoncoreasync_helperspy-line-129-in-_pseudo_sync_runner-corosendnone-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipythoncoreinteractiveshellpy-line-3106-in-run_cell_async-has_raised-await-selfrun_ast_nodescode_astbody-cell_name-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipythoncoreinteractiveshellpy-line-3309-in-run_ast_nodes-if-await-selfrun_codecode-result-async_asy-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipythoncoreinteractiveshellpy-line-3369-in-run_code-execcode_obj-selfuser_global_ns-selfuser_ns-file-cusersgtuappdatalocaltempipykernel_129922562760679py-line-17-in-cell-line-2-modelupdate_dimages_a-images_b-file-cusersgtuproject_alldrit-mastermodelpy-line-160-in-update_d-selfforward-file-cusersgtuproject_alldrit-mastermodelpy-line-128-in-forward-output_fakeb-selfgenforward_binput_content_forb-input_attr_forb-file-cusersgtuproject_alldrit-masternetworkspy-line-230-in-forward_b-out1-selfdecb1x-z1-file-cusersgtuanaconda3envsdeeplearninglibsite-packagestorchnnmodulesmodulepy-line-1110-in-_call_impl-return-forward_callinput-kwargs-file-cusersgtuproject_alldrit-masternetworkspy-line-422-in-forward-o2-selfblk1torchcato1-z_expand-dim1-file-cusersgtuanaconda3envsdeeplearninglibsite-packagestorchnnmodulesmodulepy-line-1110-in-_call_impl-return-forward_callinput-kwargs-file-cusersgtuanaconda3envsdeeplearninglibsite-packagestorchnnmodulescontainerpy-line-141-in-forward-input-moduleinput-file-cusersgtuanaconda3envsdeeplearninglibsite-packagestorchnnmodulesmodulepy-line-1110-in-_call_impl-return-forward_callinput-kwargs-file-cusersgtuanaconda3envsdeeplearninglibsite-packagestorchnnmodulesconvpy-line-447-in-forward-return-self_conv_forwardinput-selfweight-selfbias-file-cusersgtuanaconda3envsdeeplearninglibsite-packagestorchnnmodulesconvpy-line-443-in-_conv_forward-return-fconv2dinput-weight-bias-selfstride-triggered-internally-at-ccbpytorch_1000000000000worktorchcsrcautogradpython_anomaly_modecpp104-variable_execution_enginerun_backward-calls-into-the-c-engine-to-run-the-backward-pass-1" class="anchor" href="#epoch-050-87-13671573-11030113-279itscusersgtuanaconda3envsdeeplearninglibsite-packagestorchautograd__init__py173-userwarning-error-detected-in-convolutionbackward0-traceback-of-forward-call-that-caused-the-error-file-cusersgtuanaconda3envsdeeplearninglibrunpypy-line-194-in-_run_module_as_main-return-_run_codecode-main_globals-none-file-cusersgtuanaconda3envsdeeplearninglibrunpypy-line-87-in-_run_code-execcode-run_globals-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipykernel_launcherpy-line-16-in-module-applaunch_new_instance-file-cusersgtuanaconda3envsdeeplearninglibsite-packagestraitletsconfigapplicationpy-line-846-in-launch_instance-appstart-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipykernelkernelapppy-line-677-in-start-selfio_loopstart-file-cusersgtuanaconda3envsdeeplearninglibsite-packagestornadoplatformasynciopy-line-199-in-start-selfasyncio_looprun_forever-file-cusersgtuanaconda3envsdeeplearninglibasynciobase_eventspy-line-570-in-run_forever-self_run_once-file-cusersgtuanaconda3envsdeeplearninglibasynciobase_eventspy-line-1859-in-_run_once-handle_run-file-cusersgtuanaconda3envsdeeplearninglibasyncioeventspy-line-81-in-_run-self_contextrunself_callback-self_args-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipykernelkernelbasepy-line-471-in-dispatch_queue-await-selfprocess_one-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipykernelkernelbasepy-line-460-in-process_one-await-dispatchargs-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipykernelkernelbasepy-line-367-in-dispatch_shell-await-result-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipykernelkernelbasepy-line-662-in-execute_request-reply_content-await-reply_content-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipykernelipkernelpy-line-360-in-do_execute-res-shellrun_cellcode-store_historystore_history-silentsilent-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipykernelzmqshellpy-line-532-in-run_cell-return-superrun_cellargs-kwargs-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipythoncoreinteractiveshellpy-line-2863-in-run_cell-result-self_run_cell-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipythoncoreinteractiveshellpy-line-2909-in-_run_cell-return-runnercoro-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipythoncoreasync_helperspy-line-129-in-_pseudo_sync_runner-corosendnone-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipythoncoreinteractiveshellpy-line-3106-in-run_cell_async-has_raised-await-selfrun_ast_nodescode_astbody-cell_name-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipythoncoreinteractiveshellpy-line-3309-in-run_ast_nodes-if-await-selfrun_codecode-result-async_asy-file-cusersgtuanaconda3envsdeeplearninglibsite-packagesipythoncoreinteractiveshellpy-line-3369-in-run_code-execcode_obj-selfuser_global_ns-selfuser_ns-file-cusersgtuappdatalocaltempipykernel_129922562760679py-line-17-in-cell-line-2-modelupdate_dimages_a-images_b-file-cusersgtuproject_alldrit-mastermodelpy-line-160-in-update_d-selfforward-file-cusersgtuproject_alldrit-mastermodelpy-line-128-in-forward-output_fakeb-selfgenforward_binput_content_forb-input_attr_forb-file-cusersgtuproject_alldrit-masternetworkspy-line-230-in-forward_b-out1-selfdecb1x-z1-file-cusersgtuanaconda3envsdeeplearninglibsite-packagestorchnnmodulesmodulepy-line-1110-in-_call_impl-return-forward_callinput-kwargs-file-cusersgtuproject_alldrit-masternetworkspy-line-422-in-forward-o2-selfblk1torchcato1-z_expand-dim1-file-cusersgtuanaconda3envsdeeplearninglibsite-packagestorchnnmodulesmodulepy-line-1110-in-_call_impl-return-forward_callinput-kwargs-file-cusersgtuanaconda3envsdeeplearninglibsite-packagestorchnnmodulescontainerpy-line-141-in-forward-input-moduleinput-file-cusersgtuanaconda3envsdeeplearninglibsite-packagestorchnnmodulesmodulepy-line-1110-in-_call_impl-return-forward_callinput-kwargs-file-cusersgtuanaconda3envsdeeplearninglibsite-packagestorchnnmodulesconvpy-line-447-in-forward-return-self_conv_forwardinput-selfweight-selfbias-file-cusersgtuanaconda3envsdeeplearninglibsite-packagestorchnnmodulesconvpy-line-443-in-_conv_forward-return-fconv2dinput-weight-bias-selfstride-triggered-internally-at-ccbpytorch_1000000000000worktorchcsrcautogradpython_anomaly_modecpp104-variable_execution_enginerun_backward-calls-into-the-c-engine-to-run-the-backward-pass-1"></a>Epoch: 0/50:  87%|████████▋ | 1367/1573 [11:03&lt;01:13,  2.79it/s]C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\torch\autograd_<em>init</em>_.py:173: UserWarning: Error detected in ConvolutionBackward0. Traceback of forward call that caused the error:<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\runpy.py”, line 194, in _run_module_as_main<br>
return _run_code(code, main_globals, None,<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\runpy.py”, line 87, in _run_code<br>
exec(code, run_globals)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\ipykernel_launcher.py”, line 16, in <br>
app.launch_new_instance()<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\traitlets\config\application.py”, line 846, in launch_instance<br>
app.start()<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\ipykernel\kernelapp.py”, line 677, in start<br>
self.io_loop.start()<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\tornado\platform\asyncio.py”, line 199, in start<br>
self.asyncio_loop.run_forever()<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\asyncio\base_events.py”, line 570, in run_forever<br>
self._run_once()<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\asyncio\base_events.py”, line 1859, in _run_once<br>
handle._run()<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\asyncio\events.py”, line 81, in _run<br>
self._context.run(self._callback, *self._args)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\ipykernel\kernelbase.py”, line 471, in dispatch_queue<br>
await self.process_one()<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\ipykernel\kernelbase.py”, line 460, in process_one<br>
await dispatch(*args)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\ipykernel\kernelbase.py”, line 367, in dispatch_shell<br>
await result<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\ipykernel\kernelbase.py”, line 662, in execute_request<br>
reply_content = await reply_content<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\ipykernel\ipkernel.py”, line 360, in do_execute<br>
res = shell.run_cell(code, store_history=store_history, silent=silent)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\ipykernel\zmqshell.py”, line 532, in run_cell<br>
return super().run_cell(*args, **kwargs)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\IPython\core\interactiveshell.py”, line 2863, in run_cell<br>
result = self._run_cell(<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\IPython\core\interactiveshell.py”, line 2909, in _run_cell<br>
return runner(coro)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\IPython\core\async_helpers.py”, line 129, in <em>pseudo_sync_runner<br>
coro.send(None)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\IPython\core\interactiveshell.py”, line 3106, in run_cell_async<br>
has_raised = await self.run_ast_nodes(code_ast.body, cell_name,<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\IPython\core\interactiveshell.py”, line 3309, in run_ast_nodes<br>
if await self.run_code(code, result, async</em>=asy):<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\IPython\core\interactiveshell.py”, line 3369, in run_code<br>
exec(code_obj, self.user_global_ns, self.user_ns)<br>
File “C:\Users\gtu\AppData\Local\Temp\ipykernel_12992\2562760679.py”, line 17, in &lt;cell line: 2&gt;<br>
model.update_D(images_a, images_b)<br>
File “C:\Users\gtu\Project_All\DRIT-master\model.py”, line 160, in update_D<br>
self.forward()<br>
File “C:\Users\gtu\Project_All\DRIT-master\model.py”, line 128, in forward<br>
output_fakeB = self.gen.forward_b(input_content_forB, input_attr_forB)<br>
File “C:\Users\gtu\Project_All\DRIT-master\networks.py”, line 230, in forward_b<br>
out1 = self.decB1(x, z1)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\torch\nn\modules\module.py”, line 1110, in _call_impl<br>
return forward_call(*input, **kwargs)<br>
File “C:\Users\gtu\Project_All\DRIT-master\networks.py”, line 422, in forward<br>
o2 = self.blk1(torch.cat([o1, z_expand], dim=1))<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\torch\nn\modules\module.py”, line 1110, in _call_impl<br>
return forward_call(*input, **kwargs)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\torch\nn\modules\container.py”, line 141, in forward<br>
input = module(input)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\torch\nn\modules\module.py”, line 1110, in _call_impl<br>
return forward_call(*input, **kwargs)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\torch\nn\modules\conv.py”, line 447, in forward<br>
return self._conv_forward(input, self.weight, self.bias)<br>
File “C:\Users\gtu\anaconda3\envs\Deeplearning\lib\site-packages\torch\nn\modules\conv.py”, line 443, in _conv_forward<br>
return F.conv2d(input, weight, bias, self.stride,<br>
(Triggered internally at  C:\cb\pytorch_1000000000000\work\torch\csrc\autograd\python_anomaly_mode.cpp:104.)<br>
Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass</h2>
<p>RuntimeError                              Traceback (most recent call last)<br>
Input In [6], in &lt;cell line: 2&gt;()<br>
16             else:<br>
17                 model.update_D(images_a, images_b)<br>
—&gt; 18                 model.update_EG()<br>
20 #         print(‘total_it: %d (ep %d, it %d), lr %08f’ % (total_it, ep, it, model.gen_opt.param_groups[0][‘lr’]))<br>
21             total_it += 1</p>
<p>File ~\Project_All\DRIT-master\model.py:241, in DRIT.update_EG(self)<br>
236 loss_G = loss_G_GAN_A + loss_G_GAN_B + loss_G_GAN_Acontent + loss_G_GAN_Bcontent + loss_G_L1_AA + loss_G_L1_BB + loss_G_L1_A + loss_G_L1_B<br>
237 <span class="hashtag">#loss_G</span> = loss_G_GAN_A + loss_G_GAN_B + loss_G_GAN_Acontent + loss_G_GAN_Bcontent + loss_G_L1_AA + loss_G_L1_BB<br>
238<br>
239<br>
240 <span class="hashtag">#do</span> backward()<br>
 → 241 loss_G.backward(retain_graph=True)<br>
242 <span class="hashtag">#self</span>.backward_EG()<br>
243<br>
244<br>
245 # do optimisation<br>
246 self.enc_c_opt.step()</p>
<p>File ~\anaconda3\envs\Deeplearning\lib\site-packages\torch_tensor.py:363, in Tensor.backward(self, gradient, retain_graph, create_graph, inputs)<br>
354 if has_torch_function_unary(self):<br>
355     return handle_torch_function(<br>
356         Tensor.backward,<br>
357         (self,),<br>
(…)<br>
361         create_graph=create_graph,<br>
362         inputs=inputs)<br>
 → 363 torch.autograd.backward(self, gradient, retain_graph, create_graph, inputs=inputs)</p>
<p>File ~\anaconda3\envs\Deeplearning\lib\site-packages\torch\autograd_<em>init</em>_.py:173, in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)<br>
168     retain_graph = create_graph<br>
170 # The reason we repeat same the comment below is that<br>
171 # some Python versions print out the first line of a multi-line function<br>
172 # calls in the traceback and some print out the last line<br>
 → 173 Variable.<em>execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass<br>
174     tensors, grad_tensors</em>, retain_graph, create_graph, inputs,<br>
175     allow_unreachable=True, accumulate_grad=True)</p>
<p>RuntimeError: Function ‘ConvolutionBackward0’ returned nan values in its 1th output.</p>

AlphaBetaGamma96: <aside class="quote no-group" data-username="Sarmad_GTU" data-post="11" data-topic="163088">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/sarmad_gtu/48/41574_2.png" class="avatar"> Sarmad_GTU:</div>
<blockquote>
<p>o2 = self.blk1(torch.cat([o1, z_expand], dim=1))</p>
</blockquote>
</aside>
<p>Can you check the <code>self.blk1</code> definition for any Conv layers? That seems to be the last line before the error emerges?</p>

Sarmad_GTU: <aside class="quote no-group" data-username="AlphaBetaGamma96" data-post="12" data-topic="163088">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/letter_avatar_proxy/v4/letter/a/3da27b/48.png" class="avatar"> AlphaBetaGamma96:</div>
<blockquote>
<p>Can you check the <code>self.blk1</code> definition for any Conv layers? That seems to be the last line before the error emerges?</p>
</blockquote>
</aside>
<p>it is just:</p>
<p>self.blk1 = nn.Sequential(<br>
self.conv1x1(dim + dim_extra, dim + dim_extra),<br>
nn.ReLU(inplace=False),<br>
self.conv1x1(dim + dim_extra, dim),<br>
nn.ReLU(inplace=False))</p>
<p>def conv1x1(self, dim_in, dim_out):<br>
return nn.Conv2d(dim_in, dim_out, kernel_size=1, stride=1, padding=0)</p>