Gradient clipping does not prevent exploding loss

arian: <p>Before clipping the gradient value by below lines I could see very big gradient values:<br>
<code>grad_list = [torch.max(x.grad.data).cpu().detach().numpy() for x in net.parameters() if x.grad is not None] print("Maximum Gradient: ", max(grad_list))</code></p>
<p>so I perform gradient clipping:<br>
<code>torch.nn.utils.clip_grad_value_(net.parameters(), 1.05)</code></p>
<p>However I would get 1.05 for for maximum value of gradient, yet loss gets bigger and bigger.<br>
My network:</p>
<pre><code class="lang-auto">import torch
import torch.nn as nn
import numpy as np
import torchvision

class block(nn.Module):

    def __init__(self,begin):
        super(block,self).__init__()

        if begin==True:
            self.cnn1=nn.Conv2d(in_channels=64,out_channels=128, kernel_size=3,stride=1,padding=1)
        else:
            self.cnn1=nn.Conv2d(in_channels=128,out_channels=128, kernel_size=3,stride=1,padding=1)
        nn.init.xavier_normal(self.cnn1.weight) 
        self.bn1=nn.BatchNorm2d(128)
        self.non_linearity1=nn.CELU(alpha=1.0, inplace=False)
        
        self.cnn2=nn.Conv2d(in_channels=128, out_channels=196, kernel_size=3,stride=1,padding=1)
        nn.init.xavier_normal(self.cnn2.weight)
        self.bn2=nn.BatchNorm2d(196)
        self.non_linearity2=nn.CELU(alpha=1.0, inplace=False)
        
        self.cnn3=nn.Conv2d(in_channels=196, out_channels=128, kernel_size=3,stride=1,padding=1)
        nn.init.xavier_normal(self.cnn3.weight)
        self.bn3=nn.BatchNorm2d(128)
        self.non_linearity3=nn.CELU(alpha=1.0, inplace=False)
        
        self.pool=nn.MaxPool2d(kernel_size=2)

    def forward(self,x):
        
        x=self.cnn1(x)
        x=self.bn1(x)
        x=self.non_linearity1(x)
        x=self.cnn2(x)
        x=self.bn2(x)
        x=self.non_linearity2(x)
        x=self.cnn3(x)
        x=self.bn3(x)
        x=self.non_linearity3(x)
        x=self.pool(x)
        return x
    

class Anti_spoof_net_CNN(nn.Module):
  
    def __init__ (self):
        super(Anti_spoof_net_CNN,self).__init__()
            
        self.resize_32 = nn.Upsample(size=32, mode='nearest')
        self.resize_64 = nn.Upsample(size=64, mode='nearest')

        self.cnn0=nn.Conv2d(in_channels=3,out_channels=64, kernel_size=3,stride=1,padding=1)
        nn.init.xavier_normal(self.cnn0.weight) 
        self.bn0=nn.BatchNorm2d(64)
        self.non_linearity0=nn.CELU(alpha=1.0, inplace=False)

        self.block1=block(True)
        self.block2=block(False)
        self.block3=block(False)
        
        #Feature map:
        self.cnn4=nn.Conv2d(in_channels=384,out_channels=128, kernel_size=3,stride=1,padding=1)
        self.cnn5=nn.Conv2d(in_channels=128,out_channels=3, kernel_size=3,stride=1,padding=1)
        self.cnn6=nn.Conv2d(in_channels=3,out_channels=1, kernel_size=3,stride=1,padding=1)
        
        #Depth map:
        self.cnn7=nn.Conv2d(in_channels=384,out_channels=128, kernel_size=3,stride=1,padding=1)
        self.cnn8=nn.Conv2d(in_channels=128,out_channels=64, kernel_size=3,stride=1,padding=1)
        self.cnn9=nn.Conv2d(in_channels=64,out_channels=1, kernel_size=3,stride=1,padding=1)



    def forward(self,x):

        x=self.cnn0(x)
        x=self.bn0(x)
        x=self.non_linearity0(x)
        
        #Block1
        x=self.block1(x)
        X1=self.resize_64(x)
        
        #Block2
        x=self.block2(x)
        X2=x
        
        #Block3:
        x=self.block3(x)
        X3=self.resize_64(x)
        
        X=torch.cat((X1,X2,X3),1)
        
        #Feature map:
        T=self.cnn4(X)
        T=self.cnn5(T)
        T=self.cnn6(T)
        T=self.resize_32(T)
        
        #Depth map:
        D=self.cnn7(X)
        D=self.cnn8(D)
        D=self.cnn9(D)
        D=self.resize_32(D)
        
        return D,T
</code></pre>

Usama_Hasan: <p>Hy Arian,</p>
<aside class="quote no-group" data-username="arian" data-post="1" data-topic="90129">
<div class="title">
<div class="quote-controls"></div>
<img loading="lazy" alt="" width="24" height="24" src="https://discuss.pytorch.org/user_avatar/discuss.pytorch.org/arian/48/23760_2.png" class="avatar"> arian:</div>
<blockquote>
<p>yet loss gets bigger and bigger.</p>
</blockquote>
</aside>
<p>If you’re talking about the training loss that would be weird, but if the validation loss is getting bigger that would have happened due to overfitting.<br>
Please be specific about the loss.</p>

arian: <p>No it was training loss,but the weird thing is I solve the issue by decreasing learning rate from 1e-3 to 1e-5. Thank you for your response.</p>