The train_loss and the valid_loss will not converge to a local minimum

AlfonsoGiso: <p>Hi everyone,<br>
I hope you can help me. I’m doing a project for the university and I’m having some problems. I’m using HydraPlusNet to solve a problem of PAR and I need to fine tune the net. In particular, from the original 26 classes I need to classify only 2, by training the net with a custom dataset.<br>
During the training the train_loss and the validation_loss will not converge to a local minimum. In particular, the training phase will reach a minimum, i.e 0.1 on train_loss and 0.31 on val_loss but then both of the loss will start to encrease.</p>
<p>This is the code i use to train, that comes from the github repository of HydraPlusNet<br>
import os<br>
import torch<br>
import torch.utils.data as data<br>
from PIL import Image<br>
import matplotlib.pyplot as plt<br>
import torch.utils<br>
import torchvision<br>
import torch.nn as nn<br>
import torch.nn.functional as F<br>
import <a href="http://scipy.io" rel="noopener nofollow ugc">scipy.io</a> as scio<br>
import torchvision.transforms as transforms</p>
<p>from lib.AF import AF<br>
from lib.MNet import MNet<br>
from lib.Hydraplus import HP<br>
from lib import dataload<br>
from tqdm import tqdm</p>
<p>from torch.autograd import Variable<br>
import argparse<br>
import logging<br>
import pdb<br>
import numpy as np<br>
from visdom import Visdom</p>
<p>viz = Visdom()<br>
win = viz.line(<br>
Y=np.array([0.2]),<br>
name=“1”,<br>
opts=dict(title=“Training Loss”)<br>
)<br>
win2 = viz.line(<br>
Y=np.array([0.2]),<br>
name=“1”,<br>
opts=dict(title=“Validation Loss”)<br>
)</p>
<p>def freeze_layers(net):<br>
“”“Function to freeze the first layers of the model.”“”<br>
# List all the parameters<br>
params = list(net.parameters())<br>
total_layers = len(params)<br>
freeze_until = total_layers // 2  # Freeze first half</p>
<pre><code>for i, param in enumerate(params):
    if i &lt; freeze_until:
        param.requires_grad = False
    else:
        param.requires_grad = True
</code></pre>
<p>def parse_args():<br>
parser = argparse.ArgumentParser()<br>
parser.add_argument(‘-m’, help=“choose model”, choices=[‘MNet’, ‘AF1’, ‘AF2’, ‘AF3’, ‘HP’])</p>
<pre><code># pre-trained checkpoint
parser.add_argument('-r', dest='r', help="resume training", default=False)
parser.add_argument('-checkpoint', dest='checkpoint', help="load weight path", default=None)
parser.add_argument('-mpath', dest='mpath', help="load MNet weight path", default=None)
parser.add_argument('-af1path', dest='af1path', help="load AF1 weight path", default=None)
parser.add_argument('-af2path', dest='af2path', help="load AF2 weight path", default=None)
parser.add_argument('-af3path', dest='af3path', help="load AF3 weight path", default=None)

# training hyper-parameters
parser.add_argument('-nw', dest='nw', help="number of workers for dataloader",
                    default=0, type=int)
parser.add_argument('-bs', dest='bs', help="batch size",
                    default=100, type=int)
parser.add_argument('-lr', dest='lr', help="learning rate",
                    default=0.001, type=float)
parser.add_argument('-mGPUs', dest='mGPUs',
                    help='whether use multiple GPUs',
                    action='store_true')

args = parser.parse_args()

return args
</code></pre>
<h1><a name="function-to-save-checkpoint-1" class="anchor" href="#function-to-save-checkpoint-1"></a>Function to save checkpoint</h1>
<p>def checkpoint_save(args_m, state_dict, epoch):<br>
save_path = “./checkpoint/” + args_m + “<em>epoch</em>{}”.format(epoch)<br>
torch.save(state_dict, save_path)<br>
return save_path</p>
<h1><a name="function-to-initialize-weights-2" class="anchor" href="#function-to-initialize-weights-2"></a>Function to initialize weights</h1>
<p>def weight_init(m):<br>
if isinstance(m, nn.Conv2d):<br>
nn.init.xavier_normal_(m.weight.data)</p>
<h1><a name="function-for-early-stopping-3" class="anchor" href="#function-for-early-stopping-3"></a>Function for early stopping</h1>
<p>def validate(net, val_loader, criterion, checkpoint_path):<br>
net.eval()<br>
val_loss = 0.0<br>
total_batches = len(val_loader)<br>
net.load_state_dict(torch.load(checkpoint_path))<br>
with torch.no_grad():<br>
for i, (inputs, labels, _) in enumerate(val_loader):<br>
inputs, labels = inputs.cuda(), labels.cuda()<br>
outputs, confidence = net(inputs)  # Get the primary output<br>
labels_one_hot = F.one_hot(labels, num_classes=2).float()<br>
val_loss += criterion(outputs, labels_one_hot).item()<br>
<span class="hashtag-raw">#if</span> i % 1000 == 0:<br>
return val_loss / len(val_loader)</p>
<p>def main():<br>
args = parse_args()</p>
<pre><code>mytransform = transforms.Compose(
    [transforms.RandomHorizontalFlip(),
     transforms.Resize((299, 299)),
     transforms.ToTensor()]
)

# Load dataset
data_set = dataload.myImageFloder(
    root="/home/alfonso/Desktop/Dumping_Garbage_Dataset.v8i.retinanet/train/train/",
    annotation_file="/home/alfonso/Desktop/Dumping_Garbage_Dataset.v8i.retinanet/train/_annotations.csv",
    transform=mytransform,
    mode='train'
)
validation_set = dataload.myImageFloder(
    root="/home/alfonso/Desktop/Dumping_Garbage_Dataset.v8i.retinanet/valid/valid/",
    annotation_file="/home/alfonso/Desktop/Dumping_Garbage_Dataset.v8i.retinanet/valid/_annotations.csv",
    transform=mytransform,
    mode='valid'
)


# Create data loaders
train_loader = torch.utils.data.DataLoader(data_set, batch_size=args.bs, shuffle=True, num_workers=args.nw)
val_loader = torch.utils.data.DataLoader(validation_set, batch_size=args.bs, shuffle=False, num_workers=args.nw)

print('Image numbers {}'.format(len(data_set)))

# Define the training model
if args.m == 'MNet':
    net = MNet()
    if not args.r:
        net.apply(weight_init)
    freeze_layers(net)
elif 'AF' in args.m:
    net = AF(af_name=args.m)
    if not args.r:
        net.MNet.load_state_dict(torch.load(args.mpath))
    for param in net.MNet.parameters():
        param.requires_grad = False
elif args.m == 'HP':
    net = HP()
    if not args.r:
        net.MNet.load_state_dict(torch.load(args.mpath))
        net.AF1.load_state_dict(torch.load(args.af1path))
        net.AF2.load_state_dict(torch.load(args.af2path))
        net.AF3.load_state_dict(torch.load(args.af3path))
    for param in net.MNet.parameters():
        param.requires_grad = False
    for param in net.AF1.parameters():
        param.requires_grad = False
    for param in net.AF2.parameters():
        param.requires_grad = False
    for param in net.AF3.parameters():
        param.requires_grad = False

# Resume training and load the checkpoint from last training
start_epoch = 1
if args.r:
    net.load_state_dict(torch.load(args.checkpoint))
    numeric_filter = filter(str.isdigit, args.checkpoint)
    numeric_string = "".join(numeric_filter)
    start_epoch = int(numeric_string) + 1

net.cuda()
if args.mGPUs:
    net = nn.DataParallel(net)

net.train()

# Loss function and optimizer
loss_cls_weight = [1, 1]  # Carrying, NotCarrying
weight = torch.Tensor(loss_cls_weight)
criterion = nn.BCEWithLogitsLoss(weight=weight)
criterion.cuda()
optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad, net.parameters()), lr=args.lr, momentum=0.9,  weight_decay=1e-4)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=5, factor=0.5)

# Initialize logging
logging.basicConfig(level=logging.DEBUG, filename='./result/training_log/' + args.m + '.log',
                    datefmt='%Y/%m/%d %H:%M:%S', format='%(asctime)s - %(message)s')
logger = logging.getLogger(__name__)

# Early stopping parameters
best_val_loss = float('inf')
patience = 25
epochs_without_improvement = 0
running_loss = 0.0
val_loss=0.0
# Training loop
for epoch in range(start_epoch, 100):
    total_batches = len(train_loader)

     # Wrap train_loader with tqdm for progress bar
    
    with tqdm(total=total_batches, desc=f'Epoch {epoch}', ncols=100) as pbar:
        for i, data in enumerate(train_loader, 0):
            inputs, labels, _ = data
            inputs, labels = Variable(inputs).cuda(), Variable(labels).cuda()
            optimizer.zero_grad()
            outputs,confidence= net(inputs)  # Ottieni solo x0 dalla rete
            labels_one_hot = F.one_hot(labels, num_classes=2).float()
            loss = criterion(outputs, labels_one_hot)
            loss.backward()
            optimizer.step()
            running_loss = loss.data.item()
            pbar.set_postfix(loss=loss.item(), val_loss=val_loss)  # Update progress bar with current losses
            pbar.update()  # Update progress bar


            #if i % 1000 == 0:
    logger.info('[%d %5d] loss: %.6f' % (epoch, i + 1, loss))
    viz.line(
                X=np.array([epoch + i / 5000.0]),
                Y=np.array([running_loss]),
                win=win,
                update="append"
            )
        # Validation loop
    checkpoint_path=checkpoint_save(args.m, net.state_dict(), epoch)
    val_loss = validate(net, val_loader, criterion,checkpoint_path)
    scheduler.step(val_loss)
    viz.line(
                X=np.array([epoch + i / 5000.0]),
                Y=np.array([val_loss]),
                win=win2,
                update="append")
    logger.info('Validation Loss: %.6f' % val_loss)
    pbar.set_postfix(loss=loss.item(), val_loss=val_loss)  # Update progress bar with current losses
    pbar.update()  # Update progress bar
    if i % 1000 == 0:
        logger.info('[%d %5d] loss: %.6f' % (epoch, i + 1, loss))
        
    # Check for early stopping
    if val_loss &lt; best_val_loss:
        best_val_loss = val_loss
        epochs_without_improvement = 0
        print("\nCheckpoint Saved at epoch "+ str(epoch) + "\n")
        checkpoint_path=checkpoint_save(args.m, net.state_dict(), epoch)
    else:
        epochs_without_improvement += 1
        if epochs_without_improvement &gt;= 10 :
            for param_group in optimizer.param_groups:
                param_group['lr'] = param_group['lr'] * 10**-5
                print("\nLearning Rate decreased\n")
    if epochs_without_improvement &gt;= patience:
        logger.info('Early stopping at epoch %d' % epoch)
        break

    # Save checkpoint
    #if epoch % 1 == 0:
        #if args.mGPUs:
        #    checkpoint_save(args.m, net.module.state_dict(), epoch)
        #else:
        #    checkpoint_save(args.m, net.state_dict(), epoch)
</code></pre>
<p>if <strong>name</strong> == ‘<strong>main</strong>’:<br>
main()</p>
<p>and this is the Net</p>
<p>import torch<br>
import torch.nn as nn<br>
import torch.nn.functional as F</p>
<p>class MNet(nn.Module):<br>
def <strong>init</strong>(self, num_classes=2, feat_out=False):<br>
super(MNet, self).<strong>init</strong>()<br>
self.Conv2d_1_7x7_s2 = BasicConv2d(3, 32, kernel_size=7, stride=2, padding=3)<br>
self.Conv2d_2_1x1 = BasicConv2d(32, 32, kernel_size=1)<br>
self.Conv2d_3_3x3 = BasicConv2d(32, 96, kernel_size=3, padding=1)</p>
<pre><code>    self.incept_block_1 = InceptBlock1()
    self.incept_block_2 = InceptBlock2()
    self.incept_block_3 = InceptBlock3()

    self.final_fc = nn.Linear(512, num_classes)  # Aggiornato il numero di classi qui

    self.feat_out = feat_out  # output intermediate features for AF branches
    
    # Inizializzazione dei pesi
    self.apply(weight_init)

def forward(self, x):
    # 3 x 299 x 299
    x = self.Conv2d_1_7x7_s2(x)
    # 32 x 155 x 155
    x = F.max_pool2d(x, kernel_size=3, stride=2)
    # 32 x 74 x74
    x = self.Conv2d_2_1x1(x)
    # 32 x 74 x74
    x = self.Conv2d_3_3x3(x)
    # 96 x 74 x 74
    x0 = F.max_pool2d(x, kernel_size=3, stride=2)
    # x0=96 x 36 x 36

    x1 = self.incept_block_1(x0)
    # x1=256 x 18 x 18
    x2 = self.incept_block_2(x1)
    # x2 = 502 x 9 x9
    x3 = self.incept_block_3(x2)
    # x3 = 512 x 9 x9

    x = F.avg_pool2d(x3, kernel_size=9, stride=1)
    # 512 x1 x1
    x = F.dropout(x, training=self.training)
    # 1 x 1 x 512
    x = x.view(x.size(0), -1)
    # 512
    pred_class = self.final_fc(x)

    confidence = F.softmax(pred_class,dim=1)
    if self.feat_out:
        return x0, x1, x2, x3
    else:
        return pred_class,confidence
</code></pre>
<p>class BasicConv2d(nn.Module):<br>
def <strong>init</strong>(self, in_channels, out_channels, **kwargs):<br>
super(BasicConv2d, self).<strong>init</strong>()<br>
self.conv = nn.Conv2d(in_channels, out_channels, bias=False, **kwargs)<br>
self.bn = nn.BatchNorm2d(out_channels, eps=0.001)</p>
<pre><code>def forward(self, x):
    x = self.conv(x)
    x = self.bn(x)
    return F.relu(x, inplace=True)
</code></pre>
<p>class InceptBlock1(nn.Module):<br>
def <strong>init</strong>(self):<br>
super(InceptBlock1, self).<strong>init</strong>()<br>
self.module_a = InceptionA(in_channels=96, b_1x1_out=32, b_5x5_1_out=32, b_5x5_2_out=32,<br>
b_3x3_1_out=32, b_3x3_2_out=48, b_3x3_3_out=48, b_pool_out=16)  # C_out =128<br>
self.module_b = InceptionB(in_channels=128, b_1x1_1_out=64, b_1x1_2_out=80,<br>
b_3x3_1_out=32, b_3x3_2_out=48, b_3x3_3_out=48)</p>
<pre><code>def forward(self, x):
    x = self.module_a(x)
    x = self.module_b(x)
    return x
</code></pre>
<p>class InceptBlock2(nn.Module):<br>
def <strong>init</strong>(self):<br>
super(InceptBlock2, self).<strong>init</strong>()<br>
self.module_a = InceptionA(in_channels=256, b_1x1_out=112, b_5x5_1_out=32, b_5x5_2_out=48,<br>
b_3x3_1_out=48, b_3x3_2_out=64, b_3x3_3_out=64, b_pool_out=64)  # C_out = 288<br>
self.module_b = InceptionB(in_channels=288, b_1x1_1_out=64, b_1x1_2_out=86,<br>
b_3x3_1_out=96, b_3x3_2_out=128, b_3x3_3_out=128)  # C_out = 502??!</p>
<pre><code>def forward(self, x):
    x = self.module_a(x)
    x = self.module_b(x)
    return x
</code></pre>
<p>class InceptBlock3(nn.Module):<br>
def <strong>init</strong>(self):<br>
super(InceptBlock3, self).<strong>init</strong>()<br>
self.module_a = InceptionA(in_channels=502, b_1x1_out=176, b_5x5_1_out=96, b_5x5_2_out=160,<br>
b_3x3_1_out=80, b_3x3_2_out=112, b_3x3_3_out=112, b_pool_out=64)<br>
self.module_b = InceptionA(in_channels=512, b_1x1_out=176, b_5x5_1_out=96, b_5x5_2_out=160,<br>
b_3x3_1_out=96, b_3x3_2_out=112, b_3x3_3_out=112, b_pool_out=64)</p>
<pre><code>def forward(self, x):
    x = self.module_a(x)
    x = self.module_b(x)
    return x
</code></pre>
<p>class InceptionA(nn.Module):<br>
def <strong>init</strong>(<br>
self,<br>
in_channels,<br>
b_1x1_out,<br>
b_5x5_1_out,<br>
b_5x5_2_out,<br>
b_3x3_1_out,<br>
b_3x3_2_out,<br>
b_3x3_3_out,<br>
b_pool_out<br>
):<br>
super(InceptionA, self).<strong>init</strong>()<br>
self.branch1x1 = BasicConv2d(in_channels, b_1x1_out, kernel_size=1)  # H_out = H_in</p>
<pre><code>    self.branch5x5_1 = BasicConv2d(in_channels, b_5x5_1_out, kernel_size=1)
    self.branch5x5_2 = BasicConv2d(b_5x5_1_out, b_5x5_2_out, kernel_size=3, padding=1)  # H_out = H_in

    self.branch3x3dbl_1 = BasicConv2d(in_channels, b_3x3_1_out, kernel_size=1)
    self.branch3x3dbl_2 = BasicConv2d(b_3x3_1_out, b_3x3_2_out, kernel_size=3, padding=1)
    self.branch3x3dbl_3 = BasicConv2d(b_3x3_2_out, b_3x3_3_out, kernel_size=3, padding=1)

    self.branch_pool = BasicConv2d(in_channels, b_pool_out, kernel_size=1)

def forward(self, x):
    branch1x1 = self.branch1x1(x)

    branch5x5 = self.branch5x5_1(x)
    branch5x5 = self.branch5x5_2(branch5x5)

    branch3x3dbl = self.branch3x3dbl_1(x)
    branch3x3dbl = self.branch3x3dbl_2(branch3x3dbl)
    branch3x3dbl = self.branch3x3dbl_3(branch3x3dbl)

    branch_pool = F.avg_pool2d(x, kernel_size=1, stride=1)
    branch_pool = self.branch_pool(branch_pool)

    outputs = [branch1x1, branch5x5, branch3x3dbl, branch_pool]
    return torch.cat(outputs, 1)
</code></pre>
<p>class InceptionB(nn.Module):<br>
def <strong>init</strong>(<br>
self,<br>
in_channels,<br>
b_1x1_1_out,<br>
b_1x1_2_out,<br>
b_3x3_1_out,<br>
b_3x3_2_out,<br>
b_3x3_3_out<br>
):<br>
super(InceptionB, self).<strong>init</strong>()<br>
self.branch1x1_1 = BasicConv2d(in_channels, b_1x1_1_out, kernel_size=1)<br>
self.branch1x1_2 = BasicConv2d(b_1x1_1_out, b_1x1_2_out, kernel_size=3, stride=2, padding=1)</p>
<pre><code>    self.branch3x3dbl_1 = BasicConv2d(in_channels, b_3x3_1_out, kernel_size=1)
    self.branch3x3dbl_2 = BasicConv2d(b_3x3_1_out, b_3x3_2_out, kernel_size=3, padding=1)
    self.branch3x3dbl_3 = BasicConv2d(b_3x3_2_out, b_3x3_3_out, kernel_size=3, stride=2, padding=1)

def forward(self, x):
    branch3x3 = self.branch1x1_1(x)
    branch3x3 = self.branch1x1_2(branch3x3)

    branch3x3dbl = self.branch3x3dbl_1(x)
    branch3x3dbl = self.branch3x3dbl_2(branch3x3dbl)
    branch3x3dbl = self.branch3x3dbl_3(branch3x3dbl)

    branch_pool = F.max_pool2d(x, kernel_size=3, stride=2, padding=1)

    outputs = [branch3x3, branch3x3dbl, branch_pool]
    return torch.cat(outputs, 1)
</code></pre>
<p>def weight_init(m):<br>
if isinstance(m, nn.Conv2d):<br>
nn.init.xavier_normal_(m.weight.data)</p>