Title: Binary image classifier in pytorch progress bar and way to check if the training was valid
Tags: python,pytorch

Question: I would like to build and train a binary classifier in PyTorch that reads images from the path process them and trains a classifier using their labels. My images can be found in the following folder:
-data
  - class_1_folder
  - class_2_folder

Hence, to read them in tensors I am doing the following:
PATH = "data/"

transform = transforms.Compose([transforms.Resize(256),
                            transforms.RandomCrop(224),
                            transforms.ToTensor(),
                            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224,0.225])])

dataset = datasets.ImageFolder(PATH, transform=transform)

dataloader = torch.utils.data.DataLoader(dataset, batch_size=356, shuffle=True)
images, labels = next(iter(dataloader))

This code actually reads the images and performs some necessary transformations-preprocess. Next step to create a model and perform the training:
acc = model_train(images, labels, images, labels)

With model_train to be:
import pdb
import torch
from torchvision import datasets, transforms, models
from matplotlib import pyplot as plt
from torchvision.transforms.functional import to_pil_image
import torch.nn as nn
import numpy as np
import torch.optim as optim
import tqdm
import copy
from time import sleep

def my_model():
   device = "cuda" if torch.cuda.is_available() else "cpu"

   model = models.resnet18(pretrained=False)
   num_features = model.fc.in_features
   model.fc = nn.Linear(num_features, 1)  # Binary classifier with 2 output classes

   # Move the model to the device
   model = model.to(device)
   return model

def model_train(X_train, y_train, X_val, y_val):

   model = my_model()
   dtype = torch.FloatTensor
   loss_fn = nn.CrossEntropyLoss().type(dtype) # binary cross entropy
   optimizer = optim.Adam(model.parameters(), lr=0.0001)

   n_epochs = 20   # number of epochs to run
   batch_size = 10  # size of each batch
   batch_start = torch.arange(0, len(X_train), batch_size)

   # Hold the best model
   best_acc = - np.inf   # init to negative infinity
   best_weights = None

   for epoch in range(n_epochs):
       model.train()
       with tqdm.tqdm(batch_start, unit="batch", mininterval=0, disable=True) as bar:
           bar.set_description(f"Epoch {epoch}")
           for start in bar:
               # take a batch
               bar.set_description(f"Epoch {epoch}")
               X_batch = X_train[start:start+batch_size]
               y_batch = y_train[start:start+batch_size]
               # forward pass
               y_pred = model(X_batch)
               y_pred = torch.max(y_pred, 1)[0]
            
               loss = loss_fn(y_pred, y_batch.float())
               # backward pass
               print(loss)
            
               optimizer.zero_grad()
               loss.backward()
               # update weights
               optimizer.step()
               # print progress
               acc = (y_pred.round() == y_batch).float().mean()
               bar.set_postfix(
                   loss=float(loss),
                   acc=float(acc)
               )
            
               bar.set_postfix(loss=loss.item(), accuracy=100. * acc)
               sleep(0.1)
       # evaluate accuracy at end of each epoch
       model.eval()
       y_pred = model(X_val)
       acc = (y_pred.round() == y_val).float().mean()
       acc = float(acc)
       if acc > best_acc:
           best_acc = acc
           best_weights = copy.deepcopy(model.state_dict())
   # restore model and return best accuracy
   torch.save(model.state_dict(), "model/my_model.pth")
   model.load_state_dict(best_weights)
   return best_acc

I am trying to understand how I can correctly portray the progress bar during training and second, how can I validate that the training process took place correctly. For the latter, I have noticed a weird behavior. For class zero I am getting always zero loss while for class one it's between range 13-24. It seems to be incorrect, however, I am sure how to dive deeper!
    tensor(-0., grad_fn=<DivBackward1>)
tensor([-0.0986, -0.0806, -0.0161,  0.0287, -0.0279,  0.0083, -0.0526, -0.1393,
        -0.2082, -0.0141], grad_fn=<MaxBackward0>)
tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
torch.float32
torch.int64
tensor(-0., grad_fn=<DivBackward1>)
tensor([-0.1779,  0.0936, -0.0341, -0.1531, -0.1222, -0.1169, -0.0160, -0.0674,
         0.1230, -0.1181], grad_fn=<MaxBackward0>)
tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
torch.float32
torch.int64
tensor(-0., grad_fn=<DivBackward1>)
tensor([-0.0438, -0.1269, -0.1624, -0.0976, -0.0132, -0.1944, -0.0034, -0.0454,
        -0.1559,  0.0657], grad_fn=<MaxBackward0>)
tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
torch.float32
torch.int64
tensor(-0., grad_fn=<DivBackward1>)
tensor([-0.1655,  0.0222, -0.0801, -0.1390, -0.0905, -0.1472, -0.0395, -0.0180,
        -0.1492,  0.0914], grad_fn=<MaxBackward0>)
tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
torch.float32
torch.int64
tensor(-0., grad_fn=<DivBackward1>)
tensor([-0.7035, -0.1989,  0.0921, -0.1082, -0.2588, -0.3557,  0.3093,  0.0909,
         0.1603,  0.1838], grad_fn=<MaxBackward0>)
tensor([0, 1, 1, 1, 1, 1, 1, 1, 1, 1])
torch.float32
torch.int64
tensor(20.4545, grad_fn=<DivBackward1>)
tensor([-0.4783, -0.1027, -0.0357,  0.0882, -0.2955, -0.0968,  0.3323, -0.0472,
         0.1017, -0.2186], grad_fn=<MaxBackward0>)
tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1])
torch.float32
torch.int64
tensor(23.2550, grad_fn=<DivBackward1>)
tensor([ 0.1554, -0.2664,  0.1419,  0.0203,  0.0895, -0.0085, -0.2867, -0.1957,
        -0.1315, -0.2340], grad_fn=<MaxBackward0>)
tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1])
torch.float32
torch.int64
tensor(23.1584, grad_fn=<DivBackward1>)
tensor([-0.0406, -0.2144,  0.1997,  0.2196, -0.3464,  0.1311, -0.0743, -0.2440,
        -0.1751, -0.2371], grad_fn=<MaxBackward0>)
tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1])
torch.float32
torch.int64
tensor(23.2112, grad_fn=<DivBackward1>)
tensor([-0.0080, -0.1138, -0.1035,  0.0697, -0.1745, -0.1438, -0.2360, -0.1308,
         0.0146,  0.1209], grad_fn=<MaxBackward0>)
tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1])
torch.float32
torch.int64
tensor(23.0853, grad_fn=<DivBackward1>)
tensor([-0.1235,  0.0081, -0.1073, -0.1036, -0.2037, -0.1204, -0.0570, -0.1146,
         0.0849,  0.0798], grad_fn=<MaxBackward0>)
tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1])
torch.float32
torch.int64
tensor(23.0666, grad_fn=<DivBackward1>)
tensor([-0.0660, -0.0832, -0.0414, -0.0334, -0.0123, -0.0203, -0.0549, -0.0747,
        -0.0779, -0.1629], grad_fn=<MaxBackward0>)

What can be wrong in this case?

Answer: From the code, it seems that both training and validation datasets have same samples in them. If I am not wrong, then model gets trained for 20 epochs for a dataset chunk of 356 samples. Before starting to train, you can select any one random sample from a sample batch -> extract raw data back from the dataset tensors -> validate manually if x and y samples match or not? This will ensure that model is being trained on proper data and there is no error in conversion from images to tensors. Secondly check if both training and validation samples are different or same? Another thing I can figure out is that torch.save(model.state_dict(), "model/my_model.pth") is being called after model.load_state_dict(best_weights). So this will save latest weights and then load the model with weights of best accuracy. The weights of best accuracy are not being saved at all!.