Validation, Accuracy and Batch Size

ykukkim: <p>Hi all,</p>
<p>I am trying to train my model but I am having difficulties as to get validation, accuracy right.</p>
<p>The accuracy of the model changes depending on the size of batch, the lower the batch size is the more accurate it gets. Perhaps my accuracy method is not correct. For instance, I get 100% accruacy for 10 batch size, but if I increase to 64 then it would go down to 5%, so there must be something wrong here.</p>
<p>My model(LSTM) is for binary classification for a time-series data; datasets are imbalance, meaning that I do not have a constant length of the dataset as well as there are more 0’s than 1’s, approximately 110:1.</p>
<p>Furthermore, I don’t quite know how to improve my training as the loss is quite big (~0.9).</p>
<p>Here I attach links for cells:</p>
<p><a href="https://colab.research.google.com/drive/1Uxg9ivBHBKF23FYE6Ei4DpuGQdANNlf5#scrollTo=ZUxmkRUQmXUo&amp;line=1&amp;uniqifier=1" rel="nofollow noopener">Neural Network Model (LSTM)</a></p>
<p><a href="https://colab.research.google.com/drive/1Uxg9ivBHBKF23FYE6Ei4DpuGQdANNlf5#scrollTo=HzmBiauqnWcV&amp;line=28&amp;uniqifier=1" rel="nofollow noopener">Training Structure</a></p>
<p>Any comments will be greatly appreciated!</p>
<p>Thanks!</p>

ilhamfp: <p>Hi <a class="mention" href="/u/ykukkim">@ykukkim</a>,</p>
<p>Your Google colab link is restricted.</p>

ykukkim: <p>Thank you for your reply!</p>
<pre><code class="lang-auto">class Trainer:

    def __init__(self, config):

        self.cuda = config.cuda
        self.device = config.device
        self.lr = config.lr
        self.epochs = config.epochs
        self.save_model = config.save_model
        self.batch_size = config.batch_size
        self.log_interval = config.log_interval

        self.globaliter = 0

        self.train_loader = DataLoader(CoolDataset(r"data/csv/train/",config.seq_length,config.input_size),batch_size=self.batch_size, drop_last=True, shuffle=False)
        self.val_loader = DataLoader(CoolDataset(r"data/csv/val/",config.seq_length,config.input_size),batch_size=self.batch_size, drop_last=True, shuffle=False)
        self.test_loader = DataLoader(CoolDataset(r"data/csv/test/",config.seq_length,config.input_size),batch_size=self.batch_size, drop_last=True, shuffle = False)
        self.model = Network(config).float()
        self.model = self.model.to(self.device)
        self.optimizer = optim.Adam(self.model.parameters(), lr=self.lr)
        self.pos_weight_factor = torch.tensor(110)
        self.criterion = nn.BCEWithLogitsLoss(reduction = 'none',pos_weight=self.pos_weight_factor)

        kwargs = {'num_workers': 4, 'pin_memory': True} if self.cuda else {}

    def train(self, epoch):

        self.model.train()

        for batch_idx, (data, target) in enumerate(self.train_loader):
            self.globaliter += 1
            self.optimizer.zero_grad()
            
            data, target = data.to(self.device), target.to(self.device)
            predictions = self.model(data.float())
            loss_val =  self.criterion(predictions.float(), target.float())
            loss = loss_val.mean()
            loss.backward()

            self.optimizer.step()
            accuracy = 0
            correct_indx = 0
            target_indx = (target == 1).nonzero()
            output_indx = (loss_val &gt;= 50).nonzero()
            if output_indx.shape[0] &gt;= target_indx.shape[0]:
              for i in range(0, (target_indx.shape[0])):
                if (output_indx[i] == target_indx[i]).sum():
                    correct_indx += 1
            elif output_indx.shape[0] &lt; target_indx.shape[0]:
              for i in range(0, output_indx.shape[0]):
                if (output_indx[i] == target_indx[i]).sum():
                    correct_indx += 1

            # pdb.set_trace()
            # for i in range(0, (output_indx.shape[0]-1)):
            #   if output_indx[i] &gt; 50:
            #       correct_indx += 1

            accuracy = correct_indx/target_indx.shape[0]
            if batch_idx % self.log_interval == 0:
              print('\nTrain Epoch: {}\tLoss: {:.6f}\tAccuracy: {:}\t{} \t{} \t{}' 
                    .format(epoch, loss.item(),accuracy * 100,output_indx.shape[0],correct_indx,target_indx.shape[0]))
            
              with train_summary_writer.as_default():
                  summary.scalar('loss', loss.item(), step=self.globaliter)

    def val(self, epoch):
       
        self.model.eval()
        val_loss = 0
        with torch.no_grad():
            for data, target in self.val_loader:

                data, target = data.to(self.device), target.to(self.device)
                predictions = self.model(data.float())
                val_loss += self.criterion(predictions.float(), target.float())
                mean_val_loss = val_loss.mean()

                accuracy = 0
                # correct_indx = 0
                # target_indx = (target == 1).nonzero() 
                # output_indx = (val_loss[target==1])
    
                # for i in range(0, (output_indx.shape[0]-1)):
                #   if output_indx[i] &gt; 50:
                #       correct_indx += 1

                correct_indx = 0
                target_indx = (target == 1).nonzero()
                # pdb.set_trace()
                output_indx = (val_loss &gt;= 50).nonzero()
                if output_indx.shape[0] &gt;= target_indx.shape[0]:
                  for i in range(0, (target_indx.shape[0])):
                    if (output_indx[i] == target_indx[i]).sum():
                        correct_indx += 1
                elif output_indx.shape[0] &lt; target_indx.shape[0]:
                  for i in range(0, output_indx.shape[0]):
                    if (output_indx[i] == target_indx[i]).sum():
                        correct_indx += 1

                accuracy = correct_indx/target_indx.shape[0]

            print('Val Set: Average loss: {:.4f}\tAccuracy: {:}'.format(mean_val_loss,accuracy * 100))

            with val_summary_writer.as_default():
                summary.scalar('val_loss', mean_val_loss.item(), step=self.globaliter)
                summary.scalar('accuracy', accuracy, step=self.globaliter)
          

</code></pre>
<pre><code class="lang-auto">class Network(nn.Module):
    # TO DO
    def __init__(self, config):
        super(Network, self).__init__()

        # Configuration
        self.input_size = config.input_size
        self.hidden_size = config.hidden_size
        self.output_size = config.output_size
        self.batch_size = config.batch_size
        self.num_layers = config.num_layers
        self.seq_length = config.seq_length
        self.device = config.device

        # Model construct
        self.lstm = nn.LSTM(self.input_size, self.hidden_size, self.num_layers, batch_first=True)
        self.dropout = nn.Dropout(0.2)

    def forward(self, x):

        hidden, cell = self.init_hidden()
        out, (hn, cn) = self.lstm(x,(hidden, cell))

        # out = torch.sigmoid(out[:,:,-1])
        return out[:,:,-1]

    def init_hidden(self):
        weight = next((self.parameters())).data
        hidden,cell = (weight.new(self.num_layers, self.batch_size, self.hidden_size).zero_().to(self.device),
                       weight.new(self.num_layers, self.batch_size, self.hidden_size).zero_().to(self.device))

        return hidden, cell

</code></pre>
<pre><code class="lang-auto">model_config = Config(
    cuda=True if torch.cuda.is_available() else False,
    device=torch.device("cuda" if torch.cuda.is_available() else "cpu"),
    input_size=80,
    batch_size=10,
    seq_length = 128,
    hidden_size= 64,
    output_size=1,
    num_layers=3,
    lr=0.01,
    epochs=50,
    log_interval=100,
    save_model=True
)
</code></pre>