Training loss keeps going up

Cai: <p>using a simple DNN to predict Tox21 toxicity.<br>
with following model</p>
<pre><code class="lang-auto">class DNN(nn.Module):
    def __init__(self, n_features, n_layers, n_classes):
        super(DNN, self).__init__()
        self.layers = nn.ModuleList()
        in_features = n_features
        n_hidden = in_features//2
        for i in range(n_layers):
            self.layers.append(nn.Linear(in_features, n_hidden))
            self.layers.append(nn.GELU())
            self.layers.append(nn.LayerNorm(n_hidden))
            self.layers.append(nn.Dropout(0.5))
            in_features = n_hidden
            n_hidden = n_hidden // 2
            
        self.layers = nn.Sequential(*self.layers)
  
        self.classifier = nn.Linear(n_hidden *2 , n_classes)
        
        self._init_weights()


    def _init_weights(self):
        for m in self.modules():
            if isinstance(m, nn.Linear):
                nn.init.kaiming_uniform_(m.weight)
                if isinstance(m, nn.Linear) and m.bias is not None:
                   nn.init.constant_(m.bias, 0)
            elif isinstance(m, nn.LayerNorm):
                nn.init.constant_(m.bias, 0)
                nn.init.constant_(m.weight, 1.0)

    def forward(self, x):
        x = self.layers(x)
        output = self.classifier(x)
        return output

model = DNN(n_features=1024, n_layers=3, n_classes=12)
</code></pre>
<p>Tune and train</p>
<pre><code class="lang-auto">#hyperparameters
batch_size = 128
n_epochs = 3
learning_rate = 1e-5
weight_decay = 1e-6
num_workers = 2

# X_train y_train are numpy arrays. 
trainset = TensorDataset(torch.from_numpy(X_train).float(), torch.from_numpy(y_train).float())
testset = TensorDataset(torch.from_numpy(X_train).float(), torch.from_numpy(y_train).float())
trainloader = DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=num_workers)
testloader = DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=num_workers)
loader = (trainloader, testloader)

def train(model, loader, weight_decay):
    model = model.to(device)   
    train_loader, valid_loader = loader
    best_valid = 0.1

    optimizer = opt.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)
    scheduler = CosineAnnealingLR(optimizer, T_max=n_epochs)
    criterion = nn.BCEWithLogitsLoss()
 
    Loss = []
    Loss_valid = []
    # train_scores = []
    # valid_scores = []
    losses= 0
    for epoch in range(n_epochs):
        for _,(x, y)in enumerate(train_loader):
            # model.train()
            x, y = x.to(device), y.to(device)

            output = model(x)
           
            mask = torch.where(y &lt; 0, torch.zeros_like(y), torch.ones_like(y))
            # mask = (y != -1).float()       
            # mask out unknowns
            output = output * mask
            y = y * mask
 
            loss = criterion(output, y)     
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            losses += loss.item()
            
            # pred = torch.sigmoid(output).detach().cpu().numpy()
            # y = y.detach().cpu().numpy()

            # try:
            #     train_scores.append(roc_auc_score(y.ravel(), pred.ravel()))
            # except:
            #     pass

        losstrain = losses/len(train_loader)
        Loss.append(losstrain)

        # Evaluate on validation set
        valid,valid_score = evaluate(model, valid_loader, criterion)
        Loss_valid.append(valid)
        # valid_scores.append(valid_score)
    scheduler.step()
</code></pre>
<p>but got the following peculiar results.What could be the problem?<br>
<div class="lightbox-wrapper"><a class="lightbox" href="https://discuss.pytorch.org/uploads/default/original/3X/4/3/439929b28be46c3b2bf9206b890a82c004919cbd.png" data-download-href="https://discuss.pytorch.org/uploads/default/439929b28be46c3b2bf9206b890a82c004919cbd" title="image"><img src="https://discuss.pytorch.org/uploads/default/optimized/3X/4/3/439929b28be46c3b2bf9206b890a82c004919cbd_2_690x478.png" alt="image" data-base62-sha1="9E08HZX7NYPtkiOKfT7Rh1B1R49" width="690" height="478" srcset="https://discuss.pytorch.org/uploads/default/optimized/3X/4/3/439929b28be46c3b2bf9206b890a82c004919cbd_2_690x478.png, https://discuss.pytorch.org/uploads/default/optimized/3X/4/3/439929b28be46c3b2bf9206b890a82c004919cbd_2_1035x717.png 1.5x, https://discuss.pytorch.org/uploads/default/optimized/3X/4/3/439929b28be46c3b2bf9206b890a82c004919cbd_2_1380x956.png 2x" data-dominant-color="898989"><div class="meta"><svg class="fa d-icon d-icon-far-image svg-icon" aria-hidden="true"><use href="#far-image"></use></svg><span class="filename">image</span><span class="informations">1390×964 126 KB</span><svg class="fa d-icon d-icon-discourse-expand svg-icon" aria-hidden="true"><use href="#discourse-expand"></use></svg></div></a></div></p>

ptrblck: <p>You are resetting the <code>losses</code> only once before the training starts <code>losses= 0</code> and are then accumulating the batch losses to it. Later you are dividing by the number of batches, so I guess you might want to reset <code>losses</code> once per epoch.</p>