Repository: Lightning-AI/pytorch-lightning

Issue:
ModelCheckpoint does not save checkpoints
This same code worked in the past version, but now it doesn't save the checkpoints anymore.

I tried with 
MODEL_OUTPUT = 'example/hello'
MODEL_OUTPUT = 'example/hello/'
MODEL_OUTPUT = 'example/hello/weights'

with 'example/hello/' as an existing folder 


```python
checkpoint_callback = ModelCheckpoint(
        filepath='%s/' % MODEL_OUTPUT,
        save_best_only=True,
        verbose=True,
        monitor='val_loss',
        mode='min'
    )

    early_stop_callback = EarlyStopping(monitor='val_loss', patience=5)

    tt_logger = CMLLogger(LOG_DIR)

    trainer = Trainer(logger=tt_logger,
                      row_log_interval=2,
                      checkpoint_callback=checkpoint_callback,
                      early_stop_callback=early_stop_callback,
                      gradient_clip_val=0.5,
                      gpus=gpus,
                      check_val_every_n_epoch=1,
                      max_nb_epochs=99999,
                      train_percent_check=train_frac,
                      log_save_interval=5,
                     )
```

**Desktop (please complete the following information):**
 - OS:linux/mac


Discussion:
met this problem:) fixed by set train_percent_check=1
Unfortunately I cannot set train_percent_check to 1 :(
what train_frac are you using? are you using the latest version on master?
The latest published release. I noticed now that train_frac is set to 1 in all of the examples above.

I forgot the mention that I'm subclassing a class that extends LightningModule as my model, in previous version this works fine.
could you post full code or something? or code to reproduce? 
Sure, here are the train method and how I structured the class layout, I cannot post the network architecture but I don't think it is relevant at all

```python
def fit_model(model, train_size, gpus=False):

    remove_folder_rec(LOG_DIR)
    remove_folder_rec(MODEL_OUTPUT)

    if gpus:
        gpus = str(1)
    else:
        gpus = None

    train_frac = min(1, 1 / (train_size / MAX_EXAMPLES_PER_EPOCHS))

    checkpoint_callback = ModelCheckpoint(
        filepath='%s/weights' % MODEL_OUTPUT,
        save_best_only=True,
        verbose=True,
        monitor='val_loss',
        mode='min'
    )

    early_stop_callback = EarlyStopping(monitor='val_loss', patience=5)

    tt_logger = CustomLogger(LOG_DIR)

    trainer = Trainer(logger=tt_logger,
                      row_log_interval=2,
                      checkpoint_callback=checkpoint_callback,
                      early_stop_callback=early_stop_callback,
                      gradient_clip_val=0.5,
                      gpus=gpus,
                      check_val_every_n_epoch=1,
                      max_nb_epochs=99999,
                      train_percent_check=train_frac,
                      log_save_interval=5,
                     )

    # Add error handling here
    # try:
    trainer.fit(model)
    # except Exception as e:
    #     print(str(e))
    #     print('early stopped')
    return trainer
```

```python
class BaseModel(pl.LightningModule):

    def __init__(self,
                 batch_size,
                 gpu,
                 train_dataset=None,
                 val_dataset=None,
                 test_dataset=None):
        super(BaseModel, self).__init__()
        self.train_dataset = train_dataset
        self.val_dataset = val_dataset
        self.test_dataset = test_dataset
        self.gpu = gpu
        self.batch_size = batch_size

    def forward(self, inp, training=False):
        return NotImplemented

    def training_step(self, batch, batch_nb):
        return NotImplemented

    def validation_step(self, batch, batch_nb):
        return NotImplemented

    def validation_end(self, outputs):
        return NotImplemented

    def test_step(self, batch, batch_nb):
        return NotImplemented

    def test_end(self, outputs):
        return NotImplemented

    def configure_optimizers(self):
        return NotImplemented

    @pl.data_loader
    def train_dataloader(self):
        return DataLoader(self.train_dataset, shuffle=True, pin_memory=self.gpu, batch_size=self.batch_size)

    @pl.data_loader
    def val_dataloader(self):
        return DataLoader(self.val_dataset, shuffle=True, pin_memory=self.gpu, batch_size=self.batch_size)

    @pl.data_loader
    def test_dataloader(self):
        return DataLoader(self.test_dataset, shuffle=True, pin_memory=self.gpu, batch_size=self.batch_size)
```

```python
class TextCNN(BaseModel):

    def __init__(self,                 
                 gpu=False,
                 train_dataset=None,
                 val_dataset=None,
                 test_dataset=None):
        super(TextCNN, self).__init__(
            1, gpu, train_dataset, val_dataset, test_dataset)

        ## Layers initializations is defined here...

    def forward(self, inp, training=False):
       ...
       ...
        return output

    def training_step(self, batch, batch_nb):
        x = torch.squeeze(batch['x'], dim=0).float()
        y = torch.squeeze(batch['y'], dim=0).long()

        output = self.forward(x, training=True)
        loss = self.loss(output, y)
        acc = accuracy(output, y)

        return {
            'loss': loss,
            'progress_bar': {
                'train_loss': loss,
                'train_acc': acc
            },
            'log': {
                'train_loss': loss,
                'train_acc': acc
            }
        }

    def validation_step(self, batch, batch_nb):
        x = torch.squeeze(batch['x'], dim=0).float()
        y = torch.squeeze(batch['y'], dim=0).long()

        output = self.forward(x)
        return {'batch_val_loss': self.loss(output, y),
                'batch_val_acc': accuracy(output, y)}

    def validation_end(self, outputs):
        avg_loss = torch.stack([x['batch_val_loss'] for x in outputs]).mean()
        avg_acc = torch.stack([x['batch_val_acc'] for x in outputs]).mean()

        return {
            'log': {
                'val_loss': avg_loss,
                'val_acc': avg_acc,
            },
            'progress_bar': {
                'val_loss': avg_loss,
                'val_acc': avg_acc
            }
        }

    def test_step(self, batch, batch_nb):
        x = torch.squeeze(batch['x'], dim=0).float()
        y = torch.squeeze(batch['y'], dim=0).long()

        output = self.forward(x)
        return {'batch_test_loss': self.loss(output, y),
                'batch_test_acc': accuracy(output, y)}

    def test_end(self, outputs):
        avg_loss = torch.stack([x['batch_test_loss'] for x in outputs]).mean()
        avg_acc = torch.stack([x['batch_test_acc'] for x in outputs]).mean()

        return {
            'log': {
                'test_loss': avg_loss,
                'test_acc': avg_acc,
            },
            'progress_bar': {
                'test_loss': avg_loss,
                'test_acc': avg_acc
            }
        }

    def configure_optimizers(self):
        return [torch.optim.Adam(self.parameters(), lr=3e-04)]

```
Well, you seem to monitor `val_loss` in your callback, but you do not return it in the **separate** key in `validation_end`, if you're using latest version, this may be a problem, check sources https://github.com/williamFalcon/pytorch-lightning/blob/0fac2d64cf4c6c31d3b21605a272ace01d0c51a2/pytorch_lightning/trainer/trainer.py#L1304

Resolving patch:
diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py
--- a/pytorch_lightning/trainer/trainer.py
+++ b/pytorch_lightning/trainer/trainer.py
@@ -1364,6 +1364,10 @@ def __process_output(self, output, train=False):
             if self.use_dp or self.use_ddp2:
                 loss = reduce_distributed_output(loss, self.num_gpus)
 
+        # use every metric passed in as a candidate for callback
+        callback_metrics.update(progress_bar_metrics)
+        callback_metrics.update(log_metrics)
+
         return loss, progress_bar_metrics, log_metrics, callback_metrics
 
     def __clip_gradients(self):


Test patch: