Repository: Lightning-AI/pytorch-lightning

Issue:
Bug in validation_epoch_end
## 🐛 Bug

In the [documentation](https://pytorch-lightning.readthedocs.io/en/latest/lightning_module.html?highlight=validation_epoch_end#validation-epoch-end), `validation_epoch_end` is described as running at the end of a validation epoch and does not need to necessarily return anything.

When running a slightly modified version of the [example](https://pytorch-lightning.readthedocs.io/en/latest/new_project.html) in the docs, `validation_epoch_end` seemingly runs once on a single batch of validation data and returns an error if nothing is returned.

### To Reproduce

A MWE:

```
import os
import torch
import torch.nn.functional as F
from torchvision.datasets import MNIST
from torchvision import transforms
from torch.utils.data import DataLoader
import pytorch_lightning as pl
from torch.utils.data import random_split

class LitModel(pl.LightningModule):

    def __init__(self):
        super().__init__()
        self.layer_1 = torch.nn.Linear(28 * 28, 128)
        self.layer_2 = torch.nn.Linear(128, 10)

    def forward(self, x):
        x = x.view(x.size(0), -1)
        x = self.layer_1(x)
        x = F.relu(x)
        x = self.layer_2(x)
        return x

    def configure_optimizers(self):
        optimizer = torch.optim.Adam(self.parameters(), lr=1e-3)
        return optimizer

    def training_step(self, batch, batch_idx):
        x, y = batch
        y_hat = self(x)
        loss = F.cross_entropy(y_hat, y)

        return {'loss': loss}

    def validation_step(self, batch, batch_idx):
        x, y = batch
        y_hat = self(x)
        loss = F.cross_entropy(y_hat, y)
        
        return {"val_loss": loss}

    def validation_epoch_end(self, validation_step_outputs):
        print("Length of outputs: {}".format(len(validation_step_outputs)))
        # a dummy return value to avoid error
        return {"val_loss": 0}

model = LitModel()

dataset = MNIST(os.getcwd(), download=True, train=False, transform=transforms.ToTensor())
train, val = random_split(dataset, [9000, 1000])

train_loader = DataLoader(train)
val_loader = DataLoader(val)

# train
trainer = pl.Trainer(progress_bar_refresh_rate=0, max_epochs=5, weights_summary=None, gpus=1)
trainer.fit(model, train_loader, val_loader)
```

#### Output

```
GPU available: True, used: True
TPU available: False, using: 0 TPU cores
CUDA_VISIBLE_DEVICES: [0]
/usr/local/lib/python3.6/dist-packages/pytorch_lightning/utilities/distributed.py:37: UserWarning: Could not log computational graph since the `model.example_input_array` attribute is not set or `input_array` was not given
  warnings.warn(*args, **kwargs)
Length of outputs: 1
Length of outputs: 1000
Length of outputs: 1000
Length of outputs: 1000
Length of outputs: 1000
Saving latest checkpoint..
Length of outputs: 1000
```

If no return value is given in `validation_epoch_end`, the following error occurs

```
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-183-013dcacba267> in <module>()
     10 # train
     11 trainer = pl.Trainer(progress_bar_refresh_rate=0, max_epochs=5, weights_summary=None, gpus=1)
---> 12 trainer.fit(model, train_loader, val_loader)

7 frames
/usr/local/lib/python3.6/dist-packages/pytorch_lightning/trainer/states.py in wrapped_fn(self, *args, **kwargs)
     46             if entering is not None:
     47                 self.state = entering
---> 48             result = fn(self, *args, **kwargs)
     49 
     50             # The INTERRUPTED state can be set inside the run function. To indicate that run was interrupted

/usr/local/lib/python3.6/dist-packages/pytorch_lightning/trainer/trainer.py in fit(self, model, train_dataloader, val_dataloaders, datamodule)
   1071             self.accelerator_backend = GPUBackend(self)
   1072             model = self.accelerator_backend.setup(model)
-> 1073             results = self.accelerator_backend.train(model)
   1074 
   1075         elif self.use_tpu:

/usr/local/lib/python3.6/dist-packages/pytorch_lightning/accelerators/gpu_backend.py in train(self, model)
     49 
     50     def train(self, model):
---> 51         results = self.trainer.run_pretrain_routine(model)
     52         return results
     53 

/usr/local/lib/python3.6/dist-packages/pytorch_lightning/trainer/trainer.py in run_pretrain_routine(self, model)
   1222 
   1223         # run a few val batches before training starts
-> 1224         self._run_sanity_check(ref_model, model)
   1225 
   1226         # clear cache before training

/usr/local/lib/python3.6/dist-packages/pytorch_lightning/trainer/trainer.py in _run_sanity_check(self, ref_model, model)
   1255             num_loaders = len(self.val_dataloaders)
   1256             max_batches = [self.num_sanity_val_steps] * num_loaders
-> 1257             eval_results = self._evaluate(model, self.val_dataloaders, max_batches, False)
   1258 
   1259             # allow no returns from eval

/usr/local/lib/python3.6/dist-packages/pytorch_lightning/trainer/evaluation_loop.py in _evaluate(self, model, dataloaders, max_batches, test_mode)
    397 
    398         # log callback metrics
--> 399         self.__update_callback_metrics(eval_results, using_eval_result)
    400 
    401         # Write predictions to disk if they're available.

/usr/local/lib/python3.6/dist-packages/pytorch_lightning/trainer/evaluation_loop.py in __update_callback_metrics(self, eval_results, using_eval_result)
    429                         flat = {'val_loss': eval_result}
    430                     else:
--> 431                         flat = flatten_dict(eval_result)
    432                     self.callback_metrics.update(flat)
    433             else:

/usr/local/lib/python3.6/dist-packages/pytorch_lightning/utilities/parsing.py in flatten_dict(source, result)
    113         result = {}
    114 
--> 115     for k, v in source.items():
    116         if isinstance(v, dict):
    117             _ = flatten_dict(v, result)

AttributeError: 'NoneType' object has no attribute 'items'
```



### Expected behavior

The outputs in `validation_epoch_end` should contain results from all batches. Is a test validation batch running in the background, which could explain why the results of only one batch are included in the first loop.

Return values in `validation_epoch_end` were addressed in #2438 (with the same error) and from what I can tell included in the 0.8.4 release - perhaps I am missing something?


### Environment

 - PyTorch: 1.6.0
 - pytorch-lightning: 0.9.0 


Discussion:
Hi! thanks for your contribution!, great first issue!
Hi there! What you are seeing in the first output is caused by our "validation sanity check". We run a configurable number of validation steps _before_ training starts to ensure the user doesn't run into any issues after a potentially timely series of training steps. This can be configured (or turned off) with `pl.Trainer(num_sanity_val_steps=0)`.

Regarding returning `None` from `validation_epoch_end`, it appears that functionality has recently been removed, I will look into whether that was intentional or not and update the code/docs accordingly.
@teddykoker That makes sense. Thanks for the pointer for configuring this! 

Resolving patch:
diff --git a/pytorch_lightning/trainer/connectors/logger_connector.py b/pytorch_lightning/trainer/connectors/logger_connector.py
--- a/pytorch_lightning/trainer/connectors/logger_connector.py
+++ b/pytorch_lightning/trainer/connectors/logger_connector.py
@@ -104,6 +104,9 @@ def on_evaluation_epoch_end(self, eval_results, using_eval_result, test_mode):
         return self.__log_evaluation_epoch_metrics_2(eval_results, test_mode)
 
     def _log_on_evaluation_epoch_end_metrics(self, eval_results, using_eval_result):
+        if len(eval_results) > 0 and eval_results[0] is None:
+            return
+
         if using_eval_result:
             if isinstance(eval_results, list):
                 for eval_result in eval_results:
diff --git a/pytorch_lightning/trainer/logging.py b/pytorch_lightning/trainer/logging.py
--- a/pytorch_lightning/trainer/logging.py
+++ b/pytorch_lightning/trainer/logging.py
@@ -73,9 +73,10 @@ def process_output(self, output, train=False):
         # ---------------
         # all keys not progress_bar or log are candidates for callbacks
         callback_metrics = {}
-        for k, v in output.items():
-            if k not in ['progress_bar', 'log', 'hiddens']:
-                callback_metrics[k] = v
+        if output:
+            for k, v in output.items():
+                if k not in ['progress_bar', 'log', 'hiddens']:
+                    callback_metrics[k] = v
 
         if train and (self.use_dp or self.use_ddp2):
             num_gpus = self.num_gpus
@@ -136,7 +137,7 @@ def process_output(self, output, train=False):
         # ---------------
         # EXTRACT HIDDEN
         # ---------------
-        hiddens = output.get('hiddens')
+        hiddens = output.get('hiddens') if output else None
 
         # use every metric passed in as a candidate for callback
         callback_metrics.update(progress_bar_metrics)


Test patch:
diff --git a/tests/base/model_valid_epoch_ends.py b/tests/base/model_valid_epoch_ends.py
--- a/tests/base/model_valid_epoch_ends.py
+++ b/tests/base/model_valid_epoch_ends.py
@@ -33,6 +33,21 @@ def _mean(res, key):
         results = {'progress_bar': metrics_dict, 'log': metrics_dict}
         return results
 
+    def validation_epoch_end_return_none(self, outputs):
+        """
+        Called at the end of validation to aggregate outputs
+
+        Args:
+            outputs: list of individual outputs of each validation step
+        """
+        # if returned a scalar from validation_step, outputs is a list of tensor scalars
+        # we return just the average in this case (if we want)
+        def _mean(res, key):
+            # recursive mean for multilevel dicts
+            return torch.stack([x[key] if isinstance(x, dict) else _mean(x, key) for x in res]).mean()
+
+        return None
+
     def validation_epoch_end__multiple_dataloaders(self, outputs):
         """
         Called at the end of validation to aggregate outputs
diff --git a/tests/trainer/test_trainer_steps_result_return.py b/tests/trainer/test_trainer_steps_result_return.py
--- a/tests/trainer/test_trainer_steps_result_return.py
+++ b/tests/trainer/test_trainer_steps_result_return.py
@@ -623,3 +623,27 @@ def test_result_monitor_warnings(tmpdir):
 
     with pytest.warns(UserWarning, match='key of `EarlyStopping` has no effect'):
         trainer.fit(model)
+
+
+def test_eval_loop_return_none(tmpdir):
+    """
+    Tests that we warn when the monitor key is changed and we use Results obj
+    """
+    model = EvalModelTemplate()
+    model.test_step = None
+    model.training_step = model.training_step_result_obj
+    model.training_step_end = None
+    model.training_epoch_end = None
+    model.validation_step = model.validation_step_result_obj
+    model.validation_step_end = None
+    model.validation_epoch_end = model.validation_epoch_end_return_none
+    model.test_dataloader = None
+
+    trainer = Trainer(
+        default_root_dir=tmpdir,
+        max_epochs=2,
+        row_log_interval=2,
+        limit_train_batches=2,
+        weights_summary=None,
+    )
+    trainer.fit(model)