Repository: Lightning-AI/pytorch-lightning

Issue:
incorrect run on the test set with overwritten validation_end and test_epoch_end
## 🐛 Bug

If I override validation_end and test_epoch_end, TrainerEvaluationLoopMixin.evaluate works incorrectly on the test set

Suppose we override `validation_epoch_end` and `test_end`, but not `validation_end` and `test_epoch_end`. (I actually did this since I am a newbie and haven't yet figured out how everything works; also it seems `validation_end` is the same as `validation_epoch_end`, and `test_end` seems to be the same as `test_epoch_end`). Suppose I run `trainer.test(model)`. Consider lines 300-312 in [evaluation_loop.py](https://github.com/PyTorchLightning/pytorch-lightning/blob/6a0b171be4c08982a7a8b45ddffe04a726265323/pytorch_lightning/trainer/evaluation_loop.py#L300). Then we have `(test_mode and self.is_overriden('test_end', model=model)) == True`, so the first `if` block is executed, that is `eval_results = model.test_end(outputs)`. But look at the second `if` and its `elif`. We have `(test_mode and self.is_overriden('test_epoch_end', model=model)) == False`, hence the `elif` of the second `if` will also be executed, that is `eval_results = model.validation_epoch_end(outputs)`. And we will have validation results recorder as test results, which is a mistake.

This problem is present in the commit 60b8246. And the inverse problem (which happens if we override only `test_epoch_end` and `validation_end` is present in 0.7.1.



Discussion:
Hi! thanks for your contribution!, great first issue!
I can look into this.
To summarize, there is a problem when we mix `*_epoch_end` (new version) and` *_end` (old naming)  for validation and test. There is no problem if we would consequently use only one of the two, either the deprecated or the new way. Is this correct?

I propose this fix:
```
if test_mode:
    if is_overriden("test_end"):
        # get results and warn user
    elif is_overriden("test_epoch_end"):
        # get results
else: 
    # same as above but with "validation_end", "validation_epoch_end"
```
@awaelchli yes

Resolving patch:
diff --git a/pytorch_lightning/trainer/evaluation_loop.py b/pytorch_lightning/trainer/evaluation_loop.py
--- a/pytorch_lightning/trainer/evaluation_loop.py
+++ b/pytorch_lightning/trainer/evaluation_loop.py
@@ -296,20 +296,25 @@ def _evaluate(self, model: LightningModule, dataloaders, max_batches: int, test_
         if isinstance(model, (LightningDistributedDataParallel, LightningDataParallel)):
             model = model.module
 
-        # TODO: remove in v1.0.0
-        if test_mode and self.is_overriden('test_end', model=model):
-            eval_results = model.test_end(outputs)
-            warnings.warn('Method `test_end` was deprecated in 0.7.0 and will be removed 1.0.0.'
-                          ' Use `test_epoch_end` instead.', DeprecationWarning)
-        elif self.is_overriden('validation_end', model=model):
-            eval_results = model.validation_end(outputs)
-            warnings.warn('Method `validation_end` was deprecated in 0.7.0 and will be removed 1.0.0.'
-                          ' Use `validation_epoch_end` instead.', DeprecationWarning)
-
-        if test_mode and self.is_overriden('test_epoch_end', model=model):
-            eval_results = model.test_epoch_end(outputs)
-        elif self.is_overriden('validation_epoch_end', model=model):
-            eval_results = model.validation_epoch_end(outputs)
+        if test_mode:
+            if self.is_overriden('test_end', model=model):
+                # TODO: remove in v1.0.0
+                eval_results = model.test_end(outputs)
+                warnings.warn('Method `test_end` was deprecated in 0.7.0 and will be removed 1.0.0.'
+                              ' Use `test_epoch_end` instead.', DeprecationWarning)
+
+            elif self.is_overriden('test_epoch_end', model=model):
+                eval_results = model.test_epoch_end(outputs)
+
+        else:
+            if self.is_overriden('validation_end', model=model):
+                # TODO: remove in v1.0.0
+                eval_results = model.validation_end(outputs)
+                warnings.warn('Method `validation_end` was deprecated in 0.7.0 and will be removed 1.0.0.'
+                              ' Use `validation_epoch_end` instead.', DeprecationWarning)
+
+            elif self.is_overriden('validation_epoch_end', model=model):
+                eval_results = model.validation_epoch_end(outputs)
 
         # enable train mode again
         model.train()


Test patch:
diff --git a/tests/trainer/test_trainer.py b/tests/trainer/test_trainer.py
--- a/tests/trainer/test_trainer.py
+++ b/tests/trainer/test_trainer.py
@@ -528,15 +528,15 @@ def test_disabled_validation():
     class CurrentModel(LightTrainDataloader, LightValidationMixin, TestModelBase):
 
         validation_step_invoked = False
-        validation_end_invoked = False
+        validation_epoch_end_invoked = False
 
         def validation_step(self, *args, **kwargs):
             self.validation_step_invoked = True
             return super().validation_step(*args, **kwargs)
 
-        def validation_end(self, *args, **kwargs):
-            self.validation_end_invoked = True
-            return super().validation_end(*args, **kwargs)
+        def validation_epoch_end(self, *args, **kwargs):
+            self.validation_epoch_end_invoked = True
+            return super().validation_epoch_end(*args, **kwargs)
 
     hparams = tutils.get_default_hparams()
     model = CurrentModel(hparams)
@@ -555,8 +555,10 @@ def validation_end(self, *args, **kwargs):
     # check that val_percent_check=0 turns off validation
     assert result == 1, 'training failed to complete'
     assert trainer.current_epoch == 1
-    assert not model.validation_step_invoked, '`validation_step` should not run when `val_percent_check=0`'
-    assert not model.validation_end_invoked, '`validation_end` should not run when `val_percent_check=0`'
+    assert not model.validation_step_invoked, \
+        '`validation_step` should not run when `val_percent_check=0`'
+    assert not model.validation_epoch_end_invoked, \
+        '`validation_epoch_end` should not run when `val_percent_check=0`'
 
     # check that val_percent_check has no influence when fast_dev_run is turned on
     model = CurrentModel(hparams)
@@ -566,8 +568,10 @@ def validation_end(self, *args, **kwargs):
 
     assert result == 1, 'training failed to complete'
     assert trainer.current_epoch == 0
-    assert model.validation_step_invoked, 'did not run `validation_step` with `fast_dev_run=True`'
-    assert model.validation_end_invoked, 'did not run `validation_end` with `fast_dev_run=True`'
+    assert model.validation_step_invoked, \
+        'did not run `validation_step` with `fast_dev_run=True`'
+    assert model.validation_epoch_end_invoked, \
+        'did not run `validation_epoch_end` with `fast_dev_run=True`'
 
 
 def test_nan_loss_detection(tmpdir):