Repository: Lightning-AI/pytorch-lightning

Issue:
Int num_sanity_val_steps is always replaced by float limit_val_batches
The type annotations of `num_sanity_val_steps:` and `limit_val_batches` are `int` and `Union[int, float]` for _percent_ or _`num_batches`_. The minimum of _percent_ and _`num_batches`_ will be _percent_. (except when `num_batches==0`)
https://github.com/PyTorchLightning/pytorch-lightning/blob/a59e140ee814d8818d121405582133cf6b767e1a/pytorch_lightning/trainer/trainer.py#L461

Maybe we can remove the dependency of `limit_val_batches` from `num_sanity_val_steps` and revert to    
https://github.com/PyTorchLightning/pytorch-lightning/blob/1e68968ed7fb9b8f73df148dd48194d469655ea3/pytorch_lightning/trainer/trainer.py#L491



Discussion:
Hi! thanks for your contribution!, great first issue!
@williamFalcon @Borda should num_sanity_val_steps be limited by limit_val_batches? My understanding is they can be independent of one another.
@ananyahjha93 I asked william some time ago: https://github.com/PyTorchLightning/pytorch-lightning/pull/2246#issuecomment-650685045
@awaelchli Here: https://github.com/PyTorchLightning/pytorch-lightning/pull/2246#issuecomment-651398674
Shouldn't `Trainer(num_sanity_val_steps=5, limit_val_batches=3)` run 3 sanity val steps?
@awaelchli any comments on above? We can fix it then :)
It depends. What does it currently do in your example?
@awaelchli In the https://github.com/PyTorchLightning/pytorch-lightning/pull/2246#issuecomment-651398674 you said it should run for 5 val_steps but here https://github.com/PyTorchLightning/pytorch-lightning/blob/0097630a95bddc48d6fb5d3b9a58aef2e8e89b22/pytorch_lightning/trainer/trainer.py#L463-L466 it seems that it's running for 3 val_steps.

I suggest `num_sanity_val_steps` should be updated according to `limit_val_batches`(both float and int) and then we can merge https://github.com/PyTorchLightning/pytorch-lightning/pull/2892/ accordingly.

Resolving patch:
diff --git a/pytorch_lightning/callbacks/progress.py b/pytorch_lightning/callbacks/progress.py
--- a/pytorch_lightning/callbacks/progress.py
+++ b/pytorch_lightning/callbacks/progress.py
@@ -307,7 +307,7 @@ def init_test_tqdm(self) -> tqdm:
     def on_sanity_check_start(self, trainer, pl_module):
         super().on_sanity_check_start(trainer, pl_module)
         self.val_progress_bar = self.init_sanity_tqdm()
-        self.val_progress_bar.total = convert_inf(trainer.num_sanity_val_steps * len(trainer.val_dataloaders))
+        self.val_progress_bar.total = convert_inf(sum(trainer.num_sanity_val_batches))
         self.main_progress_bar = tqdm(disable=True)  # dummy progress bar
 
     def on_sanity_check_end(self, trainer, pl_module):
diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py
--- a/pytorch_lightning/trainer/trainer.py
+++ b/pytorch_lightning/trainer/trainer.py
@@ -377,6 +377,7 @@ def __init__(
         self.logged_metrics = {}
         self.num_training_batches = 0
         self.num_val_batches = []
+        self.num_sanity_val_batches = []
         self.num_test_batches = []
         self.train_dataloader = None
         self.test_dataloaders = None
@@ -463,9 +464,9 @@ def __init__(
         self.min_steps = min_steps
 
         if num_sanity_val_steps == -1:
-            self.num_sanity_val_steps = float("inf")
+            self.num_sanity_val_steps = float('inf')
         else:
-            self.num_sanity_val_steps = min(num_sanity_val_steps, limit_val_batches)
+            self.num_sanity_val_steps = num_sanity_val_steps
 
         self.reload_dataloaders_every_epoch = reload_dataloaders_every_epoch
 
@@ -1239,7 +1240,6 @@ def run_pretrain_routine(self, model: LightningModule):
         self.train()
 
     def _run_sanity_check(self, ref_model, model):
-
         using_val_step = ref_model.val_dataloader is not None and self.is_overridden('validation_step')
         should_sanity_check = using_val_step and self.num_sanity_val_steps > 0 and self.limit_val_batches > 0
 
@@ -1247,14 +1247,15 @@ def _run_sanity_check(self, ref_model, model):
         # to make sure program won't crash during val
         if should_sanity_check:
             self.reset_val_dataloader(ref_model)
+            self.num_sanity_val_batches = [
+                min(self.num_sanity_val_steps, val_batches) for val_batches in self.num_val_batches
+            ]
 
             # hook and callback
             self.running_sanity_check = True
             self.on_sanity_check_start()
 
-            num_loaders = len(self.val_dataloaders)
-            max_batches = [self.num_sanity_val_steps] * num_loaders
-            eval_results = self._evaluate(model, self.val_dataloaders, max_batches, False)
+            eval_results = self._evaluate(model, self.val_dataloaders, self.num_sanity_val_batches, False)
 
             # allow no returns from eval
             if eval_results is not None and len(eval_results) > 0:


Test patch:
diff --git a/tests/trainer/test_trainer.py b/tests/trainer/test_trainer.py
--- a/tests/trainer/test_trainer.py
+++ b/tests/trainer/test_trainer.py
@@ -907,12 +907,42 @@ def test_tpu_choice(tmpdir, tpu_cores, expected_tpu_id, error_expected):
     pytest.param(0.0),  # this should run no sanity checks
     pytest.param(1),
     pytest.param(1.0),
-    pytest.param(0.3),
+    pytest.param(0.5),
+    pytest.param(5),
 ])
 def test_num_sanity_val_steps(tmpdir, limit_val_batches):
+    """ Test that the number of sanity check batches is clipped to limit_val_batches. """
+    model = EvalModelTemplate()
+    model.validation_step = model.validation_step__multiple_dataloaders
+    model.validation_epoch_end = model.validation_epoch_end__multiple_dataloaders
+    num_sanity_val_steps = 4
+
+    trainer = Trainer(
+        default_root_dir=tmpdir,
+        num_sanity_val_steps=num_sanity_val_steps,
+        limit_val_batches=limit_val_batches,
+        max_steps=1,
+    )
+    assert trainer.num_sanity_val_steps == num_sanity_val_steps
+    val_dataloaders = model.val_dataloader__multiple_mixed_length()
+
+    with patch.object(trainer, 'evaluation_forward', wraps=trainer.evaluation_forward) as mocked:
+        trainer.fit(model, val_dataloaders=val_dataloaders)
+        assert mocked.call_count == sum(
+            min(num_sanity_val_steps, num_batches) for num_batches in trainer.num_val_batches
+        )
+
+
+@pytest.mark.parametrize(['limit_val_batches'], [
+    pytest.param(0.0),  # this should run no sanity checks
+    pytest.param(1),
+    pytest.param(1.0),
+    pytest.param(0.3),
+])
+def test_num_sanity_val_steps_neg_one(tmpdir, limit_val_batches):
     """
-    Test that num_sanity_val_steps=-1 runs through all validation data once.
-    Makes sure this setting is independent of limit_val_batches.
+    Test that num_sanity_val_steps=-1 runs through all validation data once, and as many batches as
+    limited by "limit_val_batches" Trainer argument.
     """
     model = EvalModelTemplate()
     model.validation_step = model.validation_step__multiple_dataloaders
@@ -920,7 +950,7 @@ def test_num_sanity_val_steps(tmpdir, limit_val_batches):
     trainer = Trainer(
         default_root_dir=tmpdir,
         num_sanity_val_steps=-1,
-        limit_val_batches=limit_val_batches,  # should have no influence
+        limit_val_batches=limit_val_batches,
         max_steps=1,
     )
     assert trainer.num_sanity_val_steps == float('inf')
@@ -928,7 +958,7 @@ def test_num_sanity_val_steps(tmpdir, limit_val_batches):
 
     with patch.object(trainer, 'evaluation_forward', wraps=trainer.evaluation_forward) as mocked:
         trainer.fit(model, val_dataloaders=val_dataloaders)
-        assert mocked.call_count == sum(len(dl) * (limit_val_batches > 0) for dl in val_dataloaders)
+        assert mocked.call_count == sum(trainer.num_val_batches)
 
 
 @pytest.mark.parametrize("trainer_kwargs,expected", [