Repository: Lightning-AI/pytorch-lightning

Issue:
Batched iterative dataloading disables validation
## 🐛 Bug

Setting the `batch_size` parameter for `torch.utils.data.DataLoader` to a number greater than 1, prevents `validation_step` and `validation_epoch_end` from being called.

### To Reproduce

Steps to reproduce the behavior:

1. Run `python main.py` with `bs = 1`
2. Observe exception raised in `validation_step`
3. Run `python main.py` after changing to `bs = 2`
4. Observe the model train successfully


#### Code sample

```py
import pytorch_lightning as pl
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader, IterableDataset

class Dataset(IterableDataset):
    def __init__(self):
        super().__init__()

    def __iter__(self):
        for _ in range(1024):
            yield torch.randn(20)

    def __len__(self):
        return 1024

class Model(pl.LightningModule):
    def __init__(self):
        super().__init__()
        self.fst = nn.Linear(20, 1)
        self.snd = nn.Linear(1, 20)

    def forward(self, x):
        x = self.fst(x)
        x = F.relu(x)
        x = self.snd(x)
        return x

    def training_step(self, batch, batchIdx):
        x = self.forward(batch)
        return {'loss': F.mse_loss(x, batch)}

    def validation_step(self, batch, batchIdx):
        raise NotImplementedError()
        x = self.forward(batch)
        return {'val_loss': F.mse_loss(x, batch)}

    def validation_epoch_end(self, outputs):
        return {'val_loss': torch.mean(torch.stack([x['val_loss'] for x in outputs]))}

    def configure_optimizers(self):
        return torch.optim.AdamW(self.parameters())

if __name__ == '__main__':
    trainer = pl.Trainer(num_sanity_val_steps=0)
    net = Model()
    dataset = Dataset()

    bs = 2
    trainer.fit(net, train_dataloader=DataLoader(dataset, batch_size=bs), val_dataloaders=DataLoader(dataset, batch_size=bs))
```

```
> python main.py # with bs=2, should fail
GPU available: True, used: False
TPU available: False, using: 0 TPU cores

  | Name | Type   | Params
--------------------------------
0 | fst  | Linear | 21    
1 | snd  | Linear | 40    
/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/pytorch_lightning/utilities/distributed.py:25: UserWarning: The dataloader, train dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 8 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.
  warnings.warn(*args, **kwargs)
/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/pytorch_lightning/utilities/distributed.py:25: UserWarning: The dataloader, val dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 8 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.
  warnings.warn(*args, **kwargs)
Epoch 1000:  25%|████████████████████████████████████████████████████████████▌                                                                                                                                                                                     | 512/2048 [00:00<00:01, 1396.66it/s, loss=0.890, v_num=20]
```

### Expected behavior

```
> python main.py # with bs=1
GPU available: True, used: False
TPU available: False, using: 0 TPU cores

  | Name | Type   | Params
--------------------------------
0 | fst  | Linear | 21    
1 | snd  | Linear | 40    
/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/pytorch_lightning/utilities/distributed.py:25: UserWarning: The dataloader, train dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 8 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.
  warnings.warn(*args, **kwargs)
/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/pytorch_lightning/utilities/distributed.py:25: UserWarning: The dataloader, val dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 8 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.
  warnings.warn(*args, **kwargs)
Epoch 1:  50%|████████████████Traceback (most recent call last):████████████████████████████████████████████████████████████████████████                                                                                                                          | 1024/2048 [00:00<00:00, 1337.66it/s, loss=1.032, v_num=19]
  File "main.py", line 57, in <module>
    trainer.fit(net, train_dataloader=DataLoader(dataset, batch_size=bs), val_dataloaders=DataLoader(dataset, batch_size=bs))
  File "/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 954, in fit
    self.run_pretrain_routine(model)
  File "/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 1093, in run_pretrain_routine
    self.train()
  File "/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/pytorch_lightning/trainer/training_loop.py", line 375, in train
    self.run_training_epoch()
  File "/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/pytorch_lightning/trainer/training_loop.py", line 490, in run_training_epoch
    self.run_evaluation(test_mode=self.testing)
  File "/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/pytorch_lightning/trainer/evaluation_loop.py", line 379, in run_evaluation
    eval_results = self._evaluate(self.model, dataloaders, max_batches, test_mode)
  File "/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/pytorch_lightning/trainer/evaluation_loop.py", line 281, in _evaluate
    output = self.evaluation_forward(model, batch, batch_idx, dataloader_idx, test_mode)
  File "/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/pytorch_lightning/trainer/evaluation_loop.py", line 452, in evaluation_forward
    output = model.validation_step(*args)
  File "main.py", line 39, in validation_step
    raise NotImplementedError()
NotImplementedError
Exception ignored in: <object repr() failed>
Traceback (most recent call last):
  File "/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/tqdm/std.py", line 1086, in __del__
  File "/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/tqdm/std.py", line 1293, in close
  File "/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/tqdm/std.py", line 1471, in display
  File "/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/tqdm/std.py", line 1089, in __repr__
  File "/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/tqdm/std.py", line 1433, in format_dict
TypeError: 'NoneType' object is not iterable
```

### Environment

```
Collecting environment information...
PyTorch version: 1.5.1+cu101
Is debug build: No
CUDA used to build PyTorch: 10.1

OS: Gentoo Base System release 2.7
GCC version: (Gentoo 9.3.0 p1) 9.3.0
CMake version: version 3.17.3

Python version: 3.6
Is CUDA available: Yes
CUDA runtime version: 10.1.243
GPU models and configuration: GPU 0: GeForce GT 730
Nvidia driver version: 440.82
cuDNN version: /opt/cuda/targets/x86_64-linux/lib/libcudnn.so.7.6.5

Versions of relevant libraries:
[pip3] numpy==1.19.0
[pip3] pytorch-lightning==0.8.1
[pip3] torch==1.5.1+cu101
[pip3] torchvision==0.6.1+cu101
[conda] Could not collect
```

### Additional context

Basically a reopen of #2351, as this issue is not fixed by changing batch size and dataset size.


Discussion:
Same behavior with `pytorch-lightning==0.8.3`.
I can reproduce. ~It is caused by `pl.Trainer(num_sanity_val_steps=0)`
For `num_sanity_val_steps>0` it works fine~
trying to fix his right now. another observation: only happens with iterable dataset. 
EDIT: iterable dataset that has also length defined (see comment below)
@awaelchli  thanks for looking into this! If I remember correctly, then for `num_sanity_val_steps > 0` it only raised the exception during sanity checks on my machine. When I try to raise the exception only in actual training (for example by raising the exception on the fourth time `validation_step` is called), then this completes sucessfully (i.e. does not call validation) as well.
I think the issue is that your dataset is of type Iterable but has `__len__` defined. PL interprets that wrongly. In fact, there is a weird issue with dataloaders from iterable datasets that have len defined, check out this colab that I made:
https://colab.research.google.com/drive/1RQyNLGORe4vOL_RS6khcFNObcxyFxtEm?usp=sharing
isn't it strange?

if I remove the len from the dataset definition, your code sample works. 

Resolving patch:
diff --git a/pytorch_lightning/trainer/data_loading.py b/pytorch_lightning/trainer/data_loading.py
--- a/pytorch_lightning/trainer/data_loading.py
+++ b/pytorch_lightning/trainer/data_loading.py
@@ -1,8 +1,10 @@
+import multiprocessing
 import platform
 from abc import ABC, abstractmethod
+from distutils.version import LooseVersion
 from typing import Union, List, Tuple, Callable, Optional
-import multiprocessing
 
+import torch
 import torch.distributed as torch_distrib
 from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
 from torch.utils.data.distributed import DistributedSampler
@@ -41,19 +43,33 @@
     HOROVOD_AVAILABLE = True
 
 
+def _has_iterable_dataset(dataloader: DataLoader):
+    return ITERABLE_DATASET_EXISTS and hasattr(dataloader, 'dataset') \
+        and isinstance(dataloader.dataset, IterableDataset)
+
+
 def _has_len(dataloader: DataLoader) -> bool:
     """ Checks if a given Dataloader has __len__ method implemented i.e. if
-    it is a finite dataloader or infinite dataloader """
+    it is a finite dataloader or infinite dataloader. """
+
     try:
         # try getting the length
         if len(dataloader) == 0:
             raise ValueError('`Dataloader` returned 0 length.'
                              ' Please make sure that your Dataloader at least returns 1 batch')
-        return True
+        has_len = True
     except TypeError:
-        return False
+        has_len = False
     except NotImplementedError:  # e.g. raised by torchtext if a batch_size_fn is used
-        return False
+        has_len = False
+
+    if has_len and _has_iterable_dataset(dataloader) and LooseVersion(torch.__version__) >= LooseVersion("1.4.0"):
+        rank_zero_warn(
+            'Your `IterableDataset` has `__len__` defined.'
+            ' In combination with multi-processing data loading (e.g. batch size > 1),'
+            ' this can lead to unintended side effects since the samples will be duplicated.'
+        )
+    return has_len
 
 
 class TrainerDataLoadingMixin(ABC):
@@ -128,12 +144,9 @@ def _worker_check(self, dataloader: DataLoader, name: str) -> None:
     def auto_add_sampler(self, dataloader: DataLoader, train: bool) -> DataLoader:
 
         # don't do anything if it's not a dataloader
-        # don't manipulate iterable datasets
         is_dataloader = isinstance(dataloader, DataLoader)
-
-        is_iterable_ds = False
-        if ITERABLE_DATASET_EXISTS and hasattr(dataloader, 'dataset'):
-            is_iterable_ds = isinstance(dataloader.dataset, IterableDataset)
+        # don't manipulate iterable datasets
+        is_iterable_ds = _has_iterable_dataset(dataloader)
 
         if not is_dataloader or is_iterable_ds:
             return dataloader
@@ -285,11 +298,7 @@ def _reset_eval_dataloader(
         # datasets could be none, 1 or 2+
         if len(dataloaders) != 0:
             for i, dataloader in enumerate(dataloaders):
-                try:
-                    num_batches = len(dataloader)
-                except (TypeError, NotImplementedError):
-                    num_batches = float('inf')
-
+                num_batches = len(dataloader) if _has_len(dataloader) else float('inf')
                 self._worker_check(dataloader, f'{mode} dataloader {i}')
 
                 # percent or num_steps


Test patch:
diff --git a/tests/trainer/test_dataloaders.py b/tests/trainer/test_dataloaders.py
--- a/tests/trainer/test_dataloaders.py
+++ b/tests/trainer/test_dataloaders.py
@@ -2,11 +2,13 @@
 
 import pytest
 import torch
+from packaging.version import parse
 from torch.utils.data.dataloader import DataLoader
-from torch.utils.data.dataset import Subset
+from torch.utils.data.dataset import Subset, IterableDataset
 
 import tests.base.develop_pipelines as tpipes
 from pytorch_lightning import Trainer
+from pytorch_lightning.trainer.data_loading import _has_len, _has_iterable_dataset
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
 from tests.base import EvalModelTemplate
 
@@ -487,6 +489,36 @@ def test_warning_with_few_workers(tmpdir, ckpt_path):
         trainer.test(**test_options)
 
 
+@pytest.mark.xfail(
+    parse(torch.__version__) < parse("1.4.0"),
+    reason="IterableDataset with __len__ before 1.4 raises",
+)
+def test_warning_with_iterable_dataset_and_len(tmpdir):
+    """ Tests that a warning messages is shown when an IterableDataset defines `__len__`. """
+    model = EvalModelTemplate()
+    original_dataset = model.train_dataloader().dataset
+
+    class IterableWithLen(IterableDataset):
+
+        def __iter__(self):
+            return iter(original_dataset)
+
+        def __len__(self):
+            return len(original_dataset)
+
+    dataloader = DataLoader(IterableWithLen(), batch_size=16)
+    assert _has_len(dataloader)
+    assert _has_iterable_dataset(dataloader)
+    trainer = Trainer(
+        default_root_dir=tmpdir,
+        max_steps=3,
+    )
+    with pytest.warns(UserWarning, match='Your `IterableDataset` has `__len__` defined.'):
+        trainer.fit(model, train_dataloader=dataloader, val_dataloaders=[dataloader])
+    with pytest.warns(UserWarning, match='Your `IterableDataset` has `__len__` defined.'):
+        trainer.test(model, test_dataloaders=[dataloader])
+
+
 @pytest.mark.skipif(torch.cuda.device_count() < 2, reason='Test requires multiple GPUs')
 def test_dataloader_reinit_for_subclass():