# Buggy code
def _reset_eval_dataloader(
    self, model: LightningModule, mode: str
) -> Tuple[List[Union[int, float]], List[DataLoader]]:
    """Generic method to reset a dataloader for evaluation.

    Args:
        model: The current `LightningModule`
        mode: Either `'val'` or `'test'`

    Returns:
        Tuple (num_batches, dataloaders)
    """
    # use the training loader as val and test when overfitting
    if self.overfit_batches > 0:
        dataloaders = self.request_dataloader(getattr(model, "train_dataloader"))
    else:
        dataloaders = self.request_dataloader(getattr(model, f"{mode}_dataloader"))

    if not isinstance(dataloaders, list):
        dataloaders = [dataloaders]

    for loader_i in range(len(dataloaders)):
        loader = dataloaders[loader_i]

        # shuffling in val and test set is bad practice
        if (
            mode in ("val", "test")
            and hasattr(loader, "sampler")
            and isinstance(loader.sampler, RandomSampler)
        ):
            # when overfitting, the dataloader should not have sampler
            if self.overfit_batches > 0:
                rank_zero_warn(
                    "You requested to overfit but enabled training dataloader shuffling."
                    " We are turning it off for you."
                )
                dataloaders[loader_i] = self.replace_sampler(
                    loader, SequentialSampler(loader.dataset)
                )

            else:
                rank_zero_warn(
                    f"Your {mode}_dataloader has `shuffle=True`, it is best practice to turn"
                    " this off for validation and test dataloaders."
                )

    if any([dl is None for dl in dataloaders]):
        rank_zero_warn("One of given dataloaders is None and it will be skipped.")

    # add samplers
    dataloaders = [
        self.auto_add_sampler(dl, train=False) for dl in dataloaders if dl is not None
    ]

    loader_num_batches = []

    # determine number of batches
    # datasets could be none, 1 or 2+
    if len(dataloaders) != 0:
        for i, dataloader in enumerate(dataloaders):
            try:
                num_batches = len(dataloader)
            except (TypeError, NotImplementedError):
                num_batches = float("inf")

            self._worker_check(dataloader, f"{mode} dataloader {i}")

            # percent or num_steps
            limit_eval_batches = getattr(self, f"limit_{mode}_batches")

            if num_batches != float("inf"):
                self._check_batch_limits(f"limit_{mode}_batches")

                # limit num batches either as a percent or num steps
                if isinstance(limit_eval_batches, float):
                    num_batches = int(num_batches * limit_eval_batches)
                else:
                    num_batches = limit_eval_batches

            elif limit_eval_batches not in (0.0, 1.0):
                raise MisconfigurationException(
                    "When using an infinite DataLoader (e.g. with an IterableDataset"
                    f" or when DataLoader does not implement `__len__`) for `limit_{mode}_batches`,"
                    f" `Trainer(limit_{mode}_batches)` must be `0.0` or `1.0`."
                )

            if (
                num_batches == 0
                and limit_eval_batches > 0.0
                and isinstance(limit_eval_batches, float)
            ):
                min_pct = 1.0 / len(dataloader)
                raise MisconfigurationException(
                    f"you requested to check {limit_eval_batches} of the {mode} dataloader but"
                    f" {limit_eval_batches}*{num_batches} = 0. Please increase the limit_{mode}_batches."
                    f" Try at least limit_{mode}_batches={min_pct}"
                )

            loader_num_batches.append(num_batches)

    return loader_num_batches, dataloaders


# Traceback (TypeError)
python main.py # with bs=1
GPU available: True, used: False
TPU available: False, using: 0 TPU cores

| Name | Type   | Params
--------------------------------
0 | fst  | Linear | 21
1 | snd  | Linear | 40
/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/pytorch_lightning/utilities/distributed.py:25: UserWarning: The dataloader, train dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 8 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.
warnings.warn(*args, **kwargs)
/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/pytorch_lightning/utilities/distributed.py:25: UserWarning: The dataloader, val dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 8 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.
warnings.warn(*args, **kwargs)
Epoch 1:  50%|████████████████Traceback (most recent call last):████████████████████████████████████████████████████████████████████████                                                                                                                          | 1024/2048 [00:00<00:00, 1337.66it/s, loss=1.032, v_num=19]
File "main.py", line 57, in <module>
trainer.fit(net, train_dataloader=DataLoader(dataset, batch_size=bs), val_dataloaders=DataLoader(dataset, batch_size=bs))
File "/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 954, in fit
self.run_pretrain_routine(model)
File "/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 1093, in run_pretrain_routine
self.train()
File "/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/pytorch_lightning/trainer/training_loop.py", line 375, in train
self.run_training_epoch()
File "/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/pytorch_lightning/trainer/training_loop.py", line 490, in run_training_epoch
self.run_evaluation(test_mode=self.testing)
File "/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/pytorch_lightning/trainer/evaluation_loop.py", line 379, in run_evaluation
eval_results = self._evaluate(self.model, dataloaders, max_batches, test_mode)
File "/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/pytorch_lightning/trainer/evaluation_loop.py", line 281, in _evaluate
output = self.evaluation_forward(model, batch, batch_idx, dataloader_idx, test_mode)
File "/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/pytorch_lightning/trainer/evaluation_loop.py", line 452, in evaluation_forward
output = model.validation_step(*args)
File "main.py", line 39, in validation_step
raise NotImplementedError()
NotImplementedError
Exception ignored in: <object repr() failed>
Traceback (most recent call last):
File "/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/tqdm/std.py", line 1086, in __del__
File "/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/tqdm/std.py", line 1293, in close
File "/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/tqdm/std.py", line 1471, in display
File "/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/tqdm/std.py", line 1089, in __repr__
File "/home/constantin/.virtualenvs/tensor/lib64/python3.6/site-packages/tqdm/std.py", line 1433, in format_dict
TypeError: 'NoneType' object is not iterable

# Fixed code
def _reset_eval_dataloader(
    self, model: LightningModule, mode: str
) -> Tuple[List[Union[int, float]], List[DataLoader]]:
    """Generic method to reset a dataloader for evaluation.

    Args:
        model: The current `LightningModule`
        mode: Either `'val'` or `'test'`

    Returns:
        Tuple (num_batches, dataloaders)
    """
    # use the training loader as val and test when overfitting
    if self.overfit_batches > 0:
        dataloaders = self.request_dataloader(getattr(model, "train_dataloader"))
    else:
        dataloaders = self.request_dataloader(getattr(model, f"{mode}_dataloader"))

    if not isinstance(dataloaders, list):
        dataloaders = [dataloaders]

    for loader_i in range(len(dataloaders)):
        loader = dataloaders[loader_i]

        # shuffling in val and test set is bad practice
        if (
            mode in ("val", "test")
            and hasattr(loader, "sampler")
            and isinstance(loader.sampler, RandomSampler)
        ):
            # when overfitting, the dataloader should not have sampler
            if self.overfit_batches > 0:
                rank_zero_warn(
                    "You requested to overfit but enabled training dataloader shuffling."
                    " We are turning it off for you."
                )
                dataloaders[loader_i] = self.replace_sampler(
                    loader, SequentialSampler(loader.dataset)
                )

            else:
                rank_zero_warn(
                    f"Your {mode}_dataloader has `shuffle=True`, it is best practice to turn"
                    " this off for validation and test dataloaders."
                )

    if any([dl is None for dl in dataloaders]):
        rank_zero_warn("One of given dataloaders is None and it will be skipped.")

    # add samplers
    dataloaders = [
        self.auto_add_sampler(dl, train=False) for dl in dataloaders if dl is not None
    ]

    loader_num_batches = []

    # determine number of batches
    # datasets could be none, 1 or 2+
    if len(dataloaders) != 0:
        for i, dataloader in enumerate(dataloaders):
            num_batches = len(dataloader) if _has_len(dataloader) else float("inf")
            self._worker_check(dataloader, f"{mode} dataloader {i}")

            # percent or num_steps
            limit_eval_batches = getattr(self, f"limit_{mode}_batches")

            if num_batches != float("inf"):
                self._check_batch_limits(f"limit_{mode}_batches")

                # limit num batches either as a percent or num steps
                if isinstance(limit_eval_batches, float):
                    num_batches = int(num_batches * limit_eval_batches)
                else:
                    num_batches = limit_eval_batches

            elif limit_eval_batches not in (0.0, 1.0):
                raise MisconfigurationException(
                    "When using an infinite DataLoader (e.g. with an IterableDataset"
                    f" or when DataLoader does not implement `__len__`) for `limit_{mode}_batches`,"
                    f" `Trainer(limit_{mode}_batches)` must be `0.0` or `1.0`."
                )

            if (
                num_batches == 0
                and limit_eval_batches > 0.0
                and isinstance(limit_eval_batches, float)
            ):
                min_pct = 1.0 / len(dataloader)
                raise MisconfigurationException(
                    f"you requested to check {limit_eval_batches} of the {mode} dataloader but"
                    f" {limit_eval_batches}*{num_batches} = 0. Please increase the limit_{mode}_batches."
                    f" Try at least limit_{mode}_batches={min_pct}"
                )

            loader_num_batches.append(num_batches)

    return loader_num_batches, dataloaders