# Buggy code
def _reset_eval_dataloader(
    self, model: LightningModule, mode: str
) -> Tuple[List[Union[int, float]], List[DataLoader]]:
    """Generic method to reset a dataloader for evaluation.

    Args:
        model: The current `LightningModule`
        mode: Either `'val'` or `'test'`

    Returns:
        Tuple (num_batches, dataloaders)
    """
    # use the training loader as val and test when overfitting
    loader_name = f"{mode}_dataloader"
    if self.overfit_batches > 0:
        loader_name = "train_dataloader"

    # load loaders
    dataloaders = self.request_dataloader(getattr(model, loader_name))

    if not isinstance(dataloaders, list):
        dataloaders = [dataloaders]

    self.dev_debugger.track_load_dataloader_call(loader_name, dataloaders=dataloaders)

    for loader_i in range(len(dataloaders)):
        loader = dataloaders[loader_i]

        # shuffling in val and test set is bad practice
        if (
            mode in ("val", "test")
            and hasattr(loader, "sampler")
            and isinstance(loader.sampler, RandomSampler)
        ):
            # when overfitting, the dataloader should not have sampler
            if self.overfit_batches > 0:
                rank_zero_warn(
                    "You requested to overfit but enabled test/val dataloader shuffling."
                    " We are turning it off for you."
                )
                dataloaders[loader_i] = self.replace_sampler(
                    loader, SequentialSampler(loader.dataset)
                )

            else:
                rank_zero_warn(
                    f"Your {mode}_dataloader has `shuffle=True`, it is best practice to turn"
                    " this off for validation and test dataloaders."
                )

    if any([dl is None for dl in dataloaders]):
        rank_zero_warn("One of given dataloaders is None and it will be skipped.")

    # add samplers
    dataloaders = [
        self.auto_add_sampler(dl, train=False) for dl in dataloaders if dl is not None
    ]

    loader_num_batches = []

    # determine number of batches
    # datasets could be none, 1 or 2+
    if len(dataloaders) != 0:
        for i, dataloader in enumerate(dataloaders):
            num_batches = len(dataloader) if has_len(dataloader) else float("inf")
            self._worker_check(dataloader, f"{mode} dataloader {i}")

            # percent or num_steps
            limit_eval_batches = getattr(self, f"limit_{mode}_batches")

            # limit num batches either as a percent or num steps
            if isinstance(limit_eval_batches, int) or limit_eval_batches == 0.0:
                num_batches = min(num_batches, int(limit_eval_batches))
            elif num_batches != float("inf"):
                num_batches = int(num_batches * limit_eval_batches)
            elif limit_eval_batches != 1.0:
                raise MisconfigurationException(
                    "When using an IterableDataset for `limit_{mode}_batches`,"
                    f" `Trainer(limit_{mode}_batches)` must be `0.0`, `1.0` or an int. An int k specifies"
                    f" `num_{mode}_batches` to use."
                )

            if (
                num_batches == 0
                and limit_eval_batches > 0.0
                and isinstance(limit_eval_batches, float)
            ):
                min_pct = 1.0 / len(dataloader)
                raise MisconfigurationException(
                    f"you requested to check {limit_eval_batches} of the {mode} dataloader but"
                    f" {limit_eval_batches}*{num_batches} < 1. Please increase the limit_{mode}_batches."
                    f" Try at least limit_{mode}_batches={min_pct}"
                )

            loader_num_batches.append(num_batches)

    return loader_num_batches, dataloaders


# Traceback (TypeError)
TypeError                                 Traceback (most recent call last)

<ipython-input-5-c33b987ae54f> in <module>()
1 trainer = pl.Trainer(overfit_batches=0.5)
----> 2 trainer.fit(model)

3 frames

/usr/local/lib/python3.6/dist-packages/pytorch_lightning/trainer/trainer.py in fit(self, model, train_dataloader, val_dataloaders)
1018             self.optimizers, self.lr_schedulers, self.optimizer_frequencies = self.init_optimizers(model)
1019
-> 1020             self.run_pretrain_routine(model)
1021
1022         # callbacks

/usr/local/lib/python3.6/dist-packages/pytorch_lightning/trainer/trainer.py in run_pretrain_routine(self, model)
1137                                           self.val_dataloaders,
1138                                           max_batches,
-> 1139                                           False)
1140
1141             # allow no returns from eval

/usr/local/lib/python3.6/dist-packages/pytorch_lightning/trainer/evaluation_loop.py in _evaluate(self, model, dataloaders, max_batches, test_mode)
291                         output = self.evaluation_forward(model, batch, batch_idx, dataloader_idx, test_mode)
292                 else:
--> 293                     output = self.evaluation_forward(model, batch, batch_idx, dataloader_idx, test_mode)
294
295                 # on dp / ddp2 might still want to do something with the batch parts

/usr/local/lib/python3.6/dist-packages/pytorch_lightning/trainer/evaluation_loop.py in evaluation_forward(self, model, batch, batch_idx, dataloader_idx, test_mode)
483             output = model.test_step(*args)
484         else:
--> 485             output = model.validation_step(*args)
486
487         return output

TypeError: validation_step() missing 1 required positional argument: 'dataloader_idx'

# Fixed code
def _reset_eval_dataloader(
    self, model: LightningModule, mode: str
) -> Tuple[List[Union[int, float]], List[DataLoader]]:
    """Generic method to reset a dataloader for evaluation.

    Args:
        model: The current `LightningModule`
        mode: Either `'val'` or `'test'`

    Returns:
        Tuple (num_batches, dataloaders)
    """
    # always get the loaders first so we can count how many there are
    loader_name = f"{mode}_dataloader"
    dataloaders = self.request_dataloader(getattr(model, loader_name))

    # when overfitting use the training loader as val and test
    # duplicate it the numb of times needed to match the train loaders
    if self.overfit_batches > 0:
        num_loaders = len(dataloaders)
        train_dataloader = self.request_dataloader(getattr(model, "train_dataloader"))
        dataloaders = [deepcopy(train_dataloader) for _ in range(num_loaders)]

    if not isinstance(dataloaders, list):
        dataloaders = [dataloaders]

    self.dev_debugger.track_load_dataloader_call(loader_name, dataloaders=dataloaders)

    for loader_i in range(len(dataloaders)):
        loader = dataloaders[loader_i]

        # shuffling in val and test set is bad practice
        if (
            mode in ("val", "test")
            and hasattr(loader, "sampler")
            and isinstance(loader.sampler, RandomSampler)
        ):
            # when overfitting, the dataloader should not have sampler
            if self.overfit_batches > 0:
                rank_zero_warn(
                    "You requested to overfit but enabled test/val dataloader shuffling."
                    " We are turning it off for you."
                )
                dataloaders[loader_i] = self.replace_sampler(
                    loader, SequentialSampler(loader.dataset)
                )

            else:
                rank_zero_warn(
                    f"Your {mode}_dataloader has `shuffle=True`, it is best practice to turn"
                    " this off for validation and test dataloaders."
                )

    if any([dl is None for dl in dataloaders]):
        rank_zero_warn("One of given dataloaders is None and it will be skipped.")

    # add samplers
    dataloaders = [
        self.auto_add_sampler(dl, train=False) for dl in dataloaders if dl is not None
    ]

    loader_num_batches = []

    # determine number of batches
    # datasets could be none, 1 or 2+
    if len(dataloaders) != 0:
        for i, dataloader in enumerate(dataloaders):
            num_batches = len(dataloader) if has_len(dataloader) else float("inf")
            self._worker_check(dataloader, f"{mode} dataloader {i}")

            # percent or num_steps
            limit_eval_batches = getattr(self, f"limit_{mode}_batches")

            # limit num batches either as a percent or num steps
            if isinstance(limit_eval_batches, int) or limit_eval_batches == 0.0:
                num_batches = min(num_batches, int(limit_eval_batches))
            elif num_batches != float("inf"):
                num_batches = int(num_batches * limit_eval_batches)
            elif limit_eval_batches != 1.0:
                raise MisconfigurationException(
                    "When using an IterableDataset for `limit_{mode}_batches`,"
                    f" `Trainer(limit_{mode}_batches)` must be `0.0`, `1.0` or an int. An int k specifies"
                    f" `num_{mode}_batches` to use."
                )

            if (
                num_batches == 0
                and limit_eval_batches > 0.0
                and isinstance(limit_eval_batches, float)
            ):
                min_pct = 1.0 / len(dataloader)
                raise MisconfigurationException(
                    f"you requested to check {limit_eval_batches} of the {mode} dataloader but"
                    f" {limit_eval_batches}*{num_batches} < 1. Please increase the limit_{mode}_batches."
                    f" Try at least limit_{mode}_batches={min_pct}"
                )

            loader_num_batches.append(num_batches)

    return loader_num_batches, dataloaders