Repository: Lightning-AI/pytorch-lightning

Issue:
Additional dataloader created and discarded when training with reload_dataloaders_every_epoch
## 🐛 Bug

I am training with reload_dataloaders_every_epoch and I've noticed it instantiates an extra DataLoader before training for which nothing is run. This is an issue for me as I am training with chunks that get loaded every epoch and it is messing with the order I load them in especially if I reload a checkpoint; it would be an issue for people that generate a new dataset every epoch as they waste computation. The tqdm bar also keeps the information of the first, discarded DataLoader (in the screenshot, the number of iterations is the same for both whereas they should be different sizes)

![image](https://user-images.githubusercontent.com/26709476/76968780-be917200-6929-11ea-82c5-6850a6f6e678.png)

### To Reproduce

Run the code sample below, which runs for one epoch and displays a message every time a DataLoader is created.

A DataLoader gets instantiated a first time line 286 in `training_loop.py` outside of the epoch loop (that's the usual time it gets instantiated when not reloading every epoch. Then when using `reload_dataloaders_every_epoch` another one is created at the start of every epoch line 386, inside the loop, so for the first epoch there's an extra one.

#### Code sample

```
import torch
import pytorch_lightning as pl
from torch.utils.data import DataLoader, Dataset
from time import sleep

class MinimalDataset(Dataset):

    def __init__(self, index):
        self.data = torch.Tensor(64 * index, 1024)

    def __getitem__(self, item):
        return self.data[item]

    def __len__(self):
        return len(self.data)

class MinimalModule(pl.LightningModule):

    def __init__(self):
        super(MinimalModule, self).__init__()
        self.nn = torch.nn.Linear(1024, 1)
        self.current_index = 0

    def forward(self, batch):
        return self.nn(batch)

    def training_step(self, batch, batch_idx):
        sleep(0.1)
        loss = self.nn(batch)[0]
        return {'loss': loss}

    def validation_step(self, batch, batch_idx):
        loss = self.nn(batch)[0]
        return {'val_loss': loss}

    def configure_optimizers(self):
        return torch.optim.Adam(self.parameters(), lr=0.01)

    def train_dataloader(self):
        # REQUIRED
        self.current_index += 1
        print(f"initializing DataLoader n{self.current_index}")
        data_loader = DataLoader(MinimalDataset(self.current_index))
        return data_loader
    
model = MinimalModule()
trainer = pl.Trainer(reload_dataloaders_every_epoch=True, num_sanity_val_steps=0, val_check_interval=8, max_epochs=1)

trainer.fit(model)
```

### Expected behavior

Only one dataloader should be created; two are. The tqdm bar should show 128 iterations as that is the dataset size the second time; but it shows 64 instead (I added the sleep(0.1) to leave time to observe that)

### Environment

PyTorch version: 1.4.0
Is debug build: No
CUDA used to build PyTorch: 10.1

OS: Ubuntu 18.04.4 LTS
GCC version: (Ubuntu 7.5.0-3ubuntu1~18.04) 7.5.0
CMake version: Could not collect

Python version: 3.6
Is CUDA available: Yes
CUDA runtime version: Could not collectepoch_end
GPU models and configuration: GPU 0: GeForce RTX 2070 with Max-Q Design
Nvidia driver version: 435.21
cuDNN version: Could not collect

Versions of relevant libraries:
[pip3] numpy==1.18.1
[pip3] pytorch-lightning==0.7.1
[pip3] torch==1.4.0
[pip3] torchvision==0.4.2
[conda] Could not collect

### Additional context




Discussion:
Hi! thanks for your contribution!, great first issue!

Resolving patch:
diff --git a/pytorch_lightning/trainer/evaluation_loop.py b/pytorch_lightning/trainer/evaluation_loop.py
--- a/pytorch_lightning/trainer/evaluation_loop.py
+++ b/pytorch_lightning/trainer/evaluation_loop.py
@@ -338,14 +338,14 @@ def run_evaluation(self, test_mode: bool = False):
 
         # select dataloaders
         if test_mode:
-            if self.reload_dataloaders_every_epoch or self.test_dataloaders is None:
+            if self.test_dataloaders is None:
                 self.reset_test_dataloader(model)
 
             dataloaders = self.test_dataloaders
             max_batches = self.num_test_batches
         else:
             # val
-            if self.reload_dataloaders_every_epoch or self.val_dataloaders is None:
+            if self.val_dataloaders is None:
                 self.reset_val_dataloader(model)
 
             dataloaders = self.val_dataloaders
@@ -399,6 +399,15 @@ def run_evaluation(self, test_mode: bool = False):
         else:
             self.val_progress_bar.close()
 
+        # eventual dataset reloading
+        if test_mode:
+            if self.reload_dataloaders_every_epoch:
+                self.reset_test_dataloader(model)
+        else:
+            # val
+            if self.reload_dataloaders_every_epoch:
+                self.reset_val_dataloader(model)
+
         # Validation/Test end callbacks
         if test_mode:
             self.on_test_end()
diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py
--- a/pytorch_lightning/trainer/trainer.py
+++ b/pytorch_lightning/trainer/trainer.py
@@ -275,7 +275,6 @@ def __init__(
                           " and this method will be removed in v0.8.0", DeprecationWarning)
             self.gradient_clip = gradient_clip
 
-        self.reload_dataloaders_every_epoch = reload_dataloaders_every_epoch
         self.progress_bar_refresh_rate = progress_bar_refresh_rate
         self.check_val_every_n_epoch = check_val_every_n_epoch
         self.track_grad_norm = track_grad_norm
@@ -320,6 +319,8 @@ def __init__(
                           " NaN grads will be printed automatically when detected.",
                           DeprecationWarning)
 
+        self.reload_dataloaders_every_epoch = reload_dataloaders_every_epoch
+
         self.truncated_bptt_steps = truncated_bptt_steps
         self.resume_from_checkpoint = resume_from_checkpoint
         self.shown_warnings = set()
diff --git a/pytorch_lightning/trainer/training_loop.py b/pytorch_lightning/trainer/training_loop.py
--- a/pytorch_lightning/trainer/training_loop.py
+++ b/pytorch_lightning/trainer/training_loop.py
@@ -289,7 +289,9 @@ def train(self):
         model = self.get_model()
 
         # load data
-        self.reset_train_dataloader(model)
+        # if reload_dataloaders_every_epoch, this is moved to the epoch loop
+        if not self.reload_dataloaders_every_epoch:
+            self.reset_train_dataloader(model)
         self.reset_val_dataloader(model)
 
         # Train start events
@@ -305,6 +307,9 @@ def train(self):
         try:
             # run all epochs
             for epoch in range(self.current_epoch, self.max_epochs):
+                # reset train dataloader
+                if self.reload_dataloaders_every_epoch:
+                    self.reset_train_dataloader(model)
                 # set seed for distributed sampler (enables shuffling for each epoch)
                 if self.use_ddp \
                         and hasattr(self.train_dataloader.sampler, 'set_epoch'):
@@ -393,10 +398,6 @@ def run_training_epoch(self):
             if self.is_function_implemented('on_epoch_start'):
                 self.get_model().on_epoch_start()
 
-        # reset train dataloader
-        if self.reload_dataloaders_every_epoch:
-            self.reset_train_dataloader(self.get_model())
-
         # track local dataloader so TPU can wrap each epoch
         train_dataloader = self.train_dataloader
 


Test patch: