DDP crashing with small sets
🐛 Bug
When I try to use DDP with a small validation set, I get crashes. If you run the following code:

 Code Sample 
import pytorch_lightning as pl
import torch.nn as nn
import torch

class RandomDataset(torch.utils.data.Dataset):
    def __init__(self, length: int):
        self.length = length
    def __len__(self):
        return self.length
    def __getitem__(self, idx: int):
        return torch.rand(100), torch.rand(5)

class TestModule(pl.LightningModule):
    def __init__(self):
        super().__init__()
        self.network = nn.Linear(100, 5)
        self.loss = nn.MSELoss()

    def configure_optimizers(self):
        return torch.optim.Adam(self.network.parameters(), 1e-3)
    def training_step(self, batch, batch_idx: int) -> float:
        X, y = batch
        y_hat = self.network.forward(X)
        return self.loss(y, y_hat)
    def validation_step(self, batch, batch_idx: int) -> None:
        X, y = batch
        y_hat = self.network.forward(X)

if __name__ == "__main__":
    train_loader = torch.utils.data.DataLoader(RandomDataset(50), batch_size=4)
    val_loader = torch.utils.data.DataLoader(RandomDataset(3), batch_size=1)

    mod = TestModule()

    trainer = pl.Trainer(
        callbacks=[pl.callbacks.ProgressBar()],
        fast_dev_run=True,
        accelerator="ddp_cpu",
        num_processes=7)
    trainer.fit(mod, train_loader, val_loader)

fails with

 Stack Trace 
Traceback (most recent call last):
  File "/opt/anaconda/envs/ctrldev/lib/python3.6/site-packages/torch/multiprocessing/spawn.py", line 19, in _wrap
    fn(i, *args)
  File "/opt/anaconda/envs/ctrldev/lib/python3.6/site-packages/pytorch_lightning/accelerators/ddp_cpu_spawn_accelerator.py", line 147, in ddp_train
    results = self.train_or_test()
  File "/opt/anaconda/envs/ctrldev/lib/python3.6/site-packages/pytorch_lightning/accelerators/accelerator.py", line 66, in train_or_test
    results = self.trainer.train()
  File "/opt/anaconda/envs/ctrldev/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 494, in train
    self.train_loop.run_training_epoch()
  File "/opt/anaconda/envs/ctrldev/lib/python3.6/site-packages/pytorch_lightning/trainer/training_loop.py", line 589, in run_training_epoch
    self.trainer.run_evaluation(test_mode=False)
  File "/opt/anaconda/envs/ctrldev/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 567, in run_evaluation
    for batch_idx, batch in enumerate(dataloader):
  File "/opt/anaconda/envs/ctrldev/lib/python3.6/site-packages/torch/utils/data/dataloader.py", line 435, in __next__
    data = self._next_data()
  File "/opt/anaconda/envs/ctrldev/lib/python3.6/site-packages/torch/utils/data/dataloader.py", line 474, in _next_data
    index = self._next_index()  # may raise StopIteration
  File "/opt/anaconda/envs/ctrldev/lib/python3.6/site-packages/torch/utils/data/dataloader.py", line 427, in _next_index
    return next(self._sampler_iter)  # may raise StopIteration
  File "/opt/anaconda/envs/ctrldev/lib/python3.6/site-packages/torch/utils/data/sampler.py", line 227, in __iter__
    for idx in self.sampler:
  File "/opt/anaconda/envs/ctrldev/lib/python3.6/site-packages/torch/utils/data/distributed.py", line 107, in __iter__
    assert len(indices) == self.total_size
AssertionError


Interestingly enough, this seems to depend on num_process -- if num_process=6, then this code does not crash, but at num_process=7 it starts crashing (despite the # of samples in the validation dataset only being 3).
Environment
* CUDA:
	- GPU:
	- available:         False
	- version:           None
* Packages:
	- numpy:             1.16.5
	- pyTorch_debug:     True
	- pyTorch_version:   1.7.0
	- pytorch-lightning: 1.0.8
	- tqdm:              4.51.0
* System:
	- OS:                Linux
	- architecture:
		- 64bit
		- 
	- processor:         x86_64
	- python:            3.6.10
	- version:           #1 SMP Tue Nov 24 19:16:53 UTC 2020