slow new epoch start with setting ddp, num_workers, gpus
❓ Questions and Help
Before asking:

search the issues.
search the docs.


What is your question?
I am training MNIST with below code. 1 GPU training is ok.
But it shows slow start of new epoch when num_workers is a large number and the number of gpus > 2.
Even dataloading itself is slower than with 1gpu.
Code
import torch
from torch import nn
import pytorch_lightning as pl
from torchvision import datasets, transforms
from torch.utils.data import DataLoader, random_split
from torchvision.datasets import MNIST
from torch.nn import functional as F
import torch.distributed as dist
import os, sys
class LightningMNISTClassifier(pl.LightningModule):
def init(self):
super(LightningMNISTClassifier, self).init()
self.layer_1 = nn.Linear(28*28, 128)
self.layer_2 = nn.Linear(128, 256)
self.layer_3 = nn.Linear(256, 10)
def forward(self, x):
    batch_size, channels, width, height = x.size()
    x = x.view(batch_size, -1)
    x = self.layer_1(x)
    x = torch.relu(x)
    x = self.layer_2(x)
    x = torch.relu(x)
    x = self.layer_3(x)
    x = torch.log_softmax(x, dim=-1)
    return x

def prepare_data(self):
    transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081,))])
    mnist_train = MNIST(os.getcwd(), train=True, download=True, transform=transform)
    self.mnist_test = MNIST(os.getcwd(), train=False, download=True, transform=transform)
    self.mnist_train, self.mnist_val = random_split(mnist_train, [55000,5000])

def train_dataloader(self):
    data_loader2 = DataLoader(self.mnist_train, batch_size=64, num_workers=7, shuffle=True) 
    # data_loader2 = DataLoader(self.mnist_train, batch_size=64, shuffle=True) 
    return data_loader2
def val_dataloader(self):
    return DataLoader(self.mnist_val, batch_size=64)

# def test_dataloader(self):
#     return DataLoader(self.mnist_test, batch_size=64)

def configure_optimizers(self):
    optimizer = torch.optim.Adam(self.parameters(), lr=1e-3)
    return optimizer

def cross_entropy_loss(self, logits, labels):
    return F.nll_loss(logits, labels)

def training_step(self, batch, batch_idx):
    x, y = batch
    logits = self.forward(x)
    loss = self.cross_entropy_loss(logits, y)
    logs = {'train_loss': loss}
    return {"loss": loss, "log": logs}
def validation_step(self, batch, batch_idx):
    x, y = batch
    logits = self.forward(x)
    loss = self.cross_entropy_loss(logits, y)
    return {"val_loss": loss}

def validation_epoch_end(self, outputs):
    avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()
    tensorboard_logs = {"val_loss": avg_loss}
    return {"avg_val_loss": avg_loss, 'log':tensorboard_logs}

if name == 'main':
model = LightningMNISTClassifier()
trainer = pl.Trainer(gpus=4, distributed_backend='ddp')

trainer.fit(model)

What have you tried?
Horovod backend does not show slow start of new epoch.
What's your environment?

OS: ubuntu 18.04
Packaging pip
Version pytorch 1.5.0, 0.7.6
I found that the slow deprecationwarnings shown above  are due to the torchvision library. I changed to a simple dataset and the slow start disappeared until now.