question: slow new epoch start with setting ddp, num_workers, gpus
context: ❓ Questions and Help
  Before asking:
  
  search the issues.
  search the docs.
  
  
  What is your question?
  I am training MNIST with below code. 1 GPU training is ok.
  But it shows slow start of new epoch when num_workers is a large number and the number of gpus > 2.
  Even dataloading itself is slower than with 1gpu.
  Code
  import torch
  from torch import nn
  import pytorch_lightning as pl
  from torchvision import datasets, transforms
  from torch.utils.data import DataLoader, random_split
  from torchvision.datasets import MNIST
  from torch.nn import functional as F
  import torch.distributed as dist
  import os, sys
  class LightningMNISTClassifier(pl.LightningModule):
  def init(self):
  super(LightningMNISTClassifier, self).init()
  self.layer_1 = nn.Linear(28*28, 128)
  self.layer_2 = nn.Linear(128, 256)
  self.layer_3 = nn.Linear(256, 10)
  def forward(self, x):
      batch_size, channels, width, height = x.size()
      x = x.view(batch_size, -1)
      x = self.layer_1(x)
      x = torch.relu(x)
      x = self.layer_2(x)
      x = torch.relu(x)
      x = self.layer_3(x)
      x = torch.log_softmax(x, dim=-1)
      return x
  
  def prepare_data(self):
      transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081,))])
      mnist_train = MNIST(os.getcwd(), train=True, download=True, transform=transform)
      self.mnist_test = MNIST(os.getcwd(), train=False, download=True, transform=transform)
      self.mnist_train, self.mnist_val = random_split(mnist_train, [55000,5000])
  
  def train_dataloader(self):
      data_loader2 = DataLoader(self.mnist_train, batch_size=64, num_workers=7, shuffle=True) 
      # data_loader2 = DataLoader(self.mnist_train, batch_size=64, shuffle=True) 
      return data_loader2
  def val_dataloader(self):
      return DataLoader(self.mnist_val, batch_size=64)
  
  # def test_dataloader(self):
  #     return DataLoader(self.mnist_test, batch_size=64)
  
  def configure_optimizers(self):
      optimizer = torch.optim.Adam(self.parameters(), lr=1e-3)
      return optimizer
  
  def cross_entropy_loss(self, logits, labels):
      return F.nll_loss(logits, labels)
  
  def training_step(self, batch, batch_idx):
      x, y = batch
      logits = self.forward(x)
      loss = self.cross_entropy_loss(logits, y)
      logs = {'train_loss': loss}
      return {"loss": loss, "log": logs}
  def validation_step(self, batch, batch_idx):
      x, y = batch
      logits = self.forward(x)
      loss = self.cross_entropy_loss(logits, y)
      return {"val_loss": loss}
  
  def validation_epoch_end(self, outputs):
      avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()
      tensorboard_logs = {"val_loss": avg_loss}
      return {"avg_val_loss": avg_loss, 'log':tensorboard_logs}
  
  if name == 'main':
  model = LightningMNISTClassifier()
  trainer = pl.Trainer(gpus=4, distributed_backend='ddp')
  
  trainer.fit(model)
  
  What have you tried?
  Horovod backend does not show slow start of new epoch.
  What's your environment?
  
  OS: ubuntu 18.04
  Packaging pip
  Version pytorch 1.5.0, 0.7.6
answer: I found that the slow deprecationwarnings shown above  are due to the torchvision library. I changed to a simple dataset and the slow start disappeared until now.