Multi-gpu DDP backend tries to delete model checkpoints from all processes
🐛 Bug
When running single node multi-gpu distributed training on standard MNIST example the following error appears in model checkpoint callback. Debugging showed that both worker processes tried to delete the same checkpoint file (instead of just rank 0) and os.remove fired an exception.
To Reproduce
Steps to reproduce the behavior:
app.py:
"""
see https://pytorch-lightning.readthedocs.io/en/latest/introduction_guide.html
"""
import pathlib

import hydra
import pytorch_lightning as pl
import torch
from omegaconf import OmegaConf
from torch.nn import functional as F
from torch.optim import Adam
from torch.utils.data import DataLoader, random_split
from torchvision import datasets, transforms


class LitMNIST(pl.LightningModule):
    def __init__(self):
        super().__init__()
        self.layer_1 = torch.nn.Linear(28 * 28, 128)
        self.layer_2 = torch.nn.Linear(128, 256)
        self.layer_3 = torch.nn.Linear(256, 10)

        self.train_dataset = None
        self.val_dataset = None
        self.test_dataset = None

    def forward(self, x):
        batch_size, channels, width, height = x.size()
        x = x.view(batch_size, -1)
        x = self.layer_1(x)
        x = F.relu(x)
        x = self.layer_2(x)
        x = F.relu(x)
        x = self.layer_3(x)
        x = F.log_softmax(x, dim=1)
        return x

    def prepare_data(self):
        # transform
        transform = transforms.Compose(
            [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])

        # download
        data_dir = pathlib.Path.home() / 'data'
        mnist_train = datasets.MNIST(data_dir, train=True,
                                     download=True, transform=transform)
        mnist_test = datasets.MNIST(data_dir, train=False,
                                    download=True, transform=transform)

        # train/val split
        mnist_train, mnist_val = random_split(mnist_train, [55000, 5000])

        # assign to use in dataloaders
        self.train_dataset = mnist_train
        self.val_dataset = mnist_val
        self.test_dataset = mnist_test

    def train_dataloader(self):
        return DataLoader(self.train_dataset, batch_size=64)

    def val_dataloader(self):
        return DataLoader(self.val_dataset, batch_size=64)

    def test_dataloader(self):
        return DataLoader(self.test_dataset, batch_size=64)

    def configure_optimizers(self):
        return Adam(self.parameters(), lr=1e-3)

    def training_step(self, batch, batch_idx):
        x, y = batch
        logits = self(x)
        loss = F.nll_loss(logits, y)

        # add logging
        logs = {'loss': loss}
        return {'loss': loss, 'log': logs}

    def validation_step(self, batch, batch_idx):
        x, y = batch
        logits = self(x)
        loss = F.nll_loss(logits, y)
        return {'val_loss': loss}

    def validation_epoch_end(self, outputs):
        avg_loss = torch.stack(  # pylint: disable=no-member
            [x['val_loss'] for x in outputs]).mean()
        tensorboard_logs = {'val_loss': avg_loss}
        return {'avg_val_loss': avg_loss, 'log': tensorboard_logs}

    def test_step(self, batch, batch_idx):
        x, y = batch
        logits = self(x)
        loss = F.nll_loss(logits, y)
        return {'val_loss': loss}

    def test_epoch_end(self, outputs):
        avg_loss = torch.stack(  # pylint: disable=no-member
            [x['val_loss'] for x in outputs]).mean()
        tensorboard_logs = {'val_loss': avg_loss}
        return {'avg_val_loss': avg_loss, 'log': tensorboard_logs}


@hydra.main(config_path='conf.yaml')
def main(conf: OmegaConf):
    model = LitMNIST()

    gpus = int(conf.gpus) if isinstance(conf.gpus, str) else conf.gpus
    num_nodes = (int(conf.num_nodes)
                 if isinstance(conf.num_nodes, str)
                 else conf.num_nodes)

    trainer = pl.Trainer(gpus=gpus,
                         num_nodes=num_nodes,
                         distributed_backend=conf.distributed_backend,
                         max_epochs=3)
    trainer.fit(model)


if __name__ == '__main__':
    main()  # pylint: disable=no-value-for-parameter


conf.yaml:
gpus: 2
num_nodes: 1
distributed_backend: ddp




run python app.py


see error:


-- Process 1 terminated with the following error:
Traceback (most recent call last):
  File "/home/ubuntu/anaconda3/envs/nightly_pt/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 19, in _wrap
    fn(i, *args)
  File "/home/ubuntu/anaconda3/envs/nightly_pt/lib/python3.7/site-packages/pytorch_lightning/trainer/distrib_data_parallel.py", line 342, in ddp_train
    self.run_pretrain_routine(model)
  File "/home/ubuntu/anaconda3/envs/nightly_pt/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 830, in run_pretrain_routine
    self.train()
  File "/home/ubuntu/anaconda3/envs/nightly_pt/lib/python3.7/site-packages/pytorch_lightning/trainer/training_loop.py", line 343, in train
    self.run_training_epoch()
  File "/home/ubuntu/anaconda3/envs/nightly_pt/lib/python3.7/site-packages/pytorch_lightning/trainer/training_loop.py", line 452, in run_training_epoch
    self.call_checkpoint_callback()
  File "/home/ubuntu/anaconda3/envs/nightly_pt/lib/python3.7/site-packages/pytorch_lightning/trainer/training_loop.py", line 737, in call_checkpoint_callback
    self.checkpoint_callback.on_validation_end(self, self.get_model())
  File "/home/ubuntu/anaconda3/envs/nightly_pt/lib/python3.7/site-packages/pytorch_lightning/callbacks/model_checkpoint.py", line 204, in on_validation_end
    self._do_check_save(filepath, current, epoch)
  File "/home/ubuntu/anaconda3/envs/nightly_pt/lib/python3.7/site-packages/pytorch_lightning/callbacks/model_checkpoint.py", line 221, in _do_check_save
    self._del_model(delpath)
  File "/home/ubuntu/anaconda3/envs/nightly_pt/lib/python3.7/site-packages/pytorch_lightning/callbacks/model_checkpoint.py", line 121, in _del_model
    os.remove(filepath)
FileNotFoundError: [Errno 2] No such file or directory: '/home/ubuntu/pytorch-lightning-intro-guide/outputs/2020-04-01/18-27-41/lightning_logs/version_1/checkpoints/epoch=0.ckpt'

Expected behavior
No errors
Environment

PyTorch Version (e.g., 1.0): 1.0
Lightning Version: 0.7.1
OS (e.g., Linux): Ubuntu 16
How you installed PyTorch (conda, pip, source): pip
Python version: 3.7
CUDA/cuDNN version:
GPU models and configuration: K80
Additional Configuration: the node is not managed by SLURM