# Buggy code
def dp_train(self, model):
    # CHOOSE OPTIMIZER
    # allow for lr schedulers as well
    self.optimizers, self.lr_schedulers, self.optimizer_frequencies = (
        self.init_optimizers(model)
    )

    model.cuda(self.root_gpu)

    # hack forward to do autocast for the user
    model_autocast_original_forward = model.forward
    if self.use_amp and self.use_native_amp:
        # wrap the user's forward in autocast and give it back at the end
        model.forward = torch.cuda.amp.autocast()(model.forward)

    # TODO: remove in v0.8.0
    # check for this bug (amp + dp + !01 doesn't work)
    # https://github.com/NVIDIA/apex/issues/227
    if self.use_dp and self.use_amp and not self.use_native_amp:
        if self.amp_level == "O2":
            raise MisconfigurationException(
                f"Amp level {self.amp_level} with DataParallel is not supported."
                f" See this note from NVIDIA for more info: https://github.com/NVIDIA/apex/issues/227."
                f" We recommend you switch to ddp if you want to use amp"
            )
        else:
            model, optimizers = model.configure_apex(
                amp, model, self.optimizers, self.amp_level
            )
            self.reinit_scheduler_properties(optimizers, self.lr_schedulers)

    # create list of device ids
    device_ids = self.data_parallel_device_ids
    if isinstance(device_ids, int):
        device_ids = list(range(device_ids))

    # set dp device
    torch.cuda.set_device(self.root_gpu)

    model = LightningDataParallel(model, device_ids=device_ids)

    self.run_pretrain_routine(model)

    model.forward = model_autocast_original_forward


# Traceback (AssertionError)
GPU available: True, used: True
TPU available: False, using: 0 TPU cores
CUDA_VISIBLE_DEVICES: [0,1,2,3]
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "/opt/conda/lib/python3.6/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/opt/conda/lib/python3.6/site-packages/pl_examples/basic_examples/gpu_template.py", line 80, in <module>
main(hyperparams)
File "/opt/conda/lib/python3.6/site-packages/pl_examples/basic_examples/gpu_template.py", line 41, in main
trainer.fit(model)
File "/opt/conda/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 860, in fit
self.barrier('fit_prepare_data')
File "/opt/conda/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 1261, in barrier
torch_distrib.barrier()
File "/opt/conda/lib/python3.6/site-packages/torch/distributed/distributed_c10d.py", line 1484, in barrier
_check_default_pg()
File "/opt/conda/lib/python3.6/site-packages/torch/distributed/distributed_c10d.py", line 187, in _check_default_pg
"Default process group is not initialized"
AssertionError: Default process group is not initialized

# Fixed code
def dp_train(self, model):
    # call setup after the ddp process has connected
    self.setup("fit")
    if self.is_function_implemented("setup", model):
        model.setup("fit")

    # CHOOSE OPTIMIZER
    # allow for lr schedulers as well
    self.optimizers, self.lr_schedulers, self.optimizer_frequencies = (
        self.init_optimizers(model)
    )

    model.cuda(self.root_gpu)

    # hack forward to do autocast for the user
    model_autocast_original_forward = model.forward
    if self.use_amp and self.use_native_amp:
        # wrap the user's forward in autocast and give it back at the end
        model.forward = torch.cuda.amp.autocast()(model.forward)

    # TODO: remove in v0.8.0
    # check for this bug (amp + dp + !01 doesn't work)
    # https://github.com/NVIDIA/apex/issues/227
    if self.use_dp and self.use_amp and not self.use_native_amp:
        if self.amp_level == "O2":
            raise MisconfigurationException(
                f"Amp level {self.amp_level} with DataParallel is not supported."
                f" See this note from NVIDIA for more info: https://github.com/NVIDIA/apex/issues/227."
                f" We recommend you switch to ddp if you want to use amp"
            )
        else:
            model, optimizers = model.configure_apex(
                amp, model, self.optimizers, self.amp_level
            )
            self.reinit_scheduler_properties(optimizers, self.lr_schedulers)

    # create list of device ids
    device_ids = self.data_parallel_device_ids
    if isinstance(device_ids, int):
        device_ids = list(range(device_ids))

    # set dp device
    torch.cuda.set_device(self.root_gpu)

    model = LightningDataParallel(model, device_ids=device_ids)

    self.run_pretrain_routine(model)

    model.forward = model_autocast_original_forward