gurudesh/pytracebugs huggingface dataset:
# Buggy code
def __run_ddp_spawn(self, model, nprocs):
    self.set_random_port()

    # pass in a state q
    smp = mp.get_context("spawn")
    q = smp.SimpleQueue()

    mp.spawn(
        self.ddp_train,
        nprocs=nprocs,
        args=(
            q,
            model,
        ),
    )

    # restore main state with best weights
    best_path = q.get()
    results = q.get()
    if best_path is not None and len(best_path) > 0:
        self.checkpoint_callback.best_model_path = best_path
        model.load_from_checkpoint(best_path)

    self.model = model
    return results


# Traceback (AttributeError)
-- Process 0 terminated with the following error:
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/site-packages/torch/multiprocessing/spawn.py", line 20, in _wrap
fn(i, *args)
File "/opt/conda/lib/python3.6/site-packages/pytorch_lightning/trainer/distrib_data_parallel.py", line 562, in ddp_train
q.put(self.checkpoint_callback.best_model_path)
AttributeError: 'NoneType' object has no attribute 'best_model_path'

# Fixed code
def __run_ddp_spawn(self, model, nprocs):
    self.set_random_port()

    # pass in a state q
    smp = mp.get_context("spawn")
    q = smp.SimpleQueue()

    mp.spawn(
        self.ddp_train,
        nprocs=nprocs,
        args=(
            q,
            model,
        ),
    )

    # restore main state with best weights
    best_path = q.get()
    results = q.get()
    last_path = q.get()

    # transfer back the best path to the trainer
    self.checkpoint_callback.best_model_path = best_path

    # load last weights
    if last_path is not None and not self.testing:
        torch.load(last_path, map_location=lambda storage, loc: storage)

    self.model = model
    return results
# Buggy code
def teardown(self, model):
    # restore main state with best weights
    best_path = self.mp_queue.get()
    results = self.mp_queue.get()
    last_path = self.mp_queue.get()

    # transfer back the best path to the trainer
    self.trainer.checkpoint_callback.best_model_path = best_path
    # todo, pass also bets score

    # load last weights
    if last_path is not None and not self.trainer.testing:
        ckpt = torch.load(last_path, map_location=lambda storage, loc: storage)
        model.load_state_dict(ckpt)

    self.trainer.model = model
    return results


# Traceback (AttributeError)
-- Process 0 terminated with the following error:
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/site-packages/torch/multiprocessing/spawn.py", line 20, in _wrap
fn(i, *args)
File "/opt/conda/lib/python3.6/site-packages/pytorch_lightning/trainer/distrib_data_parallel.py", line 562, in ddp_train
q.put(self.checkpoint_callback.best_model_path)
AttributeError: 'NoneType' object has no attribute 'best_model_path'

# Fixed code
def teardown(self, model):
    # restore main state with best weights
    best_path = self.mp_queue.get()
    results = self.mp_queue.get()
    last_path = self.mp_queue.get()

    # transfer back the best path to the trainer
    if self.trainer.checkpoint_callback:
        self.trainer.checkpoint_callback.best_model_path = best_path
    # todo, pass also bets score

    # load last weights
    if last_path is not None and not self.trainer.testing:
        ckpt = torch.load(last_path, map_location=lambda storage, loc: storage)
        model.load_state_dict(ckpt)

    self.trainer.model = model
    return results
# Buggy code
def _forward_example_input(self) -> None:
    """Run the example input through each layer to get input- and output sizes."""
    model = self._model
    trainer = self._model.trainer

    input_ = model.example_input_array
    input_ = model.transfer_batch_to_device(input_, model.device)

    if trainer is not None and trainer.use_amp:
        if NATIVE_AMP_AVALAIBLE:
            model.forward = torch.cuda.amp.autocast()(model.forward)

    mode = model.training
    model.eval()
    with torch.no_grad():
        # let the model hooks collect the input- and output shapes
        if isinstance(input_, (list, tuple)):
            model(*input_)
        elif isinstance(input_, dict):
            model(**input_)
        else:
            model(input_)
    model.train(mode)  # restore mode of module


# Traceback (ModuleNotFoundError)
GPU available: False, used: False
TPU available: True, using: 8 TPU cores
Traceback (most recent call last):
File "bert_ner/light/fp16_debug.py", line 16, in <module>
trainer = pl.Trainer(tpu_cores=8, precision=16)
File "/anaconda3/envs/torch-xla-1.5/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 607, in __init__
self.init_amp()
File "/anaconda3/envs/torch-xla-1.5/lib/python3.6/site-packages/pytorch_lightning/trainer/auto_mix_precision.py", line 27, in init_amp
"You set `use_amp=True` but do not have apex installed."
ModuleNotFoundError: You set `use_amp=True` but do not have apex installed.Install apex first using this guide and rerun with use_amp=True:https://github.com/NVIDIA/apex#linux his run will NOT use 16 bit precision

# Fixed code
def _forward_example_input(self) -> None:
    """Run the example input through each layer to get input- and output sizes."""
    model = self._model
    trainer = self._model.trainer

    input_ = model.example_input_array
    input_ = model.transfer_batch_to_device(input_, model.device)

    if trainer is not None and trainer.use_amp and not trainer.use_tpu:
        if NATIVE_AMP_AVALAIBLE:
            model.forward = torch.cuda.amp.autocast()(model.forward)

    mode = model.training
    model.eval()
    with torch.no_grad():
        # let the model hooks collect the input- and output shapes
        if isinstance(input_, (list, tuple)):
            model(*input_)
        elif isinstance(input_, dict):
            model(**input_)
        else:
            model(input_)
    model.train(mode)  # restore mode of module
# Buggy code
def run_batch_backward_pass(self, split_batch, batch_idx, opt_idx, optimizer):
    # ------------------
    # GRAD NORMS
    # ------------------
    # track gradient norms when requested
    grad_norm_dic = {}
    if batch_idx % self.row_log_interval == 0:
        if float(self.track_grad_norm) > 0:
            model = self.get_model()
            grad_norm_dic = model.grad_norm(self.track_grad_norm)

    # ------------------
    # CLIP GRADS
    # ------------------
    if self.use_amp and NATIVE_AMP_AVALAIBLE:
        self.scaler.unscale_(optimizer)
    self.clip_gradients()

    # ------------------
    # .STEP + ZERO_GRAD
    # ------------------
    self.call_optimizer_step(optimizer, opt_idx, batch_idx, split_batch)

    return grad_norm_dic


# Traceback (ModuleNotFoundError)
GPU available: False, used: False
TPU available: True, using: 8 TPU cores
Traceback (most recent call last):
File "bert_ner/light/fp16_debug.py", line 16, in <module>
trainer = pl.Trainer(tpu_cores=8, precision=16)
File "/anaconda3/envs/torch-xla-1.5/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 607, in __init__
self.init_amp()
File "/anaconda3/envs/torch-xla-1.5/lib/python3.6/site-packages/pytorch_lightning/trainer/auto_mix_precision.py", line 27, in init_amp
"You set `use_amp=True` but do not have apex installed."
ModuleNotFoundError: You set `use_amp=True` but do not have apex installed.Install apex first using this guide and rerun with use_amp=True:https://github.com/NVIDIA/apex#linux his run will NOT use 16 bit precision

# Fixed code
def run_batch_backward_pass(self, split_batch, batch_idx, opt_idx, optimizer):
    # ------------------
    # GRAD NORMS
    # ------------------
    # track gradient norms when requested
    grad_norm_dic = {}
    if batch_idx % self.row_log_interval == 0:
        if float(self.track_grad_norm) > 0:
            model = self.get_model()
            grad_norm_dic = model.grad_norm(self.track_grad_norm)

    # ------------------
    # CLIP GRADS
    # ------------------
    if self.use_amp and NATIVE_AMP_AVALAIBLE and not self.use_tpu:
        self.scaler.unscale_(optimizer)
    self.clip_gradients()

    # ------------------
    # .STEP + ZERO_GRAD
    # ------------------
    self.call_optimizer_step(optimizer, opt_idx, batch_idx, split_batch)

    return grad_norm_dic
# Buggy code
def teardown(self, stage: str):
    """
    Called at the end of fit and test.

    Args:
        step: either 'fit' or 'test'
    """


# Traceback (AssertionError)
GPU available: True, used: True
TPU available: False, using: 0 TPU cores
CUDA_VISIBLE_DEVICES: [0,1,2,3]
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "/opt/conda/lib/python3.6/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/opt/conda/lib/python3.6/site-packages/pl_examples/basic_examples/gpu_template.py", line 80, in <module>
main(hyperparams)
File "/opt/conda/lib/python3.6/site-packages/pl_examples/basic_examples/gpu_template.py", line 41, in main
trainer.fit(model)
File "/opt/conda/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 860, in fit
self.barrier('fit_prepare_data')
File "/opt/conda/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 1261, in barrier
torch_distrib.barrier()
File "/opt/conda/lib/python3.6/site-packages/torch/distributed/distributed_c10d.py", line 1484, in barrier
_check_default_pg()
File "/opt/conda/lib/python3.6/site-packages/torch/distributed/distributed_c10d.py", line 187, in _check_default_pg
"Default process group is not initialized"
AssertionError: Default process group is not initialized

# Fixed code
def teardown(self, stage: str):
    """
    Called at the end of fit and test.

    Args:
        stage: either 'fit' or 'test'
    """
# Buggy code
def single_gpu_train(self, model):
    model.cuda(self.root_gpu)

    # CHOOSE OPTIMIZER
    # allow for lr schedulers as well
    self.optimizers, self.lr_schedulers, self.optimizer_frequencies = (
        self.init_optimizers(model)
    )

    # TODO: update for 0.8.0
    if self.use_amp and not self.use_native_amp:
        # An example
        model, optimizers = model.configure_apex(
            amp, model, self.optimizers, self.amp_level
        )
        self.optimizers = optimizers
        self.reinit_scheduler_properties(self.optimizers, self.lr_schedulers)

    self.run_pretrain_routine(model)


# Traceback (AssertionError)
GPU available: True, used: True
TPU available: False, using: 0 TPU cores
CUDA_VISIBLE_DEVICES: [0,1,2,3]
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "/opt/conda/lib/python3.6/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/opt/conda/lib/python3.6/site-packages/pl_examples/basic_examples/gpu_template.py", line 80, in <module>
main(hyperparams)
File "/opt/conda/lib/python3.6/site-packages/pl_examples/basic_examples/gpu_template.py", line 41, in main
trainer.fit(model)
File "/opt/conda/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 860, in fit
self.barrier('fit_prepare_data')
File "/opt/conda/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 1261, in barrier
torch_distrib.barrier()
File "/opt/conda/lib/python3.6/site-packages/torch/distributed/distributed_c10d.py", line 1484, in barrier
_check_default_pg()
File "/opt/conda/lib/python3.6/site-packages/torch/distributed/distributed_c10d.py", line 187, in _check_default_pg
"Default process group is not initialized"
AssertionError: Default process group is not initialized

# Fixed code
def single_gpu_train(self, model):
    # call setup
    self.setup("fit")
    if self.is_function_implemented("setup", model):
        model.setup("fit")

    model.cuda(self.root_gpu)

    # CHOOSE OPTIMIZER
    # allow for lr schedulers as well
    self.optimizers, self.lr_schedulers, self.optimizer_frequencies = (
        self.init_optimizers(model)
    )

    # TODO: update for 0.8.0
    if self.use_amp and not self.use_native_amp:
        # An example
        model, optimizers = model.configure_apex(
            amp, model, self.optimizers, self.amp_level
        )
        self.optimizers = optimizers
        self.reinit_scheduler_properties(self.optimizers, self.lr_schedulers)

    self.run_pretrain_routine(model)
# Buggy code
def is_function_implemented(self, *args):
    """Warning: this is just empty shell for code implemented in other class."""


# Traceback (AssertionError)
GPU available: True, used: True
TPU available: False, using: 0 TPU cores
CUDA_VISIBLE_DEVICES: [0,1,2,3]
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "/opt/conda/lib/python3.6/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/opt/conda/lib/python3.6/site-packages/pl_examples/basic_examples/gpu_template.py", line 80, in <module>
main(hyperparams)
File "/opt/conda/lib/python3.6/site-packages/pl_examples/basic_examples/gpu_template.py", line 41, in main
trainer.fit(model)
File "/opt/conda/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 860, in fit
self.barrier('fit_prepare_data')
File "/opt/conda/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 1261, in barrier
torch_distrib.barrier()
File "/opt/conda/lib/python3.6/site-packages/torch/distributed/distributed_c10d.py", line 1484, in barrier
_check_default_pg()
File "/opt/conda/lib/python3.6/site-packages/torch/distributed/distributed_c10d.py", line 187, in _check_default_pg
"Default process group is not initialized"
AssertionError: Default process group is not initialized

# Fixed code
def is_function_implemented(self, *args, **kwargs):
    """Warning: this is just empty shell for code implemented in other class."""
# Buggy code
def run_training_teardown(self):
    if hasattr(self, "_teardown_already_run") and self._teardown_already_run:
        return
    # Train end events
    with self.profiler.profile("on_train_end"):
        # callbacks
        self.on_train_end()
        # model hooks
        if self.is_function_implemented("on_train_end"):
            self.get_model().on_train_end()

    if self.logger is not None:
        self.logger.finalize("success")

    # summarize profile results
    self.profiler.describe()
    self._teardown_already_run = True


# Traceback (AssertionError)
GPU available: True, used: True
TPU available: False, using: 0 TPU cores
CUDA_VISIBLE_DEVICES: [0,1,2,3]
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "/opt/conda/lib/python3.6/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/opt/conda/lib/python3.6/site-packages/pl_examples/basic_examples/gpu_template.py", line 80, in <module>
main(hyperparams)
File "/opt/conda/lib/python3.6/site-packages/pl_examples/basic_examples/gpu_template.py", line 41, in main
trainer.fit(model)
File "/opt/conda/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 860, in fit
self.barrier('fit_prepare_data')
File "/opt/conda/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 1261, in barrier
torch_distrib.barrier()
File "/opt/conda/lib/python3.6/site-packages/torch/distributed/distributed_c10d.py", line 1484, in barrier
_check_default_pg()
File "/opt/conda/lib/python3.6/site-packages/torch/distributed/distributed_c10d.py", line 187, in _check_default_pg
"Default process group is not initialized"
AssertionError: Default process group is not initialized

# Fixed code
def run_training_teardown(self):
    if hasattr(self, "_teardown_already_run") and self._teardown_already_run:
        return

    # clean up dist group
    if self.use_ddp or self.use_ddp2:
        torch_distrib.destroy_process_group()

    # Train end events
    with self.profiler.profile("on_train_end"):
        # callbacks
        self.on_train_end()
        # model hooks
        if self.is_function_implemented("on_train_end"):
            self.get_model().on_train_end()

    if self.logger is not None:
        self.logger.finalize("success")

    # summarize profile results
    self.profiler.describe()
    self._teardown_already_run = True
# Buggy code
def hparams(self, hp: Union[dict, Namespace, Any]):
    self.save_hyperparameters(hp, frame=inspect.currentframe().f_back.f_back)


# Traceback (ValueError)
Traceback (most recent call last):
File "train_test.py", line 37, in <module>
model = module(hparams, some_data)
File "train_test.py", line 16, in __init__
self.hparams = hparams
File "/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py", line 638, in __setattr__
object.__setattr__(self, name, value)
File "/opt/conda/lib/python3.6/site-packages/pytorch_lightning/core/lightning.py", line 1695, in hparams
self.save_hyperparameters(hp, frame=inspect.currentframe().f_back.f_back)
File "/opt/conda/lib/python3.6/site-packages/pytorch_lightning/core/lightning.py", line 1662, in save_hyperparameters
cand_names = [k for k, v in init_args.items() if v == hp]
File "/opt/conda/lib/python3.6/site-packages/pytorch_lightning/core/lightning.py", line 1662, in <listcomp>
cand_names = [k for k, v in init_args.items() if v == hp]
File "/opt/conda/lib/python3.6/site-packages/pandas/core/generic.py", line 1479, in __nonzero__
f"The truth value of a {type(self).__name__} is ambiguous. "
ValueError: The truth value of a DataFrame is ambiguous. Use a.empty, a.bool(), a.item(), a.any() or a.all().

# Fixed code
def hparams(self, hp: Union[dict, Namespace, Any]):
    hparams_assignment_name = self.__get_hparams_assignment_variable()
    self._hparams_name = hparams_assignment_name
    self._set_hparams(hp)
# Buggy code
def main(hparams):
    """
    Main training routine specific for this project
    :param hparams:
    """
    # ------------------------
    # 1 INIT LIGHTNING MODEL
    # ------------------------
    model = LightningTemplateModel(hparams)

    # ------------------------
    # 2 INIT TRAINER
    # ------------------------
    trainer = pl.Trainer(
        max_epochs=hparams.epochs,
        gpus=hparams.gpus,
        distributed_backend=hparams.distributed_backend,
        precision=16 if hparams.use_16bit else 32,
    )

    # ------------------------
    # 3 START TRAINING
    # ------------------------
    trainer.fit(model)


# Traceback (AttributeError)
Traceback (most recent call last):
File "/usr/lib/python3.6/runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "/usr/lib/python3.6/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/home/anthony/Downloads/pytorch-lightning/pl_examples/basic_examples/cpu_template.py", line 53, in <module>
main(args)
File "/home/anthony/Downloads/pytorch-lightning/pl_examples/basic_examples/cpu_template.py", line 34, in main
trainer.fit(model)
File "/home/anthony/Downloads/pytorch-lightning/pytorch_lightning/trainer/trainer.py", line 952, in fit
self.run_pretrain_routine(model)
File "/home/anthony/Downloads/pytorch-lightning/pytorch_lightning/trainer/trainer.py", line 1063, in run_pretrain_routine
self.reset_val_dataloader(ref_model)
File "/home/anthony/Downloads/pytorch-lightning/pytorch_lightning/trainer/data_loading.py", line 331, in reset_val_dataloader
self._reset_eval_dataloader(model, 'val')
File "/home/anthony/Downloads/pytorch-lightning/pytorch_lightning/trainer/data_loading.py", line 253, in _reset_eval_dataloader
dataloaders = self.request_dataloader(getattr(model, f'{mode}_dataloader'))
File "/home/anthony/Downloads/pytorch-lightning/pytorch_lightning/trainer/data_loading.py", line 352, in request_dataloader
dataloader = dataloader_fx()
File "/home/anthony/Downloads/pytorch-lightning/pl_examples/models/lightning_template.py", line 158, in val_dataloader
return DataLoader(self.mnist_test, batch_size=self.batch_size, num_workers=4)
File "/home/anthony/.cache/pypoetry/virtualenvs/robotics-zp-60jGk-py3.6/lib/python3.6/site-packages/torch/nn/modules/module.py", line 594, in __getattr__
type(self).__name__, name))
AttributeError: 'LightningTemplateModel' object has no attribute 'mnist_test'

# Fixed code
def main(hparams):
    """
    Main training routine specific for this project
    :param hparams:
    """
    # ------------------------
    # 1 INIT LIGHTNING MODEL
    # ------------------------
    model = LightningTemplateModel(**vars(hparams))

    # ------------------------
    # 2 INIT TRAINER
    # ------------------------
    trainer = pl.Trainer(
        max_epochs=hparams.epochs,
        gpus=hparams.gpus,
        distributed_backend=hparams.distributed_backend,
        precision=16 if hparams.use_16bit else 32,
    )

    # ------------------------
    # 3 START TRAINING
    # ------------------------
    trainer.fit(model)
# Buggy code
def __getstate__(self):
    state = self.__dict__.copy()
    # cannot be pickled
    state["_experiment"] = None
    # args needed to reload correct experiment
    state["_id"] = self.experiment.id
    return state


# Traceback (AttributeError)
-- Process 0 terminated with the following error:
Traceback (most recent call last):
File "/home/rmrao/anaconda3/lib/python3.6/site-packages/torch/multiprocessing/spawn.py", line 19, in _wrap
fn(i, *args)
File "/home/rmrao/anaconda3/lib/python3.6/site-packages/pytorch_lightning/trainer/distrib_data_parallel.py", line 331, in ddp_train
self.run_pretrain_routine(model)
File "/home/rmrao/anaconda3/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 757, in run_pretrain_routine
self.logger.log_hyperparams(ref_model.hparams)
File "/home/rmrao/anaconda3/lib/python3.6/site-packages/pytorch_lightning/logging/base.py", line 14, in wrapped_fn
fn(self, *args, **kwargs)
File "/home/rmrao/anaconda3/lib/python3.6/site-packages/pytorch_lightning/logging/wandb.py", line 79, in log_hyperparams
self.experiment.config.update(params)
AttributeError: 'NoneType' object has no attribute 'config'

# Fixed code
def __getstate__(self):
    state = self.__dict__.copy()
    # args needed to reload correct experiment
    state["_id"] = self._experiment.id if self._experiment is not None else None

    # cannot be pickled
    state["_experiment"] = None
    return state
# Buggy code
def experiment(self) -> Run:
    r"""

    Actual wandb object. To use wandb features do the following.

    Example::

        self.logger.experiment.some_wandb_function()

    """
    if self._experiment is None:
        if self._offline:
            os.environ["WANDB_MODE"] = "dryrun"
        self._experiment = wandb.init(
            name=self._name,
            dir=self._save_dir,
            project=self._project,
            anonymous=self._anonymous,
            id=self._id,
            resume="allow",
            tags=self._tags,
            entity=self._entity,
        )
        # save checkpoints in wandb dir to upload on W&B servers
        if self._log_model:
            self.save_dir = self._experiment.dir
    return self._experiment


# Traceback (AttributeError)
-- Process 0 terminated with the following error:
Traceback (most recent call last):
File "/home/rmrao/anaconda3/lib/python3.6/site-packages/torch/multiprocessing/spawn.py", line 19, in _wrap
fn(i, *args)
File "/home/rmrao/anaconda3/lib/python3.6/site-packages/pytorch_lightning/trainer/distrib_data_parallel.py", line 331, in ddp_train
self.run_pretrain_routine(model)
File "/home/rmrao/anaconda3/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 757, in run_pretrain_routine
self.logger.log_hyperparams(ref_model.hparams)
File "/home/rmrao/anaconda3/lib/python3.6/site-packages/pytorch_lightning/logging/base.py", line 14, in wrapped_fn
fn(self, *args, **kwargs)
File "/home/rmrao/anaconda3/lib/python3.6/site-packages/pytorch_lightning/logging/wandb.py", line 79, in log_hyperparams
self.experiment.config.update(params)
AttributeError: 'NoneType' object has no attribute 'config'

# Fixed code
def experiment(self) -> Run:
    r"""

    Actual wandb object. To use wandb features do the following.

    Example::

        self.logger.experiment.some_wandb_function()

    """
    if self._experiment is None:
        if self._offline:
            os.environ["WANDB_MODE"] = "dryrun"
        self._experiment = wandb.init(
            name=self._name,
            dir=self._save_dir,
            project=self._project,
            anonymous=self._anonymous,
            reinit=True,
            id=self._id,
            resume="allow",
            tags=self._tags,
            entity=self._entity,
        )
        # save checkpoints in wandb dir to upload on W&B servers
        if self._log_model:
            self.save_dir = self._experiment.dir
    return self._experiment
# Buggy code
def name(self) -> str:
    return self.experiment.project_name()


# Traceback (AttributeError)
-- Process 0 terminated with the following error:
Traceback (most recent call last):
File "/home/rmrao/anaconda3/lib/python3.6/site-packages/torch/multiprocessing/spawn.py", line 19, in _wrap
fn(i, *args)
File "/home/rmrao/anaconda3/lib/python3.6/site-packages/pytorch_lightning/trainer/distrib_data_parallel.py", line 331, in ddp_train
self.run_pretrain_routine(model)
File "/home/rmrao/anaconda3/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 757, in run_pretrain_routine
self.logger.log_hyperparams(ref_model.hparams)
File "/home/rmrao/anaconda3/lib/python3.6/site-packages/pytorch_lightning/logging/base.py", line 14, in wrapped_fn
fn(self, *args, **kwargs)
File "/home/rmrao/anaconda3/lib/python3.6/site-packages/pytorch_lightning/logging/wandb.py", line 79, in log_hyperparams
self.experiment.config.update(params)
AttributeError: 'NoneType' object has no attribute 'config'

# Fixed code
def name(self) -> str:
    # don't create an experiment if we don't have one
    name = self._experiment.project_name() if self._experiment else None
    return name
# Buggy code
def version(self) -> str:
    return self.experiment.id


# Traceback (AttributeError)
-- Process 0 terminated with the following error:
Traceback (most recent call last):
File "/home/rmrao/anaconda3/lib/python3.6/site-packages/torch/multiprocessing/spawn.py", line 19, in _wrap
fn(i, *args)
File "/home/rmrao/anaconda3/lib/python3.6/site-packages/pytorch_lightning/trainer/distrib_data_parallel.py", line 331, in ddp_train
self.run_pretrain_routine(model)
File "/home/rmrao/anaconda3/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 757, in run_pretrain_routine
self.logger.log_hyperparams(ref_model.hparams)
File "/home/rmrao/anaconda3/lib/python3.6/site-packages/pytorch_lightning/logging/base.py", line 14, in wrapped_fn
fn(self, *args, **kwargs)
File "/home/rmrao/anaconda3/lib/python3.6/site-packages/pytorch_lightning/logging/wandb.py", line 79, in log_hyperparams
self.experiment.config.update(params)
AttributeError: 'NoneType' object has no attribute 'config'

# Fixed code
def version(self) -> str:
    # don't create an experiment if we don't have one
    return self._experiment.id if self._experiment else None
# Buggy code
def __getstate__(self):
    state = self.__dict__.copy()
    # cannot be pickled
    state["_experiment"] = None
    return state


# Traceback (Exception)
Traceback (most recent call last):
File "pl.py", line 146, in <module>
main()
File "pl.py", line 103, in main
trainer.fit(model)
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 751, in fit
mp.spawn(self.ddp_train, nprocs=self.num_processes, args=(model,))
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 200, in spawn
return start_processes(fn, args, nprocs, join, daemon, start_method='spawn')
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 158, in start_processes
while not context.join():
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 119, in join
raise Exception(msg)
Exception:

-- Process 1 terminated with the following error:
Traceback (most recent call last):
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 20, in _wrap
fn(i, *args)
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/pytorch_lightning/trainer/distrib_data_parallel.py", line 370, in ddp_train
self.run_pretrain_routine(model)
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 871, in run_pretrain_routine
self.configure_checkpoint_callback()
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/pytorch_lightning/trainer/callback_config.py", line 54, in configure_checkpoint_callback
self.logger.name,
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/pytorch_lightning/loggers/neptune.py", line 267, in name
return self.experiment.name
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/pytorch_lightning/loggers/neptune.py", line 230, in experiment
**self._kwargs)
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/neptune/__init__.py", line 222, in create_experiment
raise Uninitialized()
neptune.exceptions.Uninitialized: You must initialize neptune-client first. For more information, please visit: https://github.com/neptune-ai/neptune-client#initialize-neptune

# Fixed code
def __getstate__(self):
    state = self.__dict__.copy()

    # Experiment cannot be pickled, and additionally its ID cannot be pickled in offline mode
    state["_experiment"] = None
    if self.offline_mode:
        state["_experiment_id"] = None

    return state
# Buggy code
def name(self) -> str:
    if self.mode == "offline":
        return "offline-name"
    else:
        return self.experiment.name


# Traceback (Exception)
Traceback (most recent call last):
File "pl.py", line 146, in <module>
main()
File "pl.py", line 103, in main
trainer.fit(model)
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 751, in fit
mp.spawn(self.ddp_train, nprocs=self.num_processes, args=(model,))
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 200, in spawn
return start_processes(fn, args, nprocs, join, daemon, start_method='spawn')
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 158, in start_processes
while not context.join():
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 119, in join
raise Exception(msg)
Exception:

-- Process 1 terminated with the following error:
Traceback (most recent call last):
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 20, in _wrap
fn(i, *args)
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/pytorch_lightning/trainer/distrib_data_parallel.py", line 370, in ddp_train
self.run_pretrain_routine(model)
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 871, in run_pretrain_routine
self.configure_checkpoint_callback()
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/pytorch_lightning/trainer/callback_config.py", line 54, in configure_checkpoint_callback
self.logger.name,
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/pytorch_lightning/loggers/neptune.py", line 267, in name
return self.experiment.name
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/pytorch_lightning/loggers/neptune.py", line 230, in experiment
**self._kwargs)
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/neptune/__init__.py", line 222, in create_experiment
raise Uninitialized()
neptune.exceptions.Uninitialized: You must initialize neptune-client first. For more information, please visit: https://github.com/neptune-ai/neptune-client#initialize-neptune

# Fixed code
def name(self) -> str:
    if self.offline_mode:
        return "offline-name"
    else:
        return self.experiment.name
# Buggy code
def version(self) -> str:
    if self.mode == "offline":
        return "offline-id-1234"
    else:
        return self.experiment.id


# Traceback (Exception)
Traceback (most recent call last):
File "pl.py", line 146, in <module>
main()
File "pl.py", line 103, in main
trainer.fit(model)
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 751, in fit
mp.spawn(self.ddp_train, nprocs=self.num_processes, args=(model,))
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 200, in spawn
return start_processes(fn, args, nprocs, join, daemon, start_method='spawn')
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 158, in start_processes
while not context.join():
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 119, in join
raise Exception(msg)
Exception:

-- Process 1 terminated with the following error:
Traceback (most recent call last):
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/torch/multiprocessing/spawn.py", line 20, in _wrap
fn(i, *args)
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/pytorch_lightning/trainer/distrib_data_parallel.py", line 370, in ddp_train
self.run_pretrain_routine(model)
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 871, in run_pretrain_routine
self.configure_checkpoint_callback()
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/pytorch_lightning/trainer/callback_config.py", line 54, in configure_checkpoint_callback
self.logger.name,
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/pytorch_lightning/loggers/neptune.py", line 267, in name
return self.experiment.name
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/pytorch_lightning/loggers/neptune.py", line 230, in experiment
**self._kwargs)
File "/home/hirune/anaconda3/envs/PANDA/lib/python3.7/site-packages/neptune/__init__.py", line 222, in create_experiment
raise Uninitialized()
neptune.exceptions.Uninitialized: You must initialize neptune-client first. For more information, please visit: https://github.com/neptune-ai/neptune-client#initialize-neptune

# Fixed code
def version(self) -> str:
    if self.offline_mode:
        return "offline-id-1234"
    else:
        return self.experiment.id
# Buggy code
def auto_add_sampler(self, dataloader: DataLoader, train: bool) -> DataLoader:
    # don't do anything if it's not a dataloader
    if not isinstance(dataloader, DataLoader):
        return dataloader

    need_dist_sampler = self.use_ddp or self.use_ddp2 or self.use_tpu

    if need_dist_sampler:
        skip_keys = ["sampler", "batch_sampler", "dataset_kind"]

        dl_args = {
            k: v
            for k, v in dataloader.__dict__.items()
            if not k.startswith("_") and k not in skip_keys
        }

        if self.use_tpu:
            sampler = DistributedSampler(
                dataloader.dataset,
                num_replicas=xm.xrt_world_size(),
                rank=xm.get_ordinal(),
            )
        else:
            sampler = DistributedSampler(dataloader.dataset)

        dl_args["sampler"] = sampler
        dataloader = type(dataloader)(**dl_args)

    return dataloader


# Traceback (TypeError)
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/site-packages/torch/multiprocessing/spawn.py", line 19, in _wrap
fn(i, *args)
File "/opt/conda/lib/python3.6/site-packages/pytorch_lightning/trainer/distrib_data_parallel.py", line 345, in ddp_train
self.run_pretrain_routine(model)
File "/opt/conda/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 864, in run_pretrain_routine
self.train()
File "/opt/conda/lib/python3.6/site-packages/pytorch_lightning/trainer/training_loop.py", line 296, in train
self.reset_train_dataloader(model)
File "/opt/conda/lib/python3.6/site-packages/pytorch_lightning/trainer/data_loading.py", line 128, in reset_train_dataloader
self.train_dataloader = self.auto_add_sampler(self.train_dataloader, train=True)
File "/opt/conda/lib/python3.6/site-packages/pytorch_lightning/trainer/data_loading.py", line 112, in auto_add_sampler
dataloader = type(dataloader)(**dl_args)
File "../main/dataset.py", line 15, in __init__
super().__init__(*args, **kwargs)
TypeError: __init__() got an unexpected keyword argument 'iterator'

# Fixed code
def auto_add_sampler(self, dataloader: DataLoader, train: bool) -> DataLoader:
    # don't do anything if it's not a dataloader
    if not isinstance(dataloader, DataLoader):
        return dataloader
    need_dist_sampler = self.use_ddp or self.use_ddp2 or self.use_tpu
    if self.replace_sampler_ddp and need_dist_sampler:
        skip_keys = ["sampler", "batch_sampler", "dataset_kind"]

        dl_args = {
            k: v
            for k, v in dataloader.__dict__.items()
            if not k.startswith("_") and k not in skip_keys
        }

        if self.use_tpu:
            sampler = DistributedSampler(
                dataloader.dataset,
                num_replicas=xm.xrt_world_size(),
                rank=xm.get_ordinal(),
            )
        else:
            sampler = DistributedSampler(dataloader.dataset)

        dl_args["sampler"] = sampler
        dataloader = type(dataloader)(**dl_args)

    return dataloader
# Buggy code
def check_gpus_data_type(gpus):
    """
    :param gpus: gpus parameter as passed to the Trainer
        Function checks that it is one of: None, Int, String or List
        Throws otherwise
    :return: return unmodified gpus variable
    """

    if gpus is not None and type(gpus) not in (int, str, list):
        raise MisconfigurationException(
            "GPUs must be int, string or list of ints or None."
        )


# Traceback (pytorch_lightning.utilities.debugging.MisconfigurationException)
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/opt/anaconda/miniconda3/envs/ai/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 366, in __init__
self.data_parallel_device_ids = parse_gpu_ids(self.gpus)
File "/opt/anaconda/miniconda3/envs/ai/lib/python3.7/site-packages/pytorch_lightning/trainer/distrib_parts.py", line 613, in parse_gpu_ids
check_gpus_data_type(gpus)
File "/opt/anaconda/miniconda3/envs/ai/lib/python3.7/site-packages/pytorch_lightning/trainer/distrib_parts.py", line 561, in check_gpus_data_type
raise MisconfigurationException("GPUs must be int, string or list of ints or None.")
pytorch_lightning.utilities.debugging.MisconfigurationException: GPUs must be int, string or list of ints or None.

# Fixed code
def check_gpus_data_type(gpus):
    """
    :param gpus: gpus parameter as passed to the Trainer
        Function checks that it is one of: None, Int, String or List
        Throws otherwise
    :return: return unmodified gpus variable
    """

    if gpus is not None and (
        not isinstance(gpus, (int, str, list)) or isinstance(gpus, bool)
    ):
        raise MisconfigurationException(
            "GPUs must be int, string or list of ints or None."
        )
# Buggy code
def _get_next_version(self):
    root_dir = os.path.join(self.save_dir, self.name)
    existing_versions = []
    for d in os.listdir(root_dir):
        if os.path.isdir(os.path.join(root_dir, d)) and d.startswith("version_"):
            existing_versions.append(int(d.split("_")[1]))

    if len(existing_versions) == 0:
        return 0

    return max(existing_versions) + 1


# Traceback (FileNotFoundError)
Exception:

-- Process 0 terminated with the following error:
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/envs/nightly_pt/lib/python3.6/site-packages/torch/multiprocessing/spawn.py", line 19, in _wrap
fn(i, *args)
File "/home/ubuntu/anaconda3/envs/nightly_pt/lib/python3.6/site-packages/pytorch_lightning/trainer/distrib_data_parallel.py", line 342, in ddp_train
self.run_pretrain_routine(model)
File "/home/ubuntu/anaconda3/envs/nightly_pt/lib/python3.6/site-packages/pytorch_lightning/trainer/trainer.py", line 777, in run_pretrain_routine
self.configure_checkpoint_callback()
File "/home/ubuntu/anaconda3/envs/nightly_pt/lib/python3.6/site-packages/pytorch_lightning/trainer/callback_config.py", line 45, in configure_checkpoint_callback
f'version_{self.logger.version}',
File "/home/ubuntu/anaconda3/envs/nightly_pt/lib/python3.6/site-packages/pytorch_lightning/loggers/tensorboard.py", line 161, in version
self._version = self._get_next_version()
File "/home/ubuntu/anaconda3/envs/nightly_pt/lib/python3.6/site-packages/pytorch_lightning/loggers/tensorboard.py", line 167, in _get_next_version
for d in os.listdir(root_dir):
FileNotFoundError: [Errno 2] No such file or directory: '/home/ubuntu/pytorch-lightning-intro-guide/outputs/2020-04-04/15-53-26/lightning_logs'

# Fixed code
def _get_next_version(self):
    root_dir = os.path.join(self.save_dir, self.name)

    if not os.path.isdir(root_dir):
        log.warning("Missing logger folder: %s", root_dir)
        return 0

    existing_versions = []
    for d in os.listdir(root_dir):
        if os.path.isdir(os.path.join(root_dir, d)) and d.startswith("version_"):
            existing_versions.append(int(d.split("_")[1]))

    if len(existing_versions) == 0:
        return 0

    return max(existing_versions) + 1
# Buggy code
def load_spawn_weights(self, original_model):
    """
    Load the temp weights saved in the process
    To recover the trained model from the ddp process we load the saved weights
    :param model:
    :return:
    """
    # load weights saved in ddp
    path = os.path.join(self.default_save_path, "__temp_weight_ddp_end.ckpt")
    loaded_model = original_model.__class__.load_from_checkpoint(path)

    # copy loaded weights to old model
    original_model.load_state_dict(loaded_model.state_dict())

    # remove ddp weights
    os.remove(path)

    return loaded_model


# Traceback (FileNotFoundError)
Traceback (most recent call last):
File "app.py", line 166, in <module>
main_()  # pylint: disable=no-value-for-parameter
File "app.py", line 162, in main_
trainer.fit(model)
File "/home/ubuntu/anaconda3/envs/nightly_pt/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 593, in fit
self.load_spawn_weights(model)
File "/home/ubuntu/anaconda3/envs/nightly_pt/lib/python3.7/site-packages/pytorch_lightning/trainer/distrib_data_parallel.py", line 368, in load_spawn_weights
loaded_model = original_model.__class__.load_from_checkpoint(path)
File "/home/ubuntu/anaconda3/envs/nightly_pt/lib/python3.7/site-packages/pytorch_lightning/core/lightning.py", line 1353, in load_from_checkpoint
checkpoint = torch.load(checkpoint_path, map_location=lambda storage, loc: storage)
File "/home/ubuntu/anaconda3/envs/nightly_pt/lib/python3.7/site-packages/torch/serialization.py", line 525, in load
with _open_file_like(f, 'rb') as opened_file:
File "/home/ubuntu/anaconda3/envs/nightly_pt/lib/python3.7/site-packages/torch/serialization.py", line 212, in _open_file_like
return _open_file(name_or_buffer, mode)
File "/home/ubuntu/anaconda3/envs/nightly_pt/lib/python3.7/site-packages/torch/serialization.py", line 193, in __init__
super(_open_file, self).__init__(open(name, mode))
FileNotFoundError: [Errno 2] No such file or directory: '/home/ubuntu/pytorch-lightning-intro-guide/__temp_weight_ddp_end.ckpt'

# Fixed code
def load_spawn_weights(self, original_model):
    """
    Load the temp weights saved in the process
    To recover the trained model from the ddp process we load the saved weights
    :param model:
    :return:
    """

    loaded_model = original_model

    if self.proc_rank == 0:
        # load weights saved in ddp
        path = os.path.join(self.default_save_path, "__temp_weight_ddp_end.ckpt")
        loaded_model = original_model.__class__.load_from_checkpoint(path)

        # copy loaded weights to old model
        original_model.load_state_dict(loaded_model.state_dict())

        # remove ddp weights
        os.remove(path)

    return loaded_model
# Buggy code
def _has_len(dataloader: DataLoader) -> bool:
    try:
        # try getting the length
        _ = len(dataloader)
        return True
    except TypeError:
        return False


# Traceback (RuntimeError)
Traceback (most recent call last):
File "UNet_WaveProp.py", line 174, in <module>
trainer.fit(model)
File "/mnt/RDS/home/code/pytorch-lightning/pytorch_lightning/trainer/trainer.py", line 629, in fit
self.run_pretrain_routine(model)
File "/mnt/RDS/home/code/pytorch-lightning/pytorch_lightning/trainer/trainer.py", line 809, in run_pretrain_routine
False)
File "/mnt/RDS/home/code/pytorch-lightning/pytorch_lightning/trainer/evaluation_loop.py", line 300, in evaluate
eval_results = model.validation_epoch_end(outputs)
File "UNet_WaveProp.py", line 138, in validation_epoch_end
avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()
RuntimeError: stack expects a non-empty TensorList

# Fixed code
def _has_len(dataloader: DataLoader) -> bool:
    """Checks if a given Dataloader has __len__ method implemented i.e. if
    it is a finite dataloader or infinite dataloader"""
    try:
        # try getting the length
        if len(dataloader) == 0:
            raise ValueError(
                "Dataloader returned 0 length. Please make sure"
                " that your Dataloader atleast returns 1 batch"
            )
        return True
    except TypeError:
        return False
# Buggy code
def tpu_train(self, tpu_core_idx, model):
    # put model on tpu
    model.to(xm.xla_device())

    # get the appropriate tpu ranks
    self.tpu_local_core_rank = xm.get_local_ordinal()
    self.tpu_global_core_rank = xm.get_ordinal()

    # avoid duplicating progress bar
    self.show_progress_bar = self.show_progress_bar and self.tpu_global_core_rank == 0

    # track current tpu
    self.current_tpu_idx = tpu_core_idx
    self.proc_rank = self.tpu_local_core_rank

    # CHOOSE OPTIMIZER
    # allow for lr schedulers as well
    self.optimizers, self.lr_schedulers = self.init_optimizers(
        model.configure_optimizers()
    )

    # init 16 bit for TPU
    if self.precision == 16:
        os.environ["XLA_USE_BF16"] = 1

    m = (
        f"INIT TPU local core: {self.tpu_local_core_rank}, "
        f"global rank: {self.tpu_global_core_rank}"
    )
    log.info(m)
    self.run_pretrain_routine(model)

    self.save_spawn_weights(model)


# Traceback (TypeError)
Exception in device=TPU:0: str expected, not int
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/torch_xla/distributed/xla_multiprocessing.py", line 119, in _start_fn
fn(gindex, *args)
File "/usr/local/lib/python3.6/dist-packages/pytorch_lightning/trainer/distrib_parts.py", line 492, in tpu_train
os.environ['XLA_USE_BF16'] = 1
File "/usr/lib/python3.6/os.py", line 674, in __setitem__
value = self.encodevalue(value)
File "/usr/lib/python3.6/os.py", line 744, in encode
raise TypeError("str expected, not %s" % type(value).__name__)
TypeError: str expected, not int

# Fixed code
def tpu_train(self, tpu_core_idx, model):
    # put model on tpu
    model.to(xm.xla_device())

    # get the appropriate tpu ranks
    self.tpu_local_core_rank = xm.get_local_ordinal()
    self.tpu_global_core_rank = xm.get_ordinal()

    # avoid duplicating progress bar
    self.show_progress_bar = self.show_progress_bar and self.tpu_global_core_rank == 0

    # track current tpu
    self.current_tpu_idx = tpu_core_idx
    self.proc_rank = self.tpu_local_core_rank

    # CHOOSE OPTIMIZER
    # allow for lr schedulers as well
    self.optimizers, self.lr_schedulers = self.init_optimizers(
        model.configure_optimizers()
    )

    # init 16 bit for TPU
    if self.precision == 16:
        os.environ["XLA_USE_BF16"] = str(1)

    m = (
        f"INIT TPU local core: {self.tpu_local_core_rank}, "
        f"global rank: {self.tpu_global_core_rank}"
    )
    log.info(m)
    self.run_pretrain_routine(model)

    self.save_spawn_weights(model)