gurudesh/pytracebugs huggingface dataset:
# Buggy code
def barrier(self, *args, **kwargs):
    hvd.join()


# Traceback (RuntimeError)
RuntimeError                              Traceback (most recent call last)
<ipython-input-17-587e2a9e3858> in <module>
----> 1 trainer.test(datamodule=dm)

/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in test(self, model, test_dataloaders, ckpt_path, verbose, datamodule)
922             results = self.__test_given_model(model, test_dataloaders)
923         else:
--> 924             results = self.__test_using_best_weights(ckpt_path, test_dataloaders)
925
926         self.teardown('test')

/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in __test_using_best_weights(self, ckpt_path, test_dataloaders)
950                 return {}
951             if not self._device_type == DeviceType.TPU:
--> 952                 self.accelerator.barrier()
953
954             ckpt = pl_load(ckpt_path, map_location=lambda storage, loc: storage)

/opt/conda/lib/python3.7/site-packages/pytorch_lightning/accelerators/accelerator.py in barrier(self, name)
375
376     def barrier(self, name: Optional[str] = None) -> None:
--> 377         self.training_type_plugin.barrier(name=name)
378
379     def broadcast(self, obj: object, src: int = 0) -> object:

/opt/conda/lib/python3.7/site-packages/pytorch_lightning/plugins/training_type/tpu_spawn.py in barrier(self, name)
113
114     def barrier(self, name: Optional[str] = None) -> None:
--> 115         rendezvous(f"pl.Trainer.{name}")
116
117     def transfer_distrib_spawn_state_on_fit_end(self, results):

/opt/conda/lib/python3.7/site-packages/torch_xla/core/xla_model.py in rendezvous(tag, payload, replicas)
859     ordinal `i` at position `i` in the returned tuple.
860   """
--> 861   return torch_xla._XLAC._xla_rendezvous(get_ordinal(), tag, payload, replicas)
862
863

RuntimeError: tensorflow/compiler/xla/xla_client/mesh_service.cc:316 : Check failed: impl_->channel->WaitForConnected( std::chrono::system_clock::now() + std::chrono::seconds(connect_wait_seconds))

# Fixed code
def barrier(self, *args, **kwargs):
    if torch_distrib.is_initialized():
        hvd.join()
# Buggy code
def barrier(self, name: Optional[str] = None) -> None:
    rendezvous(f"pl.Trainer.{name}")


# Traceback (RuntimeError)
RuntimeError                              Traceback (most recent call last)
<ipython-input-17-587e2a9e3858> in <module>
----> 1 trainer.test(datamodule=dm)

/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in test(self, model, test_dataloaders, ckpt_path, verbose, datamodule)
922             results = self.__test_given_model(model, test_dataloaders)
923         else:
--> 924             results = self.__test_using_best_weights(ckpt_path, test_dataloaders)
925
926         self.teardown('test')

/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in __test_using_best_weights(self, ckpt_path, test_dataloaders)
950                 return {}
951             if not self._device_type == DeviceType.TPU:
--> 952                 self.accelerator.barrier()
953
954             ckpt = pl_load(ckpt_path, map_location=lambda storage, loc: storage)

/opt/conda/lib/python3.7/site-packages/pytorch_lightning/accelerators/accelerator.py in barrier(self, name)
375
376     def barrier(self, name: Optional[str] = None) -> None:
--> 377         self.training_type_plugin.barrier(name=name)
378
379     def broadcast(self, obj: object, src: int = 0) -> object:

/opt/conda/lib/python3.7/site-packages/pytorch_lightning/plugins/training_type/tpu_spawn.py in barrier(self, name)
113
114     def barrier(self, name: Optional[str] = None) -> None:
--> 115         rendezvous(f"pl.Trainer.{name}")
116
117     def transfer_distrib_spawn_state_on_fit_end(self, results):

/opt/conda/lib/python3.7/site-packages/torch_xla/core/xla_model.py in rendezvous(tag, payload, replicas)
859     ordinal `i` at position `i` in the returned tuple.
860   """
--> 861   return torch_xla._XLAC._xla_rendezvous(get_ordinal(), tag, payload, replicas)
862
863

RuntimeError: tensorflow/compiler/xla/xla_client/mesh_service.cc:316 : Check failed: impl_->channel->WaitForConnected( std::chrono::system_clock::now() + std::chrono::seconds(connect_wait_seconds))

# Fixed code
def barrier(self, name: Optional[str] = None) -> None:
    if torch_distrib.is_initialized():
        rendezvous(f"pl.Trainer.{name}")
# Buggy code
def broadcast(self, obj, src=0):
    obj = hvd.broadcast_object(obj, src)
    return obj


# Traceback (FileNotFoundError)
Traceback (most recent call last):
File "/home/wojciech/tmp/pytorch-lightining/main.py", line 16, in <module>
result = trainer.test()
File "/home/wojciech/miniconda3/envs/ml/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 721, in test
results = self.__test_using_best_weights(ckpt_path, test_dataloaders)
File "/home/wojciech/miniconda3/envs/ml/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 751, in __test_using_best_weights
ckpt = pl_load(ckpt_path, map_location=lambda storage, loc: storage)
File "/home/wojciech/miniconda3/envs/ml/lib/python3.8/site-packages/pytorch_lightning/utilities/cloud_io.py", line 31, in load
with fs.open(path_or_url, "rb") as f:
File "/home/wojciech/miniconda3/envs/ml/lib/python3.8/site-packages/fsspec/spec.py", line 897, in open
f = self._open(
File "/home/wojciech/miniconda3/envs/ml/lib/python3.8/site-packages/fsspec/implementations/local.py", line 115, in _open
return LocalFileOpener(path, mode, fs=self, **kwargs)
File "/home/wojciech/miniconda3/envs/ml/lib/python3.8/site-packages/fsspec/implementations/local.py", line 197, in __init__
self._open()
File "/home/wojciech/miniconda3/envs/ml/lib/python3.8/site-packages/fsspec/implementations/local.py", line 202, in _open
self.f = open(self.path, mode=self.mode)
FileNotFoundError: [Errno 2] No such file or directory: '/home/wojciech/tmp/pytorch-lightining/lightning_logs/version_10/checkpoints/epoch=0-v0.ckpt'

# Fixed code
def broadcast(self, obj, src=0):
    self.barrier()
    obj = hvd.broadcast_object(obj, src)
    return obj
# Buggy code
def broadcast(self, obj, src=0):
    buffer = io.BytesIO()
    torch.save(obj, buffer)
    data = bytearray(buffer.getbuffer())
    data_tensor = torch.tensor(data).to(xm.xla_device(), dtype=torch.float)
    data = xm.all_gather(data_tensor)
    buffer = io.BytesIO(data.cpu().byte().numpy())
    obj = torch.load(buffer)
    return obj


# Traceback (FileNotFoundError)
Traceback (most recent call last):
File "/home/wojciech/tmp/pytorch-lightining/main.py", line 16, in <module>
result = trainer.test()
File "/home/wojciech/miniconda3/envs/ml/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 721, in test
results = self.__test_using_best_weights(ckpt_path, test_dataloaders)
File "/home/wojciech/miniconda3/envs/ml/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 751, in __test_using_best_weights
ckpt = pl_load(ckpt_path, map_location=lambda storage, loc: storage)
File "/home/wojciech/miniconda3/envs/ml/lib/python3.8/site-packages/pytorch_lightning/utilities/cloud_io.py", line 31, in load
with fs.open(path_or_url, "rb") as f:
File "/home/wojciech/miniconda3/envs/ml/lib/python3.8/site-packages/fsspec/spec.py", line 897, in open
f = self._open(
File "/home/wojciech/miniconda3/envs/ml/lib/python3.8/site-packages/fsspec/implementations/local.py", line 115, in _open
return LocalFileOpener(path, mode, fs=self, **kwargs)
File "/home/wojciech/miniconda3/envs/ml/lib/python3.8/site-packages/fsspec/implementations/local.py", line 197, in __init__
self._open()
File "/home/wojciech/miniconda3/envs/ml/lib/python3.8/site-packages/fsspec/implementations/local.py", line 202, in _open
self.f = open(self.path, mode=self.mode)
FileNotFoundError: [Errno 2] No such file or directory: '/home/wojciech/tmp/pytorch-lightining/lightning_logs/version_10/checkpoints/epoch=0-v0.ckpt'

# Fixed code
def broadcast(self, obj, src=0):
    if self.trainer.tpu_id is not None:
        # running on a single core
        return obj
    buffer = io.BytesIO()
    torch.save(obj, buffer)
    data = bytearray(buffer.getbuffer())
    data_tensor = torch.tensor(data).to(xm.xla_device(), dtype=torch.float)
    data = xm.all_gather(data_tensor)
    buffer = io.BytesIO(data.cpu().byte().numpy())
    obj = torch.load(buffer)
    return obj
# Buggy code
def _get_metric_interpolated_filepath_name(
    self,
    ckpt_name_metrics: Dict[str, Any],
    epoch: int,
    step: int,
    del_filepath: Optional[str] = None,
) -> str:
    filepath = self.format_checkpoint_name(epoch, step, ckpt_name_metrics)

    version_cnt = 0
    while self._fs.exists(filepath) and filepath != del_filepath:
        filepath = self.format_checkpoint_name(
            epoch, step, ckpt_name_metrics, ver=version_cnt
        )
        version_cnt += 1

    return filepath


# Traceback (FileNotFoundError)
Traceback (most recent call last):
File "/home/wojciech/tmp/pytorch-lightining/main.py", line 16, in <module>
result = trainer.test()
File "/home/wojciech/miniconda3/envs/ml/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 721, in test
results = self.__test_using_best_weights(ckpt_path, test_dataloaders)
File "/home/wojciech/miniconda3/envs/ml/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 751, in __test_using_best_weights
ckpt = pl_load(ckpt_path, map_location=lambda storage, loc: storage)
File "/home/wojciech/miniconda3/envs/ml/lib/python3.8/site-packages/pytorch_lightning/utilities/cloud_io.py", line 31, in load
with fs.open(path_or_url, "rb") as f:
File "/home/wojciech/miniconda3/envs/ml/lib/python3.8/site-packages/fsspec/spec.py", line 897, in open
f = self._open(
File "/home/wojciech/miniconda3/envs/ml/lib/python3.8/site-packages/fsspec/implementations/local.py", line 115, in _open
return LocalFileOpener(path, mode, fs=self, **kwargs)
File "/home/wojciech/miniconda3/envs/ml/lib/python3.8/site-packages/fsspec/implementations/local.py", line 197, in __init__
self._open()
File "/home/wojciech/miniconda3/envs/ml/lib/python3.8/site-packages/fsspec/implementations/local.py", line 202, in _open
self.f = open(self.path, mode=self.mode)
FileNotFoundError: [Errno 2] No such file or directory: '/home/wojciech/tmp/pytorch-lightining/lightning_logs/version_10/checkpoints/epoch=0-v0.ckpt'

# Fixed code
def _get_metric_interpolated_filepath_name(
    self,
    ckpt_name_metrics: Dict[str, Any],
    epoch: int,
    step: int,
    trainer,
    del_filepath: Optional[str] = None,
) -> str:
    filepath = self.format_checkpoint_name(epoch, step, ckpt_name_metrics)

    version_cnt = 0
    while self.file_exists(filepath, trainer) and filepath != del_filepath:
        filepath = self.format_checkpoint_name(
            epoch, step, ckpt_name_metrics, ver=version_cnt
        )
        version_cnt += 1

    return filepath
# Buggy code
def log_metrics(
    self, metrics: Dict[str, Union[torch.Tensor, float]], step: Optional[int] = None
) -> None:
    """
    Log metrics (numeric values) in Neptune experiments.

    Args:
        metrics: Dictionary with metric names as keys and measured quantities as values
        step: Step number at which the metrics should be recorded, must be strictly increasing
    """
    assert rank_zero_only.rank == 0, "experiment tried to log from global_rank != 0"

    metrics = self._add_prefix(metrics)
    for key, val in metrics.items():
        self.log_metric(key, val, step=step)


# Traceback (neptune.api_exceptions.ChannelsValuesSendBatchError)
Failed to send channel value.
Traceback (most recent call last):
File "/home/wojciech/miniconda3/envs/ml/lib/python3.7/site-packages/neptune/internal/channels/channels_values_sender.py", line 156, in _send_values
self._experiment._send_channels_values(channels_with_values)
File "/home/wojciech/miniconda3/envs/ml/lib/python3.7/site-packages/neptune/experiments.py", line 1167, in _send_channels_values
self._backend.send_channels_values(self, channels_with_values)
File "/home/wojciech/miniconda3/envs/ml/lib/python3.7/site-packages/neptune/utils.py", line 211, in wrapper
return func(*args, **kwargs)
File "/home/wojciech/miniconda3/envs/ml/lib/python3.7/site-packages/neptune/internal/backends/hosted_neptune_backend.py", line 571, in send_channels_values
raise ChannelsValuesSendBatchError(experiment.id, batch_errors)
neptune.api_exceptions.ChannelsValuesSendBatchError: Received batch errors sending channels' values to experiment SAN-28. Cause: Error(code=400, message='X-coordinates must be strictly increasing for channel: e968f192-b466-4419-89ce-469fdc4cf86f. Invalid point: InputChannelValue(timestamp=2020-12-14T15:53:20.877Z, x=0.0, numericValue=0.0, textValue=null, image', type=None) (metricId: 'e968f192-b466-4419-89ce-469fdc4cf86f', x: 0.0) Skipping 1 values.
/home/wojciech/miniconda3/envs/ml/lib/python3.7/site-packages/pytorch_lightning/utilities/distributed.py:49: UserWarning: The dataloader, test dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 6 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.
warnings.warn(*args, **kwargs)

# Fixed code
def log_metrics(
    self, metrics: Dict[str, Union[torch.Tensor, float]], step: Optional[int] = None
) -> None:
    """
    Log metrics (numeric values) in Neptune experiments.

    Args:
        metrics: Dictionary with metric names as keys and measured quantities as values
        step: Step number at which the metrics should be recorded, currently ignored
    """
    assert rank_zero_only.rank == 0, "experiment tried to log from global_rank != 0"

    metrics = self._add_prefix(metrics)
    for key, val in metrics.items():
        # `step` is ignored because Neptune expects strictly increasing step values which
        # Lighting does not always guarantee.
        self.log_metric(key, val)
# Buggy code
def log_text(self, log_name: str, text: str, step: Optional[int] = None) -> None:
    """
    Log text data in Neptune experiments.

    Args:
        log_name: The name of log, i.e. mse, my_text_data, timing_info.
        text: The value of the log (data-point).
        step: Step number at which the metrics should be recorded, must be strictly increasing
    """
    self.log_metric(log_name, text, step=step)


# Traceback (neptune.api_exceptions.ChannelsValuesSendBatchError)
Failed to send channel value.
Traceback (most recent call last):
File "/home/wojciech/miniconda3/envs/ml/lib/python3.7/site-packages/neptune/internal/channels/channels_values_sender.py", line 156, in _send_values
self._experiment._send_channels_values(channels_with_values)
File "/home/wojciech/miniconda3/envs/ml/lib/python3.7/site-packages/neptune/experiments.py", line 1167, in _send_channels_values
self._backend.send_channels_values(self, channels_with_values)
File "/home/wojciech/miniconda3/envs/ml/lib/python3.7/site-packages/neptune/utils.py", line 211, in wrapper
return func(*args, **kwargs)
File "/home/wojciech/miniconda3/envs/ml/lib/python3.7/site-packages/neptune/internal/backends/hosted_neptune_backend.py", line 571, in send_channels_values
raise ChannelsValuesSendBatchError(experiment.id, batch_errors)
neptune.api_exceptions.ChannelsValuesSendBatchError: Received batch errors sending channels' values to experiment SAN-28. Cause: Error(code=400, message='X-coordinates must be strictly increasing for channel: e968f192-b466-4419-89ce-469fdc4cf86f. Invalid point: InputChannelValue(timestamp=2020-12-14T15:53:20.877Z, x=0.0, numericValue=0.0, textValue=null, image', type=None) (metricId: 'e968f192-b466-4419-89ce-469fdc4cf86f', x: 0.0) Skipping 1 values.
/home/wojciech/miniconda3/envs/ml/lib/python3.7/site-packages/pytorch_lightning/utilities/distributed.py:49: UserWarning: The dataloader, test dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 6 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.
warnings.warn(*args, **kwargs)

# Fixed code
def log_text(self, log_name: str, text: str, step: Optional[int] = None) -> None:
    """
    Log text data in Neptune experiments.

    Args:
        log_name: The name of log, i.e. mse, my_text_data, timing_info.
        text: The value of the log (data-point).
        step: Step number at which the metrics should be recorded, must be strictly increasing
    """
    self.experiment.log_text(log_name, text, step=step)
# Buggy code
def _save_top_k_checkpoints(self, trainer, pl_module, metrics):
    current = metrics.get(self.monitor)
    epoch = metrics.get("epoch")
    step = metrics.get("step")

    if not isinstance(current, torch.Tensor) and current is not None:
        current = torch.tensor(current, device=pl_module.device)

    if self.check_monitor_top_k(current):
        self._update_best_and_save(current, epoch, step, trainer, pl_module, metrics)
    elif self.verbose:
        rank_zero_info(
            f"Epoch {epoch:d}, step {step:d}: {self.monitor} was not in top {self.save_top_k}"
        )


# Traceback (RuntimeError)
Traceback (most recent call last):
File "pytorch_lightning/trainer/trainer.py", line 521, in train
self.train_loop.run_training_epoch()
File "pytorch_lightning/trainer/training_loop.py", line 590, in run_training_epoch
self.trainer.run_evaluation(test_mode=False)
File "pytorch_lightning/trainer/trainer.py", line 628, in run_evaluation
self.evaluation_loop.on_evaluation_end()
File "pytorch_lightning/trainer/evaluation_loop.py", line 111, in on_evaluation_end
self.trainer.call_hook('on_validation_end', *args, **kwargs)
File "pytorch_lightning/trainer/trainer.py", line 887, in call_hook
trainer_hook(*args, **kwargs)
File "pytorch_lightning/trainer/callback_hook.py", line 177, in on_validation_end
callback.on_validation_end(self, self.get_model())
File "pytorch_lightning/callbacks/early_stopping.py", line 164, in on_validation_end
self._run_early_stopping_check(trainer, pl_module)
File "pytorch_lightning/callbacks/early_stopping.py", line 205, in _run_early_stopping_check
current = torch.tensor(current, device=pl_module.device)
RuntimeError: Could not infer dtype of Accuracy

# Fixed code
def _save_top_k_checkpoints(self, trainer, pl_module, metrics):
    current = metrics.get(self.monitor)
    epoch = metrics.get("epoch")
    step = metrics.get("step")

    if current is not None:
        if isinstance(current, Metric):
            current = current.compute()
        elif isinstance(current, numbers.Number):
            current = torch.tensor(current, device=pl_module.device, dtype=torch.float)

    if self.check_monitor_top_k(current):
        self._update_best_and_save(current, epoch, step, trainer, pl_module, metrics)
    elif self.verbose:
        rank_zero_info(
            f"Epoch {epoch:d}, step {step:d}: {self.monitor} was not in top {self.save_top_k}"
        )
# Buggy code
def get_forked_metrics(self, add_dataloader_idx=False):
    """
    Gets the metrics to log at the end of epoch
    """
    result = {}

    meta = self["meta"]
    for k, options in meta.items():
        if k == "_internal":
            continue

        dl_key = self._add_dataloader_idx(
            k, options["dataloader_idx"], add_dataloader_idx
        )

        if options["forked"]:
            result[dl_key] = self[k]

    return result


# Traceback (RuntimeError)
Traceback (most recent call last):
File "pytorch_lightning/trainer/trainer.py", line 521, in train
self.train_loop.run_training_epoch()
File "pytorch_lightning/trainer/training_loop.py", line 590, in run_training_epoch
self.trainer.run_evaluation(test_mode=False)
File "pytorch_lightning/trainer/trainer.py", line 628, in run_evaluation
self.evaluation_loop.on_evaluation_end()
File "pytorch_lightning/trainer/evaluation_loop.py", line 111, in on_evaluation_end
self.trainer.call_hook('on_validation_end', *args, **kwargs)
File "pytorch_lightning/trainer/trainer.py", line 887, in call_hook
trainer_hook(*args, **kwargs)
File "pytorch_lightning/trainer/callback_hook.py", line 177, in on_validation_end
callback.on_validation_end(self, self.get_model())
File "pytorch_lightning/callbacks/early_stopping.py", line 164, in on_validation_end
self._run_early_stopping_check(trainer, pl_module)
File "pytorch_lightning/callbacks/early_stopping.py", line 205, in _run_early_stopping_check
current = torch.tensor(current, device=pl_module.device)
RuntimeError: Could not infer dtype of Accuracy

# Fixed code
def get_forked_metrics(self, add_dataloader_idx=False):
    """
    Gets the metrics to log at the end of epoch
    """
    result = {}

    meta = self["meta"]
    for k, options in meta.items():
        if k == "_internal":
            continue

        dl_key = self._add_dataloader_idx(
            k, options["dataloader_idx"], add_dataloader_idx
        )

        if options["forked"]:
            if isinstance(self[k], Metric):
                result[dl_key] = self[k].compute().detach()
            else:
                result[dl_key] = self[k]

    return result
# Buggy code
def pl_multi_process(func):
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        queue = Queue()
        proc = Process(target=inner_f, args=(queue, func, *args), kwargs=kwargs)
        proc.start()
        proc.join(20)
        try:
            return queue.get_nowait()
        except q.Empty:
            traceback.print_exc()
            return False

    return wrapper


# Traceback (MisconfigurationException)
GPU available: False, used: False
TPU available: True, using: 1 TPU cores
training on 1 TPU cores
---------------------------------------------------------------------------
MisconfigurationException                 Traceback (most recent call last)
<ipython-input-11-1f9f6fbe4f6c> in <module>()
----> 1 test_x(tmpdir)

2 frames
/usr/local/lib/python3.6/dist-packages/pytorch_lightning/accelerators/tpu_accelerator.py in setup(self, model)
61         # TODO: Move this check to Trainer __init__ or device parser
62         if not TPU_AVAILABLE:
---> 63             raise MisconfigurationException('PyTorch XLA not installed.')
64
65         # see: https://discuss.pytorch.org/t/segfault-with-multiprocessing-queue/81292/2

MisconfigurationException: PyTorch XLA not installed.

# Fixed code
def pl_multi_process(func):
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        queue = Queue()
        proc = Process(target=inner_f, args=(queue, func, *args), kwargs=kwargs)
        proc.start()
        proc.join(TPU_CHECK_TIMEOUT)
        try:
            return queue.get_nowait()
        except q.Empty:
            traceback.print_exc()
            return False

    return wrapper
# Buggy code
def wrapper(*args, **kwargs):
    queue = Queue()
    proc = Process(target=inner_f, args=(queue, func, *args), kwargs=kwargs)
    proc.start()
    proc.join(20)
    try:
        return queue.get_nowait()
    except q.Empty:
        traceback.print_exc()
        return False


# Traceback (MisconfigurationException)
GPU available: False, used: False
TPU available: True, using: 1 TPU cores
training on 1 TPU cores
---------------------------------------------------------------------------
MisconfigurationException                 Traceback (most recent call last)
<ipython-input-11-1f9f6fbe4f6c> in <module>()
----> 1 test_x(tmpdir)

2 frames
/usr/local/lib/python3.6/dist-packages/pytorch_lightning/accelerators/tpu_accelerator.py in setup(self, model)
61         # TODO: Move this check to Trainer __init__ or device parser
62         if not TPU_AVAILABLE:
---> 63             raise MisconfigurationException('PyTorch XLA not installed.')
64
65         # see: https://discuss.pytorch.org/t/segfault-with-multiprocessing-queue/81292/2

MisconfigurationException: PyTorch XLA not installed.

# Fixed code
def wrapper(*args, **kwargs):
    queue = Queue()
    proc = Process(target=inner_f, args=(queue, func, *args), kwargs=kwargs)
    proc.start()
    proc.join(TPU_CHECK_TIMEOUT)
    try:
        return queue.get_nowait()
    except q.Empty:
        traceback.print_exc()
        return False
# Buggy code
def is_picklable(obj: object) -> bool:
    """Tests if an object can be pickled"""

    try:
        pickle.dumps(obj)
        return True
    except pickle.PicklingError:
        return False


# Traceback (AttributeError)
Traceback (most recent call last):
File "/home/chirag/miniconda3/envs/ml/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/home/chirag/miniconda3/envs/ml/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/home/chirag/Projects/mingle/social-processes/run/run_synthetic_social.py", line 120, in <module>
main()
File "/home/chirag/Projects/mingle/social-processes/run/run_synthetic_social.py", line 116, in main
trainer.fit(process, datamodule=dm)
File "/home/chirag/miniconda3/envs/ml/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 425, in fit
self.train_loop.setup_fit(model, train_dataloader, val_dataloaders, datamodule)
File "/home/chirag/miniconda3/envs/ml/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py", line 90, in setup_fit
parsing.clean_namespace(model.hparams)
File "/home/chirag/miniconda3/envs/ml/lib/python3.8/site-packages/pytorch_lightning/utilities/parsing.py", line 75, in clean_namespace
del_attrs = [k for k, v in hparams_dict.items() if not is_picklable(v)]
File "/home/chirag/miniconda3/envs/ml/lib/python3.8/site-packages/pytorch_lightning/utilities/parsing.py", line 75, in <listcomp>
del_attrs = [k for k, v in hparams_dict.items() if not is_picklable(v)]
File "/home/chirag/miniconda3/envs/ml/lib/python3.8/site-packages/pytorch_lightning/utilities/parsing.py", line 62, in is_picklable
pickle.dumps(obj)
AttributeError: Can't pickle local object 'ArgumentParser.__init__.<locals>.identity'

# Fixed code
def is_picklable(obj: object) -> bool:
    """Tests if an object can be pickled"""

    try:
        pickle.dumps(obj)
        return True
    except (pickle.PicklingError, AttributeError):
        return False
# Buggy code
def log_hyperparams(self, params: Union[Dict[str, Any], Namespace]) -> None:
    params = self._convert_params(params)
    params = self._flatten_dict(params)
    self.experiment.config.update(params, allow_val_change=True)


# Traceback (TypeError)
Traceback (most recent call last):
File "wandblogger_issue.py", line 12, in <module>
wandb_logger.log_hyperparams(vars(args))
File "/home/groups/mignot/miniconda3/envs/pl/lib/python3.7/site-packages/pytorch_lightning/utilities/distributed.py", line 35, in wrapped_fn
return fn(*args, **kwargs)
File "/home/groups/mignot/miniconda3/envs/pl/lib/python3.7/site-packages/pytorch_lightning/loggers/wandb.py", line 138, in log_hyperparams
self.experiment.config.update(params, allow_val_change=True)
File "/home/groups/mignot/miniconda3/envs/pl/lib/python3.7/site-packages/wandb/sdk/wandb_config.py", line 87, in update
self._callback(data=self._as_dict())
File "/home/groups/mignot/miniconda3/envs/pl/lib/python3.7/site-packages/wandb/sdk/wandb_run.py", line 587, in _config_callback
self._backend.interface.publish_config(data)
File "/home/groups/mignot/miniconda3/envs/pl/lib/python3.7/site-packages/wandb/interface/interface.py", line 496, in publish_config
cfg = self._make_config(config_dict)
File "/home/groups/mignot/miniconda3/envs/pl/lib/python3.7/site-packages/wandb/interface/interface.py", line 232, in _make_config
update.value_json = json_dumps_safer(json_friendly(v)[0])
File "/home/groups/mignot/miniconda3/envs/pl/lib/python3.7/site-packages/wandb/util.py", line 524, in json_dumps_safer
return json.dumps(obj, cls=WandBJSONEncoder, **kwargs)
File "/home/groups/mignot/miniconda3/envs/pl/lib/python3.7/json/__init__.py", line 238, in dumps
**kw).encode(obj)
File "/home/groups/mignot/miniconda3/envs/pl/lib/python3.7/json/encoder.py", line 199, in encode
chunks = self.iterencode(o, _one_shot=True)
File "/home/groups/mignot/miniconda3/envs/pl/lib/python3.7/json/encoder.py", line 257, in iterencode
return _iterencode(o, 0)
File "/home/groups/mignot/miniconda3/envs/pl/lib/python3.7/site-packages/wandb/util.py", line 480, in default
return json.JSONEncoder.default(self, obj)
File "/home/groups/mignot/miniconda3/envs/pl/lib/python3.7/json/encoder.py", line 179, in default
raise TypeError(f'Object of type {o.__class__.__name__} '
TypeError: Object of type function is not JSON serializable

# Fixed code
def log_hyperparams(self, params: Union[Dict[str, Any], Namespace]) -> None:
    params = self._convert_params(params)
    params = self._flatten_dict(params)
    params = self._sanitize_callable_params(params)
    self.experiment.config.update(params, allow_val_change=True)
# Buggy code
def teardown(self):
    pass


# Traceback (AssertionError)
Traceback (most recent call last):
File "/home/jovyan/transformers/reproduce.py", line 139, in <module>
run_test()
File "/home/jovyan/transformers/reproduce.py", line 135, in run_test
trainer.checkpoint_callback.best_model_path), f'Could not find checkpoint at rank {trainer.global_rank}'
AssertionError: Could not find checkpoint at rank 1

# Fixed code
def teardown(self):
    # Ensure if necessary all processes are finished
    self.barrier()
# Buggy code
def teardown(self):
    # replace the original fwd function
    self.trainer.model.forward = self.model_autocast_original_forward


# Traceback (AssertionError)
Traceback (most recent call last):
File "/home/jovyan/transformers/reproduce.py", line 139, in <module>
run_test()
File "/home/jovyan/transformers/reproduce.py", line 135, in run_test
trainer.checkpoint_callback.best_model_path), f'Could not find checkpoint at rank {trainer.global_rank}'
AssertionError: Could not find checkpoint at rank 1

# Fixed code
def teardown(self):
    # replace the original fwd function
    self.trainer.model.forward = self.model_autocast_original_forward
    self.barrier()
# Buggy code
def __init__(self, trainer):
    self.trainer = trainer
    self.early_stopping_accumulator = None
    self.checkpoint_accumulator = None
    self.accumulated_loss = None
    self._teardown_already_run = False
    self.running_loss = TensorRunningAccum(window_length=20)


# Traceback (AttributeError)
...
File "/venv/lib/python3.6/site-packages/pytorch_lightning/trainer/training_loop.py", line 499, in run_training_epoch
│    self.run_training_epoch_end(epoch_output)
│  File "/venv/lib/python3.6/site-packages/pytorch_lightning/trainer/training_loop.py", line 533, in run_training_epoch_end
│    _processed_outputs = self.process_output(epoch_output)
│  File "/venv/lib/python3.6/site-packages/pytorch_lightning/trainer/logging.py", line 106, in process_output
│    for k, v in output.items():
│AttributeError: 'NoneType' object has no attribute 'items'
│Exception ignored in: <object repr() failed>
│Traceback (most recent call last):
│  File "/venv/lib/python3.6/site-packages/tqdm/std.py", line 1086, in __del__
│  File "/venv/lib/python3.6/site-packages/tqdm/std.py", line 1293, in close
│  File "/venv/lib/python3.6/site-packages/tqdm/std.py", line 1471, in display
│  File "/venv/lib/python3.6/site-packages/tqdm/std.py", line 1089, in __repr__
│  File "/venv/lib/python3.6/site-packages/tqdm/std.py", line 1433, in format_dict
│TypeError: 'NoneType' object is not iterable```

The fix that I've found is to return a dictionary according to the expected typed return, as follows:

# Fixed code
def __init__(self, trainer):
    self.trainer = trainer
    self.early_stopping_accumulator = None
    self.checkpoint_accumulator = None
    self.accumulated_loss = None
    self.warning_cache = WarningCache()
    self._teardown_already_run = False
    self.running_loss = TensorRunningAccum(window_length=20)
# Buggy code
def training_step_and_backward(
    self, split_batch, batch_idx, opt_idx, optimizer, hiddens
):
    """
    wrap the forward step in a closure so second order methods work
    """
    # lightning module hook
    result = self.training_step(split_batch, batch_idx, opt_idx, hiddens)

    # backward pass
    self.backward(result, optimizer, opt_idx)

    # hook
    self.on_after_backward(result.training_step_output, batch_idx, result.loss)

    return result


# Traceback (AttributeError)
...
File "/venv/lib/python3.6/site-packages/pytorch_lightning/trainer/training_loop.py", line 499, in run_training_epoch
│    self.run_training_epoch_end(epoch_output)
│  File "/venv/lib/python3.6/site-packages/pytorch_lightning/trainer/training_loop.py", line 533, in run_training_epoch_end
│    _processed_outputs = self.process_output(epoch_output)
│  File "/venv/lib/python3.6/site-packages/pytorch_lightning/trainer/logging.py", line 106, in process_output
│    for k, v in output.items():
│AttributeError: 'NoneType' object has no attribute 'items'
│Exception ignored in: <object repr() failed>
│Traceback (most recent call last):
│  File "/venv/lib/python3.6/site-packages/tqdm/std.py", line 1086, in __del__
│  File "/venv/lib/python3.6/site-packages/tqdm/std.py", line 1293, in close
│  File "/venv/lib/python3.6/site-packages/tqdm/std.py", line 1471, in display
│  File "/venv/lib/python3.6/site-packages/tqdm/std.py", line 1089, in __repr__
│  File "/venv/lib/python3.6/site-packages/tqdm/std.py", line 1433, in format_dict
│TypeError: 'NoneType' object is not iterable```

The fix that I've found is to return a dictionary according to the expected typed return, as follows:

# Fixed code
def training_step_and_backward(
    self, split_batch, batch_idx, opt_idx, optimizer, hiddens
):
    """
    wrap the forward step in a closure so second order methods work
    """
    # lightning module hook
    result = self.training_step(split_batch, batch_idx, opt_idx, hiddens)

    if result is None:
        self.warning_cache.warn(
            "training_step returned None if it was on purpose, ignore this warning..."
        )
        return None

    # backward pass
    self.backward(result, optimizer, opt_idx)

    # hook
    self.on_after_backward(result.training_step_output, batch_idx, result.loss)

    return result
# Buggy code
def get_num_classes(
    pred: torch.Tensor,
    target: torch.Tensor,
    num_classes: Optional[int] = None,
) -> int:
    """
    Calculates the number of classes for a given prediction and target tensor.

        Args:
            pred: predicted values
            target: true labels
            num_classes: number of classes if known

        Return:
            An integer that represents the number of classes.
    """
    num_target_classes = int(target.max().detach().item() + 1)
    num_pred_classes = int(pred.max().detach().item() + 1)
    num_all_classes = max(num_target_classes, num_pred_classes)

    if num_classes is None:
        num_classes = num_all_classes
    elif num_classes != num_all_classes:
        rank_zero_warn(
            f"You have set {num_classes} number of classes if different from"
            f" predicted ({num_pred_classes}) and target ({num_target_classes}) number of classes"
        )
    return num_classes


# Traceback (RuntimeError)
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-16-5d9063513e76> in <module>
5
6 # calculates accuracy across all GPUs and all Nodes used in training
----> 7 accuracy(pred, target)

~/anaconda3/envs/pl/lib/python3.7/site-packages/pytorch_lightning/metrics/functional/classification.py in accuracy(pred, target, num_classes, reduction)
268     """
269     if not (target > 0).any() and num_classes is None:
--> 270         raise RuntimeError("cannot infer num_classes when target is all zero")
271
272     tps, fps, tns, fns, sups = stat_scores_multiple_classes(

RuntimeError: cannot infer num_classes when target is all zero

# Fixed code
def get_num_classes(
    pred: torch.Tensor,
    target: torch.Tensor,
    num_classes: Optional[int] = None,
) -> int:
    """
    Calculates the number of classes for a given prediction and target tensor.

        Args:
            pred: predicted values
            target: true labels
            num_classes: number of classes if known

        Return:
            An integer that represents the number of classes.
    """
    num_target_classes = int(target.max().detach().item() + 1)
    num_pred_classes = int(pred.max().detach().item() + 1)
    num_all_classes = max(num_target_classes, num_pred_classes)

    if num_classes is None:
        num_classes = num_all_classes
    elif num_classes != num_all_classes:
        rank_zero_warn(
            f"You have set {num_classes} number of classes which is"
            f" different from predicted ({num_pred_classes}) and"
            f" target ({num_target_classes}) number of classes",
            RuntimeWarning,
        )
    return num_classes
# Buggy code
def accuracy(
    pred: torch.Tensor,
    target: torch.Tensor,
    num_classes: Optional[int] = None,
    class_reduction: str = "micro",
) -> torch.Tensor:
    """
    Computes the accuracy classification score

    Args:
        pred: predicted labels
        target: ground truth labels
        num_classes: number of classes
        class_reduction: method to reduce metric score over labels

            - ``'micro'``: calculate metrics globally (default)
            - ``'macro'``: calculate metrics for each label, and find their unweighted mean.
            - ``'weighted'``: calculate metrics for each label, and find their weighted mean.
            - ``'none'``: returns calculated metric per class

    Return:
         A Tensor with the accuracy score.

    Example:

        >>> x = torch.tensor([0, 1, 2, 3])
        >>> y = torch.tensor([0, 1, 2, 2])
        >>> accuracy(x, y)
        tensor(0.7500)

    """
    if not (target > 0).any() and num_classes is None:
        raise RuntimeError("cannot infer num_classes when target is all zero")

    tps, fps, tns, fns, sups = stat_scores_multiple_classes(
        pred=pred, target=target, num_classes=num_classes
    )

    return class_reduce(tps, sups, sups, class_reduction=class_reduction)


# Traceback (RuntimeError)
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-16-5d9063513e76> in <module>
5
6 # calculates accuracy across all GPUs and all Nodes used in training
----> 7 accuracy(pred, target)

~/anaconda3/envs/pl/lib/python3.7/site-packages/pytorch_lightning/metrics/functional/classification.py in accuracy(pred, target, num_classes, reduction)
268     """
269     if not (target > 0).any() and num_classes is None:
--> 270         raise RuntimeError("cannot infer num_classes when target is all zero")
271
272     tps, fps, tns, fns, sups = stat_scores_multiple_classes(

RuntimeError: cannot infer num_classes when target is all zero

# Fixed code
def accuracy(
    pred: torch.Tensor,
    target: torch.Tensor,
    num_classes: Optional[int] = None,
    class_reduction: str = "micro",
) -> torch.Tensor:
    """
    Computes the accuracy classification score

    Args:
        pred: predicted labels
        target: ground truth labels
        num_classes: number of classes
        class_reduction: method to reduce metric score over labels

            - ``'micro'``: calculate metrics globally (default)
            - ``'macro'``: calculate metrics for each label, and find their unweighted mean.
            - ``'weighted'``: calculate metrics for each label, and find their weighted mean.
            - ``'none'``: returns calculated metric per class

    Return:
         A Tensor with the accuracy score.

    Example:

        >>> x = torch.tensor([0, 1, 2, 3])
        >>> y = torch.tensor([0, 1, 2, 2])
        >>> accuracy(x, y)
        tensor(0.7500)

    """
    tps, fps, tns, fns, sups = stat_scores_multiple_classes(
        pred=pred, target=target, num_classes=num_classes
    )

    return class_reduce(tps, sups, sups, class_reduction=class_reduction)
# Buggy code
def teardown(self):
    model = self.trainer.model

    # restore main state with best weights
    best_path = self.mp_queue.get()
    results = self.mp_queue.get()
    last_path = self.mp_queue.get()

    # transfer back the best path to the trainer
    self.trainer.checkpoint_callback.best_model_path = best_path
    # todo, pass also bets score

    # load last weights
    if last_path and not self.trainer.testing:
        ckpt = torch.load(last_path, map_location=lambda storage, loc: storage)
        model.load_state_dict(ckpt)

    self.trainer.model = model

    # when training completes, load the weights back in main process
    self.__load_weights_on_main_process()
    return results


# Traceback (AttributeError)
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-6-6de8b6915beb> in <module>
1 trainer = Trainer(tpu_cores=8, precision=16, max_epochs=5, checkpoint_callback=False, distributed_backend='tpu')
----> 2 trainer.fit(model)

/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/states.py in wrapped_fn(self, *args, **kwargs)
32             if entering is not None:
33                 self.state = entering
---> 34             result = fn(self, *args, **kwargs)
35
36             # The INTERRUPTED state can be set inside the run function. To indicate that run was interrupted

/opt/conda/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py in fit(self, model, train_dataloader, val_dataloaders, datamodule)
1058             self.accelerator_backend.setup()
1059             self.accelerator_backend.train(model)
-> 1060             self.accelerator_backend.teardown(model)
1061
1062         else:

/opt/conda/lib/python3.7/site-packages/pytorch_lightning/accelerators/tpu_backend.py in teardown(self, model)
60
61         # transfer back the best path to the trainer
---> 62         self.trainer.checkpoint_callback.best_model_path = best_path
63         # todo, pass also bets score
64

AttributeError: 'NoneType' object has no attribute 'best_model_path'

# Fixed code
def teardown(self):
    model = self.trainer.model

    # restore main state with best weights
    best_path = self.mp_queue.get()
    results = self.mp_queue.get()
    last_path = self.mp_queue.get()

    # transfer back the best path to the trainer
    if self.trainer.checkpoint_callback is not None:
        self.trainer.checkpoint_callback.best_model_path = best_path
    # todo, pass also bets score

    # load last weights
    if last_path and not self.trainer.testing:
        ckpt = torch.load(last_path, map_location=lambda storage, loc: storage)
        model.load_state_dict(ckpt)

    self.trainer.model = model

    # when training completes, load the weights back in main process
    self.__load_weights_on_main_process()
    return results
# Buggy code
def __call__(cls, *args, **kwargs):
    """A wrapper for LightningDataModule that:

    1. Runs user defined subclass's __init__
    2. Assures prepare_data() runs on rank 0
    3. Lets you check prepare_data and setup to see if they've been called
    """

    # Track prepare_data calls and make sure it runs on rank zero
    cls.prepare_data = track_data_hook_calls(rank_zero_only(cls.prepare_data))
    # Track setup calls
    cls.setup = track_data_hook_calls(cls.setup)

    # Get instance of LightningDataModule by mocking its __init__ via __call__
    obj = type.__call__(cls, *args, **kwargs)

    return obj


# Traceback (RecursionError)
RecursionError occured in the 998-th iteration!
Traceback (most recent call last):
File "test_dm.py", line 18, in <module>
dm.setup()
File "/workspace/src/.venv/lib/python3.8/site-packages/pytorch_lightning/core/datamodule.py", line 85, in wrapped_fn
return fn(*args, **kwargs)
File "/workspace/src/.venv/lib/python3.8/site-packages/pytorch_lightning/core/datamodule.py", line 85, in wrapped_fn
return fn(*args, **kwargs)
File "/workspace/src/.venv/lib/python3.8/site-packages/pytorch_lightning/core/datamodule.py", line 85, in wrapped_fn
return fn(*args, **kwargs)
[Previous line repeated 995 more times]
File "/workspace/src/.venv/lib/python3.8/site-packages/pytorch_lightning/core/datamodule.py", line 69, in wrapped_fn
if fn.__name__ == 'setup':
RecursionError: maximum recursion depth exceeded in comparison

# Fixed code
def __call__(cls, *args, **kwargs):
    """A wrapper for LightningDataModule that:

    1. Runs user defined subclass's __init__
    2. Assures prepare_data() runs on rank 0
    3. Lets you check prepare_data and setup to see if they've been called
    """
    if not cls.__has_added_checks:
        cls.__has_added_checks = True
        # Track prepare_data calls and make sure it runs on rank zero
        cls.prepare_data = track_data_hook_calls(rank_zero_only(cls.prepare_data))
        # Track setup calls
        cls.setup = track_data_hook_calls(cls.setup)

    # Get instance of LightningDataModule by mocking its __init__ via __call__
    obj = type.__call__(cls, *args, **kwargs)

    return obj
# Buggy code
def clean_namespace(hparams):
    """Removes all functions from hparams so we can pickle."""

    if isinstance(hparams, Namespace):
        del_attrs = []
        for k in hparams.__dict__:
            if callable(getattr(hparams, k)):
                del_attrs.append(k)

        for k in del_attrs:
            delattr(hparams, k)

    elif isinstance(hparams, dict):
        del_attrs = []
        for k, v in hparams.items():
            if callable(v):
                del_attrs.append(k)

        for k in del_attrs:
            del hparams[k]


# Traceback (KeyError)
self.hparams
"criterion": BCELoss()
"filters":   64
"layers":    2
"optimizer": functools.partial(<class 'torch.optim.adam.Adam'>, lr=0.001)
GPU available: True, used: False
TPU available: False, using: 0 TPU cores
self.hparams
"filters": 64
"layers":  2
Traceback (most recent call last):
File "lightning_hparams_bug.py", line 61, in <module>
trainer.fit(model)  # KeyError: 'optimizer'
File "/home/*user*/anaconda3/envs/onseilake/lib/python3.7/site-packages/pytorch_lightning/trainer/trainer.py", line 965, in fit
self.optimizers, self.lr_schedulers, self.optimizer_frequencies = self.init_optimizers(model)
File "/home/*user*/anaconda3/envs/onseilake/lib/python3.7/site-packages/pytorch_lightning/trainer/optimizers.py", line 18, in init_optimizers
optim_conf = model.configure_optimizers()
File "lightning_hparams_bug.py", line 51, in configure_optimizers
optimizer = self.hparams["optimizer"](self.parameters())
KeyError: 'optimizer'

# Fixed code
def clean_namespace(hparams):
    """Removes all unpicklable entries from hparams"""

    hparams_dict = hparams
    if isinstance(hparams, Namespace):
        hparams_dict = hparams.__dict__

    del_attrs = [k for k, v in hparams_dict.items() if not is_picklable(v)]

    for k in del_attrs:
        rank_zero_warn(
            f"attribute '{k}' removed from hparams because it cannot be pickled",
            UserWarning,
        )
        del hparams_dict[k]
# Buggy code
def save_spawn_weights(self, model):
    """
    Dump a temporary checkpoint after ddp ends to get weights out of the process
    :param model:
    :return:
    """
    if self.is_global_zero:
        path = os.path.join(self.default_root_dir, "__temp_weight_ddp_end.ckpt")
        self.save_checkpoint(path)


# Traceback (AttributeError)
-- Process 0 terminated with the following error:
Traceback (most recent call last):
File "/opt/conda/lib/python3.6/site-packages/torch/multiprocessing/spawn.py", line 20, in _wrap
fn(i, *args)
File "/opt/conda/lib/python3.6/site-packages/pytorch_lightning/trainer/distrib_data_parallel.py", line 562, in ddp_train
q.put(self.checkpoint_callback.best_model_path)
AttributeError: 'NoneType' object has no attribute 'best_model_path'

# Fixed code
def save_spawn_weights(self, model):
    """
    Dump a temporary checkpoint after ddp ends to get weights out of the process
    :param model:
    :return:
    """
    if self.is_global_zero:
        path = os.path.join(self.default_root_dir, "__temp_weight_ddp_end.ckpt")
        self.save_checkpoint(path)
        return path