# Buggy code
def wrapped_func(*args, **kwargs):
    # return cached value
    if self._computed is not None:
        return self._computed

    if (
        self._to_sync
        and torch.distributed.is_available()  # noqa: W503
        and torch.distributed.is_initialized()  # noqa: W503
    ):
        self._sync_dist()

    self._computed = compute(*args, **kwargs)
    self.reset()

    return self._computed


# Traceback (RuntimeError)
Epoch 7:   0% 0/8 [00:00<?, ?it/s, loss=0.480, v_num=50]Traceback (most recent call last):
File "main.py", line 72, in <module>
main()
File "main.py", line 68, in main
trainer.fit(model, data_module)
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/states.py", line 48, in wrapped_fn
result = fn(self, *args, **kwargs)
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1016, in fit
results = self.accelerator_backend.train()
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/accelerators/horovod_backend.py", line 108, in train
result = self.trainer.run_pretrain_routine(self.trainer.model)
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1237, in run_pretrain_routine
self.train()
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py", line 396, in train
self.run_training_epoch()
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py", line 484, in run_training_epoch
batch_output = self.run_training_batch(batch, batch_idx)
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py", line 832, in run_training_batch
opt_closure_result = self.optimizer_closure(
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py", line 1065, in optimizer_closure
model_ref.backward(self, closure_loss, optimizer, opt_idx)
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/core/hooks.py", line 312, in backward
loss.backward()
File "/home/aca10027xu/.local/lib/python3.8/site-packages/torch/tensor.py", line 185, in backward
torch.autograd.backward(self, gradient, retain_graph, create_graph)
File "/home/aca10027xu/.local/lib/python3.8/site-packages/torch/autograd/__init__.py", line 125, in backward
Variable._execution_engine.run_backward(
RuntimeError: Horovod has been shut down. This was caused by an exception on one of the ranks or an attempt to allreduce, allgather or broadcast a tensor after one of the ranks finished execution. If the shutdown was caused by an exception, you should see the exception in the log before the first shutdown message.
Exception ignored in: <function tqdm.__del__ at 0x2b61156e6820>
Traceback (most recent call last):
File "/home/aca10027xu/.local/lib/python3.8/site-packages/tqdm/std.py", line 1086, in __del__
File "/home/aca10027xu/.local/lib/python3.8/site-packages/tqdm/std.py", line 1293, in close
File "/home/aca10027xu/.local/lib/python3.8/site-packages/tqdm/std.py", line 1471, in display
File "/home/aca10027xu/.local/lib/python3.8/site-packages/tqdm/std.py", line 1089, in __repr__
File "/home/aca10027xu/.local/lib/python3.8/site-packages/tqdm/std.py", line 1433, in format_dict
TypeError: cannot unpack non-iterable NoneType object

# Fixed code
def wrapped_func(*args, **kwargs):
    # return cached value
    if self._computed is not None:
        return self._computed

    dist_sync_fn = self.dist_sync_fn
    if (
        dist_sync_fn is None
        and torch.distributed.is_available()
        and torch.distributed.is_initialized()
    ):
        # User provided a bool, so we assume DDP if available
        dist_sync_fn = gather_all_tensors

    if self._to_sync and dist_sync_fn is not None:
        self._sync_dist(dist_sync_fn)

    self._computed = compute(*args, **kwargs)
    self.reset()

    return self._computed