# Buggy code
def _wrap_compute(self, compute):
    @functools.wraps(compute)
    def wrapped_func(*args, **kwargs):
        # return cached value
        if self._computed is not None:
            return self._computed

        if (
            self._to_sync
            and torch.distributed.is_available()  # noqa: W503
            and torch.distributed.is_initialized()  # noqa: W503
        ):
            self._sync_dist()

        self._computed = compute(*args, **kwargs)
        self.reset()

        return self._computed

    return wrapped_func


# Traceback (RuntimeError)
Epoch 7:   0% 0/8 [00:00<?, ?it/s, loss=0.480, v_num=50]Traceback (most recent call last):
File "main.py", line 72, in <module>
main()
File "main.py", line 68, in main
trainer.fit(model, data_module)
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/states.py", line 48, in wrapped_fn
result = fn(self, *args, **kwargs)
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1016, in fit
results = self.accelerator_backend.train()
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/accelerators/horovod_backend.py", line 108, in train
result = self.trainer.run_pretrain_routine(self.trainer.model)
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1237, in run_pretrain_routine
self.train()
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py", line 396, in train
self.run_training_epoch()
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py", line 484, in run_training_epoch
batch_output = self.run_training_batch(batch, batch_idx)
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py", line 832, in run_training_batch
opt_closure_result = self.optimizer_closure(
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py", line 1065, in optimizer_closure
model_ref.backward(self, closure_loss, optimizer, opt_idx)
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/core/hooks.py", line 312, in backward
loss.backward()
File "/home/aca10027xu/.local/lib/python3.8/site-packages/torch/tensor.py", line 185, in backward
torch.autograd.backward(self, gradient, retain_graph, create_graph)
File "/home/aca10027xu/.local/lib/python3.8/site-packages/torch/autograd/__init__.py", line 125, in backward
Variable._execution_engine.run_backward(
RuntimeError: Horovod has been shut down. This was caused by an exception on one of the ranks or an attempt to allreduce, allgather or broadcast a tensor after one of the ranks finished execution. If the shutdown was caused by an exception, you should see the exception in the log before the first shutdown message.
Exception ignored in: <function tqdm.__del__ at 0x2b61156e6820>
Traceback (most recent call last):
File "/home/aca10027xu/.local/lib/python3.8/site-packages/tqdm/std.py", line 1086, in __del__
File "/home/aca10027xu/.local/lib/python3.8/site-packages/tqdm/std.py", line 1293, in close
File "/home/aca10027xu/.local/lib/python3.8/site-packages/tqdm/std.py", line 1471, in display
File "/home/aca10027xu/.local/lib/python3.8/site-packages/tqdm/std.py", line 1089, in __repr__
File "/home/aca10027xu/.local/lib/python3.8/site-packages/tqdm/std.py", line 1433, in format_dict
TypeError: cannot unpack non-iterable NoneType object

# Fixed code
def _wrap_compute(self, compute):
    @functools.wraps(compute)
    def wrapped_func(*args, **kwargs):
        # return cached value
        if self._computed is not None:
            return self._computed

        dist_sync_fn = self.dist_sync_fn
        if (
            dist_sync_fn is None
            and torch.distributed.is_available()
            and torch.distributed.is_initialized()
        ):
            # User provided a bool, so we assume DDP if available
            dist_sync_fn = gather_all_tensors

        if self._to_sync and dist_sync_fn is not None:
            self._sync_dist(dist_sync_fn)

        self._computed = compute(*args, **kwargs)
        self.reset()

        return self._computed

    return wrapped_func