# Buggy code
def _sync_dist(self):
    input_dict = {attr: getattr(self, attr) for attr in self._reductions.keys()}
    output_dict = apply_to_collection(
        input_dict,
        torch.Tensor,
        gather_all_tensors_if_available,
        group=self.process_group,
    )

    for attr, reduction_fn in self._reductions.items():
        # pre-processing ops (stack or flatten for inputs)
        if isinstance(output_dict[attr][0], torch.Tensor):
            output_dict[attr] = torch.stack(output_dict[attr])
        elif isinstance(output_dict[attr][0], list):
            output_dict[attr] = _flatten(output_dict[attr])

        assert isinstance(reduction_fn, (Callable)) or reduction_fn is None
        reduced = (
            reduction_fn(output_dict[attr])
            if reduction_fn is not None
            else output_dict[attr]
        )
        setattr(self, attr, reduced)


# Traceback (RuntimeError)
Epoch 7:   0% 0/8 [00:00<?, ?it/s, loss=0.480, v_num=50]Traceback (most recent call last):
File "main.py", line 72, in <module>
main()
File "main.py", line 68, in main
trainer.fit(model, data_module)
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/states.py", line 48, in wrapped_fn
result = fn(self, *args, **kwargs)
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1016, in fit
results = self.accelerator_backend.train()
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/accelerators/horovod_backend.py", line 108, in train
result = self.trainer.run_pretrain_routine(self.trainer.model)
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py", line 1237, in run_pretrain_routine
self.train()
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py", line 396, in train
self.run_training_epoch()
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py", line 484, in run_training_epoch
batch_output = self.run_training_batch(batch, batch_idx)
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py", line 832, in run_training_batch
opt_closure_result = self.optimizer_closure(
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/training_loop.py", line 1065, in optimizer_closure
model_ref.backward(self, closure_loss, optimizer, opt_idx)
File "/home/aca10027xu/.local/lib/python3.8/site-packages/pytorch_lightning/core/hooks.py", line 312, in backward
loss.backward()
File "/home/aca10027xu/.local/lib/python3.8/site-packages/torch/tensor.py", line 185, in backward
torch.autograd.backward(self, gradient, retain_graph, create_graph)
File "/home/aca10027xu/.local/lib/python3.8/site-packages/torch/autograd/__init__.py", line 125, in backward
Variable._execution_engine.run_backward(
RuntimeError: Horovod has been shut down. This was caused by an exception on one of the ranks or an attempt to allreduce, allgather or broadcast a tensor after one of the ranks finished execution. If the shutdown was caused by an exception, you should see the exception in the log before the first shutdown message.
Exception ignored in: <function tqdm.__del__ at 0x2b61156e6820>
Traceback (most recent call last):
File "/home/aca10027xu/.local/lib/python3.8/site-packages/tqdm/std.py", line 1086, in __del__
File "/home/aca10027xu/.local/lib/python3.8/site-packages/tqdm/std.py", line 1293, in close
File "/home/aca10027xu/.local/lib/python3.8/site-packages/tqdm/std.py", line 1471, in display
File "/home/aca10027xu/.local/lib/python3.8/site-packages/tqdm/std.py", line 1089, in __repr__
File "/home/aca10027xu/.local/lib/python3.8/site-packages/tqdm/std.py", line 1433, in format_dict
TypeError: cannot unpack non-iterable NoneType object

# Fixed code
def _sync_dist(self, dist_sync_fn=gather_all_tensors):
    input_dict = {attr: getattr(self, attr) for attr in self._reductions.keys()}
    output_dict = apply_to_collection(
        input_dict,
        torch.Tensor,
        dist_sync_fn,
        group=self.process_group,
    )

    for attr, reduction_fn in self._reductions.items():
        # pre-processing ops (stack or flatten for inputs)
        if isinstance(output_dict[attr][0], torch.Tensor):
            output_dict[attr] = torch.stack(output_dict[attr])
        elif isinstance(output_dict[attr][0], list):
            output_dict[attr] = _flatten(output_dict[attr])

        assert isinstance(reduction_fn, (Callable)) or reduction_fn is None
        reduced = (
            reduction_fn(output_dict[attr])
            if reduction_fn is not None
            else output_dict[attr]
        )
        setattr(self, attr, reduced)