# Buggy code
def write(text):
    if index is not None:
        text = prepend_context(text, index, prefix)
    dst_stream.write(text)
    dst_stream.flush()


# Traceback (UnicodeDecodeError)
Tue Oct 13 10:54:25 2020[0]<stderr>:[10/13/2020 10:54:25 - INFO - __main__ -   start running validation...
Tue Oct 13 10:54:25 2020[0]<stderr>:  0%|          | 0/1327 [00:00<?, ?it/s]
Tue Oct 13 10:55:56 2020[0]<stderr>:
Exception in thread Thread-17:derr>: 65%|██████▌   | 866/1327 [01:17<00:47,  9.80it/s]
Traceback (most recent call last):
File "/data/anaconda3/envs/vcr/lib/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "/data/anaconda3/envs/vcr/lib/python3.6/threading.py", line 864, in run
self._target(*self._args, **self._kwargs)
File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/runner/common/util/safe_shell_exec.py", line 104, in forward_stream
text = text.decode('utf-8')
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe2 in position 999: unexpected end of data

Tue Oct 13 10:58:56 2020[3]<stderr>:[2020-10-13 10:58:56.365049: E /dockerdata/app/tmp/pip-install-998q9kn4/horovod/horovod/common/operations.cc:525] Horovod background loop uncaught exception: [/dockerdata/app/tmp/pip-install-998q9kn4/horovod/third_party/compatible_gloo/gloo/transport/tcp/unbound_buffer.cc:84] Timed out waiting 30000ms for recv operation to complete
Tue Oct 13 10:58:56 2020[2]<stderr>:[2020-10-13 10:58:56.366492: E /dockerdata/app/tmp/pip-install-998q9kn4/horovod/horovod/common/operations.cc:525] Horovod background loop uncaught exception: [/dockerdata/app/tmp/pip-install-998q9kn4/horovod/third_party/compatible_gloo/gloo/transport/tcp/unbound_buffer.cc:84] Timed out waiting 30000ms for recv operation to complete
Tue Oct 13 10:58:56 2020[1]<stderr>:[2020-10-13 10:58:56.366674: E /dockerdata/app/tmp/pip-install-998q9kn4/horovod/horovod/common/operations.cc:525] Horovod background loop uncaught exception: [/dockerdata/app/tmp/pip-install-998q9kn4/horovod/third_party/compatible_gloo/gloo/transport/tcp/unbound_buffer.cc:136] Timed out waiting 30000ms for send operation to complete
Tue Oct 13 10:58:56 2020[3]<stderr>:Traceback (most recent call last):
Tue Oct 13 10:58:56 2020[1]<stderr>:Traceback (most recent call last):
Tue Oct 13 10:58:56 2020[3]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 622, in synchronize
Tue Oct 13 10:58:56 2020[2]<stderr>:Traceback (most recent call last):
Tue Oct 13 10:58:56 2020[3]<stderr>:    mpi_lib.horovod_torch_wait_and_clear(handle)
Tue Oct 13 10:58:56 2020[2]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 622, in synchronize
Tue Oct 13 10:58:56 2020[3]<stderr>:RuntimeError: Horovod has been shut down. This was caused by an exception on one of the ranks or an attempt to allreduce, allgather or broadcast a tensor after one of the ranks finished execution. If the shutdown was caused by an exception, you should see the exception in the log before the first shutdown message.
Tue Oct 13 10:58:56 2020[1]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 622, in synchronize
Tue Oct 13 10:58:56 2020[3]<stderr>:
Tue Oct 13 10:58:56 2020[1]<stderr>:    mpi_lib.horovod_torch_wait_and_clear(handle)
Tue Oct 13 10:58:56 2020[2]<stderr>:    mpi_lib.horovod_torch_wait_and_clear(handle)
Tue Oct 13 10:58:57 2020[3]<stderr>:During handling of the above exception, another exception occurred:
Tue Oct 13 10:58:57 2020[2]<stderr>:RuntimeError: Horovod has been shut down. This was caused by an exception on one of the ranks or an attempt to allreduce, allgather or broadcast a tensor after one of the ranks finished execution. If the shutdown was caused by an exception, you should see the exception in the log before the first shutdown message.
Tue Oct 13 10:58:57 2020[1]<stderr>:RuntimeError: Horovod has been shut down. This was caused by an exception on one of the ranks or an attempt to allreduce, allgather or broadcast a tensor after one of the ranks finished execution. If the shutdown was caused by an exception, you should see the exception in the log before the first shutdown message.
Tue Oct 13 10:58:57 2020[3]<stderr>:
Tue Oct 13 10:58:57 2020[1]<stderr>:
Tue Oct 13 10:58:57 2020[2]<stderr>:
Tue Oct 13 10:58:57 2020[3]<stderr>:Traceback (most recent call last):
Tue Oct 13 10:58:57 2020[2]<stderr>:During handling of the above exception, another exception occurred:
Tue Oct 13 10:58:57 2020[1]<stderr>:During handling of the above exception, another exception occurred:
Tue Oct 13 10:58:57 2020[3]<stderr>:  File "train_vcr.py", line 332, in <module>
Tue Oct 13 10:58:57 2020[1]<stderr>:
Tue Oct 13 10:58:57 2020[3]<stderr>:    main(parse_cmd_args(TrainingOpts))
Tue Oct 13 10:58:57 2020[1]<stderr>:Traceback (most recent call last):
Tue Oct 13 10:58:57 2020[3]<stderr>:  File "train_vcr.py", line 260, in main
Tue Oct 13 10:58:57 2020[1]<stderr>:  File "train_vcr.py", line 332, in <module>
Tue Oct 13 10:58:57 2020[3]<stderr>:    val_log = validate(model, val_dataloader)
Tue Oct 13 10:58:57 2020[1]<stderr>:    main(parse_cmd_args(TrainingOpts))
Tue Oct 13 10:58:57 2020[3]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/torch/autograd/grad_mode.py", line 49, in decorate_no_grad
Tue Oct 13 10:58:57 2020[1]<stderr>:  File "train_vcr.py", line 260, in main
Tue Oct 13 10:58:57 2020[3]<stderr>:    return func(*args, **kwargs)
Tue Oct 13 10:58:57 2020[1]<stderr>:    val_log = validate(model, val_dataloader)
Tue Oct 13 10:58:58 2020[3]<stderr>:  File "train_vcr.py", line 310, in validate
Tue Oct 13 10:58:58 2020[1]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/torch/autograd/grad_mode.py", line 49, in decorate_no_grad
Tue Oct 13 10:58:57 2020[2]<stderr>:
Tue Oct 13 10:58:58 2020[3]<stderr>:    qa_loss = sum(all_gather_list(qa_loss))
Tue Oct 13 10:58:58 2020[1]<stderr>:    return func(*args, **kwargs)
Tue Oct 13 10:58:58 2020[2]<stderr>:Traceback (most recent call last):
Tue Oct 13 10:58:58 2020[1]<stderr>:  File "train_vcr.py", line 310, in validate
Tue Oct 13 10:58:58 2020[2]<stderr>:  File "train_vcr.py", line 332, in <module>
Tue Oct 13 10:58:58 2020[3]<stderr>:  File "/data/cdp_algo_ceph_ssd/users/haoyanhuo/vcr/utils/distributed.py", line 233, in all_gather_list
Tue Oct 13 10:58:58 2020[2]<stderr>:    main(parse_cmd_args(TrainingOpts))
Tue Oct 13 10:58:58 2020[3]<stderr>:    max_size = hvd.allgather(torch.tensor([enc_size]).cuda()).max().item()
Tue Oct 13 10:58:58 2020[1]<stderr>:    qa_loss = sum(all_gather_list(qa_loss))
Tue Oct 13 10:58:58 2020[2]<stderr>:  File "train_vcr.py", line 260, in main
Tue Oct 13 10:58:58 2020[3]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 365, in allgather
Tue Oct 13 10:58:58 2020[2]<stderr>:    val_log = validate(model, val_dataloader)
Tue Oct 13 10:58:58 2020[3]<stderr>:    return HorovodAllgather.apply(tensor, name)
Tue Oct 13 10:58:58 2020[1]<stderr>:  File "/data/cdp_algo_ceph_ssd/users/haoyanhuo/vcr/utils/distributed.py", line 233, in all_gather_list
Tue Oct 13 10:58:58 2020[2]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/torch/autograd/grad_mode.py", line 49, in decorate_no_grad
Tue Oct 13 10:58:58 2020[1]<stderr>:    max_size = hvd.allgather(torch.tensor([enc_size]).cuda()).max().item()
Tue Oct 13 10:58:58 2020[3]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 328, in forward
Tue Oct 13 10:58:59 2020[2]<stderr>:    return func(*args, **kwargs)
Tue Oct 13 10:58:59 2020[3]<stderr>:    return synchronize(handle)
Tue Oct 13 10:58:59 2020[2]<stderr>:  File "train_vcr.py", line 310, in validate
Tue Oct 13 10:58:59 2020[3]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 626, in synchronize
Tue Oct 13 10:58:59 2020[2]<stderr>:    qa_loss = sum(all_gather_list(qa_loss))
Tue Oct 13 10:58:59 2020[3]<stderr>:    raise HorovodInternalError(e)
Tue Oct 13 10:58:59 2020[2]<stderr>:  File "/data/cdp_algo_ceph_ssd/users/haoyanhuo/vcr/utils/distributed.py", line 233, in all_gather_list
Tue Oct 13 10:58:59 2020[3]<stderr>:horovod.common.exceptions.HorovodInternalError: Horovod has been shut down. This was causedby an exception on one of the ranks or an attempt to allreduce, allgather or broadcast a tensor after one of the ranks finishedexecution. If the shutdown was caused by an exception, you should see the exception in the log before the first shutdown message.
Tue Oct 13 10:58:59 2020[1]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 365, in allgather
Tue Oct 13 10:58:59 2020[2]<stderr>:    max_size = hvd.allgather(torch.tensor([enc_size]).cuda()).max().item()
Tue Oct 13 10:58:59 2020[2]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 365, in allgather
Tue Oct 13 10:58:59 2020[1]<stderr>:    return HorovodAllgather.apply(tensor, name)
Tue Oct 13 10:58:59 2020[1]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 328, in forward
Tue Oct 13 10:58:59 2020[2]<stderr>:    return HorovodAllgather.apply(tensor, name)
Tue Oct 13 10:58:59 2020[2]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 328, in forward
Tue Oct 13 10:58:59 2020[1]<stderr>:    return synchronize(handle)
Tue Oct 13 10:58:59 2020[1]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 626, in synchronize
Tue Oct 13 10:58:59 2020[2]<stderr>:    return synchronize(handle)
Tue Oct 13 10:58:59 2020[1]<stderr>:    raise HorovodInternalError(e)
Tue Oct 13 10:59:00 2020[2]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 626, in synchronize
Tue Oct 13 10:59:00 2020[1]<stderr>:horovod.common.exceptions.HorovodInternalError: Horovod has been shut down. This was causedby an exception on one of the ranks or an attempt to allreduce, allgather or broadcast a tensor after one of the ranks finishedexecution. If the shutdown was caused by an exception, you should see the exception in the log before the first shutdown message.
Tue Oct 13 10:59:00 2020[2]<stderr>:    raise HorovodInternalError(e)
Tue Oct 13 10:59:00 2020[2]<stderr>:horovod.common.exceptions.HorovodInternalError: Horovod has been shut down. This was causedby an exception on one of the ranks or an attempt to allreduce, allgather or broadcast a tensor after one of the ranks finishedexecution. If the shutdown was caused by an exception, you should see the exception in the log before the first shutdown message.

# Fixed code
def write(text):
    if index is not None and prefix is not None:
        context = get_context(index, prefix)
        dst_stream.write(context)
    dst_stream.write(text)
    dst_stream.flush()