# Buggy code
def _exec_middleman(command, env, exit_event, stdout, stderr, rw):
    stdout_r, stdout_w = stdout
    stderr_r, stderr_w = stderr
    r, w = rw

    # Close unused file descriptors to enforce PIPE behavior.
    stdout_r.close()
    stderr_r.close()
    w.close()
    os.setsid()

    executor_shell = subprocess.Popen(
        command, shell=True, env=env, stdout=stdout_w, stderr=stderr_w
    )

    # we don't bother stopping the on_event thread, this process sys.exits soon
    # so the on_event thread has to be a deamon thread
    on_event(
        exit_event,
        terminate_executor_shell_and_children,
        args=(executor_shell.pid,),
        daemon=True,
    )

    def kill_executor_children_if_parent_dies():
        # This read blocks until the pipe is closed on the other side
        # due to parent process termination (for any reason, including -9).
        os.read(r.fileno(), 1)
        terminate_executor_shell_and_children(executor_shell.pid)

    in_thread(kill_executor_children_if_parent_dies)

    exit_code = executor_shell.wait()
    if exit_code < 0:
        # See: https://www.gnu.org/software/bash/manual/html_node/Exit-Status.html
        exit_code = 128 + abs(exit_code)

    sys.exit(exit_code)


# Traceback (UnicodeDecodeError)
Tue Oct 13 10:54:25 2020[0]<stderr>:[10/13/2020 10:54:25 - INFO - __main__ -   start running validation...
Tue Oct 13 10:54:25 2020[0]<stderr>:  0%|          | 0/1327 [00:00<?, ?it/s]
Tue Oct 13 10:55:56 2020[0]<stderr>:
Exception in thread Thread-17:derr>: 65%|██████▌   | 866/1327 [01:17<00:47,  9.80it/s]
Traceback (most recent call last):
File "/data/anaconda3/envs/vcr/lib/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "/data/anaconda3/envs/vcr/lib/python3.6/threading.py", line 864, in run
self._target(*self._args, **self._kwargs)
File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/runner/common/util/safe_shell_exec.py", line 104, in forward_stream
text = text.decode('utf-8')
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe2 in position 999: unexpected end of data

Tue Oct 13 10:58:56 2020[3]<stderr>:[2020-10-13 10:58:56.365049: E /dockerdata/app/tmp/pip-install-998q9kn4/horovod/horovod/common/operations.cc:525] Horovod background loop uncaught exception: [/dockerdata/app/tmp/pip-install-998q9kn4/horovod/third_party/compatible_gloo/gloo/transport/tcp/unbound_buffer.cc:84] Timed out waiting 30000ms for recv operation to complete
Tue Oct 13 10:58:56 2020[2]<stderr>:[2020-10-13 10:58:56.366492: E /dockerdata/app/tmp/pip-install-998q9kn4/horovod/horovod/common/operations.cc:525] Horovod background loop uncaught exception: [/dockerdata/app/tmp/pip-install-998q9kn4/horovod/third_party/compatible_gloo/gloo/transport/tcp/unbound_buffer.cc:84] Timed out waiting 30000ms for recv operation to complete
Tue Oct 13 10:58:56 2020[1]<stderr>:[2020-10-13 10:58:56.366674: E /dockerdata/app/tmp/pip-install-998q9kn4/horovod/horovod/common/operations.cc:525] Horovod background loop uncaught exception: [/dockerdata/app/tmp/pip-install-998q9kn4/horovod/third_party/compatible_gloo/gloo/transport/tcp/unbound_buffer.cc:136] Timed out waiting 30000ms for send operation to complete
Tue Oct 13 10:58:56 2020[3]<stderr>:Traceback (most recent call last):
Tue Oct 13 10:58:56 2020[1]<stderr>:Traceback (most recent call last):
Tue Oct 13 10:58:56 2020[3]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 622, in synchronize
Tue Oct 13 10:58:56 2020[2]<stderr>:Traceback (most recent call last):
Tue Oct 13 10:58:56 2020[3]<stderr>:    mpi_lib.horovod_torch_wait_and_clear(handle)
Tue Oct 13 10:58:56 2020[2]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 622, in synchronize
Tue Oct 13 10:58:56 2020[3]<stderr>:RuntimeError: Horovod has been shut down. This was caused by an exception on one of the ranks or an attempt to allreduce, allgather or broadcast a tensor after one of the ranks finished execution. If the shutdown was caused by an exception, you should see the exception in the log before the first shutdown message.
Tue Oct 13 10:58:56 2020[1]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 622, in synchronize
Tue Oct 13 10:58:56 2020[3]<stderr>:
Tue Oct 13 10:58:56 2020[1]<stderr>:    mpi_lib.horovod_torch_wait_and_clear(handle)
Tue Oct 13 10:58:56 2020[2]<stderr>:    mpi_lib.horovod_torch_wait_and_clear(handle)
Tue Oct 13 10:58:57 2020[3]<stderr>:During handling of the above exception, another exception occurred:
Tue Oct 13 10:58:57 2020[2]<stderr>:RuntimeError: Horovod has been shut down. This was caused by an exception on one of the ranks or an attempt to allreduce, allgather or broadcast a tensor after one of the ranks finished execution. If the shutdown was caused by an exception, you should see the exception in the log before the first shutdown message.
Tue Oct 13 10:58:57 2020[1]<stderr>:RuntimeError: Horovod has been shut down. This was caused by an exception on one of the ranks or an attempt to allreduce, allgather or broadcast a tensor after one of the ranks finished execution. If the shutdown was caused by an exception, you should see the exception in the log before the first shutdown message.
Tue Oct 13 10:58:57 2020[3]<stderr>:
Tue Oct 13 10:58:57 2020[1]<stderr>:
Tue Oct 13 10:58:57 2020[2]<stderr>:
Tue Oct 13 10:58:57 2020[3]<stderr>:Traceback (most recent call last):
Tue Oct 13 10:58:57 2020[2]<stderr>:During handling of the above exception, another exception occurred:
Tue Oct 13 10:58:57 2020[1]<stderr>:During handling of the above exception, another exception occurred:
Tue Oct 13 10:58:57 2020[3]<stderr>:  File "train_vcr.py", line 332, in <module>
Tue Oct 13 10:58:57 2020[1]<stderr>:
Tue Oct 13 10:58:57 2020[3]<stderr>:    main(parse_cmd_args(TrainingOpts))
Tue Oct 13 10:58:57 2020[1]<stderr>:Traceback (most recent call last):
Tue Oct 13 10:58:57 2020[3]<stderr>:  File "train_vcr.py", line 260, in main
Tue Oct 13 10:58:57 2020[1]<stderr>:  File "train_vcr.py", line 332, in <module>
Tue Oct 13 10:58:57 2020[3]<stderr>:    val_log = validate(model, val_dataloader)
Tue Oct 13 10:58:57 2020[1]<stderr>:    main(parse_cmd_args(TrainingOpts))
Tue Oct 13 10:58:57 2020[3]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/torch/autograd/grad_mode.py", line 49, in decorate_no_grad
Tue Oct 13 10:58:57 2020[1]<stderr>:  File "train_vcr.py", line 260, in main
Tue Oct 13 10:58:57 2020[3]<stderr>:    return func(*args, **kwargs)
Tue Oct 13 10:58:57 2020[1]<stderr>:    val_log = validate(model, val_dataloader)
Tue Oct 13 10:58:58 2020[3]<stderr>:  File "train_vcr.py", line 310, in validate
Tue Oct 13 10:58:58 2020[1]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/torch/autograd/grad_mode.py", line 49, in decorate_no_grad
Tue Oct 13 10:58:57 2020[2]<stderr>:
Tue Oct 13 10:58:58 2020[3]<stderr>:    qa_loss = sum(all_gather_list(qa_loss))
Tue Oct 13 10:58:58 2020[1]<stderr>:    return func(*args, **kwargs)
Tue Oct 13 10:58:58 2020[2]<stderr>:Traceback (most recent call last):
Tue Oct 13 10:58:58 2020[1]<stderr>:  File "train_vcr.py", line 310, in validate
Tue Oct 13 10:58:58 2020[2]<stderr>:  File "train_vcr.py", line 332, in <module>
Tue Oct 13 10:58:58 2020[3]<stderr>:  File "/data/cdp_algo_ceph_ssd/users/haoyanhuo/vcr/utils/distributed.py", line 233, in all_gather_list
Tue Oct 13 10:58:58 2020[2]<stderr>:    main(parse_cmd_args(TrainingOpts))
Tue Oct 13 10:58:58 2020[3]<stderr>:    max_size = hvd.allgather(torch.tensor([enc_size]).cuda()).max().item()
Tue Oct 13 10:58:58 2020[1]<stderr>:    qa_loss = sum(all_gather_list(qa_loss))
Tue Oct 13 10:58:58 2020[2]<stderr>:  File "train_vcr.py", line 260, in main
Tue Oct 13 10:58:58 2020[3]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 365, in allgather
Tue Oct 13 10:58:58 2020[2]<stderr>:    val_log = validate(model, val_dataloader)
Tue Oct 13 10:58:58 2020[3]<stderr>:    return HorovodAllgather.apply(tensor, name)
Tue Oct 13 10:58:58 2020[1]<stderr>:  File "/data/cdp_algo_ceph_ssd/users/haoyanhuo/vcr/utils/distributed.py", line 233, in all_gather_list
Tue Oct 13 10:58:58 2020[2]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/torch/autograd/grad_mode.py", line 49, in decorate_no_grad
Tue Oct 13 10:58:58 2020[1]<stderr>:    max_size = hvd.allgather(torch.tensor([enc_size]).cuda()).max().item()
Tue Oct 13 10:58:58 2020[3]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 328, in forward
Tue Oct 13 10:58:59 2020[2]<stderr>:    return func(*args, **kwargs)
Tue Oct 13 10:58:59 2020[3]<stderr>:    return synchronize(handle)
Tue Oct 13 10:58:59 2020[2]<stderr>:  File "train_vcr.py", line 310, in validate
Tue Oct 13 10:58:59 2020[3]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 626, in synchronize
Tue Oct 13 10:58:59 2020[2]<stderr>:    qa_loss = sum(all_gather_list(qa_loss))
Tue Oct 13 10:58:59 2020[3]<stderr>:    raise HorovodInternalError(e)
Tue Oct 13 10:58:59 2020[2]<stderr>:  File "/data/cdp_algo_ceph_ssd/users/haoyanhuo/vcr/utils/distributed.py", line 233, in all_gather_list
Tue Oct 13 10:58:59 2020[3]<stderr>:horovod.common.exceptions.HorovodInternalError: Horovod has been shut down. This was causedby an exception on one of the ranks or an attempt to allreduce, allgather or broadcast a tensor after one of the ranks finishedexecution. If the shutdown was caused by an exception, you should see the exception in the log before the first shutdown message.
Tue Oct 13 10:58:59 2020[1]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 365, in allgather
Tue Oct 13 10:58:59 2020[2]<stderr>:    max_size = hvd.allgather(torch.tensor([enc_size]).cuda()).max().item()
Tue Oct 13 10:58:59 2020[2]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 365, in allgather
Tue Oct 13 10:58:59 2020[1]<stderr>:    return HorovodAllgather.apply(tensor, name)
Tue Oct 13 10:58:59 2020[1]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 328, in forward
Tue Oct 13 10:58:59 2020[2]<stderr>:    return HorovodAllgather.apply(tensor, name)
Tue Oct 13 10:58:59 2020[2]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 328, in forward
Tue Oct 13 10:58:59 2020[1]<stderr>:    return synchronize(handle)
Tue Oct 13 10:58:59 2020[1]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 626, in synchronize
Tue Oct 13 10:58:59 2020[2]<stderr>:    return synchronize(handle)
Tue Oct 13 10:58:59 2020[1]<stderr>:    raise HorovodInternalError(e)
Tue Oct 13 10:59:00 2020[2]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 626, in synchronize
Tue Oct 13 10:59:00 2020[1]<stderr>:horovod.common.exceptions.HorovodInternalError: Horovod has been shut down. This was causedby an exception on one of the ranks or an attempt to allreduce, allgather or broadcast a tensor after one of the ranks finishedexecution. If the shutdown was caused by an exception, you should see the exception in the log before the first shutdown message.
Tue Oct 13 10:59:00 2020[2]<stderr>:    raise HorovodInternalError(e)
Tue Oct 13 10:59:00 2020[2]<stderr>:horovod.common.exceptions.HorovodInternalError: Horovod has been shut down. This was causedby an exception on one of the ranks or an attempt to allreduce, allgather or broadcast a tensor after one of the ranks finishedexecution. If the shutdown was caused by an exception, you should see the exception in the log before the first shutdown message.

# Fixed code
def _exec_middleman(command, env, exit_event, stdout, stderr, rw):
    stdout_r, stdout_w = stdout
    stderr_r, stderr_w = stderr
    r, w = rw

    # Close unused file descriptors to enforce PIPE behavior.
    stdout_r.close()
    stderr_r.close()
    w.close()
    os.setsid()

    executor_shell = subprocess.Popen(
        command, shell=True, env=env, stdout=stdout_w, stderr=stderr_w
    )

    # we don't bother stopping the on_event thread, this process sys.exits soon
    # so the on_event thread has to be a daemon thread
    on_event(
        exit_event,
        terminate_executor_shell_and_children,
        args=(executor_shell.pid,),
        daemon=True,
    )

    def kill_executor_children_if_parent_dies():
        # This read blocks until the pipe is closed on the other side
        # due to parent process termination (for any reason, including -9).
        os.read(r.fileno(), 1)
        terminate_executor_shell_and_children(executor_shell.pid)

    in_thread(kill_executor_children_if_parent_dies)

    exit_code = executor_shell.wait()
    if exit_code < 0:
        # See: https://www.gnu.org/software/bash/manual/html_node/Exit-Status.html
        exit_code = 128 + abs(exit_code)

    sys.exit(exit_code)