# Buggy code
def execute(
    command,
    env=None,
    stdout=None,
    stderr=None,
    index=None,
    events=None,
    prefix_output_with_timestamp=False,
):
    ctx = multiprocessing.get_context("spawn")

    # When this event is set, signal to middleman to terminate its children and exit.
    exit_event = _create_event(ctx)

    # Make a pipe for the subprocess stdout/stderr.
    (stdout_r, stdout_w) = ctx.Pipe()
    (stderr_r, stderr_w) = ctx.Pipe()

    # This Pipe is how we ensure that the executed process is properly terminated (not orphaned) if
    # the parent process is hard killed (-9). If the parent (this process) is killed for any reason,
    # this Pipe will be closed, which can be detected by the middleman. When the middleman sees the
    # closed Pipe, it will issue a SIGTERM to the subprocess executing the command. The assumption
    # here is that users will be inclined to hard kill this process, not the middleman.
    (r, w) = ctx.Pipe()

    middleman = ctx.Process(
        target=_exec_middleman,
        args=(
            command,
            env,
            exit_event,
            (stdout_r, stdout_w),
            (stderr_r, stderr_w),
            (r, w),
        ),
    )
    middleman.start()

    # Close unused file descriptors to enforce PIPE behavior.
    r.close()
    stdout_w.close()
    stderr_w.close()

    # Redirect command stdout & stderr to provided streams or sys.stdout/sys.stderr.
    # This is useful for Jupyter Notebook that uses custom sys.stdout/sys.stderr or
    # for redirecting to a file on disk.
    if stdout is None:
        stdout = sys.stdout
    if stderr is None:
        stderr = sys.stderr

    stdout_fwd = in_thread(
        target=forward_stream,
        args=(stdout_r, stdout, "stdout", index, prefix_output_with_timestamp),
    )
    stderr_fwd = in_thread(
        target=forward_stream,
        args=(stderr_r, stderr, "stderr", index, prefix_output_with_timestamp),
    )

    # TODO: Currently this requires explicitly declaration of the events and signal handler to set
    #  the event (gloo_run.py:_launch_jobs()). Need to figure out a generalized way to hide this behind
    #  interfaces.
    stop = threading.Event()
    events = events or []
    for event in events:
        on_event(event, exit_event.set, stop=stop, silent=True)

    try:
        middleman.join()
    except:
        # interrupted, send middleman TERM signal which will terminate children
        exit_event.set()
        while True:
            try:
                middleman.join()
                break
            except:
                # interrupted, wait for middleman to finish
                pass
    finally:
        stop.set()

    stdout_fwd.join()
    stderr_fwd.join()

    return middleman.exitcode


# Traceback (UnicodeDecodeError)
Tue Oct 13 10:54:25 2020[0]<stderr>:[10/13/2020 10:54:25 - INFO - __main__ -   start running validation...
Tue Oct 13 10:54:25 2020[0]<stderr>:  0%|          | 0/1327 [00:00<?, ?it/s]
Tue Oct 13 10:55:56 2020[0]<stderr>:
Exception in thread Thread-17:derr>: 65%|██████▌   | 866/1327 [01:17<00:47,  9.80it/s]
Traceback (most recent call last):
File "/data/anaconda3/envs/vcr/lib/python3.6/threading.py", line 916, in _bootstrap_inner
self.run()
File "/data/anaconda3/envs/vcr/lib/python3.6/threading.py", line 864, in run
self._target(*self._args, **self._kwargs)
File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/runner/common/util/safe_shell_exec.py", line 104, in forward_stream
text = text.decode('utf-8')
UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe2 in position 999: unexpected end of data

Tue Oct 13 10:58:56 2020[3]<stderr>:[2020-10-13 10:58:56.365049: E /dockerdata/app/tmp/pip-install-998q9kn4/horovod/horovod/common/operations.cc:525] Horovod background loop uncaught exception: [/dockerdata/app/tmp/pip-install-998q9kn4/horovod/third_party/compatible_gloo/gloo/transport/tcp/unbound_buffer.cc:84] Timed out waiting 30000ms for recv operation to complete
Tue Oct 13 10:58:56 2020[2]<stderr>:[2020-10-13 10:58:56.366492: E /dockerdata/app/tmp/pip-install-998q9kn4/horovod/horovod/common/operations.cc:525] Horovod background loop uncaught exception: [/dockerdata/app/tmp/pip-install-998q9kn4/horovod/third_party/compatible_gloo/gloo/transport/tcp/unbound_buffer.cc:84] Timed out waiting 30000ms for recv operation to complete
Tue Oct 13 10:58:56 2020[1]<stderr>:[2020-10-13 10:58:56.366674: E /dockerdata/app/tmp/pip-install-998q9kn4/horovod/horovod/common/operations.cc:525] Horovod background loop uncaught exception: [/dockerdata/app/tmp/pip-install-998q9kn4/horovod/third_party/compatible_gloo/gloo/transport/tcp/unbound_buffer.cc:136] Timed out waiting 30000ms for send operation to complete
Tue Oct 13 10:58:56 2020[3]<stderr>:Traceback (most recent call last):
Tue Oct 13 10:58:56 2020[1]<stderr>:Traceback (most recent call last):
Tue Oct 13 10:58:56 2020[3]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 622, in synchronize
Tue Oct 13 10:58:56 2020[2]<stderr>:Traceback (most recent call last):
Tue Oct 13 10:58:56 2020[3]<stderr>:    mpi_lib.horovod_torch_wait_and_clear(handle)
Tue Oct 13 10:58:56 2020[2]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 622, in synchronize
Tue Oct 13 10:58:56 2020[3]<stderr>:RuntimeError: Horovod has been shut down. This was caused by an exception on one of the ranks or an attempt to allreduce, allgather or broadcast a tensor after one of the ranks finished execution. If the shutdown was caused by an exception, you should see the exception in the log before the first shutdown message.
Tue Oct 13 10:58:56 2020[1]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 622, in synchronize
Tue Oct 13 10:58:56 2020[3]<stderr>:
Tue Oct 13 10:58:56 2020[1]<stderr>:    mpi_lib.horovod_torch_wait_and_clear(handle)
Tue Oct 13 10:58:56 2020[2]<stderr>:    mpi_lib.horovod_torch_wait_and_clear(handle)
Tue Oct 13 10:58:57 2020[3]<stderr>:During handling of the above exception, another exception occurred:
Tue Oct 13 10:58:57 2020[2]<stderr>:RuntimeError: Horovod has been shut down. This was caused by an exception on one of the ranks or an attempt to allreduce, allgather or broadcast a tensor after one of the ranks finished execution. If the shutdown was caused by an exception, you should see the exception in the log before the first shutdown message.
Tue Oct 13 10:58:57 2020[1]<stderr>:RuntimeError: Horovod has been shut down. This was caused by an exception on one of the ranks or an attempt to allreduce, allgather or broadcast a tensor after one of the ranks finished execution. If the shutdown was caused by an exception, you should see the exception in the log before the first shutdown message.
Tue Oct 13 10:58:57 2020[3]<stderr>:
Tue Oct 13 10:58:57 2020[1]<stderr>:
Tue Oct 13 10:58:57 2020[2]<stderr>:
Tue Oct 13 10:58:57 2020[3]<stderr>:Traceback (most recent call last):
Tue Oct 13 10:58:57 2020[2]<stderr>:During handling of the above exception, another exception occurred:
Tue Oct 13 10:58:57 2020[1]<stderr>:During handling of the above exception, another exception occurred:
Tue Oct 13 10:58:57 2020[3]<stderr>:  File "train_vcr.py", line 332, in <module>
Tue Oct 13 10:58:57 2020[1]<stderr>:
Tue Oct 13 10:58:57 2020[3]<stderr>:    main(parse_cmd_args(TrainingOpts))
Tue Oct 13 10:58:57 2020[1]<stderr>:Traceback (most recent call last):
Tue Oct 13 10:58:57 2020[3]<stderr>:  File "train_vcr.py", line 260, in main
Tue Oct 13 10:58:57 2020[1]<stderr>:  File "train_vcr.py", line 332, in <module>
Tue Oct 13 10:58:57 2020[3]<stderr>:    val_log = validate(model, val_dataloader)
Tue Oct 13 10:58:57 2020[1]<stderr>:    main(parse_cmd_args(TrainingOpts))
Tue Oct 13 10:58:57 2020[3]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/torch/autograd/grad_mode.py", line 49, in decorate_no_grad
Tue Oct 13 10:58:57 2020[1]<stderr>:  File "train_vcr.py", line 260, in main
Tue Oct 13 10:58:57 2020[3]<stderr>:    return func(*args, **kwargs)
Tue Oct 13 10:58:57 2020[1]<stderr>:    val_log = validate(model, val_dataloader)
Tue Oct 13 10:58:58 2020[3]<stderr>:  File "train_vcr.py", line 310, in validate
Tue Oct 13 10:58:58 2020[1]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/torch/autograd/grad_mode.py", line 49, in decorate_no_grad
Tue Oct 13 10:58:57 2020[2]<stderr>:
Tue Oct 13 10:58:58 2020[3]<stderr>:    qa_loss = sum(all_gather_list(qa_loss))
Tue Oct 13 10:58:58 2020[1]<stderr>:    return func(*args, **kwargs)
Tue Oct 13 10:58:58 2020[2]<stderr>:Traceback (most recent call last):
Tue Oct 13 10:58:58 2020[1]<stderr>:  File "train_vcr.py", line 310, in validate
Tue Oct 13 10:58:58 2020[2]<stderr>:  File "train_vcr.py", line 332, in <module>
Tue Oct 13 10:58:58 2020[3]<stderr>:  File "/data/cdp_algo_ceph_ssd/users/haoyanhuo/vcr/utils/distributed.py", line 233, in all_gather_list
Tue Oct 13 10:58:58 2020[2]<stderr>:    main(parse_cmd_args(TrainingOpts))
Tue Oct 13 10:58:58 2020[3]<stderr>:    max_size = hvd.allgather(torch.tensor([enc_size]).cuda()).max().item()
Tue Oct 13 10:58:58 2020[1]<stderr>:    qa_loss = sum(all_gather_list(qa_loss))
Tue Oct 13 10:58:58 2020[2]<stderr>:  File "train_vcr.py", line 260, in main
Tue Oct 13 10:58:58 2020[3]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 365, in allgather
Tue Oct 13 10:58:58 2020[2]<stderr>:    val_log = validate(model, val_dataloader)
Tue Oct 13 10:58:58 2020[3]<stderr>:    return HorovodAllgather.apply(tensor, name)
Tue Oct 13 10:58:58 2020[1]<stderr>:  File "/data/cdp_algo_ceph_ssd/users/haoyanhuo/vcr/utils/distributed.py", line 233, in all_gather_list
Tue Oct 13 10:58:58 2020[2]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/torch/autograd/grad_mode.py", line 49, in decorate_no_grad
Tue Oct 13 10:58:58 2020[1]<stderr>:    max_size = hvd.allgather(torch.tensor([enc_size]).cuda()).max().item()
Tue Oct 13 10:58:58 2020[3]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 328, in forward
Tue Oct 13 10:58:59 2020[2]<stderr>:    return func(*args, **kwargs)
Tue Oct 13 10:58:59 2020[3]<stderr>:    return synchronize(handle)
Tue Oct 13 10:58:59 2020[2]<stderr>:  File "train_vcr.py", line 310, in validate
Tue Oct 13 10:58:59 2020[3]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 626, in synchronize
Tue Oct 13 10:58:59 2020[2]<stderr>:    qa_loss = sum(all_gather_list(qa_loss))
Tue Oct 13 10:58:59 2020[3]<stderr>:    raise HorovodInternalError(e)
Tue Oct 13 10:58:59 2020[2]<stderr>:  File "/data/cdp_algo_ceph_ssd/users/haoyanhuo/vcr/utils/distributed.py", line 233, in all_gather_list
Tue Oct 13 10:58:59 2020[3]<stderr>:horovod.common.exceptions.HorovodInternalError: Horovod has been shut down. This was causedby an exception on one of the ranks or an attempt to allreduce, allgather or broadcast a tensor after one of the ranks finishedexecution. If the shutdown was caused by an exception, you should see the exception in the log before the first shutdown message.
Tue Oct 13 10:58:59 2020[1]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 365, in allgather
Tue Oct 13 10:58:59 2020[2]<stderr>:    max_size = hvd.allgather(torch.tensor([enc_size]).cuda()).max().item()
Tue Oct 13 10:58:59 2020[2]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 365, in allgather
Tue Oct 13 10:58:59 2020[1]<stderr>:    return HorovodAllgather.apply(tensor, name)
Tue Oct 13 10:58:59 2020[1]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 328, in forward
Tue Oct 13 10:58:59 2020[2]<stderr>:    return HorovodAllgather.apply(tensor, name)
Tue Oct 13 10:58:59 2020[2]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 328, in forward
Tue Oct 13 10:58:59 2020[1]<stderr>:    return synchronize(handle)
Tue Oct 13 10:58:59 2020[1]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 626, in synchronize
Tue Oct 13 10:58:59 2020[2]<stderr>:    return synchronize(handle)
Tue Oct 13 10:58:59 2020[1]<stderr>:    raise HorovodInternalError(e)
Tue Oct 13 10:59:00 2020[2]<stderr>:  File "/data/anaconda3/envs/vcr/lib/python3.6/site-packages/horovod/torch/mpi_ops.py", line 626, in synchronize
Tue Oct 13 10:59:00 2020[1]<stderr>:horovod.common.exceptions.HorovodInternalError: Horovod has been shut down. This was causedby an exception on one of the ranks or an attempt to allreduce, allgather or broadcast a tensor after one of the ranks finishedexecution. If the shutdown was caused by an exception, you should see the exception in the log before the first shutdown message.
Tue Oct 13 10:59:00 2020[2]<stderr>:    raise HorovodInternalError(e)
Tue Oct 13 10:59:00 2020[2]<stderr>:horovod.common.exceptions.HorovodInternalError: Horovod has been shut down. This was causedby an exception on one of the ranks or an attempt to allreduce, allgather or broadcast a tensor after one of the ranks finishedexecution. If the shutdown was caused by an exception, you should see the exception in the log before the first shutdown message.

# Fixed code
def execute(
    command,
    env=None,
    stdout=None,
    stderr=None,
    index=None,
    events=None,
    prefix_output_with_timestamp=False,
):
    """
    Execute the given command and forward stdout and stderr of the command to the given
    stdout and stderr text streams, or sys.stdout and sys.stderr, respectively, if None given.
    Prefixes each line with index and timestamp if index is not None. The timestamp
    can be disabled with prefix_output_with_timestamp set False.
    The command will be terminated when any of the given events are set.

    :param command: command to execute
    :param env: environment variables to execute command with
    :param stdout: stdout text stream, sys.stdout if None
    :param stderr: stderr text stream, sys.stderr if None
    :param index: index used to prepend text streams
    :param events: events to terminate the command
    :param prefix_output_with_timestamp: prepend text streams with timestamp if True
    :return: command's exit code
    """
    ctx = multiprocessing.get_context("spawn")

    # When this event is set, signal to middleman to terminate its children and exit.
    exit_event = _create_event(ctx)

    # Make a pipe for the subprocess stdout/stderr.
    (stdout_r, stdout_w) = ctx.Pipe()
    (stderr_r, stderr_w) = ctx.Pipe()

    # This Pipe is how we ensure that the executed process is properly terminated (not orphaned) if
    # the parent process is hard killed (-9). If the parent (this process) is killed for any reason,
    # this Pipe will be closed, which can be detected by the middleman. When the middleman sees the
    # closed Pipe, it will issue a SIGTERM to the subprocess executing the command. The assumption
    # here is that users will be inclined to hard kill this process, not the middleman.
    (r, w) = ctx.Pipe()

    middleman = ctx.Process(
        target=_exec_middleman,
        args=(
            command,
            env,
            exit_event,
            (stdout_r, stdout_w),
            (stderr_r, stderr_w),
            (r, w),
        ),
    )
    middleman.start()

    # Close unused file descriptors to enforce PIPE behavior.
    r.close()
    stdout_w.close()
    stderr_w.close()

    # Redirect command stdout & stderr to provided streams or sys.stdout/sys.stderr.
    # This is useful for Jupyter Notebook that uses custom sys.stdout/sys.stderr or
    # for redirecting to a file on disk.
    if stdout is None:
        stdout = sys.stdout
    if stderr is None:
        stderr = sys.stderr

    stdout_fwd = in_thread(
        target=prefix_connection,
        args=(stdout_r, stdout, "stdout", index, prefix_output_with_timestamp),
    )
    stderr_fwd = in_thread(
        target=prefix_connection,
        args=(stderr_r, stderr, "stderr", index, prefix_output_with_timestamp),
    )

    # TODO: Currently this requires explicitly declaration of the events and signal handler to set
    #  the event (gloo_run.py:_launch_jobs()). Need to figure out a generalized way to hide this behind
    #  interfaces.
    stop = threading.Event()
    events = events or []
    for event in events:
        on_event(event, exit_event.set, stop=stop, silent=True)

    try:
        middleman.join()
    except:
        # interrupted, send middleman TERM signal which will terminate children
        exit_event.set()
        while True:
            try:
                middleman.join()
                break
            except:
                # interrupted, wait for middleman to finish
                pass
    finally:
        stop.set()

    stdout_fwd.join()
    stderr_fwd.join()

    return middleman.exitcode