# Buggy code
def rsync_to_node(node_id, is_head_node):
    updater = NodeUpdaterThread(
        node_id=node_id,
        provider_config=config["provider"],
        provider=provider,
        auth_config=config["auth"],
        cluster_name=config["cluster_name"],
        file_mounts=config["file_mounts"],
        initialization_commands=[],
        setup_commands=[],
        ray_start_commands=[],
        runtime_hash="",
        use_internal_ip=use_internal_ip,
        process_runner=_runner,
        file_mounts_contents_hash="",
        is_head_node=is_head_node,
        rsync_options={
            "rsync_exclude": config.get("rsync_exclude"),
            "rsync_filter": config.get("rsync_filter"),
        },
        docker_config=config.get("docker"),
    )
    if down:
        rsync = updater.rsync_down
    else:
        rsync = updater.rsync_up

    if source and target:
        # print rsync progress for single file rsync
        cmd_output_util.set_output_redirected(False)
        set_rsync_silent(False)
        rsync(source, target, is_file_mount)
    else:
        updater.sync_file_mounts(rsync)


# Traceback (ray.tune.error.TuneError)
2020-11-19 19:42:27,243 VINFO updater.py:460 -- `rsync`ed /root/ray_results/tune_mnist_pbt/DEFAULT_1d092_00006_6_layer_1_size=128,layer_2_size=64_2020-11-19_19-41-58/ (remote) to /root/ray_results/tune_mn
ist_pbt/DEFAULT_1d092_00006_6_layer_1_size=128,layer_2_size=64_2020-11-19_19-41-58/ (local)
2020-11-19 19:42:27,247 ERROR trial_runner.py:868 -- Trial DEFAULT_1d092_00006: Error handling checkpoint /root/ray_results/2020-11-19_19-42-24w11y5gdl/checkpoint_1/
Traceback (most recent call last):
File "/root/anaconda3/lib/python3.7/site-packages/ray/tune/trial_runner.py", line 864, in _process_trial_save
trial.on_checkpoint(trial.saving_to)
File "/root/anaconda3/lib/python3.7/site-packages/ray/tune/trial.py", line 498, in on_checkpoint
self, checkpoint.value))
ray.tune.error.TuneError: Trial DEFAULT_1d092_00006: Checkpoint path /root/ray_results/2020-11-19_19-42-24w11y5gdl/checkpoint_1/ not found after successful sync down.
2020-11-19 19:42:27,250 WARNING util.py:140 -- The `process_trial_save` operation took 1.05049467086792 seconds to complete, which may be a performance bottleneck.

# Fixed code
def rsync_to_node(node_id, is_head_node):
    updater = NodeUpdaterThread(
        node_id=node_id,
        provider_config=config["provider"],
        provider=provider,
        auth_config=config["auth"],
        cluster_name=config["cluster_name"],
        file_mounts=config["file_mounts"],
        initialization_commands=[],
        setup_commands=[],
        ray_start_commands=[],
        runtime_hash="",
        use_internal_ip=use_internal_ip,
        process_runner=_runner,
        file_mounts_contents_hash="",
        is_head_node=is_head_node,
        rsync_options={
            "rsync_exclude": config.get("rsync_exclude"),
            "rsync_filter": config.get("rsync_filter"),
        },
        docker_config=config.get("docker"),
    )
    if down:
        rsync = updater.rsync_down
    else:
        rsync = updater.rsync_up

    if source and target:
        # print rsync progress for single file rsync
        if cli_logger.verbosity > 0:
            cmd_output_util.set_output_redirected(False)
            set_rsync_silent(False)
        rsync(source, target, is_file_mount)
    else:
        updater.sync_file_mounts(rsync)