# Buggy code
def on_trial_complete(
    self, iteration: int, trials: List["Trial"], trial: "Trial", **info
):
    trial_syncer = self._get_trial_syncer(trial)
    if NODE_IP in trial.last_result:
        trainable_ip = trial.last_result[NODE_IP]
    else:
        trainable_ip = ray.get(trial.runner.get_current_ip.remote())
    trial_syncer.set_worker_ip(trainable_ip)
    trial_syncer.sync_down_if_needed()


# Traceback (OSError)
2020-11-21 02:02:05,077 ERROR syncer.py:190 -- Sync execution failed.
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/ray/tune/syncer.py", line 187, in sync_down
self._local_dir)
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/ray/tune/sync_client.py", line 197, in sync_down
return self._execute(self.sync_down_template, source, target)
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/ray/tune/sync_client.py", line 243, in _execute
final_cmd, shell=True, stderr=subprocess.PIPE, stdout=self.logfile)
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/subprocess.py", line 729, in __init__
restore_signals, start_new_session)
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/subprocess.py", line 1254, in _execute_child
errpipe_read, errpipe_write = os.pipe()
OSError: [Errno 24] Too many open files
Traceback (most recent call last):
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/bin/rllib", line 8, in <module>
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/ray/rllib/scripts.py", line 34, in cli
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/ray/rllib/train.py", line 215, in run
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/ray/tune/tune.py", line 496, in run_experiments
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/ray/tune/tune.py", line 415, in run
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/ray/tune/trial_runner.py", line 361, in step
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/ray/tune/callback.py", line 180, in on_trial_start
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/ray/tune/logger.py", line 428, in on_trial_start
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/ray/tune/logger.py", line 635, in log_trial_start
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/tensorboardX/writer.py", line 275, in __init__
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/tensorboardX/writer.py", line 327, in _get_file_writer
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/tensorboardX/writer.py", line 95, in __init__
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/tensorboardX/event_file_writer.py", line 105, in __init__
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/multiprocessing/context.py", line 102, in Queue
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/multiprocessing/queues.py", line 42, in __init__
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/multiprocessing/context.py", line 67, in Lock
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/multiprocessing/synchronize.py", line 162, in __init__
File "/home/ubuntu/anaconda3/envs/tensorflow2_p36/lib/python3.6/multiprocessing/synchronize.py", line 59, in __init__
OSError: [Errno 24] Too many open files

# Fixed code
def on_trial_complete(
    self, iteration: int, trials: List["Trial"], trial: "Trial", **info
):
    trial_syncer = self._get_trial_syncer(trial)
    if NODE_IP in trial.last_result:
        trainable_ip = trial.last_result[NODE_IP]
    else:
        trainable_ip = ray.get(trial.runner.get_current_ip.remote())
    trial_syncer.set_worker_ip(trainable_ip)
    trial_syncer.sync_down_if_needed()
    trial_syncer.close()