# Buggy code
def __call__(self, samples: SampleBatchType) -> (SampleBatchType, List[dict]):
    _check_sample_batch_type(samples)

    # Handle everything as if multiagent
    if isinstance(samples, SampleBatch):
        samples = MultiAgentBatch({DEFAULT_POLICY_ID: samples}, samples.count)

    metrics = _get_shared_metrics()
    load_timer = metrics.timers[LOAD_BATCH_TIMER]
    learn_timer = metrics.timers[LEARN_ON_BATCH_TIMER]
    with load_timer:
        # (1) Load data into GPUs.
        num_loaded_tuples = {}
        for policy_id, batch in samples.policy_batches.items():
            if policy_id not in self.policies:
                continue

            policy = self.workers.local_worker().get_policy(policy_id)
            policy._debug_vars()
            tuples = policy._get_loss_inputs_dict(batch, shuffle=self.shuffle_sequences)
            data_keys = list(policy._loss_input_dict_no_rnn.values())
            if policy._state_inputs:
                state_keys = policy._state_inputs + [policy._seq_lens]
            else:
                state_keys = []
            num_loaded_tuples[policy_id] = self.optimizers[policy_id].load_data(
                self.sess,
                [tuples[k] for k in data_keys],
                [tuples[k] for k in state_keys],
            )

    with learn_timer:
        # (2) Execute minibatch SGD on loaded data.
        fetches = {}
        for policy_id, tuples_per_device in num_loaded_tuples.items():
            optimizer = self.optimizers[policy_id]
            num_batches = max(
                1, int(tuples_per_device) // int(self.per_device_batch_size)
            )
            logger.debug("== sgd epochs for {} ==".format(policy_id))
            for i in range(self.num_sgd_iter):
                iter_extra_fetches = defaultdict(list)
                permutation = np.random.permutation(num_batches)
                for batch_index in range(num_batches):
                    batch_fetches = optimizer.optimize(
                        self.sess, permutation[batch_index] * self.per_device_batch_size
                    )
                    for k, v in batch_fetches[LEARNER_STATS_KEY].items():
                        iter_extra_fetches[k].append(v)
                if logger.getEffectiveLevel() <= logging.DEBUG:
                    avg = averaged(iter_extra_fetches)
                    logger.debug("{} {}".format(i, avg))
            fetches[policy_id] = averaged(iter_extra_fetches, axis=0)

    load_timer.push_units_processed(samples.count)
    learn_timer.push_units_processed(samples.count)

    metrics.counters[STEPS_TRAINED_COUNTER] += samples.count
    metrics.info[LEARNER_INFO] = fetches
    if self.workers.remote_workers():
        with metrics.timers[WORKER_UPDATE_TIMER]:
            weights = ray.put(self.workers.local_worker().get_weights(self.policies))
            for e in self.workers.remote_workers():
                e.set_weights.remote(weights, _get_global_vars())
    # Also update global vars of the local worker.
    self.workers.local_worker().set_global_vars(_get_global_vars())
    return samples, fetches


# Traceback (ValueError)
Traceback (most recent call last):
File "/home/justinkterry/.local/lib/python3.6/site-packages/ray/tune/trial_runner.py", line 519, in _process_trial
result = self.trial_executor.fetch_result(trial)
File "/home/justinkterry/.local/lib/python3.6/site-packages/ray/tune/ray_trial_executor.py", line 497, in fetch_result
result = ray.get(trial_future[0], timeout=DEFAULT_GET_TIMEOUT)
File "/home/justinkterry/.local/lib/python3.6/site-packages/ray/worker.py", line 1379, in get
raise value.as_instanceof_cause()
ray.exceptions.RayTaskError(ValueError): ray::PPO.train() (pid=50904, ip=129.2.189.150)
File "python/ray/_raylet.pyx", line 463, in ray._raylet.execute_task
File "python/ray/_raylet.pyx", line 415, in ray._raylet.execute_task.function_executor
File "/home/justinkterry/.local/lib/python3.6/site-packages/ray/rllib/agents/trainer.py", line 508, in train
raise e
File "/home/justinkterry/.local/lib/python3.6/site-packages/ray/rllib/agents/trainer.py", line 494, in train
result = Trainable.train(self)
File "/home/justinkterry/.local/lib/python3.6/site-packages/ray/tune/trainable.py", line 183, in train
result = self.step()
File "/home/justinkterry/.local/lib/python3.6/site-packages/ray/rllib/agents/trainer_template.py", line 147, in step
res = next(self.train_exec_impl)
File "/home/justinkterry/.local/lib/python3.6/site-packages/ray/util/iter.py", line 756, in __next__
return next(self.built_iterator)
File "/home/justinkterry/.local/lib/python3.6/site-packages/ray/util/iter.py", line 783, in apply_foreach
for item in it:
File "/home/justinkterry/.local/lib/python3.6/site-packages/ray/util/iter.py", line 783, in apply_foreach
for item in it:
File "/home/justinkterry/.local/lib/python3.6/site-packages/ray/util/iter.py", line 843, in apply_filter
for item in it:
File "/home/justinkterry/.local/lib/python3.6/site-packages/ray/util/iter.py", line 843, in apply_filter
for item in it:
File "/home/justinkterry/.local/lib/python3.6/site-packages/ray/util/iter.py", line 783, in apply_foreach
for item in it:
File "/home/justinkterry/.local/lib/python3.6/site-packages/ray/util/iter.py", line 783, in apply_foreach
for item in it:
File "/home/justinkterry/.local/lib/python3.6/site-packages/ray/util/iter.py", line 791, in apply_foreach
result = fn(item)
File "/home/justinkterry/.local/lib/python3.6/site-packages/ray/rllib/execution/train_ops.py", line 191, in __call__
[tuples[k] for k in state_keys]))
File "/home/justinkterry/.local/lib/python3.6/site-packages/ray/rllib/execution/multi_gpu_impl.py", line 218, in load_data
sess.run([t.init_op for t in self._towers], feed_dict=feed_dict)
File "/home/justinkterry/.local/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 950, in run
run_metadata_ptr)
File "/home/justinkterry/.local/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1142, in _run
np_val = np.asarray(subfeed_val, dtype=subfeed_dtype)
File "/home/justinkterry/.local/lib/python3.6/site-packages/numpy/core/_asarray.py", line 83, in asarray
return array(a, dtype, copy=False, order=order)
ValueError: could not convert string to float: 'BCJNGGhAv0cAAAAAAAA71wUAAGGABZW0RwABAPMZjBJudW1weS5jb3JlLm51bWVyaWOUjAtfZnJvbWJ1ZmZlcpSTlCiWQC8ATwAAgD8EAP///////////////////////7u/jpNnPxXgcz+cPX/IEjIx4N1dBAB/QGxlP+DKe1QALgBMAAQEAH9+LGI/Rnx9VAAqBEwABQgAL6lqUAAqOeHdXQQAf9QcRz/SPnVUACb/CNgYWj/0OTo/FXIRP267Dz+4oiY/vgVuUAAmsXaAEz9aXzM/OSdcmAEA+AAv7+5QACcAQAAMBAA/1LlwoAAmCDwBBAwAP6kteVAAJgycAD/OZGNMACYA2BUIUAB/I8BeP/YheFAAKgTkAH9Tw14/m3FyTAAStRDyfj+/aGY/62VjBADxCKqHYz9zSng/57RaP288TT8VSlw/LiljBAB/wjJmP1DuflwABuKGTlQ/uqsDP6mYtT6KiwQA8QTfgdg+CT8yP8McMT+IO90+zzy1BAD5BK8gtT5mRAE/nj9UP4SYbj9x7GIEAAAUAPEAHKt+P6WfWT9qWUI/o4g/BADxBL/LTT+shHQ/5Sp3P0/iTj+f2T8EAPEI1qxCP8yPVj/9tXA/U5AZP6rsuz4U7rQEALGZsLc+0TMZPxzxcbwANTqOY7QCMahPehQAAJQBAAwABEgBACQAABQA8QB4+n8/x4loP5uwSD+bKkAEAHFCsUg/P0hlHAAAJAAALAAEOAAARAAAFAAPBAD///////////////////////+/r8GFaD/j7XI/NnvwEwcP5BIZBPAUdaxzYT/wdnZ0Ew9EABkILAAEUAAACABvfApeP2BLHBcnAFgAAEgACAQAfx0VYD+1g3yoACYEEBQICAAvCkYgFy//ANBTVj+14jU/TAQSP3O0IqAAJvEEOEYRP2+7Dz9+RRc/mbY3PwOVWxQVAAwBDwQAISYXUwgYBEgBP5O2XqAAJgg4AQQMAD+CTmZQACYElAAICAAvK3soFxMPEAEBCEgAAAwAb3BNZD+w+iwDJwTkAAAIAG9pgGE/OhXkGisAlAAABABvC/FpPywD3BoT4rtVaT/fSSk/3UsXP/xSBADwBd8MHj9Ep1E/rZQ6P6oOAj87bvI+BAC//F/yPq7jET+0uF+UAQbi95VoP4uhGj/sPO8+biAEAPEESaUCP0N5TT+sL18/1WcgP27EDwQA8QhQrBA/RiMsP6c8az9KYTs/AQsXP8iKFgQAAAwAABQA8QCJAnk/tMh/P3xgYz+Ctl0EAGJrCHo/ou2IFwB8FwBAAQAEAACUFwC4AfEACul4P1VRNT9FGPY+oBXwBABxhN3xPkqzMRwAACQAACwAADQAAAQAAEQAABQADwQA////////////////////////b2+Z13U/l4FAFCcMsBIAzBJ/BZxfP+wyboQvMgBMFAQEAACELy86SKwAKwRMAAQIAC9JFzAvKwRIAAgIACLtuuATDBwBDxAAEQBAAQQEAL94J1o/r5E6P0klP+wUBg9UAA0EtBcAfC+v13ETP58HMz/hyHAXJwi0FwT0AADIFA+IAA0PIAAFBEQACAgAP2gIafAAJgQ8AQgIAD9MfXNQACYImAAADABvm3JiP3/3hAInAEQABAQAAIAvLygUHDAnAFQBBOgAAJgAL/gQTAArAFAA/wxEbU0/R6ouP1HyMz/ACjU/6AM1PyhPQD+nuHJEAQYAnC8ApC8EsC8ACADxBGRLYz+8OHg/VoNBP2Vu7D7JEssEAL/5A8s+WoIIPyo6X1AABqKeP1Q/kqMDP05KtC8AvC/xBJJT2D6jKzI/SWZ/P/tQbz+9slcEAHGEalw/cTV1pBcArBcAtBcMwBe1TMB3P8rdWT9lrUK4L3H7G04/yql0hAAAfBcESAEEzAIApBcPwBchMZSMBXBH8RaUjAVkdHlwZZSTlIwCZjSUiYiHlFKUKEsDjAE8lE5OTkr/////BQDwBEsAdJRiTdARhZSMAUOUdJRSlC4AAAAA'
Result for PPO_pistonball_v3_f4d39_00000:
{}

# Fixed code
def __call__(self, samples: SampleBatchType) -> (SampleBatchType, List[dict]):
    _check_sample_batch_type(samples)

    # Handle everything as if multiagent
    if isinstance(samples, SampleBatch):
        samples = MultiAgentBatch({DEFAULT_POLICY_ID: samples}, samples.count)

    metrics = _get_shared_metrics()
    load_timer = metrics.timers[LOAD_BATCH_TIMER]
    learn_timer = metrics.timers[LEARN_ON_BATCH_TIMER]
    with load_timer:
        # (1) Load data into GPUs.
        num_loaded_tuples = {}
        for policy_id, batch in samples.policy_batches.items():
            # Not a policy-to-train.
            if policy_id not in self.policies:
                continue

            # Decompress SampleBatch, in case some columns are compressed.
            batch.decompress_if_needed()

            policy = self.workers.local_worker().get_policy(policy_id)
            policy._debug_vars()
            tuples = policy._get_loss_inputs_dict(batch, shuffle=self.shuffle_sequences)
            data_keys = list(policy._loss_input_dict_no_rnn.values())
            if policy._state_inputs:
                state_keys = policy._state_inputs + [policy._seq_lens]
            else:
                state_keys = []
            num_loaded_tuples[policy_id] = self.optimizers[policy_id].load_data(
                self.sess,
                [tuples[k] for k in data_keys],
                [tuples[k] for k in state_keys],
            )

    with learn_timer:
        # (2) Execute minibatch SGD on loaded data.
        fetches = {}
        for policy_id, tuples_per_device in num_loaded_tuples.items():
            optimizer = self.optimizers[policy_id]
            num_batches = max(
                1, int(tuples_per_device) // int(self.per_device_batch_size)
            )
            logger.debug("== sgd epochs for {} ==".format(policy_id))
            for i in range(self.num_sgd_iter):
                iter_extra_fetches = defaultdict(list)
                permutation = np.random.permutation(num_batches)
                for batch_index in range(num_batches):
                    batch_fetches = optimizer.optimize(
                        self.sess, permutation[batch_index] * self.per_device_batch_size
                    )
                    for k, v in batch_fetches[LEARNER_STATS_KEY].items():
                        iter_extra_fetches[k].append(v)
                if logger.getEffectiveLevel() <= logging.DEBUG:
                    avg = averaged(iter_extra_fetches)
                    logger.debug("{} {}".format(i, avg))
            fetches[policy_id] = averaged(iter_extra_fetches, axis=0)

    load_timer.push_units_processed(samples.count)
    learn_timer.push_units_processed(samples.count)

    metrics.counters[STEPS_TRAINED_COUNTER] += samples.count
    metrics.info[LEARNER_INFO] = fetches
    if self.workers.remote_workers():
        with metrics.timers[WORKER_UPDATE_TIMER]:
            weights = ray.put(self.workers.local_worker().get_weights(self.policies))
            for e in self.workers.remote_workers():
                e.set_weights.remote(weights, _get_global_vars())
    # Also update global vars of the local worker.
    self.workers.local_worker().set_global_vars(_get_global_vars())
    return samples, fetches