# Buggy code
def get_gpu_ids():
    """Get the IDs of the GPUs that are available to the worker.

    If the CUDA_VISIBLE_DEVICES environment variable was set when the worker
    started up, then the IDs returned by this method will be a subset of the
    IDs in CUDA_VISIBLE_DEVICES. If not, the IDs will fall in the range
    [0, NUM_GPUS - 1], where NUM_GPUS is the number of GPUs that the node has.

    Returns:
        A list of GPU IDs.
    """

    # TODO(ilr) Handle inserting resources in local mode
    all_resource_ids = global_worker.core_worker.resource_ids()
    assigned_ids = [resource_id for resource_id, _ in all_resource_ids.get("GPU", [])]
    # If the user had already set CUDA_VISIBLE_DEVICES, then respect that (in
    # the sense that only GPU IDs that appear in CUDA_VISIBLE_DEVICES should be
    # returned).
    if global_worker.original_gpu_ids is not None:
        assigned_ids = [
            global_worker.original_gpu_ids[gpu_id] for gpu_id in assigned_ids
        ]

    return assigned_ids


# Traceback (AssertionError)
$ CUDA_VISIBLE_DEVICES=1 python ./scripts/ray_cuda_issue.py 1  # 0.8.5
...
2020-06-08 18:11:57.675648: E tensorflow/stream_executor/cuda/cuda_driver.cc:351] failed call to cuInit: CUDA_ERROR_NO_DEVICE: no CUDA-capable device is detected
...
[]

...
2020-06-08 18:11:57,694 ERROR trial_runner.py:519 -- Trial DebugRunner_00000: Error processing event.
Traceback (most recent call last):
File "/users/krinen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/trial_runner.py", line 467, in _process_trial
result = self.trial_executor.fetch_result(trial)
File "/users/krinen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/ray_trial_executor.py", line 431, in fetch_result
result = ray.get(trial_future[0], DEFAULT_GET_TIMEOUT)
File "/users/krinen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/worker.py", line 1515, in get
raise value.as_instanceof_cause()
ray.exceptions.RayTaskError(AssertionError): ray::DebugRunner.train() (pid=11658, ip=163.1.88.121)
File "python/ray/_raylet.pyx", line 463, in ray._raylet.execute_task
File "python/ray/_raylet.pyx", line 417, in ray._raylet.execute_task.function_executor
File "/users/krinen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/trainable.py", line 261, in train
result = self._train()
File "./scripts/ray_cuda_issue.py", line 21, in _train
AssertionError: ('1', '')
== Status ==
Memory usage on this node: 307.8/503.8 GiB
Using FIFO scheduling algorithm.
Resources requested: 0/40 CPUs, 0/1 GPUs, 0.0/130.32 GiB heap, 0.0/41.26 GiB objects
Result logdir: /users/krinen/ray_results/DebugRunner
Number of trials: 1 (1 ERROR)
+-------------------+----------+-------+
| Trial name        | status   | loc   |
|-------------------+----------+-------|
| DebugRunner_00000 | ERROR    |       |
+-------------------+----------+-------+
Number of errored trials: 1
+-------------------+--------------+-------------------------------------------------------------------------------------------+
| Trial name        |   # failures | error file                                                                                |
|-------------------+--------------+-------------------------------------------------------------------------------------------|
| DebugRunner_00000 |            1 | /users/krinen/ray_results/DebugRunner/DebugRunner_0_2020-06-08_18-11-54uby7h2dg/error.txt |
+-------------------+--------------+-------------------------------------------------------------------------------------------+

Traceback (most recent call last):
File "./scripts/ray_cuda_issue.py", line 30, in <module>
tune.run(DebugRunner, resources_per_trial={'gpu': 1})
File "/users/krinen/conda/envs/softlearning-tf2/lib/python3.7/site-packages/ray/tune/tune.py", line 347, in run
raise TuneError("Trials did not complete", incomplete_trials)
ray.tune.error.TuneError: ('Trials did not complete', [DebugRunner_00000])

# Fixed code
def get_gpu_ids():
    """Get the IDs of the GPUs that are available to the worker.

    If the CUDA_VISIBLE_DEVICES environment variable was set when the worker
    started up, then the IDs returned by this method will be a subset of the
    IDs in CUDA_VISIBLE_DEVICES. If not, the IDs will fall in the range
    [0, NUM_GPUS - 1], where NUM_GPUS is the number of GPUs that the node has.

    Returns:
        A list of GPU IDs.
    """

    # TODO(ilr) Handle inserting resources in local mode
    all_resource_ids = global_worker.core_worker.resource_ids()
    assigned_ids = [resource_id for resource_id, _ in all_resource_ids.get("GPU", [])]
    # If the user had already set CUDA_VISIBLE_DEVICES, then respect that (in
    # the sense that only GPU IDs that appear in CUDA_VISIBLE_DEVICES should be
    # returned).
    if global_worker.original_gpu_ids is not None:
        assigned_ids = [
            global_worker.original_gpu_ids[gpu_id] for gpu_id in assigned_ids
        ]
        # Give all GPUs in local_mode.
        if global_worker.mode == LOCAL_MODE:
            max_gpus = global_worker.node.get_resource_spec().num_gpus
            return global_worker.original_gpu_ids[:max_gpus]

    return assigned_ids