Repository: ray-project/ray

Issue:
Resnet example with GPU crashes
Running the resnet example locally on a g2.2xlarge EC2 instance with the following options:
python ray/examples/resnet/resnet_main.py     
--eval_dir=/tmp/resnet-model/eval     
--train_data_path=cifar-10-batches-bin/data_batch*    
 --eval_data_path=cifar-10-batches-bin/test_batch.bin     
--dataset=cifar10     
--num_gpus=1

produces the following output:

I tensorflow/stream_executor/dso_loader.cc:135] successfully opened CUDA library libcublas.so.8.0 locally
I tensorflow/stream_executor/dso_loader.cc:135] successfully opened CUDA library libcudnn.so.5 locally
I tensorflow/stream_executor/dso_loader.cc:135] successfully opened CUDA library libcufft.so.8.0 locally
I tensorflow/stream_executor/dso_loader.cc:135] successfully opened CUDA library libcuda.so.1 locally
I tensorflow/stream_executor/dso_loader.cc:135] successfully opened CUDA library libcurand.so.8.0 locally
Waiting for redis server at 127.0.0.1:57431 to respond...
Waiting for redis server at 127.0.0.1:43581 to respond...
Starting local scheduler with 8 CPUs, 1 GPUs
View the web UI at http://localhost:8890/notebooks/ray_ui80356.ipynb
The log files for tensorboard are stored at ip 172.31.2.122.
Starting training loop. Use Ctrl-C to exit.
Traceback (most recent call last):
  File "ray/examples/resnet/resnet_main.py", line 248, in <module>
    train()
  File "ray/examples/resnet/resnet_main.py", line 231, in train
    for actor in train_actors])
  File "/usr/local/lib/python2.7/dist-packages/ray/worker.py", line 1983, in get
    raise RayGetError(object_ids[i], value)
ray.worker.RayGetError: Could not get objectid ObjectID(819d1c29782d0330f0bae9b966c4fa4f7909c849). It was created by remote function compute_steps which failed with:

Remote function compute_steps failed with:

Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/ray/worker.py", line 732, in _process_task
    self.actors[task.actor_id().id()], *arguments)
  File "ray/examples/resnet/resnet_main.py", line 113, in compute_steps
    self.model.variables.sess.run(self.model.train_op)
  File "/home/ubuntu/.local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 767, in run
    run_metadata_ptr)
  File "/home/ubuntu/.local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 965, in _run
    feed_dict_string, options, run_metadata)
  File "/home/ubuntu/.local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1015, in _do_run
    target_list, options, run_metadata)
  File "/home/ubuntu/.local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1035, in _do_call
    raise type(e)(node_def, op, message)
OutOfRangeError: FIFOQueue '_1_fifo_queue' is closed and has insufficient elements (requested 128, current size 0)
	 [[Node: fifo_queue_DequeueMany = QueueDequeueManyV2[component_types=[DT_FLOAT, DT_INT32], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/cpu:0"](fifo_queue, fifo_queue_DequeueMany/n/_2345)]]
	 [[Node: fifo_queue_DequeueMany/_2349 = _HostRecv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/gpu:0", send_device="/job:localhost/replica:0/task:0/cpu:0", send_device_incarnation=1, tensor_name="edge_5921_fifo_queue_DequeueMany", tensor_type=DT_INT32, _device="/job:localhost/replica:0/task:0/gpu:0"]()]]

Caused by op u'fifo_queue_DequeueMany', defined at:
  File "/usr/local/lib/python2.7/dist-packages/ray/workers/default_worker.py", line 105, in <module>
    ray.worker.global_worker.main_loop()
  File "/usr/local/lib/python2.7/dist-packages/ray/worker.py", line 880, in main_loop
    self._wait_for_and_process_task(task)
  File "/usr/local/lib/python2.7/dist-packages/ray/worker.py", line 833, in _wait_for_and_process_task
    self._process_task(task)
  File "/usr/local/lib/python2.7/dist-packages/ray/worker.py", line 732, in _process_task
    self.actors[task.actor_id().id()], *arguments)
  File "ray/examples/resnet/resnet_main.py", line 96, in __init__
    False)
  File "/home/ubuntu/ray/examples/resnet/cifar_input.py", line 105, in build_input
    images, labels = example_queue.dequeue_many(batch_size)
  File "/home/ubuntu/.local/lib/python2.7/site-packages/tensorflow/python/ops/data_flow_ops.py", line 458, in dequeue_many
    self._queue_ref, n=n, component_types=self._dtypes, name=name)
  File "/home/ubuntu/.local/lib/python2.7/site-packages/tensorflow/python/ops/gen_data_flow_ops.py", line 1310, in _queue_dequeue_many_v2
    timeout_ms=timeout_ms, name=name)
  File "/home/ubuntu/.local/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 763, in apply_op
    op_def=op_def)
  File "/home/ubuntu/.local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2327, in create_op
    original_op=self._default_original_op, op_def=op_def)
  File "/home/ubuntu/.local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1226, in __init__
    self._traceback = _extract_stack()

OutOfRangeError (see above for traceback): FIFOQueue '_1_fifo_queue' is closed and has insufficient elements (requested 128, current size 0)
	 [[Node: fifo_queue_DequeueMany = QueueDequeueManyV2[component_types=[DT_FLOAT, DT_INT32], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/cpu:0"](fifo_queue, fifo_queue_DequeueMany/n/_2345)]]
	 [[Node: fifo_queue_DequeueMany/_2349 = _HostRecv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/gpu:0", send_device="/job:localhost/replica:0/task:0/cpu:0", send_device_incarnation=1, tensor_name="edge_5921_fifo_queue_DequeueMany", tensor_type=DT_INT32, _device="/job:localhost/replica:0/task:0/gpu:0"]()]]


Remote function compute_steps failed with:

Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/ray/worker.py", line 732, in _process_task
    self.actors[task.actor_id().id()], *arguments)
  File "ray/examples/resnet/resnet_main.py", line 113, in compute_steps
    self.model.variables.sess.run(self.model.train_op)
  File "/home/ubuntu/.local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 767, in run
    run_metadata_ptr)
  File "/home/ubuntu/.local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 965, in _run
    feed_dict_string, options, run_metadata)
  File "/home/ubuntu/.local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1015, in _do_run
    target_list, options, run_metadata)
  File "/home/ubuntu/.local/lib/python2.7/site-packages/tensorflow/python/client/session.py", line 1035, in _do_call
    raise type(e)(node_def, op, message)
OutOfRangeError: FIFOQueue '_1_fifo_queue' is closed and has insufficient elements (requested 128, current size 0)
	 [[Node: fifo_queue_DequeueMany = QueueDequeueManyV2[component_types=[DT_FLOAT, DT_INT32], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/cpu:0"](fifo_queue, fifo_queue_DequeueMany/n/_2345)]]
	 [[Node: fifo_queue_DequeueMany/_2349 = _HostRecv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/gpu:0", send_device="/job:localhost/replica:0/task:0/cpu:0", send_device_incarnation=1, tensor_name="edge_5921_fifo_queue_DequeueMany", tensor_type=DT_INT32, _device="/job:localhost/replica:0/task:0/gpu:0"]()]]

Caused by op u'fifo_queue_DequeueMany', defined at:
  File "/usr/local/lib/python2.7/dist-packages/ray/workers/default_worker.py", line 105, in <module>
    ray.worker.global_worker.main_loop()
  File "/usr/local/lib/python2.7/dist-packages/ray/worker.py", line 880, in main_loop
    self._wait_for_and_process_task(task)
  File "/usr/local/lib/python2.7/dist-packages/ray/worker.py", line 833, in _wait_for_and_process_task
    self._process_task(task)
  File "/usr/local/lib/python2.7/dist-packages/ray/worker.py", line 732, in _process_task
    self.actors[task.actor_id().id()], *arguments)
  File "ray/examples/resnet/resnet_main.py", line 96, in __init__
    False)
  File "/home/ubuntu/ray/examples/resnet/cifar_input.py", line 105, in build_input
    images, labels = example_queue.dequeue_many(batch_size)
  File "/home/ubuntu/.local/lib/python2.7/site-packages/tensorflow/python/ops/data_flow_ops.py", line 458, in dequeue_many
    self._queue_ref, n=n, component_types=self._dtypes, name=name)
  File "/home/ubuntu/.local/lib/python2.7/site-packages/tensorflow/python/ops/gen_data_flow_ops.py", line 1310, in _queue_dequeue_many_v2
    timeout_ms=timeout_ms, name=name)
  File "/home/ubuntu/.local/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py", line 763, in apply_op
    op_def=op_def)
  File "/home/ubuntu/.local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2327, in create_op
    original_op=self._default_original_op, op_def=op_def)
  File "/home/ubuntu/.local/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 1226, in __init__
    self._traceback = _extract_stack()

OutOfRangeError (see above for traceback): FIFOQueue '_1_fifo_queue' is closed and has insufficient elements (requested 128, current size 0)
	 [[Node: fifo_queue_DequeueMany = QueueDequeueManyV2[component_types=[DT_FLOAT, DT_INT32], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/cpu:0"](fifo_queue, fifo_queue_DequeueMany/n/_2345)]]
	 [[Node: fifo_queue_DequeueMany/_2349 = _HostRecv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/gpu:0", send_device="/job:localhost/replica:0/task:0/cpu:0", send_device_incarnation=1, tensor_name="edge_5921_fifo_queue_DequeueMany", tensor_type=DT_INT32, _device="/job:localhost/replica:0/task:0/gpu:0"]()]]



  You can inspect errors by running

      ray.error_info()

  If this driver is hanging, start a new one with

      ray.init(redis_address="127.0.0.1:57431")
  
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/ray/workers/default_worker.py", line 105, in <module>
    ray.worker.global_worker.main_loop()
  File "/usr/local/lib/python2.7/dist-packages/ray/worker.py", line 880, in main_loop
    self._wait_for_and_process_task(task)
  File "/usr/local/lib/python2.7/dist-packages/ray/worker.py", line 836, in _wait_for_and_process_task
    flush_log()
  File "/usr/local/lib/python2.7/dist-packages/ray/worker.py", line 1949, in flush_log
    worker.local_scheduler_client.log_event(event_log_key,
AttributeError: 'Worker' object has no attribute 'local_scheduler_client'

  This error is unexpected and should not have happened. Somehow a worker
  crashed in an unanticipated way causing the main_loop to throw an exception,
  which is being caught in "python/ray/workers/default_worker.py".
  

  You can inspect errors by running

      ray.error_info()

  If this driver is hanging, start a new one with

      ray.init(redis_address="127.0.0.1:57431")
  
[1]+  Terminated              python ray/examples/resnet/resnet_main.py --eval_dir=/tmp/resnet-model/eval --train_data_path=cifar-10-batches-bin/data_batch* --eval_data_path=cifar-10-batches-bin/test_batch.bin --dataset=cifar10 --num_gpus=0 &> output.txt



Discussion:
Running this without gpu seems to work fine.
Interesting, it looks like this may be the same issue as #446. @Wapaul1 @pcmoritz do you have any ideas about this? It's easily reproducible.
Same problem with same error
@xc9 @maxim28, can you share some details so we can reproduce the problem? What versions os TensorFlow, CUDA, cuDNN are you using? Which OS and Python version? What type of GPU? Is this on EC2? If so, which instance type? Any other important details about your setup?

If it's on EC2, perhaps you could share an AMI where we can reproduce the problem.

We're thinking of fixing this by replacing the queues with the TensorFlow Dataset API https://www.tensorflow.org/versions/r1.3/programmers_guide/datasets.

Resolving patch:
diff --git a/examples/resnet/cifar_input.py b/examples/resnet/cifar_input.py
--- a/examples/resnet/cifar_input.py
+++ b/examples/resnet/cifar_input.py
@@ -31,24 +31,27 @@ def build_data(data_path, size, dataset):
     image_bytes = image_size * image_size * depth
     record_bytes = label_bytes + label_offset + image_bytes
 
-    data_files = tf.gfile.Glob(data_path)
-    file_queue = tf.train.string_input_producer(data_files, shuffle=True)
+    def load_transform(value):
+        # Convert these examples to dense labels and processed images.
+        record = tf.reshape(tf.decode_raw(value, tf.uint8), [record_bytes])
+        label = tf.cast(tf.slice(record, [label_offset], [label_bytes]),
+                        tf.int32)
+        # Convert from string to [depth * height * width] to
+        # [depth, height, width].
+        depth_major = tf.reshape(
+            tf.slice(record, [label_bytes], [image_bytes]),
+            [depth, image_size, image_size])
+        # Convert from [depth, height, width] to [height, width, depth].
+        image = tf.cast(tf.transpose(depth_major, [1, 2, 0]), tf.float32)
+        return (image, label)
     # Read examples from files in the filename queue.
-    reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
-    _, value = reader.read(file_queue)
-
-    # Convert these examples to dense labels and processed images.
-    record = tf.reshape(tf.decode_raw(value, tf.uint8), [record_bytes])
-    label = tf.cast(tf.slice(record, [label_offset], [label_bytes]), tf.int32)
-    # Convert from string to [depth * height * width] to
-    # [depth, height, width].
-    depth_major = tf.reshape(tf.slice(record, [label_bytes], [image_bytes]),
-                             [depth, image_size, image_size])
-    # Convert from [depth, height, width] to [height, width, depth].
-    image = tf.cast(tf.transpose(depth_major, [1, 2, 0]), tf.float32)
-    queue = tf.train.shuffle_batch([image, label], size, size, 0,
-                                   num_threads=16)
-    return queue
+    data_files = tf.gfile.Glob(data_path)
+    data = tf.contrib.data.FixedLengthRecordDataset(data_files,
+                                                    record_bytes=record_bytes)
+    data = data.map(load_transform)
+    data = data.batch(size)
+    iterator = data.make_one_shot_iterator()
+    return iterator.get_next()
 
 
 def build_input(data, batch_size, dataset, train):
@@ -67,42 +70,35 @@ def build_input(data, batch_size, dataset, train):
     Raises:
       ValueError: When the specified dataset is not supported.
     """
-    images_constant = tf.constant(data[0])
-    labels_constant = tf.constant(data[1])
     image_size = 32
     depth = 3
     num_classes = 10 if dataset == "cifar10" else 100
-    image, label = tf.train.slice_input_producer([images_constant,
-                                                  labels_constant],
-                                                 capacity=16 * batch_size)
-    if train:
+    images, labels = data
+    num_samples = images.shape[0] - images.shape[0] % batch_size
+    dataset = tf.contrib.data.Dataset.from_tensor_slices(
+        (images[:num_samples], labels[:num_samples]))
+
+    def map_train(image, label):
         image = tf.image.resize_image_with_crop_or_pad(image, image_size + 4,
                                                        image_size + 4)
         image = tf.random_crop(image, [image_size, image_size, 3])
         image = tf.image.random_flip_left_right(image)
         image = tf.image.per_image_standardization(image)
-        example_queue = tf.RandomShuffleQueue(
-            capacity=16 * batch_size,
-            min_after_dequeue=8 * batch_size,
-            dtypes=[tf.float32, tf.int32],
-            shapes=[[image_size, image_size, depth], [1]])
-        num_threads = 16
-    else:
+        return (image, label)
+
+    def map_test(image, label):
         image = tf.image.resize_image_with_crop_or_pad(image, image_size,
                                                        image_size)
         image = tf.image.per_image_standardization(image)
-        example_queue = tf.FIFOQueue(
-            3 * batch_size,
-            dtypes=[tf.float32, tf.int32],
-            shapes=[[image_size, image_size, depth], [1]])
-        num_threads = 1
-
-    example_enqueue_op = example_queue.enqueue([image, label])
-    tf.train.add_queue_runner(tf.train.queue_runner.QueueRunner(
-        example_queue, [example_enqueue_op] * num_threads))
-
-    # Read "batch" labels + images from the example queue.
-    images, labels = example_queue.dequeue_many(batch_size)
+        return (image, label)
+
+    dataset = dataset.map(map_train if train else map_test)
+    dataset = dataset.batch(batch_size)
+    dataset = dataset.repeat()
+    if train:
+        dataset = dataset.shuffle(buffer_size=16 * batch_size)
+    images, labels = dataset.make_one_shot_iterator().get_next()
+    images = tf.reshape(images, [batch_size, image_size, image_size, depth])
     labels = tf.reshape(labels, [batch_size, 1])
     indices = tf.reshape(tf.range(0, batch_size, 1), [batch_size, 1])
     labels = tf.sparse_to_dense(
diff --git a/examples/resnet/resnet_main.py b/examples/resnet/resnet_main.py
--- a/examples/resnet/resnet_main.py
+++ b/examples/resnet/resnet_main.py
@@ -15,9 +15,11 @@
 import cifar_input
 import resnet_model
 
-# Tensorflow must be at least version 1.0.0 for the example to work.
-if int(tf.__version__.split(".")[0]) < 1:
-    raise Exception("Your Tensorflow version is less than 1.0.0. Please "
+# Tensorflow must be at least version 1.2.0 for the example to work.
+tf_major = int(tf.__version__.split(".")[0])
+tf_minor = int(tf.__version__.split(".")[1])
+if (tf_major < 1) or (tf_major == 1 and tf_minor < 2):
+    raise Exception("Your Tensorflow version is less than 1.2.0. Please "
                     "update Tensorflow to the latest version.")
 
 parser = argparse.ArgumentParser(description="Run the ResNet example.")
@@ -50,12 +52,9 @@ def get_data(path, size, dataset):
     # This only uses the cpu.
     os.environ["CUDA_VISIBLE_DEVICES"] = ""
     with tf.device("/cpu:0"):
-        queue = cifar_input.build_data(path, size, dataset)
+        dataset = cifar_input.build_data(path, size, dataset)
         sess = tf.Session()
-        coord = tf.train.Coordinator()
-        tf.train.start_queue_runners(sess, coord=coord)
-        images, labels = sess.run(queue)
-        coord.request_stop()
+        images, labels = sess.run(dataset)
         sess.close()
         return images, labels
 
@@ -86,12 +85,9 @@ def __init__(self, data, dataset, num_gpus):
             # Only a single actor in this case.
             tf.set_random_seed(1)
 
-        input_images = data[0]
-        input_labels = data[1]
         with tf.device("/gpu:0" if num_gpus > 0 else "/cpu:0"):
             # Build the model.
-            images, labels = cifar_input.build_input([input_images,
-                                                      input_labels],
+            images, labels = cifar_input.build_input(data,
                                                      hps.batch_size, dataset,
                                                      False)
             self.model = resnet_model.ResNet(hps, images, labels, "train")
@@ -100,8 +96,6 @@ def __init__(self, data, dataset, num_gpus):
             config.gpu_options.allow_growth = True
             sess = tf.Session(config=config)
             self.model.variables.set_session(sess)
-            self.coord = tf.train.Coordinator()
-            tf.train.start_queue_runners(sess, coord=self.coord)
             init = tf.global_variables_initializer()
             sess.run(init)
             self.steps = 10
@@ -123,6 +117,7 @@ def get_weights(self):
 @ray.remote
 class ResNetTestActor(object):
     def __init__(self, data, dataset, eval_batch_count, eval_dir):
+        os.environ["CUDA_VISIBLE_DEVICES"] = ""
         hps = resnet_model.HParams(
             batch_size=100,
             num_classes=100 if dataset == "cifar100" else 10,
@@ -134,12 +129,9 @@ def __init__(self, data, dataset, eval_batch_count, eval_dir):
             relu_leakiness=0.1,
             optimizer="mom",
             num_gpus=0)
-        input_images = data[0]
-        input_labels = data[1]
         with tf.device("/cpu:0"):
             # Builds the testing network.
-            images, labels = cifar_input.build_input([input_images,
-                                                      input_labels],
+            images, labels = cifar_input.build_input(data,
                                                      hps.batch_size, dataset,
                                                      False)
             self.model = resnet_model.ResNet(hps, images, labels, "eval")
@@ -148,8 +140,6 @@ def __init__(self, data, dataset, eval_batch_count, eval_dir):
             config.gpu_options.allow_growth = True
             sess = tf.Session(config=config)
             self.model.variables.set_session(sess)
-            self.coord = tf.train.Coordinator()
-            tf.train.start_queue_runners(sess, coord=self.coord)
             init = tf.global_variables_initializer()
             sess.run(init)
 


Test patch: