Repository: ray-project/ray

Issue:
Error in parameter server examples when multiple workers try to download MNIST at the same time.
To reproduce the example, run the following (making sure that `ray/examples/parameter_server/` does not have a copy of the MNIST data set)

```
cd ray/examples/parameter_server/
python async_parameter_server.py
```

Some tasks threw the following error

```
Remote function __main__.worker_task failed with:

Traceback (most recent call last):
  File "async_parameter_server.py", line 40, in worker_task
    mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
  File "/home/ubuntu/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py", line 245, in read_data_sets
    source_url + TRAIN_LABELS)
  File "/home/ubuntu/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/base.py", line 209, in maybe_download
    gfile.Copy(temp_file_name, filepath)
  File "/home/ubuntu/anaconda3/lib/python3.6/site-packages/tensorflow/python/lib/io/file_io.py", line 385, in copy
    compat.as_bytes(oldpath), compat.as_bytes(newpath), overwrite, status)
  File "/home/ubuntu/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/errors_impl.py", line 473, in __exit__
    c_api.TF_GetCode(self.status.status))
tensorflow.python.framework.errors_impl.AlreadyExistsError: file already exists
```

The same error can probably occur with the sync parameter server as well.


Discussion:

Resolving patch:
diff --git a/examples/parameter_server/async_parameter_server.py b/examples/parameter_server/async_parameter_server.py
--- a/examples/parameter_server/async_parameter_server.py
+++ b/examples/parameter_server/async_parameter_server.py
@@ -3,11 +3,9 @@
 from __future__ import print_function
 
 import argparse
-from tensorflow.examples.tutorials.mnist import input_data
 import time
 
 import ray
-
 import model
 
 parser = argparse.ArgumentParser(description="Run the asynchronous parameter "
@@ -35,9 +33,9 @@ def pull(self, keys):
 
 
 @ray.remote
-def worker_task(ps, batch_size=50):
+def worker_task(ps, worker_index, batch_size=50):
     # Download MNIST.
-    mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
+    mnist = model.download_mnist_retry(seed=worker_index)
 
     # Initialize the model.
     net = model.SimpleCNN()
@@ -65,10 +63,10 @@ def worker_task(ps, batch_size=50):
     ps = ParameterServer.remote(all_keys, all_values)
 
     # Start some training tasks.
-    worker_tasks = [worker_task.remote(ps) for _ in range(args.num_workers)]
+    worker_tasks = [worker_task.remote(ps, i) for i in range(args.num_workers)]
 
     # Download MNIST.
-    mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
+    mnist = model.download_mnist_retry()
 
     i = 0
     while True:
diff --git a/examples/parameter_server/model.py b/examples/parameter_server/model.py
--- a/examples/parameter_server/model.py
+++ b/examples/parameter_server/model.py
@@ -8,6 +8,18 @@
 
 import ray
 import tensorflow as tf
+from tensorflow.examples.tutorials.mnist import input_data
+import time
+
+
+def download_mnist_retry(seed=0, max_num_retries=20):
+    for _ in range(max_num_retries):
+        try:
+            return input_data.read_data_sets("MNIST_data", one_hot=True,
+                                             seed=seed)
+        except tf.errors.AlreadyExistsError:
+            time.sleep(1)
+    raise Exception("Failed to download MNIST.")
 
 
 class SimpleCNN(object):
diff --git a/examples/parameter_server/sync_parameter_server.py b/examples/parameter_server/sync_parameter_server.py
--- a/examples/parameter_server/sync_parameter_server.py
+++ b/examples/parameter_server/sync_parameter_server.py
@@ -3,9 +3,7 @@
 from __future__ import print_function
 
 import argparse
-
 import numpy as np
-from tensorflow.examples.tutorials.mnist import input_data
 
 import ray
 import model
@@ -36,8 +34,7 @@ class Worker(object):
     def __init__(self, worker_index, batch_size=50):
         self.worker_index = worker_index
         self.batch_size = batch_size
-        self.mnist = input_data.read_data_sets("MNIST_data", one_hot=True,
-                                               seed=worker_index)
+        self.mnist = model.download_mnist_retry(seed=worker_index)
         self.net = model.SimpleCNN()
 
     def compute_gradients(self, weights):
@@ -60,7 +57,7 @@ def compute_gradients(self, weights):
                for worker_index in range(args.num_workers)]
 
     # Download MNIST.
-    mnist = input_data.read_data_sets("MNIST_data", one_hot=True)
+    mnist = model.download_mnist_retry()
 
     i = 0
     current_weights = ps.get_weights.remote()


Test patch: