Repository: mars-project/mars

Issue:
[BUG][WORKER] Got AttributeError when submitting job to cluster
**Describe the bug**
 
When I submitted a job to cluster, the worker reported AttributeError and the job failed.

Error message:

```
Unexpected error occurred in executing bd974beb010abfa6964bdee22c5d2080
Traceback (most recent call last):
  File "/home/admin/work/_public-mars-0.1.0b1.zip/mars/worker/execution.py", line 457, in _handle_rejection
    six.reraise(*exc)
  File "/home/admin/work/_public-mars-0.1.0b1.zip/mars/lib/six.py", line 703, in reraise
    raise value
  File "/home/admin/work/_public-mars-0.1.0b1.zip/mars/promise.py", line 86, in _wrapped
    result = func(*args, **kwargs)
  File "/home/admin/work/_public-mars-0.1.0b1.zip/mars/utils.py", line 287, in _wrapped
    return func(*args, **kwargs)
  File "/home/admin/work/_public-mars-0.1.0b1.zip/mars/worker/execution.py", line 328, in _prepare_inputs
    for w in worker_results.children:
AttributeError: 'tuple' object has no attribute 'children'
```

**To Reproduce**

More than one worker, and the data need to be exchanged from one worker to another worker, the error can be reproduced.

**Expected behavior**

The `worker_results` is a tuple, and do have no attribute `children`.

**Additional context**

We need to figure out a test in which more than 1 workers created, and some data needs to be exchanged between workers.


Discussion:

Resolving patch:
diff --git a/mars/worker/execution.py b/mars/worker/execution.py
--- a/mars/worker/execution.py
+++ b/mars/worker/execution.py
@@ -322,13 +322,11 @@ def _prepare_inputs(*_):
                     .get_chunk_meta(session_id, chunk.key)
                 if chunk_meta is None:
                     raise DependencyMissing('Dependency %s not met on sending.' % chunk.key)
-                worker_results = chunk_meta.workers
 
                 worker_priorities = []
-                for w in worker_results.children:
-                    _, worker_ip = w.key.rsplit('/', 1)
+                for w in chunk_meta.workers:
                     # todo sort workers by speed of network and other possible factors
-                    worker_priorities.append((worker_ip, (0, )))
+                    worker_priorities.append((w, (0, )))
 
                 transfer_keys.append(chunk.key)
 


Test patch:
diff --git a/mars/scheduler/tests/test_main.py b/mars/scheduler/tests/test_main.py
--- a/mars/scheduler/tests/test_main.py
+++ b/mars/scheduler/tests/test_main.py
@@ -55,13 +55,20 @@ def tearDownClass(cls):
 
     def setUp(self):
         scheduler_port = str(get_next_port())
-        proc_worker = subprocess.Popen([sys.executable, '-m', 'mars.worker',
-                                        '-a', '127.0.0.1',
-                                        '--cpu-procs', '2',
-                                        '--level', 'debug',
-                                        '--cache-mem', '16m',
-                                        '--schedulers', '127.0.0.1:' + scheduler_port,
-                                        '--ignore-avail-mem'])
+        proc_worker1 = subprocess.Popen([sys.executable, '-m', 'mars.worker',
+                                         '-a', '127.0.0.1',
+                                         '--cpu-procs', '2',
+                                         '--level', 'debug',
+                                         '--cache-mem', '16m',
+                                         '--schedulers', '127.0.0.1:' + scheduler_port,
+                                         '--ignore-avail-mem'])
+        proc_worker2 = subprocess.Popen([sys.executable, '-m', 'mars.worker',
+                                         '-a', '127.0.0.1',
+                                         '--cpu-procs', '2',
+                                         '--level', 'debug',
+                                         '--cache-mem', '16m',
+                                         '--schedulers', '127.0.0.1:' + scheduler_port,
+                                         '--ignore-avail-mem'])
         proc_scheduler = subprocess.Popen([sys.executable, '-m', 'mars.scheduler',
                                            '-H', '127.0.0.1',
                                            '--level', 'debug',
@@ -69,7 +76,7 @@ def setUp(self):
                                            '--format', '%(asctime)-15s %(message)s'])
 
         self.scheduler_port = scheduler_port
-        self.proc_worker = proc_worker
+        self.proc_workers = [proc_worker1, proc_worker2]
         self.proc_scheduler = proc_scheduler
 
         time.sleep(2)
@@ -89,7 +96,7 @@ def setUp(self):
 
         check_time = time.time()
         while True:
-            if not resource_ref.get_worker_count():
+            if resource_ref.get_worker_count() < 2:
                 time.sleep(0.5)
                 self.check_process_statuses()
                 if time.time() - check_time > 20:
@@ -101,7 +108,7 @@ def setUp(self):
         gevent.hub.Hub.NOT_ERROR = (Exception,)
 
     def tearDown(self):
-        procs = (self.proc_worker, self.proc_scheduler)
+        procs = (self.proc_scheduler,) + tuple(self.proc_workers)
         for p in procs:
             p.send_signal(signal.SIGINT)
 
@@ -120,8 +127,9 @@ def tearDown(self):
     def check_process_statuses(self):
         if self.proc_scheduler.poll() is not None:
             raise SystemError('Scheduler not started. exit code %s' % self.proc_scheduler.poll())
-        if self.proc_worker.poll() is not None:
-            raise SystemError('Worker not started. exit code %s' % self.proc_worker.poll())
+        for worker_proc in self.proc_workers:
+            if worker_proc.poll() is not None:
+                raise SystemError('Worker not started. exit code %s' % worker_proc.poll())
 
     def wait_for_termination(self, session_ref, graph_key):
         check_time = time.time()