Repository: mars-project/mars

Issue:
[BUG] Fail in assigning workers in LGBM training when no eval_sets available
Fail in assigning workers in LGBM training when no eval_sets available. The  stack is

```
Error
Traceback (most recent call last):
  File "/Users/wenjun.swj/Code/mars/mars/scheduler/graph.py", line 395, in _execute_graph
    self.prepare_graph(compose=compose)
  File "/Users/wenjun.swj/Code/mars/mars/utils.py", line 353, in _wrapped
    return func(*args, **kwargs)
  File "/Users/wenjun.swj/Code/mars/mars/utils.py", line 493, in inner
    return func(*args, **kwargs)
  File "/Users/wenjun.swj/Code/mars/mars/scheduler/graph.py", line 633, in prepare_graph
    self._target_tileable_datas + fetch_tileables, tileable_graph)
  File "/Users/wenjun.swj/Code/mars/mars/utils.py", line 399, in _wrapped
    return func(*args, **kwargs)
  File "/Users/wenjun.swj/Code/mars/mars/utils.py", line 493, in inner
    return func(*args, **kwargs)
  File "/Users/wenjun.swj/Code/mars/mars/tiles.py", line 350, in build
    tileables, tileable_graph=tileable_graph)
  File "/Users/wenjun.swj/Code/mars/mars/utils.py", line 399, in _wrapped
    return func(*args, **kwargs)
  File "/Users/wenjun.swj/Code/mars/mars/utils.py", line 493, in inner
    return func(*args, **kwargs)
  File "/Users/wenjun.swj/Code/mars/mars/tiles.py", line 263, in build
    self._on_tile_failure(tileable_data.op, exc_info)
  File "/Users/wenjun.swj/Code/mars/mars/tiles.py", line 302, in inner
    raise exc_info[1].with_traceback(exc_info[2]) from None
  File "/Users/wenjun.swj/Code/mars/mars/tiles.py", line 243, in build
    tiled = self._tile(tileable_data, tileable_graph)
  File "/Users/wenjun.swj/Code/mars/mars/tiles.py", line 338, in _tile
    return super()._tile(tileable_data, tileable_graph)
  File "/Users/wenjun.swj/Code/mars/mars/tiles.py", line 203, in _tile
    tds = on_tile(tileable_data.op.outputs, tds)
  File "/Users/wenjun.swj/Code/mars/mars/scheduler/graph.py", line 615, in on_tile
    return self.context.wraps(handler.dispatch)(first.op)
  File "/Users/wenjun.swj/Code/mars/mars/context.py", line 68, in h
    return func(*args, **kwargs)
  File "/Users/wenjun.swj/Code/mars/mars/utils.py", line 399, in _wrapped
    return func(*args, **kwargs)
  File "/Users/wenjun.swj/Code/mars/mars/tiles.py", line 119, in dispatch
    tiled = op_cls.tile(op)
  File "/Users/wenjun.swj/Code/mars/mars/learn/contrib/lightgbm/train.py", line 207, in tile
    extra_workers = reduce(operator.or_, (set(w) for w in eval_workers_list)) - set(workers)
TypeError: reduce() of empty sequence with no initial value

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Users/wenjun.swj/miniconda3/lib/python3.7/unittest/case.py", line 59, in testPartExecutor
    yield
  File "/Users/wenjun.swj/miniconda3/lib/python3.7/unittest/case.py", line 628, in run
    testMethod()
  File "/Users/wenjun.swj/Code/mars/mars/learn/contrib/lightgbm/tests/integrated/test_distributed_lightgbm.py", line 52, in testDistributedLGBMClassifier
    classifier.fit(X, y, session=sess, run_kwargs=run_kwargs)
  File "/Users/wenjun.swj/Code/mars/mars/learn/contrib/lightgbm/classifier.py", line 32, in fit
    session=session, run_kwargs=run_kwargs, **kwargs)
  File "/Users/wenjun.swj/Code/mars/mars/learn/contrib/lightgbm/train.py", line 343, in train
    ret = op().execute(session=session, **run_kwargs).fetch(session=session)
  File "/Users/wenjun.swj/Code/mars/mars/core.py", line 367, in execute
    session.run(self, **kw)
  File "/Users/wenjun.swj/Code/mars/mars/session.py", line 428, in run
    result = self._sess.run(*tileables, **kw)
  File "/Users/wenjun.swj/Code/mars/mars/web/session.py", line 187, in run
    if self._check_response_finished(graph_url, timeout_val):
  File "/Users/wenjun.swj/Code/mars/mars/web/session.py", line 146, in _check_response_finished
    raise ExecutionFailed('Graph execution failed.') from exc
mars.errors.ExecutionFailed: 'Graph execution failed.'
```


Discussion:

Resolving patch:
diff --git a/mars/learn/contrib/lightgbm/train.py b/mars/learn/contrib/lightgbm/train.py
--- a/mars/learn/contrib/lightgbm/train.py
+++ b/mars/learn/contrib/lightgbm/train.py
@@ -203,20 +203,21 @@ def tile(cls, op: "LGBMTrain"):
                             getattr(op, arg).chunks, workers).items():
                         worker_to_args[worker][arg] = chunk
 
-            eval_workers_list = [cls._get_data_chunks_workers(ctx, d) for d in op.eval_datas]
-            extra_workers = reduce(operator.or_, (set(w) for w in eval_workers_list)) - set(workers)
-            worker_remap = dict(zip(extra_workers, itertools.cycle(workers)))
-            if worker_remap:
-                eval_workers_list = [[worker_remap.get(w, w) for w in wl] for wl in eval_workers_list]
-
-            for arg in ['_eval_datas', '_eval_labels', '_eval_sample_weights', '_eval_init_scores']:
-                if getattr(op, arg):
-                    for tileable, eval_workers in zip(getattr(op, arg), eval_workers_list):
-                        for worker, chunk in cls._concat_chunks_by_worker(
-                                tileable.chunks, eval_workers).items():
-                            if arg not in worker_to_args[worker]:
-                                worker_to_args[worker][arg] = []
-                            worker_to_args[worker][arg].append(chunk)
+            if op.eval_datas:
+                eval_workers_list = [cls._get_data_chunks_workers(ctx, d) for d in op.eval_datas]
+                extra_workers = reduce(operator.or_, (set(w) for w in eval_workers_list)) - set(workers)
+                worker_remap = dict(zip(extra_workers, itertools.cycle(workers)))
+                if worker_remap:
+                    eval_workers_list = [[worker_remap.get(w, w) for w in wl] for wl in eval_workers_list]
+
+                for arg in ['_eval_datas', '_eval_labels', '_eval_sample_weights', '_eval_init_scores']:
+                    if getattr(op, arg):
+                        for tileable, eval_workers in zip(getattr(op, arg), eval_workers_list):
+                            for worker, chunk in cls._concat_chunks_by_worker(
+                                    tileable.chunks, eval_workers).items():
+                                if arg not in worker_to_args[worker]:
+                                    worker_to_args[worker][arg] = []
+                                worker_to_args[worker][arg].append(chunk)
 
             out_chunks = []
             for worker in workers:


Test patch:
diff --git a/mars/learn/contrib/lightgbm/tests/integrated/test_distributed_lightgbm.py b/mars/learn/contrib/lightgbm/tests/integrated/test_distributed_lightgbm.py
--- a/mars/learn/contrib/lightgbm/tests/integrated/test_distributed_lightgbm.py
+++ b/mars/learn/contrib/lightgbm/tests/integrated/test_distributed_lightgbm.py
@@ -48,6 +48,15 @@ def testDistributedLGBMClassifier(self):
 
             X, y = self.X, self.y
             y = (y * 10).astype(mt.int32)
+            classifier = LGBMClassifier(n_estimators=2)
+            classifier.fit(X, y, session=sess, run_kwargs=run_kwargs)
+            prediction = classifier.predict(X, session=sess, run_kwargs=run_kwargs)
+
+            self.assertEqual(prediction.ndim, 1)
+            self.assertEqual(prediction.shape[0], len(self.X))
+
+            self.assertIsInstance(prediction, mt.Tensor)
+
             classifier = LGBMClassifier(n_estimators=2)
             classifier.fit(X, y, eval_set=[(X, y)], session=sess, run_kwargs=run_kwargs)
             prediction = classifier.predict(X, session=sess, run_kwargs=run_kwargs)