Repository: mars-project/mars

Issue:
[BUG] lightgbm TypeError: a bytes-like object is required, not 'list'
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**
There is a `TypeError: a bytes-like object is required, not 'list'` when i run a `lightgbm.LGBMClassifier.fit` on a 3 nodes Mars cluster.

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version: python 3.7.9
2. The version of Mars you use: 0.10.0
3. Versions of crucial packages, such as numpy, scipy and pandas: numpy 1.21.6, pandas 1.3.5, lightgbm 3.32
4. Full stack of the error.
5. Minimized code to reproduce the error.

4 and 5 are as follows:
I launched a 3 nodes Mars cluster with 1 supervisor and 3 workers. The Supervisor and a worker are on the same node, the other 2 worker are on two different nodes.
`Breast_cancer_data.csv` is from https://www.kaggle.com/code/prashant111/lightgbm-classifier-in-python/input
```python
In [1]: import pandas as pd
   ...: import mars.dataframe as md
   ...:
   ...: import mars
   ...: session = mars.new_session(f"http://6.0.55.34:9008")
   ...:
   ...: df = pd.read_csv("./Breast_cancer_data.csv")
   ...: mdf = md.DataFrame(data=df, chunk_size=300)
   ...: X = mdf[['mean_radius','mean_texture','mean_perimeter','mean_area','mean_smoothness']]
   ...: y = mdf['diagnosis']
   ...:
   ...: from mars.learn.contrib import lightgbm as lgb
   ...:
   ...: gbm = lgb.LGBMClassifier(importance_type='gain')
   ...: gbm.fit(X, y)
Metric is not initialized, please call `init_metrics()` before using metrics.
100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 100.00/100 [00:00<00:00, 705.12it/s]
100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 100.00/100 [00:03<00:00, 28.08it/s]
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-1-dfd6546be0ac> in <module>
     13
     14 gbm = lgb.LGBMClassifier(importance_type='gain')
---> 15 gbm.fit(X, y)

~/.pyenv/versions/3.7.9/envs/gmars379/lib/python3.7/site-packages/mars/learn/contrib/lightgbm/classifier.py in fit(self, X, y, sample_weight, init_score, eval_set, eval_sample_weight, eval_init_score, session, run_kwargs, **kwargs)
     53                 session=session,
     54                 run_kwargs=run_kwargs,
---> 55                 **kwargs
     56             )
     57

~/.pyenv/versions/3.7.9/envs/gmars379/lib/python3.7/site-packages/mars/learn/contrib/lightgbm/_train.py in train(params, train_set, eval_sets, **kwargs)
    454     ret = op().execute(session=session, **run_kwargs).fetch(session=session)
    455
--> 456     bst = pickle.loads(ret)
    457     evals_result.update(bst.evals_result_ or {})
    458     return bst

TypeError: a bytes-like object is required, not 'list'
```

**Expected behavior**
A clear and concise description of what you expected to happen.

**Additional context**
Add any other context about the problem here.



Discussion:

Resolving patch:
diff --git a/mars/learn/contrib/lightgbm/_train.py b/mars/learn/contrib/lightgbm/_train.py
--- a/mars/learn/contrib/lightgbm/_train.py
+++ b/mars/learn/contrib/lightgbm/_train.py
@@ -436,7 +436,8 @@ def train(params, train_set, eval_sets=None, **kwargs):
         sample_weights.append(train_kw["sample_weight"])
         init_scores.append(train_kw["init_score"])
 
-    op = LGBMTrain(
+    train_cls = kwargs.pop("train_cls", LGBMTrain)
+    op = train_cls(
         params=params,
         data=datas[0],
         label=labels[0],
@@ -452,6 +453,10 @@ def train(params, train_set, eval_sets=None, **kwargs):
         kwds=kwargs,
     )
     ret = op().execute(session=session, **run_kwargs).fetch(session=session)
+    # Note: There may be multiple results if running on clusters, so we should
+    # combine the results if so.
+    if isinstance(ret, list):
+        ret = b"".join(ret)
 
     bst = pickle.loads(ret)
     evals_result.update(bst.evals_result_ or {})
diff --git a/mars/metrics/api.py b/mars/metrics/api.py
--- a/mars/metrics/api.py
+++ b/mars/metrics/api.py
@@ -119,7 +119,7 @@ def record(self, value=1, tags: Optional[Dict[str, str]] = None):
             self._metric.record(value, tags)
         elif not self._log_not_init_error:
             self._log_not_init_error = True
-            logger.warning(
+            logger.info(
                 "Metric is not initialized, please call `init_metrics()` before using metrics."
             )
 


Test patch:
diff --git a/mars/learn/contrib/lightgbm/tests/test_classifier.py b/mars/learn/contrib/lightgbm/tests/test_classifier.py
--- a/mars/learn/contrib/lightgbm/tests/test_classifier.py
+++ b/mars/learn/contrib/lightgbm/tests/test_classifier.py
@@ -170,3 +170,64 @@ def test_local_classifier_from_to_parquet(setup):
         expected = classifier.predict(X)
         expected = np.stack([1 - expected, expected]).argmax(axis=0)
         np.testing.assert_array_equal(ret, expected)
+
+
+@pytest.mark.skipif(lightgbm is None, reason="LightGBM not installed")
+def test_classifier_on_multiple_machines(setup):
+    from .._train import LGBMTrain
+
+    class MockLGMBTrain(LGBMTrain):
+        @classmethod
+        def execute(cls, ctx, op: "LGBMTrain"):
+            super().execute(ctx, op)
+            # Note: There may be a list result when running on multiple
+            # machines, here just make an array of length 1 to simulate
+            # this scenario.
+            ctx[op.outputs[0].key] = [ctx[op.outputs[0].key]]
+
+    from ..core import LGBMModelType
+    from .._train import train
+    from ....utils import check_consistent_length
+
+    class MockLGBMClassifier(LGBMClassifier, lightgbm.LGBMClassifier):
+        def fit(
+            self,
+            X,
+            y,
+            sample_weight=None,
+            init_score=None,
+            eval_set=None,
+            eval_sample_weight=None,
+            eval_init_score=None,
+            session=None,
+            run_kwargs=None,
+            **kwargs,
+        ):
+            check_consistent_length(X, y, session=session, run_kwargs=run_kwargs)
+            params = self.get_params(True)
+            model = train(
+                params,
+                self._wrap_train_tuple(X, y, sample_weight, init_score),
+                eval_sets=self._wrap_eval_tuples(
+                    eval_set, eval_sample_weight, eval_init_score
+                ),
+                model_type=LGBMModelType.CLASSIFIER,
+                session=session,
+                run_kwargs=run_kwargs,
+                train_cls=MockLGMBTrain,
+                **kwargs,
+            )
+
+            self.set_params(**model.get_params())
+            self._copy_extra_params(model, self)
+            return self
+
+    y_data = (y * 10).astype(mt.int32)
+    classifier = MockLGBMClassifier(n_estimators=2)
+    classifier.fit(X, y_data, eval_set=[(X, y_data)], verbose=True)
+    prediction = classifier.predict(X)
+
+    assert prediction.ndim == 1
+    assert prediction.shape[0] == len(X)
+
+    assert isinstance(prediction, mt.Tensor)