Repository: mars-project/mars

Issue:
lightgbm raised TypeError when the same model is fitted again
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**
mars version is 0.5.1, run lightgbm in cluster, schedule node error:

```
Attempt 4: Unexpected error TypeError occurred in executing operand affdad0be8e3430b7b6088cd112ed634 in 10.xxx:8083
Traceback (most recent call last):
  File "/data/platform/anaconda3/envs/mars-dev/lib/python3.7/site-packages/mars/promise.py", line 100, in _wrapped
    result = func(*args, **kwargs)
  File "/data/platform/anaconda3/envs/mars-dev/lib/python3.7/site-packages/mars/worker/calc.py", line 299, in <lambda>
    .then(lambda context_dict: _start_calc(context_dict)) \
  File "/data/platform/anaconda3/envs/mars-dev/lib/python3.7/site-packages/mars/worker/calc.py", line 273, in _start_calc
    return self._calc_results(session_id, graph_key, graph, context_dict, chunk_targets)
  File "/data/platform/anaconda3/envs/mars-dev/lib/python3.7/site-packages/mars/utils.py", line 365, in _wrapped
    return func(*args, **kwargs)
  File "/data/platform/anaconda3/envs/mars-dev/lib/python3.7/site-packages/mars/worker/calc.py", line 197, in _calc_results
    chunk_targets, retval=False).result()
  File "src/gevent/event.py", line 383, in gevent._gevent_cevent.AsyncResult.result
  File "src/gevent/event.py", line 305, in gevent._gevent_cevent.AsyncResult.get
  File "src/gevent/event.py", line 335, in gevent._gevent_cevent.AsyncResult.get
  File "src/gevent/event.py", line 323, in gevent._gevent_cevent.AsyncResult.get
  File "src/gevent/event.py", line 303, in gevent._gevent_cevent.AsyncResult._raise_exception
  File "/data/platform/anaconda3/envs/mars-dev/lib/python3.7/site-packages/gevent/_compat.py", line 65, in reraise
    raise value.with_traceback(tb)
  File "/data/platform/anaconda3/envs/mars-dev/lib/python3.7/site-packages/gevent/threadpool.py", line 142, in __run_task
    thread_result.set(func(*args, **kwargs))
  File "mars/actors/pool/gevent_pool.pyx", line 127, in mars.actors.pool.gevent_pool.GeventThreadPool._wrap_watch.inner
    result = fn(*args, **kwargs)
  File "/data/platform/anaconda3/envs/mars-dev/lib/python3.7/site-packages/mars/executor.py", line 690, in execute_graph
    res = graph_execution.execute(retval)
  File "/data/platform/anaconda3/envs/mars-dev/lib/python3.7/site-packages/mars/executor.py", line 571, in execute
    future.result()
  File "/data/platform/anaconda3/envs/mars-dev/lib/python3.7/concurrent/futures/_base.py", line 435, in result
    return self.__get_result()
  File "/data/platform/anaconda3/envs/mars-dev/lib/python3.7/concurrent/futures/_base.py", line 384, in __get_result
    raise self._exception
  File "/data/platform/anaconda3/envs/mars-dev/lib/python3.7/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/data/platform/anaconda3/envs/mars-dev/lib/python3.7/site-packages/mars/utils.py", line 439, in _inner
    return func(*args, **kwargs)
  File "/data/platform/anaconda3/envs/mars-dev/lib/python3.7/site-packages/mars/executor.py", line 443, in _execute_operand
    Executor.handle(first_op, results, self._mock)
  File "/data/platform/anaconda3/envs/mars-dev/lib/python3.7/site-packages/mars/executor.py", line 641, in handle
    return runner(results, op)
  File "/data/platform/anaconda3/envs/mars-dev/lib/python3.7/site-packages/mars/learn/contrib/lightgbm/train.py", line 298, in execute
    eval_init_score=eval_init_score, **op.kwds)
  File "/data/platform/anaconda3/envs/mars-dev/lib/python3.7/site-packages/lightgbm/sklearn.py", line 760, in fit
    callbacks=callbacks, init_model=init_model)
  File "/data/platform/anaconda3/envs/mars-dev/lib/python3.7/site-packages/lightgbm/sklearn.py", line 600, in fit
    callbacks=callbacks, init_model=init_model)
  File "/data/platform/anaconda3/envs/mars-dev/lib/python3.7/site-packages/lightgbm/engine.py", line 231, in train
    booster = Booster(params=params, train_set=train_set)
  File "/data/platform/anaconda3/envs/mars-dev/lib/python3.7/site-packages/lightgbm/basic.py", line 1983, in __init__
    train_set.construct()
  File "/data/platform/anaconda3/envs/mars-dev/lib/python3.7/site-packages/lightgbm/basic.py", line 1325, in construct
    categorical_feature=self.categorical_feature, params=self.params)
  File "/data/platform/anaconda3/envs/mars-dev/lib/python3.7/site-packages/lightgbm/basic.py", line 1102, in _lazy_init
    params_str = param_dict_to_str(params)
  File "/data/platform/anaconda3/envs/mars-dev/lib/python3.7/site-packages/lightgbm/basic.py", line 156, in param_dict_to_str
    % (key, type(val).__name__))
TypeError: Unknown type of parameter:out_dtype_, got:dtype
```

Conda List:
```
apipkg             1.5
attrs              20.2.0
bokeh              2.2.1
brotlipy           0.7.0
certifi            2020.6.20
cffi               1.14.3
chardet            3.0.4
cloudpickle        1.6.0
coverage           5.3
cryptography       3.1
Cython             0.29.21
execnet            1.7.1
gevent             20.6.2
greenlet           0.4.16
idna               2.10
importlib-metadata 1.7.0
iniconfig          0.0.0
Jinja2             2.11.2
joblib             0.16.0
lightgbm           3.0.0
lz4                2.2.1
MarkupSafe         1.1.1
mkl-fft            1.2.0
mkl-random         1.1.1
mkl-service        2.3.0
mock               4.0.2
more-itertools     8.5.0
numexpr            2.7.1
numpy              1.19.1
olefile            0.46
packaging          20.4
pandas             1.1.1
Pillow             7.2.0
pip                20.2.2
pluggy             0.13.1
protobuf           3.13.0
psutil             5.7.2
py                 1.9.0
pyarrow            0.11.1
pycparser          2.20
pymars             0.5.1
pyOpenSSL          19.1.0
pyparsing          2.4.7
PySocks            1.7.1
pytest             6.0.2
pytest-cov         2.10.1
pytest-forked      1.3.0
pytest-timeout     1.4.2
pytest-xdist       2.1.0
python-dateutil    2.8.1
pytz               2020.1
PyYAML             5.3.1
requests           2.24.0
scikit-learn       0.23.2
scipy              1.5.2
setuptools         49.6.0.post20200814
six                1.15.0
threadpoolctl      2.1.0
toml               0.10.1
tornado            6.0.4
typing-extensions  3.7.4.3
urllib3            1.25.10
wheel              0.35.1
zipp               3.1.0
zope.event         4.4
zope.interface     5.1.0
```

```python
import mars.tensor as mt
import mars.dataframe as md
from mars.session import new_session
sess = new_session('http://10.xxx:8082').as_default()

a = mt.ones((2000, 2000), chunk_size=200)
b = mt.inner(a, a)
b.execute()  # submit tensor to cluster

df = md.DataFrame(a).sum()

df.execute()# submit DataFrame to cluster
z = mt.zeros((3, 3))
t = mt.ones((3, 3))
mt.ExecutableTuple([z, t]).execute()

from sklearn.datasets import load_boston

boston = load_boston()

import mars.dataframe as md

data = md.DataFrame(boston.data, columns=boston.feature_names)

data.head().execute()

from mars.learn.model_selection import train_test_split

X_train, X_test, y_train, y_test = \
    train_test_split(data, boston.target, train_size=0.7, random_state=0)

from mars.learn.contrib import lightgbm as lgb

lg_reg = lgb.LGBMRegressor(colsample_bytree=0.3, learning_rate=0.1, max_depth=5, reg_alpha=10, n_estimators=10)

lg_reg.fit(X_train, y_train)

lg_reg.fit(X_train,y_train, session=sess) # error

```


Discussion:
> Please provide minimal code to reproduce this stack.

from mars.learn.contrib import lightgbm as lgb

lg_reg = lgb.LGBMRegressor(colsample_bytree=0.3, learning_rate=0.1, max_depth=5, reg_alpha=10, n_estimators=10)

lg_reg.fit(X_train, y_train)

lg_reg.predict(X_test)

lg_reg.fit(X_train,y_train, session=sess)
The issue is caused by passing training results into fit function, which is not recognized. Constructing a new ``lg_reg`` can be a workaround.

We will fix it in the next release. Thanks for reporting.

Resolving patch:
diff --git a/mars/learn/contrib/lightgbm/train.py b/mars/learn/contrib/lightgbm/train.py
--- a/mars/learn/contrib/lightgbm/train.py
+++ b/mars/learn/contrib/lightgbm/train.py
@@ -277,6 +277,8 @@ def execute(cls, ctx, op: "LGBMTrain"):
             eval_init_score = eval_init_score or None
 
         params = op.params.copy()
+        # if model is trained, remove unsupported parameters
+        params.pop('out_dtype_', None)
         if ctx.running_mode == RunningMode.distributed:
             params['machines'] = ','.join(op.lgbm_endpoints)
             params['time_out'] = op.timeout


Test patch:
diff --git a/mars/learn/contrib/lightgbm/tests/integrated/test_distributed_lightgbm.py b/mars/learn/contrib/lightgbm/tests/integrated/test_distributed_lightgbm.py
--- a/mars/learn/contrib/lightgbm/tests/integrated/test_distributed_lightgbm.py
+++ b/mars/learn/contrib/lightgbm/tests/integrated/test_distributed_lightgbm.py
@@ -55,6 +55,13 @@ def testDistributedLGBMClassifier(self):
             self.assertEqual(prediction.ndim, 1)
             self.assertEqual(prediction.shape[0], len(self.X))
 
+            # fi on fitted model shall work well
+            classifier.fit(X, y, session=sess, run_kwargs=run_kwargs)
+            prediction = classifier.predict(X, session=sess, run_kwargs=run_kwargs)
+
+            self.assertEqual(prediction.ndim, 1)
+            self.assertEqual(prediction.shape[0], len(self.X))
+
             self.assertIsInstance(prediction, mt.Tensor)
 
             classifier = LGBMClassifier(n_estimators=2)
diff --git a/mars/learn/tests/integrated/base.py b/mars/learn/tests/integrated/base.py
--- a/mars/learn/tests/integrated/base.py
+++ b/mars/learn/tests/integrated/base.py
@@ -75,6 +75,7 @@ def _start_distributed_env(self, n_workers=2):
                                            '-H', '127.0.0.1',
                                            '-p', scheduler_port,
                                            '-Dscheduler.default_cpu_usage=0',
+                                           '-Dscheduler.retry_delay=5',
                                            '--log-level', 'debug',
                                            '--log-format', 'SCH %(asctime)-15s %(message)s'])
         self.proc_scheduler = proc_scheduler