Repository: optuna/optuna

Issue:
OptunaSearchCV does not support `scikit-learn>=0.22.1`.
## Expected behavior


The `OptunaSearchCV` does not work with the latest scikit-learn (v0.22.1). This is due to the update of the internal interface in [this commit](https://github.com/scikit-learn/scikit-learn/commit/46001e7cfb163b73a83d5adf73fb454e8a3c64b6).  More concretely, it removed `sklearn.model_selection._validation._index_param_value` which is used in `OptunaSearchCV` and added `sklearn.utils.validation._check_fit_params` instead.

`OptunaSearchCV` uses some other internal interface of scikit-learn, and I think it would be better to stop using them to avoid similar issues.

## Environment

- Optuna version: 0.19.0
- Python version: 3.7.5
- OS : `Linux 75b723335ed2 4.9.184-linuxkit #1 SMP Tue Jul 2 22:58:16 UTC 2019 x86_64 GNU/Linux`
- scikit-learn versions: `scikit-learn==0.22.1`


## Reproducible examples

Setup
```bash
$ docker run -it --rm python:3.7 bash
$ pip install scikit-learn
$ pip install optuna
```

Reproducible code
```python
>>> import sklearn
>>> import optuna.integration.sklearn

# The following line raises import error even if sklearn has been installed.
>>> optuna.integration.sklearn._check_sklearn_availability()
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/local/lib/python3.8/site-packages/optuna/integration/sklearn.py", line 65, in _check_sklearn_availability
    raise ImportError(
ImportError: scikit-learn is not available. Please install scikit-learn to use this feature. scikit-learn can be installed by executing `$ pip install scikit-learn>=0.19.0`. For further information, please refer to the installation guide of scikit-learn. (The actual import error is as follows: cannot import name '_index_param_value' from 'sklearn.model_selection._validation' (/usr/local/lib/python3.8/site-packages/sklearn/model_selection/_validation.py))

# The actual cause of the above error.
>>> from sklearn.model_selection._validation import _index_param_value
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
ImportError: cannot import name '_index_param_value' from 'sklearn.model_selection._validation' (/usr/local/lib/python3.7/site-packages/sklearn/model_selection/_validation.py)

```



Discussion:

Resolving patch:
diff --git a/optuna/integration/sklearn.py b/optuna/integration/sklearn.py
--- a/optuna/integration/sklearn.py
+++ b/optuna/integration/sklearn.py
@@ -7,11 +7,11 @@
 import numpy as np
 
 try:
+    from sklearn import __version__ as _sklearn_version
     from sklearn.base import BaseEstimator
     from sklearn.base import clone
     from sklearn.base import is_classifier
     from sklearn.metrics.scorer import check_scoring
-    from sklearn.model_selection._validation import _index_param_value
     from sklearn.model_selection import BaseCrossValidator  # NOQA
     from sklearn.model_selection import check_cv
     from sklearn.model_selection import cross_validate
@@ -29,14 +29,20 @@
     _import_error = e
     _available = False
 
-from optuna import distributions
-from optuna import exceptions
-from optuna import logging
+if _available:
+    if _sklearn_version >= '0.22.1':
+        from sklearn.utils.validation import _check_fit_params as _sklearn_check_fit_params
+    else:
+        from sklearn.utils.validation import _index_param_value as _sklearn_index_param_value
+
+from optuna import distributions  # NOQA
+from optuna import exceptions  # NOQA
+from optuna import logging  # NOQA
 from optuna import samplers  # NOQA
-from optuna import structs
-from optuna import study as study_module
+from optuna import structs  # NOQA
+from optuna import study as study_module  # NOQA
 from optuna import trial as trial_module  # NOQA
-from optuna import type_checking
+from optuna import type_checking  # NOQA
 
 if type_checking.TYPE_CHECKING:
     import pandas as pd  # NOQA
@@ -56,6 +62,25 @@
 _logger = logging.get_logger(__name__)
 
 
+def _check_fit_params(
+    X,  # type: TwoDimArrayLikeType
+    fit_params,  # type: Dict
+    indices  # type: OneDimArrayLikeType
+):
+    # type: (...) -> Dict
+
+    if _sklearn_version >= '0.22.1':
+        return _sklearn_check_fit_params(X, fit_params, indices)
+    else:  # '_sklearn_version < 0.22.1'
+        return {
+            key: _sklearn_index_param_value(
+                X,
+                value,
+                indices
+            ) for key, value in fit_params.items()
+        }
+
+
 def _check_sklearn_availability():
     # type: () -> None
 
@@ -841,13 +866,7 @@ def fit(
         fit_params_res = fit_params
 
         if fit_params_res is not None:
-            fit_params_res = {
-                key: _index_param_value(
-                    X,
-                    value,
-                    self.sample_indices_
-                ) for key, value in fit_params.items()
-            }
+            fit_params_res = _check_fit_params(X, fit_params, self.sample_indices_)
 
         classifier = is_classifier(self.estimator)
         cv = check_cv(self.cv, y_res, classifier)
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -65,9 +65,7 @@ def get_extras_require():
         ],
         'doctest': [
             'pandas',
-            # TODO(Yanase): Update sklearn integration to support v0.22.1 or newer.
-            # See https://github.com/optuna/optuna/issues/825 for further details.
-            'scikit-learn>=0.19.0,<=0.22.0',
+            'scikit-learn>=0.19.0',
         ],
         'document': [
             'lightgbm',
@@ -81,9 +79,7 @@ def get_extras_require():
             'mlflow',
             'mxnet',
             'scikit-image',
-            # TODO(Yanase): Update sklearn integration to support v0.22.1 or newer.
-            # See https://github.com/optuna/optuna/issues/825 for further details.
-            'scikit-learn<=0.22.0',
+            'scikit-learn',
             'xgboost',
         ] + (['fastai<2'] if (3, 5) < sys.version_info[:2] < (3, 8) else [])
         + ([
@@ -107,9 +103,7 @@ def get_extras_require():
             'pandas',
             'plotly>=4.0.0',
             'pytest',
-            # TODO(Yanase): Update sklearn integration to support v0.22.1 or newer.
-            # See https://github.com/optuna/optuna/issues/825 for further details.
-            'scikit-learn>=0.19.0,<=0.22.0',
+            'scikit-learn>=0.19.0',
             'scikit-optimize',
             'xgboost',
         ] + (['fastai<2'] if (3, 5) < sys.version_info[:2] < (3, 8) else [])


Test patch:
diff --git a/tests/integration_tests/test_sklearn.py b/tests/integration_tests/test_sklearn.py
--- a/tests/integration_tests/test_sklearn.py
+++ b/tests/integration_tests/test_sklearn.py
@@ -6,11 +6,14 @@
 from optuna import distributions
 from optuna import integration
 
+import numpy as np
+
 
 @pytest.mark.parametrize('enable_pruning', [True, False])
+@pytest.mark.parametrize('fit_params', ['', 'coef_init'])
 @pytest.mark.filterwarnings('ignore::UserWarning')
-def test_optuna_search(enable_pruning):
-    # type: (bool) -> None
+def test_optuna_search(enable_pruning, fit_params):
+    # type: (bool, str) -> None
 
     X, y = make_blobs(n_samples=10)
     est = SGDClassifier(max_iter=5, tol=1e-03)
@@ -29,7 +32,11 @@ def test_optuna_search(enable_pruning):
     with pytest.raises(NotFittedError):
         optuna_search._check_is_fitted()
 
-    optuna_search.fit(X, y)
+    if fit_params == 'coef_init' and not enable_pruning:
+        optuna_search.fit(X, y, coef_init=np.ones((3, 2), dtype=np.float64))
+    else:
+        optuna_search.fit(X, y)
+
     optuna_search.trials_dataframe()
     optuna_search.decision_function(X)
     optuna_search.predict(X)