Repository: scikit-optimize/scikit-optimize (python)

Issue:
Scikit-optimize master GaussianProcessRegressor is incompatible with scikit-learn 1.0
`scikit-learn` 0.19 depreciated the `y_train_mean` attribute of `GaussianProcessRegressor`. At the time it was made private, and so scikit-optimize continued to rely on it. #467 was never merged to properly fix that.
Scikit-learn 1.0 removed the attribute, breaking scikit-optimize.

**Version**
- `python==3.9.2`
- `scikit-learn==1.0`
- `scikit-optimize==0.9.dev0` (installed with `pip install git+https://github.com/scikit-optimize/scikit-optimize.git`)

**How to reproduce**
```bash
mkdir test
cd test
python3 -m venv venv
source venv/bin/activate
pip install "scikit-learn==1.0" git+https://github.com/scikit-optimize/scikit-optimize.git
```

Write the following script to `test.py`:
```python
import scipy
import sklearn
import skopt
from sklearn import model_selection, svm
from sklearn.datasets import load_digits

print("scipy:", scipy.__version__)
print("scikit-learn:", sklearn.__version__)
print("scikit-optimize:", skopt.__version__)

X, y = load_digits(n_class=10, return_X_y=True)
X_train, X_test, y_train, y_test = model_selection.train_test_split(
    X, y, train_size=0.75, test_size=0.25, random_state=0
)

opt = skopt.BayesSearchCV(
    svm.SVC(),
    {
        "C": (1e-6, 1e6, "log-uniform"),
        "gamma": (1e-6, 1e6, "log-uniform"),
        "kernel": ["linear", "rbf"],
    },
    n_iter=32,
    cv=3,
)

opt.fit(X_train, y_train)
print("Best train score:", opt.best_score_)
print("Tests score:", opt.score(X_test, y_test))
```

and run it in the test venv:
```bash
python test.py
```


Pull request:
[MRG] Fix GaussianProcessRegressor for scikit-learn 1.0
This PR fix compatibility with scikit-learn 1.0:

- Fixes #1061
- Fix tests using outdated scikit-learn API to load datasets

Patch:
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 3b307cc..af8df0e 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -10,9 +10,9 @@ jobs:
       - MINICONDA_PATH: ~/miniconda
       - CONDA_ENV_NAME: testenv
       - PYTHON_VERSION: 3.6
-      - NUMPY_VERSION: 1.13.3
+      - NUMPY_VERSION: 1.14.6
       - SKLEARN_VERSION: 0.20.4
-      - SCIPY_VERSION: 0.19.1
+      - SCIPY_VERSION: 1.1.0
       - MATPLOTLIB_VERSION: 2.1.1
       # on conda, this is the latest for python 3.5
       # The following places need to be in sync with regard to Cython version:
diff --git a/doc/whats_new/v0.9.rst b/doc/whats_new/v0.9.rst
index 2085fe0..6e8e599 100644
--- a/doc/whats_new/v0.9.rst
+++ b/doc/whats_new/v0.9.rst
@@ -14,3 +14,9 @@ Version 0.9.0
   :pr:`988`
 - |API| Deprecate :class:`skopt.searchcv.BayesSearchCV` parameter `iid=`.
     :pr:`988`
+
+:mod:`skopt.learning.gaussian_process.gpr`
+------------------------------------------
+- |Fix| Fix :class:`skopt.learning.gaussian_process.gpr.GaussianProcessRegressor`
+  for scikit-learn >= 1.0
+  :pr:`1063`
\ No newline at end of file
diff --git a/examples/sklearn-gridsearchcv-replacement.py b/examples/sklearn-gridsearchcv-replacement.py
index 2b07281..2d52b87 100644
--- a/examples/sklearn-gridsearchcv-replacement.py
+++ b/examples/sklearn-gridsearchcv-replacement.py
@@ -198,7 +198,7 @@ from skopt import BayesSearchCV
 from sklearn.datasets import load_iris
 from sklearn.svm import SVC
 
-X, y = load_iris(True)
+X, y = load_iris(return_X_y=True)
 
 searchcv = BayesSearchCV(
     SVC(),
diff --git a/skopt/learning/gaussian_process/gpr.py b/skopt/learning/gaussian_process/gpr.py
index 6b13bb6..34902d7 100644
--- a/skopt/learning/gaussian_process/gpr.py
+++ b/skopt/learning/gaussian_process/gpr.py
@@ -224,10 +224,10 @@ class GaussianProcessRegressor(sk_GaussianProcessRegressor):
         self.K_inv_ = L_inv.dot(L_inv.T)
 
         # Fix deprecation warning #462
-        if int(sklearn.__version__[2:4]) >= 23:
+        if sklearn.__version__ >= "0.23":
             self.y_train_std_ = self._y_train_std
             self.y_train_mean_ = self._y_train_mean
-        elif int(sklearn.__version__[2:4]) >= 19:
+        elif sklearn.__version__ >= "0.19":
             self.y_train_mean_ = self._y_train_mean
             self.y_train_std_ = 1
         else:
diff --git a/skopt/searchcv.py b/skopt/searchcv.py
index a37bc79..391b14c 100644
--- a/skopt/searchcv.py
+++ b/skopt/searchcv.py
@@ -157,7 +157,7 @@ class BayesSearchCV(BaseSearchCV):
     >>> from sklearn.svm import SVC
     >>> from sklearn.model_selection import train_test_split
     >>>
-    >>> X, y = load_iris(True)
+    >>> X, y = load_iris(return_X_y=True)
     >>> X_train, X_test, y_train, y_test = train_test_split(X, y,
     ...                                                     train_size=0.75,
     ...                                                     random_state=0)


Test patch:
diff --git a/skopt/tests/test_plots.py b/skopt/tests/test_plots.py
index 3fb57d9..6a909ac 100644
--- a/skopt/tests/test_plots.py
+++ b/skopt/tests/test_plots.py
@@ -39,7 +39,8 @@ def test_plots_work():
                                      **{dim.name: val
                                         for dim, val in zip(SPACE, params)
                                         if dim.name != 'dummy'})
-        return -np.mean(cross_val_score(clf, *load_breast_cancer(True)))
+        X, y = load_breast_cancer(return_X_y=True)
+        return -np.mean(cross_val_score(clf, X, y))
 
     res = gp_minimize(objective, SPACE, n_calls=10, random_state=3)
 
@@ -112,7 +113,8 @@ def test_plots_work_without_cat():
                                      **{dim.name: val
                                         for dim, val in zip(SPACE, params)
                                         if dim.name != 'dummy'})
-        return -np.mean(cross_val_score(clf, *load_breast_cancer(True)))
+        X, y = load_breast_cancer(return_X_y=True)
+        return -np.mean(cross_val_score(clf, X, y))
 
     res = gp_minimize(objective, SPACE, n_calls=10, random_state=3)
     plots.plot_convergence(res)
diff --git a/skopt/tests/test_searchcv.py b/skopt/tests/test_searchcv.py
index 67c15ee..8dcb6f4 100644
--- a/skopt/tests/test_searchcv.py
+++ b/skopt/tests/test_searchcv.py
@@ -93,7 +93,7 @@ def test_searchcv_runs(surrogate, n_jobs, n_points, cv=None):
 
     """
 
-    X, y = load_iris(True)
+    X, y = load_iris(return_X_y=True)
     X_train, X_test, y_train, y_test = train_test_split(
         X, y, train_size=0.75, random_state=0
     )
@@ -140,7 +140,7 @@ def test_searchcv_runs_multiple_subspaces():
     multiple subspaces are given.
     """
 
-    X, y = load_iris(True)
+    X, y = load_iris(return_X_y=True)
     X_train, X_test, y_train, y_test = train_test_split(
         X, y, train_size=0.75, random_state=0
     )
@@ -193,7 +193,7 @@ def test_searchcv_sklearn_compatibility():
     such as clone, set_params, get_params.
     """
 
-    X, y = load_iris(True)
+    X, y = load_iris(return_X_y=True)
     X_train, X_test, y_train, y_test = train_test_split(
         X, y, train_size=0.75, random_state=0
     )
@@ -257,7 +257,7 @@ def test_searchcv_reproducibility():
     random state.
     """
 
-    X, y = load_iris(True)
+    X, y = load_iris(return_X_y=True)
     X_train, X_test, y_train, y_test = train_test_split(
         X, y, train_size=0.75, random_state=0
     )
@@ -305,7 +305,7 @@ def test_searchcv_rank():
     random state.
     """
 
-    X, y = load_iris(True)
+    X, y = load_iris(return_X_y=True)
     X_train, X_test, y_train, y_test = train_test_split(
         X, y, train_size=0.75, random_state=0
     )
@@ -341,7 +341,7 @@ def test_searchcv_refit():
     random state.
     """
 
-    X, y = load_iris(True)
+    X, y = load_iris(return_X_y=True)
     X_train, X_test, y_train, y_test = train_test_split(
         X, y, train_size=0.75, random_state=0
     )
@@ -383,7 +383,7 @@ def test_searchcv_callback():
     # Test whether callback is used in BayesSearchCV and
     # whether is can be used to interrupt the search loop
 
-    X, y = load_iris(True)
+    X, y = load_iris(return_X_y=True)
     opt = BayesSearchCV(
         DecisionTreeClassifier(),
         {