Gridsearch return nan as score value

### Describe the bug

I'm trying to train random forest regressor and Greadsearchcv in JupyterLab. Whenever I tried to tune the model, it return NaN as the scoring result. 

> In Google Colab, the score value was returned.

When I set `error_score='raise'`, I have the following output:
```
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
Input In [62], in <cell line: 1>()
----> 1 rfr_tune_cu.fit(X_train, y_train)

File /usr/local/lib/python3.9/dist-packages/sklearn/model_selection/_search.py:875, in BaseSearchCV.fit(self, X, y, groups, **fit_params)
    869     results = self._format_results(
    870         all_candidate_params, n_splits, all_out, all_more_results
    871     )
    873     return results
--> 875 self._run_search(evaluate_candidates)
    877 # multimetric is determined here because in the case of a callable
    878 # self.scoring the return type is only known after calling
    879 first_test_score = all_out[0]["test_scores"]

File /usr/local/lib/python3.9/dist-packages/sklearn/model_selection/_search.py:1375, in GridSearchCV._run_search(self, evaluate_candidates)
   1373 def _run_search(self, evaluate_candidates):
   1374     """Search all candidates in param_grid"""
-> 1375     evaluate_candidates(ParameterGrid(self.param_grid))

File /usr/local/lib/python3.9/dist-packages/sklearn/model_selection/_search.py:822, in BaseSearchCV.fit.<locals>.evaluate_candidates(candidate_params, cv, more_results)
    814 if self.verbose > 0:
    815     print(
    816         "Fitting {0} folds for each of {1} candidates,"
    817         " totalling {2} fits".format(
    818             n_splits, n_candidates, n_candidates * n_splits
    819         )
    820     )
--> 822 out = parallel(
    823     delayed(_fit_and_score)(
    824         clone(base_estimator),
    825         X,
    826         y,
    827         train=train,
    828         test=test,
    829         parameters=parameters,
    830         split_progress=(split_idx, n_splits),
    831         candidate_progress=(cand_idx, n_candidates),
    832         **fit_and_score_kwargs,
    833     )
    834     for (cand_idx, parameters), (split_idx, (train, test)) in product(
    835         enumerate(candidate_params), enumerate(cv.split(X, y, groups))
    836     )
    837 )
    839 if len(out) < 1:
    840     raise ValueError(
    841         "No fits were performed. "
    842         "Was the CV iterator empty? "
    843         "Were there no candidates?"
    844     )

File /usr/local/lib/python3.9/dist-packages/joblib/parallel.py:1043, in Parallel.__call__(self, iterable)
   1034 try:
   1035     # Only set self._iterating to True if at least a batch
   1036     # was dispatched. In particular this covers the edge
   (...)
   1040     # was very quick and its callback already dispatched all the
   1041     # remaining jobs.
   1042     self._iterating = False
-> 1043     if self.dispatch_one_batch(iterator):
   1044         self._iterating = self._original_iterator is not None
   1046     while self.dispatch_one_batch(iterator):

File /usr/local/lib/python3.9/dist-packages/joblib/parallel.py:861, in Parallel.dispatch_one_batch(self, iterator)
    859     return False
    860 else:
--> 861     self._dispatch(tasks)
    862     return True

File /usr/local/lib/python3.9/dist-packages/joblib/parallel.py:779, in Parallel._dispatch(self, batch)
    777 with self._lock:
    778     job_idx = len(self._jobs)
--> 779     job = self._backend.apply_async(batch, callback=cb)
    780     # A job can complete so quickly than its callback is
    781     # called before we get here, causing self._jobs to
    782     # grow. To ensure correct results ordering, .insert is
    783     # used (rather than .append) in the following line
    784     self._jobs.insert(job_idx, job)

File /usr/local/lib/python3.9/dist-packages/joblib/_parallel_backends.py:208, in SequentialBackend.apply_async(self, func, callback)
    206 def apply_async(self, func, callback=None):
    207     """Schedule a func to be run"""
--> 208     result = ImmediateResult(func)
    209     if callback:
    210         callback(result)

File /usr/local/lib/python3.9/dist-packages/joblib/_parallel_backends.py:572, in ImmediateResult.__init__(self, batch)
    569 def __init__(self, batch):
    570     # Don't delay the application, to avoid keeping the input
    571     # arguments in memory
--> 572     self.results = batch()

File /usr/local/lib/python3.9/dist-packages/joblib/parallel.py:262, in BatchedCalls.__call__(self)
    258 def __call__(self):
    259     # Set the default nested backend to self._backend but do not set the
    260     # change the default number of processes to -1
    261     with parallel_backend(self._backend, n_jobs=self._n_jobs):
--> 262         return [func(*args, **kwargs)
    263                 for func, args, kwargs in self.items]

File /usr/local/lib/python3.9/dist-packages/joblib/parallel.py:262, in <listcomp>(.0)
    258 def __call__(self):
    259     # Set the default nested backend to self._backend but do not set the
    260     # change the default number of processes to -1
    261     with parallel_backend(self._backend, n_jobs=self._n_jobs):
--> 262         return [func(*args, **kwargs)
    263                 for func, args, kwargs in self.items]

File /usr/local/lib/python3.9/dist-packages/sklearn/utils/fixes.py:117, in _FuncWrapper.__call__(self, *args, **kwargs)
    115 def __call__(self, *args, **kwargs):
    116     with config_context(**self.config):
--> 117         return self.function(*args, **kwargs)

File /usr/local/lib/python3.9/dist-packages/sklearn/model_selection/_validation.py:686, in _fit_and_score(estimator, X, y, scorer, train, test, verbose, parameters, fit_params, return_train_score, return_parameters, return_n_test_samples, return_times, return_estimator, split_progress, candidate_progress, error_score)
    684         estimator.fit(X_train, **fit_params)
    685     else:
--> 686         estimator.fit(X_train, y_train, **fit_params)
    688 except Exception:
    689     # Note fit time as time until error
    690     fit_time = time.time() - start_time

File /usr/local/lib/python3.9/dist-packages/nvtx/nvtx.py:101, in annotate.__call__.<locals>.inner(*args, **kwargs)
     98 @wraps(func)
     99 def inner(*args, **kwargs):
    100     libnvtx_push_range(self.attributes, self.domain.handle)
--> 101     result = func(*args, **kwargs)
    102     libnvtx_pop_range(self.domain.handle)
    103     return result

File /usr/local/lib/python3.9/dist-packages/cuml/internals/api_decorators.py:415, in BaseReturnAnyDecorator.__call__.<locals>.inner(*args, **kwargs)
    411 @wraps(func)
    412 def inner(*args, **kwargs):
    414     with self._recreate_cm(func, args):
--> 415         return func(*args, **kwargs)

File randomforestregressor.pyx:459, in cuml.ensemble.randomforestregressor.RandomForestRegressor.fit()

RuntimeError: CUDA error encountered at: file=/project/cpp/src/decisiontree/batched-levelalgo/quantiles.cuh line=52: call='cub::DeviceRadixSort::SortKeys( nullptr, temp_storage_bytes, data, sorted_column.data(), n_rows, 0, 8 * sizeof(T), stream)', Reason=cudaErrorInvalidDeviceFunction:invalid device function
Obtained 64 stack frames
#0 in /usr/local/lib/python3.9/dist-packages/cuml/common/../libcuml++.so(_ZN4raft9exception18collect_call_stackEv+0x38) [0x7f077ca11438]
#1 in /usr/local/lib/python3.9/dist-packages/cuml/common/../libcuml++.so(_ZN4raft10cuda_errorC1ERKSs+0x38) [0x7f077ca12708]
#2 in /usr/local/lib/python3.9/dist-packages/cuml/common/../libcuml++.so(_ZN2ML2DT16computeQuantilesIfEEDaRKN4raft8handle_tEPKT_iii+0xe7d) [0x7f077d3ece9d]
#3 in /usr/local/lib/python3.9/dist-packages/cuml/common/../libcuml++.so(_ZN2ML12RandomForestIffE3fitERKN4raft8handle_tEPKfiiPfiRPNS_20RandomForestMetaDataIffEE+0x264) [0x7f077d3ee384]
#4 in /usr/local/lib/python3.9/dist-packages/cuml/common/../libcuml++.so(_ZN2ML3fitERKN4raft8handle_tERPNS_20RandomForestMetaDataIffEEPfiiS8_NS_9RF_paramsEi+0x1e3) [0x7f077d398c93]
#5 in /usr/local/lib/python3.9/dist-packages/cuml/ensemble/randomforestregressor.cpython-39-x86_64-linux-gnu.so(+0x32fb7) [0x7f076958dfb7]
#6 in /usr/local/bin/python3(PyObject_Call+0x20b) [0x629d6b]
#7 in /usr/local/bin/python3(_PyEval_EvalFrameDefault+0x3cb1) [0x5adc51]
#8 in /usr/local/bin/python3(_PyFunction_Vectorcall+0x404) [0x62a714]
#9 in /usr/local/bin/python3(PyObject_Call+0x1c2) [0x629d22]
#10 in /usr/local/bin/python3(_PyEval_EvalFrameDefault+0x3cb1) [0x5adc51]
#11 in /usr/local/bin/python3(_PyFunction_Vectorcall+0x404) [0x62a714]
#12 in /usr/local/bin/python3() [0x547058]
#13 in /usr/local/bin/python3(PyObject_Call+0x1c2) [0x629d22]
#14 in /usr/local/bin/python3(_PyEval_EvalFrameDefault+0x3cb1) [0x5adc51]
#15 in /usr/local/bin/python3(_PyFunction_Vectorcall+0x404) [0x62a714]
#16 in /usr/local/bin/python3(PyObject_Call+0xac) [0x629c0c]
#17 in /usr/local/bin/python3(_PyEval_EvalFrameDefault+0x3cb1) [0x5adc51]
#18 in /usr/local/bin/python3(_PyFunction_Vectorcall+0x404) [0x62a714]
#19 in /usr/local/bin/python3(_PyObject_FastCallDictTstate+0xb8) [0x62d798]
#20 in /usr/local/bin/python3(_PyObject_Call_Prepend+0x5a) [0x62d86a]
#21 in /usr/local/bin/python3() [0x6e0be7]
#22 in /usr/local/bin/python3(PyObject_Call+0x20b) [0x629d6b]
#23 in /usr/local/bin/python3(_PyEval_EvalFrameDefault+0x3cb1) [0x5adc51]
#24 in /usr/local/bin/python3(_PyFunction_Vectorcall+0x404) [0x62a714]
#25 in /usr/local/bin/python3(_PyEval_EvalFrameDefault+0x39f) [0x5aa33f]
#26 in /usr/local/bin/python3() [0x629f53]
#27 in /usr/local/bin/python3(_PyFunction_Vectorcall+0x7f) [0x62a38f]
#28 in /usr/local/bin/python3(_PyObject_FastCallDictTstate+0x59) [0x62d739]
#29 in /usr/local/bin/python3(_PyObject_Call_Prepend+0x5a) [0x62d86a]
#30 in /usr/local/bin/python3() [0x6e0be7]
#31 in /usr/local/bin/python3(_PyObject_MakeTpCall+0x33e) [0x62bd6e]
#32 in /usr/local/bin/python3(_PyEval_EvalFrameDefault+0x4a14) [0x5ae9b4]
#33 in /usr/local/bin/python3() [0x629f53]
#34 in /usr/local/bin/python3() [0x5d6a00]
#35 in /usr/local/bin/python3(_PyObject_MakeTpCall+0x3a3) [0x62bdd3]
#36 in /usr/local/bin/python3(_PyEval_EvalFrameDefault+0x4a14) [0x5ae9b4]
#37 in /usr/local/bin/python3() [0x5a8c44]
#38 in /usr/local/bin/python3() [0x546fc1]
#39 in /usr/local/bin/python3(_PyEval_EvalFrameDefault+0x113e) [0x5ab0de]
#40 in /usr/local/bin/python3() [0x629f53]
#41 in /usr/local/bin/python3(_PyFunction_Vectorcall+0x7f) [0x62a38f]
#42 in /usr/local/bin/python3(_PyEval_EvalFrameDefault+0x635) [0x5aa5d5]
#43 in /usr/local/bin/python3() [0x629f53]
#44 in /usr/local/bin/python3(_PyFunction_Vectorcall+0x7f) [0x62a38f]
#45 in /usr/local/bin/python3(_PyEval_EvalFrameDefault+0x635) [0x5aa5d5]
#46 in /usr/local/bin/python3(_PyFunction_Vectorcall+0x404) [0x62a714]
#47 in /usr/local/bin/python3(_PyObject_FastCallDictTstate+0x59) [0x62d739]
#48 in /usr/local/bin/python3(_PyObject_Call_Prepend+0x5a) [0x62d86a]
#49 in /usr/local/bin/python3() [0x6e0be7]
#50 in /usr/local/bin/python3(_PyObject_MakeTpCall+0x2aa) [0x62bcda]
#51 in /usr/local/bin/python3(_PyEval_EvalFrameDefault+0x4a14) [0x5ae9b4]
#52 in /usr/local/bin/python3(_PyFunction_Vectorcall+0x404) [0x62a714]
#53 in /usr/local/bin/python3(_PyEval_EvalFrameDefault+0x39f) [0x5aa33f]
#54 in /usr/local/bin/python3() [0x629f53]
#55 in /usr/local/bin/python3(_PyFunction_Vectorcall+0x7f) [0x62a38f]
#56 in /usr/local/bin/python3(_PyEval_EvalFrameDefault+0x635) [0x5aa5d5]
#57 in /usr/local/bin/python3(_PyFunction_Vectorcall+0x404) [0x62a714]
#58 in /usr/local/bin/python3(_PyEval_EvalFrameDefault+0x635) [0x5aa5d5]
#59 in /usr/local/bin/python3() [0x5a8c44]
#60 in /usr/local/bin/python3(_PyEval_EvalCodeWithName+0x4b) [0x5a9b2b]
#61 in /usr/local/bin/python3(PyEval_EvalCode+0x27) [0x6d0007]
#62 in /usr/local/bin/python3() [0x646bc1]
#63 in /usr/local/bin/python3() [0x53939f]


```





 





### Steps/Code to Reproduce

```
from cuml.ensemble import RandomForestRegressor as curfr
from sklearn.model_selection import GridSearchCV

param_grid2 = {
    #'split_criterion': ['mse', 'poisson', 'gamma', 'inverse_gaussian'],
    'bootstrap': [True, False],
    'max_depth': [16, 30, 50, 80, 90, 100 ],
    'max_features': ['auto', 'sqrt', 'log2'],
    'min_samples_leaf': [1, 3, 4, 5],
    'min_samples_split': [2,4, 8, 10, 12],
    'n_estimators': [100, 200, 300, 400 ,500], # 1000
}

# Create a based model
rfr_cu = curfr() 

# Instantiate the grid search model
rfr_tune_cu = GridSearchCV(rfr_cu , param_grid2, cv = 3, n_jobs = 1,   verbose = 3, #error_score='raise',
                           return_train_score =False )

rfr_tune_cu.fit(X_train, y_train)



```

### Expected Results

```
Fitting 3 folds for each of 3600 candidates, totalling 10800 fits
[CV 1/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=2, n_estimators=100;, 
=> score=value
 total time=   0.0s
........
```

### Actual Results

```
Fitting 3 folds for each of 3600 candidates, totalling 10800 fits
[CV 1/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=2, n_estimators=100;, score=nan total time=   0.0s
[CV 2/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=2, n_estimators=100;, score=nan total time=   0.0s
[CV 3/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=2, n_estimators=100;, score=nan total time=   0.0s
[CV 1/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=2, n_estimators=200;, score=nan total time=   0.0s
[CV 2/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=2, n_estimators=200;, score=nan total time=   0.0s
[CV 3/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=2, n_estimators=200;, score=nan total time=   0.0s
[CV 1/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=2, n_estimators=300;, score=nan total time=   0.0s
[CV 2/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=2, n_estimators=300;, score=nan total time=   0.0s
[CV 3/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=2, n_estimators=300;, score=nan total time=   0.0s
[CV 1/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=2, n_estimators=400;, score=nan total time=   0.0s
[CV 2/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=2, n_estimators=400;, score=nan total time=   0.0s
[CV 3/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=2, n_estimators=400;, score=nan total time=   0.0s
[CV 1/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=2, n_estimators=500;, score=nan total time=   0.0s
[CV 2/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=2, n_estimators=500;, score=nan total time=   0.0s
[CV 3/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=2, n_estimators=500;, score=nan total time=   0.0s
[CV 1/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=100;, score=nan total time=   0.0s
[CV 2/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=100;, score=nan total time=   0.0s
[CV 3/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=100;, score=nan total time=   0.0s
[CV 1/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=200;, score=nan total time=   0.0s
[CV 2/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=200;, score=nan total time=   0.0s
[CV 3/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=200;, score=nan total time=   0.0s
[CV 1/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=300;, score=nan total time=   0.0s
[CV 2/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=300;, score=nan total time=   0.0s
[CV 3/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=300;, score=nan total time=   0.0s
[CV 1/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=400;, score=nan total time=   0.0s
[CV 2/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=400;, score=nan total time=   0.0s
[CV 3/3] END bootstrap=True, max_depth=16, max_features=auto, min_samples_leaf=1, min_samples_split=4, n_estimators=400;, score=nan total time=   0.0s

```

### Versions

```shell
System:
    python: 3.9.13 (main, May 23 2022, 22:01:06)  [GCC 9.4.0]
executable: /usr/local/bin/python3
   machine: Linux-5.4.0-122-generic-x86_64-with-glibc2.31

Python dependencies:
      sklearn: 1.1.1
          pip: 22.2.2
   setuptools: 63.1.0
        numpy: 1.23.1
        scipy: 1.8.1
       Cython: 0.29.30
       pandas: 1.4.3
   matplotlib: 3.5.2
       joblib: 1.1.0
threadpoolctl: 3.1.0

Built with OpenMP: True

threadpoolctl info:
       user_api: blas
   internal_api: openblas
         prefix: libopenblas
       filepath: /usr/local/lib/python3.9/dist-packages/numpy.libs/libopenblas64_p-r0-742d56dc.3.20.so
        version: 0.3.20
threading_layer: pthreads
   architecture: Haswell
    num_threads: 8

       user_api: openmp
   internal_api: openmp
         prefix: libgomp
       filepath: /usr/local/lib/python3.9/dist-packages/scikit_learn.libs/libgomp-a34b3233.so.1.0.0
        version: None
    num_threads: 8

       user_api: blas
   internal_api: openblas
         prefix: libopenblas
       filepath: /usr/local/lib/python3.9/dist-packages/scipy.libs/libopenblasp-r0-8b9e111f.3.17.so
        version: 0.3.17
threading_layer: pthreads
   architecture: Haswell
    num_threads: 8

       user_api: openmp
   internal_api: openmp
         prefix: libgomp
       filepath: /usr/lib/x86_64-linux-gnu/libgomp.so.1.0.0
        version: None
    num_threads: 8
```

COMMENT:
You get `nan` because the fitting of `RandomForest` is failing. This is a normal behaviour.
Then you are using the `RandomForest` from CuML (https://github.com/rapidsai/cuml)

```pytb
RuntimeError: CUDA error encountered at: file=/project/cpp/src/decisiontree/batched-levelalgo/quantiles.cuh line=52: call='cub::DeviceRadixSort::SortKeys( nullptr, temp_storage_bytes, data, sorted_column.data(), n_rows, 0, 8 * sizeof(T), stream)', Reason=cudaErrorInvalidDeviceFunction:invalid device function
```

So this is not related to scikit-learn. Check that you use CuML as expected otherwise you should open an issue on their issue tracker if this is a bug.

Labels: Bug, Needs Triage