Repository: automl/auto-sklearn

Issue:
Bug in _partial_fit_and_predict_iterative in train_evaluator.py?
Hi, 

Working with [AutoSklearn2Classifier](https://github.com/automl/auto-sklearn/blob/b2ac331c500ebef7becf372802493a7b235f7cec/autosklearn/experimental/askl2.py#L180), I recently got a strange error that might be due to a bug. This is what I am running:

```python
    n_cores = 10
    memory_limit= psutil.virtual_memory().available * 10**-6 // n_cores
    time_left_for_this_task = 7200

    predictor = AutoSklearn2Classifier(memory_limit=memory_limit,
                                       metric=autosklearn_metrics.roc_auc,
                                       n_jobs=n_cores,
                                       time_left_for_this_task=time_left_for_this_task
                                      )

    predictor.fit(x_train, y_train)
    predictor.automl_.runhistory_.data
```

where `x_train` is a pandas dataframe and `y_train` a list. Even though `fit` runs, after inspecting `runhistory_.data` it is clear that all models are crashing in line 741 of /autosklearn/evaluation/train_evaluator.py. It seems like `self.Y_train` in line 741 should be `self.X_train` instead:

https://github.com/automl/auto-sklearn/blob/b2ac331c500ebef7becf372802493a7b235f7cec/autosklearn/evaluation/train_evaluator.py#L738-L744

I found that a workaround for this issue is providing `x_train` as numpy. Also tried providing `x_train` as a list of lists or `y_train` as pandas series, but neither of them solved the issue. I am using auto-sklearn==0.14.7.

I am wondering if this is the expected behavior or actually a bug. Thanks!


Discussion:
This is a bug and should definitely be fixed, thanks for finding this!

Resolving patch:
diff --git a/autosklearn/evaluation/train_evaluator.py b/autosklearn/evaluation/train_evaluator.py
--- a/autosklearn/evaluation/train_evaluator.py
+++ b/autosklearn/evaluation/train_evaluator.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional, Sequence, Tuple, Union, cast
+from typing import Any, Dict, List, Optional, Sequence, Tuple, TypeVar, Union, cast
 
 import copy
 import json
@@ -67,6 +67,14 @@ def _get_y_array(y: SUPPORTED_TARGET_TYPES, task_type: int) -> SUPPORTED_TARGET_
         return y
 
 
+T = TypeVar("T", SUPPORTED_FEAT_TYPES, SUPPORTED_TARGET_TYPES)
+
+
+def select(data: T, indices: np.ndarray) -> T:
+    """Select into some data by indices"""
+    return data.iloc[indices] if hasattr(data, "iloc") else data[indices]
+
+
 def subsample_indices(
     train_indices: List[int],
     subsample: Optional[float],
@@ -86,11 +94,7 @@ def subsample_indices(
         # required to subsample because otherwise scikit-learn will complain
 
         if task_type in CLASSIFICATION_TASKS and task_type != MULTILABEL_CLASSIFICATION:
-            stratify: Optional[SUPPORTED_TARGET_TYPES] = (
-                Y_train.iloc[train_indices]
-                if hasattr(Y_train, "iloc")
-                else Y_train[train_indices]
-            )
+            stratify: Optional[SUPPORTED_TARGET_TYPES] = select(Y_train, train_indices)
         else:
             stratify = None
 
@@ -118,50 +122,24 @@ def _fit_with_budget(
     train_indices: List[int],
     task_type: int,
 ) -> None:
-    if (
-        budget_type == "iterations"
-        or budget_type == "mixed"
-        and model.estimator_supports_iterative_fit()
+    if budget_type == "iterations" or (
+        budget_type == "mixed" and model.estimator_supports_iterative_fit()
     ):
+        X = select(X_train, train_indices)
+        y = select(Y_train, train_indices)
+
         if model.estimator_supports_iterative_fit():
             budget_factor = model.get_max_iter()
-            Xt, fit_params = model.fit_transformer(
-                X_train.iloc[train_indices]
-                if hasattr(X_train, "iloc")
-                else X_train[train_indices],
-                Y_train.iloc[train_indices]
-                if hasattr(Y_train, "iloc")
-                else Y_train[train_indices],
-            )
+            Xt, fit_params = model.fit_transformer(X, y)
 
             n_iter = int(np.ceil(budget / 100 * budget_factor))
-            model.iterative_fit(
-                Xt,
-                Y_train.iloc[train_indices]
-                if hasattr(Y_train, "iloc")
-                else Y_train[train_indices],
-                n_iter=n_iter,
-                refit=True,
-                **fit_params,
-            )
+            model.iterative_fit(Xt, y, n_iter=n_iter, refit=True, **fit_params)
         else:
-            _fit_and_suppress_warnings(
-                logger,
-                model,
-                X_train.iloc[train_indices]
-                if hasattr(X_train, "iloc")
-                else X_train[train_indices],
-                Y_train.iloc[train_indices]
-                if hasattr(Y_train, "iloc")
-                else Y_train[train_indices],
-            )
+            _fit_and_suppress_warnings(logger, model, X, y)
 
-    elif (
-        budget_type == "subsample"
-        or budget_type == "mixed"
-        and not model.estimator_supports_iterative_fit()
+    elif budget_type == "subsample" or (
+        budget_type == "mixed" and not model.estimator_supports_iterative_fit()
     ):
-
         subsample = budget / 100
         train_indices_subset = subsample_indices(
             train_indices,
@@ -169,12 +147,9 @@ def _fit_with_budget(
             task_type,
             Y_train,
         )
-        _fit_and_suppress_warnings(
-            logger,
-            model,
-            X_train[train_indices_subset],
-            Y_train[train_indices_subset],
-        )
+        X = select(X_train, train_indices_subset)
+        y = select(Y_train, train_indices_subset)
+        _fit_and_suppress_warnings(logger, model, X, y)
 
     else:
         raise ValueError(budget_type)
@@ -289,13 +264,9 @@ def fit_predict_and_loss(self, iterative: bool = False) -> None:
                     self.Y_train,
                     groups=self.resampling_strategy_args.get("groups"),
                 ):
-                    self.X_optimization = (
-                        self.X_train.iloc[test_split]
-                        if hasattr(self.X_train, "iloc")
-                        else self.X_train[test_split]
-                    )
-                    self.Y_optimization = self.Y_train[test_split]
-                    self.Y_actual_train = self.Y_train[train_split]
+                    self.X_optimization = select(self.X_train, test_split)
+                    self.Y_optimization = select(self.Y_train, test_split)
+                    self.Y_actual_train = select(self.Y_train, train_split)
                     self._partial_fit_and_predict_iterative(
                         0,
                         train_indices=train_split,
@@ -384,37 +355,27 @@ def fit_predict_and_loss(self, iterative: bool = False) -> None:
                         model = self.models[i]
 
                         if iterations[i] == 1:
-                            self.Y_train_targets[train_indices] = (
-                                self.Y_train.iloc[train_indices]
-                                if hasattr(self.Y_train, "iloc")
-                                else self.Y_train[train_indices]
-                            )
-                            self.X_targets[i] = (
-                                self.X_train.iloc[test_indices]
-                                if hasattr(self.X_train, "iloc")
-                                else self.X_train[train_indices]
+                            self.Y_train_targets[train_indices] = select(
+                                self.Y_train, train_indices
                             )
+                            self.X_targets[i] = select(self.X_train, test_indices)
+                            self.Y_targets[i] = select(self.Y_train, test_indices)
 
-                            self.Y_targets[i] = self.Y_train[test_indices]
-
+                            # Note: Be careful moving these into variables, caused a
+                            # headache when trying to debug why things were breaking
                             Xt, fit_params = model.fit_transformer(
-                                self.X_train.iloc[train_indices]
-                                if hasattr(self.X_train, "iloc")
-                                else self.X_train[train_indices],
-                                self.Y_train.iloc[train_indices]
-                                if hasattr(self.Y_train, "iloc")
-                                else self.Y_train[train_indices],
+                                select(self.X_train, train_indices),
+                                select(self.Y_train, train_indices),
                             )
                             Xt_array[i] = Xt
                             fit_params_array[i] = fit_params
+
                         n_iter = int(2 ** iterations[i] / 2) if iterations[i] > 1 else 2
                         total_n_iterations[i] = total_n_iterations[i] + n_iter
 
                         model.iterative_fit(
                             Xt_array[i],
-                            self.Y_train.iloc[train_indices]
-                            if hasattr(self.Y_train, "iloc")
-                            else self.Y_train[train_indices],
+                            select(self.Y_train, train_indices),
                             n_iter=n_iter,
                             **fit_params_array[i],
                         )
@@ -431,9 +392,7 @@ def fit_predict_and_loss(self, iterative: bool = False) -> None:
                         train_splits[i] = train_indices
 
                         train_loss = self._loss(
-                            self.Y_train.iloc[train_indices]
-                            if hasattr(self.Y_train, "iloc")
-                            else self.Y_train[train_indices],
+                            select(self.Y_train, train_indices),
                             train_pred,
                             X_data=Xt_array[i],
                         )
@@ -608,13 +567,10 @@ def fit_predict_and_loss(self, iterative: bool = False) -> None:
                 Y_test_pred[i] = test_pred
                 train_splits[i] = train_split
 
-                train_loss = self._loss(
-                    self.Y_train_targets[train_split],
-                    train_pred,
-                    X_data=self.X_train.iloc[train_split]
-                    if hasattr(self.X_train, "iloc")
-                    else self.X_train[train_split],
-                )
+                X = select(self.X_train, train_split)
+                y = select(self.Y_train_targets, train_split)
+
+                train_loss = self._loss(y, train_pred, X_data=X)
                 train_losses.append(train_loss)
                 # number of training data points for this fold. Used for weighting
                 # the average.
@@ -622,7 +578,9 @@ def fit_predict_and_loss(self, iterative: bool = False) -> None:
 
                 # Compute validation loss of this fold and store it.
                 optimization_loss = self._loss(
-                    self.Y_targets[i], opt_pred, X_data=self.X_targets[i]
+                    self.Y_targets[i],
+                    opt_pred,
+                    X_data=self.X_targets[i],
                 )
                 opt_losses.append(optimization_loss)
                 # number of optimization data points for this fold. Used for weighting
@@ -745,9 +703,9 @@ def partial_fit_predict_and_loss(self, fold: int, iterative: bool = False) -> No
                 break
 
         if self.num_cv_folds > 1:
-            self.X_optimization = self.X_train[test_split]
-            self.Y_optimization = self.Y_train[test_split]
-            self.Y_actual_train = self.Y_train[train_split]
+            self.X_optimization = select(self.X_train, test_split)
+            self.Y_optimization = select(self.Y_train, test_split)
+            self.Y_actual_train = select(self.Y_train, train_split)
 
         if iterative:
             self._partial_fit_and_predict_iterative(
@@ -770,8 +728,12 @@ def partial_fit_predict_and_loss(self, fold: int, iterative: bool = False) -> No
                 test_indices=test_split,
                 add_model_to_self=True,
             )
-            train_loss = self._loss(self.Y_actual_train, train_pred)
-            loss = self._loss(self.Y_targets[fold], opt_pred)
+
+            # This is my best guess at what the X_data for these should be
+            X_train = select(self.X_train, train_split)  # From above (only cv?)
+            X_fold = self.X_targets[fold]
+            train_loss = self._loss(self.Y_actual_train, train_pred, X_data=X_train)
+            loss = self._loss(self.Y_targets[fold], opt_pred, X_data=X_fold)
 
             if self.model.estimator_supports_iterative_fit():
                 model_max_iter = self.model.get_max_iter()
@@ -811,20 +773,15 @@ def _partial_fit_and_predict_iterative(
         file_output = True if self.num_cv_folds == 1 else False
 
         if model.estimator_supports_iterative_fit():
-            Xt, fit_params = model.fit_transformer(
-                self.X_train.iloc[train_indices]
-                if hasattr(self.Y_train, "iloc")
-                else self.X_train[train_indices],
-                self.Y_train.iloc[train_indices]
-                if hasattr(self.Y_train, "iloc")
-                else self.Y_train[train_indices],
-            )
+            X = select(self.X_train, train_indices)
+            y = select(self.Y_train, train_indices)
 
-            self.Y_train_targets[train_indices] = (
-                self.Y_train.iloc[train_indices]
-                if hasattr(self.Y_train, "iloc")
-                else self.Y_train[train_indices]
-            )
+            X_test = select(self.X_train, test_indices)
+            y_test = select(self.Y_train, test_indices)
+
+            Xt, fit_params = model.fit_transformer(X, y)
+
+            self.Y_train_targets[train_indices] = y
 
             iteration = 1
             total_n_iteration = 0
@@ -842,14 +799,8 @@ def _partial_fit_and_predict_iterative(
             ):
                 n_iter = int(2**iteration / 2) if iteration > 1 else 2
                 total_n_iteration += n_iter
-                model.iterative_fit(
-                    Xt,
-                    self.Y_train.iloc[train_indices]
-                    if hasattr(self.Y_train, "iloc")
-                    else self.Y_train[train_indices],
-                    n_iter=n_iter,
-                    **fit_params,
-                )
+                model.iterative_fit(Xt, y, n_iter=n_iter, **fit_params)
+
                 (Y_train_pred, Y_optimization_pred, Y_test_pred,) = self._predict(
                     model,
                     train_indices=train_indices,
@@ -859,13 +810,9 @@ def _partial_fit_and_predict_iterative(
                 if add_model_to_self:
                     self.model = model
 
-                train_loss = self._loss(
-                    self.Y_train.iloc[train_indices]
-                    if hasattr(self.Y_train, "iloc")
-                    else self.Y_train[train_indices],
-                    Y_train_pred,
-                )
-                loss = self._loss(self.Y_train[test_indices], Y_optimization_pred)
+                train_loss = self._loss(y, Y_train_pred, X_data=X)
+                loss = self._loss(y_test, Y_optimization_pred, X_data=X_test)
+
                 additional_run_info = model.get_additional_run_info()
 
                 model_current_iter = model.get_current_iter()
@@ -902,13 +849,15 @@ def _partial_fit_and_predict_iterative(
             ) = self._partial_fit_and_predict_standard(
                 fold, train_indices, test_indices, add_model_to_self
             )
-            train_loss = self._loss(
-                self.Y_train.iloc[train_indices]
-                if hasattr(self.Y_train, "iloc")
-                else self.Y_train[train_indices],
-                Y_train_pred,
-            )
-            loss = self._loss(self.Y_train[test_indices], Y_optimization_pred)
+
+            X = select(self.X_train, train_indices)
+            y = select(self.Y_train, train_indices)
+            train_loss = self._loss(y, Y_train_pred, X_data=X)
+
+            X_test = select(self.X_train, test_indices)
+            y_test = select(self.Y_train, test_indices)
+            loss = self._loss(y_test, Y_optimization_pred, X_data=X_test)
+
             if self.model.estimator_supports_iterative_fit():
                 model_max_iter = self.model.get_max_iter()
                 model_current_iter = self.model.get_current_iter()
@@ -946,37 +895,18 @@ def _partial_fit_and_predict_standard(
 
         self.indices[fold] = (train_indices, test_indices)
 
-        _fit_and_suppress_warnings(
-            self.logger,
-            model,
-            self.X_train.iloc[train_indices]
-            if hasattr(self.X_train, "iloc")
-            else self.X_train[train_indices],
-            self.Y_train.iloc[train_indices]
-            if hasattr(self.Y_train, "iloc")
-            else self.Y_train[train_indices],
-        )
+        X = select(self.X_train, train_indices)
+        y = select(self.Y_train, train_indices)
+        _fit_and_suppress_warnings(self.logger, model, X, y)
 
         if add_model_to_self:
             self.model = model
         else:
             self.models[fold] = model
 
-        self.X_targets[fold] = (
-            self.X_train.iloc[test_indices]
-            if hasattr(self.X_train, "iloc")
-            else self.X_train[test_indices]
-        )
-        self.Y_targets[fold] = (
-            self.Y_train.iloc[test_indices]
-            if hasattr(self.Y_train, "iloc")
-            else self.Y_train[test_indices]
-        )
-        self.Y_train_targets[train_indices] = (
-            self.Y_train.iloc[train_indices]
-            if hasattr(self.Y_train, "iloc")
-            else self.Y_train[train_indices]
-        )
+        self.X_targets[fold] = select(self.X_train, test_indices)
+        self.Y_targets[fold] = select(self.Y_train, test_indices)
+        self.Y_train_targets[train_indices] = select(self.Y_train, train_indices)
 
         train_pred, opt_pred, test_pred = self._predict(
             model=model,
@@ -1010,13 +940,9 @@ def _partial_fit_and_predict_budget(
 
         model = self._get_model(self.feat_type)
         self.indices[fold] = (train_indices, test_indices)
-        self.X_targets[fold] = self.X_train[test_indices]
-        self.Y_targets[fold] = self.Y_train[test_indices]
-        self.Y_train_targets[train_indices] = (
-            self.Y_train.iloc[train_indices]
-            if hasattr(self.Y_train, "iloc")
-            else self.Y_train[train_indices],
-        )
+        self.X_targets[fold] = select(self.X_train, test_indices)
+        self.Y_targets[fold] = select(self.Y_train, test_indices)
+        self.Y_train_targets[train_indices] = select(self.Y_train, train_indices)
 
         _fit_with_budget(
             X_train=self.X_train,
@@ -1051,38 +977,20 @@ def _partial_fit_and_predict_budget(
     def _predict(
         self, model: BaseEstimator, test_indices: List[int], train_indices: List[int]
     ) -> Tuple[PIPELINE_DATA_DTYPE, PIPELINE_DATA_DTYPE, PIPELINE_DATA_DTYPE]:
-        train_pred = self.predict_function(
-            self.X_train.iloc[train_indices]
-            if hasattr(self.X_train, "iloc")
-            else self.X_train[train_indices],
-            model,
-            self.task_type,
-            self.Y_train.iloc[train_indices]
-            if hasattr(self.Y_train, "iloc")
-            else self.Y_train[train_indices],
-        )
+        y_train = select(self.Y_train, train_indices)
+        X_train = select(self.X_train, train_indices)
+        X_test = select(self.X_train, test_indices)
 
-        opt_pred = self.predict_function(
-            self.X_train.iloc[test_indices]
-            if hasattr(self.X_train, "iloc")
-            else self.X_train[test_indices],
-            model,
-            self.task_type,
-            self.Y_train.iloc[train_indices]
-            if hasattr(self.Y_train, "iloc")
-            else self.Y_train[train_indices],
-        )
+        # The y_train here does not correspond to the X, there to ensure output shape
+        # will be correct in case labels were missing from some split
+        train_pred = self.predict_function(X_train, model, self.task_type, y_train)
+        opt_pred = self.predict_function(X_test, model, self.task_type, y_train)
 
+        # This is the test data the user can pass in
         if self.X_test is not None:
-            X_test = self.X_test.copy()
-            test_pred = self.predict_function(
-                X_test,
-                model,
-                self.task_type,
-                self.Y_train.iloc[train_indices]
-                if hasattr(self.Y_train, "iloc")
-                else self.Y_train[train_indices],
-            )
+            # See comment above about y_train
+            X_user = self.X_test.copy()
+            test_pred = self.predict_function(X_user, model, self.task_type, y_train)
         else:
             test_pred = None
 


Test patch: