Repository: automl/auto-sklearn

Issue:
Fit fails when input data has categorical columns
## Describe the bug ##
auto-sklearn fails when input data has categorical data. I have changed example_pandas_train_test.py to use OpenML dataset, data_id : 1558 and also updated categorical and numerical list var.  
Changes done:
```
X, y = sklearn.datasets.fetch_openml(data_id=1558, return_X_y=True, as_frame=False)
X = pd.DataFrame(
    data=X,
    columns=['V' + str(i) for i in range(1, 17)]
)
desired_boolean_columns = ['']
desired_categorical_columns = ['V2', 'V3', 'V4', 'V5', 'V7', 'V8', 'V9', 'V11', 'V16']
desired_numerical_columns = ['V1', 'V6', 'V10', 'V12', 'V13', 'V14', 'V15']
```


As per understanding, categorical columns are not encoded. That's why it fails.


## Actual behavior, stacktrace or logfile ##
```
Traceback (most recent call last):
  File "example_pandas_train_test.py", line 105, in <module>
    cls.fit(X_train, y_train, X_test, y_test)
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/estimators.py", line 587, in fit
    dataset_name=dataset_name,
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/estimators.py", line 346, in fit
    self.automl_.fit(load_models=True, **kwargs)
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/automl.py", line 1154, in fit
    load_models=load_models,
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/automl.py", line 607, in fit
    _proc_smac.run_smbo()
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/smbo.py", line 374, in run_smbo
    metalearning_configurations = self.get_metalearning_suggestions()
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/smbo.py", line 584, in get_metalearning_suggestions
    self.datamanager.perform1HotEncoding()
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/data/abstract_data_manager.py", line 94, in perform1HotEncoding
    data=data)
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/data/abstract_data_manager.py", line 28, in perform_one_hot_encoding
    rvals.append(encoder.fit_transform(data[0]))
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/pipeline/components/data_preprocessing/data_preprocessing.py", line 117, in fit_transform
    return self.fit(X, y).transform(X)
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/pipeline/components/data_preprocessing/data_preprocessing.py", line 105, in fit
    self.column_transformer.fit(X)
  File "Desktop/mls/lib/python3.7/site-packages/sklearn/compose/_column_transformer.py", line 484, in fit
    self.fit_transform(X, y=y)
  File "Desktop/mls/lib/python3.7/site-packages/sklearn/compose/_column_transformer.py", line 518, in fit_transform
    result = self._fit_transform(X, y, _fit_transform_one)
  File "Desktop/mls/lib/python3.7/site-packages/sklearn/compose/_column_transformer.py", line 457, in _fit_transform
    self._iter(fitted=fitted, replace_strings=True), 1))
  File "Desktop/mls/lib/python3.7/site-packages/joblib/parallel.py", line 1029, in __call__
    if self.dispatch_one_batch(iterator):
  File "Desktop/mls/lib/python3.7/site-packages/joblib/parallel.py", line 847, in dispatch_one_batch
    self._dispatch(tasks)
  File "Desktop/mls/lib/python3.7/site-packages/joblib/parallel.py", line 765, in _dispatch
    job = self._backend.apply_async(batch, callback=cb)
  File "Desktop/mls/lib/python3.7/site-packages/joblib/_parallel_backends.py", line 206, in apply_async
    result = ImmediateResult(func)
  File "Desktop/mls/lib/python3.7/site-packages/joblib/_parallel_backends.py", line 570, in __init__
    self.results = batch()
  File "Desktop/mls/lib/python3.7/site-packages/joblib/parallel.py", line 253, in __call__
    for func, args, kwargs in self.items]
  File "Desktop/mls/lib/python3.7/site-packages/sklearn/pipeline.py", line 728, in _fit_transform_one
    res = transformer.fit_transform(X, y, **fit_params)
  File "Desktop/mls/lib/python3.7/site-packages/sklearn/pipeline.py", line 385, in fit_transform
    Xt, fit_params = self._fit(X, y, **fit_params)
  File "Desktop/mls/lib/python3.7/site-packages/sklearn/pipeline.py", line 315, in _fit
    **fit_params_steps[name])
  File "Desktop/mls/lib/python3.7/site-packages/joblib/memory.py", line 352, in __call__
    return self.func(*args, **kwargs)
  File "Desktop/mls/lib/python3.7/site-packages/sklearn/pipeline.py", line 728, in _fit_transform_one
    res = transformer.fit_transform(X, y, **fit_params)
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/pipeline/components/data_preprocessing/category_shift/category_shift.py", line 30, in fit_transform
    return self.fit(X, y).transform(X)
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/pipeline/components/data_preprocessing/category_shift/category_shift.py", line 21, in fit
    self.preprocessor.fit(X, y)
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/pipeline/implementations/CategoryShift.py", line 26, in fit
    self._convert_and_check_X(X)
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/pipeline/implementations/CategoryShift.py", line 16, in _convert_and_check_X
    raise ValueError('Categories should be non-negative numbers. '
ValueError: Categories should be non-negative numbers. NOTE: floats will be casted to integers.
```

## Environment and installation: ##

Please give details about your installation:

* Is your installation in a virtual environment or conda environment? virtual environment
* Python version: 3.7
* Auto-sklearn version: 0.10.0

Fit fails when input data has categorical columns
## Describe the bug ##
auto-sklearn fails when input data has categorical data. I have changed example_pandas_train_test.py to use OpenML dataset, data_id : 1558 and also updated categorical and numerical list var.  
Changes done:
```
X, y = sklearn.datasets.fetch_openml(data_id=1558, return_X_y=True, as_frame=False)
X = pd.DataFrame(
    data=X,
    columns=['V' + str(i) for i in range(1, 17)]
)
desired_boolean_columns = ['']
desired_categorical_columns = ['V2', 'V3', 'V4', 'V5', 'V7', 'V8', 'V9', 'V11', 'V16']
desired_numerical_columns = ['V1', 'V6', 'V10', 'V12', 'V13', 'V14', 'V15']
```


As per understanding, categorical columns are not encoded. That's why it fails.


## Actual behavior, stacktrace or logfile ##
```
Traceback (most recent call last):
  File "example_pandas_train_test.py", line 105, in <module>
    cls.fit(X_train, y_train, X_test, y_test)
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/estimators.py", line 587, in fit
    dataset_name=dataset_name,
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/estimators.py", line 346, in fit
    self.automl_.fit(load_models=True, **kwargs)
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/automl.py", line 1154, in fit
    load_models=load_models,
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/automl.py", line 607, in fit
    _proc_smac.run_smbo()
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/smbo.py", line 374, in run_smbo
    metalearning_configurations = self.get_metalearning_suggestions()
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/smbo.py", line 584, in get_metalearning_suggestions
    self.datamanager.perform1HotEncoding()
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/data/abstract_data_manager.py", line 94, in perform1HotEncoding
    data=data)
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/data/abstract_data_manager.py", line 28, in perform_one_hot_encoding
    rvals.append(encoder.fit_transform(data[0]))
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/pipeline/components/data_preprocessing/data_preprocessing.py", line 117, in fit_transform
    return self.fit(X, y).transform(X)
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/pipeline/components/data_preprocessing/data_preprocessing.py", line 105, in fit
    self.column_transformer.fit(X)
  File "Desktop/mls/lib/python3.7/site-packages/sklearn/compose/_column_transformer.py", line 484, in fit
    self.fit_transform(X, y=y)
  File "Desktop/mls/lib/python3.7/site-packages/sklearn/compose/_column_transformer.py", line 518, in fit_transform
    result = self._fit_transform(X, y, _fit_transform_one)
  File "Desktop/mls/lib/python3.7/site-packages/sklearn/compose/_column_transformer.py", line 457, in _fit_transform
    self._iter(fitted=fitted, replace_strings=True), 1))
  File "Desktop/mls/lib/python3.7/site-packages/joblib/parallel.py", line 1029, in __call__
    if self.dispatch_one_batch(iterator):
  File "Desktop/mls/lib/python3.7/site-packages/joblib/parallel.py", line 847, in dispatch_one_batch
    self._dispatch(tasks)
  File "Desktop/mls/lib/python3.7/site-packages/joblib/parallel.py", line 765, in _dispatch
    job = self._backend.apply_async(batch, callback=cb)
  File "Desktop/mls/lib/python3.7/site-packages/joblib/_parallel_backends.py", line 206, in apply_async
    result = ImmediateResult(func)
  File "Desktop/mls/lib/python3.7/site-packages/joblib/_parallel_backends.py", line 570, in __init__
    self.results = batch()
  File "Desktop/mls/lib/python3.7/site-packages/joblib/parallel.py", line 253, in __call__
    for func, args, kwargs in self.items]
  File "Desktop/mls/lib/python3.7/site-packages/sklearn/pipeline.py", line 728, in _fit_transform_one
    res = transformer.fit_transform(X, y, **fit_params)
  File "Desktop/mls/lib/python3.7/site-packages/sklearn/pipeline.py", line 385, in fit_transform
    Xt, fit_params = self._fit(X, y, **fit_params)
  File "Desktop/mls/lib/python3.7/site-packages/sklearn/pipeline.py", line 315, in _fit
    **fit_params_steps[name])
  File "Desktop/mls/lib/python3.7/site-packages/joblib/memory.py", line 352, in __call__
    return self.func(*args, **kwargs)
  File "Desktop/mls/lib/python3.7/site-packages/sklearn/pipeline.py", line 728, in _fit_transform_one
    res = transformer.fit_transform(X, y, **fit_params)
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/pipeline/components/data_preprocessing/category_shift/category_shift.py", line 30, in fit_transform
    return self.fit(X, y).transform(X)
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/pipeline/components/data_preprocessing/category_shift/category_shift.py", line 21, in fit
    self.preprocessor.fit(X, y)
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/pipeline/implementations/CategoryShift.py", line 26, in fit
    self._convert_and_check_X(X)
  File "Desktop/mls/lib/python3.7/site-packages/autosklearn/pipeline/implementations/CategoryShift.py", line 16, in _convert_and_check_X
    raise ValueError('Categories should be non-negative numbers. '
ValueError: Categories should be non-negative numbers. NOTE: floats will be casted to integers.
```

## Environment and installation: ##

Please give details about your installation:

* Is your installation in a virtual environment or conda environment? virtual environment
* Python version: 3.7
* Auto-sklearn version: 0.10.0



Discussion:

Resolving patch:
diff --git a/autosklearn/data/validation.py b/autosklearn/data/validation.py
--- a/autosklearn/data/validation.py
+++ b/autosklearn/data/validation.py
@@ -1,5 +1,6 @@
 # -*- encoding: utf-8 -*-
 
+import functools
 import warnings
 from typing import List, Optional, Tuple, Union
 
@@ -365,6 +366,25 @@ def _check_and_encode_features(
                 assert self.feature_encoder is not None
                 self.feature_encoder.fit(X)
 
+                # The column transformer reoders the feature types - we therefore need to change
+                # it as well
+                def comparator(cmp1, cmp2):
+                    if (
+                        cmp1 == 'categorical' and cmp2 == 'categorical'
+                        or cmp1 == 'numerical' and cmp2 == 'numerical'
+                    ):
+                        return 0
+                    elif cmp1 == 'categorical' and cmp2 == 'numerical':
+                        return -1
+                    elif cmp1 == 'numerical' and cmp2 == 'categorical':
+                        return 1
+                    else:
+                        raise ValueError((cmp1, cmp2))
+                self.feature_types = sorted(
+                    self.feature_types,
+                    key=functools.cmp_to_key(comparator)
+                )
+
         if self.feature_encoder:
             try:
                 X = self.feature_encoder.transform(X)


Test patch:
diff --git a/test/test_automl/test_estimators.py b/test/test_automl/test_estimators.py
--- a/test/test_automl/test_estimators.py
+++ b/test/test_automl/test_estimators.py
@@ -563,7 +563,7 @@ def test_cv_regression(self):
         self._setUp(tmp)
         self._setUp(output)
 
-        X_train, Y_train, X_test, Y_test = putil.get_dataset('boston', train_size_maximum=300)
+        X_train, Y_train, X_test, Y_test = putil.get_dataset('diabetes', train_size_maximum=400)
         automl = AutoSklearnRegressor(time_left_for_this_task=60,
                                       per_run_time_limit=10,
                                       resampling_strategy='cv',
@@ -572,7 +572,7 @@ def test_cv_regression(self):
 
         automl.fit(X_train, Y_train)
         predictions = automl.predict(X_test)
-        self.assertEqual(predictions.shape, (206,))
+        self.assertEqual(predictions.shape, (148,))
         score = r2(Y_test, predictions)
         print(Y_test)
         print(predictions)
diff --git a/test/test_data/test_validation.py b/test/test_data/test_validation.py
--- a/test/test_data/test_validation.py
+++ b/test/test_data/test_validation.py
@@ -538,6 +538,14 @@ def test_big_dataset_encoding(self):
         # No change to numerical columns
         np.testing.assert_array_equal(x['carbon'].to_numpy(), x_t[:, 3])
 
+        # Categorical columns are sorted to the beginning
+        self.assertEqual(
+            validator.feature_types,
+            (['categorical'] * 3) + (['numerical'] * 7)
+        )
+        self.assertEqual(x.iloc[0, 6], 610)
+        np.testing.assert_array_equal(x_t[0], [0, 0, 0, 8, 0, 0, 0.7, 610, 0, np.NaN])
+
         return
 
     def test_join_and_check(self):