Repository: automl/auto-sklearn

Issue:
Issue in extending auto-sklearn with new categorical encoders
I am trying to extend auto-sklearn with new categorical encoders like catboost encoder, target encoder etc. I am currently using the scikit-learn-contrib/category_encoder package to do so.  For spot check, I am just using the catboost encoder and forbidding other encoders (OHE) to see if I get acceptable results. 

Here's the script for catboost encoder (which I have added under autosklearn->pipeline->data_preprocessing->categorical-encoding)

```from ConfigSpace.configuration_space import ConfigurationSpace

from autosklearn.pipeline.components.base import AutoSklearnPreprocessingAlgorithm
from autosklearn.pipeline.constants import DENSE, SPARSE, UNSIGNED_DATA, INPUT
import category_encoders.cat_boost as catboost_enc

class CatBoostEncoder(AutoSklearnPreprocessingAlgorithm):
    def __init__(self, random_state=None):
        self.random_state = random_state

    def fit(self, X, y=None):
        self.preprocessor = catboost_enc()
        self.preprocessor.fit(X, y)
        return self

    def transform(self, X):
        if self.preprocessor is None:
            raise NotImplementedError()
        return self.preprocessor.transform(X)

    def fit_transform(self, X, y=None):
        return self.fit(X, y).transform(X)

    @staticmethod
    def get_properties(dataset_properties=None):
        return {'shortname': 'CatBoostEnc',
                'name': 'CatBoost Encoder',
                'handles_regression': True,
                'handles_classification': True,
                'handles_multiclass': True,
                'handles_multilabel': True,
                'handles_multioutput': True,
                # TODO find out of this is right!
                'handles_sparse': True,
                'handles_dense': True,
                'input': (DENSE, SPARSE, UNSIGNED_DATA),
                'output': (INPUT,), }

    @staticmethod
    def get_hyperparameter_search_space(dataset_properties=None):
        return ConfigurationSpace()
```


For spot check I print the encoders available during the run-time, which are:

<img width="477" alt="Screenshot 2020-11-02 at 2 40 27 PM" src="https://user-images.githubusercontent.com/69512026/97849973-6624ae80-1d19-11eb-989c-ece4414e9830.png">

But the output pipeline looks something like this:

<img width="680" alt="Screenshot 2020-11-02 at 2 43 42 PM" src="https://user-images.githubusercontent.com/69512026/97850289-d5020780-1d19-11eb-9760-3b35621a9887.png">


Is there something that I am missing here? Is there a link I can refer to to add new categorical encoders? Would appreciate any help.

Thanks


Discussion:

Resolving patch:
diff --git a/autosklearn/__version__.py b/autosklearn/__version__.py
--- a/autosklearn/__version__.py
+++ b/autosklearn/__version__.py
@@ -1,4 +1,4 @@
 """Version information."""
 
 # The following line *must* be the last in the module, exactly as formatted:
-__version__ = "0.11.0"
+__version__ = "0.11.1"
diff --git a/autosklearn/evaluation/__init__.py b/autosklearn/evaluation/__init__.py
--- a/autosklearn/evaluation/__init__.py
+++ b/autosklearn/evaluation/__init__.py
@@ -1,12 +1,14 @@
 # -*- encoding: utf-8 -*-
 import functools
+import json
 import math
 import multiprocessing
 from queue import Empty
 import time
 import traceback
-from typing import Tuple
+from typing import Dict, List, Optional, Tuple, Union
 
+from ConfigSpace import Configuration
 import numpy as np
 import pynisher
 from smac.runhistory.runhistory import RunInfo, RunValue
@@ -81,6 +83,14 @@ def get_cost_of_crash(metric):
     return worst_possible_result
 
 
+def _encode_exit_status(exit_status):
+    try:
+        json.dumps(exit_status)
+        return exit_status
+    except (TypeError, OverflowError):
+        return str(exit_status)
+
+
 # TODO potentially log all inputs to this class to pickle them in order to do
 # easier debugging of potential crashes
 class ExecuteTaFuncWithQueue(AbstractTAFunc):
@@ -224,11 +234,15 @@ def run_wrapper(
 
         return super().run_wrapper(run_info=run_info)
 
-    def run(self, config, instance=None,
-            cutoff=None,
-            seed=12345,
-            budget=0.0,
-            instance_specific=None):
+    def run(
+        self,
+        config: Configuration,
+        instance: Optional[str] = None,
+        cutoff: Optional[float] = None,
+        seed: int = 12345,
+        budget: float = 0.0,
+        instance_specific: Optional[str] = None,
+    ) -> Tuple[StatusType, float, float, Dict[str, Union[int, float, str, Dict, List, Tuple]]]:
 
         queue = multiprocessing.Queue()
 
@@ -272,11 +286,19 @@ def run(self, config, instance=None,
             obj_kwargs['resampling_strategy'] = self.resampling_strategy
             obj_kwargs['resampling_strategy_args'] = self.resampling_strategy_args
 
-        obj = pynisher.enforce_limits(**arguments)(self.ta)
-        obj(**obj_kwargs)
-
-        if obj.exit_status in (pynisher.TimeoutException,
-                               pynisher.MemorylimitException):
+        try:
+            obj = pynisher.enforce_limits(**arguments)(self.ta)
+            obj(**obj_kwargs)
+        except Exception as e:
+            exception_traceback = traceback.format_exc()
+            error_message = repr(e)
+            additional_info = {
+                'traceback': exception_traceback,
+                'error': error_message
+            }
+            return StatusType.CRASHED, self.cost_for_crash, 0.0, additional_info
+
+        if obj.exit_status in (pynisher.TimeoutException, pynisher.MemorylimitException):
             # Even if the pynisher thinks that a timeout or memout occured,
             # it can be that the target algorithm wrote something into the queue
             #  - then we treat it as a succesful run
@@ -309,8 +331,7 @@ def run(self, config, instance=None,
                 elif obj.exit_status is pynisher.MemorylimitException:
                     status = StatusType.MEMOUT
                     additional_run_info = {
-                        'error': 'Memout (used more than %d MB).' %
-                                 self.memory_limit
+                        'error': 'Memout (used more than %d MB).' % self.memory_limit
                     }
                 else:
                     raise ValueError(obj.exit_status)
@@ -322,7 +343,7 @@ def run(self, config, instance=None,
             cost = self.worst_possible_result
             additional_run_info = {'error': 'Your configuration of '
                                             'auto-sklearn does not work!',
-                                   'exit_status': obj.exit_status,
+                                   'exit_status': _encode_exit_status(obj.exit_status),
                                    'subprocess_stdout': obj.stdout,
                                    'subprocess_stderr': obj.stderr,
                                    }
@@ -343,14 +364,14 @@ def run(self, config, instance=None,
                                                   'because the pynisher exit ' \
                                                   'status %s is unknown.' % \
                                                   str(obj.exit_status)
-                    additional_run_info['exit_status'] = obj.exit_status
+                    additional_run_info['exit_status'] = _encode_exit_status(obj.exit_status)
                     additional_run_info['subprocess_stdout'] = obj.stdout
                     additional_run_info['subprocess_stderr'] = obj.stderr
             except Empty:
                 info = None
                 additional_run_info = {
                     'error': 'Result queue is empty',
-                    'exit_status': obj.exit_status,
+                    'exit_status': _encode_exit_status(obj.exit_status),
                     'subprocess_stdout': obj.stdout,
                     'subprocess_stderr': obj.stderr,
                     'exitcode': obj.exitcode
diff --git a/autosklearn/pipeline/components/data_preprocessing/data_preprocessing.py b/autosklearn/pipeline/components/data_preprocessing/data_preprocessing.py
--- a/autosklearn/pipeline/components/data_preprocessing/data_preprocessing.py
+++ b/autosklearn/pipeline/components/data_preprocessing/data_preprocessing.py
@@ -102,7 +102,7 @@ def fit(self, X, y=None):
             transformers=sklearn_transf_spec,
             sparse_threshold=float(self.sparse_),
             )
-        self.column_transformer.fit(X)
+        self.column_transformer.fit(X, y)
         return self
 
     def transform(self, X):


Test patch: