Repository: automl/auto-sklearn

Issue:
Improvement: parallelize `AutoML.predict`
Prediction with an ensemble can take a long time depending on the number of models. A quick win would be to use a `multiprocessing.Pool.map` to replace this for loop in `AutoML.predict` (line 455-470):

```
all_predictions = []
for identifier in self.ensemble_.get_model_identifiers():
    model = self.models_[identifier]

    X_ = X.copy()
    if self._task in REGRESSION_TASKS:
        prediction = model.predict(X_)
    else:
        prediction = model.predict_proba(X_)

    if len(prediction.shape) < 1 or len(X_.shape) < 1 or \
            X_.shape[0] < 1 or prediction.shape[0] != X_.shape[0]:
        self._logger.warning("Prediction shape for model %s is %s "
                             "while X_.shape is %s" %
                             (model, str(prediction.shape),
                              str(X_.shape)))
    all_predictions.append(prediction)
```


Discussion:
Indeed, this would help, although I think one should use `sklearn.externals.joblib`. A PR would be highly appreciated.

This is related to #75.
Unfortunately, I don't have a lot of time on my hands right now, but I might give it a go soon-ish. If I do, I will look into using `joblib` as you suggest.

Another interesting note is that some models (I'm looking at you Naive Bayes) run out of memory predicting on a large number of rows (e.g., `X.shape[0] > 100e3`). Hence, chunking the predictions into batches of 20k rows seams like a good idea to incorporate into this PR.
The underlying [predict function](https://github.com/automl/auto-sklearn/blob/master/autosklearn/pipeline/base.py#L155) support chunking and is already used while fitting the models at optimization time. I have never experienced this at prediction time though, that's why it's not active.
All right, I'll see if I can free up some time over the weekend for a PR that uses `joblib` and sets the `batch_size`.
That'd be great. I'll be on holiday from next week, so I won't be able to respond and merge this for a bit.

Resolving patch:
diff --git a/autosklearn/automl.py b/autosklearn/automl.py
--- a/autosklearn/automl.py
+++ b/autosklearn/automl.py
@@ -14,6 +14,8 @@
 from sklearn.base import BaseEstimator
 from smac.tae.execute_ta_run import StatusType
 from smac.stats.stats import Stats
+from sklearn.grid_search import _CVScoreTuple
+from sklearn.externals import joblib
 
 from autosklearn.constants import *
 from autosklearn.metrics import Scorer
@@ -28,6 +30,29 @@
 from autosklearn.util.hash import hash_numpy_array
 
 
+def _model_predict(self, X, batch_size, identifier):
+    def send_warnings_to_log(
+            message, category, filename, lineno, file=None, line=None):
+        self._logger.debug('%s:%s: %s:%s' %
+                       (filename, lineno, category.__name__, message))
+        return
+    model = self.models_[identifier]
+    X_ = X.copy()
+    with warnings.catch_warnings():
+        warnings.showwarning = send_warnings_to_log
+        if self._task in REGRESSION_TASKS:
+            prediction = model.predict(X_, batch_size=batch_size)
+        else:
+            prediction = model.predict_proba(X_, batch_size=batch_size)
+    if len(prediction.shape) < 1 or len(X_.shape) < 1 or \
+            X_.shape[0] < 1 or prediction.shape[0] != X_.shape[0]:
+        self._logger.warning("Prediction shape for model %s is %s "
+                             "while X_.shape is %s" %
+                             (model, str(prediction.shape),
+                              str(X_.shape)))
+    return prediction
+
+
 class AutoML(BaseEstimator):
 
     def __init__(self,
@@ -484,7 +509,22 @@ def send_warnings_to_log(message, category, filename, lineno,
         self._can_predict = True
         return self
 
-    def predict(self, X):
+    def predict(self, X, batch_size=None, n_jobs=1):
+        """predict.
+
+        Parameters
+        ----------
+        X: array-like, shape = (n_samples, n_features)
+
+        batch_size: int or None, defaults to None
+            batch_size controls whether the pipelines will be
+            called on small chunks of the data. Useful when calling the
+            predict method on the whole array X results in a MemoryError.
+
+        n_jobs: int, defaults to 1
+            Parallelize the predictions across the models with n_jobs
+            processes.
+        """
         if self._keep_models is not True:
             raise ValueError(
                 "Predict can only be called if 'keep_models==True'")
@@ -499,31 +539,11 @@ def predict(self, X):
                 self.ensemble_ is None:
             self._load_models()
 
-        def send_warnings_to_log(message, category, filename, lineno,
-                                 file=None, line=None):
-            self._logger.debug('%s:%s: %s:%s' %
-                               (filename, lineno, category.__name__, message))
-            return
-
-        all_predictions = []
-        for identifier in self.ensemble_.get_model_identifiers():
-            model = self.models_[identifier]
-
-            X_ = X.copy()
-            with warnings.catch_warnings():
-                warnings.showwarning = send_warnings_to_log
-                if self._task in REGRESSION_TASKS:
-                    prediction = model.predict(X_)
-                else:
-                    prediction = model.predict_proba(X_)
-
-            if len(prediction.shape) < 1 or len(X_.shape) < 1 or \
-                    X_.shape[0] < 1 or prediction.shape[0] != X_.shape[0]:
-                self._logger.warning("Prediction shape for model %s is %s "
-                                     "while X_.shape is %s" %
-                                     (model, str(prediction.shape),
-                                      str(X_.shape)))
-            all_predictions.append(prediction)
+        # Parallelize predictions across models with n_jobs processes.
+        # Each process computes predictions in chunks of batch_size rows.
+        all_predictions = joblib.Parallel(n_jobs=n_jobs)(
+            joblib.delayed(_model_predict)(self, X, batch_size, identifier)
+            for identifier in self.ensemble_.get_model_identifiers())
 
         if len(all_predictions) == 0:
             raise ValueError('Something went wrong generating the predictions. '
diff --git a/autosklearn/estimators.py b/autosklearn/estimators.py
--- a/autosklearn/estimators.py
+++ b/autosklearn/estimators.py
@@ -65,8 +65,8 @@ def fit_ensemble(self, y, task=None, metric=None, precision='32',
                                          dataset_name, ensemble_nbest,
                                          ensemble_size)
 
-    def predict(self, X):
-        return self._automl.predict(X)
+    def predict(self, X, batch_size=None, n_jobs=1):
+        return self._automl.predict(X, batch_size=batch_size, n_jobs=n_jobs)
 
     def score(self, X, y):
         return self._automl.score(X, y)
@@ -353,7 +353,7 @@ def fit(self, X, y,
                                                       feat_type=feat_type,
                                                       dataset_name=dataset_name)
 
-    def predict(self, X):
+    def predict(self, X, batch_size=None, n_jobs=1):
         """Predict classes for X.
 
         Parameters
@@ -366,9 +366,10 @@ def predict(self, X):
             The predicted classes.
 
         """
-        return super(AutoSklearnClassifier, self).predict(X)
+        return super(AutoSklearnClassifier, self).predict(
+            X, batch_size=batch_size, n_jobs=n_jobs)
 
-    def predict_proba(self, X):
+    def predict_proba(self, X, batch_size=None, n_jobs=1):
 
         """Predict probabilities of classes for all samples X.
 
@@ -381,7 +382,8 @@ def predict_proba(self, X):
         y : array of shape = [n_samples, n_classes] or [n_samples, n_labels]
             The predicted class probabilities.
         """
-        return self._automl.predict_proba(X)
+        return self._automl.predict_proba(
+            X, batch_size=batch_size, n_jobs=n_jobs)
 
 
 class AutoSklearnRegressor(AutoSklearnEstimator):
@@ -434,7 +436,7 @@ def fit(self, X, y,
                                                      feat_type=feat_type,
                                                      dataset_name=dataset_name)
 
-    def predict(self, X):
+    def predict(self, X, batch_size=None, n_jobs=1):
         """Predict regression target for X.
 
         Parameters
@@ -447,7 +449,8 @@ def predict(self, X):
             The predicted values.
 
         """
-        return super(AutoSklearnRegressor, self).predict(X)
+        return super(AutoSklearnRegressor, self).predict(
+            X, batch_size=batch_size, n_jobs=n_jobs)
 
 
 class AutoMLClassifier(AutoMLDecorator):
@@ -529,8 +532,11 @@ def _process_target_classes(self, y):
 
         return y
 
-    def predict(self, X):
-        predicted_probabilities = self._automl.predict(X)
+
+    def predict(self, X, batch_size=None, n_jobs=1):
+        predicted_probabilities = self._automl.predict(
+            X, batch_size=batch_size, n_jobs=n_jobs)
+
         if self._n_outputs == 1:
             predicted_indexes = np.argmax(predicted_probabilities, axis=1)
             predicted_classes = self._classes[0].take(predicted_indexes)
@@ -547,8 +553,8 @@ def predict(self, X):
 
             return predicted_classes
 
-    def predict_proba(self, X):
-        return self._automl.predict(X)
+    def predict_proba(self, X, batch_size=None, n_jobs=1):
+        return self._automl.predict(X, batch_size=batch_size, n_jobs=n_jobs)
 
 
 class AutoMLRegressor(AutoMLDecorator):


Test patch: