Repository: automl/auto-sklearn

Issue:
Parallel Usage with n_jobs=-1
Hello,

I tried to use auto-sklearn in parallel as suggested here: https://github.com/automl/auto-sklearn/blob/master/examples/example_parallel_n_jobs.py

Unfortunately, I faced two issues. First when setting n_jobs=-1 I get "IndexError: list index out of range". However, when setting it to the number of cores in my machine (e.g. 24) the error disappears. Second, I can see some very high peaks at the cpu utilization even when n_jobs=1 (sometimes even around 100% for some minutes within an 8h run) and I wonder what happens when setting n_jobs=24 (I have 24 cores at my machine) or any other high value. I already tried it and after a few minutes the cpu utilization was way over 100%. Is there any suggestion on how many cores it is safe to use?

I would be very happy about any suggestion, thanks!


Discussion:

Resolving patch:
diff --git a/autosklearn/estimators.py b/autosklearn/estimators.py
--- a/autosklearn/estimators.py
+++ b/autosklearn/estimators.py
@@ -6,6 +6,7 @@
 import numpy as np
 from sklearn.base import BaseEstimator
 from sklearn.utils.multiclass import type_of_target
+import joblib
 
 from autosklearn.automl import AutoMLClassifier, AutoMLRegressor, BaseAutoML
 from autosklearn.util.backend import create, get_randomized_directory_names
@@ -341,7 +342,11 @@ def fit(self, **kwargs):
                 output_directory=self.output_folder,
             )
 
-            self._n_jobs = self.n_jobs
+            if self.n_jobs == -1:
+                self._n_jobs = joblib.cpu_count()
+            else:
+                self._n_jobs = self.n_jobs
+
             shared_mode = True
             seeds = set()
             for i in range(self._n_jobs):
@@ -508,7 +513,6 @@ def refit(self, X, y):
         self._automl[0].refit(X, y)
         return self
 
-
     def predict(self, X, batch_size=None, n_jobs=1):
         return self._automl[0].predict(X, batch_size=batch_size, n_jobs=n_jobs)
 


Test patch:
diff --git a/test/test_automl/test_estimators.py b/test/test_automl/test_estimators.py
--- a/test/test_automl/test_estimators.py
+++ b/test/test_automl/test_estimators.py
@@ -9,7 +9,10 @@
 import numpy as np
 import numpy.ma as npma
 
+from joblib import cpu_count
+
 import autosklearn.pipeline.util as putil
+import autosklearn.estimators  # noqa F401
 from autosklearn.estimators import AutoSklearnEstimator
 from autosklearn.classification import AutoSklearnClassifier
 from autosklearn.regression import AutoSklearnRegressor
@@ -17,9 +20,9 @@
 from autosklearn.automl import AutoMLClassifier, AutoML
 from autosklearn.util.backend import Backend, BackendContext
 from autosklearn.constants import BINARY_CLASSIFICATION
-sys.path.append(os.path.dirname(__file__))
 from base import Base
 
+sys.path.append(os.path.dirname(__file__))
 
 class ArrayReturningDummyPredictor(object):
     def __init__(self, test):
@@ -479,6 +482,17 @@ def test_fit_n_jobs_2(self):
 
         self.assertEqual(len(seeds), 1)
 
+    @unittest.mock.patch('autosklearn.estimators.AutoSklearnEstimator.build_automl')
+    def test_fit_n_jobs_negative(self, build_automl_patch):
+        n_cores = cpu_count()
+        cls = AutoSklearnEstimator(n_jobs=-1)
+        cls.fit()
+        self.assertEqual(len(cls._automl), n_cores)
+
+    def test_get_number_of_available_cores(self):
+        n_cores = cpu_count()
+        self.assertGreaterEqual(n_cores, 1)
+
 
 class AutoMLClassifierTest(Base, unittest.TestCase):
     @unittest.mock.patch('autosklearn.automl.AutoML.predict')
diff --git a/test/test_metalearning/pyMetaLearn/test_meta_features_sparse.py b/test/test_metalearning/pyMetaLearn/test_meta_features_sparse.py
--- a/test/test_metalearning/pyMetaLearn/test_meta_features_sparse.py
+++ b/test/test_metalearning/pyMetaLearn/test_meta_features_sparse.py
@@ -6,16 +6,16 @@
 # Make the super class importable
 sys.path.append(os.path.dirname(__file__))
 
-import arff
-import numpy as np
-from scipy import sparse
-from sklearn.impute import SimpleImputer
+import arff  # noqa: E402
+import numpy as np  # noqa: E402
+from scipy import sparse  # noqa: E402
+from sklearn.impute import SimpleImputer  # noqa: E402
 
-from autosklearn.pipeline.implementations.OneHotEncoder import OneHotEncoder
-from sklearn.preprocessing import StandardScaler
+from autosklearn.pipeline.implementations.OneHotEncoder import OneHotEncoder  # noqa: E402
+from sklearn.preprocessing import StandardScaler  # noqa: E402
 
-import autosklearn.metalearning.metafeatures.metafeatures as meta_features
-import test_meta_features
+import autosklearn.metalearning.metafeatures.metafeatures as meta_features  # noqa: E402
+import test_meta_features  # noqa: E402
 
 
 class SparseMetaFeaturesTest(test_meta_features.MetaFeaturesTest,