Repository: automl/auto-sklearn

Issue:
auto-sklearn deleted my entire home directory necessitating Linux reinstall
I run Ubuntu.
I installed pyrfr via pip. I then installed auto-sklearn via pip.
I ran a basic example with minimal modifications.

I received the following error, then my entire home/$username/* directory was deleted. I think I need to reinstall Ubuntu because of auto-sklearn. Maybe there is a notice in the docs somewhere that the output dir must be deleteable, but I don't think that is reasonable.

FileNotFoundError: [Errno 2] No such file or directory: '/home/$username/.local/lib/python3.5/site-packages/autosklearn/util/logging.yaml'
[WARNING] [14:08:38:autosklearn.util.backend] Could not delete output dir: /home/$username

I am guessing that file was not found because it was deleted.
I cannot produce the entire trace because my Linux box is destroyed and I am typing this on second computer.


Discussion:
An easy fix is to only permit using empty/currently nonexistent directories as tmp and output dir.
First of all, please excuse us for not safeguarding against auto-sklearn running havoc and deleting your home directory. In order to somehow find out how this happened, do you remember to what location you set the `tmp_dir`?
I do not remember to what I set the tmp_dir. It was not the home dir, as far as I remember.

Besides that, the code works great. 
Hm, comparing the output you pasted to the [backed file](https://github.com/automl/auto-sklearn/blob/master/autosklearn/util/backend.py) I think that the `output_dir` was set to the home directory. However, auto-sklearn should really catch this and not start deleting files it doesn't own.

Resolving patch:
diff --git a/autosklearn/automl.py b/autosklearn/automl.py
--- a/autosklearn/automl.py
+++ b/autosklearn/automl.py
@@ -145,9 +145,7 @@ def fit(
         dataset_name=None,
         only_return_configuration_space=False,
     ):
-        if not self._shared_mode:
-            self._backend.context.delete_directories()
-        else:
+        if self._shared_mode:
             # If this fails, it's likely that this is the first call to get
             # the data manager
             try:
@@ -156,8 +154,6 @@ def fit(
             except IOError:
                 pass
 
-        self._backend.context.create_directories()
-
         if dataset_name is None:
             dataset_name = hash_array_or_matrix(X)
 
diff --git a/autosklearn/estimators.py b/autosklearn/estimators.py
--- a/autosklearn/estimators.py
+++ b/autosklearn/estimators.py
@@ -217,7 +217,8 @@ def build_automl(self):
         backend = create(temporary_directory=self.tmp_folder,
                          output_directory=self.output_folder,
                          delete_tmp_folder_after_terminate=self.delete_tmp_folder_after_terminate,
-                         delete_output_folder_after_terminate=self.delete_output_folder_after_terminate)
+                         delete_output_folder_after_terminate=self.delete_output_folder_after_terminate,
+                         shared_mode = self.shared_mode)
         automl = self._get_automl_class()(
             backend=backend,
             time_left_for_this_task=self.time_left_for_this_task,
diff --git a/autosklearn/util/backend.py b/autosklearn/util/backend.py
--- a/autosklearn/util/backend.py
+++ b/autosklearn/util/backend.py
@@ -18,10 +18,12 @@
 def create(temporary_directory,
            output_directory,
            delete_tmp_folder_after_terminate=True,
-           delete_output_folder_after_terminate=True):
+           delete_output_folder_after_terminate=True,
+           shared_mode=False):
     context = BackendContext(temporary_directory, output_directory,
                              delete_tmp_folder_after_terminate,
-                             delete_output_folder_after_terminate)
+                             delete_output_folder_after_terminate,
+                             shared_mode)
     backend = Backend(context)
 
     return backend
@@ -33,20 +35,35 @@ def __init__(self,
                  temporary_directory,
                  output_directory,
                  delete_tmp_folder_after_terminate,
-                 delete_output_folder_after_terminate):
-        self._prepare_directories(temporary_directory, output_directory)
+                 delete_output_folder_after_terminate,
+                 shared_mode=False):
+
+        # Check that the names of tmp_dir and output_dir is not the same.
+        if temporary_directory == output_directory \
+            and temporary_directory is not None:
+            raise ValueError("The temporary and the output directory "
+                             "must be different.")
+
         self.delete_tmp_folder_after_terminate = delete_tmp_folder_after_terminate
         self.delete_output_folder_after_terminate = delete_output_folder_after_terminate
+        self.shared_mode = shared_mode
+        # attributes to check that directories were created by autosklearn.
+        self._tmp_dir_created = False
+        self._output_dir_created = False
+
+        self._prepare_directories(temporary_directory, output_directory)
         self._logger = logging.get_logger(__name__)
         self.create_directories()
 
     @property
     def output_directory(self):
-        return self.__output_directory
+        # make sure that tilde does not appear on the path.
+        return os.path.expanduser(os.path.expandvars(self.__output_directory))
 
     @property
     def temporary_directory(self):
-        return self.__temporary_directory
+        # make sure that tilde does not appear on the path.
+        return os.path.expanduser(os.path.expandvars(self.__temporary_directory))
 
     def _prepare_directories(self, temporary_directory, output_directory):
         random_number = random.randint(0, 10000)
@@ -61,20 +78,39 @@ def _prepare_directories(self, temporary_directory, output_directory):
             else '/tmp/autosklearn_output_%d_%d' % (pid, random_number)
 
     def create_directories(self):
-        try:
+        if self.shared_mode:
+            # If shared_mode == True, the tmp and output dir will be shared
+            # by different instances of auto-sklearn.
+            try:
+                os.makedirs(self.temporary_directory)
+            except OSError:
+                pass
+            try:
+                os.makedirs(self.output_directory)
+            except OSError:
+                pass
+
+        else:
+            # Exception is raised if self.temporary_directory already exists.
             os.makedirs(self.temporary_directory)
-        except OSError:
-            pass
-        try:
+            self._tmp_dir_created = True
+
+            # Exception is raised if self.output_directory already exists.
             os.makedirs(self.output_directory)
-        except OSError:
-            pass
+            self._output_dir_created = True
+
 
     def __del__(self):
         self.delete_directories(force=False)
 
     def delete_directories(self, force=True):
         if self.delete_output_folder_after_terminate or force:
+            if self._output_dir_created is False and self.shared_mode is False:
+                raise ValueError("Failed to delete output dir: %s "
+                              "because auto-sklearn did not create it. "
+                              "Please make sure that the specified output "
+                              "dir does not exist when instantiating "
+                              "auto-sklearn." % self.output_directory)
             try:
                 shutil.rmtree(self.output_directory)
             except Exception:
@@ -86,6 +122,12 @@ def delete_directories(self, force=True):
                           self.output_directory)
 
         if self.delete_tmp_folder_after_terminate or force:
+            if self._tmp_dir_created is False and self.shared_mode is False:
+                raise ValueError("Failed to delete tmp dir: % s "
+                              "because auto-sklearn did not create it. "
+                              "Please make sure that the specified tmp "
+                              "dir does not exist when instantiating "
+                              "auto-sklearn." % self.temporary_directory)
             try:
                 shutil.rmtree(self.temporary_directory)
             except Exception:
@@ -418,4 +460,4 @@ def write_txt_file(self, filepath, data, name):
                 self.logger.debug('Created %s file %s' % (name, filepath))
             else:
                 self.logger.debug('%s file already present %s' %
-                                  (name, filepath))
+                                  (name, filepath))
\ No newline at end of file


Test patch:
diff --git a/test/test_automl/base.py b/test/test_automl/base.py
--- a/test/test_automl/base.py
+++ b/test/test_automl/base.py
@@ -3,6 +3,7 @@
 import shutil
 import time
 import unittest
+from autosklearn.util.backend import create
 
 
 class Base(unittest.TestCase):
@@ -19,25 +20,33 @@ def setUp(self):
         except Exception:
             self.travis = False
 
-    def _setUp(self, output):
-        if os.path.exists(output):
+    def _setUp(self, dir):
+        if os.path.exists(dir):
             for i in range(10):
                 try:
-                    shutil.rmtree(output)
+                    shutil.rmtree(dir)
                     break
                 except OSError:
                     time.sleep(1)
-        try:
-            os.makedirs(output)
-        except OSError:
-            pass
 
+    def _create_backend(self, test_name):
+        tmp = os.path.join(self.test_dir, '..', '.tmp._%s' % test_name)
+        output = os.path.join(self.test_dir, '..', '.output._%s' % test_name)
+        # Make sure the folders we wanna create do not already exist.
+        self._setUp(tmp)
+        self._setUp(output)
+        backend = create(tmp, output)
+        return backend
 
-    def _tearDown(self, output):
-        if os.path.exists(output):
+    def _tearDown(self, dir):
+        """
+        Delete the temporary and the output directories manually
+        in case they are not deleted.
+        """
+        if os.path.exists(dir):
             for i in range(10):
                 try:
-                    shutil.rmtree(output)
+                    shutil.rmtree(dir)
                     break
                 except OSError:
                     time.sleep(1)
\ No newline at end of file
diff --git a/test/test_automl/test_automl.py b/test/test_automl/test_automl.py
--- a/test/test_automl/test_automl.py
+++ b/test/test_automl/test_automl.py
@@ -43,14 +43,12 @@ def setUp(self):
         self.automl._delete_output_directories = lambda: 0
 
     def test_refit_shuffle_on_fail(self):
-        output = os.path.join(self.test_dir, '..', '.tmp_refit_shuffle_on_fail')
-        context = BackendContext(output, output, False, False)
-        backend = Backend(context)
+        backend_api = self._create_backend('test_refit_shuffle_on_fail')
 
         failing_model = unittest.mock.Mock()
         failing_model.fit.side_effect = [ValueError(), ValueError(), None]
 
-        auto = AutoML(backend, 20, 5)
+        auto = AutoML(backend_api, 20, 5)
         ensemble_mock = unittest.mock.Mock()
         auto.ensemble_ = ensemble_mock
         ensemble_mock.get_selected_model_identifiers.return_value = [1]
@@ -63,6 +61,10 @@ def test_refit_shuffle_on_fail(self):
 
         self.assertEqual(failing_model.fit.call_count, 3)
 
+        del auto
+        self._tearDown(backend_api.temporary_directory)
+        self._tearDown(backend_api.output_directory)
+
     def test_only_loads_ensemble_models(self):
         identifiers = [(1, 2), (3, 4)]
 
@@ -96,11 +98,9 @@ def test_raises_if_no_models(self):
         self.automl._load_models()
 
     def test_fit(self):
-        output = os.path.join(self.test_dir, '..', '.tmp_test_fit')
-        self._setUp(output)
+        backend_api = self._create_backend('test_fit')
 
         X_train, Y_train, X_test, Y_test = putil.get_dataset('iris')
-        backend_api = backend.create(output, output)
         automl = autosklearn.automl.AutoML(backend_api, 20, 5)
         automl.fit(
             X_train, Y_train, metric=accuracy, task=MULTICLASS_CLASSIFICATION,
@@ -110,7 +110,8 @@ def test_fit(self):
         self.assertEqual(automl._task, MULTICLASS_CLASSIFICATION)
 
         del automl
-        self._tearDown(output)
+        self._tearDown(backend_api.temporary_directory)
+        self._tearDown(backend_api.output_directory)
 
     def test_fit_roar(self):
         def get_roar_object_callback(
@@ -129,11 +130,9 @@ def get_roar_object_callback(
                 tae_runner=ta,
             )
 
-        output = os.path.join(self.test_dir, '..', '.tmp_test_fit_roar')
-        self._setUp(output)
+        backend_api = self._create_backend('test_fit_roar')
 
         X_train, Y_train, X_test, Y_test = putil.get_dataset('iris')
-        backend_api = backend.create(output, output)
         automl = autosklearn.automl.AutoML(
             backend=backend_api,
             time_left_for_this_task=20,
@@ -149,16 +148,15 @@ def get_roar_object_callback(
         self.assertEqual(automl._task, MULTICLASS_CLASSIFICATION)
 
         del automl
-        self._tearDown(output)
+        self._tearDown(backend_api.temporary_directory)
+        self._tearDown(backend_api.output_directory)
 
     def test_binary_score_and_include(self):
         """
         Test fix for binary classification prediction
         taking the index 1 of second dimension in prediction matrix
         """
-
-        output = os.path.join(self.test_dir, '..', '.tmp_test_binary_score')
-        self._setUp(output)
+        backend_api = self._create_backend('test_binary_score_and_include')
 
         data = sklearn.datasets.make_classification(
             n_samples=400, n_features=10, n_redundant=1, n_informative=3,
@@ -168,7 +166,6 @@ def test_binary_score_and_include(self):
         X_test = data[0][200:]
         Y_test = data[1][200:]
 
-        backend_api = backend.create(output, output)
         automl = autosklearn.automl.AutoML(backend_api, 20, 5,
                                            include_estimators=['sgd'],
                                            include_preprocessors=['no_preprocessing'])
@@ -182,18 +179,17 @@ def test_binary_score_and_include(self):
         self.assertGreaterEqual(score, 0.4)
 
         del automl
-        self._tearDown(output)
+        self._tearDown(backend_api.temporary_directory)
+        self._tearDown(backend_api.output_directory)
 
     def test_automl_outputs(self):
-        output = os.path.join(self.test_dir, '..',
-                              '.tmp_test_automl_outputs')
-        self._setUp(output)
+        backend_api = self._create_backend('test_automl_outputs')
+
         name = '31_bac'
         dataset = os.path.join(self.test_dir, '..', '.data', name)
-        data_manager_file = os.path.join(output, '.auto-sklearn',
+        data_manager_file = os.path.join(backend_api.temporary_directory, '.auto-sklearn',
                                          'datamanager.pkl')
 
-        backend_api = backend.create(output, output)
         auto = autosklearn.automl.AutoML(
             backend_api, 20, 5,
             initial_configurations_via_metalearning=0,
@@ -212,43 +208,41 @@ def test_automl_outputs(self):
                    'start_time_100', 'datamanager.pkl',
                    'predictions_ensemble',
                    'ensembles', 'predictions_test', 'models']
-        self.assertEqual(sorted(os.listdir(os.path.join(output,
+        self.assertEqual(sorted(os.listdir(os.path.join(backend_api.temporary_directory,
                                                         '.auto-sklearn'))),
                          sorted(fixture))
 
         # At least one ensemble, one validation, one test prediction and one
         # model and one ensemble
-        fixture = os.listdir(os.path.join(output, '.auto-sklearn',
+        fixture = os.listdir(os.path.join(backend_api.temporary_directory, '.auto-sklearn',
                                           'predictions_ensemble'))
         self.assertIn('predictions_ensemble_100_1.npy', fixture)
 
-        fixture = os.listdir(os.path.join(output, '.auto-sklearn',
+        fixture = os.listdir(os.path.join(backend_api.temporary_directory, '.auto-sklearn',
                                           'models'))
         self.assertIn('100.1.model', fixture)
 
-        fixture = os.listdir(os.path.join(output, '.auto-sklearn',
+        fixture = os.listdir(os.path.join(backend_api.temporary_directory, '.auto-sklearn',
                                           'ensembles'))
         self.assertIn('100.0.ensemble', fixture)
 
         # Start time
-        start_time_file_path = os.path.join(output, '.auto-sklearn',
+        start_time_file_path = os.path.join(backend_api.temporary_directory, '.auto-sklearn',
                                             "start_time_100")
         with open(start_time_file_path, 'r') as fh:
             start_time = float(fh.read())
         self.assertGreaterEqual(time.time() - start_time, 10)
 
         del auto
-        self._tearDown(output)
+        self._tearDown(backend_api.temporary_directory)
+        self._tearDown(backend_api.output_directory)
 
     def test_do_dummy_prediction(self):
         for name in ['401_bac', '31_bac', 'adult', 'cadata']:
-            output = os.path.join(self.test_dir, '..',
-                                  '.tmp_test_do_dummy_prediction')
-            self._setUp(output)
+            backend_api = self._create_backend('test_do_dummy_prediction')
 
             dataset = os.path.join(self.test_dir, '..', '.data', name)
 
-            backend_api = backend.create(output, output)
             auto = autosklearn.automl.AutoML(
                 backend_api, 20, 5,
                 initial_configurations_via_metalearning=25)
@@ -260,14 +254,17 @@ def test_do_dummy_prediction(self):
             auto._do_dummy_prediction(D, 1)
 
             # Ensure that the dummy predictions are not in the current working
-            # directory, but in the output directory (under output)
+            # directory, but in the temporary directory.
             self.assertFalse(os.path.exists(os.path.join(os.getcwd(),
                                                          '.auto-sklearn')))
             self.assertTrue(os.path.exists(os.path.join(
-                output, '.auto-sklearn', 'predictions_ensemble',
+                backend_api.temporary_directory, '.auto-sklearn', 'predictions_ensemble',
                 'predictions_ensemble_1_1.npy')))
 
             del auto
-            self._tearDown(output)
+            self._tearDown(backend_api.temporary_directory)
+            self._tearDown(backend_api.output_directory)
 
 
+if __name__=="__main__":
+    unittest.main()
diff --git a/test/test_automl/test_estimators.py b/test/test_automl/test_estimators.py
--- a/test/test_automl/test_estimators.py
+++ b/test/test_automl/test_estimators.py
@@ -86,7 +86,9 @@ def test_feat_type_wrong_arguments(self):
                                 X=X, y=y, feat_type=['Car']*100)
 
     def test_fit_pSMAC(self):
-        output = os.path.join(self.test_dir, '..', '.tmp_estimator_fit_pSMAC')
+        tmp = os.path.join(self.test_dir, '..', '.tmp_estimator_fit_pSMAC')
+        output = os.path.join(self.test_dir, '..', '.out_estimator_fit_pSMAC')
+        self._setUp(tmp)
         self._setUp(output)
 
         X_train, Y_train, X_test, Y_test = putil.get_dataset('digits')
@@ -99,7 +101,7 @@ def test_fit_pSMAC(self):
             time_left_for_this_task=20,
             per_run_time_limit=5,
             output_folder=output,
-            tmp_folder=output,
+            tmp_folder=tmp,
             shared_mode=True,
             seed=1,
             initial_configurations_via_metalearning=0,
@@ -109,7 +111,7 @@ def test_fit_pSMAC(self):
         # Create a 'dummy model' for the first run, which has an accuracy of
         # more than 99%; it should be in the final ensemble if the ensemble
         # building of the second AutoSklearn classifier works correct
-        true_targets_ensemble_path = os.path.join(output, '.auto-sklearn',
+        true_targets_ensemble_path = os.path.join(tmp, '.auto-sklearn',
                                                   'true_targets_ensemble.npy')
         with open(true_targets_ensemble_path, 'rb') as fh:
             true_targets_ensemble = np.load(fh)
@@ -120,7 +122,7 @@ def test_fit_pSMAC(self):
         for i, value in enumerate(true_targets_ensemble):
             probas[i, value] = 1.0
         dummy_predictions_path = os.path.join(
-            output,
+            tmp,
             '.auto-sklearn',
             'predictions_ensemble',
             'predictions_ensemble_1_00030.npy',
@@ -133,7 +135,7 @@ def test_fit_pSMAC(self):
             probas_test[i, value - 1] = 1.0
 
         dummy = ArrayReturningDummyPredictor(probas_test)
-        context = BackendContext(output, output, False, False)
+        context = BackendContext(tmp, output, False, False, True)
         backend = Backend(context)
         backend.save_model(dummy, 30, 1)
 
@@ -141,7 +143,7 @@ def test_fit_pSMAC(self):
             time_left_for_this_task=20,
             per_run_time_limit=5,
             output_folder=output,
-            tmp_folder=output,
+            tmp_folder=tmp,
             shared_mode=True,
             seed=2,
             initial_configurations_via_metalearning=0,
@@ -158,7 +160,7 @@ def test_fit_pSMAC(self):
         predictions = automl.predict(X_test)
         score = sklearn.metrics.accuracy_score(Y_test, predictions)
 
-        self.assertEqual(len(os.listdir(os.path.join(output, '.auto-sklearn',
+        self.assertEqual(len(os.listdir(os.path.join(tmp, '.auto-sklearn',
                                                      'ensembles'))), 1)
         self.assertGreaterEqual(score, 0.90)
         self.assertEqual(automl._automl._task, MULTICLASS_CLASSIFICATION)
@@ -168,19 +170,22 @@ def test_fit_pSMAC(self):
         self.assertIn(ArrayReturningDummyPredictor, classifier_types)
 
         del automl
+        self._tearDown(tmp)
         self._tearDown(output)
 
     def test_cv_results(self):
         # TODO restructure and actually use real SMAC output from a long run
         # to do this unittest!
-        output = os.path.join(self.test_dir, '..', '.tmp_cv_results')
+        tmp = os.path.join(self.test_dir, '..', '.tmp_cv_results')
+        output = os.path.join(self.test_dir, '..', '.out_cv_results')
+        self._setUp(tmp)
         self._setUp(output)
         X_train, Y_train, X_test, Y_test = putil.get_dataset('iris')
 
         cls = AutoSklearnClassifier(time_left_for_this_task=20,
                                     per_run_time_limit=5,
                                     output_folder=output,
-                                    tmp_folder=output,
+                                    tmp_folder=tmp,
                                     shared_mode=False,
                                     seed=1,
                                     initial_configurations_via_metalearning=0,
@@ -195,6 +200,7 @@ def test_cv_results(self):
         self.assertTrue([isinstance(val, npma.MaskedArray) for key, val in
                          cv_results.items() if key.startswith('param_')])
         del cls
+        self._tearDown(tmp)
         self._tearDown(output)
 
         
@@ -249,13 +255,15 @@ def test_multilabel_prediction(self, predict_mock):
         np.testing.assert_array_equal(expected_result, actual_result)
 
     def test_can_pickle_classifier(self):
-        output = os.path.join(self.test_dir, '..', '.tmp_can_pickle')
+        tmp = os.path.join(self.test_dir, '..', '.tmp_can_pickle')
+        output = os.path.join(self.test_dir, '..', '.out_can_pickle')
+        self._setUp(tmp)
         self._setUp(output)
 
         X_train, Y_train, X_test, Y_test = putil.get_dataset('iris')
         automl = AutoSklearnClassifier(time_left_for_this_task=20,
                                        per_run_time_limit=5,
-                                       tmp_folder=output,
+                                       tmp_folder=tmp,
                                        output_folder=output)
         automl.fit(X_train, Y_train)
 
@@ -295,14 +303,16 @@ def test_can_pickle_classifier(self):
         self.assertEqual(initial_accuracy, restored_accuracy)
 
     def test_multilabel(self):
-        output = os.path.join(self.test_dir, '..', '.tmp_multilabel_fit')
+        tmp = os.path.join(self.test_dir, '..', '.tmp_multilabel_fit')
+        output = os.path.join(self.test_dir, '..', '.out_multilabel_fit')
+        self._setUp(tmp)
         self._setUp(output)
 
         X_train, Y_train, X_test, Y_test = putil.get_dataset(
             'iris', make_multilabel=True)
         automl = AutoSklearnClassifier(time_left_for_this_task=20,
                                        per_run_time_limit=5,
-                                       tmp_folder=output,
+                                       tmp_folder=tmp,
                                        output_folder=output)
 
         automl.fit(X_train, Y_train)
@@ -314,14 +324,16 @@ def test_multilabel(self):
         self.assertAlmostEqual(np.mean(probs), 0.33, places=1)
 
     def test_binary(self):
+        tmp = os.path.join(self.test_dir, '..', '.out_binary_fit')
         output = os.path.join(self.test_dir, '..', '.tmp_binary_fit')
         self._setUp(output)
+        self._setUp(tmp)
 
         X_train, Y_train, X_test, Y_test = putil.get_dataset(
             'iris', make_binary=True)
         automl = AutoSklearnClassifier(time_left_for_this_task=20,
                                        per_run_time_limit=5,
-                                       tmp_folder=output,
+                                       tmp_folder=tmp,
                                        output_folder=output)
 
         automl.fit(X_train, Y_train, X_test=X_test, y_test=Y_test,
@@ -356,13 +368,15 @@ def test_conversion_of_list_to_np(self, fit_ensemble, refit, fit):
 
 class AutoMLRegressorTest(Base, unittest.TestCase):
     def test_regression(self):
-        output = os.path.join(self.test_dir, '..', '.tmp_regression_fit')
+        tmp = os.path.join(self.test_dir, '..', '.tmp_regression_fit')
+        output = os.path.join(self.test_dir, '..', '.out_regression_fit')
+        self._setUp(tmp)
         self._setUp(output)
 
         X_train, Y_train, X_test, Y_test = putil.get_dataset('boston')
         automl = AutoSklearnRegressor(time_left_for_this_task=20,
                                       per_run_time_limit=5,
-                                      tmp_folder=output,
+                                      tmp_folder=tmp,
                                       output_folder=output)
 
         automl.fit(X_train, Y_train)
diff --git a/test/test_evaluation/test_train_evaluator.py b/test/test_evaluation/test_train_evaluator.py
--- a/test/test_evaluation/test_train_evaluator.py
+++ b/test/test_evaluation/test_train_evaluator.py
@@ -56,10 +56,11 @@ def test_holdout(self, pipeline_mock):
         pipeline_mock.predict_proba.side_effect = lambda X, batch_size: np.tile([0.6, 0.4], (len(X), 1))
         pipeline_mock.side_effect = lambda **kwargs: pipeline_mock
         pipeline_mock.get_additional_run_info.return_value = None
-        output_dir = os.path.join(os.getcwd(), '.test_holdout')
+        tmp_dir = os.path.join(os.getcwd(), '.out_test_holdout')
+        output_dir = os.path.join(os.getcwd(), '.tmp_test_holdout')
 
         configuration = unittest.mock.Mock(spec=Configuration)
-        backend_api = backend.create(output_dir, output_dir)
+        backend_api = backend.create(tmp_dir, output_dir)
         backend_api.load_datamanager = lambda: D
         queue_ = multiprocessing.Queue()
 
@@ -118,10 +119,11 @@ def configuration_fully_fitted(self):
         pipeline_mock.predict_proba.side_effect = lambda X, batch_size: np.tile([0.6, 0.4], (len(X), 1))
         pipeline_mock.get_additional_run_info.return_value = None
         pipeline_mock.side_effect = lambda **kwargs: pipeline_mock
-        output_dir = os.path.join(os.getcwd(), '.test_iterative_holdout')
+        tmp_dir = os.path.join(os.getcwd(), '.tmp_test_iterative_holdout')
+        output_dir = os.path.join(os.getcwd(), '.out_test_iterative_holdout')
 
         configuration = unittest.mock.Mock(spec=Configuration)
-        backend_api = backend.create(output_dir, output_dir)
+        backend_api = backend.create(tmp_dir, output_dir)
         backend_api.load_datamanager = lambda: D
         queue_ = multiprocessing.Queue()
 
@@ -199,10 +201,11 @@ def configuration_fully_fitted(self):
         pipeline_mock.predict_proba.side_effect = lambda X, batch_size: np.tile([0.6, 0.4], (len(X), 1))
         pipeline_mock.side_effect = lambda **kwargs: pipeline_mock
         pipeline_mock.get_additional_run_info.return_value = None
-        output_dir = os.path.join(os.getcwd(), '.test_iterative_holdout_interuption')
+        tmp_dir = os.path.join(os.getcwd(), '.tmp_test_iterative_holdout_interuption')
+        output_dir = os.path.join(os.getcwd(), '.out_test_iterative_holdout_interuption')
 
         configuration = unittest.mock.Mock(spec=Configuration)
-        backend_api = backend.create(output_dir, output_dir)
+        backend_api = backend.create(tmp_dir, output_dir)
         backend_api.load_datamanager = lambda: D
         queue_ = multiprocessing.Queue()
 
@@ -266,10 +269,11 @@ def test_iterative_holdout_not_iterative(self, pipeline_mock):
         pipeline_mock.predict_proba.side_effect = lambda X, batch_size: np.tile([0.6, 0.4], (len(X), 1))
         pipeline_mock.side_effect = lambda **kwargs: pipeline_mock
         pipeline_mock.get_additional_run_info.return_value = None
-        output_dir = os.path.join(os.getcwd(), '.test_iterative_holdout_not_iterative')
+        tmp_dir = os.path.join(os.getcwd(), '.tmp_test_iterative_holdout_not_iterative')
+        output_dir = os.path.join(os.getcwd(), '.out_test_iterative_holdout_not_iterative')
 
         configuration = unittest.mock.Mock(spec=Configuration)
-        backend_api = backend.create(output_dir, output_dir)
+        backend_api = backend.create(tmp_dir, output_dir)
         backend_api.load_datamanager = lambda: D
         queue_ = multiprocessing.Queue()
 
@@ -306,10 +310,11 @@ def test_cv(self, pipeline_mock):
         pipeline_mock.predict_proba.side_effect = lambda X, batch_size: np.tile([0.6, 0.4], (len(X), 1))
         pipeline_mock.side_effect = lambda **kwargs: pipeline_mock
         pipeline_mock.get_additional_run_info.return_value = None
-        output_dir = os.path.join(os.getcwd(), '.test_cv')
+        tmp_dir = os.path.join(os.getcwd(), '.tmp_test_cv')
+        output_dir = os.path.join(os.getcwd(), '.out_test_cv')
 
         configuration = unittest.mock.Mock(spec=Configuration)
-        backend_api = backend.create(output_dir, output_dir)
+        backend_api = backend.create(tmp_dir, output_dir)
         backend_api.load_datamanager = lambda : D
         queue_ = multiprocessing.Queue()
 
@@ -353,12 +358,13 @@ def test_partial_cv(self, pipeline_mock):
         pipeline_mock.predict_proba.side_effect = lambda X, batch_size: np.tile([0.6, 0.4], (len(X), 1))
         pipeline_mock.side_effect = lambda **kwargs: pipeline_mock
         pipeline_mock.get_additional_run_info.return_value = None
-        output_dir = os.path.join(os.getcwd(), '.test_partial_cv')
+        tmp_dir = os.path.join(os.getcwd(), '.tmp_test_partial_cv')
+        output_dir = os.path.join(os.getcwd(), '.out_test_partial_cv')
         D = get_binary_classification_datamanager()
         D.name = 'test'
 
         configuration = unittest.mock.Mock(spec=Configuration)
-        backend_api = backend.create(output_dir, output_dir)
+        backend_api = backend.create(tmp_dir, output_dir)
         backend_api.load_datamanager = lambda: D
         queue_ = multiprocessing.Queue()
 
@@ -411,10 +417,11 @@ def configuration_fully_fitted(self):
         pipeline_mock.predict_proba.side_effect = lambda X, batch_size: np.tile([0.6, 0.4], (len(X), 1))
         pipeline_mock.get_additional_run_info.return_value = None
         pipeline_mock.side_effect = lambda **kwargs: pipeline_mock
-        output_dir = os.path.join(os.getcwd(), '.test_iterative_partial_cv')
+        tmp_dir = os.path.join(os.getcwd(), '.tmp_test_iterative_partial_cv')
+        output_dir = os.path.join(os.getcwd(), '.out_test_iterative_partial_cv')
 
         configuration = unittest.mock.Mock(spec=Configuration)
-        backend_api = backend.create(output_dir, output_dir)
+        backend_api = backend.create(tmp_dir, output_dir)
         backend_api.load_datamanager = lambda: D
         queue_ = multiprocessing.Queue()