Repository: keras-team/autokeras

Issue:
in search.py train() function, handle other exceptions and log the error
<!---
**If you are reporting a bug:**
*  Verify that your issue is not being currently addressed by other issues or pull requests.
*  Please note that Auto-Keras is only compatible with **Python 3.6**.
*  Tag the issue with the `bug report` tag.
-->

### Bug Description
<!---
A clear and concise description of what the bug is.
-->
Any error occurred in the train() function should not block the search.
The search should just discard the current architecture and search for new ones.

### Reproducing Steps
Steps to reproduce the behavior:
  * Step 1: ...
  * Step 2: ...

### Expected Behavior
<!---
A clear and concise description of what you expected to happen.
-->
Any exception, they should all return None, None, None. And should all push (None, None, None) into the queue.

### Setup Details
Include the details about the versions of:
 - OS type and version:
 - Python: 
 - autokeras: <!--- e.g. 0.2.1, 0.2.0, master-->
 - scikit-learn:
 - numpy:
 - keras:
 - scipy:
 - tensorflow:
 - pytorch:

### Additional context
<!---
Add any other context about the problem here.
-->



Discussion:

Resolving patch:
diff --git a/autokeras/__init__.py b/autokeras/__init__.py
--- a/autokeras/__init__.py
+++ b/autokeras/__init__.py
@@ -1,8 +1,7 @@
 from autokeras.image.image_supervised import ImageClassifier, ImageRegressor, PortableImageSupervised
 from autokeras.text.text_supervised import TextClassifier, TextRegressor
-# from autokeras.tabular.tabular_supervised import TabularClassifier, TabularRegressor
-
 from autokeras.net_module import CnnModule, MlpModule
-
-# from autokeras.pretrained.object_detector import ObjectDetector
-# from autokeras.pretrained.face_detector import FaceDetector
\ No newline at end of file
+try:
+    from autokeras.tabular.tabular_supervised import TabularClassifier, TabularRegressor
+except OSError:
+    print('You need to "brew install libomp".')
diff --git a/autokeras/predefined_model.py b/autokeras/predefined_model.py
--- a/autokeras/predefined_model.py
+++ b/autokeras/predefined_model.py
@@ -1,4 +1,3 @@
-import os
 import numpy as np
 import torch
 from abc import ABC, abstractmethod
@@ -13,6 +12,7 @@
 from autokeras.constant import Constant
 from autokeras.preprocessor import ImageDataTransformer, OneHotEncoder
 
+
 class PredefinedModel(ABC):
     """The base class for the predefined model without architecture search
 
@@ -24,6 +24,7 @@ class PredefinedModel(ABC):
         verbose: A boolean of whether the search process will be printed to stdout.
         path: A string. The path to a directory, where the intermediate results are saved.
     """
+
     def __init__(self, y_encoder=OneHotEncoder, data_transformer_class=ImageDataTransformer,
                  verbose=False,
                  path=None):
@@ -81,7 +82,7 @@ def fit(self, x, y, trainer_args=None):
                                                             test_size=validation_set_size,
                                                             random_state=42)
 
-        #initialize data_transformer
+        # initialize data_transformer
         self.data_transformer = self.data_transformer_class(x_train)
         # Wrap the data into DataLoaders
         train_loader = self.data_transformer.transform_train(x_train, y_train)
@@ -122,14 +123,6 @@ def evaluate(self, x_test, y_test):
         y_predict = self.predict(x_test)
         return self.metric().evaluate(y_predict, y_test)
 
-    def save(self, model_path):
-        """Save the model as keras format.
-
-        Args:
-            model_path: the path to save model.
-        """
-        self.graph.produce_keras_model().save(model_path)
-
 
 class PredefinedResnet(PredefinedModel):
     def _init_generator(self, n_output_node, input_shape):
@@ -138,4 +131,4 @@ def _init_generator(self, n_output_node, input_shape):
 
 class PredefinedDensenet(PredefinedModel):
     def _init_generator(self, n_output_node, input_shape):
-        return DenseNetGenerator(n_output_node, input_shape)
\ No newline at end of file
+        return DenseNetGenerator(n_output_node, input_shape)
diff --git a/autokeras/search.py b/autokeras/search.py
--- a/autokeras/search.py
+++ b/autokeras/search.py
@@ -4,12 +4,12 @@
 import re
 import sys
 import time
-from datetime import datetime
-from abc import abstractmethod
-
 import torch
 import torch.multiprocessing as mp
 
+
+from abc import abstractmethod
+from datetime import datetime
 from autokeras.bayesian import BayesianOptimizer
 from autokeras.constant import Constant
 from autokeras.nn.model_trainer import ModelTrainer
@@ -85,11 +85,13 @@ def __init__(self, n_output_node, input_shape, path, metric, loss, generators, v
         self.training_queue = []
         self.x_queue = []
         self.y_queue = []
+
         logging.basicConfig(filename=os.path.join(self.path, datetime.now().strftime('run_%d_%m_%Y : _%H_%M.log')),
                             format='%(asctime)s - %(filename)s - %(message)s', level=logging.DEBUG)
 
         self._timeout = None
 
+
     def load_model_by_id(self, model_id):
         return pickle_from_file(os.path.join(self.path, str(model_id) + '.graph'))
 
@@ -354,7 +356,16 @@ def train(q, graph, train_data, test_data, trainer_args, metric, loss, verbose,
         if q:
             q.put((None, None, None))
         return None, None, None
-    except TimeoutError:
+    except TimeoutError as exp:
+        logging.warning("TimeoutError occurred at train() : {0}".format(str(exp)))
         if q:
             q.put((None, None, None))
-        return None, None, None
\ No newline at end of file
+        return None, None, None
+    except Exception as exp:
+        logging.warning("Exception occurred at train() : {0}".format(str(exp)))
+        if verbose:
+            print("Exception occurred at train() : {0}".format(str(exp)))
+        if q:
+            q.put((None, None, None))
+        return None, None, None
+


Test patch:
diff --git a/tests/common.py b/tests/common.py
--- a/tests/common.py
+++ b/tests/common.py
@@ -280,6 +280,7 @@ def qsize(self):
                 if self.count > 8:
                     return 1
                 return 0
+
         # (0.5, 0.8, get_pooling_model())
         return MockQueue()
 
@@ -311,11 +312,15 @@ def mock_train(**kwargs):
     return 1, 0
 
 
+def mock_exception_handling_train(**kwargs):
+    str(kwargs)
+    raise Exception
+
+
 def mock_out_of_memory_train(**kwargs):
     str(kwargs)
     raise RuntimeError('CUDA: out of memory.')
 
-
 # def mock_nvidia_smi_output(*arg, **kwargs):
 #     return \
 #         '    Free                        : 1 MiB \n' \
diff --git a/tests/tabular/test_tabular_supervised.py b/tests/tabular/test_tabular_supervised.py
--- a/tests/tabular/test_tabular_supervised.py
+++ b/tests/tabular/test_tabular_supervised.py
@@ -7,6 +7,8 @@
 
 
 def test_fit_evalute_predict_binary_classification():
+    train_x = None
+    train_y = None
     clean_dir(TEST_TEMP_DIR)
     clf = TabularClassifier(path=TEST_TEMP_DIR)
     nsample = 59
@@ -39,7 +41,7 @@ def test_fit_evalute_predict_multiclass_classification():
     clean_dir(TEST_TEMP_DIR)
     clf = TabularClassifier(path=TEST_TEMP_DIR)
     clf.verbose = True
-    nsample = 10000
+    nsample = 1000
     [ntime, nnum, ncat] = [11, 15, 13]
     datainfo = np.array(['TIME'] * ntime + ['NUM'] * nnum + ['CAT'] * ncat)
     x_num = np.random.random([nsample, nnum])
@@ -58,7 +60,7 @@ def test_fit_evalute_predict_multiclass_classification():
 def test_fit_predict_evalute_regression():
     clean_dir(TEST_TEMP_DIR)
     clf = TabularRegressor(path=TEST_TEMP_DIR)
-    nsample = 10000
+    nsample = 1000
     [ntime, nnum, ncat] = [3, 15, 3]
     datainfo = np.array(['TIME'] * ntime + ['NUM'] * nnum + ['CAT'] * ncat)
     x_num = np.random.random([nsample, nnum])
diff --git a/tests/test_predefined_model.py b/tests/test_predefined_model.py
--- a/tests/test_predefined_model.py
+++ b/tests/test_predefined_model.py
@@ -1,14 +1,12 @@
-import os
 from unittest.mock import patch
 
-import pytest
-
 from autokeras.predefined_model import *
-from tests.common import clean_dir, mock_train, TEST_TEMP_DIR
+from tests.common import clean_dir, mock_train, TEST_TEMP_DIR, MockProcess
 
 
+@patch('torch.multiprocessing.get_context', side_effect=MockProcess)
 @patch('autokeras.search.ModelTrainer.train_model', side_effect=mock_train)
-def test_fit_predict_save(_):
+def test_fit_predict_save(_, _1):
     train_x = np.random.rand(100, 25, 25, 1)
     train_y = np.random.randint(0, 5, 100)
     for Model in [PredefinedResnet, PredefinedDensenet]:
@@ -18,8 +16,5 @@ def test_fit_predict_save(_):
         assert all(map(lambda result: result in train_y, results))
         score = clf.evaluate(train_x, train_y)
         assert score <= 1.0
-        model_path = os.path.join(TEST_TEMP_DIR, 'kerasmodel')
-        clf.save(model_path)
-        assert os.path.isfile(model_path)
 
-    clean_dir(TEST_TEMP_DIR)
\ No newline at end of file
+    clean_dir(TEST_TEMP_DIR)
diff --git a/tests/test_search.py b/tests/test_search.py
--- a/tests/test_search.py
+++ b/tests/test_search.py
@@ -6,7 +6,8 @@
 from autokeras.search import *
 from autokeras.nn.generator import CnnGenerator, MlpGenerator, ResNetGenerator
 from tests.common import clean_dir, MockProcess, get_classification_data_loaders, get_classification_data_loaders_mlp, \
-    simple_transform, TEST_TEMP_DIR, simple_transform_mlp, mock_train, mock_out_of_memory_train
+    simple_transform, TEST_TEMP_DIR, simple_transform_mlp, mock_train, mock_out_of_memory_train, \
+    mock_exception_handling_train
 
 from nas.greedy import GreedySearcher
 from nas.grid import GridSearcher
@@ -20,7 +21,7 @@ def test_bayesian_searcher(_, _1, _2):
     train_data, test_data = get_classification_data_loaders()
     clean_dir(TEST_TEMP_DIR)
     searcher = BayesianSearcher(3, (28, 28, 3), verbose=False, path=TEST_TEMP_DIR, metric=Accuracy,
-                        loss=classification_loss, generators=[CnnGenerator, CnnGenerator])
+                                loss=classification_loss, generators=[CnnGenerator, CnnGenerator])
     Constant.N_NEIGHBOURS = 1
     Constant.T_MIN = 0.8
     for _ in range(2):
@@ -29,6 +30,22 @@ def test_bayesian_searcher(_, _1, _2):
     assert len(searcher.history) == 2
 
 
+@patch('torch.multiprocessing.get_context', side_effect=MockProcess)
+@patch('autokeras.search.ModelTrainer.train_model', side_effect=mock_exception_handling_train)
+def test_exception_handling(_, _2):
+    train_data, test_data = get_classification_data_loaders()
+    clean_dir(TEST_TEMP_DIR)
+    Constant.N_NEIGHBOURS = 2
+    Constant.T_MIN = 0.8
+    Constant.BETA = 1
+    generator = BayesianSearcher(3, (28, 28, 3), verbose=True, path=TEST_TEMP_DIR, metric=Accuracy,
+                                 loss=classification_loss, generators=[CnnGenerator, ResNetGenerator])
+    for _ in range(3):
+        generator.search(train_data, test_data)
+    clean_dir(TEST_TEMP_DIR)
+    assert len(generator.history) == 0
+
+
 @patch('torch.multiprocessing.get_context', side_effect=MockProcess)
 @patch('autokeras.bayesian.transform', side_effect=simple_transform)
 @patch('autokeras.search.ModelTrainer.train_model', side_effect=mock_train)
@@ -37,7 +54,7 @@ def test_bayesian_searcher_sp(_, _1, _2, _3):
     train_data, test_data = get_classification_data_loaders()
     clean_dir(TEST_TEMP_DIR)
     searcher = BayesianSearcher(3, (28, 28, 3), verbose=False, path=TEST_TEMP_DIR, metric=Accuracy,
-                        loss=classification_loss, generators=[CnnGenerator, CnnGenerator])
+                                loss=classification_loss, generators=[CnnGenerator, CnnGenerator])
     Constant.N_NEIGHBOURS = 1
     Constant.T_MIN = 0.8
     for _ in range(2):
@@ -53,7 +70,7 @@ def test_bayesian_searcher_mlp(_, _1, _2):
     train_data, test_data = get_classification_data_loaders_mlp()
     clean_dir(TEST_TEMP_DIR)
     generator = BayesianSearcher(3, (28,), verbose=False, path=TEST_TEMP_DIR, metric=Accuracy,
-                         loss=classification_loss, generators=[MlpGenerator, MlpGenerator])
+                                 loss=classification_loss, generators=[MlpGenerator, MlpGenerator])
     Constant.N_NEIGHBOURS = 1
     Constant.T_MIN = 0.8
     for _ in range(2):
@@ -69,7 +86,7 @@ def test_greedy_searcher(_, _1, _2):
     train_data, test_data = get_classification_data_loaders()
     clean_dir(TEST_TEMP_DIR)
     searcher = GreedySearcher(3, (28, 28, 3), verbose=False, path=TEST_TEMP_DIR, metric=Accuracy,
-                        loss=classification_loss, generators=[CnnGenerator, CnnGenerator])
+                              loss=classification_loss, generators=[CnnGenerator, CnnGenerator])
     for _ in range(2):
         searcher.search(train_data, test_data)
     clean_dir(TEST_TEMP_DIR)
@@ -84,7 +101,7 @@ def test_greedy_searcher_sp(_, _1, _2, _3):
     train_data, test_data = get_classification_data_loaders()
     clean_dir(TEST_TEMP_DIR)
     searcher = GreedySearcher(3, (28, 28, 3), verbose=False, path=TEST_TEMP_DIR, metric=Accuracy,
-                        loss=classification_loss, generators=[CnnGenerator, CnnGenerator])
+                              loss=classification_loss, generators=[CnnGenerator, CnnGenerator])
     for _ in range(2):
         searcher.search(train_data, test_data)
     clean_dir(TEST_TEMP_DIR)
@@ -98,7 +115,7 @@ def test_greedy_searcher_mlp(_, _1, _2):
     train_data, test_data = get_classification_data_loaders_mlp()
     clean_dir(TEST_TEMP_DIR)
     generator = GreedySearcher(3, (28,), verbose=False, path=TEST_TEMP_DIR, metric=Accuracy,
-                         loss=classification_loss, generators=[MlpGenerator, MlpGenerator])
+                               loss=classification_loss, generators=[MlpGenerator, MlpGenerator])
     for _ in range(2):
         generator.search(train_data, test_data)
     clean_dir(TEST_TEMP_DIR)
@@ -112,7 +129,7 @@ def test_random_searcher(_, _1, _2):
     train_data, test_data = get_classification_data_loaders()
     clean_dir(TEST_TEMP_DIR)
     searcher = RandomSearcher(3, (28, 28, 3), verbose=False, path=TEST_TEMP_DIR, metric=Accuracy,
-                        loss=classification_loss, generators=[CnnGenerator, CnnGenerator])
+                              loss=classification_loss, generators=[CnnGenerator, CnnGenerator])
     for _ in range(2):
         searcher.search(train_data, test_data)
     clean_dir(TEST_TEMP_DIR)
@@ -127,7 +144,7 @@ def test_random_searcher_sp(_, _1, _2, _3):
     train_data, test_data = get_classification_data_loaders()
     clean_dir(TEST_TEMP_DIR)
     searcher = RandomSearcher(3, (28, 28, 3), verbose=False, path=TEST_TEMP_DIR, metric=Accuracy,
-                        loss=classification_loss, generators=[CnnGenerator, CnnGenerator])
+                              loss=classification_loss, generators=[CnnGenerator, CnnGenerator])
     for _ in range(2):
         searcher.search(train_data, test_data)
     clean_dir(TEST_TEMP_DIR)
@@ -141,7 +158,7 @@ def test_grid_searcher(_, _1, _2):
     train_data, test_data = get_classification_data_loaders()
     clean_dir(TEST_TEMP_DIR)
     searcher = GridSearcher(3, (28, 28, 3), verbose=True, path=TEST_TEMP_DIR, metric=Accuracy,
-                        loss=classification_loss, generators=[CnnGenerator, CnnGenerator])
+                            loss=classification_loss, generators=[CnnGenerator, CnnGenerator])
     Constant.N_NEIGHBOURS = 1
     Constant.T_MIN = 0.8
     print(len(searcher.get_search_dimensions()))
@@ -161,7 +178,7 @@ def test_max_acq(_, _2):
     Constant.T_MIN = 0.8
     Constant.BETA = 1
     generator = BayesianSearcher(3, (28, 28, 3), verbose=False, path=TEST_TEMP_DIR, metric=Accuracy,
-                         loss=classification_loss, generators=[CnnGenerator, ResNetGenerator])
+                                 loss=classification_loss, generators=[CnnGenerator, ResNetGenerator])
     for _ in range(3):
         generator.search(train_data, test_data)
     for index1, descriptor1 in enumerate(generator.descriptors):
@@ -181,7 +198,7 @@ def test_out_of_memory(_, _2):
     Constant.T_MIN = 0.8
     Constant.BETA = 1
     generator = BayesianSearcher(3, (28, 28, 3), verbose=True, path=TEST_TEMP_DIR, metric=Accuracy,
-                         loss=classification_loss, generators=[CnnGenerator, ResNetGenerator])
+                                 loss=classification_loss, generators=[CnnGenerator, ResNetGenerator])
     for _ in range(3):
         generator.search(train_data, test_data)
     clean_dir(TEST_TEMP_DIR)