Repository: keras-team/autokeras

Issue:
"numpy.linalg.linalg.LinAlgError: N-th leading minor of the array is not positive definite" exception while training ImageClassifier
<!---
**If you are reporting a bug:**
*  Verify that your issue is not being currently addressed by other issues or pull requests.
*  Please note that Auto-Keras is only compatible with **Python 3.6**.
*  Tag the issue with the `bug report` tag.
-->

### Bug Description

When training ImageClassifier on float inputs of shape 80x92x24 on a binary classification task, this exception is thrown when AutoKeras lists all created models until this point and their respected losses and metrics. Here is a more complete log + stack trace:  http://termbin.com/gb3n

### Reproducing Steps

* Create a dataset of noise shaped 80x92x24 with ~360 or so samples
*  Respective binary labels.

In my case, some values of the dataset are quite large (they vary between 0 and 1_000_000, but I'm not sure that this is related since AutoKeras performs normalization).

* Have it train on a GTX 1070Ti or equivalent in terms of vram for about an hour (can't confirm that it's GPU-related either).
* Wait for the exception


### Expected Behavior

It should work.

### Setup Details
 - OS type and version: Ubuntu 18.04
 - Python: 3.6
 - autokeras: 0.2.14 (installed manually from git)
- all dependencies are installed as stated in requirements.txt for this release, unless for:
- tensorflow 1.10, compiled locally. 



Discussion:
@Zvezdin Thank you for the report.
I think this has been fixed in the develop branch.
Will be released soon.

Resolving patch:
diff --git a/autokeras/image_supervised.py b/autokeras/image_supervised.py
--- a/autokeras/image_supervised.py
+++ b/autokeras/image_supervised.py
@@ -248,7 +248,7 @@ def fit(self, x_train=None, y_train=None, time_limit=None):
             if len(self.load_searcher().history) == 0:
                 raise TimeoutError("Search Time too short. No model was found during the search time.")
             elif self.verbose:
-                print('Time is out.')
+                print('\nTime limit for model search is reached. Ending the model search.')
 
     @abstractmethod
     def get_n_output_node(self):
@@ -315,6 +315,8 @@ def final_fit(self, x_train, y_train, x_test, y_test, trainer_args=None, retrain
 
         searcher = self.load_searcher()
         graph = searcher.load_best_model()
+        if self.verbose:
+            print('\nLoading and training the best model recorded from the search.')
 
         if retrain:
             graph.weighted = False
diff --git a/autokeras/model_trainer.py b/autokeras/model_trainer.py
--- a/autokeras/model_trainer.py
+++ b/autokeras/model_trainer.py
@@ -64,6 +64,8 @@ def __init__(self, model, **kwargs):
         self.model.to(self.device)
         self.optimizer = None
         self.early_stop = None
+        self.current_epoch = 0
+        self.current_metric_value = 0
 
     def train_model(self,
                     max_iter_num=None,
@@ -89,40 +91,17 @@ def train_model(self,
         test_loss_list = []
         self.optimizer = torch.optim.Adam(self.model.parameters())
 
-        progress_bar = None
-        if self.verbose:
-            progress_bar = tqdm(total=max_iter_num,
-                                desc='    Model    ',
-                                file=sys.stdout,
-                                leave=False,
-                                ncols=75,
-                                position=1,
-                                unit=' epoch')
-
         for epoch in range(max_iter_num):
             self._train()
             test_loss, metric_value = self._test()
+            self.current_metric_value = metric_value
             test_metric_value_list.append(metric_value)
             test_loss_list.append(test_loss)
-            if self.verbose:
-                progress_bar.update(1)
-                if epoch == 0:
-                    header = ['Epoch', 'Loss', 'Accuracy']
-                    line = '|'.join(x.center(24) for x in header)
-                    progress_bar.write('+' + '-' * len(line) + '+')
-                    progress_bar.write('|' + line + '|')
-                    progress_bar.write('+' + '-' * len(line) + '+')
-                r = [epoch + 1, test_loss, metric_value]
-                line = '|'.join(str(x).center(24) for x in r)
-                progress_bar.write('|' + line + '|')
-                progress_bar.write('+' + '-' * len(line) + '+')
             decreasing = self.early_stop.on_epoch_end(test_loss)
             if not decreasing:
                 if self.verbose:
                     print('\nNo loss decrease after {} epochs.\n'.format(max_no_improvement_num))
                 break
-        if self.verbose:
-            progress_bar.close()
         last_num = min(max_no_improvement_num, max_iter_num)
         return (sum(test_loss_list[-last_num:]) / last_num,
                 sum(test_metric_value_list[-last_num:]) / last_num)
@@ -130,14 +109,15 @@ def train_model(self,
     def _train(self):
         self.model.train()
         loader = self.train_loader
+        self.current_epoch += 1
 
         cp_loader = deepcopy(loader)
         if self.verbose:
             progress_bar = tqdm(total=len(cp_loader),
-                                desc='Current Epoch',
+                                desc='Epoch-' + str(self.current_epoch) + ', Current Metric - ' + str(self.current_metric_value),
                                 file=sys.stdout,
                                 leave=False,
-                                ncols=75,
+                                ncols=100,
                                 position=0,
                                 unit=' batch')
 
diff --git a/autokeras/search.py b/autokeras/search.py
--- a/autokeras/search.py
+++ b/autokeras/search.py
@@ -129,7 +129,9 @@ def add_model(self, metric_value, loss, graph, model_id):
             line = '|'.join(x.center(24) for x in header)
             print('+' + '-' * len(line) + '+')
             print('|' + line + '|')
-            for i, r in enumerate(self.history):
+
+            if self.history:
+                r = self.history[-1]
                 print('+' + '-' * len(line) + '+')
                 line = '|'.join(str(r[x]).center(24) for x in idx)
                 print('|' + line + '|')
@@ -219,7 +221,7 @@ def search(self, train_data, test_data, timeout=60 * 60 * 24):
             if not re.search('out of memory', str(e)):
                 raise e
             if self.verbose:
-                print('out of memory')
+                print('\nCurrent model size is too big. Discontinuing training this model to search for other models.')
             Constant.MAX_MODEL_SIZE = graph.size() - 1
             return
         finally:


Test patch: