Repository: keras-team/autokeras

Issue:
Minimum hardware required 
What is the minimal hardware required to run a search ?  On CIFAR, a Nvidia GTX 1080 8Go doesn't  seems to be enough.  The GPU memory used varies a lot according to the model built. Is there a way to limit the model build space or something like that ?

```
  File "cifar.py", line 17, in <module> clf.fit(x_train, y_train, time_limit=12 * 60 * 60)  
  File "/myPath/python3.6/site-packages/autokeras/image_supervised.py", line 238, in fit run_searcher_once(train_data, test_data, self.path, int(time_remain))  
  File "/myPath/python3.6/site-packages/autokeras/image_supervised.py", line 41, in run_searcher_once searcher.search(train_data, test_data, timeout)  
  File "/myPath/python3.6/site-packages/autokeras/search.py", line 177, in search self.metric, self.loss, self.verbose)]))  
  File "/myPath/python3.6/site-packages/autokeras/search.py", line 280, in train verbose).train_model(**trainer_args)  
  File "/myPath/python3.6/site-packages/autokeras/utils.py", line 124, in train_model self._train()  
  File "/myPath/python3.6/site-packages/autokeras/utils.py", line 169, in _train loss.backward()  
  File "/myPath/python3.6/site-packages/torch/tensor.py", line 93, in backward torch.autograd.backward(self, gradient, retain_graph, create_graph)  
  File "/myPath/python3.6/site-packages/torch/autograd/__init__.py", line 90, in backward allow_unreachable=True)  # allow_unreachable flag  
  RuntimeError: CUDA error: out of memory```


Discussion:
@MattVil @fliessen 
You can import autokeras.constant.Constant. Then change the value of Constant.MAX_MODEL_WIDTH and Constant.MAX_MODEL_DEPTH to limit the memory usage.

We are trying to find more complicated ways to limit the memory.
Will close this issue afterward.

thanks.
This looks more like a memory allocation/deallocation problem.
I am running out of memory on a regular basis. Using max_model_width and depth does not help. Neither does limiting 'max_iter_num'. It simply delays the problem.

The good thing is that you build autokeras with snapshots that allow to continue training after an error. In such a case all memory is deallocated and if I restart/continue training, then I am able to train the model for quite some time again before it happens again.

I am not so sure if this problem can be solved by PyTorchs cuda management (https://pytorch.org/docs/stable/notes/cuda.html). Compared to tensorflows GPU management it is still quite poor. 

I would suggest to implement full GPU memory deallocation after each model or after a defineable threshold of GPU memory.

Resolving patch:
diff --git a/autokeras/constant.py b/autokeras/constant.py
--- a/autokeras/constant.py
+++ b/autokeras/constant.py
@@ -10,7 +10,7 @@ class Constant:
     KERNEL_LAMBDA = 0.1
     T_MIN = 0.0001
     N_NEIGHBOURS = 8
-    MAX_MODEL_SIZE = (1 << 17)
+    MAX_MODEL_SIZE = (1 << 25)
 
     # Model Defaults
 
diff --git a/autokeras/search.py b/autokeras/search.py
--- a/autokeras/search.py
+++ b/autokeras/search.py
@@ -1,4 +1,5 @@
 import os
+import re
 import time
 
 import torch
@@ -172,13 +173,17 @@ def search(self, train_data, test_data, timeout=60 * 60 * 24):
             print('╘' + '=' * 46 + '╛')
         mp.set_start_method('spawn', force=True)
         pool = mp.Pool(1)
-        train_results = pool.map_async(train, [(graph, train_data, test_data, self.trainer_args,
-                                                os.path.join(self.path, str(model_id) + '.png'),
-                                                self.metric, self.loss, self.verbose)])
-
-        # Do the search in current thread.
         try:
+            train_results = pool.map_async(train, [(graph, train_data, test_data, self.trainer_args,
+                                                    os.path.join(self.path, str(model_id) + '.png'),
+                                                    self.metric, self.loss, self.verbose)])
+
+            # Do the search in current thread.
+            searched = False
+            new_graph = None
+            new_father_id = None
             if not self.training_queue:
+                searched = True
                 new_graph, new_father_id = self.bo.optimize_acq(self.search_tree.adj_list.keys(),
                                                                 self.descriptors,
                                                                 timeout)
@@ -190,40 +195,50 @@ def search(self, train_data, test_data, timeout=60 * 60 * 24):
                 self.training_queue.append((new_graph, new_father_id, new_model_id))
                 self.descriptors.append(new_graph.extract_descriptor())
 
-                if self.verbose:
-                    cell_size = [24, 49]
-                    header = ['Father Model ID', 'Added Operation']
-                    line = '|'.join(str(x).center(cell_size[i]) for i, x in enumerate(header))
-                    print('\n' + '+' + '-' * len(line) + '+')
-                    print('|' + line + '|')
-                    print('+' + '-' * len(line) + '+')
-                    for i in range(len(new_graph.operation_history)):
-                        if i == len(new_graph.operation_history) // 2:
-                            r = [new_father_id, new_graph.operation_history[i]]
-                        else:
-                            r = [' ', new_graph.operation_history[i]]
-                        line = '|'.join(str(x).center(cell_size[i]) for i, x in enumerate(r))
-                        print('|' + line + '|')
-                    print('+' + '-' * len(line) + '+')
             remaining_time = timeout - (time.time() - start_time)
-            if remaining_time > 0:
-                metric_value, loss, graph = train_results.get(timeout=remaining_time)[0]
-            else:
+            if remaining_time <= 0:
                 raise TimeoutError
+
+            metric_value, loss, graph = train_results.get(timeout=remaining_time)[0]
+
+            if self.verbose and searched:
+                cell_size = [24, 49]
+                header = ['Father Model ID', 'Added Operation']
+                line = '|'.join(str(x).center(cell_size[i]) for i, x in enumerate(header))
+                print('\n' + '+' + '-' * len(line) + '+')
+                print('|' + line + '|')
+                print('+' + '-' * len(line) + '+')
+                for i in range(len(new_graph.operation_history)):
+                    if i == len(new_graph.operation_history) // 2:
+                        r = [new_father_id, new_graph.operation_history[i]]
+                    else:
+                        r = [' ', new_graph.operation_history[i]]
+                    line = '|'.join(str(x).center(cell_size[i]) for i, x in enumerate(r))
+                    print('|' + line + '|')
+                print('+' + '-' * len(line) + '+')
+
+            self.add_model(metric_value, loss, graph, model_id)
+            self.search_tree.add_child(father_id, model_id)
+            self.bo.fit(self.x_queue, self.y_queue)
+            self.x_queue = []
+            self.y_queue = []
+
+            pickle_to_file(self, os.path.join(self.path, 'searcher'))
+            self.export_json(os.path.join(self.path, 'history.json'))
+
         except (mp.TimeoutError, TimeoutError) as e:
             raise TimeoutError from e
+        except RuntimeError as e:
+            if not re.search('out of memory', str(e)):
+                raise e
+            if self.verbose:
+                print('out of memory')
+            Constant.MAX_MODEL_SIZE = graph.size() - 1
+            return
         finally:
             # terminate and join the subprocess to prevent any resource leak
             pool.close()
             pool.join()
-        self.add_model(metric_value, loss, graph, model_id)
-        self.search_tree.add_child(father_id, model_id)
-        self.bo.fit(self.x_queue, self.y_queue)
-        self.x_queue = []
-        self.y_queue = []
-
-        pickle_to_file(self, os.path.join(self.path, 'searcher'))
-        self.export_json(os.path.join(self.path, 'history.json'))
 
     def export_json(self, path):
         data = dict()


Test patch:
diff --git a/tests/common.py b/tests/common.py
--- a/tests/common.py
+++ b/tests/common.py
@@ -247,6 +247,11 @@ def close(self):
         pass
 
 
+class MockMemoryOutProcess(MockProcess):
+    def map_async(self, a, b):
+        raise RuntimeError('cuda: out of memory.')
+
+
 def simple_transform(graph):
     graph.to_wider_model(5, 64)
     return [deepcopy(graph)]
diff --git a/tests/test_search.py b/tests/test_search.py
--- a/tests/test_search.py
+++ b/tests/test_search.py
@@ -5,7 +5,7 @@
 from autokeras.search import *
 
 from tests.common import clean_dir, MockProcess, get_classification_data_loaders, get_add_skip_model, \
-    get_concat_skip_model, simple_transform
+    get_concat_skip_model, simple_transform, MockMemoryOutProcess
 
 default_test_path = 'tests/resources/temp'
 
@@ -85,3 +85,19 @@ def test_max_acq(_, _1):
             assert edit_distance(descriptor1, descriptor2, 1) != 0
 
     clean_dir(default_test_path)
+
+
+@patch('torch.multiprocessing.Pool', new=MockMemoryOutProcess)
+@patch('autokeras.bayesian.transform', side_effect=simple_transform)
+@patch('autokeras.search.ModelTrainer.train_model', side_effect=mock_train)
+def test_out_of_memory(_, _1):
+    train_data, test_data = get_classification_data_loaders()
+    clean_dir(default_test_path)
+    searcher = Searcher(3, (28, 28, 3), verbose=False, path=default_test_path, metric=Accuracy,
+                        loss=classification_loss)
+    Constant.N_NEIGHBOURS = 1
+    Constant.T_MIN = 0.8
+    for _ in range(4):
+        searcher.search(train_data, test_data)
+    clean_dir(default_test_path)
+    assert len(searcher.history) == 0