Repository: keras-team/autokeras

Issue:
MemoryError on mnist.py

### Bug Description
When running mnist.py after some time, suddenly an error occurred : 

.......Traceback (most recent call last):
  File "mnist.py", line 10, in <module>
    clf.fit(x_train, y_train, time_limit=12 * 60 * 60)
  File "/home/jdiez/lib/anaconda3/envs/autokeras/lib/python3.6/site-packages/autokeras/image_supervised.py", line 238, in fit
    run_searcher_once(train_data, test_data, self.path, int(time_remain))
  File "/home/jdiez/lib/anaconda3/envs/autokeras/lib/python3.6/site-packages/autokeras/image_supervised.py", line 41, in run_searcher_once
    searcher.search(train_data, test_data, timeout)
  File "/home/jdiez/lib/anaconda3/envs/autokeras/lib/python3.6/site-packages/autokeras/search.py", line 175, in search
    timeout)
  File "/home/jdiez/lib/anaconda3/envs/autokeras/lib/python3.6/site-packages/autokeras/bayesian.py", line 257, in optimize_acq
    for temp_graph in transform(elem.graph):
  File "/home/jdiez/lib/anaconda3/envs/autokeras/lib/python3.6/site-packages/autokeras/net_transformer.py", line 85, in transform
    graphs.append(to_deeper_graph(deepcopy(graph)))
  File "/home/jdiez/lib/anaconda3/envs/autokeras/lib/python3.6/site-packages/autokeras/net_transformer.py", line 66, in to_deeper_graph
    graph.to_conv_deeper_model(layer_id, randint(1, 2) * 2 + 1)
  File "/home/jdiez/lib/anaconda3/envs/autokeras/lib/python3.6/site-packages/autokeras/graph.py", line 313, in to_conv_deeper_model
    new_layers = deeper_conv_block(target, kernel_size, self.weighted)
  File "/home/jdiez/lib/anaconda3/envs/autokeras/lib/python3.6/site-packages/autokeras/layer_transformer.py", line 11, in deeper_conv_block
    weight = np.zeros((n_filters, n_filters) + filter_shape)
MemoryError

Process finished with exit code 1

### Reproducing Steps
Steps to reproduce the behavior:
  * Step 1: Follow the instruction steps for 0.2.4 install
  * Step 2: Run mnist.py

### Expected Behavior
Run without error

### Setup Details
Include the details about the versions of:
 - OS type and version: Ubuntu 16.04 
 - Python: 3.6
 - autokeras: 0.2.4
 - scikit-learn: 0.19.1
 - numpy: 1.14.5
 - keras: 2.2.2
 - scipy: 1.1.0
 - tensorflow: 1.8.0
 - pytorch: 0.4.1 (py36_cuda9.2.148_cudnn7.1.4_1)

### Additional context
Torch, Tensorflow and Keras are all working fine insisde this conda env.
The same error occurs when adapting the code for CIFAR10, Fashion-MNIST




Discussion:
I've encountered the same bug running:

Debian 9.5
TensorFlow 1.10
Python 3.6.6
keras 2.2.2

The error is not always reproducible. Sometimes running mnist.py results in no errors and a graceful exit after the 20th model converges. This is using Nvidia K80 on Google Cloud.

Resolving patch:
diff --git a/autokeras/bayesian.py b/autokeras/bayesian.py
--- a/autokeras/bayesian.py
+++ b/autokeras/bayesian.py
@@ -245,15 +245,15 @@ def optimize_acq(self, model_ids, descriptors, timeout):
         t_min = self.t_min
         alpha = 0.9
         opt_acq = self._get_init_opt_acq_value()
-        remaining_time = timeout - (time.time() - start_time)
+        remaining_time = timeout
         while not pq.empty() and t > t_min and remaining_time > 0:
             elem = pq.get()
             if self.metric.higher_better():
-                temp_exp = min((elem.metric_value - opt_acq) / t, 709.0)
+                temp_exp = min((elem.metric_value - opt_acq) / t, 1.0)
             else:
-                temp_exp = min((opt_acq - elem.metric_value) / t, 709.0)
+                temp_exp = min((opt_acq - elem.metric_value) / t, 1.0)
             ap = math.exp(temp_exp)
-            if ap > random.uniform(0, 1):
+            if ap >= random.uniform(0, 1):
                 for temp_graph in transform(elem.graph):
                     if contain(descriptors, temp_graph.extract_descriptor()):
                         continue
diff --git a/autokeras/constant.py b/autokeras/constant.py
--- a/autokeras/constant.py
+++ b/autokeras/constant.py
@@ -10,8 +10,8 @@ class Constant:
     KERNEL_LAMBDA = 0.1
     T_MIN = 0.0001
     N_NEIGHBOURS = 8
-    # T_MIN = 0.8
-    # N_NEIGHBOURS = 1
+    MAX_MODEL_WIDTH = 2048
+    MAX_MODEL_DEPTH = 15
 
     # Model Defaults
 
diff --git a/autokeras/net_transformer.py b/autokeras/net_transformer.py
--- a/autokeras/net_transformer.py
+++ b/autokeras/net_transformer.py
@@ -5,11 +5,16 @@
 from autokeras.graph import NetworkDescriptor
 
 from autokeras.constant import Constant
-from autokeras.layers import is_layer
+from autokeras.layers import is_layer, layer_width
 
 
 def to_wider_graph(graph):
     weighted_layer_ids = graph.wide_layer_ids()
+    weighted_layer_ids = list(filter(lambda x: layer_width(graph.layer_list[x]) * 2 <= Constant.MAX_MODEL_WIDTH,
+                                     weighted_layer_ids))
+
+    if len(weighted_layer_ids) == 0:
+        return None
     # n_wider_layer = randint(1, len(weighted_layer_ids))
     # wider_layers = sample(weighted_layer_ids, n_wider_layer)
     wider_layers = sample(weighted_layer_ids, 1)
@@ -56,6 +61,9 @@ def to_skip_connection_graph(graph):
 
 def to_deeper_graph(graph):
     weighted_layer_ids = graph.deep_layer_ids()
+    if len(weighted_layer_ids) >= Constant.MAX_MODEL_DEPTH:
+        return None
+
     deeper_layer_ids = sample(weighted_layer_ids, 1)
     # n_deeper_layer = randint(1, len(weighted_layer_ids))
     # deeper_layer_ids = sample(weighted_layer_ids, n_deeper_layer)
@@ -87,6 +95,7 @@ def transform(graph):
             graphs.append(to_wider_graph(deepcopy(graph)))
         elif a == 2:
             graphs.append(to_skip_connection_graph(deepcopy(graph)))
+    graphs = list(filter(lambda x: x, graphs))
     return list(filter(lambda x: legal_graph(x), graphs))
 
 


Test patch:
diff --git a/tests/test_net_transformer.py b/tests/test_net_transformer.py
--- a/tests/test_net_transformer.py
+++ b/tests/test_net_transformer.py
@@ -53,3 +53,11 @@ def test_default_transform():
     model(torch.Tensor(get_conv_data()))
     assert len(graphs) == 1
     assert len(graphs[0].layer_list) == 43
+
+
+def test_search_space_limit():
+    graph = CnnGenerator(10, (32, 32, 3)).generate(model_len=3, model_width=2048)
+    assert to_wider_graph(graph) is None
+
+    graph = CnnGenerator(10, (32, 32, 3)).generate(model_len=14)
+    assert to_deeper_graph(graph) is None