Repository: keras-team/autokeras

Issue:
Update the greedy tuner to consider the conditional search space
I just started using Autokeras ver 1.0.2 in anaconda with Python 3.7 and tensorflow-gpu ver. 2.1.0 on Windows 10. I have a dual RTX Titan (NVlinked) setup and using Spyder ver. 4.0.1

I wanted to do some simple binary classification for RGB images:

clf = ak.ImageClassifier()
clf.fit(x_train, y_train, epochs=10)

Training starts ok but after 3 trials, this happens:
INFO:tensorflow:Oracle triggered exit

I have tried different image datasets, including the MNIST example dataset but the exit trigger still happens after 3 trials, regardless of the val_acc. I can't figure out what is stopping Autokeras from continue to train beyond 3 trials (default is 100 trials?).

Looks like similar issue to https://github.com/keras-team/autokeras/issues/997
but problem is it is happening for version 1.0.2


Discussion:
This might because of the greedy tuning algorithms have some bugs. You can try to use the AutoModel API and use random search as tuner.
I am having the same problem. I downgraded my AutoKeras from 1.0.2 to 1.0.1, set my **max_trials=10**, epochs=1 (for testing purposes) and I was still getting the Oracle error. **BUT** if I set the **max_trials=12** and I set metrics=['accuracy'] then it would complete all 12 trials without any issues. This also works without metrics=['accuracy'], but it seems to error out at iteration 20 instead of 10. **This was also with kernel restarts in-between each max_trial test**. I am unsure what this means, but hopefully, it helps in the debugging process. 

Unfortunately, there is a bug in 1.0.1 that makes it difficult to get the best model. clf.get_best_model does not choose the best one. To obtain all models found by AutoKeras one needs to do the following:

`max_trials=12

clf=ak.ImageClassifier(max_trials=max_trials, metrics=['accuracy'])

clf.fit(X_train, y_train, epochs=1, verbose=0)

found_models = clf.tuner.get_best_models(max_trials)

for model in found_models:
    print(model.evaluate(X_test, y_test))`

Resolving patch:
diff --git a/autokeras/tuners/greedy.py b/autokeras/tuners/greedy.py
--- a/autokeras/tuners/greedy.py
+++ b/autokeras/tuners/greedy.py
@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import copy
-
 import kerastuner
 import numpy as np
 
@@ -25,7 +23,7 @@ def __init__(self):
         super().__init__()
         self.num_leaves = 0
         self.children = {}
-        self.hp = None
+        self.hp_name = None
 
     def is_leaf(self):
         return len(self.children) == 0
@@ -36,10 +34,8 @@ def __init__(self):
         super().__init__()
         self.root = TrieNode()
 
-    def insert(self, hp):
-        name = hp.name
-        long_name = name
-        names = long_name.split("/")
+    def insert(self, hp_name):
+        names = hp_name.split("/")
 
         new_word = False
         current_node = self.root
@@ -50,7 +46,7 @@ def insert(self, hp):
                 new_word = True
             current_node = current_node.children[name]
             nodes_on_path.append(current_node)
-        current_node.hp = hp
+        current_node.hp_name = hp_name
 
         if new_word:
             for node in nodes_on_path:
@@ -66,12 +62,12 @@ def _get_all_nodes(self, node):
             ret += self._get_all_nodes(value)
         return ret
 
-    def get_hps(self, node):
+    def get_hp_names(self, node):
         if node.is_leaf():
-            return [node.hp]
+            return [node.hp_name]
         ret = []
         for key, value in node.children.items():
-            ret += self.get_hps(value)
+            ret += self.get_hp_names(value)
         return ret
 
 
@@ -113,10 +109,11 @@ def set_state(self, state):
         self._tried_initial_hps = state["tried_initial_hps"]
 
     def _select_hps(self):
-        # TODO: consider condition_scopes.
         trie = Trie()
-        for hp in self.hyperparameters.space:
-            trie.insert(hp)
+        best_hps = self._get_best_hps()
+        for hp in best_hps.space:
+            if best_hps.is_active(hp):
+                trie.insert(hp.name)
         all_nodes = trie.nodes
 
         if len(all_nodes) <= 1:
@@ -127,7 +124,7 @@ def _select_hps(self):
         probabilities = probabilities / sum_p
         node = np.random.choice(all_nodes, p=probabilities)
 
-        return trie.get_hps(node)
+        return trie.get_hp_names(node)
 
     def _next_initial_hps(self):
         for index, hps in enumerate(self.initial_hps):
@@ -144,8 +141,8 @@ def _populate_space(self, trial_id):
             }
 
         for i in range(self._max_collisions):
-            hp_list = self._select_hps()
-            values = self._generate_hp_values(hp_list)
+            hp_names = self._select_hps()
+            values = self._generate_hp_values(hp_names)
             # Reached max collisions.
             if values is None:
                 continue
@@ -160,21 +157,31 @@ def _populate_space(self, trial_id):
             "values": None,
         }
 
-    def _generate_hp_values(self, hp_list):
+    def _get_best_hps(self):
         best_trials = self.get_best_trials()
         if best_trials:
-            best_hps = best_trials[0].hyperparameters
+            return best_trials[0].hyperparameters
         else:
-            best_hps = self.hyperparameters
+            return self.hyperparameters
+
+    def _generate_hp_values(self, hp_names):
+        best_hps = self._get_best_hps()
 
         collisions = 0
         while True:
-            hps = copy.deepcopy(best_hps)
+            hps = kerastuner.HyperParameters()
             # Generate a set of random values.
-            for hp in hp_list:
-                # TODO: Check is_active for hp.
-                hps.values[hp.name] = hp.random_sample(self._seed_state)
-                self._seed_state += 1
+            for hp in best_hps.space:
+                hps.merge([hp])
+                # if not active, do nothing.
+                # if active, check if selected to be changed.
+                if hps.is_active(hp):
+                    # if was active and not selected, do nothing.
+                    if best_hps.is_active(hp.name) and hp.name not in hp_names:
+                        continue
+                    # if was not active or selected, sample.
+                    hps.values[hp.name] = hp.random_sample(self._seed_state)
+                    self._seed_state += 1
             values = hps.values
             # Keep trying until the set of values is unique,
             # or until we exit due to too many collisions.


Test patch:
diff --git a/tests/autokeras/tuners/greedy_test.py b/tests/autokeras/tuners/greedy_test.py
--- a/tests/autokeras/tuners/greedy_test.py
+++ b/tests/autokeras/tuners/greedy_test.py
@@ -42,7 +42,6 @@ def test_greedy_oracle_populate_different_values(get_best_trials):
     values_a = oracle._populate_space("a")["values"]
     values_b = oracle._populate_space("b")["values"]
 
-    assert set(values_a.keys()) == set(values_b.keys())
     assert not all([values_a[key] == values_b[key] for key in values_a])