Repository: keras-team/autokeras

Issue:
KeyError: 'classification_head_1/spatial_reduction_1/reduction_type'
AutoKeras version: master
KerasTuner version: master
TensorFlow version: 2.2.0

I am running a multi label image classifier based on the example code here: https://autokeras.com/tutorial/image_classification/

I was able to get the example code working with these dependency versions.

Now I am trying to modify the code to run on my own dataset. I have loaded in my own images and labels. I have set `multi_label=True` since my dataset can have more than one label per image.

My process seems to be pretty standard. After loading in the data and labels, I transform them into numpy arrays, then use `MultiLabelBinarizer` to pre-process, split into test and train sets, and run the image classiifer. Code is here:

```
data = np.array(data, dtype="float") / 255.0
labels = np.array(labels)

# binarize the labels using scikit-learn's special multi-label
# binarizer implementation
print("[INFO] class labels:")
mlb = MultiLabelBinarizer()
labels = mlb.fit_transform(labels)
# loop over each of the possible class labels and show them
for (i, label) in enumerate(mlb.classes_):
    print("{}. {}".format(i + 1, label))

(x_train, x_test, y_train, y_test) = train_test_split(data, labels, test_size=0.2, random_state=42)
print(x_train.shape)  # (60000, 28, 28)
print(y_train.shape)  # (60000,)
print(y_train[:3])  # array([7, 2, 1], dtype=uint8)
```

Afterwards, code is straight from the example, except setting `multi_label=True`:

```
# Initialize the image classifier.
clf = ak.ImageClassifier(max_trials=3, multi_label=True)  # It tries 10 different models.
# Feed the image classifier with training data.
clf.fit(x_train, y_train, epochs=3)


# Predict with the best model.
predicted_y = clf.predict(x_test)
print(predicted_y)


# Evaluate the best model with testing data.
print(clf.evaluate(x_test, y_test))
```

The code runs fine for a few trials. Then, I get this error:

```
KeyError: 'classification_head_1/spatial_reduction_1/reduction_type'
[Warning] Invalid model 0/5
```

Here is the full stack trace:

```
Starting new trial
Traceback (most recent call last):
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py", line 105, in build
    model = self.hypermodel.build(hp)
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py", line 65, in _build_wrapper
    return self._build(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/graph.py", line 235, in build
    outputs = block.build(hp, inputs=temp_inputs)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/engine/block.py", line 34, in _build_wrapper
    return super()._build_wrapper(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py", line 65, in _build_wrapper
    return self._build(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/blocks/heads.py", line 82, in build
    output_node = reduction.SpatialReduction().build(hp, output_node)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/engine/block.py", line 34, in _build_wrapper
    return super()._build_wrapper(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py", line 65, in _build_wrapper
    return self._build(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/blocks/reduction.py", line 106, in build
    default='global_avg')
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hyperparameters.py", line 691, in Choice
    return self._retrieve(hp)
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hyperparameters.py", line 619, in _retrieve
    return self.values[hp.name]
KeyError: 'classification_head_1/spatial_reduction_1/reduction_type'
[Warning] Invalid model 0/5
Traceback (most recent call last):
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py", line 105, in build
    model = self.hypermodel.build(hp)
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py", line 65, in _build_wrapper
    return self._build(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/graph.py", line 235, in build
    outputs = block.build(hp, inputs=temp_inputs)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/engine/block.py", line 34, in _build_wrapper
    return super()._build_wrapper(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py", line 65, in _build_wrapper
    return self._build(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/blocks/heads.py", line 82, in build
    output_node = reduction.SpatialReduction().build(hp, output_node)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/engine/block.py", line 34, in _build_wrapper
    return super()._build_wrapper(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py", line 65, in _build_wrapper
    return self._build(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/blocks/reduction.py", line 106, in build
    default='global_avg')
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hyperparameters.py", line 691, in Choice
    return self._retrieve(hp)
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hyperparameters.py", line 619, in _retrieve
    return self.values[hp.name]
KeyError: 'classification_head_1/spatial_reduction_1/reduction_type'
[Warning] Invalid model 1/5
Traceback (most recent call last):
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py", line 105, in build
    model = self.hypermodel.build(hp)
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py", line 65, in _build_wrapper
    return self._build(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/graph.py", line 235, in build
    outputs = block.build(hp, inputs=temp_inputs)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/engine/block.py", line 34, in _build_wrapper
    return super()._build_wrapper(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py", line 65, in _build_wrapper
    return self._build(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/blocks/heads.py", line 82, in build
    output_node = reduction.SpatialReduction().build(hp, output_node)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/engine/block.py", line 34, in _build_wrapper
    return super()._build_wrapper(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py", line 65, in _build_wrapper
    return self._build(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/blocks/reduction.py", line 106, in build
    default='global_avg')
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hyperparameters.py", line 691, in Choice
    return self._retrieve(hp)
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hyperparameters.py", line 619, in _retrieve
    return self.values[hp.name]
KeyError: 'classification_head_1/spatial_reduction_1/reduction_type'
[Warning] Invalid model 2/5
Traceback (most recent call last):
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py", line 105, in build
    model = self.hypermodel.build(hp)
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py", line 65, in _build_wrapper
    return self._build(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/graph.py", line 235, in build
    outputs = block.build(hp, inputs=temp_inputs)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/engine/block.py", line 34, in _build_wrapper
    return super()._build_wrapper(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py", line 65, in _build_wrapper
    return self._build(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/blocks/heads.py", line 82, in build
    output_node = reduction.SpatialReduction().build(hp, output_node)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/engine/block.py", line 34, in _build_wrapper
    return super()._build_wrapper(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py", line 65, in _build_wrapper
    return self._build(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/blocks/reduction.py", line 106, in build
    default='global_avg')
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hyperparameters.py", line 691, in Choice
    return self._retrieve(hp)
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hyperparameters.py", line 619, in _retrieve
    return self.values[hp.name]
KeyError: 'classification_head_1/spatial_reduction_1/reduction_type'
[Warning] Invalid model 3/5
Traceback (most recent call last):
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py", line 105, in build
    model = self.hypermodel.build(hp)
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py", line 65, in _build_wrapper
    return self._build(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/graph.py", line 235, in build
    outputs = block.build(hp, inputs=temp_inputs)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/engine/block.py", line 34, in _build_wrapper
    return super()._build_wrapper(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py", line 65, in _build_wrapper
    return self._build(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/blocks/heads.py", line 82, in build
    output_node = reduction.SpatialReduction().build(hp, output_node)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/engine/block.py", line 34, in _build_wrapper
    return super()._build_wrapper(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py", line 65, in _build_wrapper
    return self._build(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/blocks/reduction.py", line 106, in build
    default='global_avg')
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hyperparameters.py", line 691, in Choice
    return self._retrieve(hp)
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hyperparameters.py", line 619, in _retrieve
    return self.values[hp.name]
KeyError: 'classification_head_1/spatial_reduction_1/reduction_type'
[Warning] Invalid model 4/5
Traceback (most recent call last):
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py", line 105, in build
    model = self.hypermodel.build(hp)
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py", line 65, in _build_wrapper
    return self._build(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/graph.py", line 235, in build
    outputs = block.build(hp, inputs=temp_inputs)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/engine/block.py", line 34, in _build_wrapper
    return super()._build_wrapper(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py", line 65, in _build_wrapper
    return self._build(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/blocks/heads.py", line 82, in build
    output_node = reduction.SpatialReduction().build(hp, output_node)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/engine/block.py", line 34, in _build_wrapper
    return super()._build_wrapper(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py", line 65, in _build_wrapper
    return self._build(hp, *args, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/autokeras/blocks/reduction.py", line 106, in build
    default='global_avg')
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hyperparameters.py", line 691, in Choice
    return self._retrieve(hp)
  File "/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hyperparameters.py", line 619, in _retrieve
    return self.values[hp.name]
KeyError: 'classification_head_1/spatial_reduction_1/reduction_type'
[Warning] Invalid model 5/5
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py in build(self, hp)
    104                 with maybe_distribute(self.distribution_strategy):
--> 105                     model = self.hypermodel.build(hp)
    106             except:

17 frames
KeyError: 'classification_head_1/spatial_reduction_1/reduction_type'

During handling of the above exception, another exception occurred:

RuntimeError                              Traceback (most recent call last)
/usr/local/lib/python3.6/dist-packages/kerastuner/engine/hypermodel.py in build(self, hp)
    113                 if i == self._max_fail_streak:
    114                     raise RuntimeError(
--> 115                         'Too many failed attempts to build model.')
    116                 continue
    117 

RuntimeError: Too many failed attempts to build model.
```

How can I resolve this?


Discussion:
Thank you for the report! I think it is fixed in the latest release 1.0.3. If it doesn't, please let me know. Thanks.
I'm having the same issue, but with Image Classifier. My autokeras version is 1.0.3
I have the same issue with Image Classifier. Also version is 1.0.3. My dataset is like:
```
tdataset = tf.data.Dataset.from_tensor_slices((tfilenames, tannotations))
vdataset = tf.data.Dataset.from_tensor_slices((vfilenames, vannotations))
...
tdataset = tdataset.map(BasicPreprocessor(im_size, 0))
vdataset = vdataset.map(BasicPreprocessor(im_size, 1))
```

Preprocessor is:
```
    @tf.function
    def preprocess_annotated(self, data, annotation):
        if self.pipeline == 0 or self.pipeline == 1:
            image_string = tf.io.read_file(data)
            image = tf.image.decode_image(image_string, channels=3)
            image.set_shape(shape=[None, None, 3])
            images = self.adjust_pixelwise(image)
        else:
            images = data

        images = self.resize_to(image=images, sz=self.images_size, keep_scales=self.keep_scales)

        return images, annotation
```
where `self.resize_to` is call of `tf.image.resize(images=image, size=(sz, sz))`; `self.adjust_pixelwise` is:
```
result = tf.divide(image, tf.constant(2.0))
result = tf.add(result, tf.constant(0.5))
result = tf.image.convert_image_dtype(result, tf.uint8)
return result
```
The other code is from Get Started example.
Sometimes it works well but when trials reach something with 'classification_head_1/spatial_reduction_1/reduction_type' I get the error mentioned earlier. Once i hade nearly 60 trials finished without this error and I think there was no `spatial_reduction_1` block inside any trial. And most of the times I have only nearly 3-7 trials working correctly and then an error appears.

Any workarounds or news about issue, @haifeng-jin?
I might because of the `overwrite=False`. Just change it to True may solve the problem.
@haifeng-jin 
For me issue happens when I start training without interrupting and resuming the process, wothout any previously saved checkpoints. That is why the result is the same with `overwrite=False` or with `overwrite=True`.
> I might because of the `overwrite=False`. Just change it to True may solve the problem.

I'll try with this, if it does not solve the issue, I will notify.

As @I-Kryachko , my issue is starting training from scratch.

And finally, I have to say that the error didn't show up in the last 3-4 trials I made, but I got a run out of memory error. So not sure what was going on with the previous issue.
Update: I can't even finish a trial, I get an error of running out of memory which disappears if I do not use `overwrite=True`

Update 2: two trials later I do not run out of memory (for now, it's only in the first trial). 

Sorry if you fell overwhelmed by my many comments, I am just try to give as much information as possible to contribute to Autokeras.
Thanks everyone!
As I talked to chyt, we don't see this error any more with ak 1.0.3 and kt 1.0.2rc0.
Not sure how to reproduce it. I am running totally fine locally.
Sometimes I run ok too. It happens for me at ak 1.0.3 when tuner decides to make certain improvents on get started task. Two tasks which I run one by one do not run the same layers, so sometimes I have error.

Resolving patch:
diff --git a/autokeras/engine/adapter.py b/autokeras/engine/adapter.py
--- a/autokeras/engine/adapter.py
+++ b/autokeras/engine/adapter.py
@@ -72,7 +72,7 @@ def fit_transform(self, dataset):
         return dataset
 
     def _record_dataset_shape(self, dataset):
-        self.shape = data_utils.dataset_shape(dataset)[1:]
+        self.shape = data_utils.dataset_shape(dataset)[1:].as_list()
 
     def transform(self, dataset):
         """Transform the input dataset to tf.data.Dataset.
diff --git a/autokeras/engine/tuner.py b/autokeras/engine/tuner.py
--- a/autokeras/engine/tuner.py
+++ b/autokeras/engine/tuner.py
@@ -35,9 +35,10 @@ def __init__(self,
                  hypermodel,
                  preprocessors=None,
                  **kwargs):
+        # Initialize before super() for reload to work.
+        self._finished = False
         super().__init__(oracle, hypermodel, **kwargs)
         self.preprocessors = nest.flatten(preprocessors)
-        self._finished = False
         # Save or load the HyperModel.
         self.hypermodel.hypermodel.save(os.path.join(self.project_dir, 'graph'))
 
@@ -152,6 +153,17 @@ def search(self,
         model.save_weights(self.best_model_path)
         self._finished = True
 
+    def get_state(self):
+        state = super().get_state()
+        state.update({
+            'finished': self._finished,
+        })
+        return state
+
+    def set_state(self, state):
+        super().set_state(state)
+        self._finished = state.get('finished')
+
     @staticmethod
     def _remove_early_stopping(callbacks):
         return [copy.deepcopy(callbacks)
diff --git a/autokeras/tuners/greedy.py b/autokeras/tuners/greedy.py
--- a/autokeras/tuners/greedy.py
+++ b/autokeras/tuners/greedy.py
@@ -5,6 +5,7 @@
 import numpy as np
 
 from autokeras import blocks
+from autokeras import graph
 from autokeras.engine import tuner as tuner_module
 
 
@@ -59,6 +60,36 @@ def __init__(self,
         self._tried_so_far = set()
         self._max_collisions = 5
 
+    def get_state(self):
+        state = super().get_state()
+        new_hp_names = {}
+        for key, value in self._hp_names.items():
+            new_hp_names[key] = list(value)
+        state.update({
+            'initial_hps': self.initial_hps,
+            'tried_initial_hps': self._tried_initial_hps,
+            'hypermodel': self.hypermodel.get_config(),
+            'hp_names': new_hp_names,
+            'seed': self.seed,
+            'seed_state': self._seed_state,
+            'tried_so_far': list(self._tried_so_far),
+            'max_collisions': self._max_collisions,
+        })
+        return state
+
+    def set_state(self, state):
+        super().set_state(state)
+        self.initial_hps = state['initial_hps']
+        self._tried_initial_hps = state['tried_initial_hps']
+        self.hypermodel = graph.Graph.from_config(state['hypermodel'])
+        new_hp_names = state['hp_names']
+        for key, value in new_hp_names.items():
+            self._hp_names[key] = value
+        self.seed = state['seed']
+        self._seed_state = state['seed_state']
+        self._tried_so_far = set(state['tried_so_far'])
+        self._max_collisions = state['max_collisions']
+
     def update_space(self, hyperparameters):
         # Get the block names.
         self.hypermodel.build(hyperparameters)


Test patch:
diff --git a/tests/autokeras/adapters/input_adapter_test.py b/tests/autokeras/adapters/input_adapter_test.py
--- a/tests/autokeras/adapters/input_adapter_test.py
+++ b/tests/autokeras/adapters/input_adapter_test.py
@@ -75,12 +75,20 @@ def test_partial_column_types():
     assert adapter.column_types['fare'] == 'categorical'
 
 
-def test_image_input():
+def test_image_input_adapter_transform_to_dataset():
     x = utils.generate_data()
     adapter = input_adapter.ImageInputAdapter()
     assert isinstance(adapter.transform(x), tf.data.Dataset)
 
 
+def test_image_input_adapter_shape_is_list():
+    x = utils.generate_data()
+    adapter = input_adapter.ImageInputAdapter()
+    adapter.fit_transform(x)
+    assert isinstance(adapter.shape, list)
+    assert all(map(lambda x: isinstance(x, int), adapter.shape))
+
+
 def test_image_input_with_three_dim():
     x = utils.generate_data(shape=(32, 32))
     adapter = input_adapter.ImageInputAdapter()
diff --git a/tests/autokeras/engine/tuner_test.py b/tests/autokeras/engine/tuner_test.py
--- a/tests/autokeras/engine/tuner_test.py
+++ b/tests/autokeras/engine/tuner_test.py
@@ -101,7 +101,12 @@ def test_tuner_not_call_super_search_with_overwrite(
         directory=tmp_path)
 
     tuner.search(epochs=10)
+    tuner.save()
     super_search.reset_mock()
+
+    tuner = greedy.Greedy(
+        hypermodel=utils.build_graph(),
+        directory=tmp_path)
     tuner.search(epochs=10)
 
     super_search.assert_not_called()
diff --git a/tests/autokeras/tuners/greedy_test.py b/tests/autokeras/tuners/greedy_test.py
--- a/tests/autokeras/tuners/greedy_test.py
+++ b/tests/autokeras/tuners/greedy_test.py
@@ -2,29 +2,26 @@
 
 import kerastuner
 
+from autokeras import graph as graph_module
 from autokeras.tuners import greedy
 from tests import utils
 
 
-def test_greedy_oracle_state():
-    graph = utils.build_graph()
+def test_greedy_oracle_state_hypermodel_is_graph():
     oracle = greedy.GreedyOracle(
-        hypermodel=graph,
+        hypermodel=utils.build_graph(),
         objective='val_loss',
     )
-    oracle.hypermodel = graph
     oracle.set_state(oracle.get_state())
-    assert oracle.hypermodel is graph
+    assert isinstance(oracle.hypermodel, graph_module.Graph)
 
 
 @mock.patch('autokeras.tuners.greedy.GreedyOracle.get_best_trials')
 def test_greedy_oracle(fn):
-    graph = utils.build_graph()
     oracle = greedy.GreedyOracle(
-        hypermodel=graph,
+        hypermodel=utils.build_graph(),
         objective='val_loss',
     )
-    oracle.hypermodel = graph
     trial = mock.Mock()
     hp = kerastuner.HyperParameters()
     trial.hyperparameters = hp