Repository: keras-team/autokeras (language: python)

Issue:
Refactor AutoKeras to support use preprocessor and model layers within the same block



Discussion / hints:

Fix patch:
diff --git a/autokeras/prototype/__init__.py b/autokeras/prototype/__init__.py
new file mode 100644
index 000000000..3d57915f1
--- /dev/null
+++ b/autokeras/prototype/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2020 The AutoKeras Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/autokeras/prototype/base_block.py b/autokeras/prototype/base_block.py
new file mode 100644
index 000000000..72111b182
--- /dev/null
+++ b/autokeras/prototype/base_block.py
@@ -0,0 +1,21 @@
+# Copyright 2020 The AutoKeras Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from autokeras.engine import block
+
+
+class BaseBlock(block.Block):
+    # Renaming the autokeras.engine.block to BaseBlock.
+    # Not open for extension.
+    pass
diff --git a/autokeras/prototype/block.py b/autokeras/prototype/block.py
new file mode 100644
index 000000000..f86e60a48
--- /dev/null
+++ b/autokeras/prototype/block.py
@@ -0,0 +1,34 @@
+# Copyright 2020 The AutoKeras Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from autokeras.prototype import base_block
+from autokeras.prototype import graph_state
+
+
+def is_dataset(inputs):
+    raise NotImplementedError
+
+
+def convert_to_keras_inputs(inputs):
+    raise NotImplementedError
+
+
+class Block(base_block.BaseBlock):
+    # Open for extension. The same as the old Block class.
+    def _build_wrapper(self, hp, inputs, *args, **kwargs):
+        # Accept only KerasTensor.
+        if is_dataset(inputs):
+            inputs = convert_to_keras_inputs(inputs)
+        graph_state.get_state().register_inputs(inputs)
+        return super()._build_wrapper(hp, inputs, *args, **kwargs)
diff --git a/autokeras/prototype/flexible_block.py b/autokeras/prototype/flexible_block.py
new file mode 100644
index 000000000..a522ce7a3
--- /dev/null
+++ b/autokeras/prototype/flexible_block.py
@@ -0,0 +1,24 @@
+# Copyright 2020 The AutoKeras Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from autokeras.prototype import base_block
+
+
+class FlexibleBlock(base_block.BaseBlock):
+    # Open for extension.
+    def _build_wrapper(self, hp, inputs, *args, **kwargs):
+        # Accept both KerasTensor & Dataset.
+        # If the inputs are datasets, build the block into a model and pass
+        # through it.
+        pass
diff --git a/autokeras/prototype/graph.py b/autokeras/prototype/graph.py
new file mode 100644
index 000000000..b84beb7a3
--- /dev/null
+++ b/autokeras/prototype/graph.py
@@ -0,0 +1,69 @@
+# Copyright 2020 The AutoKeras Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from tensorflow import keras
+from tensorflow import nest
+
+from autokeras import graph
+from autokeras.prototype import graph_state
+
+
+class Graph(graph.Graph):
+    def __init__(self, inputs=None, outputs=None, **kwargs):
+        super().__init__(inputs, outputs, **kwargs)
+
+    def build(self, hp):
+        """Build the HyperModel into a Keras Model."""
+        graph_state.init_state()
+        self.compile()
+        keras_nodes = {}
+        keras_input_nodes = []
+
+        # Preparing the inputs of the pipeline.
+        for node in self.inputs:
+            node_id = self._node_to_id[node]
+            input_node = node.build_node(hp)
+            output_node = node.build(hp, input_node)
+            keras_input_nodes.append(input_node)
+            keras_nodes[node_id] = output_node
+
+        # Connecting through the blocks.
+        # Don't check the block type to deal with the output since the block has
+        # sub blocks of different types. The difference should all be handled in
+        # block._build_wrapper().
+        for block in self.blocks:
+            temp_inputs = [
+                keras_nodes[self._node_to_id[input_node]]
+                for input_node in block.inputs
+            ]
+            outputs = block.build(hp, inputs=temp_inputs)
+            outputs = nest.flatten(outputs)
+            for output_node, real_output_node in zip(block.outputs, outputs):
+                keras_nodes[self._node_to_id[output_node]] = real_output_node
+
+        for output_node in self.outputs:
+            node = keras_nodes[self._node_to_id[output_node]]
+            graph_state.get_state().register_outputs(node)
+
+        model = keras.Model(
+            keras_input_nodes,
+            [
+                keras_nodes[self._node_to_id[output_node]]
+                for output_node in self.outputs
+            ],
+        )
+        self._compile_keras_model(hp, model)
+
+        pipeline = None
+        return pipeline
diff --git a/autokeras/prototype/graph_state.py b/autokeras/prototype/graph_state.py
new file mode 100644
index 000000000..13ed88c91
--- /dev/null
+++ b/autokeras/prototype/graph_state.py
@@ -0,0 +1,41 @@
+# Copyright 2020 The AutoKeras Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import threading
+
+STATE = {}
+
+
+class State:
+    def __init__(self):
+        self.registered_inputs = []
+        self.registered_outputs = []
+
+        # Passing y class info from preprocessor to postprocessor.
+        self.y_info
+
+    def register_inputs(self, inputs):
+        raise NotImplementedError
+
+    def register_outputs(self, inputs):
+        # Remember to check duplication
+        raise NotImplementedError
+
+
+def get_state():
+    return STATE[threading.get_ident()]
+
+
+def init_state():
+    STATE[threading.get_ident()] = State()
diff --git a/autokeras/prototype/postprocessor.py b/autokeras/prototype/postprocessor.py
new file mode 100644
index 000000000..11e65a06f
--- /dev/null
+++ b/autokeras/prototype/postprocessor.py
@@ -0,0 +1,26 @@
+# Copyright 2020 The AutoKeras Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from autokeras.prototype import base_block
+
+
+class Postprocessor(base_block.BaseBlock):
+    def _build_wrapper(self, hp, inputs, *args, **kwargs):
+        # Accept only KerasTensor.
+        # Return a KerasTensor.
+        # Register ConcretePostprocessor.
+        # How to register it to the right input node?
+        # Use the output_node (inputs in the args) as the id to push stack in
+        # graph state.
+        return super()._build_wrapper(hp, inputs, *args, **kwargs)
diff --git a/autokeras/prototype/preprocessor.py b/autokeras/prototype/preprocessor.py
new file mode 100644
index 000000000..df19074bc
--- /dev/null
+++ b/autokeras/prototype/preprocessor.py
@@ -0,0 +1,24 @@
+# Copyright 2020 The AutoKeras Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from autokeras.prototype import base_block
+
+
+class Preprocessor(base_block.BaseBlock):
+    def _build_wrapper(self, hp, inputs, *args, **kwargs):
+        # Accept only Dataset.
+        # Return a Dataset.
+        # Register ConcretePreprocessor.
+        # How to register it to the right input node?
+        return super()._build_wrapper(hp, inputs, *args, **kwargs)
diff --git a/docs/redesign.md b/docs/redesign.md
new file mode 100644
index 000000000..16a51a0e1
--- /dev/null
+++ b/docs/redesign.md
@@ -0,0 +1,120 @@
+# AutoKeras Redesign
+
+## Motivation
+redesign how the graph can tune preprocessor, model, and post-processor
+altogether. This is mainly to adapt to the latest best practices of Keras,
+which do not include the preprocessing layers in the main model.
+
+## The problem
+
+The following problems should all be resolved by the new design.
+Some of the problems are supported by AutoKeras today, but some are not.
+
+* Hyperparameters exists in both preprocessors.
+    * Example: Data augmentation contains hyperparameters.
+* The preprocessors needs to know if it is training, validation, or testing.
+    * Example: Data augmentation only apply when training.
+* The hyperparameter for model selection is deciding the preprocessors.
+    * Example: If choose BERT, the preprocessor should use `BertTokenizer`.
+* The preprocessor need to access the target (y) data.
+    * Example: Feature selection based on y.
+    * Example: Object detection preprocessing the images.
+* All the preprocessors applied needs to be tracked to export the preprocessors.
+* Post-processor needs information from preprocessors.
+    * Example: Post-processor to decode probability vector back to
+      classification label needs information from the y encoder.
+
+## The current design
+
+It stacks the preprocessors based on input type.
+Do not select preprocessors based on the hyperparameter value for model selection.
+
+
+## Infeasible best solution
+
+The ideal solution would be separate the search space representation (the
+`HyperModel`s) from the actual implementation (the `tf.data` operations and
+`tf.keras.Model`). We just use the `HyperModel` to get all the hyperparameter
+values and construct an abstraction of the actual implementation, which may also
+be called an intermediate representation (IR), and build the IR into the actual
+implementation. It sounds it would be easy to separate the `tf.data` operations
+and the `tf.keras.Model`. However, this is not possible to implement without
+adding significant burden for all the `autokeras.Block` subclasses
+implementations. It is because KerasTuner create hyperparameters as it builds
+the `HyperModel`. So, it is not possible to get all the hyperparameter values
+without running the code of actual `tf.data` operations and `tf.keras.Model`
+building.
+
+## The new design
+
+### Mixing the preprocessors and model
+
+No longer create the preprocessors based on the input nodes.
+Make preprocessors part of the graph.
+Each block can be build into a mix of preprocessors and parts of the model.  The
+input to `Block.build()` can be either a `KerasTensor` or `tf.data.Dataset`.
+Whenever we need to switch a `Dataset` into a `KerasTensor`, we register the
+`keras.Input` in the graph, so that we can use it to build the `keras.Model`
+later. Similar to the Keras saving mechanism of custom objects registered in
+[`_GLOBAL_CUSTOM_OBJECTS`](https://github.com/keras-team/keras/blob/v2.11.0/keras/saving/object_registration.py#L23).
+
+How to register?
+Using `Block._build_wrapper()`. If a dataset is passed to a block that suppose
+to be part of the Keras model, it should do the switch from dataset to Keras
+input.
+In `Block._build_wrapper()`, it should try to modify some constant value from
+another module to put the input node in.
+
+Note: we require, wherever switching from dataset to a Keras input, it has to
+rely on the `Block._build_wrapper()` to do the job.
+
+### Export the preprocessor
+
+We keep the current design of building `HyperPreprocessor` into `Preprocessor`.
+Whenever a preprocessing is done on the `Dataset`, we record the input and
+output dataset and the preprocessor, so that we can track and reconstruct the
+preprocessing computation graph (not rebuilding the preprocessors from the
+`HyperProcessor`s because fit preprocessors takes time. We can directly use the
+already built preprocessors) and export the prerpocessors.  The current saving
+mechanism of the preprocessors can also be kept.
+
+How to track?
+Similar to register `input`s above. In `HyperPreprocessor._build_wrapper()`, it
+register the input and output datasets and the preprocessor built processed
+them.
+
+### About handling the target values
+
+`y` can be either treat separately or also be part of the graph. Treating
+separately would be much easier. The preprocessing and postprocessing needs to
+share info, but they don't share info with the rest of the graph. Some
+preprocessors needs `y`.
+
+The analyzer and heads information flow can be kept without touch. The analyzer
+analyzes the data and pass the configs into the heads, for example, use sigmoid
+or softmax for classification.
+
+However, the coupling between heads and preprocessing and postprocessing of `y`
+may be decoupled. The information needed for pre & post processing came from the
+analysers. No need to route it into the heads. Routing through heads would make
+implementing custom heads harder.
+
+Although, we get the analysers by the heads, no information is actually passed
+from the heads to the analysers. This design is only because the users are
+specifying the `y`s using the heads.
+
+### Reduce unnecessary dataset conversions
+
+Have more strict type and shape checks, which reduce the overhead of the
+preprocessors to reshape and type convert the dataset.  First, analyze the
+dataset. If doesn't meet the requirements, raise a clear error.
+
+### Accomendations in KerasTuner
+
+Seems no accomendations needed. Should continue to override
+`Tuner._build_and_fit_model()`.
+
+## TODOs
+
+* Remove prototype directory.
+* Remove prototype from coverage exclusion in setup.cfg.
\ No newline at end of file
diff --git a/setup.cfg b/setup.cfg
index 6104ce0c0..04cdc7f20 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -16,7 +16,9 @@ exclude_lines =
     pragma: no cover
     @abstract
     raise NotImplementedError
-omit = *test*
+omit = 
+    *test*
+    autokeras/prototype/*
 
 [flake8]
 


Test patch:
diff --git a/autokeras/prototype/graph_test.py b/autokeras/prototype/graph_test.py
new file mode 100644
index 000000000..3d57915f1
--- /dev/null
+++ b/autokeras/prototype/graph_test.py
@@ -0,0 +1,13 @@
+# Copyright 2020 The AutoKeras Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.