Repository: keras-team/autokeras

Issue:
Adopt preprocessing layers for categorical encoding



Discussion:

Resolving patch:
diff --git a/autokeras/hypermodels/heads.py b/autokeras/hypermodels/heads.py
--- a/autokeras/hypermodels/heads.py
+++ b/autokeras/hypermodels/heads.py
@@ -1,10 +1,10 @@
 from typing import Optional
 
+from tensorflow.keras import activations
 from tensorflow.keras import layers
 from tensorflow.python.util import nest
 
 from autokeras import adapters
-from autokeras import keras_layers
 from autokeras.engine import head as head_module
 from autokeras.hypermodels import reduction
 from autokeras.utils import types
@@ -96,7 +96,8 @@ def build(self, hp, inputs=None):
             output_node = layers.Dropout(dropout_rate)(output_node)
         output_node = layers.Dense(self.output_shape[-1])(output_node)
         if self.loss == 'binary_crossentropy':
-            output_node = keras_layers.Sigmoid(name=self.name)(output_node)
+            output_node = layers.Activation(activations.sigmoid,
+                                            name=self.name)(output_node)
         else:
             output_node = layers.Softmax(name=self.name)(output_node)
         return output_node
diff --git a/autokeras/hypermodels/preprocessing.py b/autokeras/hypermodels/preprocessing.py
--- a/autokeras/hypermodels/preprocessing.py
+++ b/autokeras/hypermodels/preprocessing.py
@@ -207,4 +207,4 @@ def build(self, hp, inputs=None):
                 encoding.append(keras_layers.INT)
             else:
                 encoding.append(keras_layers.NONE)
-        return keras_layers.CategoricalEncoding(encoding)(input_node)
+        return keras_layers.MultiColumnCategoricalEncoding(encoding)(input_node)
diff --git a/autokeras/keras_layers.py b/autokeras/keras_layers.py
--- a/autokeras/keras_layers.py
+++ b/autokeras/keras_layers.py
@@ -1,10 +1,8 @@
-import collections
 import inspect
 
-import numpy as np
 import tensorflow as tf
-from tensorflow.keras import backend as K
 from tensorflow.keras.layers.experimental import preprocessing
+from tensorflow.python.keras.layers.preprocessing import index_lookup
 from tensorflow.python.util import nest
 
 CombinerPreprocessingLayer = inspect.getmro(preprocessing.Normalization)[1]
@@ -15,7 +13,7 @@
 ONE_HOT = 'one-hot'
 
 
-class CategoricalEncoding(CombinerPreprocessingLayer):
+class MultiColumnCategoricalEncoding(preprocessing.PreprocessingLayer):
     """Encode the categorical features to numerical features.
 
     # Arguments
@@ -26,119 +24,51 @@ class CategoricalEncoding(CombinerPreprocessingLayer):
     """
 
     # TODO: Support one-hot encoding.
+    # TODO: Support frequency encoding.
 
     def __init__(self, encoding, **kwargs):
-        super().__init__(
-            combiner=CategoricalEncodingCombiner(encoding),
-            **kwargs)
+        super().__init__(**kwargs)
         self.encoding = encoding
-        self.tables = {
-            str(index): tf.lookup.experimental.DenseHashTable(
-                key_dtype=tf.string,
-                value_dtype=tf.int64,
-                default_value=-3,
-                empty_key='-2',
-                deleted_key='-1'
-            )
-            for index, method in enumerate(self.encoding)
-            if method in [INT, ONE_HOT]
-        }
-
-        for key, table in self.tables.items():
-            tracked_table = self._add_trackable(table, trainable=False)
-            tracked_table.shape = tf.TensorShape((0,))
-
-    def _set_state_variables(self, updates):
-        for key, vocab in updates.items():
-            self.tables[key].insert(
-                np.array(vocab, dtype=np.str),
-                np.arange(len(vocab))
-            )
+        self.encoding_layers = []
+        for encoding in self.encoding:
+            if encoding == NONE:
+                self.encoding_layers.append(None)
+            elif encoding == INT:
+                self.encoding_layers.append(index_lookup.IndexLookup())
+            elif encoding == ONE_HOT:
+                self.encoding_layers.append(None)
+
+    def build(self, input_shape):
+        for encoding_layer in self.encoding_layers:
+            if encoding_layer is not None:
+                encoding_layer.build(tf.TensorShape([1]))
 
     def call(self, inputs):
-        inputs = nest.flatten(inputs)[0]
-        outputs = []
-        for index in range(len(self.encoding)):
-            col = tf.slice(inputs, [0, index], [-1, 1])
-            if self.encoding[index] in [INT, ONE_HOT]:
-                col = self.tables[str(index)].lookup(col)
-                col = tf.cast(col, tf.float32)
+        input_nodes = nest.flatten(inputs)[0]
+        split_inputs = tf.split(input_nodes, [1] * len(self.encoding), axis=-1)
+        output_nodes = []
+        for input_node, encoding_layer in zip(split_inputs, self.encoding_layers):
+            if encoding_layer is None:
+                output_nodes.append(tf.strings.to_number(input_node, tf.float32))
             else:
-                col = tf.strings.to_number(col, tf.float32)
-            outputs.append(col)
-        outputs = tf.concat(outputs, axis=-1)
-        outputs.set_shape(inputs.shape)
-        return outputs
+                output_nodes.append(tf.cast(encoding_layer(input_node), tf.float32))
+        return tf.keras.layers.Concatenate()(output_nodes)
 
-    def compute_output_shape(self, input_shape):
-        return input_shape
-
-    def compute_output_signature(self, input_spec):
-        return input_spec
+    def adapt(self, data):
+        for index, encoding_layer in enumerate(self.encoding_layers):
+            if encoding_layer is None:
+                continue
+            data_column = data.map(lambda x: tf.slice(x, [0, index], [-1, 1]))
+            encoding_layer.adapt(data_column)
 
     def get_config(self):
-        config = {'encoding': self.encoding}
+        config = {
+            'encoding': self.encoding,
+        }
         base_config = super().get_config()
         return dict(list(base_config.items()) + list(config.items()))
 
 
-class CategoricalEncodingCombiner(Combiner):
-
-    def __init__(self, encoding):
-        self.encoding = encoding
-
-    def compute(self, values, accumulator=None):
-        if accumulator is None:
-            accumulator = collections.defaultdict(set)
-        for line in K.get_value(values):
-            for index, value in enumerate(line):
-                if self.encoding[index] in [INT, ONE_HOT]:
-                    accumulator[str(index)].add(value)
-        return accumulator
-
-    def merge(self, accumulators):
-        base_accumulator = collections.defaultdict(set)
-        for accumulator in accumulators:
-            for index, method in enumerate(self.encoding):
-                if method in [INT, ONE_HOT]:
-                    base_accumulator[index] = base_accumulator[index].union(
-                        accumulator[index])
-        return base_accumulator
-
-    def extract(self, accumulator):
-        return {
-            key: list(value)
-            for key, value in accumulator.items()
-        }
-
-    def restore(self, output):
-        return {
-            key: set(value)
-            for key, value in output
-        }
-
-    def serialize(self, accumulator):
-        pass
-
-    def deserialize(self, encoded_accumulator):
-        pass
-
-
-class Sigmoid(tf.keras.layers.Layer):
-    """Sigmoid activation function."""
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    def call(self, inputs):
-        return tf.keras.activations.sigmoid(inputs)
-
-    def compute_output_shape(self, input_shape):
-        return input_shape
-
-
 CUSTOM_OBJECTS = {
-    'CategoricalEncoding': CategoricalEncoding,
-    'Sigmoid': Sigmoid,
-    'Normalization': preprocessing.Normalization
+    'MultiColumnCategoricalEncoding': MultiColumnCategoricalEncoding,
 }


Test patch:
diff --git a/tests/autokeras/keras_layers_test.py b/tests/autokeras/keras_layers_test.py
--- a/tests/autokeras/keras_layers_test.py
+++ b/tests/autokeras/keras_layers_test.py
@@ -4,13 +4,15 @@
 from autokeras import keras_layers as layer_module
 
 
-def test_feature_encoder_layer():
-    data = np.array([['a', 'ab'], ['b', 'bc'], ['a', 'bc']])
+def test_feature_encoder_layer(tmp_path):
+    data = np.array([['a', 'ab', 2.1], ['b', 'bc', 1.0], ['a', 'bc', 3.5]])
+    data2 = np.array([['a', 'ab', 2.1], ['x', 'bc', 1.0], ['a', 'bc', 3.5]])
 
-    input_node = tf.keras.Input(shape=(2,), dtype=tf.string)
-    layer = layer_module.CategoricalEncoding([
+    input_node = tf.keras.Input(shape=(3,), dtype=tf.string)
+    layer = layer_module.MultiColumnCategoricalEncoding([
         layer_module.INT,
         layer_module.INT,
+        layer_module.NONE,
     ])
     hidden_node = layer(input_node)
     output_node = tf.keras.layers.Dense(1, activation='sigmoid')(hidden_node)
@@ -20,7 +22,7 @@ def test_feature_encoder_layer():
         (tf.data.Dataset.from_tensor_slices(data).batch(32),),
         (tf.data.Dataset.from_tensor_slices(np.random.rand(3, 1)).batch(32),),
     ))
-    layer.adapt(data)
+    layer.adapt(tf.data.Dataset.from_tensor_slices(data).batch(32))
 
     model.fit(data, np.random.rand(3, 1), epochs=1)
 
@@ -28,7 +30,7 @@ def test_feature_encoder_layer():
 
     model2 = tf.keras.Model(input_node, hidden_node)
     result = model2.predict(data)
-    print(result)
+    model2.predict(data2)
     assert result[0][0] == result[2][0]
     assert result[0][0] != result[1][0]
     assert result[0][1] != result[1][1]