Repository: keras-team/keras-nlp

Issue:
Register our BERT components as a serializable Keras objects
We would like to be able to annotate our new BERT models and components with `@keras.utils.register_keras_serializable(package="keras_nlp")`. This will allow the actual python object to restore after saving, which will be import to access things like `model.tokenizer` or `model.token_embedding` after restoring.

Discussion:

Patch:
diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml
index feadfd73..a9983527 100644
--- a/.github/workflows/actions.yml
+++ b/.github/workflows/actions.yml
@@ -8,10 +8,7 @@ on:
 jobs:
   build:
     name: Run tests
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [windows-latest, ubuntu-latest]
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python 3.7
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index 7ff94048..ee581042 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -9,10 +9,7 @@ on:
 jobs:
   build:
     name: Run tests
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [windows-latest, ubuntu-latest]
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python 3.7
diff --git a/keras_nlp/models/__init__.py b/keras_nlp/models/__init__.py
index 32c34492..06766d79 100644
--- a/keras_nlp/models/__init__.py
+++ b/keras_nlp/models/__init__.py
@@ -20,6 +20,11 @@ from keras_nlp.models.bert.bert_models import BertSmall
 from keras_nlp.models.bert.bert_models import BertTiny
 from keras_nlp.models.bert.bert_preprocessing import BertPreprocessor
 from keras_nlp.models.bert.bert_tasks import BertClassifier
+from keras_nlp.models.distilbert.distilbert_models import DistilBertBase
+from keras_nlp.models.distilbert.distilbert_models import DistilBertCustom
+from keras_nlp.models.distilbert.distilbert_preprocessing import (
+    DistilBertPreprocessor,
+)
 from keras_nlp.models.gpt2.gpt2_models import Gpt2Base
 from keras_nlp.models.gpt2.gpt2_models import Gpt2Custom
 from keras_nlp.models.gpt2.gpt2_models import Gpt2ExtraLarge
diff --git a/keras_nlp/models/bert/bert_models.py b/keras_nlp/models/bert/bert_models.py
index 016d8427..1586e271 100644
--- a/keras_nlp/models/bert/bert_models.py
+++ b/keras_nlp/models/bert/bert_models.py
@@ -71,6 +71,7 @@ def _handle_pretrained_model_arguments(bert_variant, weights, vocabulary_size):
     return weights, vocabulary_size
 
 
+@keras.utils.register_keras_serializable(package="keras_nlp")
 class BertCustom(keras.Model):
     """BERT encoder network with custom hyperparmeters.
 
@@ -248,21 +249,22 @@ class BertCustom(keras.Model):
         self.cls_token_index = cls_token_index
 
     def get_config(self):
-        config = super().get_config()
-        config.update(
-            {
-                "vocabulary_size": self.vocabulary_size,
-                "hidden_dim": self.hidden_dim,
-                "intermediate_dim": self.intermediate_dim,
-                "num_layers": self.num_layers,
-                "num_heads": self.num_heads,
-                "max_sequence_length": self.max_sequence_length,
-                "num_segments": self.num_segments,
-                "dropout": self.dropout,
-                "cls_token_index": self.cls_token_index,
-            }
-        )
-        return config
+        return {
+            "vocabulary_size": self.vocabulary_size,
+            "hidden_dim": self.hidden_dim,
+            "intermediate_dim": self.intermediate_dim,
+            "num_layers": self.num_layers,
+            "num_heads": self.num_heads,
+            "max_sequence_length": self.max_sequence_length,
+            "num_segments": self.num_segments,
+            "dropout": self.dropout,
+            "name": self.name,
+            "trainable": self.trainable,
+        }
+
+    @classmethod
+    def from_config(cls, config):
+        return cls(**config)
 
 
 MODEL_DOCSTRING = """Bert "{type}" architecture.
diff --git a/keras_nlp/models/bert/bert_tasks.py b/keras_nlp/models/bert/bert_tasks.py
index 3d6abf77..e1f1e570 100644
--- a/keras_nlp/models/bert/bert_tasks.py
+++ b/keras_nlp/models/bert/bert_tasks.py
@@ -67,6 +67,7 @@ CLASSIFIER_DOCSTRING = """BERT encoder model with a classification head.
 """
 
 
+@keras.utils.register_keras_serializable(package="keras_nlp")
 class BertClassifier(keras.Model):
     def __init__(
         self,
@@ -104,6 +105,20 @@ class BertClassifier(keras.Model):
         self.backbone = backbone
         self.num_classes = num_classes
 
+    def get_config(self):
+        return {
+            "backbone": keras.layers.serialize(self.backbone),
+            "num_classes": self.num_classes,
+            "name": self.name,
+            "trainable": self.trainable,
+        }
+
+    @classmethod
+    def from_config(cls, config):
+        if "backbone" in config:
+            config["backbone"] = keras.layers.deserialize(config["backbone"])
+        return cls(**config)
+
 
 setattr(
     BertClassifier,
diff --git a/keras_nlp/models/distilbert/__init__.py b/keras_nlp/models/distilbert/__init__.py
new file mode 100644
index 00000000..6e4df4e7
--- /dev/null
+++ b/keras_nlp/models/distilbert/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2022 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/keras_nlp/models/distilbert/distilbert_models.py b/keras_nlp/models/distilbert/distilbert_models.py
new file mode 100644
index 00000000..042c1a62
--- /dev/null
+++ b/keras_nlp/models/distilbert/distilbert_models.py
@@ -0,0 +1,234 @@
+# Copyright 2022 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""DistilBERT backbone models."""
+
+
+import tensorflow as tf
+from tensorflow import keras
+
+from keras_nlp.layers.token_and_position_embedding import (
+    TokenAndPositionEmbedding,
+)
+from keras_nlp.layers.transformer_encoder import TransformerEncoder
+
+
+def distilbert_kernel_initializer(stddev=0.02):
+    return keras.initializers.TruncatedNormal(stddev=stddev)
+
+
+class DistilBertCustom(keras.Model):
+    """DistilBERT encoder network with custom hyperparmeters.
+
+    This network implements a bi-directional Transformer-based encoder as
+    described in ["DistilBERT, a distilled version of BERT: smaller, faster,
+    cheaper and lighter"](https://arxiv.org/abs/1910.01108). It includes the
+    embedding lookups and transformer layers, but not the masked language model
+    or classification task networks.
+
+    This class gives a fully customizable DistilBERT model with any number of layers,
+    heads, and embedding dimensions. For specific DistilBERT architectures defined in
+    the paper, see, for example, `keras_nlp.models.DistilBertBase`.
+
+    Args:
+        vocabulary_size: int. The size of the token vocabulary.
+        num_layers: int. The number of transformer layers.
+        num_heads: int. The number of attention heads for each transformer.
+            The hidden size must be divisible by the number of attention heads.
+        hidden_dim: int. The size of the transformer encoding and pooler layers.
+        intermediate_dim: int. The output dimension of the first Dense layer in
+            a two-layer feedforward network for each transformer.
+        dropout: float. Dropout probability for the Transformer encoder.
+        max_sequence_length: int. The maximum sequence length that this encoder
+            can consume. If None, `max_sequence_length` uses the value from
+            sequence length. This determines the variable shape for positional
+            embeddings.
+        name: string, optional. Name of the model.
+        trainable: boolean, optional. If the model's variables should be
+            trainable.
+
+    Examples:
+    ```python
+    # Randomly initialized DistilBERT encoder
+    model = keras_nlp.models.DistilBertCustom(
+        vocabulary_size=30522,
+        num_layers=12,
+        num_heads=12,
+        hidden_dim=768,
+        intermediate_dim=3072,
+        max_sequence_length=12,
+        name="encoder",
+    )
+
+    # Call encoder on the inputs
+    input_data = {
+        "token_ids": tf.random.uniform(
+            shape=(1, 12), dtype=tf.int64, maxval=model.vocabulary_size
+        ),
+        "padding_mask": tf.constant(
+            [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0], shape=(1, 12)
+        ),
+    }
+    output = model(input_data)
+    ```
+    """
+
+    def __init__(
+        self,
+        vocabulary_size,
+        num_layers,
+        num_heads,
+        hidden_dim,
+        intermediate_dim,
+        dropout=0.1,
+        max_sequence_length=512,
+        name=None,
+        trainable=True,
+    ):
+
+        # Inputs
+        token_id_input = keras.Input(
+            shape=(None,), dtype="int32", name="token_ids"
+        )
+        padding_mask = keras.Input(
+            shape=(None,), dtype="int32", name="padding_mask"
+        )
+
+        # Embed tokens and positions.
+        x = TokenAndPositionEmbedding(
+            vocabulary_size=vocabulary_size,
+            sequence_length=max_sequence_length,
+            embedding_dim=hidden_dim,
+            embeddings_initializer=distilbert_kernel_initializer(),
+            name="token_and_position_embedding",
+        )(token_id_input)
+
+        # Normalize and apply dropout to embeddings.
+        x = keras.layers.LayerNormalization(
+            axis=-1,
+            epsilon=1e-12,
+            dtype=tf.float32,
+            name="embeddings_layer_norm",
+        )(x)
+        x = keras.layers.Dropout(
+            dropout,
+            name="embeddings_dropout",
+        )(x)
+
+        # Apply successive transformer encoder blocks.
+        for i in range(num_layers):
+            x = TransformerEncoder(
+                num_heads=num_heads,
+                intermediate_dim=intermediate_dim,
+                activation="gelu",
+                dropout=dropout,
+                kernel_initializer=distilbert_kernel_initializer(),
+                name=f"transformer_layer_{i}",
+            )(x, padding_mask=padding_mask)
+
+        # Instantiate using Functional API Model constructor
+        super().__init__(
+            inputs={
+                "token_ids": token_id_input,
+                "padding_mask": padding_mask,
+            },
+            outputs=x,
+            name=name,
+            trainable=trainable,
+        )
+        # All references to `self` below this line
+        self.vocabulary_size = vocabulary_size
+        self.num_layers = num_layers
+        self.num_heads = num_heads
+        self.hidden_dim = hidden_dim
+        self.intermediate_dim = intermediate_dim
+        self.dropout = dropout
+        self.max_sequence_length = max_sequence_length
+
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "vocabulary_size": self.vocabulary_size,
+                "num_layers": self.num_layers,
+                "num_heads": self.num_heads,
+                "hidden_dim": self.hidden_dim,
+                "intermediate_dim": self.intermediate_dim,
+                "dropout": self.dropout,
+                "max_sequence_length": self.max_sequence_length,
+            }
+        )
+        return config
+
+
+MODEL_DOCSTRING = """DistilBert "{type}" architecture.
+
+This network implements a bi-directional Transformer-based encoder as
+described in ["DistilBERT, a distilled version of BERT: smaller, faster,
+cheaper and lighter"](https://arxiv.org/abs/1910.01108). It includes the
+embedding lookups and transformer layers, but not the masked language model
+or classification task networks.
+
+Args:
+    vocabulary_size: int, optional. The size of the token vocabulary.
+    name: string, optional. Name of the model.
+    trainable: boolean, optional. If the model's variables should be
+        trainable.
+
+Examples:
+```python
+# Randomly initialized DistilBert{type} encoder
+model = keras_nlp.models.DistilBert{type}(vocabulary_size=10000)
+
+# Call encoder on the inputs.
+input_data = {{
+    "token_ids": tf.random.uniform(
+        shape=(1, 512), dtype=tf.int64, maxval=model.vocabulary_size
+    ),
+    "padding_mask": tf.constant([1] * 512, shape=(1, 512)),
+}}
+output = model(input_data)
+```
+"""
+
+
+def DistilBertBase(vocabulary_size, name=None, trainable=True):
+
+    model = DistilBertCustom(
+        vocabulary_size=vocabulary_size,
+        num_layers=6,
+        num_heads=12,
+        hidden_dim=768,
+        intermediate_dim=3072,
+        dropout=0.1,
+        max_sequence_length=512,
+        name=name,
+        trainable=trainable,
+    )
+
+    return model
+
+
+def model_class_by_name(classname):
+    """Return model class given the class name."""
+    return {
+        "DistilBertBase": DistilBertBase,
+    }[classname]
+
+
+setattr(
+    DistilBertBase,
+    "__doc__",
+    MODEL_DOCSTRING.format(type="Base"),
+)
diff --git a/keras_nlp/models/distilbert/distilbert_preprocessing.py b/keras_nlp/models/distilbert/distilbert_preprocessing.py
new file mode 100644
index 00000000..f1c00cbb
--- /dev/null
+++ b/keras_nlp/models/distilbert/distilbert_preprocessing.py
@@ -0,0 +1,160 @@
+# Copyright 2022 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""DistilBERT preprocessing layers."""
+
+from tensorflow import keras
+
+from keras_nlp.layers.multi_segment_packer import MultiSegmentPacker
+from keras_nlp.tokenizers.word_piece_tokenizer import WordPieceTokenizer
+
+PREPROCESSOR_DOCSTRING = """DistilBERT preprocessor with pretrained vocabularies.
+
+This preprocessing layer will do three things:
+
+ - Tokenize any number of inputs using a
+   `keras_nlp.tokenizers.WordPieceTokenizer`.
+ - Pack the inputs together using a `keras_nlp.layers.MultiSegmentPacker`.
+   with the appropriate `"[CLS]"`, `"[SEP]"` and `"[PAD]"` tokens.
+ - Construct a dictionary of with keys `"token_ids"`, `"padding_mask"`, that can
+   be passed directly to a DistilBERT model.
+
+This layer will accept either a tuple of (possibly batched) inputs, or a single
+input tensor. If a single tensor is passed, it will be packed equivalently to
+a tuple with a single element.
+
+The WordPiece tokenizer can be accessed via the `tokenizer` property on this
+layer, and can be used directly for custom packing on inputs.
+
+Args:
+    vocabulary: One of a list of vocabulary terms, a vocabulary filename, or
+        the name of the pretrained vocabulary.
+    lowercase: If `True`, input will be lowercase before tokenization. If
+        `vocabulary` is set to a pretrained vocabulary, this parameter will
+        be inferred.
+    sequence_length: The length of the packed inputs. Only used if
+        `pack_inputs` is True.
+    truncate: The algorithm to truncate a list of batched segments to fit
+        within `sequence_length`. Only used if
+        `pack_inputs` is True. The value can be either `round_robin` or
+        `waterfall`:
+            - `"round_robin"`: Available space is assigned one token at a
+                time in a round-robin fashion to the inputs that still need
+                some, until the limit is reached.
+            - `"waterfall"`: The allocation of the budget is done using a
+                "waterfall" algorithm that allocates quota in a
+                left-to-right manner and fills up the buckets until we run
+                out of budget. It supports an arbitrary number of segments.
+
+Examples:
+```python
+preprocessor = keras_nlp.models.DistilBertPreprocessor(vocabulary="uncased_en")
+
+# Tokenize and pack a single sentence directly.
+preprocessor("The quick brown fox jumped.")
+
+# Tokenize and pack a multiple sentence directly.
+preprocessor(("The quick brown fox jumped.", "Call me Ishmael."))
+
+# Map a dataset to preprocess a single sentence.
+features = ["The quick brown fox jumped.", "I forgot my homework."]
+labels = [0, 1]
+ds = tf.data.Dataset.from_tensor_slices((features, labels))
+ds = ds.map(
+    lambda x, y: (preprocessor(x), y),
+    num_parallel_calls=tf.data.AUTOTUNE,
+)
+
+# Map a dataset to preprocess a multiple sentences.
+first_sentences = ["The quick brown fox jumped.", "Call me Ishmael."]
+second_sentences = ["The fox tripped.", "Oh look, a whale."]
+labels = [1, 1]
+ds = tf.data.Dataset.from_tensor_slices(((first_sentences, second_sentences), labels))
+ds = ds.map(
+    lambda x, y: (preprocessor(x), y),
+    num_parallel_calls=tf.data.AUTOTUNE,
+)
+```
+"""
+
+
+class DistilBertPreprocessor(keras.layers.Layer):
+    def __init__(
+        self,
+        vocabulary="uncased_en",
+        lowercase=False,
+        sequence_length=512,
+        truncate="round_robin",
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.tokenizer = WordPieceTokenizer(
+            vocabulary=vocabulary,
+            lowercase=lowercase,
+        )
+
+        # Check for necessary special tokens.
+        cls_token = "[CLS]"
+        sep_token = "[SEP]"
+        pad_token = "[PAD]"
+        for token in [cls_token, pad_token, sep_token]:
+            if token not in self.tokenizer.get_vocabulary():
+                raise ValueError(
+                    f"Cannot find token `'{token}'` in the provided "
+                    f"`vocabulary`. Please provide `'{token}'` in your "
+                    "`vocabulary` or use a pretrained `vocabulary` name."
+                )
+
+        self.pad_token_id = self.tokenizer.token_to_id(pad_token)
+        self.packer = MultiSegmentPacker(
+            start_value=self.tokenizer.token_to_id(cls_token),
+            end_value=self.tokenizer.token_to_id(sep_token),
+            pad_value=self.pad_token_id,
+            truncate=truncate,
+            sequence_length=sequence_length,
+        )
+
+    def vocabulary_size(self):
+        """Returns the vocabulary size of the tokenizer."""
+        return self.tokenizer.vocabulary_size()
+
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "vocabulary": self.tokenizer.vocabulary,
+                "lowercase": self.tokenizer.lowercase,
+                "sequence_length": self.packer.sequence_length,
+                "truncate": self.packer.truncate,
+            }
+        )
+        return config
+
+    def call(self, inputs):
+        if not isinstance(inputs, (list, tuple)):
+            inputs = [inputs]
+
+        inputs = [self.tokenizer(x) for x in inputs]
+        token_ids, _ = self.packer(inputs)
+        return {
+            "token_ids": token_ids,
+            "padding_mask": token_ids != self.pad_token_id,
+        }
+
+
+setattr(
+    DistilBertPreprocessor,
+    "__doc__",
+    PREPROCESSOR_DOCSTRING,
+)


Test patch:
diff --git a/keras_nlp/models/bert/bert_models_test.py b/keras_nlp/models/bert/bert_models_test.py
index dab9abd8..740d225f 100644
--- a/keras_nlp/models/bert/bert_models_test.py
+++ b/keras_nlp/models/bert/bert_models_test.py
@@ -120,12 +120,19 @@ class BertTest(tf.test.TestCase, parameterized.TestCase):
                 name="encoder",
             )
 
-    def test_saving_model(self):
+    @parameterized.named_parameters(
+        ("save_format_tf", "tf"), ("save_format_h5", "h5")
+    )
+    def test_saving_model(self, save_format):
         model_output = self.model(self.input_batch)
         save_path = os.path.join(self.get_temp_dir(), "model")
-        self.model.save(save_path)
+        self.model.save(save_path, save_format)
         restored_model = keras.models.load_model(save_path)
 
+        # Check we got the real object back.
+        self.assertIsInstance(restored_model, BertCustom)
+
+        # Check that output matches.
         restored_output = restored_model(self.input_batch)
         self.assertAllClose(
             model_output["pooled_output"], restored_output["pooled_output"]
diff --git a/keras_nlp/models/bert/bert_tasks_test.py b/keras_nlp/models/bert/bert_tasks_test.py
index 26204f68..eca12ebc 100644
--- a/keras_nlp/models/bert/bert_tasks_test.py
+++ b/keras_nlp/models/bert/bert_tasks_test.py
@@ -76,11 +76,18 @@ class BertClassifierTest(tf.test.TestCase, parameterized.TestCase):
         self.classifier.compile(jit_compile=jit_compile)
         self.classifier.predict(self.input_dataset)
 
-    def test_saving_model(self):
+    @parameterized.named_parameters(
+        ("save_format_tf", "tf"), ("save_format_h5", "h5")
+    )
+    def test_saving_model(self, save_format):
         model_output = self.classifier(self.input_batch)
         save_path = os.path.join(self.get_temp_dir(), "model")
-        self.classifier.save(save_path)
+        self.classifier.save(save_path, save_format)
         restored_model = keras.models.load_model(save_path)
 
+        # Check we got the real object back.
+        self.assertIsInstance(restored_model, BertClassifier)
+
+        # Check that output matches.
         restored_output = restored_model(self.input_batch)
         self.assertAllClose(model_output, restored_output)
diff --git a/keras_nlp/models/distilbert/distilbert_models_test.py b/keras_nlp/models/distilbert/distilbert_models_test.py
new file mode 100644
index 00000000..6425f6a5
--- /dev/null
+++ b/keras_nlp/models/distilbert/distilbert_models_test.py
@@ -0,0 +1,101 @@
+# Copyright 2022 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Test for DistilBERT backbone models."""
+
+import os
+
+import tensorflow as tf
+from absl.testing import parameterized
+from tensorflow import keras
+
+from keras_nlp.models.distilbert.distilbert_models import DistilBertBase
+from keras_nlp.models.distilbert.distilbert_models import DistilBertCustom
+
+
+class DistilBertTest(tf.test.TestCase, parameterized.TestCase):
+    def setUp(self):
+        self.model = DistilBertCustom(
+            vocabulary_size=1000,
+            num_layers=2,
+            num_heads=2,
+            hidden_dim=64,
+            intermediate_dim=128,
+            max_sequence_length=128,
+            name="encoder",
+        )
+        self.batch_size = 8
+        self.input_batch = {
+            "token_ids": tf.ones(
+                (self.batch_size, self.model.max_sequence_length), dtype="int32"
+            ),
+            "padding_mask": tf.ones(
+                (self.batch_size, self.model.max_sequence_length), dtype="int32"
+            ),
+        }
+
+        self.input_dataset = tf.data.Dataset.from_tensor_slices(
+            self.input_batch
+        ).batch(2)
+
+    def test_valid_call_distilbert(self):
+        self.model(self.input_batch)
+
+    def test_variable_sequence_length_call_distilbert(self):
+        for seq_length in (25, 50, 75):
+            input_data = {
+                "token_ids": tf.ones(
+                    (self.batch_size, seq_length), dtype="int32"
+                ),
+                "padding_mask": tf.ones(
+                    (self.batch_size, seq_length), dtype="int32"
+                ),
+            }
+            self.model(input_data)
+
+    def test_valid_call_distilbert_base(self):
+        model = DistilBertBase(vocabulary_size=1000, name="encoder")
+        input_data = {
+            "token_ids": tf.ones(
+                (self.batch_size, self.model.max_sequence_length), dtype="int32"
+            ),
+            "padding_mask": tf.ones(
+                (self.batch_size, self.model.max_sequence_length), dtype="int32"
+            ),
+        }
+        model(input_data)
+
+    @parameterized.named_parameters(
+        ("jit_compile_false", False), ("jit_compile_true", True)
+    )
+    def test_distilbert_base_compile(self, jit_compile):
+        model = DistilBertBase(vocabulary_size=1000, name="encoder")
+        model.compile(jit_compile=jit_compile)
+        model.predict(self.input_batch)
+
+    @parameterized.named_parameters(
+        ("jit_compile_false", False), ("jit_compile_true", True)
+    )
+    def test_distilbert_base_compile_batched_ds(self, jit_compile):
+        model = DistilBertBase(vocabulary_size=1000, name="encoder")
+        model.compile(jit_compile=jit_compile)
+        model.predict(self.input_dataset)
+
+    def test_saving_model(self):
+        model_output = self.model(self.input_batch)
+        save_path = os.path.join(self.get_temp_dir(), "model")
+        self.model.save(save_path)
+        restored_model = keras.models.load_model(save_path)
+
+        restored_output = restored_model(self.input_batch)
+        self.assertAllClose(model_output, restored_output)
diff --git a/keras_nlp/models/distilbert/distilbert_preprocessing_test.py b/keras_nlp/models/distilbert/distilbert_preprocessing_test.py
new file mode 100644
index 00000000..03bc2bdb
--- /dev/null
+++ b/keras_nlp/models/distilbert/distilbert_preprocessing_test.py
@@ -0,0 +1,57 @@
+# Copyright 2022 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for DistilBERT preprocessing layers."""
+
+import tensorflow as tf
+
+from keras_nlp.models.distilbert.distilbert_preprocessing import (
+    DistilBertPreprocessor,
+)
+
+
+class DistilBertPreprocessorTest(tf.test.TestCase):
+    def setUp(self):
+        self.vocab = ["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"]
+        self.vocab += ["THE", "QUICK", "BROWN", "FOX"]
+        self.vocab += ["the", "quick", "brown", "fox"]
+
+    def test_tokenize(self):
+        input_data = ["THE QUICK BROWN FOX."]
+        preprocessor = DistilBertPreprocessor(
+            vocabulary=self.vocab,
+            sequence_length=8,
+        )
+        output = preprocessor(input_data)
+        self.assertAllEqual(output["token_ids"], [2, 5, 6, 7, 8, 1, 3, 0])
+        self.assertAllEqual(output["padding_mask"], [1, 1, 1, 1, 1, 1, 1, 0])
+
+    def test_lowercase(self):
+        input_data = ["THE QUICK BROWN FOX."]
+        preprocessor = DistilBertPreprocessor(
+            vocabulary=self.vocab,
+            sequence_length=8,
+            lowercase=True,
+        )
+        output = preprocessor(input_data)
+        self.assertAllEqual(output["token_ids"], [2, 9, 10, 11, 12, 1, 3, 0])
+
+    def test_detokenize(self):
+        input_data = [[5, 6, 7, 8]]
+        preprocessor = DistilBertPreprocessor(vocabulary=self.vocab)
+        output = preprocessor.tokenizer.detokenize(input_data)
+        self.assertAllEqual(output, ["THE QUICK BROWN FOX"])
+
+    def test_vocabulary_size(self):
+        preprocessor = DistilBertPreprocessor(vocabulary=self.vocab)
+        self.assertEqual(preprocessor.vocabulary_size(), 13)