Repository: keras-team/keras-nlp
Pull request #1365 (2023-12-12 22:24:06)

Kaggle Presets

mattdangerw: Just a note, we should not merge this via the "Squash and merge" button. I will rebase and push it to master so we can keep our commit history for the feature branch. Useful to bisect and go from commit -> PR via the link in the message.

Patch:
diff --git a/keras_nlp/layers/modeling/token_and_position_embedding.py b/keras_nlp/layers/modeling/token_and_position_embedding.py
index f3dffe34..bb7107f9 100644
--- a/keras_nlp/layers/modeling/token_and_position_embedding.py
+++ b/keras_nlp/layers/modeling/token_and_position_embedding.py
@@ -122,7 +122,7 @@ class TokenAndPositionEmbedding(keras.layers.Layer):
                 ),
                 "tie_weights": self.token_embedding.tie_weights,
                 "mask_zero": self.token_embedding.mask_zero,
-            },
+            }
         )
         return config
 
diff --git a/keras_nlp/layers/preprocessing/preprocessing_layer.py b/keras_nlp/layers/preprocessing/preprocessing_layer.py
index 63c40713..d6101da1 100644
--- a/keras_nlp/layers/preprocessing/preprocessing_layer.py
+++ b/keras_nlp/layers/preprocessing/preprocessing_layer.py
@@ -29,7 +29,12 @@ class PreprocessingLayer(keras.layers.Layer):
         super().__init__(**kwargs)
         self._convert_input_args = False
         self._allow_non_tensor_positional_args = True
-        self.built = True
+        # Most pre-preprocessing has no build.
+        if not hasattr(self, "build"):
+            self.built = True
+
+    def get_build_config(self):
+        return None
 
     def __call__(self, *args, **kwargs):
         # Always place on CPU for preprocessing, to avoid expensive back and
diff --git a/keras_nlp/models/albert/albert_masked_lm_preprocessor.py b/keras_nlp/models/albert/albert_masked_lm_preprocessor.py
index 9a52d28a..89cf1344 100644
--- a/keras_nlp/models/albert/albert_masked_lm_preprocessor.py
+++ b/keras_nlp/models/albert/albert_masked_lm_preprocessor.py
@@ -131,18 +131,27 @@ class AlbertMaskedLMPreprocessor(AlbertPreprocessor):
             truncate=truncate,
             **kwargs,
         )
-
+        self.mask_selection_rate = mask_selection_rate
+        self.mask_selection_length = mask_selection_length
+        self.mask_token_rate = mask_token_rate
+        self.random_token_rate = random_token_rate
+        self.masker = None
+
+    def build(self, input_shape):
+        super().build(input_shape)
+        # Defer masker creation to `build()` so that we can be sure tokenizer
+        # assets have loaded when restoring a saved model.
         self.masker = MaskedLMMaskGenerator(
-            mask_selection_rate=mask_selection_rate,
-            mask_selection_length=mask_selection_length,
-            mask_token_rate=mask_token_rate,
-            random_token_rate=random_token_rate,
-            vocabulary_size=tokenizer.vocabulary_size(),
-            mask_token_id=tokenizer.mask_token_id,
+            mask_selection_rate=self.mask_selection_rate,
+            mask_selection_length=self.mask_selection_length,
+            mask_token_rate=self.mask_token_rate,
+            random_token_rate=self.random_token_rate,
+            vocabulary_size=self.tokenizer.vocabulary_size(),
+            mask_token_id=self.tokenizer.mask_token_id,
             unselectable_token_ids=[
-                tokenizer.cls_token_id,
-                tokenizer.sep_token_id,
-                tokenizer.pad_token_id,
+                self.tokenizer.cls_token_id,
+                self.tokenizer.sep_token_id,
+                self.tokenizer.pad_token_id,
             ],
         )
 
@@ -150,10 +159,10 @@ class AlbertMaskedLMPreprocessor(AlbertPreprocessor):
         config = super().get_config()
         config.update(
             {
-                "mask_selection_rate": self.masker.mask_selection_rate,
-                "mask_selection_length": self.masker.mask_selection_length,
-                "mask_token_rate": self.masker.mask_token_rate,
-                "random_token_rate": self.masker.random_token_rate,
+                "mask_selection_rate": self.mask_selection_rate,
+                "mask_selection_length": self.mask_selection_length,
+                "mask_token_rate": self.mask_token_rate,
+                "random_token_rate": self.random_token_rate,
             }
         )
         return config
diff --git a/keras_nlp/models/albert/albert_preprocessor.py b/keras_nlp/models/albert/albert_preprocessor.py
index 4849aab3..5d5628a7 100644
--- a/keras_nlp/models/albert/albert_preprocessor.py
+++ b/keras_nlp/models/albert/albert_preprocessor.py
@@ -158,20 +158,28 @@ class AlbertPreprocessor(Preprocessor):
     ):
         super().__init__(**kwargs)
         self.tokenizer = tokenizer
+        self.truncate = truncate
+        self.sequence_length = sequence_length
+        self.packer = None
+
+    def build(self, input_shape):
+        # Defer packer creation to `build()` so that we can be sure tokenizer
+        # assets have loaded when restoring a saved model.
         self.packer = MultiSegmentPacker(
             start_value=self.tokenizer.cls_token_id,
             end_value=self.tokenizer.sep_token_id,
             pad_value=self.tokenizer.pad_token_id,
-            truncate=truncate,
-            sequence_length=sequence_length,
+            truncate=self.truncate,
+            sequence_length=self.sequence_length,
         )
+        self.built = True
 
     def get_config(self):
         config = super().get_config()
         config.update(
             {
-                "sequence_length": self.packer.sequence_length,
-                "truncate": self.packer.truncate,
+                "sequence_length": self.sequence_length,
+                "truncate": self.truncate,
             }
         )
         return config
diff --git a/keras_nlp/models/albert/albert_presets.py b/keras_nlp/models/albert/albert_presets.py
index 34126f52..3314241f 100644
--- a/keras_nlp/models/albert/albert_presets.py
+++ b/keras_nlp/models/albert/albert_presets.py
@@ -26,24 +26,7 @@ backbone_presets = {
             "path": "albert",
             "model_card": "https://github.com/google-research/albert/blob/master/README.md",
         },
-        "config": {
-            "vocabulary_size": 30000,
-            "num_layers": 12,
-            "num_heads": 12,
-            "num_groups": 1,
-            "num_inner_repetitions": 1,
-            "embedding_dim": 128,
-            "hidden_dim": 768,
-            "intermediate_dim": 3072,
-            "dropout": 0.0,
-            "max_sequence_length": 512,
-            "num_segments": 2,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/albert_base_en_uncased/v1/model.h5",
-        "weights_hash": "b83ccf3418dd84adc569324183176813",
-        "spm_proto_url": "https://storage.googleapis.com/keras-nlp/models/albert_base_en_uncased/v1/vocab.spm",
-        "spm_proto_hash": "73e62ff8e90f951f24c8b907913039a5",
+        "kaggle_handle": "kaggle://keras/albert/albert_base_en_uncased/2",
     },
     "albert_large_en_uncased": {
         "metadata": {
@@ -56,24 +39,7 @@ backbone_presets = {
             "path": "albert",
             "model_card": "https://github.com/google-research/albert/blob/master/README.md",
         },
-        "config": {
-            "vocabulary_size": 30000,
-            "num_layers": 24,
-            "num_heads": 16,
-            "num_groups": 1,
-            "num_inner_repetitions": 1,
-            "embedding_dim": 128,
-            "hidden_dim": 1024,
-            "intermediate_dim": 4096,
-            "dropout": 0,
-            "max_sequence_length": 512,
-            "num_segments": 2,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/albert_large_en_uncased/v1/model.h5",
-        "weights_hash": "c7754804efb245f06dd6e7ced32e082c",
-        "spm_proto_url": "https://storage.googleapis.com/keras-nlp/models/albert_large_en_uncased/v1/vocab.spm",
-        "spm_proto_hash": "73e62ff8e90f951f24c8b907913039a5",
+        "kaggle_handle": "kaggle://keras/albert/albert_large_en_uncased/2",
     },
     "albert_extra_large_en_uncased": {
         "metadata": {
@@ -86,24 +52,7 @@ backbone_presets = {
             "path": "albert",
             "model_card": "https://github.com/google-research/albert/blob/master/README.md",
         },
-        "config": {
-            "vocabulary_size": 30000,
-            "num_layers": 24,
-            "num_heads": 16,
-            "num_groups": 1,
-            "num_inner_repetitions": 1,
-            "embedding_dim": 128,
-            "hidden_dim": 2048,
-            "intermediate_dim": 8192,
-            "dropout": 0,
-            "max_sequence_length": 512,
-            "num_segments": 2,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/albert_extra_large_en_uncased/v1/model.h5",
-        "weights_hash": "713209be8aadfa614fd79f18c9aeb16d",
-        "spm_proto_url": "https://storage.googleapis.com/keras-nlp/models/albert_extra_large_en_uncased/v1/vocab.spm",
-        "spm_proto_hash": "73e62ff8e90f951f24c8b907913039a5",
+        "kaggle_handle": "kaggle://keras/albert/albert_extra_large_en_uncased/2",
     },
     "albert_extra_extra_large_en_uncased": {
         "metadata": {
@@ -116,23 +65,6 @@ backbone_presets = {
             "path": "albert",
             "model_card": "https://github.com/google-research/albert/blob/master/README.md",
         },
-        "config": {
-            "vocabulary_size": 30000,
-            "num_layers": 12,
-            "num_heads": 64,
-            "num_groups": 1,
-            "num_inner_repetitions": 1,
-            "embedding_dim": 128,
-            "hidden_dim": 4096,
-            "intermediate_dim": 16384,
-            "dropout": 0,
-            "max_sequence_length": 512,
-            "num_segments": 2,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/albert_extra_extra_large_en_uncased/v1/model.h5",
-        "weights_hash": "a835177b692fb6a82139f94c66db2f22",
-        "spm_proto_url": "https://storage.googleapis.com/keras-nlp/models/albert_extra_extra_large_en_uncased/v1/vocab.spm",
-        "spm_proto_hash": "73e62ff8e90f951f24c8b907913039a5",
+        "kaggle_handle": "kaggle://keras/albert/albert_extra_extra_large_en_uncased/2",
     },
 }
diff --git a/keras_nlp/models/albert/albert_tokenizer.py b/keras_nlp/models/albert/albert_tokenizer.py
index 1b85be3a..44aed44c 100644
--- a/keras_nlp/models/albert/albert_tokenizer.py
+++ b/keras_nlp/models/albert/albert_tokenizer.py
@@ -87,25 +87,38 @@ class AlbertTokenizer(SentencePieceTokenizer):
     """
 
     def __init__(self, proto, **kwargs):
+        self.cls_token = "[CLS]"
+        self.sep_token = "[SEP]"
+        self.pad_token = "<pad>"
+        self.mask_token = "[MASK]"
+
         super().__init__(proto=proto, **kwargs)
 
-        # Check for necessary special tokens.
-        cls_token = "[CLS]"
-        sep_token = "[SEP]"
-        pad_token = "<pad>"
-        mask_token = "[MASK]"
-        for token in [cls_token, sep_token, pad_token, mask_token]:
-            if token not in self.get_vocabulary():
-                raise ValueError(
-                    f"Cannot find token `'{token}'` in the provided "
-                    f"`vocabulary`. Please provide `'{token}'` in your "
-                    "`vocabulary` or use a pretrained `vocabulary` name."
-                )
-
-        self.cls_token_id = self.token_to_id(cls_token)
-        self.sep_token_id = self.token_to_id(sep_token)
-        self.pad_token_id = self.token_to_id(pad_token)
-        self.mask_token_id = self.token_to_id(mask_token)
+    def set_proto(self, proto):
+        super().set_proto(proto)
+        if proto is not None:
+            for token in [
+                self.cls_token,
+                self.sep_token,
+                self.pad_token,
+                self.mask_token,
+            ]:
+                if token not in self.get_vocabulary():
+                    raise ValueError(
+                        f"Cannot find token `'{token}'` in the provided "
+                        f"`vocabulary`. Please provide `'{token}'` in your "
+                        "`vocabulary` or use a pretrained `vocabulary` name."
+                    )
+
+            self.cls_token_id = self.token_to_id(self.cls_token)
+            self.sep_token_id = self.token_to_id(self.sep_token)
+            self.pad_token_id = self.token_to_id(self.pad_token)
+            self.mask_token_id = self.token_to_id(self.mask_token)
+        else:
+            self.cls_token_id = None
+            self.sep_token_id = None
+            self.pad_token_id = None
+            self.mask_token_id = None
 
     @classproperty
     def presets(cls):
diff --git a/keras_nlp/models/backbone.py b/keras_nlp/models/backbone.py
index a55f7673..69da5659 100644
--- a/keras_nlp/models/backbone.py
+++ b/keras_nlp/models/backbone.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
-
 from keras_nlp.backend import keras
+from keras_nlp.utils.preset_utils import check_preset_class
+from keras_nlp.utils.preset_utils import load_from_preset
 from keras_nlp.utils.python_utils import classproperty
 from keras_nlp.utils.python_utils import format_docstring
 
@@ -24,6 +24,19 @@ class Backbone(keras.Model):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._token_embedding = None
+        self._functional_layer_ids = set(
+            id(layer) for layer in self._flatten_layers()
+        )
+
+    def __dir__(self):
+        # Temporary fixes for weight saving. This mimics the following PR for
+        # older version of Keras: https://github.com/keras-team/keras/pull/18982
+        def filter_fn(attr):
+            if attr == "_layer_checkpoint_dependencies":
+                return False
+            return id(getattr(self, attr)) not in self._functional_layer_ids
+
+        return filter(filter_fn, super().__dir__())
 
     def __setattr__(self, name, value):
         # Work around torch setattr for properties.
@@ -94,35 +107,18 @@ class Backbone(keras.Model):
         )
         ```
         """
-
-        if not cls.presets:
-            raise NotImplementedError(
-                "No presets have been created for this class."
-            )
-
-        if preset not in cls.presets:
-            raise ValueError(
-                "`preset` must be one of "
-                f"""{", ".join(cls.presets)}. Received: {preset}."""
-            )
-        metadata = cls.presets[preset]
-        config = metadata["config"]
-        model = cls.from_config({**config, **kwargs})
-
-        if not load_weights:
-            return model
-
-        filename = os.path.basename(metadata["weights_url"])
-        weights = keras.utils.get_file(
-            filename,
-            metadata["weights_url"],
-            cache_subdir=os.path.join("models", preset),
-            file_hash=metadata["weights_hash"],
+        # We support short IDs for official presets, e.g. `"bert_base_en"`.
+        # Map these to a Kaggle Models handle.
+        if preset in cls.presets:
+            preset = cls.presets[preset]["kaggle_handle"]
+
+        check_preset_class(preset, cls)
+        return load_from_preset(
+            preset,
+            load_weights=load_weights,
+            config_overrides=kwargs,
         )
 
-        model.load_weights(weights)
-        return model
-
     def __init_subclass__(cls, **kwargs):
         # Use __init_subclass__ to setup a correct docstring for from_preset.
         super().__init_subclass__(**kwargs)
diff --git a/keras_nlp/models/bart/bart_preprocessor.py b/keras_nlp/models/bart/bart_preprocessor.py
index ebe0310b..ffe21488 100644
--- a/keras_nlp/models/bart/bart_preprocessor.py
+++ b/keras_nlp/models/bart/bart_preprocessor.py
@@ -140,15 +140,23 @@ class BartPreprocessor(Preprocessor):
     ):
         super().__init__(**kwargs)
         self.tokenizer = tokenizer
+        self.encoder_sequence_length = encoder_sequence_length
+        self.decoder_sequence_length = decoder_sequence_length
+        self.encoder_packer = None
+        self.decoder_packer = None
+
+    def build(self, input_shape):
+        # Defer packer creation to `build()` so that we can be sure tokenizer
+        # assets have loaded when restoring a saved model.
 
         # TODO: Use `MultiSegmentPacker` instead of `StartEndPacker` once we
         # want to move to multi-segment packing and have improved
         # `MultiSegmentPacker`'s performance.
         self.encoder_packer = StartEndPacker(
-            start_value=tokenizer.start_token_id,
-            end_value=tokenizer.end_token_id,
-            pad_value=tokenizer.pad_token_id,
-            sequence_length=encoder_sequence_length,
+            start_value=self.tokenizer.start_token_id,
+            end_value=self.tokenizer.end_token_id,
+            pad_value=self.tokenizer.pad_token_id,
+            sequence_length=self.encoder_sequence_length,
             return_padding_mask=True,
         )
 
@@ -161,19 +169,10 @@ class BartPreprocessor(Preprocessor):
             ],
             end_value=self.tokenizer.end_token_id,
             pad_value=self.tokenizer.pad_token_id,
-            sequence_length=decoder_sequence_length,
+            sequence_length=self.decoder_sequence_length,
             return_padding_mask=True,
         )
-
-    def get_config(self):
-        config = super().get_config()
-        config.update(
-            {
-                "encoder_sequence_length": self.encoder_packer.sequence_length,
-                "decoder_sequence_length": self.decoder_packer.sequence_length,
-            }
-        )
-        return config
+        self.built = True
 
     def call(self, x, y=None, sample_weight=None):
         if not (
@@ -217,6 +216,16 @@ class BartPreprocessor(Preprocessor):
 
         return pack_x_y_sample_weight(x, y, sample_weight)
 
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "encoder_sequence_length": self.encoder_sequence_length,
+                "decoder_sequence_length": self.decoder_sequence_length,
+            }
+        )
+        return config
+
     @classproperty
     def tokenizer_cls(cls):
         return BartTokenizer
@@ -224,63 +233,3 @@ class BartPreprocessor(Preprocessor):
     @classproperty
     def presets(cls):
         return copy.deepcopy(backbone_presets)
-
-    @classmethod
-    def from_preset(
-        cls,
-        preset,
-        **kwargs,
-    ):
-        # Override base class's `from_preset` to handle `encoder_sequence_length`
-        # and `decoder_sequence_length`.
-        if not cls.presets:
-            raise NotImplementedError(
-                "No presets have been created for this class."
-            )
-        if preset not in cls.presets:
-            raise ValueError(
-                "`preset` must be one of "
-                f"""{", ".join(cls.presets)}. Received: {preset}."""
-            )
-
-        tokenizer = cls.tokenizer_cls.from_preset(preset)
-
-        metadata = cls.presets[preset]
-        # For task model presets, the backbone config is nested.
-        if "backbone" in metadata["config"]:
-            backbone_config = metadata["config"]["backbone"]["config"]
-        else:
-            backbone_config = metadata["config"]
-
-        # Use model's `max_sequence_length` if either `encoder_sequence_length`
-        # or `decoder_sequence_length` are unspecified; otherwise check that
-        # `encoder_sequence_length`/`decoder_sequence_length` are not too long.
-        encoder_sequence_length = kwargs.pop("encoder_sequence_length", None)
-        decoder_sequence_length = kwargs.pop("decoder_sequence_length", None)
-        max_sequence_length = backbone_config["max_sequence_length"]
-
-        def check_sequence_length(sequence_length, name):
-            if sequence_length is not None:
-                if sequence_length > max_sequence_length:
-                    raise ValueError(
-                        f"`{name}` cannot be longer than `{preset}` "
-                        f"preset's `max_sequence_length` of {max_sequence_length}. "
-                        f"Received: {sequence_length}."
-                    )
-                return sequence_length
-            else:
-                return max_sequence_length
-
-        encoder_sequence_length = check_sequence_length(
-            encoder_sequence_length, "encoder_sequence_length"
-        )
-        decoder_sequence_length = check_sequence_length(
-            decoder_sequence_length, "decoder_sequence_length"
-        )
-
-        return cls(
-            tokenizer=tokenizer,
-            encoder_sequence_length=encoder_sequence_length,
-            decoder_sequence_length=decoder_sequence_length,
-            **kwargs,
-        )
diff --git a/keras_nlp/models/bart/bart_presets.py b/keras_nlp/models/bart/bart_presets.py
index aa06254c..6249d467 100644
--- a/keras_nlp/models/bart/bart_presets.py
+++ b/keras_nlp/models/bart/bart_presets.py
@@ -25,22 +25,7 @@ backbone_presets = {
             "path": "bart",
             "model_card": "https://github.com/facebookresearch/fairseq/blob/main/examples/bart/README.md",
         },
-        "config": {
-            "vocabulary_size": 50265,
-            "num_layers": 6,
-            "num_heads": 12,
-            "hidden_dim": 768,
-            "intermediate_dim": 3072,
-            "dropout": 0.1,
-            "max_sequence_length": 1024,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/bart_base_en/v1/model.h5",
-        "weights_hash": "5b59403f0cafafbd89680e0785791163",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/bart_base_en/v1/vocab.json",
-        "vocabulary_hash": "be4d3c6f3f5495426b2c03b334334354",
-        "merges_url": "https://storage.googleapis.com/keras-nlp/models/bart_base_en/v1/merges.txt",
-        "merges_hash": "75a37753dd7a28a2c5df80c28bf06e4e",
+        "kaggle_handle": "kaggle://keras/bart/bart_base_en/2",
     },
     "bart_large_en": {
         "metadata": {
@@ -62,13 +47,7 @@ backbone_presets = {
             "dropout": 0.1,
             "max_sequence_length": 1024,
         },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/bart_large_en/v1/model.h5",
-        "weights_hash": "6bfe7e591af8c5699ce6f9f18753af9a",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/bart_large_en/v1/vocab.json",
-        "vocabulary_hash": "cf410ee085c5c69c957bb1f6d8456596",
-        "merges_url": "https://storage.googleapis.com/keras-nlp/models/bart_large_en/v1/merges.txt",
-        "merges_hash": "75a37753dd7a28a2c5df80c28bf06e4e",
+        "kaggle_handle": "kaggle://keras/bart/bart_large_en/2",
     },
     "bart_large_en_cnn": {
         "metadata": {
@@ -90,12 +69,6 @@ backbone_presets = {
             "dropout": 0.1,
             "max_sequence_length": 1024,
         },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/bart_large_en_cnn/v1/model.h5",
-        "weights_hash": "99782ecd9365956f016096fef9afd62c",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/bart_large_en_cnn/v1/vocab.json",
-        "vocabulary_hash": "be4d3c6f3f5495426b2c03b334334354",
-        "merges_url": "https://storage.googleapis.com/keras-nlp/models/bart_large_en_cnn/v1/merges.txt",
-        "merges_hash": "75a37753dd7a28a2c5df80c28bf06e4e",
+        "kaggle_handle": "kaggle://keras/bart/bart_large_en_cnn/2",
     },
 }
diff --git a/keras_nlp/models/bart/bart_seq_2_seq_lm_preprocessor.py b/keras_nlp/models/bart/bart_seq_2_seq_lm_preprocessor.py
index 8c15de85..3d398d29 100644
--- a/keras_nlp/models/bart/bart_seq_2_seq_lm_preprocessor.py
+++ b/keras_nlp/models/bart/bart_seq_2_seq_lm_preprocessor.py
@@ -46,16 +46,6 @@ class BartSeq2SeqLMPreprocessor(BartPreprocessor):
         tokenizer: A `keras_nlp.models.BartTokenizer` instance.
         encoder_sequence_length: The length of the packed encoder inputs.
         decoder_sequence_length: The length of the packed decoder inputs.
-        truncate: string. The algorithm to truncate a list of batched segments
-            to fit within `sequence_length`. The value can be either
-            `round_robin` or `waterfall`:
-                - `"round_robin"`: Available space is assigned one token at a
-                    time in a round-robin fashion to the inputs that still need
-                    some, until the limit is reached.
-                - `"waterfall"`: The allocation of the budget is done using a
-                    "waterfall" algorithm that allocates quota in a
-                    left-to-right manner and fills up the buckets until we run
-                    out of budget. It supports an arbitrary number of segments.
 
     Call arguments:
         x: A dictionary with `encoder_text` and `decoder_text` as its keys.
@@ -137,9 +127,8 @@ class BartSeq2SeqLMPreprocessor(BartPreprocessor):
     def __init__(
         self,
         tokenizer,
-        encoder_sequence_length,
-        decoder_sequence_length,
-        truncate="round_robin",
+        encoder_sequence_length=1024,
+        decoder_sequence_length=1024,
         **kwargs
     ):
         # Since we truncate the last token from `decoder_token_ids`, we need to
@@ -156,16 +145,6 @@ class BartSeq2SeqLMPreprocessor(BartPreprocessor):
         self._encoder_sequence_length = encoder_sequence_length
         self._decoder_sequence_length = decoder_sequence_length
 
-    def get_config(self):
-        config = super().get_config()
-        config.update(
-            {
-                "encoder_sequence_length": self._encoder_sequence_length,
-                "decoder_sequence_length": self._decoder_sequence_length,
-            }
-        )
-        return config
-
     def call(self, x, y=None, sample_weight=None):
         if y is not None or sample_weight is not None:
             logging.warning(
@@ -191,10 +170,6 @@ class BartSeq2SeqLMPreprocessor(BartPreprocessor):
         sample_weight = decoder_padding_mask[..., 1:]
         return pack_x_y_sample_weight(x, y, sample_weight)
 
-    @classproperty
-    def presets(cls):
-        return copy.deepcopy(backbone_presets)
-
     def generate_preprocess(
         self,
         x,
@@ -212,6 +187,9 @@ class BartSeq2SeqLMPreprocessor(BartPreprocessor):
         the decoder sequence (as generation is expected to continue at the end
         of the inputted decoder prompt).
         """
+        if not self.built:
+            self.build(None)
+
         # If `sequence_length` is not provided, we use the default value.
         if sequence_length is None:
             sequence_length = self._decoder_sequence_length
@@ -262,6 +240,9 @@ class BartSeq2SeqLMPreprocessor(BartPreprocessor):
         padding and start/end tokens, and then converting the integer sequence
         back to a string.
         """
+        if not self.built:
+            self.build(None)
+
         decoder_token_ids, decoder_padding_mask = (
             x["decoder_token_ids"],
             x["decoder_padding_mask"],
@@ -279,3 +260,17 @@ class BartSeq2SeqLMPreprocessor(BartPreprocessor):
             decoder_token_ids, decoder_padding_mask
         )
         return self.tokenizer.detokenize(decoder_token_ids)
+
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "encoder_sequence_length": self._encoder_sequence_length,
+                "decoder_sequence_length": self._decoder_sequence_length,
+            }
+        )
+        return config
+
+    @classproperty
+    def presets(cls):
+        return copy.deepcopy(backbone_presets)
diff --git a/keras_nlp/models/bart/bart_tokenizer.py b/keras_nlp/models/bart/bart_tokenizer.py
index 0f8728d7..17fb237b 100644
--- a/keras_nlp/models/bart/bart_tokenizer.py
+++ b/keras_nlp/models/bart/bart_tokenizer.py
@@ -78,34 +78,45 @@ class BartTokenizer(BytePairTokenizer):
 
     def __init__(
         self,
-        vocabulary,
-        merges,
+        vocabulary=None,
+        merges=None,
         **kwargs,
     ):
-        # Special tokens.
-        start_token = "
"
-        pad_token = "<pad>"
-        end_token = "
"
+        self.start_token = "
"
+        self.pad_token = "<pad>"
+        self.end_token = "
"
 
         super().__init__(
             vocabulary=vocabulary,
             merges=merges,
-            unsplittable_tokens=[start_token, pad_token, end_token],
+            unsplittable_tokens=[
+                self.start_token,
+                self.pad_token,
+                self.end_token,
+            ],
             **kwargs,
         )
 
-        # Check whether special tokens are present in the vocabulary.
-        for token in [start_token, pad_token, end_token]:
-            if token not in self.get_vocabulary():
-                raise ValueError(
-                    f"Cannot find token `'{token}'` in the provided "
-                    f"`vocabulary`. Please provide `'{token}'` in your "
-                    "`vocabulary` or use a pretrained `vocabulary` name."
-                )
-
-        self.start_token_id = self.token_to_id(start_token)
-        self.pad_token_id = self.token_to_id(pad_token)
-        self.end_token_id = self.token_to_id(end_token)
+    def set_vocabulary_and_merges(self, vocabulary, merges):
+        super().set_vocabulary_and_merges(vocabulary, merges)
+
+        if vocabulary is not None:
+            # Check for necessary special tokens.
+            for token in [self.start_token, self.pad_token, self.end_token]:
+                if token not in self.vocabulary:
+                    raise ValueError(
+                        f"Cannot find token `'{token}'` in the provided "
+                        f"`vocabulary`. Please provide `'{token}'` in your "
+                        "`vocabulary` or use a pretrained `vocabulary` name."
+                    )
+
+            self.start_token_id = self.token_to_id(self.start_token)
+            self.pad_token_id = self.token_to_id(self.pad_token)
+            self.end_token_id = self.token_to_id(self.end_token)
+        else:
+            self.start_token_id = None
+            self.pad_token_id = None
+            self.end_token_id = None
 
     @classproperty
     def presets(cls):
diff --git a/keras_nlp/models/bert/bert_masked_lm_preprocessor.py b/keras_nlp/models/bert/bert_masked_lm_preprocessor.py
index 685c65c7..cdc61fba 100644
--- a/keras_nlp/models/bert/bert_masked_lm_preprocessor.py
+++ b/keras_nlp/models/bert/bert_masked_lm_preprocessor.py
@@ -134,33 +134,30 @@ class BertMaskedLMPreprocessor(BertPreprocessor):
             truncate=truncate,
             **kwargs,
         )
-
+        self.mask_selection_rate = mask_selection_rate
+        self.mask_selection_length = mask_selection_length
+        self.mask_token_rate = mask_token_rate
+        self.random_token_rate = random_token_rate
+        self.masker = None
+
+    def build(self, input_shape):
+        super().build(input_shape)
+        # Defer masker creation to `build()` so that we can be sure tokenizer
+        # assets have loaded when restoring a saved model.
         self.masker = MaskedLMMaskGenerator(
-            mask_selection_rate=mask_selection_rate,
-            mask_selection_length=mask_selection_length,
-            mask_token_rate=mask_token_rate,
-            random_token_rate=random_token_rate,
-            vocabulary_size=tokenizer.vocabulary_size(),
-            mask_token_id=tokenizer.mask_token_id,
+            mask_selection_rate=self.mask_selection_rate,
+            mask_selection_length=self.mask_selection_length,
+            mask_token_rate=self.mask_token_rate,
+            random_token_rate=self.random_token_rate,
+            vocabulary_size=self.tokenizer.vocabulary_size(),
+            mask_token_id=self.tokenizer.mask_token_id,
             unselectable_token_ids=[
-                tokenizer.cls_token_id,
-                tokenizer.sep_token_id,
-                tokenizer.pad_token_id,
+                self.tokenizer.cls_token_id,
+                self.tokenizer.sep_token_id,
+                self.tokenizer.pad_token_id,
             ],
         )
 
-    def get_config(self):
-        config = super().get_config()
-        config.update(
-            {
-                "mask_selection_rate": self.masker.mask_selection_rate,
-                "mask_selection_length": self.masker.mask_selection_length,
-                "mask_token_rate": self.masker.mask_token_rate,
-                "random_token_rate": self.masker.random_token_rate,
-            }
-        )
-        return config
-
     def call(self, x, y=None, sample_weight=None):
         if y is not None or sample_weight is not None:
             logging.warning(
@@ -187,3 +184,15 @@ class BertMaskedLMPreprocessor(BertPreprocessor):
         y = masker_outputs["mask_ids"]
         sample_weight = masker_outputs["mask_weights"]
         return pack_x_y_sample_weight(x, y, sample_weight)
+
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "mask_selection_rate": self.mask_selection_rate,
+                "mask_selection_length": self.mask_selection_length,
+                "mask_token_rate": self.mask_token_rate,
+                "random_token_rate": self.random_token_rate,
+            }
+        )
+        return config
diff --git a/keras_nlp/models/bert/bert_preprocessor.py b/keras_nlp/models/bert/bert_preprocessor.py
index 21419375..bad38f22 100644
--- a/keras_nlp/models/bert/bert_preprocessor.py
+++ b/keras_nlp/models/bert/bert_preprocessor.py
@@ -139,23 +139,21 @@ class BertPreprocessor(Preprocessor):
     ):
         super().__init__(**kwargs)
         self.tokenizer = tokenizer
+        self.sequence_length = sequence_length
+        self.truncate = truncate
+        self.packer = None
+
+    def build(self, input_shape):
+        # Defer packer creation to `build()` so that we can be sure tokenizer
+        # assets have loaded when restoring a saved model.
         self.packer = MultiSegmentPacker(
             start_value=self.tokenizer.cls_token_id,
             end_value=self.tokenizer.sep_token_id,
             pad_value=self.tokenizer.pad_token_id,
-            truncate=truncate,
-            sequence_length=sequence_length,
+            truncate=self.truncate,
+            sequence_length=self.sequence_length,
         )
-
-    def get_config(self):
-        config = super().get_config()
-        config.update(
-            {
-                "sequence_length": self.packer.sequence_length,
-                "truncate": self.packer.truncate,
-            }
-        )
-        return config
+        self.built = True
 
     def call(self, x, y=None, sample_weight=None):
         x = convert_inputs_to_list_of_tensor_segments(x)
@@ -168,6 +166,16 @@ class BertPreprocessor(Preprocessor):
         }
         return pack_x_y_sample_weight(x, y, sample_weight)
 
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "sequence_length": self.sequence_length,
+                "truncate": self.truncate,
+            }
+        )
+        return config
+
     @classproperty
     def tokenizer_cls(cls):
         return BertTokenizer
diff --git a/keras_nlp/models/bert/bert_presets.py b/keras_nlp/models/bert/bert_presets.py
index 7a3bbdce..1730e172 100644
--- a/keras_nlp/models/bert/bert_presets.py
+++ b/keras_nlp/models/bert/bert_presets.py
@@ -13,8 +13,6 @@
 # limitations under the License.
 """BERT model preset configurations."""
 
-# TODO(jbischof): document presets in keras.io and use URL in docstrings
-# Metadata for loading pretrained model weights.
 backbone_presets = {
     "bert_tiny_en_uncased": {
         "metadata": {
@@ -27,23 +25,7 @@ backbone_presets = {
             "path": "bert",
             "model_card": "https://github.com/google-research/bert/blob/master/README.md",
         },
-        "config": {
-            "vocabulary_size": 30522,
-            "num_layers": 2,
-            "num_heads": 2,
-            "hidden_dim": 128,
-            "intermediate_dim": 512,
-            "dropout": 0.1,
-            "max_sequence_length": 512,
-            "num_segments": 2,
-        },
-        "preprocessor_config": {
-            "lowercase": True,
-        },
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/bert_tiny_en_uncased/v1/model.h5",
-        "weights_hash": "c2b29fcbf8f814a0812e4ab89ef5c068",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/bert_tiny_en_uncased/v1/vocab.txt",
-        "vocabulary_hash": "64800d5d8528ce344256daf115d4965e",
+        "kaggle_handle": "kaggle://keras/bert/bert_tiny_en_uncased/2",
     },
     "bert_small_en_uncased": {
         "metadata": {
@@ -56,23 +38,7 @@ backbone_presets = {
             "path": "bert",
             "model_card": "https://github.com/google-research/bert/blob/master/README.md",
         },
-        "config": {
-            "vocabulary_size": 30522,
-            "num_layers": 4,
-            "num_heads": 8,
-            "hidden_dim": 512,
-            "intermediate_dim": 2048,
-            "dropout": 0.1,
-            "max_sequence_length": 512,
-            "num_segments": 2,
-        },
-        "preprocessor_config": {
-            "lowercase": True,
-        },
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/bert_small_en_uncased/v1/model.h5",
-        "weights_hash": "08632c9479b034f342ba2c2b7afba5f7",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/bert_small_en_uncased/v1/vocab.txt",
-        "vocabulary_hash": "64800d5d8528ce344256daf115d4965e",
+        "kaggle_handle": "kaggle://keras/bert/bert_small_en_uncased/2",
     },
     "bert_medium_en_uncased": {
         "metadata": {
@@ -85,23 +51,7 @@ backbone_presets = {
             "path": "bert",
             "model_card": "https://github.com/google-research/bert/blob/master/README.md",
         },
-        "config": {
-            "vocabulary_size": 30522,
-            "num_layers": 8,
-            "num_heads": 8,
-            "hidden_dim": 512,
-            "intermediate_dim": 2048,
-            "dropout": 0.1,
-            "max_sequence_length": 512,
-            "num_segments": 2,
-        },
-        "preprocessor_config": {
-            "lowercase": True,
-        },
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/bert_medium_en_uncased/v1/model.h5",
-        "weights_hash": "bb990e1184ec6b6185450c73833cd661",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/bert_medium_en_uncased/v1/vocab.txt",
-        "vocabulary_hash": "64800d5d8528ce344256daf115d4965e",
+        "kaggle_handle": "kaggle://keras/bert/bert_medium_en_uncased/2",
     },
     "bert_base_en_uncased": {
         "metadata": {
@@ -114,23 +64,7 @@ backbone_presets = {
             "path": "bert",
             "model_card": "https://github.com/google-research/bert/blob/master/README.md",
         },
-        "config": {
-            "vocabulary_size": 30522,
-            "num_layers": 12,
-            "num_heads": 12,
-            "hidden_dim": 768,
-            "intermediate_dim": 3072,
-            "dropout": 0.1,
-            "max_sequence_length": 512,
-            "num_segments": 2,
-        },
-        "preprocessor_config": {
-            "lowercase": True,
-        },
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/bert_base_en_uncased/v1/model.h5",
-        "weights_hash": "9b2b2139f221988759ac9cdd17050b31",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/bert_base_en_uncased/v1/vocab.txt",
-        "vocabulary_hash": "64800d5d8528ce344256daf115d4965e",
+        "kaggle_handle": "kaggle://keras/bert/bert_base_en_uncased/2",
     },
     "bert_base_en": {
         "metadata": {
@@ -143,23 +77,7 @@ backbone_presets = {
             "path": "bert",
             "model_card": "https://github.com/google-research/bert/blob/master/README.md",
         },
-        "config": {
-            "vocabulary_size": 28996,
-            "num_layers": 12,
-            "num_heads": 12,
-            "hidden_dim": 768,
-            "intermediate_dim": 3072,
-            "dropout": 0.1,
-            "max_sequence_length": 512,
-            "num_segments": 2,
-        },
-        "preprocessor_config": {
-            "lowercase": False,
-        },
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/bert_base_en/v1/model.h5",
-        "weights_hash": "f94a6cb012e18f4fb8ec92abb91864e9",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/bert_base_en/v1/vocab.txt",
-        "vocabulary_hash": "bb6ca9b42e790e5cd986bbb16444d0e0",
+        "kaggle_handle": "kaggle://keras/bert/bert_base_en/2",
     },
     "bert_base_zh": {
         "metadata": {
@@ -171,23 +89,7 @@ backbone_presets = {
             "path": "bert",
             "model_card": "https://github.com/google-research/bert/blob/master/README.md",
         },
-        "config": {
-            "vocabulary_size": 21128,
-            "num_layers": 12,
-            "num_heads": 12,
-            "hidden_dim": 768,
-            "intermediate_dim": 3072,
-            "dropout": 0.1,
-            "max_sequence_length": 512,
-            "num_segments": 2,
-        },
-        "preprocessor_config": {
-            "lowercase": False,
-        },
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/bert_base_zh/v1/model.h5",
-        "weights_hash": "79afa421e386076e62ab42dad555ab0c",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/bert_base_zh/v1/vocab.txt",
-        "vocabulary_hash": "3b5b76c4aef48ecf8cb3abaafe960f09",
+        "kaggle_handle": "kaggle://keras/bert/bert_base_zh/2",
     },
     "bert_base_multi": {
         "metadata": {
@@ -199,23 +101,7 @@ backbone_presets = {
             "path": "bert",
             "model_card": "https://github.com/google-research/bert/blob/master/README.md",
         },
-        "config": {
-            "vocabulary_size": 119547,
-            "num_layers": 12,
-            "num_heads": 12,
-            "hidden_dim": 768,
-            "intermediate_dim": 3072,
-            "dropout": 0.1,
-            "max_sequence_length": 512,
-            "num_segments": 2,
-        },
-        "preprocessor_config": {
-            "lowercase": False,
-        },
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/bert_base_multi/v1/model.h5",
-        "weights_hash": "b0631cec0a1f2513c6cfd75ba29c33aa",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/bert_base_multi/v1/vocab.txt",
-        "vocabulary_hash": "d9d865138d17f1958502ed060ecfeeb6",
+        "kaggle_handle": "kaggle://keras/bert/bert_base_multi/2",
     },
     "bert_large_en_uncased": {
         "metadata": {
@@ -228,23 +114,7 @@ backbone_presets = {
             "path": "bert",
             "model_card": "https://github.com/google-research/bert/blob/master/README.md",
         },
-        "config": {
-            "vocabulary_size": 30522,
-            "num_layers": 24,
-            "num_heads": 16,
-            "hidden_dim": 1024,
-            "intermediate_dim": 4096,
-            "dropout": 0.1,
-            "max_sequence_length": 512,
-            "num_segments": 2,
-        },
-        "preprocessor_config": {
-            "lowercase": True,
-        },
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/bert_large_en_uncased/v1/model.h5",
-        "weights_hash": "cc5cacc9565ef400ee4376105f40ddae",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/bert_large_en_uncased/v1/vocab.txt",
-        "vocabulary_hash": "64800d5d8528ce344256daf115d4965e",
+        "kaggle_handle": "kaggle://keras/bert/bert_large_en_uncased/2",
     },
     "bert_large_en": {
         "metadata": {
@@ -257,23 +127,7 @@ backbone_presets = {
             "path": "bert",
             "model_card": "https://github.com/google-research/bert/blob/master/README.md",
         },
-        "config": {
-            "vocabulary_size": 28996,
-            "num_layers": 24,
-            "num_heads": 16,
-            "hidden_dim": 1024,
-            "intermediate_dim": 4096,
-            "dropout": 0.1,
-            "max_sequence_length": 512,
-            "num_segments": 2,
-        },
-        "preprocessor_config": {
-            "lowercase": False,
-        },
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/bert_large_en/v1/model.h5",
-        "weights_hash": "8b8ab82290bbf4f8db87d4f100648890",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/bert_large_en/v1/vocab.txt",
-        "vocabulary_hash": "bb6ca9b42e790e5cd986bbb16444d0e0",
+        "kaggle_handle": "kaggle://keras/bert/bert_large_en/2",
     },
 }
 
@@ -288,29 +142,6 @@ classifier_presets = {
             "path": "bert",
             "model_card": "https://github.com/google-research/bert/blob/master/README.md",
         },
-        "config": {
-            "backbone": {
-                "class_name": "keras_nlp>BertBackbone",
-                "config": {
-                    "vocabulary_size": 30522,
-                    "hidden_dim": 128,
-                    "intermediate_dim": 512,
-                    "num_layers": 2,
-                    "num_heads": 2,
-                    "max_sequence_length": 512,
-                    "num_segments": 2,
-                    "dropout": 0.1,
-                },
-            },
-            "num_classes": 2,
-            "dropout": 0.1,
-        },
-        "preprocessor_config": {
-            "lowercase": True,
-        },
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/bert_tiny_en_uncased_sst2/v1/model.h5",
-        "weights_hash": "1f9c2d59f9e229e08f3fbd44239cfb0b",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/bert_tiny_en_uncased_sst2/v1/vocab.txt",
-        "vocabulary_hash": "64800d5d8528ce344256daf115d4965e",
+        "kaggle_handle": "kaggle://keras/bert/bert_tiny_en_uncased_sst2/3",
     }
 }
diff --git a/keras_nlp/models/bert/bert_tokenizer.py b/keras_nlp/models/bert/bert_tokenizer.py
index 5c01bf24..1b634fe9 100644
--- a/keras_nlp/models/bert/bert_tokenizer.py
+++ b/keras_nlp/models/bert/bert_tokenizer.py
@@ -74,33 +74,42 @@ class BertTokenizer(WordPieceTokenizer):
 
     def __init__(
         self,
-        vocabulary,
+        vocabulary=None,
         lowercase=False,
         **kwargs,
     ):
+        self.cls_token = "[CLS]"
+        self.sep_token = "[SEP]"
+        self.pad_token = "[PAD]"
+        self.mask_token = "[MASK]"
         super().__init__(
             vocabulary=vocabulary,
             lowercase=lowercase,
             **kwargs,
         )
 
-        # Check for necessary special tokens.
-        cls_token = "[CLS]"
-        sep_token = "[SEP]"
-        pad_token = "[PAD]"
-        mask_token = "[MASK]"
-        for token in [cls_token, pad_token, sep_token]:
-            if token not in self.get_vocabulary():
-                raise ValueError(
-                    f"Cannot find token `'{token}'` in the provided "
-                    f"`vocabulary`. Please provide `'{token}'` in your "
-                    "`vocabulary` or use a pretrained `vocabulary` name."
-                )
-
-        self.cls_token_id = self.token_to_id(cls_token)
-        self.sep_token_id = self.token_to_id(sep_token)
-        self.pad_token_id = self.token_to_id(pad_token)
-        self.mask_token_id = self.token_to_id(mask_token)
+    def set_vocabulary(self, vocabulary):
+        super().set_vocabulary(vocabulary)
+
+        if vocabulary is not None:
+            # Check for necessary special tokens.
+            for token in [self.cls_token, self.pad_token, self.sep_token]:
+                if token not in self.vocabulary:
+                    raise ValueError(
+                        f"Cannot find token `'{token}'` in the provided "
+                        f"`vocabulary`. Please provide `'{token}'` in your "
+                        "`vocabulary` or use a pretrained `vocabulary` name."
+                    )
+
+            self.cls_token_id = self.token_to_id(self.cls_token)
+            self.sep_token_id = self.token_to_id(self.sep_token)
+            self.pad_token_id = self.token_to_id(self.pad_token)
+            self.mask_token_id = self.token_to_id(self.mask_token)
+        else:
+            self.cls_token_id = None
+            self.sep_token_id = None
+            self.pad_token_id = None
+            self.mask_token_id = None
 
     @classproperty
     def presets(cls):
diff --git a/keras_nlp/models/deberta_v3/deberta_v3_backbone.py b/keras_nlp/models/deberta_v3/deberta_v3_backbone.py
index 76e2cf9d..aa5077ec 100644
--- a/keras_nlp/models/deberta_v3/deberta_v3_backbone.py
+++ b/keras_nlp/models/deberta_v3/deberta_v3_backbone.py
@@ -68,7 +68,7 @@ class DebertaV3Backbone(Backbone):
         bucket_size: int. The size of the relative position buckets. Generally
             equal to `max_sequence_length // 2`.
 
-    Example usage:
+    Example:
     ```python
     input_data = {
         "token_ids": np.ones(shape=(1, 12), dtype="int32"),
diff --git a/keras_nlp/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py b/keras_nlp/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py
index 1644c138..519b0b4f 100644
--- a/keras_nlp/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py
+++ b/keras_nlp/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py
@@ -133,17 +133,27 @@ class DebertaV3MaskedLMPreprocessor(DebertaV3Preprocessor):
             **kwargs,
         )
 
+        self.mask_selection_rate = mask_selection_rate
+        self.mask_selection_length = mask_selection_length
+        self.mask_token_rate = mask_token_rate
+        self.random_token_rate = random_token_rate
+        self.masker = None
+
+    def build(self, input_shape):
+        super().build(input_shape)
+        # Defer masker creation to `build()` so that we can be sure tokenizer
+        # assets have loaded when restoring a saved model.
         self.masker = MaskedLMMaskGenerator(
-            mask_selection_rate=mask_selection_rate,
-            mask_selection_length=mask_selection_length,
-            mask_token_rate=mask_token_rate,
-            random_token_rate=random_token_rate,
-            vocabulary_size=tokenizer.vocabulary_size(),
-            mask_token_id=tokenizer.mask_token_id,
+            mask_selection_rate=self.mask_selection_rate,
+            mask_selection_length=self.mask_selection_length,
+            mask_token_rate=self.mask_token_rate,
+            random_token_rate=self.random_token_rate,
+            vocabulary_size=self.tokenizer.vocabulary_size(),
+            mask_token_id=self.tokenizer.mask_token_id,
             unselectable_token_ids=[
-                tokenizer.cls_token_id,
-                tokenizer.sep_token_id,
-                tokenizer.pad_token_id,
+                self.tokenizer.cls_token_id,
+                self.tokenizer.sep_token_id,
+                self.tokenizer.pad_token_id,
             ],
         )
 
@@ -151,10 +161,10 @@ class DebertaV3MaskedLMPreprocessor(DebertaV3Preprocessor):
         config = super().get_config()
         config.update(
             {
-                "mask_selection_rate": self.masker.mask_selection_rate,
-                "mask_selection_length": self.masker.mask_selection_length,
-                "mask_token_rate": self.masker.mask_token_rate,
-                "random_token_rate": self.masker.random_token_rate,
+                "mask_selection_rate": self.mask_selection_rate,
+                "mask_selection_length": self.mask_selection_length,
+                "mask_token_rate": self.mask_token_rate,
+                "random_token_rate": self.random_token_rate,
             }
         )
         return config
diff --git a/keras_nlp/models/deberta_v3/deberta_v3_preprocessor.py b/keras_nlp/models/deberta_v3/deberta_v3_preprocessor.py
index dee91dcf..93f4fbbd 100644
--- a/keras_nlp/models/deberta_v3/deberta_v3_preprocessor.py
+++ b/keras_nlp/models/deberta_v3/deberta_v3_preprocessor.py
@@ -156,20 +156,28 @@ class DebertaV3Preprocessor(Preprocessor):
     ):
         super().__init__(**kwargs)
         self.tokenizer = tokenizer
+        self.truncate = truncate
+        self.sequence_length = sequence_length
+        self.packer = None
+
+    def build(self, input_shape):
+        # Defer packer creation to `build()` so that we can be sure tokenizer
+        # assets have loaded when restoring a saved model.
         self.packer = MultiSegmentPacker(
             start_value=self.tokenizer.cls_token_id,
             end_value=self.tokenizer.sep_token_id,
             pad_value=self.tokenizer.pad_token_id,
-            truncate=truncate,
-            sequence_length=sequence_length,
+            truncate=self.truncate,
+            sequence_length=self.sequence_length,
         )
+        self.built = True
 
     def get_config(self):
         config = super().get_config()
         config.update(
             {
-                "sequence_length": self.packer.sequence_length,
-                "truncate": self.packer.truncate,
+                "sequence_length": self.sequence_length,
+                "truncate": self.truncate,
             }
         )
         return config
diff --git a/keras_nlp/models/deberta_v3/deberta_v3_presets.py b/keras_nlp/models/deberta_v3/deberta_v3_presets.py
index f5df6cb5..ba8734e7 100644
--- a/keras_nlp/models/deberta_v3/deberta_v3_presets.py
+++ b/keras_nlp/models/deberta_v3/deberta_v3_presets.py
@@ -25,21 +25,7 @@ backbone_presets = {
             "path": "deberta_v3",
             "model_card": "https://huggingface.co/microsoft/deberta-v3-xsmall",
         },
-        "config": {
-            "vocabulary_size": 128100,
-            "num_layers": 12,
-            "num_heads": 6,
-            "hidden_dim": 384,
-            "intermediate_dim": 1536,
-            "dropout": 0.1,
-            "max_sequence_length": 512,
-            "bucket_size": 256,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/deberta_v3_extra_small_en/v1/model.h5",
-        "weights_hash": "d8e10327107e5c5e20b45548a5028619",
-        "spm_proto_url": "https://storage.googleapis.com/keras-nlp/models/deberta_v3_extra_small_en/v1/vocab.spm",
-        "spm_proto_hash": "1613fcbf3b82999c187b09c9db79b568",
+        "kaggle_handle": "kaggle://keras/deberta_v3/deberta_v3_extra_small_en/2",
     },
     "deberta_v3_small_en": {
         "metadata": {
@@ -52,21 +38,7 @@ backbone_presets = {
             "path": "deberta_v3",
             "model_card": "https://huggingface.co/microsoft/deberta-v3-small",
         },
-        "config": {
-            "vocabulary_size": 128100,
-            "num_layers": 6,
-            "num_heads": 12,
-            "hidden_dim": 768,
-            "intermediate_dim": 3072,
-            "dropout": 0.1,
-            "max_sequence_length": 512,
-            "bucket_size": 256,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/deberta_v3_small_en/v1/model.h5",
-        "weights_hash": "84118eb7c5a735f2061ecccaf71bb888",
-        "spm_proto_url": "https://storage.googleapis.com/keras-nlp/models/deberta_v3_small_en/v1/vocab.spm",
-        "spm_proto_hash": "1613fcbf3b82999c187b09c9db79b568",
+        "kaggle_handle": "kaggle://keras/deberta_v3/deberta_v3_small_en/2",
     },
     "deberta_v3_base_en": {
         "metadata": {
@@ -79,21 +51,7 @@ backbone_presets = {
             "path": "deberta_v3",
             "model_card": "https://huggingface.co/microsoft/deberta-v3-base",
         },
-        "config": {
-            "vocabulary_size": 128100,
-            "num_layers": 12,
-            "num_heads": 12,
-            "hidden_dim": 768,
-            "intermediate_dim": 3072,
-            "dropout": 0.1,
-            "max_sequence_length": 512,
-            "bucket_size": 256,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/deberta_v3_base_en/v1/model.h5",
-        "weights_hash": "cebce044aeed36aec9b94e3b8a255430",
-        "spm_proto_url": "https://storage.googleapis.com/keras-nlp/models/deberta_v3_base_en/v1/vocab.spm",
-        "spm_proto_hash": "1613fcbf3b82999c187b09c9db79b568",
+        "kaggle_handle": "kaggle://keras/deberta_v3/deberta_v3_base_en/2",
     },
     "deberta_v3_large_en": {
         "metadata": {
@@ -106,21 +64,7 @@ backbone_presets = {
             "path": "deberta_v3",
             "model_card": "https://huggingface.co/microsoft/deberta-v3-large",
         },
-        "config": {
-            "vocabulary_size": 128100,
-            "num_layers": 24,
-            "num_heads": 16,
-            "hidden_dim": 1024,
-            "intermediate_dim": 4096,
-            "dropout": 0.1,
-            "max_sequence_length": 512,
-            "bucket_size": 256,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/deberta_v3_large_en/v1/model.h5",
-        "weights_hash": "bce7690f358a9e39304f8c0ebc71a745",
-        "spm_proto_url": "https://storage.googleapis.com/keras-nlp/models/deberta_v3_large_en/v1/vocab.spm",
-        "spm_proto_hash": "1613fcbf3b82999c187b09c9db79b568",
+        "kaggle_handle": "kaggle://keras/deberta_v3/deberta_v3_large_en/2",
     },
     "deberta_v3_base_multi": {
         "metadata": {
@@ -133,20 +77,6 @@ backbone_presets = {
             "path": "deberta_v3",
             "model_card": "https://huggingface.co/microsoft/mdeberta-v3-base",
         },
-        "config": {
-            "vocabulary_size": 251000,
-            "num_layers": 12,
-            "num_heads": 12,
-            "hidden_dim": 768,
-            "intermediate_dim": 3072,
-            "dropout": 0.1,
-            "max_sequence_length": 512,
-            "bucket_size": 256,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/deberta_v3_base_multi/v1/model.h5",
-        "weights_hash": "26e5a824b26afd2ee336835bd337bbeb",
-        "spm_proto_url": "https://storage.googleapis.com/keras-nlp/models/deberta_v3_base_multi/v1/vocab.spm",
-        "spm_proto_hash": "b4ca07289eac48600b29529119d565e2",
+        "kaggle_handle": "kaggle://keras/deberta_v3/deberta_v3_base_multi/2",
     },
 }
diff --git a/keras_nlp/models/deberta_v3/deberta_v3_tokenizer.py b/keras_nlp/models/deberta_v3/deberta_v3_tokenizer.py
index 03c9cd58..e66c373e 100644
--- a/keras_nlp/models/deberta_v3/deberta_v3_tokenizer.py
+++ b/keras_nlp/models/deberta_v3/deberta_v3_tokenizer.py
@@ -93,33 +93,38 @@ class DebertaV3Tokenizer(SentencePieceTokenizer):
     """
 
     def __init__(self, proto, **kwargs):
+        self.cls_token = "[CLS]"
+        self.sep_token = "[SEP]"
+        self.pad_token = "[PAD]"
+        self.mask_token = "[MASK]"
+
         super().__init__(proto=proto, **kwargs)
 
-        # Check for necessary special tokens.
-        cls_token = "[CLS]"
-        sep_token = "[SEP]"
-        pad_token = "[PAD]"
-        mask_token = "[MASK]"
-
-        # We do not throw an error if `mask_token` is not present in the
-        # vocabulary.
-        for token in [cls_token, pad_token, sep_token]:
-            if token not in super().get_vocabulary():
-                raise ValueError(
-                    f"Cannot find token `'{token}'` in the provided "
-                    f"`vocabulary`. Please provide `'{token}'` in your "
-                    "`vocabulary` or use a pretrained `vocabulary` name."
-                )
-
-        self.cls_token_id = self.token_to_id(cls_token)
-        self.sep_token_id = self.token_to_id(sep_token)
-        self.pad_token_id = self.token_to_id(pad_token)
-        # If the mask token is not in the vocabulary, add it to the end of the
-        # vocabulary.
-        if mask_token in super().get_vocabulary():
-            self.mask_token_id = super().token_to_id(mask_token)
+    def set_proto(self, proto):
+        super().set_proto(proto)
+        if proto is not None:
+            for token in [self.cls_token, self.pad_token, self.sep_token]:
+                if token not in super().get_vocabulary():
+                    raise ValueError(
+                        f"Cannot find token `'{token}'` in the provided "
+                        f"`vocabulary`. Please provide `'{token}'` in your "
+                        "`vocabulary` or use a pretrained `vocabulary` name."
+                    )
+
+            self.cls_token_id = self.token_to_id(self.cls_token)
+            self.sep_token_id = self.token_to_id(self.sep_token)
+            self.pad_token_id = self.token_to_id(self.pad_token)
+            # If the mask token is not in the vocabulary, add it to the end of the
+            # vocabulary.
+            if self.mask_token in super().get_vocabulary():
+                self.mask_token_id = super().token_to_id(self.mask_token)
+            else:
+                self.mask_token_id = super().vocabulary_size()
         else:
-            self.mask_token_id = super().vocabulary_size()
+            self.cls_token_id = None
+            self.sep_token_id = None
+            self.pad_token_id = None
+            self.mask_token_id = None
 
     def vocabulary_size(self):
         sentence_piece_size = super().vocabulary_size()
diff --git a/keras_nlp/models/distil_bert/distil_bert_masked_lm_preprocessor.py b/keras_nlp/models/distil_bert/distil_bert_masked_lm_preprocessor.py
index 3fcf9bce..f1360f58 100644
--- a/keras_nlp/models/distil_bert/distil_bert_masked_lm_preprocessor.py
+++ b/keras_nlp/models/distil_bert/distil_bert_masked_lm_preprocessor.py
@@ -136,33 +136,30 @@ class DistilBertMaskedLMPreprocessor(DistilBertPreprocessor):
             truncate=truncate,
             **kwargs,
         )
-
+        self.mask_selection_rate = mask_selection_rate
+        self.mask_selection_length = mask_selection_length
+        self.mask_token_rate = mask_token_rate
+        self.random_token_rate = random_token_rate
+        self.masker = None
+
+    def build(self, input_shape):
+        super().build(input_shape)
+        # Defer masker creation to `build()` so that we can be sure tokenizer
+        # assets have loaded when restoring a saved model.
         self.masker = MaskedLMMaskGenerator(
-            mask_selection_rate=mask_selection_rate,
-            mask_selection_length=mask_selection_length,
-            mask_token_rate=mask_token_rate,
-            random_token_rate=random_token_rate,
-            vocabulary_size=tokenizer.vocabulary_size(),
-            mask_token_id=tokenizer.mask_token_id,
+            mask_selection_rate=self.mask_selection_rate,
+            mask_selection_length=self.mask_selection_length,
+            mask_token_rate=self.mask_token_rate,
+            random_token_rate=self.random_token_rate,
+            vocabulary_size=self.tokenizer.vocabulary_size(),
+            mask_token_id=self.tokenizer.mask_token_id,
             unselectable_token_ids=[
-                tokenizer.cls_token_id,
-                tokenizer.sep_token_id,
-                tokenizer.pad_token_id,
+                self.tokenizer.cls_token_id,
+                self.tokenizer.sep_token_id,
+                self.tokenizer.pad_token_id,
             ],
         )
 
-    def get_config(self):
-        config = super().get_config()
-        config.update(
-            {
-                "mask_selection_rate": self.masker.mask_selection_rate,
-                "mask_selection_length": self.masker.mask_selection_length,
-                "mask_token_rate": self.masker.mask_token_rate,
-                "random_token_rate": self.masker.random_token_rate,
-            }
-        )
-        return config
-
     def call(self, x, y=None, sample_weight=None):
         if y is not None or sample_weight is not None:
             logging.warning(
@@ -183,3 +180,15 @@ class DistilBertMaskedLMPreprocessor(DistilBertPreprocessor):
         y = masker_outputs["mask_ids"]
         sample_weight = masker_outputs["mask_weights"]
         return pack_x_y_sample_weight(x, y, sample_weight)
+
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "mask_selection_rate": self.mask_selection_rate,
+                "mask_selection_length": self.mask_selection_length,
+                "mask_token_rate": self.mask_token_rate,
+                "random_token_rate": self.random_token_rate,
+            }
+        )
+        return config
diff --git a/keras_nlp/models/distil_bert/distil_bert_preprocessor.py b/keras_nlp/models/distil_bert/distil_bert_preprocessor.py
index f2c43262..107275f8 100644
--- a/keras_nlp/models/distil_bert/distil_bert_preprocessor.py
+++ b/keras_nlp/models/distil_bert/distil_bert_preprocessor.py
@@ -127,24 +127,21 @@ class DistilBertPreprocessor(Preprocessor):
     ):
         super().__init__(**kwargs)
         self.tokenizer = tokenizer
+        self.sequence_length = sequence_length
+        self.truncate = truncate
+
+    def build(self, input_shape):
+        super().build(input_shape)
+        # Defer masker creation to `build()` so that we can be sure tokenizer
+        # assets have loaded when restoring a saved model.
         self.packer = MultiSegmentPacker(
             start_value=self.tokenizer.cls_token_id,
             end_value=self.tokenizer.sep_token_id,
             pad_value=self.tokenizer.pad_token_id,
-            truncate=truncate,
-            sequence_length=sequence_length,
+            truncate=self.truncate,
+            sequence_length=self.sequence_length,
         )
 
-    def get_config(self):
-        config = super().get_config()
-        config.update(
-            {
-                "sequence_length": self.packer.sequence_length,
-                "truncate": self.packer.truncate,
-            }
-        )
-        return config
-
     def call(self, x, y=None, sample_weight=None):
         x = convert_inputs_to_list_of_tensor_segments(x)
         x = [self.tokenizer(segment) for segment in x]
@@ -155,6 +152,16 @@ class DistilBertPreprocessor(Preprocessor):
         }
         return pack_x_y_sample_weight(x, y, sample_weight)
 
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "sequence_length": self.sequence_length,
+                "truncate": self.truncate,
+            }
+        )
+        return config
+
     @classproperty
     def tokenizer_cls(cls):
         return DistilBertTokenizer
diff --git a/keras_nlp/models/distil_bert/distil_bert_presets.py b/keras_nlp/models/distil_bert/distil_bert_presets.py
index 3f939fb6..cdbd2349 100644
--- a/keras_nlp/models/distil_bert/distil_bert_presets.py
+++ b/keras_nlp/models/distil_bert/distil_bert_presets.py
@@ -26,22 +26,7 @@ backbone_presets = {
             "path": "distil_bert",
             "model_card": "https://huggingface.co/distilbert-base-uncased",
         },
-        "config": {
-            "vocabulary_size": 30522,
-            "num_layers": 6,
-            "num_heads": 12,
-            "hidden_dim": 768,
-            "intermediate_dim": 3072,
-            "dropout": 0.1,
-            "max_sequence_length": 512,
-        },
-        "preprocessor_config": {
-            "lowercase": True,
-        },
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/distil_bert_base_en_uncased/v1/model.h5",
-        "weights_hash": "6625a649572e74086d74c46b8d0b0da3",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/distil_bert_base_en_uncased/v1/vocab.txt",
-        "vocabulary_hash": "64800d5d8528ce344256daf115d4965e",
+        "kaggle_handle": "kaggle://keras/distil_bert/distil_bert_base_en_uncased/2",
     },
     "distil_bert_base_en": {
         "metadata": {
@@ -55,22 +40,7 @@ backbone_presets = {
             "path": "distil_bert",
             "model_card": "https://huggingface.co/distilbert-base-cased",
         },
-        "config": {
-            "vocabulary_size": 28996,
-            "num_layers": 6,
-            "num_heads": 12,
-            "hidden_dim": 768,
-            "intermediate_dim": 3072,
-            "dropout": 0.1,
-            "max_sequence_length": 512,
-        },
-        "preprocessor_config": {
-            "lowercase": False,
-        },
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/distil_bert_base_en/v1/model.h5",
-        "weights_hash": "fa36aa6865978efbf85a5c8264e5eb57",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/distil_bert_base_en/v1/vocab.txt",
-        "vocabulary_hash": "bb6ca9b42e790e5cd986bbb16444d0e0",
+        "kaggle_handle": "kaggle://keras/distil_bert/distil_bert_base_en/2",
     },
     "distil_bert_base_multi": {
         "metadata": {
@@ -82,21 +52,6 @@ backbone_presets = {
             "path": "distil_bert",
             "model_card": "https://huggingface.co/distilbert-base-multilingual-cased",
         },
-        "config": {
-            "vocabulary_size": 119547,
-            "num_layers": 6,
-            "num_heads": 12,
-            "hidden_dim": 768,
-            "intermediate_dim": 3072,
-            "dropout": 0.1,
-            "max_sequence_length": 512,
-        },
-        "preprocessor_config": {
-            "lowercase": False,
-        },
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/distil_bert_base_multi/v1/model.h5",
-        "weights_hash": "c0f11095e2a6455bd3b1a6d14800a7fa",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/distil_bert_base_multi/v1/vocab.txt",
-        "vocabulary_hash": "d9d865138d17f1958502ed060ecfeeb6",
+        "kaggle_handle": "kaggle://keras/distil_bert/distil_bert_base_multi/2",
     },
 }
diff --git a/keras_nlp/models/distil_bert/distil_bert_tokenizer.py b/keras_nlp/models/distil_bert/distil_bert_tokenizer.py
index 9e344a37..4a18398a 100644
--- a/keras_nlp/models/distil_bert/distil_bert_tokenizer.py
+++ b/keras_nlp/models/distil_bert/distil_bert_tokenizer.py
@@ -76,29 +76,38 @@ class DistilBertTokenizer(WordPieceTokenizer):
         lowercase=False,
         **kwargs,
     ):
+        self.cls_token = "[CLS]"
+        self.sep_token = "[SEP]"
+        self.pad_token = "[PAD]"
+        self.mask_token = "[MASK]"
         super().__init__(
             vocabulary=vocabulary,
             lowercase=lowercase,
             **kwargs,
         )
 
-        # Check for necessary special tokens.
-        cls_token = "[CLS]"
-        sep_token = "[SEP]"
-        pad_token = "[PAD]"
-        mask_token = "[MASK]"
-        for token in [cls_token, pad_token, sep_token, mask_token]:
-            if token not in self.get_vocabulary():
-                raise ValueError(
-                    f"Cannot find token `'{token}'` in the provided "
-                    f"`vocabulary`. Please provide `'{token}'` in your "
-                    "`vocabulary` or use a pretrained `vocabulary` name."
-                )
-
-        self.cls_token_id = self.token_to_id(cls_token)
-        self.sep_token_id = self.token_to_id(sep_token)
-        self.pad_token_id = self.token_to_id(pad_token)
-        self.mask_token_id = self.token_to_id(mask_token)
+    def set_vocabulary(self, vocabulary):
+        super().set_vocabulary(vocabulary)
+
+        if vocabulary is not None:
+            # Check for necessary special tokens.
+            for token in [self.cls_token, self.pad_token, self.sep_token]:
+                if token not in self.vocabulary:
+                    raise ValueError(
+                        f"Cannot find token `'{token}'` in the provided "
+                        f"`vocabulary`. Please provide `'{token}'` in your "
+                        "`vocabulary` or use a pretrained `vocabulary` name."
+                    )
+
+            self.cls_token_id = self.token_to_id(self.cls_token)
+            self.sep_token_id = self.token_to_id(self.sep_token)
+            self.pad_token_id = self.token_to_id(self.pad_token)
+            self.mask_token_id = self.token_to_id(self.mask_token)
+        else:
+            self.cls_token_id = None
+            self.sep_token_id = None
+            self.pad_token_id = None
+            self.mask_token_id = None
 
     @classproperty
     def presets(cls):
diff --git a/keras_nlp/models/f_net/f_net_masked_lm_preprocessor.py b/keras_nlp/models/f_net/f_net_masked_lm_preprocessor.py
index 87fa1a31..51b4a4d1 100644
--- a/keras_nlp/models/f_net/f_net_masked_lm_preprocessor.py
+++ b/keras_nlp/models/f_net/f_net_masked_lm_preprocessor.py
@@ -136,18 +136,27 @@ class FNetMaskedLMPreprocessor(FNetPreprocessor):
             truncate=truncate,
             **kwargs,
         )
-
+        self.mask_selection_rate = mask_selection_rate
+        self.mask_selection_length = mask_selection_length
+        self.mask_token_rate = mask_token_rate
+        self.random_token_rate = random_token_rate
+        self.masker = None
+
+    def build(self, input_shape):
+        super().build(input_shape)
+        # Defer masker creation to `build()` so that we can be sure tokenizer
+        # assets have loaded when restoring a saved model.
         self.masker = MaskedLMMaskGenerator(
-            mask_selection_rate=mask_selection_rate,
-            mask_selection_length=mask_selection_length,
-            mask_token_rate=mask_token_rate,
-            random_token_rate=random_token_rate,
-            vocabulary_size=tokenizer.vocabulary_size(),
-            mask_token_id=tokenizer.mask_token_id,
+            mask_selection_rate=self.mask_selection_rate,
+            mask_selection_length=self.mask_selection_length,
+            mask_token_rate=self.mask_token_rate,
+            random_token_rate=self.random_token_rate,
+            vocabulary_size=self.tokenizer.vocabulary_size(),
+            mask_token_id=self.tokenizer.mask_token_id,
             unselectable_token_ids=[
-                tokenizer.cls_token_id,
-                tokenizer.sep_token_id,
-                tokenizer.pad_token_id,
+                self.tokenizer.cls_token_id,
+                self.tokenizer.sep_token_id,
+                self.tokenizer.pad_token_id,
             ],
         )
 
@@ -155,10 +164,10 @@ class FNetMaskedLMPreprocessor(FNetPreprocessor):
         config = super().get_config()
         config.update(
             {
-                "mask_selection_rate": self.masker.mask_selection_rate,
-                "mask_selection_length": self.masker.mask_selection_length,
-                "mask_token_rate": self.masker.mask_token_rate,
-                "random_token_rate": self.masker.random_token_rate,
+                "mask_selection_rate": self.mask_selection_rate,
+                "mask_selection_length": self.mask_selection_length,
+                "mask_token_rate": self.mask_token_rate,
+                "random_token_rate": self.random_token_rate,
             }
         )
         return config
diff --git a/keras_nlp/models/f_net/f_net_preprocessor.py b/keras_nlp/models/f_net/f_net_preprocessor.py
index 5ebd5d16..296493c9 100644
--- a/keras_nlp/models/f_net/f_net_preprocessor.py
+++ b/keras_nlp/models/f_net/f_net_preprocessor.py
@@ -129,20 +129,28 @@ class FNetPreprocessor(Preprocessor):
     ):
         super().__init__(**kwargs)
         self.tokenizer = tokenizer
+        self.truncate = truncate
+        self.sequence_length = sequence_length
+        self.packer = None
+
+    def build(self, input_shape):
+        # Defer packer creation to `build()` so that we can be sure tokenizer
+        # assets have loaded when restoring a saved model.
         self.packer = MultiSegmentPacker(
             start_value=self.tokenizer.cls_token_id,
             end_value=self.tokenizer.sep_token_id,
             pad_value=self.tokenizer.pad_token_id,
-            truncate=truncate,
-            sequence_length=sequence_length,
+            truncate=self.truncate,
+            sequence_length=self.sequence_length,
         )
+        self.built = True
 
     def get_config(self):
         config = super().get_config()
         config.update(
             {
-                "sequence_length": self.packer.sequence_length,
-                "truncate": self.packer.truncate,
+                "sequence_length": self.sequence_length,
+                "truncate": self.truncate,
             }
         )
         return config
diff --git a/keras_nlp/models/f_net/f_net_presets.py b/keras_nlp/models/f_net/f_net_presets.py
index b3df5f8e..0db5f1c2 100644
--- a/keras_nlp/models/f_net/f_net_presets.py
+++ b/keras_nlp/models/f_net/f_net_presets.py
@@ -25,20 +25,7 @@ backbone_presets = {
             "path": "f_net",
             "model_card": "https://github.com/google-research/google-research/blob/master/f_net/README.md",
         },
-        "config": {
-            "vocabulary_size": 32000,
-            "num_layers": 12,
-            "hidden_dim": 768,
-            "intermediate_dim": 3072,
-            "dropout": 0.1,
-            "max_sequence_length": 512,
-            "num_segments": 4,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/f_net_base_en/v1/model.h5",
-        "weights_hash": "35db90842b85a985a0e54c86c00746fe",
-        "spm_proto_url": "https://storage.googleapis.com/keras-nlp/models/f_net_base_en/v1/vocab.spm",
-        "spm_proto_hash": "71c5f4610bef1daf116998a113a01f3d",
+        "kaggle_handle": "kaggle://keras/f_net/f_net_base_en/2",
     },
     "f_net_large_en": {
         "metadata": {
@@ -51,19 +38,6 @@ backbone_presets = {
             "path": "f_net",
             "model_card": "https://github.com/google-research/google-research/blob/master/f_net/README.md",
         },
-        "config": {
-            "vocabulary_size": 32000,
-            "num_layers": 24,
-            "hidden_dim": 1024,
-            "intermediate_dim": 4096,
-            "dropout": 0.1,
-            "max_sequence_length": 512,
-            "num_segments": 4,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/f_net_large_en/v1/model.h5",
-        "weights_hash": "7ae4a3faa67ff054f8cecffb5619f779",
-        "spm_proto_url": "https://storage.googleapis.com/keras-nlp/models/f_net_large_en/v1/vocab.spm",
-        "spm_proto_hash": "71c5f4610bef1daf116998a113a01f3d",
+        "kaggle_handle": "kaggle://keras/f_net/f_net_large_en/2",
     },
 }
diff --git a/keras_nlp/models/f_net/f_net_tokenizer.py b/keras_nlp/models/f_net/f_net_tokenizer.py
index 294e02e4..ae3f569b 100644
--- a/keras_nlp/models/f_net/f_net_tokenizer.py
+++ b/keras_nlp/models/f_net/f_net_tokenizer.py
@@ -63,25 +63,37 @@ class FNetTokenizer(SentencePieceTokenizer):
     """
 
     def __init__(self, proto, **kwargs):
+        self.cls_token = "[CLS]"
+        self.sep_token = "[SEP]"
+        self.pad_token = "<pad>"
+        self.mask_token = "[MASK]"
         super().__init__(proto=proto, **kwargs)
 
-        # Check for necessary special tokens.
-        cls_token = "[CLS]"
-        sep_token = "[SEP]"
-        pad_token = "<pad>"
-        mask_token = "[MASK]"
-        for token in [cls_token, sep_token, pad_token, mask_token]:
-            if token not in self.get_vocabulary():
-                raise ValueError(
-                    f"Cannot find token `'{token}'` in the provided "
-                    f"`vocabulary`. Please provide `'{token}'` in your "
-                    "`vocabulary` or use a pretrained `vocabulary` name."
-                )
-
-        self.cls_token_id = self.token_to_id(cls_token)
-        self.sep_token_id = self.token_to_id(sep_token)
-        self.pad_token_id = self.token_to_id(pad_token)
-        self.mask_token_id = self.token_to_id(mask_token)
+    def set_proto(self, proto):
+        super().set_proto(proto)
+        if proto is not None:
+            for token in [
+                self.cls_token,
+                self.sep_token,
+                self.pad_token,
+                self.mask_token,
+            ]:
+                if token not in self.get_vocabulary():
+                    raise ValueError(
+                        f"Cannot find token `'{token}'` in the provided "
+                        f"`vocabulary`. Please provide `'{token}'` in your "
+                        "`vocabulary` or use a pretrained `vocabulary` name."
+                    )
+
+            self.cls_token_id = self.token_to_id(self.cls_token)
+            self.sep_token_id = self.token_to_id(self.sep_token)
+            self.pad_token_id = self.token_to_id(self.pad_token)
+            self.mask_token_id = self.token_to_id(self.mask_token)
+        else:
+            self.cls_token_id = None
+            self.sep_token_id = None
+            self.pad_token_id = None
+            self.mask_token_id = None
 
     @classproperty
     def presets(cls):
diff --git a/keras_nlp/models/gpt2/gpt2_backbone.py b/keras_nlp/models/gpt2/gpt2_backbone.py
index 3f357d64..89c23f71 100644
--- a/keras_nlp/models/gpt2/gpt2_backbone.py
+++ b/keras_nlp/models/gpt2/gpt2_backbone.py
@@ -62,7 +62,7 @@ class GPT2Backbone(Backbone):
             sequence length. This determines the variable shape for positional
             embeddings.
 
-    Example usage:
+    Example:
     ```python
     input_data = {
         "token_ids": np.ones(shape=(1, 12), dtype="int32"),
diff --git a/keras_nlp/models/gpt2/gpt2_causal_lm_preprocessor.py b/keras_nlp/models/gpt2/gpt2_causal_lm_preprocessor.py
index 41ea591d..97d0b42d 100644
--- a/keras_nlp/models/gpt2/gpt2_causal_lm_preprocessor.py
+++ b/keras_nlp/models/gpt2/gpt2_causal_lm_preprocessor.py
@@ -142,6 +142,9 @@ class GPT2CausalLMPreprocessor(GPT2Preprocessor):
         the sequence (as generation is expected to continue at the end of the
         inputted prompt).
         """
+        if not self.built:
+            self.build(None)
+
         x = convert_inputs_to_list_of_tensor_segments(x)[0]
         x = self.tokenizer(x)
         token_ids, padding_mask = self.packer(
@@ -162,6 +165,9 @@ class GPT2CausalLMPreprocessor(GPT2Preprocessor):
         padding and start/end tokens, and then converting the integer sequence
         back to a string.
         """
+        if not self.built:
+            self.build(None)
+
         token_ids, padding_mask = x["token_ids"], x["padding_mask"]
         token_ids = ops.convert_to_numpy(token_ids)
         padding_mask = ops.convert_to_numpy(padding_mask)
diff --git a/keras_nlp/models/gpt2/gpt2_preprocessor.py b/keras_nlp/models/gpt2/gpt2_preprocessor.py
index bb373643..29182f77 100644
--- a/keras_nlp/models/gpt2/gpt2_preprocessor.py
+++ b/keras_nlp/models/gpt2/gpt2_preprocessor.py
@@ -123,24 +123,18 @@ class GPT2Preprocessor(Preprocessor):
         self.sequence_length = sequence_length
         self.add_start_token = add_start_token
         self.add_end_token = add_end_token
+
+    def build(self, input_shape):
+        # Defer packer creation to `build()` so that we can be sure tokenizer
+        # assets have loaded when restoring a saved model.
         self.packer = StartEndPacker(
-            start_value=tokenizer.start_token_id,
-            end_value=tokenizer.end_token_id,
-            pad_value=tokenizer.pad_token_id,
-            sequence_length=sequence_length,
+            start_value=self.tokenizer.start_token_id,
+            end_value=self.tokenizer.end_token_id,
+            pad_value=self.tokenizer.pad_token_id,
+            sequence_length=self.sequence_length,
             return_padding_mask=True,
         )
-
-    def get_config(self):
-        config = super().get_config()
-        config.update(
-            {
-                "sequence_length": self.sequence_length,
-                "add_start_token": self.add_start_token,
-                "add_end_token": self.add_end_token,
-            }
-        )
-        return config
+        self.built = True
 
     def call(
         self,
@@ -170,6 +164,17 @@ class GPT2Preprocessor(Preprocessor):
         }
         return pack_x_y_sample_weight(x, y, sample_weight)
 
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "sequence_length": self.sequence_length,
+                "add_start_token": self.add_start_token,
+                "add_end_token": self.add_end_token,
+            }
+        )
+        return config
+
     @classproperty
     def presets(cls):
         return copy.deepcopy(backbone_presets)
diff --git a/keras_nlp/models/gpt2/gpt2_presets.py b/keras_nlp/models/gpt2/gpt2_presets.py
index 7101bdb1..660c021f 100644
--- a/keras_nlp/models/gpt2/gpt2_presets.py
+++ b/keras_nlp/models/gpt2/gpt2_presets.py
@@ -26,22 +26,7 @@ backbone_presets = {
             "path": "gpt2",
             "model_card": "https://github.com/openai/gpt-2/blob/master/model_card.md",
         },
-        "config": {
-            "vocabulary_size": 50257,
-            "num_layers": 12,
-            "num_heads": 12,
-            "hidden_dim": 768,
-            "intermediate_dim": 3072,
-            "dropout": 0.1,
-            "max_sequence_length": 1024,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/gpt2_base_en/v1/model.h5",
-        "weights_hash": "f4ea6e1b214516dd7de452461ee6e16e",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/gpt2_base_en/v1/vocab.json",
-        "vocabulary_hash": "dffec25a898b1f5e569bec4dffd7e5c0",
-        "merges_url": "https://storage.googleapis.com/keras-nlp/models/gpt2_base_en/v1/merges.txt",
-        "merges_hash": "75a37753dd7a28a2c5df80c28bf06e4e",
+        "kaggle_handle": "kaggle://keras/gpt2/gpt2_base_en/2",
     },
     "gpt2_medium_en": {
         "metadata": {
@@ -54,22 +39,7 @@ backbone_presets = {
             "path": "gpt2",
             "model_card": "https://github.com/openai/gpt-2/blob/master/model_card.md",
         },
-        "config": {
-            "vocabulary_size": 50257,
-            "num_layers": 24,
-            "num_heads": 16,
-            "hidden_dim": 1024,
-            "intermediate_dim": 4096,
-            "dropout": 0.1,
-            "max_sequence_length": 1024,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/gpt2_medium_en/v1/model.h5",
-        "weights_hash": "580ff9b79c04fc90e6d6f47e975c5afe",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/gpt2_medium_en/v1/vocab.json",
-        "vocabulary_hash": "dffec25a898b1f5e569bec4dffd7e5c0",
-        "merges_url": "https://storage.googleapis.com/keras-nlp/models/gpt2_medium_en/v1/merges.txt",
-        "merges_hash": "75a37753dd7a28a2c5df80c28bf06e4e",
+        "kaggle_handle": "kaggle://keras/gpt2/gpt2_medium_en/2",
     },
     "gpt2_large_en": {
         "metadata": {
@@ -82,22 +52,7 @@ backbone_presets = {
             "path": "gpt2",
             "model_card": "https://github.com/openai/gpt-2/blob/master/model_card.md",
         },
-        "config": {
-            "vocabulary_size": 50257,
-            "num_layers": 36,
-            "num_heads": 20,
-            "hidden_dim": 1280,
-            "intermediate_dim": 5120,
-            "dropout": 0.1,
-            "max_sequence_length": 1024,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/gpt2_large_en/v1/model.h5",
-        "weights_hash": "67957cb3dfc9e965960dabe068811e1a",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/gpt2_large_en/v1/vocab.json",
-        "vocabulary_hash": "dffec25a898b1f5e569bec4dffd7e5c0",
-        "merges_url": "https://storage.googleapis.com/keras-nlp/models/gpt2_large_en/v1/merges.txt",
-        "merges_hash": "75a37753dd7a28a2c5df80c28bf06e4e",
+        "kaggle_handle": "kaggle://keras/gpt2/gpt2_large_en/2",
     },
     "gpt2_extra_large_en": {
         "metadata": {
@@ -110,22 +65,7 @@ backbone_presets = {
             "path": "gpt2",
             "model_card": "https://github.com/openai/gpt-2/blob/master/model_card.md",
         },
-        "config": {
-            "vocabulary_size": 50257,
-            "num_layers": 48,
-            "num_heads": 25,
-            "hidden_dim": 1600,
-            "intermediate_dim": 6400,
-            "dropout": 0.1,
-            "max_sequence_length": 1024,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/gpt2_extra_large_en/v1/model.h5",
-        "weights_hash": "d093c1ee0d9705d845c0190909aa2917",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/gpt2_extra_large_en/v1/vocab.json",
-        "vocabulary_hash": "dffec25a898b1f5e569bec4dffd7e5c0",
-        "merges_url": "https://storage.googleapis.com/keras-nlp/models/gpt2_extra_large_en/v1/merges.txt",
-        "merges_hash": "75a37753dd7a28a2c5df80c28bf06e4e",
+        "kaggle_handle": "kaggle://keras/gpt2/gpt2_extra_large_en/2",
     },
     "gpt2_base_en_cnn_dailymail": {
         "metadata": {
@@ -137,21 +77,6 @@ backbone_presets = {
             "official_name": "GPT-2",
             "path": "gpt2",
         },
-        "config": {
-            "vocabulary_size": 50257,
-            "num_layers": 12,
-            "num_heads": 12,
-            "hidden_dim": 768,
-            "intermediate_dim": 3072,
-            "dropout": 0.1,
-            "max_sequence_length": 1024,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/gpt2_base_en_news/v1/model.h5",
-        "weights_hash": "09d86ca6e1b4213886b720a1392f2a70",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/gpt2_base_en_news/v1/vocab.json",
-        "vocabulary_hash": "dffec25a898b1f5e569bec4dffd7e5c0",
-        "merges_url": "https://storage.googleapis.com/keras-nlp/models/gpt2_base_en_news/v1/merges.txt",
-        "merges_hash": "75a37753dd7a28a2c5df80c28bf06e4e",
+        "kaggle_handle": "kaggle://keras/gpt2/gpt2_base_en_cnn_dailymail/2",
     },
 }
diff --git a/keras_nlp/models/gpt2/gpt2_tokenizer.py b/keras_nlp/models/gpt2/gpt2_tokenizer.py
index 9401b385..15b35bed 100644
--- a/keras_nlp/models/gpt2/gpt2_tokenizer.py
+++ b/keras_nlp/models/gpt2/gpt2_tokenizer.py
@@ -70,32 +70,39 @@ class GPT2Tokenizer(BytePairTokenizer):
 
     def __init__(
         self,
-        vocabulary,
-        merges,
+        vocabulary=None,
+        merges=None,
         **kwargs,
     ):
-        # Special tokens.
-        end_token = "
"
+        # GPT2 uses the same start as end token, i.e., "
".
+        self.end_token = self.start_token = "
"
 
         super().__init__(
             vocabulary=vocabulary,
             merges=merges,
-            unsplittable_tokens=[end_token],
+            unsplittable_tokens=[self.end_token],
             **kwargs,
         )
 
-        # Check whether special tokens are present in the vocabulary.
-        if end_token not in self.get_vocabulary():
-            raise ValueError(
-                f"Cannot find token `'{end_token}'` in the provided "
-                f"`vocabulary`. Please provide `'{end_token}'` in your "
-                "`vocabulary` or use a pretrained `vocabulary` name."
-            )
-
-        self.end_token_id = self.token_to_id(end_token)
-        # GPT2 uses the same start as end token, i.e., "
".
-        self.start_token_id = self.end_token_id
-        self.pad_token_id = 0
+    def set_vocabulary_and_merges(self, vocabulary, merges):
+        super().set_vocabulary_and_merges(vocabulary, merges)
+
+        if vocabulary is not None:
+            # Check for necessary special tokens.
+            if self.end_token not in self.get_vocabulary():
+                raise ValueError(
+                    f"Cannot find token `'{self.end_token}'` in the provided "
+                    f"`vocabulary`. Please provide `'{self.end_token}'` in "
+                    "your `vocabulary` or use a pretrained `vocabulary` name."
+                )
+
+            self.end_token_id = self.token_to_id(self.end_token)
+            self.start_token_id = self.end_token_id
+            self.pad_token_id = 0
+        else:
+            self.end_token_id = None
+            self.start_token_id = None
+            self.pad_token_id = None
 
     @classproperty
     def presets(cls):
diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py b/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py
index 3ed16c3f..92ff9bbb 100644
--- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py
+++ b/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py
@@ -110,6 +110,9 @@ class GPTNeoXCausalLMPreprocessor(GPTNeoXPreprocessor):
         the sequence (as generation is expected to continue at the end of the
         inputted prompt).
         """
+        if not self.built:
+            self.build(None)
+
         x = convert_inputs_to_list_of_tensor_segments(x)[0]
         x = self.tokenizer(x)
         token_ids, padding_mask = self.packer(
@@ -130,6 +133,9 @@ class GPTNeoXCausalLMPreprocessor(GPTNeoXPreprocessor):
         padding and start/end tokens, and then converting the integer sequence
         back to a string.
         """
+        if not self.built:
+            self.build(None)
+
         token_ids, padding_mask = x["token_ids"], x["padding_mask"]
         if not isinstance(token_ids, tf.Tensor):
             token_ids = ops.convert_to_numpy(token_ids)
diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_preprocessor.py b/keras_nlp/models/gpt_neo_x/gpt_neo_x_preprocessor.py
index 0dad9a05..1db4fe4c 100644
--- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_preprocessor.py
+++ b/keras_nlp/models/gpt_neo_x/gpt_neo_x_preprocessor.py
@@ -79,24 +79,19 @@ class GPTNeoXPreprocessor(Preprocessor):
         self.sequence_length = sequence_length
         self.add_start_token = add_start_token
         self.add_end_token = add_end_token
+        self.packer = None
+
+    def build(self, input_shape):
+        # Defer packer creation to `build()` so that we can be sure tokenizer
+        # assets have loaded when restoring a saved model.
         self.packer = StartEndPacker(
-            start_value=tokenizer.start_token_id,
-            end_value=tokenizer.end_token_id,
-            pad_value=tokenizer.pad_token_id,
-            sequence_length=sequence_length,
+            start_value=self.tokenizer.start_token_id,
+            end_value=self.tokenizer.end_token_id,
+            pad_value=self.tokenizer.pad_token_id,
+            sequence_length=self.sequence_length,
             return_padding_mask=True,
         )
-
-    def get_config(self):
-        config = super().get_config()
-        config.update(
-            {
-                "sequence_length": self.sequence_length,
-                "add_start_token": self.add_start_token,
-                "add_end_token": self.add_end_token,
-            }
-        )
-        return config
+        self.built = True
 
     def call(
         self,
@@ -126,6 +121,17 @@ class GPTNeoXPreprocessor(Preprocessor):
         }
         return pack_x_y_sample_weight(x, y, sample_weight)
 
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "sequence_length": self.sequence_length,
+                "add_start_token": self.add_start_token,
+                "add_end_token": self.add_end_token,
+            }
+        )
+        return config
+
     @classproperty
     def tokenizer_cls(cls):
         return GPTNeoXTokenizer
diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_tokenizer.py b/keras_nlp/models/gpt_neo_x/gpt_neo_x_tokenizer.py
index 3935d85a..d109c584 100644
--- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_tokenizer.py
+++ b/keras_nlp/models/gpt_neo_x/gpt_neo_x_tokenizer.py
@@ -45,32 +45,39 @@ class GPTNeoXTokenizer(BytePairTokenizer):
 
     def __init__(
         self,
-        vocabulary,
-        merges,
+        vocabulary=None,
+        merges=None,
         **kwargs,
     ):
-        # Special tokens.
-        end_token = "
"
+        # GPTNeoX uses the same start as end token, i.e., "
".
+        self.end_token = self.start_token = "
"
 
         super().__init__(
             vocabulary=vocabulary,
             merges=merges,
-            unsplittable_tokens=[end_token],
+            unsplittable_tokens=[self.end_token],
             **kwargs,
         )
 
-        # Check whether special tokens are present in the vocabulary.
-        if end_token not in self.get_vocabulary():
-            raise ValueError(
-                f"Cannot find token `'{end_token}'` in the provided "
-                f"`vocabulary`. Please provide `'{end_token}'` in your "
-                "`vocabulary` or use a pretrained `vocabulary` name."
-            )
+    def set_vocabulary_and_merges(self, vocabulary, merges):
+        super().set_vocabulary_and_merges(vocabulary, merges)
 
-        self.end_token_id = self.token_to_id(end_token)
-        # GPTNeoX uses the same start as end token, i.e., "
".
-        self.start_token_id = self.end_token_id
-        self.pad_token_id = 0
+        if vocabulary is not None:
+            # Check for necessary special tokens.
+            if self.end_token not in self.get_vocabulary():
+                raise ValueError(
+                    f"Cannot find token `'{self.end_token}'` in the provided "
+                    f"`vocabulary`. Please provide `'{self.end_token}'` in "
+                    "your `vocabulary` or use a pretrained `vocabulary` name."
+                )
+
+            self.end_token_id = self.token_to_id(self.end_token)
+            self.start_token_id = self.end_token_id
+            self.pad_token_id = 0
+        else:
+            self.end_token_id = None
+            self.start_token_id = None
+            self.pad_token_id = None
 
     def get_config(self):
         config = super().get_config()
diff --git a/keras_nlp/models/opt/opt_causal_lm_preprocessor.py b/keras_nlp/models/opt/opt_causal_lm_preprocessor.py
index 9cc8c7f4..1895854e 100644
--- a/keras_nlp/models/opt/opt_causal_lm_preprocessor.py
+++ b/keras_nlp/models/opt/opt_causal_lm_preprocessor.py
@@ -143,6 +143,9 @@ class OPTCausalLMPreprocessor(OPTPreprocessor):
         the sequence (as generation is expected to continue at the end of the
         inputted prompt).
         """
+        if not self.built:
+            self.build(None)
+
         x = convert_inputs_to_list_of_tensor_segments(x)[0]
         x = self.tokenizer(x)
         token_ids, padding_mask = self.packer(
@@ -163,6 +166,9 @@ class OPTCausalLMPreprocessor(OPTPreprocessor):
         padding and start/end tokens, and then converting the integer sequence
         back to a string.
         """
+        if not self.built:
+            self.build(None)
+
         token_ids, padding_mask = x["token_ids"], x["padding_mask"]
         token_ids = ops.convert_to_numpy(token_ids)
         padding_mask = ops.convert_to_numpy(padding_mask)
diff --git a/keras_nlp/models/opt/opt_preprocessor.py b/keras_nlp/models/opt/opt_preprocessor.py
index 6a6b5537..cdca9048 100644
--- a/keras_nlp/models/opt/opt_preprocessor.py
+++ b/keras_nlp/models/opt/opt_preprocessor.py
@@ -123,13 +123,19 @@ class OPTPreprocessor(Preprocessor):
         self.sequence_length = sequence_length
         self.add_start_token = add_start_token
         self.add_end_token = add_end_token
+        self.packer = None
+
+    def build(self, input_shape):
+        # Defer packer creation to `build()` so that we can be sure tokenizer
+        # assets have loaded when restoring a saved model.
         self.packer = StartEndPacker(
-            start_value=tokenizer.start_token_id,
-            end_value=tokenizer.end_token_id,
-            pad_value=tokenizer.pad_token_id,
-            sequence_length=sequence_length,
+            start_value=self.tokenizer.start_token_id,
+            end_value=self.tokenizer.end_token_id,
+            pad_value=self.tokenizer.pad_token_id,
+            sequence_length=self.sequence_length,
             return_padding_mask=True,
         )
+        self.built = True
 
     def get_config(self):
         config = super().get_config()
diff --git a/keras_nlp/models/opt/opt_presets.py b/keras_nlp/models/opt/opt_presets.py
index 7af26411..abafa147 100644
--- a/keras_nlp/models/opt/opt_presets.py
+++ b/keras_nlp/models/opt/opt_presets.py
@@ -26,22 +26,7 @@ backbone_presets = {
             "path": "opt",
             "model_card": "https://github.com/facebookresearch/metaseq/blob/main/projects/OPT/model_card.md",
         },
-        "config": {
-            "vocabulary_size": 50272,
-            "num_layers": 12,
-            "num_heads": 12,
-            "hidden_dim": 768,
-            "intermediate_dim": 3072,
-            "dropout": 0.1,
-            "max_sequence_length": 2048,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/opt_125m_en/v1/model.h5",
-        "weights_hash": "63e444998982e48da4a1a3970f4c6203",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/opt_125m_en/v1/vocab.json",
-        "vocabulary_hash": "cf410ee085c5c69c957bb1f6d8456596",
-        "merges_url": "https://storage.googleapis.com/keras-nlp/models/opt_125m_en/v1/merges.txt",
-        "merges_hash": "75a37753dd7a28a2c5df80c28bf06e4e",
+        "kaggle_handle": "kaggle://keras/opt/opt_125m_en/2",
     },
     # We skip the 350m checkpoint because it does not match the structure of
     # other checkpoints.
@@ -56,22 +41,7 @@ backbone_presets = {
             "path": "opt",
             "model_card": "https://github.com/facebookresearch/metaseq/blob/main/projects/OPT/model_card.md",
         },
-        "config": {
-            "vocabulary_size": 50272,
-            "num_layers": 24,
-            "num_heads": 32,
-            "hidden_dim": 2048,
-            "intermediate_dim": 8192,
-            "dropout": 0.1,
-            "max_sequence_length": 2048,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/opt_1.3b_en/v1/model.h5",
-        "weights_hash": "0365ac8483e99a912c9770521909ecce",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/opt_1.3b_en/v1/vocab.json",
-        "vocabulary_hash": "cf410ee085c5c69c957bb1f6d8456596",
-        "merges_url": "https://storage.googleapis.com/keras-nlp/models/opt_1.3b_en/v1/merges.txt",
-        "merges_hash": "75a37753dd7a28a2c5df80c28bf06e4e",
+        "kaggle_handle": "kaggle://keras/opt/opt_1.3b_en/2",
     },
     "opt_2.7b_en": {
         "metadata": {
@@ -84,22 +54,7 @@ backbone_presets = {
             "path": "opt",
             "model_card": "https://github.com/facebookresearch/metaseq/blob/main/projects/OPT/model_card.md",
         },
-        "config": {
-            "vocabulary_size": 50272,
-            "num_layers": 32,
-            "num_heads": 32,
-            "hidden_dim": 2560,
-            "intermediate_dim": 10240,
-            "dropout": 0.1,
-            "max_sequence_length": 2048,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/opt_2.7b_en/v1/model.h5",
-        "weights_hash": "af56da9206a95b9287356955c5bc14e7",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/opt_2.7b_en/v1/vocab.json",
-        "vocabulary_hash": "cf410ee085c5c69c957bb1f6d8456596",
-        "merges_url": "https://storage.googleapis.com/keras-nlp/models/opt_2.7b_en/v1/merges.txt",
-        "merges_hash": "75a37753dd7a28a2c5df80c28bf06e4e",
+        "kaggle_handle": "kaggle://keras/opt/opt_2.7b_en/2",
     },
     "opt_6.7b_en": {
         "metadata": {
@@ -112,21 +67,6 @@ backbone_presets = {
             "path": "opt",
             "model_card": "https://github.com/facebookresearch/metaseq/blob/main/projects/OPT/model_card.md",
         },
-        "config": {
-            "vocabulary_size": 50272,
-            "num_layers": 32,
-            "num_heads": 32,
-            "hidden_dim": 4096,
-            "intermediate_dim": 16384,
-            "dropout": 0.1,
-            "max_sequence_length": 2048,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/opt_6.7b_en/v1/model.h5",
-        "weights_hash": "543120fbe601b70e6ec04cc909781e21",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/opt_6.7b_en/v1/vocab.json",
-        "vocabulary_hash": "cf410ee085c5c69c957bb1f6d8456596",
-        "merges_url": "https://storage.googleapis.com/keras-nlp/models/opt_6.7b_en/v1/merges.txt",
-        "merges_hash": "75a37753dd7a28a2c5df80c28bf06e4e",
+        "kaggle_handle": "kaggle://keras/opt/opt_6.7b_en/2",
     },
 }
diff --git a/keras_nlp/models/opt/opt_tokenizer.py b/keras_nlp/models/opt/opt_tokenizer.py
index b15aa948..4fb62ee7 100644
--- a/keras_nlp/models/opt/opt_tokenizer.py
+++ b/keras_nlp/models/opt/opt_tokenizer.py
@@ -70,35 +70,45 @@ class OPTTokenizer(BytePairTokenizer):
 
     def __init__(
         self,
-        vocabulary,
-        merges,
+        vocabulary=None,
+        merges=None,
         **kwargs,
     ):
-        # Special tokens. We use `"
"` as both a start and end token, as OPT
-        # was only pre-trained with `"
"` marking document boundaries.
-        start_token = "
"
-        pad_token = "<pad>"
-        end_token = "
"
+        self.start_token = "
"
+        self.pad_token = "<pad>"
+        self.end_token = "
"
 
         super().__init__(
             vocabulary=vocabulary,
             merges=merges,
-            unsplittable_tokens=[start_token, pad_token, end_token],
+            unsplittable_tokens=[
+                self.start_token,
+                self.pad_token,
+                self.end_token,
+            ],
             **kwargs,
         )
 
-        # Check whether special tokens are present in the vocabulary.
-        for token in [start_token, pad_token, end_token]:
-            if token not in self.get_vocabulary():
-                raise ValueError(
-                    f"Cannot find token `'{token}'` in the provided "
-                    f"`vocabulary`. Please provide `'{token}'` in your "
-                    "`vocabulary` or use a pretrained `vocabulary` name."
-                )
-
-        self.start_token_id = self.token_to_id(start_token)
-        self.pad_token_id = self.token_to_id(pad_token)
-        self.end_token_id = self.token_to_id(end_token)
+    def set_vocabulary_and_merges(self, vocabulary, merges):
+        super().set_vocabulary_and_merges(vocabulary, merges)
+
+        if vocabulary is not None:
+            # Check for necessary special tokens.
+            for token in [self.start_token, self.pad_token, self.end_token]:
+                if token not in self.vocabulary:
+                    raise ValueError(
+                        f"Cannot find token `'{token}'` in the provided "
+                        f"`vocabulary`. Please provide `'{token}'` in your "
+                        "`vocabulary` or use a pretrained `vocabulary` name."
+                    )
+
+            self.start_token_id = self.token_to_id(self.start_token)
+            self.pad_token_id = self.token_to_id(self.pad_token)
+            self.end_token_id = self.token_to_id(self.end_token)
+        else:
+            self.start_token_id = None
+            self.pad_token_id = None
+            self.end_token_id = None
 
     @classproperty
     def presets(cls):
diff --git a/keras_nlp/models/preprocessor.py b/keras_nlp/models/preprocessor.py
index b5ea48a0..16a65e57 100644
--- a/keras_nlp/models/preprocessor.py
+++ b/keras_nlp/models/preprocessor.py
@@ -16,6 +16,8 @@ from keras_nlp.backend import keras
 from keras_nlp.layers.preprocessing.preprocessing_layer import (
     PreprocessingLayer,
 )
+from keras_nlp.utils.preset_utils import check_preset_class
+from keras_nlp.utils.preset_utils import load_from_preset
 from keras_nlp.utils.python_utils import classproperty
 from keras_nlp.utils.python_utils import format_docstring
 
@@ -81,44 +83,18 @@ class Preprocessor(PreprocessingLayer):
         )
         ```
         """
-        if not cls.presets:
-            raise NotImplementedError(
-                "No presets have been created for this class."
-            )
-        if preset not in cls.presets:
-            raise ValueError(
-                "`preset` must be one of "
-                f"""{", ".join(cls.presets)}. Received: {preset}."""
-            )
-
-        tokenizer = cls.tokenizer_cls.from_preset(preset)
-
-        metadata = cls.presets[preset]
-        # For task model presets, the backbone config is nested.
-        if "backbone" in metadata["config"]:
-            backbone_config = metadata["config"]["backbone"]["config"]
-        else:
-            backbone_config = metadata["config"]
-
-        # Use model's `max_sequence_length` if `sequence_length` unspecified;
-        # otherwise check that `sequence_length` not too long.
-        sequence_length = kwargs.pop("sequence_length", None)
-        max_sequence_length = backbone_config["max_sequence_length"]
-        if sequence_length is not None:
-            if sequence_length > max_sequence_length:
-                raise ValueError(
-                    f"`sequence_length` cannot be longer than `{preset}` "
-                    f"preset's `max_sequence_length` of {max_sequence_length}. "
-                    f"Received: {sequence_length}."
-                )
-        else:
-            sequence_length = max_sequence_length
-
-        return cls(
-            tokenizer=tokenizer,
-            sequence_length=sequence_length,
-            **kwargs,
+        # We support short IDs for official presets, e.g. `"bert_base_en"`.
+        # Map these to a Kaggle Models handle.
+        if preset in cls.presets:
+            preset = cls.presets[preset]["kaggle_handle"]
+
+        config_file = "tokenizer.json"
+        check_preset_class(preset, cls.tokenizer_cls, config_file=config_file)
+        tokenizer = load_from_preset(
+            preset,
+            config_file=config_file,
         )
+        return cls(tokenizer=tokenizer, **kwargs)
 
     def __init_subclass__(cls, **kwargs):
         # Use __init_subclass__ to setup a correct docstring for from_preset.
diff --git a/keras_nlp/models/roberta/roberta_masked_lm_preprocessor.py b/keras_nlp/models/roberta/roberta_masked_lm_preprocessor.py
index 0d5a24e1..c69c300d 100644
--- a/keras_nlp/models/roberta/roberta_masked_lm_preprocessor.py
+++ b/keras_nlp/models/roberta/roberta_masked_lm_preprocessor.py
@@ -137,32 +137,30 @@ class RobertaMaskedLMPreprocessor(RobertaPreprocessor):
             truncate=truncate,
             **kwargs,
         )
-
+        self.mask_selection_rate = mask_selection_rate
+        self.mask_selection_length = mask_selection_length
+        self.mask_token_rate = mask_token_rate
+        self.random_token_rate = random_token_rate
+        self.masker = None
+
+    def build(self, input_shape):
+        super().build(input_shape)
+        # Defer packer creation to `build()` so that we can be sure tokenizer
+        # assets have loaded when restoring a saved model.
         self.masker = MaskedLMMaskGenerator(
-            mask_selection_rate=mask_selection_rate,
-            mask_selection_length=mask_selection_length,
-            mask_token_rate=mask_token_rate,
-            random_token_rate=random_token_rate,
-            vocabulary_size=tokenizer.vocabulary_size(),
-            mask_token_id=tokenizer.mask_token_id,
+            mask_selection_rate=self.mask_selection_rate,
+            mask_selection_length=self.mask_selection_length,
+            mask_token_rate=self.mask_token_rate,
+            random_token_rate=self.random_token_rate,
+            vocabulary_size=self.tokenizer.vocabulary_size(),
+            mask_token_id=self.tokenizer.mask_token_id,
             unselectable_token_ids=[
-                tokenizer.start_token_id,
-                tokenizer.end_token_id,
-                tokenizer.pad_token_id,
+                self.tokenizer.start_token_id,
+                self.tokenizer.end_token_id,
+                self.tokenizer.pad_token_id,
             ],
         )
-
-    def get_config(self):
-        config = super().get_config()
-        config.update(
-            {
-                "mask_selection_rate": self.masker.mask_selection_rate,
-                "mask_selection_length": self.masker.mask_selection_length,
-                "mask_token_rate": self.masker.mask_token_rate,
-                "random_token_rate": self.masker.random_token_rate,
-            }
-        )
-        return config
+        self.built = True
 
     def call(self, x, y=None, sample_weight=None):
         if y is not None or sample_weight is not None:
@@ -184,3 +182,15 @@ class RobertaMaskedLMPreprocessor(RobertaPreprocessor):
         y = masker_outputs["mask_ids"]
         sample_weight = masker_outputs["mask_weights"]
         return pack_x_y_sample_weight(x, y, sample_weight)
+
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "mask_selection_rate": self.mask_selection_rate,
+                "mask_selection_length": self.mask_selection_length,
+                "mask_token_rate": self.mask_token_rate,
+                "random_token_rate": self.random_token_rate,
+            }
+        )
+        return config
diff --git a/keras_nlp/models/roberta/roberta_preprocessor.py b/keras_nlp/models/roberta/roberta_preprocessor.py
index 7e641a93..556561d1 100644
--- a/keras_nlp/models/roberta/roberta_preprocessor.py
+++ b/keras_nlp/models/roberta/roberta_preprocessor.py
@@ -143,24 +143,22 @@ class RobertaPreprocessor(Preprocessor):
         super().__init__(**kwargs)
 
         self.tokenizer = tokenizer
+        self.truncate = truncate
+        self.sequence_length = sequence_length
+        self.packer = None
+
+    def build(self, input_shape):
+        # Defer packer creation to `build()` so that we can be sure tokenizer
+        # assets have loaded when restoring a saved model.
         self.packer = MultiSegmentPacker(
             start_value=self.tokenizer.start_token_id,
             end_value=self.tokenizer.end_token_id,
             sep_value=[self.tokenizer.end_token_id] * 2,
             pad_value=self.tokenizer.pad_token_id,
-            truncate=truncate,
-            sequence_length=sequence_length,
+            truncate=self.truncate,
+            sequence_length=self.sequence_length,
         )
-
-    def get_config(self):
-        config = super().get_config()
-        config.update(
-            {
-                "sequence_length": self.packer.sequence_length,
-                "truncate": self.packer.truncate,
-            }
-        )
-        return config
+        self.built = True
 
     def call(self, x, y=None, sample_weight=None):
         x = convert_inputs_to_list_of_tensor_segments(x)
@@ -172,6 +170,16 @@ class RobertaPreprocessor(Preprocessor):
         }
         return pack_x_y_sample_weight(x, y, sample_weight)
 
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "sequence_length": self.sequence_length,
+                "truncate": self.truncate,
+            }
+        )
+        return config
+
     @classproperty
     def tokenizer_cls(cls):
         return RobertaTokenizer
diff --git a/keras_nlp/models/roberta/roberta_presets.py b/keras_nlp/models/roberta/roberta_presets.py
index f098bed5..9ed6bc41 100644
--- a/keras_nlp/models/roberta/roberta_presets.py
+++ b/keras_nlp/models/roberta/roberta_presets.py
@@ -25,22 +25,7 @@ backbone_presets = {
             "path": "roberta",
             "model_card": "https://github.com/facebookresearch/fairseq/blob/main/examples/roberta/README.md",
         },
-        "config": {
-            "vocabulary_size": 50265,
-            "num_layers": 12,
-            "num_heads": 12,
-            "hidden_dim": 768,
-            "intermediate_dim": 3072,
-            "dropout": 0.1,
-            "max_sequence_length": 512,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/roberta_base_en/v1/model.h5",
-        "weights_hash": "958eede1c7edaa9308e027be18fde7a8",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/roberta_base_en/v1/vocab.json",
-        "vocabulary_hash": "be4d3c6f3f5495426b2c03b334334354",
-        "merges_url": "https://storage.googleapis.com/keras-nlp/models/roberta_base_en/v1/merges.txt",
-        "merges_hash": "75a37753dd7a28a2c5df80c28bf06e4e",
+        "kaggle_handle": "kaggle://keras/roberta/roberta_base_en/2",
     },
     "roberta_large_en": {
         "metadata": {
@@ -53,21 +38,6 @@ backbone_presets = {
             "path": "roberta",
             "model_card": "https://github.com/facebookresearch/fairseq/blob/main/examples/roberta/README.md",
         },
-        "config": {
-            "vocabulary_size": 50265,
-            "num_layers": 24,
-            "num_heads": 16,
-            "hidden_dim": 1024,
-            "intermediate_dim": 4096,
-            "dropout": 0.1,
-            "max_sequence_length": 512,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/roberta_large_en/v1/model.h5",
-        "weights_hash": "1978b864c317a697fe62a894d3664f14",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/roberta_large_en/v1/vocab.json",
-        "vocabulary_hash": "be4d3c6f3f5495426b2c03b334334354",
-        "merges_url": "https://storage.googleapis.com/keras-nlp/models/roberta_large_en/v1/merges.txt",
-        "merges_hash": "75a37753dd7a28a2c5df80c28bf06e4e",
+        "kaggle_handle": "kaggle://keras/roberta/roberta_large_en/2",
     },
 }
diff --git a/keras_nlp/models/roberta/roberta_tokenizer.py b/keras_nlp/models/roberta/roberta_tokenizer.py
index 838f32a8..0cfabff7 100644
--- a/keras_nlp/models/roberta/roberta_tokenizer.py
+++ b/keras_nlp/models/roberta/roberta_tokenizer.py
@@ -77,36 +77,54 @@ class RobertaTokenizer(BytePairTokenizer):
 
     def __init__(
         self,
-        vocabulary,
-        merges,
+        vocabulary=None,
+        merges=None,
         **kwargs,
     ):
-        # Special tokens.
-        start_token = "
"
-        pad_token = "<pad>"
-        end_token = "
"
-        mask_token = "<mask>"
+        self.start_token = "
"
+        self.pad_token = "<pad>"
+        self.end_token = "
"
+        self.mask_token = "<mask>"
 
         super().__init__(
             vocabulary=vocabulary,
             merges=merges,
-            unsplittable_tokens=[start_token, pad_token, end_token, mask_token],
+            unsplittable_tokens=[
+                self.start_token,
+                self.pad_token,
+                self.end_token,
+                self.mask_token,
+            ],
             **kwargs,
         )
 
-        # Check whether special tokens are present in the vocabulary.
-        for token in [start_token, pad_token, end_token, mask_token]:
-            if token not in self.get_vocabulary():
-                raise ValueError(
-                    f"Cannot find token `'{token}'` in the provided "
-                    f"`vocabulary`. Please provide `'{token}'` in your "
-                    "`vocabulary` or use a pretrained `vocabulary` name."
-                )
-
-        self.start_token_id = self.token_to_id(start_token)
-        self.pad_token_id = self.token_to_id(pad_token)
-        self.end_token_id = self.token_to_id(end_token)
-        self.mask_token_id = self.token_to_id(mask_token)
+    def set_vocabulary_and_merges(self, vocabulary, merges):
+        super().set_vocabulary_and_merges(vocabulary, merges)
+
+        if vocabulary is not None:
+            # Check for necessary special tokens.
+            for token in [
+                self.start_token,
+                self.pad_token,
+                self.end_token,
+                self.mask_token,
+            ]:
+                if token not in self.vocabulary:
+                    raise ValueError(
+                        f"Cannot find token `'{token}'` in the provided "
+                        f"`vocabulary`. Please provide `'{token}'` in your "
+                        "`vocabulary` or use a pretrained `vocabulary` name."
+                    )
+
+            self.start_token_id = self.token_to_id(self.start_token)
+            self.pad_token_id = self.token_to_id(self.pad_token)
+            self.end_token_id = self.token_to_id(self.end_token)
+            self.mask_token_id = self.token_to_id(self.mask_token)
+        else:
+            self.start_token_id = None
+            self.pad_token_id = None
+            self.end_token_id = None
+            self.mask_token_id = None
 
     @classproperty
     def presets(cls):
diff --git a/keras_nlp/models/t5/t5_backbone.py b/keras_nlp/models/t5/t5_backbone.py
index 314a5d68..6e76094d 100644
--- a/keras_nlp/models/t5/t5_backbone.py
+++ b/keras_nlp/models/t5/t5_backbone.py
@@ -222,6 +222,7 @@ class T5Backbone(Backbone):
         self.activation = keras.activations.get(activation)
         self.key_value_dim = key_value_dim
         self.dropout = dropout
+        self.use_gated_activation = use_gated_activation
         self.layer_norm_epsilon = layer_norm_epsilon
         self.tie_embedding_weights = tie_embedding_weights
         self.token_embedding = token_embedding_layer
@@ -238,6 +239,7 @@ class T5Backbone(Backbone):
                 "activation": keras.activations.serialize(self.activation),
                 "key_value_dim": self.key_value_dim,
                 "dropout": self.dropout,
+                "use_gated_activation": self.use_gated_activation,
                 "layer_norm_epsilon": self.layer_norm_epsilon,
                 "tie_embedding_weights": self.tie_embedding_weights,
             }
diff --git a/keras_nlp/models/t5/t5_presets.py b/keras_nlp/models/t5/t5_presets.py
index dd2bea7a..c25bef1c 100644
--- a/keras_nlp/models/t5/t5_presets.py
+++ b/keras_nlp/models/t5/t5_presets.py
@@ -25,24 +25,7 @@ backbone_presets = {
             "path": "t5",
             "model_card": "https://github.com/google-research/text-to-text-transfer-transformer/blob/main/README.md",
         },
-        "config": {
-            "vocabulary_size": 32128,
-            "num_layers": 6,
-            "num_heads": 8,
-            "hidden_dim": 512,
-            "intermediate_dim": 2048,
-            "key_value_dim": 64,
-            "dropout": 0.1,
-            "activation": "relu",
-            "use_gated_activation": False,
-            "layer_norm_epsilon": 1e-06,
-            "tie_embedding_weights": True,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/t5_small_multi/v1/model.weights.h5",
-        "weights_hash": "2e10b5f72405d464ee55026b07e60741",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/t5_small_multi/v1/vocab.spm",
-        "vocabulary_hash": "9d15ef55d09d5a425ceb63fa31f7cae3",
+        "kaggle_handle": "kaggle://keras/t5/t5_small_multi/2",
     },
     "t5_base_multi": {
         "metadata": {
@@ -55,23 +38,7 @@ backbone_presets = {
             "path": "t5",
             "model_card": "https://github.com/google-research/text-to-text-transfer-transformer/blob/main/README.md",
         },
-        "config": {
-            "vocabulary_size": 32128,
-            "num_layers": 12,
-            "num_heads": 12,
-            "hidden_dim": 768,
-            "intermediate_dim": 3072,
-            "dropout": 0.1,
-            "activation": "relu",
-            "use_gated_activation": False,
-            "layer_norm_epsilon": 1e-06,
-            "tie_embedding_weights": True,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/t5_base_multi/v1/model.weights.h5",
-        "weights_hash": "bed6ef276cfe83d1323467051211978d",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/t5_base_multi/v1/vocab.spm",
-        "vocabulary_hash": "9d15ef55d09d5a425ceb63fa31f7cae3",
+        "kaggle_handle": "kaggle://keras/t5/t5_base_multi/2",
     },
     "t5_large_multi": {
         "metadata": {
@@ -84,23 +51,7 @@ backbone_presets = {
             "path": "t5",
             "model_card": "https://github.com/google-research/text-to-text-transfer-transformer/blob/main/README.md",
         },
-        "config": {
-            "vocabulary_size": 32128,
-            "num_layers": 24,
-            "num_heads": 16,
-            "hidden_dim": 1024,
-            "intermediate_dim": 4096,
-            "dropout": 0.1,
-            "activation": "relu",
-            "use_gated_activation": False,
-            "layer_norm_epsilon": 1e-06,
-            "tie_embedding_weights": True,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/t5_large_multi/v1/model.weights.h5",
-        "weights_hash": "7854a05c2e6812899bf6f0f104792cda",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/t5_large_multi/v1/vocab.spm",
-        "vocabulary_hash": "9d15ef55d09d5a425ceb63fa31f7cae3",
+        "kaggle_handle": "kaggle://keras/t5/t5_large_multi/2",
     },
     "flan_small_multi": {
         "metadata": {
@@ -113,24 +64,7 @@ backbone_presets = {
             "path": "t5",
             "model_card": "https://github.com/google-research/text-to-text-transfer-transformer/blob/main/README.md",
         },
-        "config": {
-            "vocabulary_size": 32128,
-            "num_layers": 8,
-            "num_heads": 6,
-            "hidden_dim": 512,
-            "intermediate_dim": 1024,
-            "key_value_dim": 64,
-            "dropout": 0.1,
-            "activation": "keras_nlp>gelu_approximate",
-            "use_gated_activation": True,
-            "layer_norm_epsilon": 1e-06,
-            "tie_embedding_weights": False,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/flan_small_multi/v1/model.weights.h5",
-        "weights_hash": "aa0fbaddb1759ef313bbc4f9e4f1e197",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/flan_small_multi/v1/vocab.spm",
-        "vocabulary_hash": "9d15ef55d09d5a425ceb63fa31f7cae3",
+        "kaggle_handle": "kaggle://keras/t5/flan_small_multi/2",
     },
     "flan_base_multi": {
         "metadata": {
@@ -143,23 +77,7 @@ backbone_presets = {
             "path": "t5",
             "model_card": "https://github.com/google-research/text-to-text-transfer-transformer/blob/main/README.md",
         },
-        "config": {
-            "vocabulary_size": 32128,
-            "num_layers": 12,
-            "num_heads": 12,
-            "hidden_dim": 768,
-            "intermediate_dim": 2048,
-            "dropout": 0.1,
-            "activation": "keras_nlp>gelu_approximate",
-            "use_gated_activation": True,
-            "layer_norm_epsilon": 1e-06,
-            "tie_embedding_weights": False,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/flan_base_multi/v1/model.weights.h5",
-        "weights_hash": "84a10bec83fd093931bb2a6264115d31",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/flan_base_multi/v1/vocab.spm",
-        "vocabulary_hash": "9d15ef55d09d5a425ceb63fa31f7cae3",
+        "kaggle_handle": "kaggle://keras/t5/flan_base_multi/2",
     },
     "flan_large_multi": {
         "metadata": {
@@ -172,22 +90,6 @@ backbone_presets = {
             "path": "t5",
             "model_card": "https://github.com/google-research/text-to-text-transfer-transformer/blob/main/README.md",
         },
-        "config": {
-            "vocabulary_size": 32128,
-            "num_layers": 24,
-            "num_heads": 16,
-            "hidden_dim": 1024,
-            "intermediate_dim": 2816,
-            "dropout": 0.1,
-            "activation": "keras_nlp>gelu_approximate",
-            "use_gated_activation": True,
-            "layer_norm_epsilon": 1e-06,
-            "tie_embedding_weights": False,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/flan_large_multi/v1/model.weights.h5",
-        "weights_hash": "513f530ce790efa7e261c0ef965f3697",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/flan_large_multi/v1/vocab.spm",
-        "vocabulary_hash": "9d15ef55d09d5a425ceb63fa31f7cae3",
+        "kaggle_handle": "kaggle://keras/t5/flan_large_multi/2",
     },
 }
diff --git a/keras_nlp/models/t5/t5_tokenizer.py b/keras_nlp/models/t5/t5_tokenizer.py
index ae9facb3..b5dee49b 100644
--- a/keras_nlp/models/t5/t5_tokenizer.py
+++ b/keras_nlp/models/t5/t5_tokenizer.py
@@ -11,9 +11,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import copy
 
 from keras_nlp.api_export import keras_nlp_export
+from keras_nlp.models.t5.t5_presets import backbone_presets
 from keras_nlp.tokenizers.sentence_piece_tokenizer import SentencePieceTokenizer
+from keras_nlp.utils.python_utils import classproperty
 
 
 @keras_nlp_export("keras_nlp.models.T5Tokenizer")
@@ -73,20 +76,30 @@ class T5Tokenizer(SentencePieceTokenizer):
     """
 
     def __init__(self, proto, **kwargs):
+        self.end_token = "
"
+        self.pad_token = "<pad>"
+
         super().__init__(proto=proto, **kwargs)
 
-        # Check for necessary special tokens.
-        end_token = "
"
-        pad_token = "<pad>"
-        for token in [pad_token]:
-            if token not in self.get_vocabulary():
-                raise ValueError(
-                    f"Cannot find token `'{token}'` in the provided "
-                    f"`vocabulary`. Please provide `'{token}'` in your "
-                    "`vocabulary` or use a pretrained `vocabulary` name."
-                )
-
-        self.pad_token_id = self.token_to_id(pad_token)
-        self.end_token_id = self.token_to_id(end_token)
-        # T5 uses the same start token as end token, i.e., "<\s>".
-        self.start_token_id = self.end_token_id
+    def set_proto(self, proto):
+        super().set_proto(proto)
+        if proto is not None:
+            for token in [self.end_token, self.pad_token]:
+                if token not in self.get_vocabulary():
+                    raise ValueError(
+                        f"Cannot find token `'{token}'` in the provided "
+                        f"`vocabulary`. Please provide `'{token}'` in your "
+                        "`vocabulary` or use a pretrained `vocabulary` name."
+                    )
+            self.end_token_id = self.token_to_id(self.end_token)
+            self.pad_token_id = self.token_to_id(self.pad_token)
+            # T5 uses the same start token as end token, i.e., "<\s>".
+            self.start_token_id = self.end_token_id
+        else:
+            self.end_token_id = None
+            self.pad_token_id = None
+            self.start_token_id = None
+
+    @classproperty
+    def presets(cls):
+        return copy.deepcopy(backbone_presets)
diff --git a/keras_nlp/models/task.py b/keras_nlp/models/task.py
index f159fbfe..ee28e3a9 100644
--- a/keras_nlp/models/task.py
+++ b/keras_nlp/models/task.py
@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
-
 from rich import console as rich_console
 from rich import markup
 from rich import table as rich_table
@@ -22,6 +20,8 @@ from keras_nlp.backend import config
 from keras_nlp.backend import keras
 from keras_nlp.utils.keras_utils import print_msg
 from keras_nlp.utils.pipeline_model import PipelineModel
+from keras_nlp.utils.preset_utils import check_preset_class
+from keras_nlp.utils.preset_utils import load_from_preset
 from keras_nlp.utils.python_utils import classproperty
 from keras_nlp.utils.python_utils import format_docstring
 
@@ -34,6 +34,19 @@ class Task(PipelineModel):
         self._backbone = None
         self._preprocessor = None
         super().__init__(*args, **kwargs)
+        self._functional_layer_ids = set(
+            id(layer) for layer in self._flatten_layers()
+        )
+
+    def __dir__(self):
+        # Temporary fixes for weight saving. This mimics the following PR for
+        # older version of Keras: https://github.com/keras-team/keras/pull/18982
+        def filter_fn(attr):
+            if attr == "_layer_checkpoint_dependencies":
+                return False
+            return id(getattr(self, attr)) not in self._functional_layer_ids
+
+        return filter(filter_fn, super().__dir__())
 
     def _check_for_loss_mismatch(self, loss):
         """Check for a softmax/from_logits mismatch after compile.
@@ -174,43 +187,43 @@ class Task(PipelineModel):
         )
         ```
         """
-        if not cls.presets:
-            raise NotImplementedError(
-                "No presets have been created for this class."
-            )
-
-        if preset not in cls.presets:
+        if "backbone" in kwargs:
             raise ValueError(
-                "`preset` must be one of "
-                f"""{", ".join(cls.presets)}. Received: {preset}."""
+                "You cannot pass a `backbone` argument to the `from_preset` "
+                f"method. Instead, call the {cls.__name__} default "
+                "constructor with a `backbone` argument. "
+                f"Received: backbone={kwargs['backbone']}."
             )
-
-        if "preprocessor" not in kwargs:
-            kwargs["preprocessor"] = cls.preprocessor_cls.from_preset(preset)
-
-        # Check if preset is backbone-only model
-        if preset in cls.backbone_cls.presets:
-            backbone = cls.backbone_cls.from_preset(preset, load_weights)
-            return cls(backbone, **kwargs)
-
-        # Otherwise must be one of class presets
-        metadata = cls.presets[preset]
-        config = metadata["config"]
-        model = cls.from_config({**config, **kwargs})
-
-        if not load_weights:
-            return model
-
-        weights = keras.utils.get_file(
-            "model.h5",
-            metadata["weights_url"],
-            cache_subdir=os.path.join("models", preset),
-            file_hash=metadata["weights_hash"],
+        # We support short IDs for official presets, e.g. `"bert_base_en"`.
+        # Map these to a Kaggle Models handle.
+        if preset in cls.presets:
+            preset = cls.presets[preset]["kaggle_handle"]
+
+        preset_cls = check_preset_class(preset, (cls, cls.backbone_cls))
+
+        # Backbone case.
+        if preset_cls == cls.backbone_cls:
+            backbone = load_from_preset(
+                preset,
+                load_weights=load_weights,
+            )
+            if "preprocessor" in kwargs:
+                preprocessor = kwargs.pop("preprocessor")
+            else:
+                tokenizer = load_from_preset(
+                    preset,
+                    config_file="tokenizer.json",
+                )
+                preprocessor = cls.preprocessor_cls(tokenizer=tokenizer)
+            return cls(backbone=backbone, preprocessor=preprocessor, **kwargs)
+
+        # Task case.
+        return load_from_preset(
+            preset,
+            load_weights=load_weights,
+            config_overrides=kwargs,
         )
 
-        model.load_weights(weights)
-        return model
-
     def __init_subclass__(cls, **kwargs):
         # Use __init_subclass__ to setup a correct docstring for from_preset.
         super().__init_subclass__(**kwargs)
diff --git a/keras_nlp/models/whisper/whisper_preprocessor.py b/keras_nlp/models/whisper/whisper_preprocessor.py
index 8545890c..abcff0d7 100644
--- a/keras_nlp/models/whisper/whisper_preprocessor.py
+++ b/keras_nlp/models/whisper/whisper_preprocessor.py
@@ -30,7 +30,6 @@ from keras_nlp.utils.keras_utils import (
 )
 from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
 from keras_nlp.utils.python_utils import classproperty
-from keras_nlp.utils.python_utils import format_docstring
 
 
 @keras_nlp_export("keras_nlp.models.WhisperPreprocessor")
@@ -49,9 +48,11 @@ class WhisperPreprocessor(Preprocessor):
         directly to a Whisper model.
 
     Args:
-        audio_feature_extractor: A `keras_nlp.models.WhisperAudioFeatureExtractor`
-            instance.
         tokenizer: A `keras_nlp.models.WhisperTokenizer` instance.
+        audio_feature_extractor: A
+            `keras_nlp.models.WhisperAudioFeatureExtractor` instance or `None`.
+            If `None` a feature extractor with default parameters will be
+            created.
         decoder_sequence_length: The length of the packed decoder inputs.
         language: string, language token. Should only be passed if your
             tokenizer is multilingual.
@@ -73,7 +74,9 @@ class WhisperPreprocessor(Preprocessor):
 
     Directly calling the layer on data.
     ```python
-    preprocessor = keras_nlp.models.WhisperPreprocessor.from_preset("whisper_tiny_en")
+    preprocessor = keras_nlp.models.WhisperPreprocessor.from_preset(
+        "whisper_tiny_en",
+    )
 
     # Preprocess unbatched inputs.
     input_data = {
@@ -153,8 +156,8 @@ class WhisperPreprocessor(Preprocessor):
 
     def __init__(
         self,
-        audio_feature_extractor,
         tokenizer,
+        audio_feature_extractor=None,
         decoder_sequence_length=448,
         language=None,
         task=None,
@@ -162,50 +165,65 @@ class WhisperPreprocessor(Preprocessor):
         **kwargs,
     ):
         super().__init__(**kwargs)
+        if audio_feature_extractor is None:
+            audio_feature_extractor = WhisperAudioFeatureExtractor()
         self.audio_feature_extractor = audio_feature_extractor
         self.tokenizer = tokenizer
+        self.decoder_sequence_length = decoder_sequence_length
+        self.language = language
+        self.task = task
+        self.no_timestamps = no_timestamps
+        self.decoder_packer = None
+
+    def build(self, input_shape):
+        # Defer packer creation to `build()` so that we can be sure tokenizer
+        # assets have loaded when restoring a saved model.
 
         # Create list of tokens to be prepended to decoder inputs.
         bos_tokens = [self.tokenizer.bos_token_id]
         if self.tokenizer.language_tokens is not None:
             if (
-                language is None
-                or language not in self.tokenizer.language_tokens
+                self.language is None
+                or self.language not in self.tokenizer.language_tokens
             ):
                 raise ValueError(
                     "You must pass a non-None value for `language` when using "
                     "a multilingual tokenizer. The value must be one of "
                     f'{",".join(self.tokenizer.language_tokens.keys())}. '
-                    f"Received: language={language}."
+                    f"Received: language={self.language}."
                 )
-            if task is None or task not in ["transcribe", "translate"]:
+            if self.task is None or self.task not in [
+                "transcribe",
+                "translate",
+            ]:
                 raise ValueError(
                     "You must pass a non-None value for `task` when using "
                     "a multilingual tokenizer. The value must be one of "
-                    f'`"transcribe"`, `"translate"`. Received: task={task}.'
+                    '`"transcribe"`, `"translate"`. '
+                    f"Received: task={self.task}."
                 )
 
-            bos_tokens += [self.tokenizer.language_tokens[language]]
+            bos_tokens += [self.tokenizer.language_tokens[self.language]]
 
-            if task == "transcribe":
+            if self.task == "transcribe":
                 bos_tokens += [self.tokenizer.special_tokens["
"]]
-            elif task == "translate":
+            elif self.task == "translate":
                 bos_tokens += [self.tokenizer.special_tokens["
"]]
         else:
-            if language is not None:
+            if self.language is not None:
                 logging.info(
                     "`tokenizer` is monolingual, and `language` has a "
                     "non-`None` value. Setting `language` to `None`."
                 )
-                language = None
-            if task is not None:
+                self.language = None
+            if self.task is not None:
                 logging.info(
                     "`tokenizer` is monolingual, and `task` has a "
                     "non-`None` value. Setting `task` to `None`."
                 )
-                task = None
+                self.task = None
 
-        if no_timestamps:
+        if self.no_timestamps:
             bos_tokens += [self.tokenizer.no_timestamps_token_id]
 
         # TODO: Use `MultiSegmentPacker` instead of `StartEndPacker` once we
@@ -215,44 +233,10 @@ class WhisperPreprocessor(Preprocessor):
             start_value=bos_tokens,
             end_value=self.tokenizer.eos_token_id,
             pad_value=self.tokenizer.pad_token_id,
-            sequence_length=decoder_sequence_length,
+            sequence_length=self.decoder_sequence_length,
             return_padding_mask=True,
         )
 
-        self.decoder_sequence_length = decoder_sequence_length
-        self.language = language
-        self.task = task
-        self.no_timestamps = no_timestamps
-
-    def get_config(self):
-        config = super().get_config()
-        config.update(
-            {
-                "audio_feature_extractor": keras.layers.serialize(
-                    self.audio_feature_extractor
-                ),
-                "decoder_sequence_length": self.decoder_sequence_length,
-                "language": self.language,
-                "task": self.task,
-                "no_timestamps": self.no_timestamps,
-            }
-        )
-        return config
-
-    @classmethod
-    def from_config(cls, config):
-        if "tokenizer" in config and isinstance(config["tokenizer"], dict):
-            config["tokenizer"] = keras.layers.deserialize(config["tokenizer"])
-
-        if "audio_feature_extractor" in config and isinstance(
-            config["audio_feature_extractor"], dict
-        ):
-            config["audio_feature_extractor"] = keras.layers.deserialize(
-                config["audio_feature_extractor"]
-            )
-
-        return cls(**config)
-
     def call(self, x, y=None, sample_weight=None, decoder_sequence_length=None):
         if not (
             isinstance(x, dict)
@@ -294,6 +278,35 @@ class WhisperPreprocessor(Preprocessor):
 
         return pack_x_y_sample_weight(x, y, sample_weight)
 
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "audio_feature_extractor": keras.layers.serialize(
+                    self.audio_feature_extractor
+                ),
+                "decoder_sequence_length": self.decoder_sequence_length,
+                "language": self.language,
+                "task": self.task,
+                "no_timestamps": self.no_timestamps,
+            }
+        )
+        return config
+
+    @classmethod
+    def from_config(cls, config):
+        if "tokenizer" in config and isinstance(config["tokenizer"], dict):
+            config["tokenizer"] = keras.layers.deserialize(config["tokenizer"])
+
+        if "audio_feature_extractor" in config and isinstance(
+            config["audio_feature_extractor"], dict
+        ):
+            config["audio_feature_extractor"] = keras.layers.deserialize(
+                config["audio_feature_extractor"]
+            )
+
+        return cls(**config)
+
     @classproperty
     def audio_feature_extractor_cls(cls):
         return WhisperAudioFeatureExtractor
@@ -305,97 +318,3 @@ class WhisperPreprocessor(Preprocessor):
     @classproperty
     def presets(cls):
         return copy.deepcopy(backbone_presets)
-
-    @classmethod
-    def from_preset(
-        cls,
-        preset,
-        language=None,
-        task=None,
-        no_timestamps=True,
-        **kwargs,
-    ):
-        """Instantiate `WhisperPreprocessor` from preset architecture.
-
-        Args:
-            preset: string. Must be one of "{{preset_names}}".
-            language: string, language token (eg., `"
"`). Should only be
-                passed if your tokenizer is multilingual.
-            task: string, task name. One of `"transcribe"`, `"translate"`.
-                Should only be passed if your tokenizer is multilingual.
-            no_timestamps: bool. If True, `"
"` will be added as
-                a special token to your input.
-
-        Examples:
-        ```python
-        # Load a preprocessor layer from a preset.
-        preprocessor = keras_nlp.models.WhisperPreprocessor.from_preset(
-            "{{example_preset_name}}",
-        )
-        ```
-        """
-        # Override base class's `from_preset` to handle audio feature extractor
-        # , `decoder_sequence_length` and special tokens.
-        if not cls.presets:
-            raise NotImplementedError(
-                "No presets have been created for this class."
-            )
-        if preset not in cls.presets:
-            raise ValueError(
-                "`preset` must be one of "
-                f"""{", ".join(cls.presets)}. Received: {preset}."""
-            )
-
-        audio_feature_extractor = cls.audio_feature_extractor_cls.from_preset(
-            preset
-        )
-        tokenizer = cls.tokenizer_cls.from_preset(preset)
-
-        metadata = cls.presets[preset]
-        # For task model presets, the backbone config is nested.
-        if "backbone" in metadata["config"]:
-            backbone_config = metadata["config"]["backbone"]["config"]
-        else:
-            backbone_config = metadata["config"]
-
-        # Use model's `max_decoder_sequence_length` if `decoder_sequence_length`
-        # is unspecified; otherwise check that `decoder_sequence_length` is not
-        # too long.
-        decoder_sequence_length = kwargs.pop("decoder_sequence_length", None)
-        max_decoder_sequence_length = backbone_config[
-            "max_decoder_sequence_length"
-        ]
-
-        def check_sequence_length(sequence_length, max_sequence_length, name):
-            if sequence_length is not None:
-                if sequence_length > max_sequence_length:
-                    raise ValueError(
-                        f"`{name}` cannot be longer than `{preset}` "
-                        f"preset's `max_{name}` of {max_sequence_length}. "
-                        f"Received: {sequence_length}."
-                    )
-                return sequence_length
-            else:
-                return max_sequence_length
-
-        decoder_sequence_length = check_sequence_length(
-            decoder_sequence_length,
-            max_decoder_sequence_length,
-            "decoder_sequence_length",
-        )
-
-        return cls(
-            audio_feature_extractor=audio_feature_extractor,
-            tokenizer=tokenizer,
-            decoder_sequence_length=decoder_sequence_length,
-            language=language,
-            task=task,
-            no_timestamps=no_timestamps,
-            **kwargs,
-        )
-
-
-format_docstring(
-    example_preset_name=next(iter(backbone_presets), ""),
-    preset_names='", "'.join(backbone_presets),
-)(WhisperPreprocessor.from_preset.__func__)
diff --git a/keras_nlp/models/whisper/whisper_presets.py b/keras_nlp/models/whisper/whisper_presets.py
index 3c385dea..7494eef8 100644
--- a/keras_nlp/models/whisper/whisper_presets.py
+++ b/keras_nlp/models/whisper/whisper_presets.py
@@ -11,131 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-MULTILINGUAL_SPECIAL_TOKENS = {
-    "
": 50258,
-    "
": 50257,
-    "
": 50363,
-    "
": 50359,
-    "
": 50358,
-}
-
-ENGLISH_SPECIAL_TOKENS = {
-    "
": 50257,
-    "
": 50256,
-    "
": 50362,
-    "
": 50358,
-    "
": 50357,
-}
-
-AUDIO_FEATURE_EXTRACTOR_CONFIG = {
-    "num_mels": 80,
-    "num_fft_bins": 400,
-    "stride": 160,
-    "sampling_rate": 16000,
-    "max_audio_length": 30,
-}
-
-LANGUAGE_TOKENS = {
-    "
": 50327,
-    "
": 50334,
-    "
": 50272,
-    "
": 50350,
-    "
": 50304,
-    "
": 50355,
-    "
": 50330,
-    "
": 50292,
-    "
": 50302,
-    "
": 50347,
-    "
": 50309,
-    "
": 50315,
-    "
": 50270,
-    "
": 50283,
-    "
": 50297,
-    "
": 50285,
-    "
": 50261,
-    "
": 50281,
-    "
": 50259,
-    "
": 50262,
-    "
": 50307,
-    "
": 50310,
-    "
": 50300,
-    "
": 50277,
-    "
": 50338,
-    "
": 50265,
-    "
": 50319,
-    "
": 50333,
-    "
": 50352,
-    "
": 50354,
-    "
": 50279,
-    "
": 50276,
-    "
": 50291,
-    "
": 50339,
-    "
": 50286,
-    "
": 50312,
-    "
": 50275,
-    "
": 50311,
-    "
": 50274,
-    "
": 50266,
-    "
": 50356,
-    "
": 50329,
-    "
": 50316,
-    "
": 50323,
-    "
": 50306,
-    "
": 50264,
-    "
": 50294,
-    "
": 50345,
-    "
": 50353,
-    "
": 50336,
-    "
": 50293,
-    "
": 50301,
-    "
": 50349,
-    "
": 50295,
-    "
": 50308,
-    "
": 50296,
-    "
": 50314,
-    "
": 50320,
-    "
": 50282,
-    "
": 50343,
-    "
": 50346,
-    "
": 50313,
-    "
": 50271,
-    "
": 50342,
-    "
": 50288,
-    "
": 50328,
-    "
": 50321,
-    "
": 50269,
-    "
": 50340,
-    "
": 50267,
-    "
": 50284,
-    "
": 50263,
-    "
": 50344,
-    "
": 50332,
-    "
": 50322,
-    "
": 50298,
-    "
": 50305,
-    "
": 50324,
-    "
": 50326,
-    "
": 50317,
-    "
": 50303,
-    "
": 50357,
-    "
": 50273,
-    "
": 50318,
-    "
": 50287,
-    "
": 50299,
-    "
": 50331,
-    "
": 50289,
-    "
": 50341,
-    "
": 50348,
-    "
": 50268,
-    "
": 50351,
-    "
": 50280,
-    "
": 50290,
-    "
": 50337,
-    "
": 50278,
-    "
": 50335,
-    "
": 50325,
-    "
": 50260,
-}
 
 # Metadata for loading pretrained model weights.
 backbone_presets = {
@@ -150,28 +25,7 @@ backbone_presets = {
             "path": "whisper",
             "model_card": "https://github.com/openai/whisper/blob/main/model-card.md",
         },
-        "config": {
-            "vocabulary_size": 51864,
-            "num_layers": 4,
-            "num_heads": 6,
-            "hidden_dim": 384,
-            "intermediate_dim": 1536,
-            "num_mels": 80,
-            "dropout": 0.0,
-            "max_encoder_sequence_length": 3000,
-            "max_decoder_sequence_length": 448,
-        },
-        "audio_feature_extractor_config": AUDIO_FEATURE_EXTRACTOR_CONFIG,
-        "preprocessor_config": {
-            "special_tokens": ENGLISH_SPECIAL_TOKENS,
-            "language_tokens": None,
-        },
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/whisper_tiny_en/v1/model.h5",
-        "weights_hash": "3dc3768ac48ec90b1029fbf52ffbacc7",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/whisper_tiny_en/v1/vocab.json",
-        "vocabulary_hash": "22377f841debacb023848b3468ea3281",
-        "merges_url": "https://storage.googleapis.com/keras-nlp/models/whisper_tiny_en/v1/merges.txt",
-        "merges_hash": "093ecf3f30371012f2e96fcfb10ea6ab",
+        "kaggle_handle": "kaggle://keras/whisper/whisper_tiny_en/2",
     },
     "whisper_base_en": {
         "metadata": {
@@ -184,28 +38,7 @@ backbone_presets = {
             "path": "whisper",
             "model_card": "https://github.com/openai/whisper/blob/main/model-card.md",
         },
-        "config": {
-            "vocabulary_size": 51864,
-            "num_layers": 6,
-            "num_heads": 8,
-            "hidden_dim": 512,
-            "intermediate_dim": 2048,
-            "num_mels": 80,
-            "dropout": 0.0,
-            "max_encoder_sequence_length": 3000,
-            "max_decoder_sequence_length": 448,
-        },
-        "audio_feature_extractor_config": AUDIO_FEATURE_EXTRACTOR_CONFIG,
-        "preprocessor_config": {
-            "special_tokens": ENGLISH_SPECIAL_TOKENS,
-            "language_tokens": None,
-        },
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/whisper_base_en/v1/model.h5",
-        "weights_hash": "799d3c143993d42f7446bafbc0f46d7d",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/whisper_base_en/v1/vocab.json",
-        "vocabulary_hash": "22377f841debacb023848b3468ea3281",
-        "merges_url": "https://storage.googleapis.com/keras-nlp/models/whisper_base_en/v1/merges.txt",
-        "merges_hash": "093ecf3f30371012f2e96fcfb10ea6ab",
+        "kaggle_handle": "kaggle://keras/whisper/whisper_base_en/2",
     },
     "whisper_small_en": {
         "metadata": {
@@ -218,28 +51,7 @@ backbone_presets = {
             "path": "whisper",
             "model_card": "https://github.com/openai/whisper/blob/main/model-card.md",
         },
-        "config": {
-            "vocabulary_size": 51864,
-            "num_layers": 12,
-            "num_heads": 12,
-            "hidden_dim": 768,
-            "intermediate_dim": 3072,
-            "num_mels": 80,
-            "dropout": 0.0,
-            "max_encoder_sequence_length": 3000,
-            "max_decoder_sequence_length": 448,
-        },
-        "audio_feature_extractor_config": AUDIO_FEATURE_EXTRACTOR_CONFIG,
-        "preprocessor_config": {
-            "special_tokens": ENGLISH_SPECIAL_TOKENS,
-            "language_tokens": None,
-        },
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/whisper_small_en/v1/model.h5",
-        "weights_hash": "b75a89225e20019d85ff5f1c362f8a49",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/whisper_small_en/v1/vocab.json",
-        "vocabulary_hash": "22377f841debacb023848b3468ea3281",
-        "merges_url": "https://storage.googleapis.com/keras-nlp/models/whisper_small_en/v1/merges.txt",
-        "merges_hash": "093ecf3f30371012f2e96fcfb10ea6ab",
+        "kaggle_handle": "kaggle://keras/whisper/whisper_small_en/2",
     },
     "whisper_medium_en": {
         "metadata": {
@@ -252,28 +64,7 @@ backbone_presets = {
             "path": "whisper",
             "model_card": "https://github.com/openai/whisper/blob/main/model-card.md",
         },
-        "config": {
-            "vocabulary_size": 51864,
-            "num_layers": 24,
-            "num_heads": 16,
-            "hidden_dim": 1024,
-            "intermediate_dim": 4096,
-            "num_mels": 80,
-            "dropout": 0.0,
-            "max_encoder_sequence_length": 3000,
-            "max_decoder_sequence_length": 448,
-        },
-        "audio_feature_extractor_config": AUDIO_FEATURE_EXTRACTOR_CONFIG,
-        "preprocessor_config": {
-            "special_tokens": ENGLISH_SPECIAL_TOKENS,
-            "language_tokens": None,
-        },
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/whisper_medium_en/v1/model.h5",
-        "weights_hash": "107184882d1cc65926815e4cc50dc5f3",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/whisper_medium_en/v1/vocab.json",
-        "vocabulary_hash": "22377f841debacb023848b3468ea3281",
-        "merges_url": "https://storage.googleapis.com/keras-nlp/models/whisper_medium_en/v1/merges.txt",
-        "merges_hash": "093ecf3f30371012f2e96fcfb10ea6ab",
+        "kaggle_handle": "kaggle://keras/whisper/whisper_medium_en/2",
     },
     "whisper_tiny_multi": {
         "metadata": {
@@ -286,28 +77,7 @@ backbone_presets = {
             "path": "whisper",
             "model_card": "https://github.com/openai/whisper/blob/main/model-card.md",
         },
-        "config": {
-            "vocabulary_size": 51865,
-            "num_layers": 4,
-            "num_heads": 6,
-            "hidden_dim": 384,
-            "intermediate_dim": 1536,
-            "num_mels": 80,
-            "dropout": 0.0,
-            "max_encoder_sequence_length": 3000,
-            "max_decoder_sequence_length": 448,
-        },
-        "audio_feature_extractor_config": AUDIO_FEATURE_EXTRACTOR_CONFIG,
-        "preprocessor_config": {
-            "special_tokens": MULTILINGUAL_SPECIAL_TOKENS,
-            "language_tokens": LANGUAGE_TOKENS,
-        },
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/whisper_tiny_multi/v1/model.h5",
-        "weights_hash": "b1279a81001ad5eb35970d1aea706396",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/whisper_tiny_multi/v1/vocab.json",
-        "vocabulary_hash": "1b87ed3e3ecd9ccfdca74e64cbe81d68",
-        "merges_url": "https://storage.googleapis.com/keras-nlp/models/whisper_tiny_multi/v1/merges.txt",
-        "merges_hash": "c7f01d4100f6211417988889bf35ccd8",
+        "kaggle_handle": "kaggle://keras/whisper/whisper_tiny_multi/2",
     },
     "whisper_base_multi": {
         "metadata": {
@@ -320,28 +90,7 @@ backbone_presets = {
             "path": "whisper",
             "model_card": "https://github.com/openai/whisper/blob/main/model-card.md",
         },
-        "config": {
-            "vocabulary_size": 51865,
-            "num_layers": 6,
-            "num_heads": 8,
-            "hidden_dim": 512,
-            "intermediate_dim": 2048,
-            "num_mels": 80,
-            "dropout": 0.0,
-            "max_encoder_sequence_length": 3000,
-            "max_decoder_sequence_length": 448,
-        },
-        "audio_feature_extractor_config": AUDIO_FEATURE_EXTRACTOR_CONFIG,
-        "preprocessor_config": {
-            "special_tokens": MULTILINGUAL_SPECIAL_TOKENS,
-            "language_tokens": LANGUAGE_TOKENS,
-        },
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/whisper_base_multi/v1/model.h5",
-        "weights_hash": "5208396e2d5efac43114a4a3d4f583ab",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/whisper_base_multi/v1/vocab.json",
-        "vocabulary_hash": "1b87ed3e3ecd9ccfdca74e64cbe81d68",
-        "merges_url": "https://storage.googleapis.com/keras-nlp/models/whisper_base_multi/v1/merges.txt",
-        "merges_hash": "c7f01d4100f6211417988889bf35ccd8",
+        "kaggle_handle": "kaggle://keras/whisper/whisper_base_multi/2",
     },
     "whisper_small_multi": {
         "metadata": {
@@ -354,28 +103,7 @@ backbone_presets = {
             "path": "whisper",
             "model_card": "https://github.com/openai/whisper/blob/main/model-card.md",
         },
-        "config": {
-            "vocabulary_size": 51865,
-            "num_layers": 12,
-            "num_heads": 12,
-            "hidden_dim": 768,
-            "intermediate_dim": 3072,
-            "num_mels": 80,
-            "dropout": 0.0,
-            "max_encoder_sequence_length": 3000,
-            "max_decoder_sequence_length": 448,
-        },
-        "audio_feature_extractor_config": AUDIO_FEATURE_EXTRACTOR_CONFIG,
-        "preprocessor_config": {
-            "special_tokens": MULTILINGUAL_SPECIAL_TOKENS,
-            "language_tokens": LANGUAGE_TOKENS,
-        },
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/whisper_small_multi/v1/model.h5",
-        "weights_hash": "c90c6a895e522056b77b924b6e907ed8",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/whisper_small_multi/v1/vocab.json",
-        "vocabulary_hash": "1b87ed3e3ecd9ccfdca74e64cbe81d68",
-        "merges_url": "https://storage.googleapis.com/keras-nlp/models/whisper_small_multi/v1/merges.txt",
-        "merges_hash": "c7f01d4100f6211417988889bf35ccd8",
+        "kaggle_handle": "kaggle://keras/whisper/whisper_small_multi/2",
     },
     "whisper_medium_multi": {
         "metadata": {
@@ -388,28 +116,7 @@ backbone_presets = {
             "path": "whisper",
             "model_card": "https://github.com/openai/whisper/blob/main/model-card.md",
         },
-        "config": {
-            "vocabulary_size": 51865,
-            "num_layers": 24,
-            "num_heads": 16,
-            "hidden_dim": 1024,
-            "intermediate_dim": 4096,
-            "num_mels": 80,
-            "dropout": 0.0,
-            "max_encoder_sequence_length": 3000,
-            "max_decoder_sequence_length": 448,
-        },
-        "audio_feature_extractor_config": AUDIO_FEATURE_EXTRACTOR_CONFIG,
-        "preprocessor_config": {
-            "special_tokens": MULTILINGUAL_SPECIAL_TOKENS,
-            "language_tokens": LANGUAGE_TOKENS,
-        },
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/whisper_medium_multi/v1/model.h5",
-        "weights_hash": "6f993f732fe397e9c5e3a96a9505a3a9",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/whisper_medium_multi/v1/vocab.json",
-        "vocabulary_hash": "1b87ed3e3ecd9ccfdca74e64cbe81d68",
-        "merges_url": "https://storage.googleapis.com/keras-nlp/models/whisper_medium_multi/v1/merges.txt",
-        "merges_hash": "c7f01d4100f6211417988889bf35ccd8",
+        "kaggle_handle": "kaggle://keras/whisper/whisper_medium_multi/2",
     },
     "whisper_large_multi": {
         "metadata": {
@@ -422,28 +129,7 @@ backbone_presets = {
             "path": "whisper",
             "model_card": "https://github.com/openai/whisper/blob/main/model-card.md",
         },
-        "config": {
-            "vocabulary_size": 51865,
-            "num_layers": 32,
-            "num_heads": 20,
-            "hidden_dim": 1280,
-            "intermediate_dim": 5120,
-            "num_mels": 80,
-            "dropout": 0.0,
-            "max_encoder_sequence_length": 3000,
-            "max_decoder_sequence_length": 448,
-        },
-        "audio_feature_extractor_config": AUDIO_FEATURE_EXTRACTOR_CONFIG,
-        "preprocessor_config": {
-            "special_tokens": MULTILINGUAL_SPECIAL_TOKENS,
-            "language_tokens": LANGUAGE_TOKENS,
-        },
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/whisper_large_multi/v1/model.h5",
-        "weights_hash": "ccab1c93c5739007868ae73fe025806d",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/whisper_large_multi/v1/vocab.json",
-        "vocabulary_hash": "1b87ed3e3ecd9ccfdca74e64cbe81d68",
-        "merges_url": "https://storage.googleapis.com/keras-nlp/models/whisper_large_multi/v1/merges.txt",
-        "merges_hash": "c7f01d4100f6211417988889bf35ccd8",
+        "kaggle_handle": "kaggle://keras/whisper/whisper_large_multi/2",
     },
     "whisper_large_multi_v2": {
         "metadata": {
@@ -457,27 +143,6 @@ backbone_presets = {
             "path": "whisper",
             "model_card": "https://github.com/openai/whisper/blob/main/model-card.md",
         },
-        "config": {
-            "vocabulary_size": 51865,
-            "num_layers": 32,
-            "num_heads": 20,
-            "hidden_dim": 1280,
-            "intermediate_dim": 5120,
-            "num_mels": 80,
-            "dropout": 0.0,
-            "max_encoder_sequence_length": 3000,
-            "max_decoder_sequence_length": 448,
-        },
-        "audio_feature_extractor_config": AUDIO_FEATURE_EXTRACTOR_CONFIG,
-        "preprocessor_config": {
-            "special_tokens": MULTILINGUAL_SPECIAL_TOKENS,
-            "language_tokens": LANGUAGE_TOKENS,
-        },
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/whisper_large_multi_v2/v1/model.h5",
-        "weights_hash": "ccab1c93c5739007868ae73fe025806d",
-        "vocabulary_url": "https://storage.googleapis.com/keras-nlp/models/whisper_large_multi_v2/v1/vocab.json",
-        "vocabulary_hash": "1b87ed3e3ecd9ccfdca74e64cbe81d68",
-        "merges_url": "https://storage.googleapis.com/keras-nlp/models/whisper_large_multi_v2/v1/merges.txt",
-        "merges_hash": "c7f01d4100f6211417988889bf35ccd8",
+        "kaggle_handle": "kaggle://keras/whisper/whisper_large_multi_v2/2",
     },
 }
diff --git a/keras_nlp/models/whisper/whisper_tokenizer.py b/keras_nlp/models/whisper/whisper_tokenizer.py
index cd4da7d1..7b68dfd7 100644
--- a/keras_nlp/models/whisper/whisper_tokenizer.py
+++ b/keras_nlp/models/whisper/whisper_tokenizer.py
@@ -52,49 +52,36 @@ class WhisperTokenizer(BytePairTokenizer):
 
     def __init__(
         self,
-        vocabulary,
-        merges,
-        special_tokens,
+        vocabulary=None,
+        merges=None,
+        special_tokens=None,
         language_tokens=None,
         **kwargs,
     ):
-        vocabulary = _load_dict(vocabulary)
-
-        # Necessary special tokens.
-        bos_token = "
"
-        eos_token = "
"
-
+        special_tokens = _load_dict(special_tokens)
         if language_tokens is not None:
-            # Multilingual tokenizer.
-            # TODO: The pad token for the multilingual tokenizer is actually
-            # "", but it errors out (OOM). After BPE is fixed, we can update
-            # this to "". For now, we will use `"
"`.
-            pad_token = "
"
             language_tokens = _load_dict(language_tokens)
 
-            # Add language tokens to the vocabulary. This makes detokenization
-            # easier for us.
-            vocabulary = {
-                **vocabulary,
-                **language_tokens,
-            }
-        else:
-            # English tokenizer.
-            pad_token = "
"
-
-        no_timestamps_token = "
"
+        # Necessary special tokens.
+        self.bos_token = "
"
+        self.eos_token = "
"
+        # TODO: The pad token for the multilingual tokenizer is actually
+        # "", but it errors out (OOM). After BPE is fixed, we can update
+        # this to "". For now, we will use `"
"`.
+        self.pad_token = "
"
+
+        self.no_timestamps_token = "
"
         # Task special tokens.
-        translate_token = "
"
-        transcribe_token = "
"
+        self.translate_token = "
"
+        self.transcribe_token = "
"
 
-        special_tokens = _load_dict(special_tokens)
         for token in [
-            bos_token,
-            eos_token,
-            pad_token,
-            no_timestamps_token,
-            translate_token,
-            transcribe_token,
+            self.bos_token,
+            self.eos_token,
+            self.pad_token,
+            self.no_timestamps_token,
+            self.translate_token,
+            self.transcribe_token,
         ]:
             if token not in special_tokens:
                 raise ValueError(
@@ -102,15 +89,16 @@ class WhisperTokenizer(BytePairTokenizer):
                     f"`special_tokens`. Please provide `'{token}'` in your "
                     "`special_tokens`."
                 )
-            # Add special tokens to `vocabulary` for easy detokenization.
-            vocabulary[token] = special_tokens[token]
 
-        self.bos_token_id = special_tokens[bos_token]
-        self.eos_token_id = special_tokens[eos_token]
-        self.pad_token_id = special_tokens[pad_token]
-        self.no_timestamps_token_id = special_tokens[no_timestamps_token]
-        self.translate_token_id = special_tokens[translate_token]
-        self.transcribe_token_id = special_tokens[transcribe_token]
+        self.bos_token_id = special_tokens[self.bos_token]
+        self.eos_token_id = special_tokens[self.eos_token]
+        self.pad_token_id = special_tokens[self.pad_token]
+        self.no_timestamps_token_id = special_tokens[self.no_timestamps_token]
+        self.translate_token_id = special_tokens[self.translate_token]
+        self.transcribe_token_id = special_tokens[self.transcribe_token]
+
+        self.special_tokens = special_tokens
+        self.language_tokens = language_tokens
 
         # TODO: Add language tokens to `unsplittable_tokens` once we figure
         # out the performance issue with a large list.
@@ -123,8 +111,43 @@ class WhisperTokenizer(BytePairTokenizer):
             **kwargs,
         )
 
-        self.special_tokens = special_tokens
-        self.language_tokens = language_tokens
+    def save_assets(self, dir_path):
+        # TODO: whisper is currently mutating it's vocabulary before passing
+        # it to the super class, so we need to restore the unmutated vocabulary
+        # before saving our assets. We should find a more robust (and memory
+        # efficient) way to do this.
+        vocabulary = self.vocabulary
+        self.vocabulary = self._initial_vocabulary
+        super().save_assets(dir_path)
+        self.vocabulary = vocabulary
+
+    def set_vocabulary_and_merges(self, vocabulary, merges):
+        if vocabulary is not None:
+            vocabulary = _load_dict(vocabulary)
+            self._initial_vocabulary = dict(vocabulary)
+
+            if self.language_tokens is not None:
+                # Multilingual tokenizer.
+                # Add language tokens to the vocabulary. This makes
+                # detokenization easier for us.
+                vocabulary = {
+                    **vocabulary,
+                    **self.language_tokens,
+                }
+
+            for token in [
+                self.bos_token,
+                self.eos_token,
+                self.pad_token,
+                self.no_timestamps_token,
+                self.translate_token,
+                self.transcribe_token,
+            ]:
+                vocabulary[token] = self.special_tokens[token]
+        else:
+            self._initial_vocabulary = None
+
+        super().set_vocabulary_and_merges(vocabulary, merges)
 
     def get_config(self):
         config = super().get_config()
diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py b/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py
index 2a8750c5..a26905e9 100644
--- a/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py
+++ b/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py
@@ -137,18 +137,27 @@ class XLMRobertaMaskedLMPreprocessor(XLMRobertaPreprocessor):
             truncate=truncate,
             **kwargs,
         )
-
+        self.mask_selection_rate = mask_selection_rate
+        self.mask_selection_length = mask_selection_length
+        self.mask_token_rate = mask_token_rate
+        self.random_token_rate = random_token_rate
+        self.masker = None
+
+    def build(self, input_shape):
+        super().build(input_shape)
+        # Defer masker creation to `build()` so that we can be sure tokenizer
+        # assets have loaded when restoring a saved model.
         self.masker = MaskedLMMaskGenerator(
-            mask_selection_rate=mask_selection_rate,
-            mask_selection_length=mask_selection_length,
-            mask_token_rate=mask_token_rate,
-            random_token_rate=random_token_rate,
-            vocabulary_size=tokenizer.vocabulary_size(),
-            mask_token_id=tokenizer.mask_token_id,
+            mask_selection_rate=self.mask_selection_rate,
+            mask_selection_length=self.mask_selection_length,
+            mask_token_rate=self.mask_token_rate,
+            random_token_rate=self.random_token_rate,
+            vocabulary_size=self.tokenizer.vocabulary_size(),
+            mask_token_id=self.tokenizer.mask_token_id,
             unselectable_token_ids=[
-                tokenizer.start_token_id,
-                tokenizer.end_token_id,
-                tokenizer.pad_token_id,
+                self.tokenizer.start_token_id,
+                self.tokenizer.end_token_id,
+                self.tokenizer.pad_token_id,
             ],
         )
 
@@ -156,10 +165,10 @@ class XLMRobertaMaskedLMPreprocessor(XLMRobertaPreprocessor):
         config = super().get_config()
         config.update(
             {
-                "mask_selection_rate": self.masker.mask_selection_rate,
-                "mask_selection_length": self.masker.mask_selection_length,
-                "mask_token_rate": self.masker.mask_token_rate,
-                "random_token_rate": self.masker.random_token_rate,
+                "mask_selection_rate": self.mask_selection_rate,
+                "mask_selection_length": self.mask_selection_length,
+                "mask_token_rate": self.mask_token_rate,
+                "random_token_rate": self.random_token_rate,
             }
         )
         return config
diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_preprocessor.py b/keras_nlp/models/xlm_roberta/xlm_roberta_preprocessor.py
index e557bd06..23b48073 100644
--- a/keras_nlp/models/xlm_roberta/xlm_roberta_preprocessor.py
+++ b/keras_nlp/models/xlm_roberta/xlm_roberta_preprocessor.py
@@ -156,22 +156,29 @@ class XLMRobertaPreprocessor(Preprocessor):
         super().__init__(**kwargs)
 
         self.tokenizer = tokenizer
+        self.truncate = truncate
+        self.sequence_length = sequence_length
+        self.packer = None
 
+    def build(self, input_shape):
+        # Defer packer creation to `build()` so that we can be sure tokenizer
+        # assets have loaded when restoring a saved model.
         self.packer = MultiSegmentPacker(
             start_value=self.tokenizer.start_token_id,
             end_value=self.tokenizer.end_token_id,
             sep_value=[self.tokenizer.end_token_id] * 2,
             pad_value=self.tokenizer.pad_token_id,
-            truncate=truncate,
-            sequence_length=sequence_length,
+            truncate=self.truncate,
+            sequence_length=self.sequence_length,
         )
+        self.built = True
 
     def get_config(self):
         config = super().get_config()
         config.update(
             {
-                "sequence_length": self.packer.sequence_length,
-                "truncate": self.packer.truncate,
+                "sequence_length": self.sequence_length,
+                "truncate": self.truncate,
             }
         )
         return config
diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_presets.py b/keras_nlp/models/xlm_roberta/xlm_roberta_presets.py
index 350c069f..d8676b46 100644
--- a/keras_nlp/models/xlm_roberta/xlm_roberta_presets.py
+++ b/keras_nlp/models/xlm_roberta/xlm_roberta_presets.py
@@ -25,20 +25,7 @@ backbone_presets = {
             "path": "xlm_roberta",
             "model_card": "https://github.com/facebookresearch/fairseq/blob/main/examples/xlmr/README.md",
         },
-        "config": {
-            "vocabulary_size": 250002,
-            "num_layers": 12,
-            "num_heads": 12,
-            "hidden_dim": 768,
-            "intermediate_dim": 3072,
-            "dropout": 0.1,
-            "max_sequence_length": 512,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/xlm_roberta_base_multi/v1/model.h5",
-        "weights_hash": "2eb6fcda5a42f0a88056213ba3d93906",
-        "spm_proto_url": "https://storage.googleapis.com/keras-nlp/models/xlm_roberta_base_multi/v1/vocab.spm",
-        "spm_proto_hash": "bf25eb5120ad92ef5c7d8596b5dc4046",
+        "kaggle_handle": "kaggle://keras/xlm_roberta/xlm_roberta_base_multi/2",
     },
     "xlm_roberta_large_multi": {
         "metadata": {
@@ -51,19 +38,6 @@ backbone_presets = {
             "path": "xlm_roberta",
             "model_card": "https://github.com/facebookresearch/fairseq/blob/main/examples/xlmr/README.md",
         },
-        "config": {
-            "vocabulary_size": 250002,
-            "num_layers": 24,
-            "num_heads": 16,
-            "hidden_dim": 1024,
-            "intermediate_dim": 4096,
-            "dropout": 0.1,
-            "max_sequence_length": 512,
-        },
-        "preprocessor_config": {},
-        "weights_url": "https://storage.googleapis.com/keras-nlp/models/xlm_roberta_large_multi/v1/model.h5",
-        "weights_hash": "276211827174b71751f2ce3a89da503a",
-        "spm_proto_url": "https://storage.googleapis.com/keras-nlp/models/xlm_roberta_large_multi/v1/vocab.spm",
-        "spm_proto_hash": "bf25eb5120ad92ef5c7d8596b5dc4046",
+        "kaggle_handle": "kaggle://keras/xlm_roberta/xlm_roberta_large_multi/2",
     },
 }
diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_tokenizer.py b/keras_nlp/models/xlm_roberta/xlm_roberta_tokenizer.py
index eacdcc73..576f30bc 100644
--- a/keras_nlp/models/xlm_roberta/xlm_roberta_tokenizer.py
+++ b/keras_nlp/models/xlm_roberta/xlm_roberta_tokenizer.py
@@ -88,8 +88,6 @@ class XLMRobertaTokenizer(SentencePieceTokenizer):
     """
 
     def __init__(self, proto, **kwargs):
-        super().__init__(proto=proto, **kwargs)
-
         # List of special tokens.
         self._vocabulary_prefix = ["
", "<pad>", "
", "<unk>"]
 
@@ -98,7 +96,15 @@ class XLMRobertaTokenizer(SentencePieceTokenizer):
         self.pad_token_id = 1  # <pad>
         self.end_token_id = 2  # 

         self.unk_token_id = 3  # <unk>
-        self.mask_token_id = self.vocabulary_size() - 1  # <mask>
+
+        super().__init__(proto=proto, **kwargs)
+
+    def set_proto(self, proto):
+        super().set_proto(proto)
+        if proto is not None:
+            self.mask_token_id = self.vocabulary_size() - 1
+        else:
+            self.mask_token_id = None
 
     def vocabulary_size(self):
         """Get the size of the tokenizer vocabulary."""
@@ -106,6 +112,7 @@ class XLMRobertaTokenizer(SentencePieceTokenizer):
 
     def get_vocabulary(self):
         """Get the size of the tokenizer vocabulary."""
+        self._check_vocabulary()
         vocabulary = tensor_to_list(
             self._sentence_piece.id_to_string(
                 tf.range(super().vocabulary_size())
@@ -115,6 +122,7 @@ class XLMRobertaTokenizer(SentencePieceTokenizer):
 
     def id_to_token(self, id):
         """Convert an integer id to a string token."""
+        self._check_vocabulary()
 
         if id == self.mask_token_id:
             return "<mask>"
@@ -132,6 +140,7 @@ class XLMRobertaTokenizer(SentencePieceTokenizer):
 
     def token_to_id(self, token):
         """Convert a string token to an integer id."""
+        self._check_vocabulary()
 
         if token in self._vocabulary_prefix:
             return self._vocabulary_prefix.index(token)
@@ -146,6 +155,7 @@ class XLMRobertaTokenizer(SentencePieceTokenizer):
         return int(spm_token_id.numpy()) + 1
 
     def tokenize(self, inputs):
+        self._check_vocabulary()
         tokens = super().tokenize(inputs)
 
         # Correct `unk_token_id` (0 -> 3). Note that we do not correct
@@ -157,6 +167,7 @@ class XLMRobertaTokenizer(SentencePieceTokenizer):
         return tf.add(tokens, 1)
 
     def detokenize(self, inputs):
+        self._check_vocabulary()
         tokens = tf.ragged.boolean_mask(
             inputs, tf.not_equal(inputs, self.mask_token_id)
         )
diff --git a/keras_nlp/tokenizers/byte_pair_tokenizer.py b/keras_nlp/tokenizers/byte_pair_tokenizer.py
index b799874d..55992a16 100644
--- a/keras_nlp/tokenizers/byte_pair_tokenizer.py
+++ b/keras_nlp/tokenizers/byte_pair_tokenizer.py
@@ -28,8 +28,9 @@ import regex as re
 import tensorflow as tf
 
 from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
 from keras_nlp.tokenizers import tokenizer
+from keras_nlp.utils.preset_utils import check_preset_class
+from keras_nlp.utils.preset_utils import load_from_preset
 from keras_nlp.utils.python_utils import classproperty
 from keras_nlp.utils.python_utils import format_docstring
 from keras_nlp.utils.tensor_utils import assert_tf_text_installed
@@ -42,6 +43,10 @@ try:
 except ImportError:
     tf_text = None
 
+VOCAB_FILENAME = "vocabulary.json"
+MERGES_FILENAME = "merges.txt"
+
+
 # As python and TF handles special spaces differently, we need to
 # manually handle special spaces during string split.
 SPECIAL_WHITESPACES = r"\x{a0}\x{2009}\x{202f}\x{3000}"
@@ -273,8 +278,8 @@ class BytePairTokenizer(tokenizer.Tokenizer):
 
     def __init__(
         self,
-        vocabulary,
-        merges,
+        vocabulary=None,
+        merges=None,
         sequence_length=None,
         add_prefix_space=False,
         unsplittable_tokens=None,
@@ -290,6 +295,48 @@ class BytePairTokenizer(tokenizer.Tokenizer):
             )
 
         super().__init__(dtype=dtype, **kwargs)
+        self.sequence_length = sequence_length
+        self.add_prefix_space = add_prefix_space
+        self.unsplittable_tokens = unsplittable_tokens
+
+        # Create byte <=> unicode mapping. This is useful for handling
+        # whitespace tokens.
+        byte_list, unicode_list = bytes_to_unicode()
+        self.byte2unicode = create_static_hashtable(
+            byte_list, unicode_list, default=""
+        )
+        self.unicode2byte = create_static_hashtable(
+            unicode_list, byte_list, default=""
+        )
+
+        self.set_vocabulary_and_merges(vocabulary, merges)
+
+    def save_assets(self, dir_path):
+        vocab_path = os.path.join(dir_path, VOCAB_FILENAME)
+        merges_path = os.path.join(dir_path, MERGES_FILENAME)
+        with open(vocab_path, "w", encoding="utf-8") as file:
+            file.write(json.dumps(dict(self.vocabulary)))
+        with open(merges_path, "w", encoding="utf-8") as file:
+            for merge in self.merges:
+                file.write(f"{merge}\n")
+
+    def load_assets(self, dir_path):
+        vocab_path = os.path.join(dir_path, VOCAB_FILENAME)
+        merges_path = os.path.join(dir_path, MERGES_FILENAME)
+        self.set_vocabulary_and_merges(vocab_path, merges_path)
+
+    def set_vocabulary_and_merges(self, vocabulary, merges):
+        """Set the vocabulary and merge rules from data or files."""
+        if vocabulary is None or merges is None:
+            # Clear vocab related state.
+            self.vocabulary = None
+            self.merges = None
+            self.cache = None
+            self.id_to_token_map = None
+            self.token_to_id_map = None
+            self.merge_ranks_lookup_default = None
+            self.merge_ranks = None
+            return
 
         if isinstance(vocabulary, str):
             with open(vocabulary, "r", encoding="utf-8") as f:
@@ -303,7 +350,7 @@ class BytePairTokenizer(tokenizer.Tokenizer):
                 f"`type(vocabulary)={type(vocabulary)}`."
             )
         if isinstance(merges, str):
-            self.merges = [bp.rstrip() for bp in tf.io.gfile.GFile(merges)]
+            self.merges = [bp.rstrip() for bp in open(merges, encoding="utf-8")]
         elif isinstance(merges, Iterable):
             self.merges = list(merges)
         else:
@@ -311,25 +358,14 @@ class BytePairTokenizer(tokenizer.Tokenizer):
                 "Merges must be a file path or a list of merge rules. "
                 f"Received: `type(merges)={type(merges)}`"
             )
-        self.sequence_length = sequence_length
-        self.add_prefix_space = add_prefix_space
-        self.unsplittable_tokens = unsplittable_tokens
-
-        # Create byte <=> unicode mapping. This is useful for handling
-        # whitespace tokens.
-        byte_list, unicode_list = bytes_to_unicode()
-        self.byte2unicode = create_static_hashtable(
-            byte_list, unicode_list, default=""
-        )
-        self.unicode2byte = create_static_hashtable(
-            unicode_list, byte_list, default=""
-        )
 
         self.cache = BytePairTokenizerCache()
-        if unsplittable_tokens:
+        if self.unsplittable_tokens:
             # Put special tokens into cache, so it won't be further split and
             # merged.
-            self.cache.insert(unsplittable_tokens, unsplittable_tokens)
+            self.cache.insert(
+                self.unsplittable_tokens, self.unsplittable_tokens
+            )
 
         # Create mapping between string tokens to int ids, and vice versa.
         byte_pairs = [x[0] for x in self.vocabulary.items()]
@@ -356,10 +392,12 @@ class BytePairTokenizer(tokenizer.Tokenizer):
 
     def get_vocabulary(self) -> List[str]:
         """Get the tokenizer vocabulary as a list of strings tokens."""
+        self._check_vocabulary()
         return self.vocabulary.keys()
 
     def vocabulary_size(self) -> int:
         """Get the size of the tokenizer vocabulary."""
+        self._check_vocabulary()
         return len(self.vocabulary)
 
     def id_to_token(self, id: int) -> str:
@@ -367,6 +405,7 @@ class BytePairTokenizer(tokenizer.Tokenizer):
         # This will be slow, but keep memory usage down compared to building a
         # dict. Assuming the main use case is looking up a few special tokens
         # early in the vocab, this should be fine.
+        self._check_vocabulary()
 
         keys = self.get_vocabulary()
         for token in keys:
@@ -376,24 +415,9 @@ class BytePairTokenizer(tokenizer.Tokenizer):
 
     def token_to_id(self, token: str) -> int:
         """Convert a string token to an integer id."""
+        self._check_vocabulary()
         return self.vocabulary[token]
 
-    def get_config(self):
-        config = super().get_config()
-        config.update(
-            {
-                # Ideally vocabulary and merge list would be saved as plain text
-                # assets in the saved model. We have no good way to support
-                # this currently, so we save the vocabulary in the config.
-                "vocabulary": self.vocabulary,
-                "merges": self.merges,
-                "sequence_length": self.sequence_length,
-                "add_prefix_space": self.add_prefix_space,
-                "unsplittable_tokens": self.unsplittable_tokens,
-            }
-        )
-        return config
-
     @tf.function
     def _bpe_merge_one_step(self, words, mask):
         """Perform one step of byte-pair merge."""
@@ -499,7 +523,16 @@ class BytePairTokenizer(tokenizer.Tokenizer):
         )
         return merged_words
 
+    def _check_vocabulary(self):
+        if self.vocabulary is None:
+            raise ValueError(
+                "No vocabulary has been set for BytePairTokenizer. Make sure "
+                "to pass `vocabulary` and `merges` arguments when creating the "
+                "layer."
+            )
+
     def tokenize(self, inputs):
+        self._check_vocabulary()
         if not isinstance(inputs, (tf.Tensor, tf.RaggedTensor)):
             inputs = tf.convert_to_tensor(inputs)
 
@@ -560,6 +593,7 @@ class BytePairTokenizer(tokenizer.Tokenizer):
         return tokens
 
     def detokenize(self, inputs):
+        self._check_vocabulary()
         inputs, unbatched, _ = convert_to_ragged_batch(inputs)
         inputs = tf.cast(inputs, self.dtype)
         unicode_text = tf.strings.reduce_join(
@@ -592,6 +626,17 @@ class BytePairTokenizer(tokenizer.Tokenizer):
         )
         self.cache.insert(tokens, tokenized_words)
 
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "sequence_length": self.sequence_length,
+                "add_prefix_space": self.add_prefix_space,
+                "unsplittable_tokens": self.unsplittable_tokens,
+            }
+        )
+        return config
+
     @classproperty
     def presets(cls):
         return {}
@@ -619,42 +664,19 @@ class BytePairTokenizer(tokenizer.Tokenizer):
         tokenizer.detokenize([5, 6, 7, 8, 9])
         ```
         """
-
-        if not cls.presets:
-            raise NotImplementedError(
-                "No presets have been created for this class"
-            )
-
-        if preset not in cls.presets:
-            raise ValueError(
-                "`preset` must be one of "
-                f"""{", ".join(cls.presets)}. Received: {preset}."""
-            )
-        metadata = cls.presets[preset]
-
-        vocabulary = keras.utils.get_file(
-            "vocab.json",
-            metadata["vocabulary_url"],
-            cache_subdir=os.path.join("models", preset),
-            file_hash=metadata["vocabulary_hash"],
-        )
-        merges = keras.utils.get_file(
-            "merges.txt",
-            metadata["merges_url"],
-            cache_subdir=os.path.join("models", preset),
-            file_hash=metadata["merges_hash"],
-        )
-
-        config = metadata["preprocessor_config"]
-        config.update(
-            {
-                "vocabulary": vocabulary,
-                "merges": merges,
-            },
+        # We support short IDs for official presets, e.g. `"bert_base_en"`.
+        # Map these to a Kaggle Models handle.
+        if preset in cls.presets:
+            preset = cls.presets[preset]["kaggle_handle"]
+
+        config_file = "tokenizer.json"
+        check_preset_class(preset, cls, config_file=config_file)
+        return load_from_preset(
+            preset,
+            config_file=config_file,
+            config_overrides=kwargs,
         )
 
-        return cls.from_config({**config, **kwargs})
-
     def __init_subclass__(cls, **kwargs):
         # Use __init_subclass__ to setup a correct docstring for from_preset.
         super().__init_subclass__(**kwargs)
diff --git a/keras_nlp/tokenizers/sentence_piece_tokenizer.py b/keras_nlp/tokenizers/sentence_piece_tokenizer.py
index 3a53c12a..ae655ace 100644
--- a/keras_nlp/tokenizers/sentence_piece_tokenizer.py
+++ b/keras_nlp/tokenizers/sentence_piece_tokenizer.py
@@ -20,8 +20,9 @@ from typing import List
 import tensorflow as tf
 
 from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
 from keras_nlp.tokenizers import tokenizer
+from keras_nlp.utils.preset_utils import check_preset_class
+from keras_nlp.utils.preset_utils import load_from_preset
 from keras_nlp.utils.python_utils import classproperty
 from keras_nlp.utils.python_utils import format_docstring
 from keras_nlp.utils.tensor_utils import assert_tf_text_installed
@@ -36,6 +37,9 @@ except ImportError:
     tf_text = None
 
 
+VOCAB_FILENAME = "vocabulary.spm"
+
+
 @keras_nlp_export("keras_nlp.tokenizers.SentencePieceTokenizer")
 class SentencePieceTokenizer(tokenizer.Tokenizer):
     """A SentencePiece tokenizer layer.
@@ -106,7 +110,7 @@ class SentencePieceTokenizer(tokenizer.Tokenizer):
 
     def __init__(
         self,
-        proto,
+        proto=None,
         sequence_length: int = None,
         dtype="int32",
         **kwargs,
@@ -121,6 +125,25 @@ class SentencePieceTokenizer(tokenizer.Tokenizer):
 
         super().__init__(dtype=dtype, **kwargs)
 
+        self.proto = None
+        self.sequence_length = sequence_length
+        self.set_proto(proto)
+
+    def save_assets(self, dir_path):
+        path = os.path.join(dir_path, VOCAB_FILENAME)
+        with open(path, "wb") as file:
+            file.write(self.proto)
+
+    def load_assets(self, dir_path):
+        path = os.path.join(dir_path, VOCAB_FILENAME)
+        self.set_proto(path)
+
+    def set_proto(self, proto):
+        if proto is None:
+            self.proto = None
+            self._sentence_piece = None
+            return
+
         if isinstance(proto, str):
             # A string could be either a filepath, or a base64 encoded byte
             # array (which we need for serialization). We will heuristically
@@ -134,7 +157,7 @@ class SentencePieceTokenizer(tokenizer.Tokenizer):
                 except binascii.Error:
                     pass
             if not is_base64:
-                proto_bytes = tf.io.gfile.GFile(proto, "rb").read()
+                proto_bytes = open(proto, "rb").read()
         elif isinstance(proto, bytes):
             proto_bytes = proto
         else:
@@ -148,18 +171,18 @@ class SentencePieceTokenizer(tokenizer.Tokenizer):
             model=proto_bytes,
             out_type=self.compute_dtype,
         )
-
         # Keras cannot serialize a bytestring, so we base64 encode the model
         # byte array as a string for saving.
-        self.proto = base64.b64encode(proto_bytes).decode("ascii")
-        self.sequence_length = sequence_length
+        self.proto = proto_bytes
 
     def vocabulary_size(self) -> int:
         """Get the size of the tokenizer vocabulary."""
+        self._check_vocabulary()
         return int(self._sentence_piece.vocab_size().numpy())
 
     def get_vocabulary(self) -> List[str]:
         """Get the tokenizer vocabulary."""
+        self._check_vocabulary()
         return tensor_to_list(
             self._sentence_piece.id_to_string(
                 tf.range(int(self._sentence_piece.vocab_size().numpy()))
@@ -168,6 +191,7 @@ class SentencePieceTokenizer(tokenizer.Tokenizer):
 
     def id_to_token(self, id: int) -> str:
         """Convert an integer id to a string token."""
+        self._check_vocabulary()
         if id >= self.vocabulary_size() or id < 0:
             raise ValueError(
                 f"`id` must be in range [0, {self.vocabulary_size() - 1}]. "
@@ -177,28 +201,40 @@ class SentencePieceTokenizer(tokenizer.Tokenizer):
 
     def token_to_id(self, token: str) -> int:
         """Convert a string token to an integer id."""
+        self._check_vocabulary()
         return int(self._sentence_piece.string_to_id(token).numpy())
 
     def get_config(self):
         config = super().get_config()
         config.update(
             {
-                # Ideally the model would be saved as a file asset in
-                # the saved model. We have no good way to support this
-                # currently, so we save the model string in the config.
-                "proto": self.proto,
+                "proto": None,  # Save vocabulary via an asset!
                 "sequence_length": self.sequence_length,
             }
         )
         return config
 
+    def _check_vocabulary(self):
+        if self.proto is None:
+            raise ValueError(
+                "No vocabulary has been set for SentencePieceTokenizer. Make "
+                "sure to pass a `proto` argument when creating the layer."
+            )
+
     def tokenize(self, inputs):
+        self._check_vocabulary()
         if not isinstance(inputs, (tf.Tensor, tf.RaggedTensor)):
             inputs = tf.convert_to_tensor(inputs)
         scalar_input = inputs.shape.rank == 0
         if scalar_input:
             inputs = tf.expand_dims(inputs, 0)
 
+        if self._sentence_piece is None:
+            raise ValueError(
+                "No vocabulary has been set for SentencePieceTokenizer. Make "
+                "sure to pass a `vocabulary` argument when creating the layer."
+            )
+
         tokens = self._sentence_piece.tokenize(inputs)
 
         # Convert to a dense output if `sequence_length` is set.
@@ -215,6 +251,7 @@ class SentencePieceTokenizer(tokenizer.Tokenizer):
         return tokens
 
     def detokenize(self, inputs):
+        self._check_vocabulary()
         inputs, unbatched, _ = convert_to_ragged_batch(inputs)
         outputs = self._sentence_piece.detokenize(inputs)
         if unbatched:
@@ -248,35 +285,19 @@ class SentencePieceTokenizer(tokenizer.Tokenizer):
         tokenizer.detokenize([5, 6, 7, 8, 9])
         ```
         """
-
-        if not cls.presets:
-            raise NotImplementedError(
-                "No presets have been created for this class"
-            )
-
-        if preset not in cls.presets:
-            raise ValueError(
-                "`preset` must be one of "
-                f"""{", ".join(cls.presets)}. Received: {preset}."""
-            )
-        metadata = cls.presets[preset]
-
-        spm_proto = keras.utils.get_file(
-            "vocab.spm",
-            metadata["spm_proto_url"],
-            cache_subdir=os.path.join("models", preset),
-            file_hash=metadata["spm_proto_hash"],
-        )
-
-        config = metadata["preprocessor_config"]
-        config.update(
-            {
-                "proto": spm_proto,
-            },
+        # We support short IDs for official presets, e.g. `"bert_base_en"`.
+        # Map these to a Kaggle Models handle.
+        if preset in cls.presets:
+            preset = cls.presets[preset]["kaggle_handle"]
+
+        config_file = "tokenizer.json"
+        check_preset_class(preset, cls, config_file=config_file)
+        return load_from_preset(
+            preset,
+            config_file=config_file,
+            config_overrides=kwargs,
         )
 
-        return cls.from_config({**config, **kwargs})
-
     def __init_subclass__(cls, **kwargs):
         # Use __init_subclass__ to setup a correct docstring for from_preset.
         super().__init_subclass__(**kwargs)
diff --git a/keras_nlp/tokenizers/word_piece_tokenizer.py b/keras_nlp/tokenizers/word_piece_tokenizer.py
index fe37bebf..75f95689 100644
--- a/keras_nlp/tokenizers/word_piece_tokenizer.py
+++ b/keras_nlp/tokenizers/word_piece_tokenizer.py
@@ -19,8 +19,9 @@ from typing import List
 import tensorflow as tf
 
 from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
 from keras_nlp.tokenizers import tokenizer
+from keras_nlp.utils.preset_utils import check_preset_class
+from keras_nlp.utils.preset_utils import load_from_preset
 from keras_nlp.utils.python_utils import classproperty
 from keras_nlp.utils.python_utils import format_docstring
 from keras_nlp.utils.tensor_utils import assert_tf_text_installed
@@ -33,6 +34,8 @@ try:
 except ImportError:
     tf_text = None
 
+VOCAB_FILENAME = "vocabulary.txt"
+
 # Matches whitespace and control characters.
 WHITESPACE_REGEX = r"|".join(
     [
@@ -312,19 +315,6 @@ class WordPieceTokenizer(tokenizer.Tokenizer):
             )
 
         super().__init__(dtype=dtype, **kwargs)
-
-        if isinstance(vocabulary, str):
-            self.vocabulary = [
-                line.rstrip() for line in tf.io.gfile.GFile(vocabulary)
-            ]
-        elif isinstance(vocabulary, Iterable):
-            # Make a copy.
-            self.vocabulary = list(vocabulary)
-        else:
-            raise ValueError(
-                "Vocabulary must be an file path or list of terms. "
-                f"Received: vocabulary={vocabulary}"
-            )
         if oov_token is None:
             raise ValueError("`oov_token` cannot be None.")
 
@@ -335,8 +325,38 @@ class WordPieceTokenizer(tokenizer.Tokenizer):
         self.split_on_cjk = split_on_cjk
         self.suffix_indicator = suffix_indicator
         self.oov_token = oov_token
+        self.set_vocabulary(vocabulary)
+
+    def save_assets(self, dir_path):
+        path = os.path.join(dir_path, VOCAB_FILENAME)
+        with open(path, "w", encoding="utf-8") as file:
+            for token in self.vocabulary:
+                file.write(f"{token}\n")
+
+    def load_assets(self, dir_path):
+        path = os.path.join(dir_path, VOCAB_FILENAME)
+        self.set_vocabulary(path)
+
+    def set_vocabulary(self, vocabulary):
+        """Set the tokenizer vocabulary to a file or list of strings."""
+        if vocabulary is None:
+            self.vocabulary = None
+            self._fast_word_piece = None
+            return
+
+        if isinstance(vocabulary, str):
+            with open(vocabulary, "r", encoding="utf-8") as file:
+                self.vocabulary = [line.rstrip() for line in file]
+        elif isinstance(vocabulary, Iterable):
+            # Make a defensive copy.
+            self.vocabulary = list(vocabulary)
+        else:
+            raise ValueError(
+                "Vocabulary must be an file path or list of terms. "
+                f"Received: vocabulary={vocabulary}"
+            )
 
-        if oov_token not in self.vocabulary:
+        if self.oov_token not in self.vocabulary:
             raise ValueError(
                 f'Cannot find `oov_token="{self.oov_token}"` in the '
                 "vocabulary.\n"
@@ -348,22 +368,25 @@ class WordPieceTokenizer(tokenizer.Tokenizer):
         self._fast_word_piece = tf_text.FastWordpieceTokenizer(
             vocab=self.vocabulary,
             token_out_type=self.compute_dtype,
-            suffix_indicator=suffix_indicator,
-            unknown_token=oov_token,
+            suffix_indicator=self.suffix_indicator,
+            unknown_token=self.oov_token,
             no_pretokenization=True,
             support_detokenization=True,
         )
 
     def get_vocabulary(self) -> List[str]:
         """Get the tokenizer vocabulary as a list of strings tokens."""
+        self._check_vocabulary()
         return self.vocabulary
 
     def vocabulary_size(self) -> int:
         """Get the size of the tokenizer vocabulary."""
+        self._check_vocabulary()
         return len(self.vocabulary)
 
     def id_to_token(self, id: int) -> str:
         """Convert an integer id to a string token."""
+        self._check_vocabulary()
         if id >= self.vocabulary_size() or id < 0:
             raise ValueError(
                 f"`id` must be in range [0, {self.vocabulary_size() - 1}]. "
@@ -376,16 +399,14 @@ class WordPieceTokenizer(tokenizer.Tokenizer):
         # This will be slow, but keep memory usage down compared to building a
         # . Assuming the main use case is looking up a few special tokens
         # early in the vocab, this should be fine.
+        self._check_vocabulary()
         return self.vocabulary.index(token)
 
     def get_config(self):
         config = super().get_config()
         config.update(
             {
-                # Ideally a vocabulary would be saved as a plain text asset in
-                # the saved model. We have no good way to support this
-                # currently, so we save the vocabulary in the config.
-                "vocabulary": self.vocabulary,
+                "vocabulary": None,  # Save vocabulary via an asset!
                 "sequence_length": self.sequence_length,
                 "lowercase": self.lowercase,
                 "strip_accents": self.strip_accents,
@@ -396,7 +417,15 @@ class WordPieceTokenizer(tokenizer.Tokenizer):
         )
         return config
 
+    def _check_vocabulary(self):
+        if self.vocabulary is None:
+            raise ValueError(
+                "No vocabulary has been set for WordPieceTokenizer. Make sure "
+                "to pass a `vocabulary` argument when creating the layer."
+            )
+
     def tokenize(self, inputs):
+        self._check_vocabulary()
         if not isinstance(inputs, (tf.Tensor, tf.RaggedTensor)):
             inputs = tf.convert_to_tensor(inputs)
 
@@ -429,6 +458,7 @@ class WordPieceTokenizer(tokenizer.Tokenizer):
         return tokens
 
     def detokenize(self, inputs):
+        self._check_vocabulary()
         inputs, unbatched, _ = convert_to_ragged_batch(inputs)
         outputs = self._fast_word_piece.detokenize(inputs)
         if unbatched:
@@ -462,35 +492,19 @@ class WordPieceTokenizer(tokenizer.Tokenizer):
         tokenizer.detokenize([5, 6, 7, 8, 9])
         ```
         """
-
-        if not cls.presets:
-            raise NotImplementedError(
-                "No presets have been created for this class"
-            )
-
-        if preset not in cls.presets:
-            raise ValueError(
-                "`preset` must be one of "
-                f"""{", ".join(cls.presets)}. Received: {preset}."""
-            )
-        metadata = cls.presets[preset]
-
-        vocabulary = keras.utils.get_file(
-            "vocab.txt",
-            metadata["vocabulary_url"],
-            cache_subdir=os.path.join("models", preset),
-            file_hash=metadata["vocabulary_hash"],
+        # We support short IDs for official presets, e.g. `"bert_base_en"`.
+        # Map these to a Kaggle Models handle.
+        if preset in cls.presets:
+            preset = cls.presets[preset]["kaggle_handle"]
+
+        config_file = "tokenizer.json"
+        check_preset_class(preset, cls, config_file=config_file)
+        return load_from_preset(
+            preset,
+            config_file=config_file,
+            config_overrides=kwargs,
         )
 
-        config = metadata["preprocessor_config"]
-        config.update(
-            {
-                "vocabulary": vocabulary,
-            },
-        )
-
-        return cls.from_config({**config, **kwargs})
-
     def __init_subclass__(cls, **kwargs):
         # Use __init_subclass__ to setup a correct docstring for from_preset.
         super().__init_subclass__(**kwargs)
diff --git a/keras_nlp/tokenizers/word_piece_tokenizer_trainer.py b/keras_nlp/tokenizers/word_piece_tokenizer_trainer.py
index 8571097e..dc90075a 100644
--- a/keras_nlp/tokenizers/word_piece_tokenizer_trainer.py
+++ b/keras_nlp/tokenizers/word_piece_tokenizer_trainer.py
@@ -172,7 +172,7 @@ def compute_word_piece_vocabulary(
     if vocabulary_output_file is not None:
         vocab_text = "".join([line + "\n" for line in vocab])
         # Write vocab to file.
-        with open(vocabulary_output_file, "w") as vocab_file:
+        with open(vocabulary_output_file, "w", encoding="utf-8") as vocab_file:
             vocab_file.write(vocab_text)
     else:
         return vocab
diff --git a/keras_nlp/utils/preset_utils.py b/keras_nlp/utils/preset_utils.py
new file mode 100644
index 00000000..2ee1e8dd
--- /dev/null
+++ b/keras_nlp/utils/preset_utils.py
@@ -0,0 +1,215 @@
+# Copyright 2023 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+import json
+import os
+
+from keras_nlp.backend import keras
+
+try:
+    import kagglehub
+except ImportError:
+    kagglehub = None
+
+KAGGLE_PREFIX = "kaggle://"
+GS_PREFIX = "gs://"
+TOKENIZER_ASSET_DIR = "assets/tokenizer"
+
+
+def get_file(preset, path):
+    """Download a preset file in necessary and return the local path."""
+    if not isinstance(preset, str):
+        raise ValueError(
+            f"A preset identifier must be a string. Received: preset={preset}"
+        )
+    if preset.startswith(KAGGLE_PREFIX):
+        if kagglehub is None:
+            raise ImportError(
+                "`from_preset()` requires the `kagglehub` package. "
+                "Please install with `pip install kagglehub`."
+            )
+        segments = preset.removeprefix(KAGGLE_PREFIX).split("/")
+        # Insert the kaggle framework into the handle.
+        if len(segments) == 3:
+            org, model, variant = segments
+            kaggle_handle = f"{org}/{model}/keras/{variant}"
+        elif len(segments) == 4:
+            org, model, variant, version = segments
+            kaggle_handle = f"{org}/{model}/keras/{variant}/{version}"
+        else:
+            raise ValueError(
+                "Unexpected kaggle preset handle. Kaggle model handles should "
+                "have the form kaggle://{org}/{model}/{variant}[/{version}]. "
+                "For example, 'kaggle://keras/bert/bert_base_en'. "
+                f"Received: preset={preset}"
+            )
+        return kagglehub.model_download(kaggle_handle, path)
+    elif preset.startswith(GS_PREFIX):
+        url = os.path.join(preset, path)
+        url = url.replace(GS_PREFIX, "https://storage.googleapis.com/")
+        subdir = preset.replace(GS_PREFIX, "gs_")
+        subdir = subdir.replace("/", "_").replace("-", "_")
+        filename = os.path.basename(path)
+        subdir = os.path.join(subdir, os.path.dirname(path))
+        return keras.utils.get_file(
+            filename,
+            url,
+            cache_subdir=os.path.join("models", subdir),
+        )
+    elif os.path.exists(preset):
+        # Assume a local filepath.
+        return os.path.join(preset, path)
+    else:
+        raise ValueError(
+            "Unknown preset identifier. A preset must be a one of:\n"
+            "1) a built in preset identifier like `'bert_base_en'`\n"
+            "2) a Kaggle Models handle like `'kaggle://keras/bert/bert_base_en'`\n"
+            "3) a path to a local preset directory like `'./bert_base_en`\n"
+            "Use `print(cls.presets.keys())` to view all built-in presets for "
+            "API symbol `cls`.\n"
+            f"Received: preset='{preset}'"
+        )
+
+
+def get_tokenizer(layer):
+    """Get the tokenizer from any KerasNLP model or layer."""
+    # Avoid circular import.
+    from keras_nlp.tokenizers.tokenizer import Tokenizer
+
+    if isinstance(layer, Tokenizer):
+        return layer
+    if hasattr(layer, "tokenizer"):
+        return layer.tokenizer
+    if hasattr(layer, "preprocessor"):
+        return getattr(layer.preprocessor, "tokenizer", None)
+    return None
+
+
+def recursive_pop(config, key):
+    """Remove a key from a nested config object"""
+    config.pop(key, None)
+    for value in config.values():
+        if isinstance(value, dict):
+            recursive_pop(value, key)
+
+
+def save_to_preset(
+    layer,
+    preset,
+    save_weights=True,
+    config_filename="config.json",
+    weights_filename="model.weights.h5",
+):
+    """Save a KerasNLP layer to a preset directory."""
+    os.makedirs(preset, exist_ok=True)
+
+    # Save tokenizers assets.
+    tokenizer = get_tokenizer(layer)
+    assets = []
+    if tokenizer:
+        asset_dir = os.path.join(preset, TOKENIZER_ASSET_DIR)
+        os.makedirs(asset_dir, exist_ok=True)
+        tokenizer.save_assets(asset_dir)
+        for asset_path in os.listdir(asset_dir):
+            assets.append(os.path.join(TOKENIZER_ASSET_DIR, asset_path))
+
+    # Optionally save weights.
+    save_weights = save_weights and hasattr(layer, "save_weights")
+    if save_weights:
+        weights_path = os.path.join(preset, weights_filename)
+        layer.save_weights(weights_path)
+
+    # Save a serialized Keras object.
+    config_path = os.path.join(preset, config_filename)
+    config = keras.saving.serialize_keras_object(layer)
+    # Include references to weights and assets.
+    config["assets"] = assets
+    config["weights"] = weights_filename if save_weights else None
+    recursive_pop(config, "compile_config")
+    recursive_pop(config, "build_config")
+    with open(config_path, "w") as config_file:
+        config_file.write(json.dumps(config, indent=4))
+
+    from keras_nlp import __version__ as keras_nlp_version
+
+    keras_version = keras.version() if hasattr(keras, "version") else None
+
+    # Save any associated metadata.
+    if config_filename == "config.json":
+        metadata = {
+            "keras_version": keras_version,
+            "keras_nlp_version": keras_nlp_version,
+            "parameter_count": layer.count_params(),
+            "date_saved": datetime.datetime.now().strftime("%Y-%m-%d@%H:%M:%S"),
+        }
+        metadata_path = os.path.join(preset, "metadata.json")
+        with open(metadata_path, "w") as metadata_file:
+            metadata_file.write(json.dumps(metadata, indent=4))
+
+
+def load_from_preset(
+    preset,
+    load_weights=True,
+    config_file="config.json",
+    config_overrides={},
+):
+    """Load a KerasNLP layer to a preset directory."""
+    # Load a serialized Keras object.
+    config_path = get_file(preset, config_file)
+    with open(config_path) as config_file:
+        config = json.load(config_file)
+    config["config"] = {**config["config"], **config_overrides}
+    layer = keras.saving.deserialize_keras_object(config)
+
+    # Load any assets for our tokenizers.
+    tokenizer = get_tokenizer(layer)
+    if tokenizer and config["assets"]:
+        for asset in config["assets"]:
+            get_file(preset, asset)
+        config_dir = os.path.dirname(config_path)
+        asset_dir = os.path.join(config_dir, TOKENIZER_ASSET_DIR)
+        tokenizer.load_assets(asset_dir)
+
+    # Optionally load weights.
+    load_weights = load_weights and config["weights"]
+    if load_weights:
+        weights_path = get_file(preset, config["weights"])
+        layer.load_weights(weights_path)
+
+    return layer
+
+
+def check_preset_class(
+    preset,
+    classes,
+    config_file="config.json",
+):
+    """Validate a preset is being loaded on the correct class."""
+    config_path = get_file(preset, config_file)
+    with open(config_path) as config_file:
+        config = json.load(config_file)
+    cls = keras.saving.get_registered_object(config["registered_name"])
+    if not isinstance(classes, (tuple, list)):
+        classes = (classes,)
+    # Allow subclasses for testing a base class, e.g.
+    # `check_preset_class(preset, Backbone)`
+    if not any(issubclass(cls, x) for x in classes):
+        raise ValueError(
+            f"Unexpected class in preset `'{preset}'`. "
+            "When calling `from_preset()` on a class object, the preset class "
+            f"much match allowed classes. Allowed classes are `{classes}`. "
+            f"Received: `{cls}`."
+        )
+    return cls
diff --git a/requirements-common.txt b/requirements-common.txt
index 5c9710de..5c2a8a3d 100644
--- a/requirements-common.txt
+++ b/requirements-common.txt
@@ -2,6 +2,7 @@
 dm-tree
 regex
 rich
+kagglehub
 # Tooling deps.
 astor
 packaging
diff --git a/setup.py b/setup.py
index 6acd5416..b246d18e 100644
--- a/setup.py
+++ b/setup.py
@@ -62,6 +62,7 @@ setup(
         "regex",
         "rich",
         "dm-tree",
+        "kagglehub",
         # Don't require tensorflow-text on MacOS, there are no binaries for ARM.
         # Also, we rely on tensorflow *transitively* through tensorflow-text.
         # This avoid a slowdown during `pip install keras-nlp` where pip would
diff --git a/tools/convert_legacy_presets.py b/tools/convert_legacy_presets.py
new file mode 100644
index 00000000..c1470cf6
--- /dev/null
+++ b/tools/convert_legacy_presets.py
@@ -0,0 +1,117 @@
+# Copyright 2023 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This script was used to convert our legacy presets into the directory format
+used by Kaggle.
+
+This script is for reference only.
+"""
+
+import os
+import re
+import shutil
+
+os.environ["KERAS_HOME"] = os.getcwd()
+
+from keras_nlp import models  # noqa: E402
+from keras_nlp.src.utils.preset_utils import save_to_preset  # noqa: E402
+
+BUCKET = "keras-nlp-kaggle"
+
+
+def to_snake_case(name):
+    name = re.sub(r"\W+", "", name)
+    name = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name)
+    name = re.sub("([a-z])([A-Z])", r"\1_\2", name).lower()
+    return name
+
+
+if __name__ == "__main__":
+    backbone_models = [
+        (models.AlbertBackbone, models.AlbertTokenizer),
+        (models.BartBackbone, models.BartTokenizer),
+        (models.BertBackbone, models.BertTokenizer),
+        (models.DebertaV3Backbone, models.DebertaV3Tokenizer),
+        (models.DistilBertBackbone, models.DistilBertTokenizer),
+        (models.FNetBackbone, models.FNetTokenizer),
+        (models.GPT2Backbone, models.GPT2Tokenizer),
+        (models.OPTBackbone, models.OPTTokenizer),
+        (models.RobertaBackbone, models.RobertaTokenizer),
+        (models.T5Backbone, models.T5Tokenizer),
+        (models.WhisperBackbone, models.WhisperTokenizer),
+        (models.XLMRobertaBackbone, models.XLMRobertaTokenizer),
+    ]
+    for backbone_cls, tokenizer_cls in backbone_models:
+        for preset in backbone_cls.presets:
+            backbone = backbone_cls.from_preset(
+                preset, name=to_snake_case(backbone_cls.__name__)
+            )
+            tokenizer = tokenizer_cls.from_preset(
+                preset, name=to_snake_case(tokenizer_cls.__name__)
+            )
+            save_to_preset(
+                backbone,
+                preset,
+                config_filename="config.json",
+            )
+            save_to_preset(
+                tokenizer,
+                preset,
+                config_filename="tokenizer.json",
+            )
+            # Delete first to clean up any exising version.
+            os.system(f"gsutil rm -rf gs://{BUCKET}/{preset}")
+            os.system(f"gsutil cp -r {preset} gs://{BUCKET}/{preset}")
+            for root, _, files in os.walk(preset):
+                for file in files:
+                    path = os.path.join(BUCKET, root, file)
+                    os.system(
+                        f"gcloud storage objects update gs://{path} "
+                        "--add-acl-grant=entity=AllUsers,role=READER"
+                    )
+            # Clean up local disk usage.
+            shutil.rmtree("models")
+            shutil.rmtree(preset)
+
+    # Handle our single task model.
+    preset = "bert_tiny_en_uncased_sst2"
+    task = models.BertClassifier.from_preset(
+        preset, name=to_snake_case(models.BertClassifier.__name__)
+    )
+    tokenizer = models.BertTokenizer.from_preset(
+        preset, name=to_snake_case(models.BertTokenizer.__name__)
+    )
+    save_to_preset(
+        task,
+        preset,
+        config_filename="config.json",
+    )
+    save_to_preset(
+        tokenizer,
+        preset,
+        config_filename="tokenizer.json",
+    )
+    # Delete first to clean up any exising version.
+    os.system(f"gsutil rm -rf gs://{BUCKET}/{preset}")
+    os.system(f"gsutil cp -r {preset} gs://{BUCKET}/{preset}")
+    for root, _, files in os.walk(preset):
+        for file in files:
+            path = os.path.join(BUCKET, root, file)
+            os.system(
+                f"gcloud storage objects update gs://{path} "
+                "--add-acl-grant=entity=AllUsers,role=READER"
+            )
+    # Clean up local disk usage.
+    shutil.rmtree("models")
+    shutil.rmtree(preset)


Test patch:
diff --git a/keras_nlp/layers/modeling/token_and_position_embedding_test.py b/keras_nlp/layers/modeling/token_and_position_embedding_test.py
index 16269b5d..122d74e1 100644
--- a/keras_nlp/layers/modeling/token_and_position_embedding_test.py
+++ b/keras_nlp/layers/modeling/token_and_position_embedding_test.py
@@ -14,7 +14,6 @@
 
 import numpy as np
 
-from keras_nlp.backend import keras
 from keras_nlp.backend import ops
 from keras_nlp.backend import random
 from keras_nlp.layers.modeling.token_and_position_embedding import (
@@ -31,7 +30,7 @@ class TokenAndPositionEmbeddingTest(TestCase):
                 "vocabulary_size": 5,
                 "sequence_length": 4,
                 "embedding_dim": 3,
-                "embeddings_initializer": keras.initializers.Constant(1.0),
+                "embeddings_initializer": "ones",
             },
             input_data=random.randint(minval=0, maxval=5, shape=(2, 4)),
             expected_output_shape=(2, 4, 3),
diff --git a/keras_nlp/models/albert/albert_classifier_test.py b/keras_nlp/models/albert/albert_classifier_test.py
index ebf8a630..5a60ff99 100644
--- a/keras_nlp/models/albert/albert_classifier_test.py
+++ b/keras_nlp/models/albert/albert_classifier_test.py
@@ -42,7 +42,7 @@ class AlbertClassifierTest(TestCase):
             hidden_dim=2,
             embedding_dim=2,
             intermediate_dim=4,
-            max_sequence_length=self.preprocessor.packer.sequence_length,
+            max_sequence_length=self.preprocessor.sequence_length,
         )
         self.init_kwargs = {
             "preprocessor": self.preprocessor,
diff --git a/keras_nlp/models/albert/albert_masked_lm_test.py b/keras_nlp/models/albert/albert_masked_lm_test.py
index f992ed2b..49e6a595 100644
--- a/keras_nlp/models/albert/albert_masked_lm_test.py
+++ b/keras_nlp/models/albert/albert_masked_lm_test.py
@@ -50,7 +50,7 @@ class AlbertMaskedLMTest(TestCase):
             hidden_dim=2,
             embedding_dim=2,
             intermediate_dim=4,
-            max_sequence_length=self.preprocessor.packer.sequence_length,
+            max_sequence_length=self.preprocessor.sequence_length,
         )
         self.init_kwargs = {
             "preprocessor": self.preprocessor,
diff --git a/keras_nlp/models/bart/bart_seq_2_seq_lm_preprocessor_test.py b/keras_nlp/models/bart/bart_seq_2_seq_lm_preprocessor_test.py
index f67dab70..33fbd5fc 100644
--- a/keras_nlp/models/bart/bart_seq_2_seq_lm_preprocessor_test.py
+++ b/keras_nlp/models/bart/bart_seq_2_seq_lm_preprocessor_test.py
@@ -21,7 +21,7 @@ from keras_nlp.models.bart.bart_tokenizer import BartTokenizer
 from keras_nlp.tests.test_case import TestCase
 
 
-class BartPreprocessorTest(TestCase):
+class BartSeq2SeqLMPreprocessorTest(TestCase):
     def setUp(self):
         self.vocab = ["
", "<pad>", "
", "air", "Ġair", "plane", "Ġat"]
         self.vocab += ["port", "<mask>"]
diff --git a/keras_nlp/models/bert/bert_classifier_test.py b/keras_nlp/models/bert/bert_classifier_test.py
index d5a767d3..53da9d4f 100644
--- a/keras_nlp/models/bert/bert_classifier_test.py
+++ b/keras_nlp/models/bert/bert_classifier_test.py
@@ -36,7 +36,7 @@ class BertClassifierTest(TestCase):
             num_heads=2,
             hidden_dim=2,
             intermediate_dim=4,
-            max_sequence_length=self.preprocessor.packer.sequence_length,
+            max_sequence_length=self.preprocessor.sequence_length,
         )
         self.init_kwargs = {
             "preprocessor": self.preprocessor,
diff --git a/keras_nlp/models/bert/bert_masked_lm_test.py b/keras_nlp/models/bert/bert_masked_lm_test.py
index 0bad92a4..dd6f41b0 100644
--- a/keras_nlp/models/bert/bert_masked_lm_test.py
+++ b/keras_nlp/models/bert/bert_masked_lm_test.py
@@ -43,7 +43,7 @@ class BertMaskedLMTest(TestCase):
             num_heads=2,
             hidden_dim=2,
             intermediate_dim=4,
-            max_sequence_length=self.preprocessor.packer.sequence_length,
+            max_sequence_length=self.preprocessor.sequence_length,
         )
         self.init_kwargs = {
             "preprocessor": self.preprocessor,
diff --git a/keras_nlp/models/deberta_v3/deberta_v3_classifier_test.py b/keras_nlp/models/deberta_v3/deberta_v3_classifier_test.py
index 0e0ab764..7d4f61e0 100644
--- a/keras_nlp/models/deberta_v3/deberta_v3_classifier_test.py
+++ b/keras_nlp/models/deberta_v3/deberta_v3_classifier_test.py
@@ -45,7 +45,7 @@ class DebertaV3ClassifierTest(TestCase):
             num_heads=2,
             hidden_dim=2,
             intermediate_dim=4,
-            max_sequence_length=self.preprocessor.packer.sequence_length,
+            max_sequence_length=self.preprocessor.sequence_length,
         )
         self.init_kwargs = {
             "preprocessor": self.preprocessor,
diff --git a/keras_nlp/models/deberta_v3/deberta_v3_masked_lm_test.py b/keras_nlp/models/deberta_v3/deberta_v3_masked_lm_test.py
index 32bf71de..b103f390 100644
--- a/keras_nlp/models/deberta_v3/deberta_v3_masked_lm_test.py
+++ b/keras_nlp/models/deberta_v3/deberta_v3_masked_lm_test.py
@@ -48,7 +48,7 @@ class DebertaV3MaskedLMTest(TestCase):
             num_heads=2,
             hidden_dim=2,
             intermediate_dim=4,
-            max_sequence_length=self.preprocessor.packer.sequence_length,
+            max_sequence_length=self.preprocessor.sequence_length,
         )
         self.init_kwargs = {
             "preprocessor": self.preprocessor,
diff --git a/keras_nlp/models/distil_bert/distil_bert_classifier_test.py b/keras_nlp/models/distil_bert/distil_bert_classifier_test.py
index 782cf765..d25f1768 100644
--- a/keras_nlp/models/distil_bert/distil_bert_classifier_test.py
+++ b/keras_nlp/models/distil_bert/distil_bert_classifier_test.py
@@ -42,7 +42,7 @@ class DistilBertClassifierTest(TestCase):
             num_heads=2,
             hidden_dim=2,
             intermediate_dim=4,
-            max_sequence_length=self.preprocessor.packer.sequence_length,
+            max_sequence_length=self.preprocessor.sequence_length,
         )
         self.init_kwargs = {
             "preprocessor": self.preprocessor,
diff --git a/keras_nlp/models/distil_bert/distil_bert_masked_lm_test.py b/keras_nlp/models/distil_bert/distil_bert_masked_lm_test.py
index 52e846f4..4aa8327a 100644
--- a/keras_nlp/models/distil_bert/distil_bert_masked_lm_test.py
+++ b/keras_nlp/models/distil_bert/distil_bert_masked_lm_test.py
@@ -47,7 +47,7 @@ class DistilBertMaskedLMTest(TestCase):
             num_heads=2,
             hidden_dim=2,
             intermediate_dim=4,
-            max_sequence_length=self.preprocessor.packer.sequence_length,
+            max_sequence_length=self.preprocessor.sequence_length,
         )
         self.init_kwargs = {
             "preprocessor": self.preprocessor,
diff --git a/keras_nlp/models/f_net/f_net_classifier_test.py b/keras_nlp/models/f_net/f_net_classifier_test.py
index c871fbcc..4defce4a 100644
--- a/keras_nlp/models/f_net/f_net_classifier_test.py
+++ b/keras_nlp/models/f_net/f_net_classifier_test.py
@@ -40,7 +40,7 @@ class FNetClassifierTest(TestCase):
             num_layers=2,
             hidden_dim=2,
             intermediate_dim=4,
-            max_sequence_length=self.preprocessor.packer.sequence_length,
+            max_sequence_length=self.preprocessor.sequence_length,
         )
         self.init_kwargs = {
             "preprocessor": self.preprocessor,
diff --git a/keras_nlp/models/f_net/f_net_masked_lm_test.py b/keras_nlp/models/f_net/f_net_masked_lm_test.py
index b4931a76..1db6b361 100644
--- a/keras_nlp/models/f_net/f_net_masked_lm_test.py
+++ b/keras_nlp/models/f_net/f_net_masked_lm_test.py
@@ -47,7 +47,7 @@ class FNetMaskedLMTest(TestCase):
             num_layers=2,
             hidden_dim=2,
             intermediate_dim=4,
-            max_sequence_length=self.preprocessor.packer.sequence_length,
+            max_sequence_length=self.preprocessor.sequence_length,
         )
         self.init_kwargs = {
             "preprocessor": self.preprocessor,
diff --git a/keras_nlp/models/gpt2/gpt2_causal_lm_test.py b/keras_nlp/models/gpt2/gpt2_causal_lm_test.py
index 7ce93150..f34b6baa 100644
--- a/keras_nlp/models/gpt2/gpt2_causal_lm_test.py
+++ b/keras_nlp/models/gpt2/gpt2_causal_lm_test.py
@@ -44,7 +44,7 @@ class GPT2CausalLMTest(TestCase):
             num_heads=2,
             hidden_dim=4,
             intermediate_dim=8,
-            max_sequence_length=self.preprocessor.packer.sequence_length,
+            max_sequence_length=self.preprocessor.sequence_length,
         )
         self.init_kwargs = {
             "preprocessor": self.preprocessor,
diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_test.py b/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_test.py
index 6857d5f4..c8839c8b 100644
--- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_test.py
+++ b/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_test.py
@@ -44,7 +44,7 @@ class GPTNeoXCausalLMTest(TestCase):
             num_heads=2,
             hidden_dim=4,
             intermediate_dim=8,
-            max_sequence_length=self.preprocessor.packer.sequence_length,
+            max_sequence_length=self.preprocessor.sequence_length,
         )
         self.init_kwargs = {
             "preprocessor": self.preprocessor,
diff --git a/keras_nlp/models/opt/opt_causal_lm_test.py b/keras_nlp/models/opt/opt_causal_lm_test.py
index e6e707e7..3ba27178 100644
--- a/keras_nlp/models/opt/opt_causal_lm_test.py
+++ b/keras_nlp/models/opt/opt_causal_lm_test.py
@@ -43,7 +43,7 @@ class OPTCausalLMTest(TestCase):
             num_heads=2,
             hidden_dim=4,
             intermediate_dim=8,
-            max_sequence_length=self.preprocessor.packer.sequence_length,
+            max_sequence_length=self.preprocessor.sequence_length,
         )
         self.init_kwargs = {
             "preprocessor": self.preprocessor,
diff --git a/keras_nlp/models/roberta/roberta_classifier_test.py b/keras_nlp/models/roberta/roberta_classifier_test.py
index 04c054f4..e85d2a37 100644
--- a/keras_nlp/models/roberta/roberta_classifier_test.py
+++ b/keras_nlp/models/roberta/roberta_classifier_test.py
@@ -40,7 +40,7 @@ class RobertaClassifierTest(TestCase):
             num_heads=2,
             hidden_dim=2,
             intermediate_dim=4,
-            max_sequence_length=self.preprocessor.packer.sequence_length,
+            max_sequence_length=self.preprocessor.sequence_length,
         )
         self.init_kwargs = {
             "preprocessor": self.preprocessor,
diff --git a/keras_nlp/models/roberta/roberta_masked_lm_test.py b/keras_nlp/models/roberta/roberta_masked_lm_test.py
index 663b5a5b..f4e410fa 100644
--- a/keras_nlp/models/roberta/roberta_masked_lm_test.py
+++ b/keras_nlp/models/roberta/roberta_masked_lm_test.py
@@ -47,7 +47,7 @@ class RobertaMaskedLMTest(TestCase):
             num_heads=2,
             hidden_dim=2,
             intermediate_dim=4,
-            max_sequence_length=self.preprocessor.packer.sequence_length,
+            max_sequence_length=self.preprocessor.sequence_length,
         )
         self.init_kwargs = {
             "preprocessor": self.preprocessor,
diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_classifier_test.py b/keras_nlp/models/xlm_roberta/xlm_roberta_classifier_test.py
index 8255a40c..bfa65002 100644
--- a/keras_nlp/models/xlm_roberta/xlm_roberta_classifier_test.py
+++ b/keras_nlp/models/xlm_roberta/xlm_roberta_classifier_test.py
@@ -47,7 +47,7 @@ class XLMRobertaClassifierTest(TestCase):
             num_heads=2,
             hidden_dim=2,
             intermediate_dim=4,
-            max_sequence_length=self.preprocessor.packer.sequence_length,
+            max_sequence_length=self.preprocessor.sequence_length,
         )
         self.init_kwargs = {
             "preprocessor": self.preprocessor,
diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm_test.py b/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm_test.py
index bcbafe4a..d9a1ce68 100644
--- a/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm_test.py
+++ b/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm_test.py
@@ -52,7 +52,7 @@ class XLMRobertaMaskedLMTest(TestCase):
             num_heads=2,
             hidden_dim=2,
             intermediate_dim=4,
-            max_sequence_length=self.preprocessor.packer.sequence_length,
+            max_sequence_length=self.preprocessor.sequence_length,
         )
         self.init_kwargs = {
             "preprocessor": self.preprocessor,
diff --git a/keras_nlp/tests/test_case.py b/keras_nlp/tests/test_case.py
index 7797ca1c..455a8569 100644
--- a/keras_nlp/tests/test_case.py
+++ b/keras_nlp/tests/test_case.py
@@ -244,6 +244,11 @@ class TestCase(tf.test.TestCase, parameterized.TestCase):
         """Check idempotency of serialize/deserialize.
 
         Not this is a much faster test than saving."""
+        run_dir_test = True
+        # Tokenizers will not initialize the tensorflow trackable system after
+        # clone, leading to some weird errors here.
+        if config.backend() == "tensorflow" and isinstance(instance, Tokenizer):
+            run_dir_test = False
         # get_config roundtrip
         cls = instance.__class__
         cfg = instance.get_config()
@@ -253,9 +258,8 @@ class TestCase(tf.test.TestCase, parameterized.TestCase):
         revived_cfg = revived_instance.get_config()
         revived_cfg_json = json.dumps(revived_cfg, sort_keys=True, indent=4)
         self.assertEqual(cfg_json, revived_cfg_json)
-        # Dir tests only work with Keras 3.
-        if config.keras_3():
-            self.assertEqual(ref_dir, dir(revived_instance))
+        if run_dir_test:
+            self.assertEqual(set(ref_dir), set(dir(revived_instance)))
 
         # serialization roundtrip
         serialized = keras.saving.serialize_keras_object(instance)
@@ -266,13 +270,12 @@ class TestCase(tf.test.TestCase, parameterized.TestCase):
         revived_cfg = revived_instance.get_config()
         revived_cfg_json = json.dumps(revived_cfg, sort_keys=True, indent=4)
         self.assertEqual(cfg_json, revived_cfg_json)
-        # Dir tests only work with Keras 3.
-        if config.keras_3():
+        if run_dir_test:
             new_dir = dir(revived_instance)[:]
             for lst in [ref_dir, new_dir]:
                 if "__annotations__" in lst:
                     lst.remove("__annotations__")
-            self.assertEqual(ref_dir, new_dir)
+            self.assertEqual(set(ref_dir), set(new_dir))
 
     def run_model_saving_test(
         self,
@@ -394,7 +397,7 @@ class TestCase(tf.test.TestCase, parameterized.TestCase):
         """Run instantiation and a forward pass for a preset."""
         self.assertRegex(cls.from_preset.__doc__, preset)
 
-        with self.assertRaises(ValueError):
+        with self.assertRaises(Exception):
             cls.from_preset("clowntown", **init_kwargs)
 
         instance = cls.from_preset(preset, **init_kwargs)
diff --git a/keras_nlp/tokenizers/byte_pair_tokenizer_test.py b/keras_nlp/tokenizers/byte_pair_tokenizer_test.py
index d5f7b376..00f8f9b8 100644
--- a/keras_nlp/tokenizers/byte_pair_tokenizer_test.py
+++ b/keras_nlp/tokenizers/byte_pair_tokenizer_test.py
@@ -164,6 +164,9 @@ class BytePairTokenizerTest(TestCase):
         cloned_tokenizer = BytePairTokenizer.from_config(
             self.tokenizer.get_config()
         )
+        cloned_tokenizer.set_vocabulary_and_merges(
+            self.tokenizer.vocabulary, self.tokenizer.merges
+        )
         self.assertAllEqual(
             self.tokenizer(input_data),
             cloned_tokenizer(input_data),
diff --git a/keras_nlp/tokenizers/sentence_piece_tokenizer_test.py b/keras_nlp/tokenizers/sentence_piece_tokenizer_test.py
index f3b39711..74477cdf 100644
--- a/keras_nlp/tokenizers/sentence_piece_tokenizer_test.py
+++ b/keras_nlp/tokenizers/sentence_piece_tokenizer_test.py
@@ -161,6 +161,7 @@ class SentencePieceTokenizerTest(TestCase):
         cloned_tokenizer = SentencePieceTokenizer.from_config(
             original_tokenizer.get_config()
         )
+        cloned_tokenizer.set_proto(original_tokenizer.proto)
         self.assertAllEqual(
             original_tokenizer(input_data),
             cloned_tokenizer(input_data),
diff --git a/keras_nlp/tokenizers/word_piece_tokenizer_test.py b/keras_nlp/tokenizers/word_piece_tokenizer_test.py
index 7ba691c5..ead098c3 100644
--- a/keras_nlp/tokenizers/word_piece_tokenizer_test.py
+++ b/keras_nlp/tokenizers/word_piece_tokenizer_test.py
@@ -190,6 +190,7 @@ class WordPieceTokenizerTest(TestCase):
         cloned_tokenizer = WordPieceTokenizer.from_config(
             original_tokenizer.get_config()
         )
+        cloned_tokenizer.set_vocabulary(original_tokenizer.get_vocabulary())
         self.assertAllEqual(
             original_tokenizer(input_data),
             cloned_tokenizer(input_data),
diff --git a/keras_nlp/tokenizers/word_piece_tokenizer_trainer_test.py b/keras_nlp/tokenizers/word_piece_tokenizer_trainer_test.py
index 62d0f4ad..03186944 100644
--- a/keras_nlp/tokenizers/word_piece_tokenizer_trainer_test.py
+++ b/keras_nlp/tokenizers/word_piece_tokenizer_trainer_test.py
@@ -177,7 +177,7 @@ class WordPieceTokenizerTrainerTest(TestCase):
             reserved_tokens=[],
         )
         vocab_from_file = []
-        with open(vocab_file, "r") as f:
+        with open(vocab_file, "r", encoding="utf-8") as f:
             for line in f:
                 vocab_from_file.append(line.strip())
         self.assertAllEqual(vocab_from_file, test_output)
diff --git a/keras_nlp/utils/preset_utils_test.py b/keras_nlp/utils/preset_utils_test.py
new file mode 100644
index 00000000..44dc39f4
--- /dev/null
+++ b/keras_nlp/utils/preset_utils_test.py
@@ -0,0 +1,107 @@
+# Copyright 2023 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+
+import pytest
+from absl.testing import parameterized
+
+from keras_nlp.models.albert.albert_classifier import AlbertClassifier
+from keras_nlp.models.backbone import Backbone
+from keras_nlp.models.bert.bert_classifier import BertClassifier
+from keras_nlp.models.roberta.roberta_classifier import RobertaClassifier
+from keras_nlp.models.task import Task
+from keras_nlp.tests.test_case import TestCase
+from keras_nlp.utils.preset_utils import check_preset_class
+from keras_nlp.utils.preset_utils import load_from_preset
+from keras_nlp.utils.preset_utils import save_to_preset
+
+
+class PresetUtilsTest(TestCase):
+    @parameterized.parameters(
+        (AlbertClassifier, "albert_base_en_uncased", "sentencepiece"),
+        (RobertaClassifier, "roberta_base_en", "bytepair"),
+        (BertClassifier, "bert_tiny_en_uncased", "wordpiece"),
+    )
+    @pytest.mark.keras_3_only
+    @pytest.mark.large
+    def test_preset_saving(self, cls, preset_name, tokenizer_type):
+        save_dir = self.get_temp_dir()
+        model = cls.from_preset(preset_name, num_classes=2)
+        save_to_preset(model, save_dir)
+
+        if tokenizer_type == "bytepair":
+            vocab_filename = "assets/tokenizer/vocabulary.json"
+            expected_assets = [
+                "assets/tokenizer/vocabulary.json",
+                "assets/tokenizer/merges.txt",
+            ]
+        elif tokenizer_type == "sentencepiece":
+            vocab_filename = "assets/tokenizer/vocabulary.spm"
+            expected_assets = ["assets/tokenizer/vocabulary.spm"]
+        else:
+            vocab_filename = "assets/tokenizer/vocabulary.txt"
+            expected_assets = ["assets/tokenizer/vocabulary.txt"]
+
+        # Check existence of files
+        self.assertTrue(os.path.exists(os.path.join(save_dir, vocab_filename)))
+        self.assertTrue(os.path.exists(os.path.join(save_dir, "config.json")))
+        self.assertTrue(
+            os.path.exists(os.path.join(save_dir, "model.weights.h5"))
+        )
+        self.assertTrue(os.path.exists(os.path.join(save_dir, "metadata.json")))
+
+        # Check the model config (`config.json`)
+        config_json = open(os.path.join(save_dir, "config.json"), "r").read()
+        self.assertTrue(
+            "build_config" not in config_json
+        )  # Test on raw json to include nested keys
+        self.assertTrue(
+            "compile_config" not in config_json
+        )  # Test on raw json to include nested keys
+        config = json.loads(config_json)
+        self.assertEqual(set(config["assets"]), set(expected_assets))
+        self.assertEqual(config["weights"], "model.weights.h5")
+
+        # Try loading the model from preset directory
+        self.assertEqual(cls, check_preset_class(save_dir, cls))
+        self.assertEqual(cls, check_preset_class(save_dir, Task))
+        with self.assertRaises(ValueError):
+            # Preset is a subclass of Task, not Backbone.
+            check_preset_class(save_dir, Backbone)
+
+        # Try loading the model from preset directory
+        restored_model = load_from_preset(save_dir)
+
+        train_data = (
+            ["the quick brown fox.", "the slow brown fox."],  # Features.
+        )
+        model_input_data = model.preprocessor(*train_data)
+        restored_model_input_data = restored_model.preprocessor(*train_data)
+
+        # Check that saved vocab is equal to the original preset vocab
+        self.assertAllClose(model_input_data, restored_model_input_data)
+
+        # Check model outputs
+        self.assertAllEqual(
+            model(model_input_data), restored_model(restored_model_input_data)
+        )
+
+    def test_preset_errors(self):
+        with self.assertRaisesRegex(ValueError, "must be a string"):
+            AlbertClassifier.from_preset(AlbertClassifier)
+
+        with self.assertRaisesRegex(ValueError, "Unknown preset identifier"):
+            AlbertClassifier.from_preset("snaggle://bort/bort/bort")