Repository: keras-team/keras-hub (language: python)

Issue:
API Generation (for code review)
* Code changes required to move sources to `keras_nlp/src` and API in `keras_nlp/api`.

Developers will run `./shell/api_gen.sh` when they make changes that affect Public API exposed via `keras_nlp_export`. Other times, this step can be ignored.

For Code Review purposes only and will not be merged to Master. We will merge #1608 to Master once this is reviewed.


Discussion / hints:

Fix patch:
diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml
index 16916c8fbd..da3c7897f3 100644
--- a/.github/workflows/actions.yml
+++ b/.github/workflows/actions.yml
@@ -104,3 +104,12 @@ jobs:
           pip install --no-deps -e "." --progress-bar off
       - name: Lint
         run: bash shell/lint.sh
+      - name: Check for API changes
+        run: |
+          bash shell/api_gen.sh
+          git status
+          clean=$(git status | grep "nothing to commit")
+          if [ -z "$clean" ]; then
+            echo "Please run shell/api_gen.sh to generate API."
+            exit 1
+          fi
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 394a1fb148..0e60522b83 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -117,13 +117,13 @@ has cuda support.
 conda create -y -n keras-nlp-cpu python=3.10
 conda activate keras-nlp-cpu
 pip install -r requirements.txt  # install deps
-python pip_build.py --install  # install keras-nlp
+pip install -e .  # install keras-nlp
 
 for backend in "jax" "torch" "tensorflow"; do
     conda create -y -n keras-nlp-${backend} python=3.10
     conda activate keras-nlp-${backend}
     pip install -r requirements-${backend}-cuda.txt  # install deps
-    python pip_build.py --install  # install keras-nlp
+    pip install -e .  # install keras-nlp
 done
 ```
 
@@ -158,6 +158,15 @@ This is automatically done if you clone using git inside WSL.
 Note that will not support Windows Shell/PowerShell for any scripts in this
 repository.
 
+## Update Public API
+
+Run API generation script when creating PRs that update `keras_nlp_export`
+public APIs. Add the files changed in `keras_nlp/api` to the same PR.
+
+```
+./shell/api_gen.sh
+```
+
 ## Testing changes
 
 KerasNLP is tested using [PyTest](https://docs.pytest.org/en/6.2.x/).
diff --git a/api_gen.py b/api_gen.py
new file mode 100644
index 0000000000..bb8957d860
--- /dev/null
+++ b/api_gen.py
@@ -0,0 +1,86 @@
+# Copyright 2023 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Script to generate keras_nlp public API in `keras_nlp/api` directory.
+
+Usage:
+
+Run via `./shell/api_gen.sh`.
+It generates API and formats user and generated APIs.
+"""
+
+import os
+import shutil
+
+import namex
+
+package = "keras_nlp"
+
+
+def ignore_files(_, filenames):
+    return [f for f in filenames if f.endswith("_test.py")]
+
+
+def copy_source_to_build_directory(root_path):
+    # Copy sources (`keras_nlp/` directory and setup files) to build dir
+    build_dir = os.path.join(root_path, "tmp_build_dir")
+    if os.path.exists(build_dir):
+        shutil.rmtree(build_dir)
+    os.mkdir(build_dir)
+    shutil.copytree(
+        package, os.path.join(build_dir, package), ignore=ignore_files
+    )
+    return build_dir
+
+
+def export_version_string(api_init_fname):
+    with open(api_init_fname) as f:
+        contents = f.read()
+    with open(api_init_fname, "w") as f:
+        contents += "from keras_nlp.src.version_utils import __version__\n"
+        f.write(contents)
+
+
+def build():
+    # Backup the `keras_nlp/__init__.py` and restore it on error in api gen.
+    root_path = os.path.dirname(os.path.abspath(__file__))
+    code_api_dir = os.path.join(root_path, package, "api")
+    # Create temp build dir
+    build_dir = copy_source_to_build_directory(root_path)
+    build_api_dir = os.path.join(build_dir, package, "api")
+    build_init_fname = os.path.join(build_dir, package, "__init__.py")
+    build_api_init_fname = os.path.join(build_api_dir, "__init__.py")
+    try:
+        os.chdir(build_dir)
+        # Generates `keras_nlp/api` directory.
+        if os.path.exists(build_api_dir):
+            shutil.rmtree(build_api_dir)
+        if os.path.exists(build_init_fname):
+            os.remove(build_init_fname)
+        os.makedirs(build_api_dir)
+        namex.generate_api_files(
+            "keras_nlp", code_directory="src", target_directory="api"
+        )
+        # Add __version__ to keras package
+        export_version_string(build_api_init_fname)
+        # Copy back the keras_nlp/api and keras_nlp/__init__.py from build dir
+        if os.path.exists(code_api_dir):
+            shutil.rmtree(code_api_dir)
+        shutil.copytree(build_api_dir, code_api_dir)
+    finally:
+        # Clean up: remove the build directory (no longer needed)
+        shutil.rmtree(build_dir)
+
+
+if __name__ == "__main__":
+    build()
diff --git a/keras_nlp/__init__.py b/keras_nlp/__init__.py
index 407a4b7a71..a96f712ed4 100644
--- a/keras_nlp/__init__.py
+++ b/keras_nlp/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2021 The KerasNLP Authors
+# Copyright 2023 The KerasNLP Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,21 +11,36 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+"""DO NOT EDIT.
 
-# sentencepiece is segfaulting on tf-nightly if tensorflow is imported first.
-# This is a temporary fix to restore our nightly testing to green, while we look
-# for a longer term solution.
-try:
-    import sentencepiece
-except ImportError:
-    pass
-
-from keras_nlp import layers
-from keras_nlp import metrics
-from keras_nlp import models
-from keras_nlp import samplers
-from keras_nlp import tokenizers
-from keras_nlp import utils
-from keras_nlp.utils.preset_utils import upload_preset
-from keras_nlp.version_utils import __version__
-from keras_nlp.version_utils import version
+This file was autogenerated. Do not edit it by hand,
+since your modifications would be overwritten.
+"""
+
+import os
+
+# Import everything from /api/ into keras.
+from keras_nlp.api import *  # noqa: F403
+from keras_nlp.api import __version__  # Import * ignores names start with "_".
+
+# Add everything in /api/ to the module search path.
+__path__.append(os.path.join(os.path.dirname(__file__), "api"))  # noqa: F405
+
+# Don't pollute namespace.
+del os
+
+
+# Never autocomplete `.src` or `.api` on an imported keras object.
+def __dir__():
+    keys = dict.fromkeys((globals().keys()))
+    keys.pop("src")
+    keys.pop("api")
+    return list(keys)
+
+
+# Don't import `.src` or `.api` during `from keras import *`.
+__all__ = [
+    name
+    for name in globals().keys()
+    if not (name.startswith("_") or name in ("src", "api"))
+]
diff --git a/keras_nlp/api/__init__.py b/keras_nlp/api/__init__.py
new file mode 100644
index 0000000000..0c3aa253e5
--- /dev/null
+++ b/keras_nlp/api/__init__.py
@@ -0,0 +1,27 @@
+# Copyright 2023 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""DO NOT EDIT.
+
+This file was autogenerated. Do not edit it by hand,
+since your modifications would be overwritten.
+"""
+
+from keras_nlp.api import layers
+from keras_nlp.api import metrics
+from keras_nlp.api import models
+from keras_nlp.api import samplers
+from keras_nlp.api import tokenizers
+from keras_nlp.src.utils.preset_utils import upload_preset
+from keras_nlp.src.version_utils import __version__
+from keras_nlp.src.version_utils import version
diff --git a/keras_nlp/api/layers/__init__.py b/keras_nlp/api/layers/__init__.py
new file mode 100644
index 0000000000..d80bd72440
--- /dev/null
+++ b/keras_nlp/api/layers/__init__.py
@@ -0,0 +1,47 @@
+# Copyright 2023 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""DO NOT EDIT.
+
+This file was autogenerated. Do not edit it by hand,
+since your modifications would be overwritten.
+"""
+
+from keras_nlp.src.layers.modeling.alibi_bias import AlibiBias
+from keras_nlp.src.layers.modeling.cached_multi_head_attention import (
+    CachedMultiHeadAttention,
+)
+from keras_nlp.src.layers.modeling.f_net_encoder import FNetEncoder
+from keras_nlp.src.layers.modeling.masked_lm_head import MaskedLMHead
+from keras_nlp.src.layers.modeling.position_embedding import PositionEmbedding
+from keras_nlp.src.layers.modeling.reversible_embedding import (
+    ReversibleEmbedding,
+)
+from keras_nlp.src.layers.modeling.rotary_embedding import RotaryEmbedding
+from keras_nlp.src.layers.modeling.sine_position_encoding import (
+    SinePositionEncoding,
+)
+from keras_nlp.src.layers.modeling.token_and_position_embedding import (
+    TokenAndPositionEmbedding,
+)
+from keras_nlp.src.layers.modeling.transformer_decoder import TransformerDecoder
+from keras_nlp.src.layers.modeling.transformer_encoder import TransformerEncoder
+from keras_nlp.src.layers.preprocessing.masked_lm_mask_generator import (
+    MaskedLMMaskGenerator,
+)
+from keras_nlp.src.layers.preprocessing.multi_segment_packer import (
+    MultiSegmentPacker,
+)
+from keras_nlp.src.layers.preprocessing.random_deletion import RandomDeletion
+from keras_nlp.src.layers.preprocessing.random_swap import RandomSwap
+from keras_nlp.src.layers.preprocessing.start_end_packer import StartEndPacker
diff --git a/keras_nlp/api/metrics/__init__.py b/keras_nlp/api/metrics/__init__.py
new file mode 100644
index 0000000000..f202964e4c
--- /dev/null
+++ b/keras_nlp/api/metrics/__init__.py
@@ -0,0 +1,24 @@
+# Copyright 2023 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""DO NOT EDIT.
+
+This file was autogenerated. Do not edit it by hand,
+since your modifications would be overwritten.
+"""
+
+from keras_nlp.src.metrics.bleu import Bleu
+from keras_nlp.src.metrics.edit_distance import EditDistance
+from keras_nlp.src.metrics.perplexity import Perplexity
+from keras_nlp.src.metrics.rouge_l import RougeL
+from keras_nlp.src.metrics.rouge_n import RougeN
diff --git a/keras_nlp/api/models/__init__.py b/keras_nlp/api/models/__init__.py
new file mode 100644
index 0000000000..5795c85532
--- /dev/null
+++ b/keras_nlp/api/models/__init__.py
@@ -0,0 +1,197 @@
+# Copyright 2023 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""DO NOT EDIT.
+
+This file was autogenerated. Do not edit it by hand,
+since your modifications would be overwritten.
+"""
+
+from keras_nlp.src.models.albert.albert_backbone import AlbertBackbone
+from keras_nlp.src.models.albert.albert_classifier import AlbertClassifier
+from keras_nlp.src.models.albert.albert_masked_lm import AlbertMaskedLM
+from keras_nlp.src.models.albert.albert_masked_lm_preprocessor import (
+    AlbertMaskedLMPreprocessor,
+)
+from keras_nlp.src.models.albert.albert_preprocessor import AlbertPreprocessor
+from keras_nlp.src.models.albert.albert_tokenizer import AlbertTokenizer
+from keras_nlp.src.models.backbone import Backbone
+from keras_nlp.src.models.bart.bart_backbone import BartBackbone
+from keras_nlp.src.models.bart.bart_preprocessor import BartPreprocessor
+from keras_nlp.src.models.bart.bart_seq_2_seq_lm import BartSeq2SeqLM
+from keras_nlp.src.models.bart.bart_seq_2_seq_lm_preprocessor import (
+    BartSeq2SeqLMPreprocessor,
+)
+from keras_nlp.src.models.bart.bart_tokenizer import BartTokenizer
+from keras_nlp.src.models.bert.bert_backbone import BertBackbone
+from keras_nlp.src.models.bert.bert_classifier import BertClassifier
+from keras_nlp.src.models.bert.bert_masked_lm import BertMaskedLM
+from keras_nlp.src.models.bert.bert_masked_lm_preprocessor import (
+    BertMaskedLMPreprocessor,
+)
+from keras_nlp.src.models.bert.bert_preprocessor import BertPreprocessor
+from keras_nlp.src.models.bert.bert_tokenizer import BertTokenizer
+from keras_nlp.src.models.bloom.bloom_backbone import BloomBackbone
+from keras_nlp.src.models.bloom.bloom_causal_lm import BloomCausalLM
+from keras_nlp.src.models.bloom.bloom_causal_lm_preprocessor import (
+    BloomCausalLMPreprocessor,
+)
+from keras_nlp.src.models.bloom.bloom_preprocessor import BloomPreprocessor
+from keras_nlp.src.models.bloom.bloom_tokenizer import BloomTokenizer
+from keras_nlp.src.models.causal_lm import CausalLM
+from keras_nlp.src.models.classifier import Classifier
+from keras_nlp.src.models.deberta_v3.deberta_v3_backbone import (
+    DebertaV3Backbone,
+)
+from keras_nlp.src.models.deberta_v3.deberta_v3_classifier import (
+    DebertaV3Classifier,
+)
+from keras_nlp.src.models.deberta_v3.deberta_v3_masked_lm import (
+    DebertaV3MaskedLM,
+)
+from keras_nlp.src.models.deberta_v3.deberta_v3_masked_lm_preprocessor import (
+    DebertaV3MaskedLMPreprocessor,
+)
+from keras_nlp.src.models.deberta_v3.deberta_v3_preprocessor import (
+    DebertaV3Preprocessor,
+)
+from keras_nlp.src.models.deberta_v3.deberta_v3_tokenizer import (
+    DebertaV3Tokenizer,
+)
+from keras_nlp.src.models.distil_bert.distil_bert_backbone import (
+    DistilBertBackbone,
+)
+from keras_nlp.src.models.distil_bert.distil_bert_classifier import (
+    DistilBertClassifier,
+)
+from keras_nlp.src.models.distil_bert.distil_bert_masked_lm import (
+    DistilBertMaskedLM,
+)
+from keras_nlp.src.models.distil_bert.distil_bert_masked_lm_preprocessor import (
+    DistilBertMaskedLMPreprocessor,
+)
+from keras_nlp.src.models.distil_bert.distil_bert_preprocessor import (
+    DistilBertPreprocessor,
+)
+from keras_nlp.src.models.distil_bert.distil_bert_tokenizer import (
+    DistilBertTokenizer,
+)
+from keras_nlp.src.models.electra.electra_backbone import ElectraBackbone
+from keras_nlp.src.models.electra.electra_preprocessor import (
+    ElectraPreprocessor,
+)
+from keras_nlp.src.models.electra.electra_tokenizer import ElectraTokenizer
+from keras_nlp.src.models.f_net.f_net_backbone import FNetBackbone
+from keras_nlp.src.models.f_net.f_net_classifier import FNetClassifier
+from keras_nlp.src.models.f_net.f_net_masked_lm import FNetMaskedLM
+from keras_nlp.src.models.f_net.f_net_masked_lm_preprocessor import (
+    FNetMaskedLMPreprocessor,
+)
+from keras_nlp.src.models.f_net.f_net_preprocessor import FNetPreprocessor
+from keras_nlp.src.models.f_net.f_net_tokenizer import FNetTokenizer
+from keras_nlp.src.models.falcon.falcon_backbone import FalconBackbone
+from keras_nlp.src.models.falcon.falcon_causal_lm_preprocessor import (
+    FalconCausalLMPreprocessor,
+)
+from keras_nlp.src.models.falcon.falcon_preprocessor import FalconPreprocessor
+from keras_nlp.src.models.falcon.falcon_tokenizer import FalconTokenizer
+from keras_nlp.src.models.gemma.gemma_backbone import GemmaBackbone
+from keras_nlp.src.models.gemma.gemma_causal_lm import GemmaCausalLM
+from keras_nlp.src.models.gemma.gemma_causal_lm_preprocessor import (
+    GemmaCausalLMPreprocessor,
+)
+from keras_nlp.src.models.gemma.gemma_preprocessor import GemmaPreprocessor
+from keras_nlp.src.models.gemma.gemma_tokenizer import GemmaTokenizer
+from keras_nlp.src.models.gpt2.gpt2_backbone import GPT2Backbone
+from keras_nlp.src.models.gpt2.gpt2_causal_lm import GPT2CausalLM
+from keras_nlp.src.models.gpt2.gpt2_causal_lm_preprocessor import (
+    GPT2CausalLMPreprocessor,
+)
+from keras_nlp.src.models.gpt2.gpt2_preprocessor import GPT2Preprocessor
+from keras_nlp.src.models.gpt2.gpt2_tokenizer import GPT2Tokenizer
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_backbone import GPTNeoXBackbone
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_causal_lm import GPTNeoXCausalLM
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_causal_lm_preprocessor import (
+    GPTNeoXCausalLMPreprocessor,
+)
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_preprocessor import (
+    GPTNeoXPreprocessor,
+)
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_tokenizer import GPTNeoXTokenizer
+from keras_nlp.src.models.llama.llama_backbone import LlamaBackbone
+from keras_nlp.src.models.llama.llama_causal_lm import LlamaCausalLM
+from keras_nlp.src.models.llama.llama_causal_lm_preprocessor import (
+    LlamaCausalLMPreprocessor,
+)
+from keras_nlp.src.models.llama.llama_preprocessor import LlamaPreprocessor
+from keras_nlp.src.models.llama.llama_tokenizer import LlamaTokenizer
+from keras_nlp.src.models.masked_lm import MaskedLM
+from keras_nlp.src.models.mistral.mistral_backbone import MistralBackbone
+from keras_nlp.src.models.mistral.mistral_causal_lm import MistralCausalLM
+from keras_nlp.src.models.mistral.mistral_causal_lm_preprocessor import (
+    MistralCausalLMPreprocessor,
+)
+from keras_nlp.src.models.mistral.mistral_preprocessor import (
+    MistralPreprocessor,
+)
+from keras_nlp.src.models.mistral.mistral_tokenizer import MistralTokenizer
+from keras_nlp.src.models.opt.opt_backbone import OPTBackbone
+from keras_nlp.src.models.opt.opt_causal_lm import OPTCausalLM
+from keras_nlp.src.models.opt.opt_causal_lm_preprocessor import (
+    OPTCausalLMPreprocessor,
+)
+from keras_nlp.src.models.opt.opt_preprocessor import OPTPreprocessor
+from keras_nlp.src.models.opt.opt_tokenizer import OPTTokenizer
+from keras_nlp.src.models.preprocessor import Preprocessor
+from keras_nlp.src.models.roberta.roberta_backbone import RobertaBackbone
+from keras_nlp.src.models.roberta.roberta_classifier import RobertaClassifier
+from keras_nlp.src.models.roberta.roberta_masked_lm import RobertaMaskedLM
+from keras_nlp.src.models.roberta.roberta_masked_lm_preprocessor import (
+    RobertaMaskedLMPreprocessor,
+)
+from keras_nlp.src.models.roberta.roberta_preprocessor import (
+    RobertaPreprocessor,
+)
+from keras_nlp.src.models.roberta.roberta_tokenizer import RobertaTokenizer
+from keras_nlp.src.models.seq_2_seq_lm import Seq2SeqLM
+from keras_nlp.src.models.t5.t5_backbone import T5Backbone
+from keras_nlp.src.models.t5.t5_tokenizer import T5Tokenizer
+from keras_nlp.src.models.task import Task
+from keras_nlp.src.models.whisper.whisper_audio_feature_extractor import (
+    WhisperAudioFeatureExtractor,
+)
+from keras_nlp.src.models.whisper.whisper_backbone import WhisperBackbone
+from keras_nlp.src.models.whisper.whisper_preprocessor import (
+    WhisperPreprocessor,
+)
+from keras_nlp.src.models.whisper.whisper_tokenizer import WhisperTokenizer
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_backbone import (
+    XLMRobertaBackbone,
+)
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_classifier import (
+    XLMRobertaClassifier,
+)
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_masked_lm import (
+    XLMRobertaMaskedLM,
+)
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_masked_lm_preprocessor import (
+    XLMRobertaMaskedLMPreprocessor,
+)
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_preprocessor import (
+    XLMRobertaPreprocessor,
+)
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_tokenizer import (
+    XLMRobertaTokenizer,
+)
+from keras_nlp.src.models.xlnet.xlnet_backbone import XLNetBackbone
+from keras_nlp.src.tokenizers.tokenizer import Tokenizer
diff --git a/keras_nlp/api/samplers/__init__.py b/keras_nlp/api/samplers/__init__.py
new file mode 100644
index 0000000000..3aa22ada0a
--- /dev/null
+++ b/keras_nlp/api/samplers/__init__.py
@@ -0,0 +1,29 @@
+# Copyright 2023 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""DO NOT EDIT.
+
+This file was autogenerated. Do not edit it by hand,
+since your modifications would be overwritten.
+"""
+
+from keras_nlp.src.samplers.beam_sampler import BeamSampler
+from keras_nlp.src.samplers.contrastive_sampler import ContrastiveSampler
+from keras_nlp.src.samplers.greedy_sampler import GreedySampler
+from keras_nlp.src.samplers.random_sampler import RandomSampler
+from keras_nlp.src.samplers.sampler import Sampler
+from keras_nlp.src.samplers.serialization import deserialize
+from keras_nlp.src.samplers.serialization import get
+from keras_nlp.src.samplers.serialization import serialize
+from keras_nlp.src.samplers.top_k_sampler import TopKSampler
+from keras_nlp.src.samplers.top_p_sampler import TopPSampler
diff --git a/keras_nlp/api/tokenizers/__init__.py b/keras_nlp/api/tokenizers/__init__.py
new file mode 100644
index 0000000000..90f2e65383
--- /dev/null
+++ b/keras_nlp/api/tokenizers/__init__.py
@@ -0,0 +1,35 @@
+# Copyright 2023 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""DO NOT EDIT.
+
+This file was autogenerated. Do not edit it by hand,
+since your modifications would be overwritten.
+"""
+
+from keras_nlp.src.tokenizers.byte_pair_tokenizer import BytePairTokenizer
+from keras_nlp.src.tokenizers.byte_tokenizer import ByteTokenizer
+from keras_nlp.src.tokenizers.sentence_piece_tokenizer import (
+    SentencePieceTokenizer,
+)
+from keras_nlp.src.tokenizers.sentence_piece_tokenizer_trainer import (
+    compute_sentence_piece_proto,
+)
+from keras_nlp.src.tokenizers.tokenizer import Tokenizer
+from keras_nlp.src.tokenizers.unicode_codepoint_tokenizer import (
+    UnicodeCodepointTokenizer,
+)
+from keras_nlp.src.tokenizers.word_piece_tokenizer import WordPieceTokenizer
+from keras_nlp.src.tokenizers.word_piece_tokenizer_trainer import (
+    compute_word_piece_vocabulary,
+)
diff --git a/keras_nlp/layers/__init__.py b/keras_nlp/layers/__init__.py
deleted file mode 100644
index 595c4eb661..0000000000
--- a/keras_nlp/layers/__init__.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# Copyright 2023 The KerasNLP Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from keras_nlp.layers.modeling.cached_multi_head_attention import (
-    CachedMultiHeadAttention,
-)
-from keras_nlp.layers.modeling.f_net_encoder import FNetEncoder
-from keras_nlp.layers.modeling.masked_lm_head import MaskedLMHead
-from keras_nlp.layers.modeling.position_embedding import PositionEmbedding
-from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding
-from keras_nlp.layers.modeling.rotary_embedding import RotaryEmbedding
-from keras_nlp.layers.modeling.sine_position_encoding import (
-    SinePositionEncoding,
-)
-from keras_nlp.layers.modeling.token_and_position_embedding import (
-    TokenAndPositionEmbedding,
-)
-from keras_nlp.layers.modeling.transformer_decoder import TransformerDecoder
-from keras_nlp.layers.modeling.transformer_encoder import TransformerEncoder
-from keras_nlp.layers.preprocessing.masked_lm_mask_generator import (
-    MaskedLMMaskGenerator,
-)
-from keras_nlp.layers.preprocessing.multi_segment_packer import (
-    MultiSegmentPacker,
-)
-from keras_nlp.layers.preprocessing.random_deletion import RandomDeletion
-from keras_nlp.layers.preprocessing.random_swap import RandomSwap
-from keras_nlp.layers.preprocessing.start_end_packer import StartEndPacker
diff --git a/keras_nlp/models/__init__.py b/keras_nlp/models/__init__.py
deleted file mode 100644
index 4139656fbf..0000000000
--- a/keras_nlp/models/__init__.py
+++ /dev/null
@@ -1,170 +0,0 @@
-# Copyright 2023 The KerasNLP Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from keras_nlp.models.albert.albert_backbone import AlbertBackbone
-from keras_nlp.models.albert.albert_classifier import AlbertClassifier
-from keras_nlp.models.albert.albert_masked_lm import AlbertMaskedLM
-from keras_nlp.models.albert.albert_masked_lm_preprocessor import (
-    AlbertMaskedLMPreprocessor,
-)
-from keras_nlp.models.albert.albert_preprocessor import AlbertPreprocessor
-from keras_nlp.models.albert.albert_tokenizer import AlbertTokenizer
-from keras_nlp.models.backbone import Backbone
-from keras_nlp.models.bart.bart_backbone import BartBackbone
-from keras_nlp.models.bart.bart_preprocessor import BartPreprocessor
-from keras_nlp.models.bart.bart_seq_2_seq_lm import BartSeq2SeqLM
-from keras_nlp.models.bart.bart_seq_2_seq_lm_preprocessor import (
-    BartSeq2SeqLMPreprocessor,
-)
-from keras_nlp.models.bart.bart_tokenizer import BartTokenizer
-from keras_nlp.models.bert.bert_backbone import BertBackbone
-from keras_nlp.models.bert.bert_classifier import BertClassifier
-from keras_nlp.models.bert.bert_masked_lm import BertMaskedLM
-from keras_nlp.models.bert.bert_masked_lm_preprocessor import (
-    BertMaskedLMPreprocessor,
-)
-from keras_nlp.models.bert.bert_preprocessor import BertPreprocessor
-from keras_nlp.models.bert.bert_tokenizer import BertTokenizer
-from keras_nlp.models.bloom.bloom_backbone import BloomBackbone
-from keras_nlp.models.bloom.bloom_causal_lm import BloomCausalLM
-from keras_nlp.models.bloom.bloom_causal_lm_preprocessor import (
-    BloomCausalLMPreprocessor,
-)
-from keras_nlp.models.bloom.bloom_preprocessor import BloomPreprocessor
-from keras_nlp.models.bloom.bloom_tokenizer import BloomTokenizer
-from keras_nlp.models.causal_lm import CausalLM
-from keras_nlp.models.classifier import Classifier
-from keras_nlp.models.deberta_v3.deberta_v3_backbone import DebertaV3Backbone
-from keras_nlp.models.deberta_v3.deberta_v3_classifier import (
-    DebertaV3Classifier,
-)
-from keras_nlp.models.deberta_v3.deberta_v3_masked_lm import DebertaV3MaskedLM
-from keras_nlp.models.deberta_v3.deberta_v3_masked_lm_preprocessor import (
-    DebertaV3MaskedLMPreprocessor,
-)
-from keras_nlp.models.deberta_v3.deberta_v3_preprocessor import (
-    DebertaV3Preprocessor,
-)
-from keras_nlp.models.deberta_v3.deberta_v3_tokenizer import DebertaV3Tokenizer
-from keras_nlp.models.distil_bert.distil_bert_backbone import DistilBertBackbone
-from keras_nlp.models.distil_bert.distil_bert_classifier import (
-    DistilBertClassifier,
-)
-from keras_nlp.models.distil_bert.distil_bert_masked_lm import (
-    DistilBertMaskedLM,
-)
-from keras_nlp.models.distil_bert.distil_bert_masked_lm_preprocessor import (
-    DistilBertMaskedLMPreprocessor,
-)
-from keras_nlp.models.distil_bert.distil_bert_preprocessor import (
-    DistilBertPreprocessor,
-)
-from keras_nlp.models.distil_bert.distil_bert_tokenizer import (
-    DistilBertTokenizer,
-)
-from keras_nlp.models.electra.electra_backbone import ElectraBackbone
-from keras_nlp.models.electra.electra_preprocessor import ElectraPreprocessor
-from keras_nlp.models.electra.electra_tokenizer import ElectraTokenizer
-from keras_nlp.models.f_net.f_net_backbone import FNetBackbone
-from keras_nlp.models.f_net.f_net_classifier import FNetClassifier
-from keras_nlp.models.f_net.f_net_masked_lm import FNetMaskedLM
-from keras_nlp.models.f_net.f_net_masked_lm_preprocessor import (
-    FNetMaskedLMPreprocessor,
-)
-from keras_nlp.models.f_net.f_net_preprocessor import FNetPreprocessor
-from keras_nlp.models.f_net.f_net_tokenizer import FNetTokenizer
-from keras_nlp.models.falcon.falcon_backbone import FalconBackbone
-from keras_nlp.models.falcon.falcon_tokenizer import FalconTokenizer
-from keras_nlp.models.gemma.gemma_backbone import GemmaBackbone
-from keras_nlp.models.gemma.gemma_causal_lm import GemmaCausalLM
-from keras_nlp.models.gemma.gemma_causal_lm_preprocessor import (
-    GemmaCausalLMPreprocessor,
-)
-from keras_nlp.models.gemma.gemma_preprocessor import GemmaPreprocessor
-from keras_nlp.models.gemma.gemma_tokenizer import GemmaTokenizer
-from keras_nlp.models.gpt2.gpt2_backbone import GPT2Backbone
-from keras_nlp.models.gpt2.gpt2_causal_lm import GPT2CausalLM
-from keras_nlp.models.gpt2.gpt2_causal_lm_preprocessor import (
-    GPT2CausalLMPreprocessor,
-)
-from keras_nlp.models.gpt2.gpt2_preprocessor import GPT2Preprocessor
-from keras_nlp.models.gpt2.gpt2_tokenizer import GPT2Tokenizer
-from keras_nlp.models.gpt_neo_x.gpt_neo_x_backbone import GPTNeoXBackbone
-from keras_nlp.models.gpt_neo_x.gpt_neo_x_causal_lm import GPTNeoXCausalLM
-from keras_nlp.models.gpt_neo_x.gpt_neo_x_causal_lm_preprocessor import (
-    GPTNeoXCausalLMPreprocessor,
-)
-from keras_nlp.models.gpt_neo_x.gpt_neo_x_preprocessor import (
-    GPTNeoXPreprocessor,
-)
-from keras_nlp.models.gpt_neo_x.gpt_neo_x_tokenizer import GPTNeoXTokenizer
-from keras_nlp.models.llama.llama_backbone import LlamaBackbone
-from keras_nlp.models.llama.llama_causal_lm import LlamaCausalLM
-from keras_nlp.models.llama.llama_causal_lm_preprocessor import (
-    LlamaCausalLMPreprocessor,
-)
-from keras_nlp.models.llama.llama_preprocessor import LlamaPreprocessor
-from keras_nlp.models.llama.llama_tokenizer import LlamaTokenizer
-from keras_nlp.models.masked_lm import MaskedLM
-from keras_nlp.models.mistral.mistral_backbone import MistralBackbone
-from keras_nlp.models.mistral.mistral_causal_lm import MistralCausalLM
-from keras_nlp.models.mistral.mistral_causal_lm_preprocessor import (
-    MistralCausalLMPreprocessor,
-)
-from keras_nlp.models.mistral.mistral_preprocessor import MistralPreprocessor
-from keras_nlp.models.mistral.mistral_tokenizer import MistralTokenizer
-from keras_nlp.models.opt.opt_backbone import OPTBackbone
-from keras_nlp.models.opt.opt_causal_lm import OPTCausalLM
-from keras_nlp.models.opt.opt_causal_lm_preprocessor import (
-    OPTCausalLMPreprocessor,
-)
-from keras_nlp.models.opt.opt_preprocessor import OPTPreprocessor
-from keras_nlp.models.opt.opt_tokenizer import OPTTokenizer
-from keras_nlp.models.preprocessor import Preprocessor
-from keras_nlp.models.roberta.roberta_backbone import RobertaBackbone
-from keras_nlp.models.roberta.roberta_classifier import RobertaClassifier
-from keras_nlp.models.roberta.roberta_masked_lm import RobertaMaskedLM
-from keras_nlp.models.roberta.roberta_masked_lm_preprocessor import (
-    RobertaMaskedLMPreprocessor,
-)
-from keras_nlp.models.roberta.roberta_preprocessor import RobertaPreprocessor
-from keras_nlp.models.roberta.roberta_tokenizer import RobertaTokenizer
-from keras_nlp.models.seq_2_seq_lm import Seq2SeqLM
-from keras_nlp.models.t5.t5_backbone import T5Backbone
-from keras_nlp.models.t5.t5_tokenizer import T5Tokenizer
-from keras_nlp.models.task import Task
-from keras_nlp.models.whisper.whisper_audio_feature_extractor import (
-    WhisperAudioFeatureExtractor,
-)
-from keras_nlp.models.whisper.whisper_backbone import WhisperBackbone
-from keras_nlp.models.whisper.whisper_preprocessor import WhisperPreprocessor
-from keras_nlp.models.whisper.whisper_tokenizer import WhisperTokenizer
-from keras_nlp.models.xlm_roberta.xlm_roberta_backbone import XLMRobertaBackbone
-from keras_nlp.models.xlm_roberta.xlm_roberta_classifier import (
-    XLMRobertaClassifier,
-)
-from keras_nlp.models.xlm_roberta.xlm_roberta_masked_lm import (
-    XLMRobertaMaskedLM,
-)
-from keras_nlp.models.xlm_roberta.xlm_roberta_masked_lm_preprocessor import (
-    XLMRobertaMaskedLMPreprocessor,
-)
-from keras_nlp.models.xlm_roberta.xlm_roberta_preprocessor import (
-    XLMRobertaPreprocessor,
-)
-from keras_nlp.models.xlm_roberta.xlm_roberta_tokenizer import (
-    XLMRobertaTokenizer,
-)
-from keras_nlp.models.xlnet.xlnet_backbone import XLNetBackbone
-from keras_nlp.tokenizers.tokenizer import Tokenizer
diff --git a/keras_nlp/samplers/__init__.py b/keras_nlp/samplers/__init__.py
deleted file mode 100644
index 79f9d20e8e..0000000000
--- a/keras_nlp/samplers/__init__.py
+++ /dev/null
@@ -1,25 +0,0 @@
-# Copyright 2023 The KerasNLP Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from keras_nlp.backend import keras
-from keras_nlp.samplers.beam_sampler import BeamSampler
-from keras_nlp.samplers.contrastive_sampler import ContrastiveSampler
-from keras_nlp.samplers.greedy_sampler import GreedySampler
-from keras_nlp.samplers.random_sampler import RandomSampler
-from keras_nlp.samplers.sampler import Sampler
-from keras_nlp.samplers.serialization import deserialize
-from keras_nlp.samplers.serialization import get
-from keras_nlp.samplers.serialization import serialize
-from keras_nlp.samplers.top_k_sampler import TopKSampler
-from keras_nlp.samplers.top_p_sampler import TopPSampler
diff --git a/keras_nlp/src/__init__.py b/keras_nlp/src/__init__.py
new file mode 100644
index 0000000000..29cb010422
--- /dev/null
+++ b/keras_nlp/src/__init__.py
@@ -0,0 +1,31 @@
+# Copyright 2021 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# sentencepiece is segfaulting on tf-nightly if tensorflow is imported first.
+# This is a temporary fix to restore our nightly testing to green, while we look
+# for a longer term solution.
+try:
+    import sentencepiece
+except ImportError:
+    pass
+
+from keras_nlp.src import layers
+from keras_nlp.src import metrics
+from keras_nlp.src import models
+from keras_nlp.src import samplers
+from keras_nlp.src import tokenizers
+from keras_nlp.src import utils
+from keras_nlp.src.utils.preset_utils import upload_preset
+from keras_nlp.src.version_utils import __version__
+from keras_nlp.src.version_utils import version
diff --git a/keras_nlp/api_export.py b/keras_nlp/src/api_export.py
similarity index 97%
rename from keras_nlp/api_export.py
rename to keras_nlp/src/api_export.py
index bbf31fe012..83deccd836 100644
--- a/keras_nlp/api_export.py
+++ b/keras_nlp/src/api_export.py
@@ -14,7 +14,7 @@
 
 import types
 
-from keras_nlp.backend import keras
+from keras_nlp.src.backend import keras
 
 try:
     import namex
diff --git a/keras_nlp/backend/__init__.py b/keras_nlp/src/backend/__init__.py
similarity index 86%
rename from keras_nlp/backend/__init__.py
rename to keras_nlp/src/backend/__init__.py
index 9be0bcb6e1..8812aacff1 100644
--- a/keras_nlp/backend/__init__.py
+++ b/keras_nlp/src/backend/__init__.py
@@ -26,12 +26,12 @@
 - `random`: `keras.random` for Keras 3 or `keras_core.ops` for Keras 2.
 """
 
-from keras_nlp.backend import config
+from keras_nlp.src.backend import config
 
 if not config.keras_3():
-    import keras_nlp.backend.keras2 as keras  # noqa: E402
+    import keras_nlp.src.backend.keras2 as keras  # noqa: E402
 else:
     import keras  # noqa: E402
 
-from keras_nlp.backend import ops  # noqa: E402
-from keras_nlp.backend import random  # noqa: E402
+from keras_nlp.src.backend import ops  # noqa: E402
+from keras_nlp.src.backend import random  # noqa: E402
diff --git a/keras_nlp/backend/config.py b/keras_nlp/src/backend/config.py
similarity index 100%
rename from keras_nlp/backend/config.py
rename to keras_nlp/src/backend/config.py
diff --git a/keras_nlp/backend/keras2.py b/keras_nlp/src/backend/keras2.py
similarity index 96%
rename from keras_nlp/backend/keras2.py
rename to keras_nlp/src/backend/keras2.py
index ae7f71d362..c142c31581 100644
--- a/keras_nlp/backend/keras2.py
+++ b/keras_nlp/src/backend/keras2.py
@@ -26,7 +26,7 @@
 from tensorflow.keras import *  # noqa: F403, F401
 from tensorflow.keras import utils
 
-from keras_nlp.backend import config  # noqa: F401
+from keras_nlp.src.backend import config  # noqa: F401
 
 # Shims to handle symbol renames for older `tf.keras` versions.
 if not hasattr(tf.keras, "saving"):
diff --git a/keras_nlp/backend/ops.py b/keras_nlp/src/backend/ops.py
similarity index 97%
rename from keras_nlp/backend/ops.py
rename to keras_nlp/src/backend/ops.py
index f36a2d2d05..1c858a448a 100644
--- a/keras_nlp/backend/ops.py
+++ b/keras_nlp/src/backend/ops.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.backend import config
+from keras_nlp.src.backend import config
 
 if config.keras_3():
     from keras.ops import *  # noqa: F403, F401
diff --git a/keras_nlp/backend/random.py b/keras_nlp/src/backend/random.py
similarity index 94%
rename from keras_nlp/backend/random.py
rename to keras_nlp/src/backend/random.py
index c4f4e6f467..7983e3b235 100644
--- a/keras_nlp/backend/random.py
+++ b/keras_nlp/src/backend/random.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.backend import config
+from keras_nlp.src.backend import config
 
 if config.keras_3():
     from keras.random import *  # noqa: F403, F401
diff --git a/keras_nlp/src/layers/__init__.py b/keras_nlp/src/layers/__init__.py
new file mode 100644
index 0000000000..3bcd9e71b1
--- /dev/null
+++ b/keras_nlp/src/layers/__init__.py
@@ -0,0 +1,41 @@
+# Copyright 2023 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from keras_nlp.src.layers.modeling.cached_multi_head_attention import (
+    CachedMultiHeadAttention,
+)
+from keras_nlp.src.layers.modeling.f_net_encoder import FNetEncoder
+from keras_nlp.src.layers.modeling.masked_lm_head import MaskedLMHead
+from keras_nlp.src.layers.modeling.position_embedding import PositionEmbedding
+from keras_nlp.src.layers.modeling.reversible_embedding import (
+    ReversibleEmbedding,
+)
+from keras_nlp.src.layers.modeling.rotary_embedding import RotaryEmbedding
+from keras_nlp.src.layers.modeling.sine_position_encoding import (
+    SinePositionEncoding,
+)
+from keras_nlp.src.layers.modeling.token_and_position_embedding import (
+    TokenAndPositionEmbedding,
+)
+from keras_nlp.src.layers.modeling.transformer_decoder import TransformerDecoder
+from keras_nlp.src.layers.modeling.transformer_encoder import TransformerEncoder
+from keras_nlp.src.layers.preprocessing.masked_lm_mask_generator import (
+    MaskedLMMaskGenerator,
+)
+from keras_nlp.src.layers.preprocessing.multi_segment_packer import (
+    MultiSegmentPacker,
+)
+from keras_nlp.src.layers.preprocessing.random_deletion import RandomDeletion
+from keras_nlp.src.layers.preprocessing.random_swap import RandomSwap
+from keras_nlp.src.layers.preprocessing.start_end_packer import StartEndPacker
diff --git a/keras_nlp/layers/modeling/__init__.py b/keras_nlp/src/layers/modeling/__init__.py
similarity index 100%
rename from keras_nlp/layers/modeling/__init__.py
rename to keras_nlp/src/layers/modeling/__init__.py
diff --git a/keras_nlp/layers/modeling/alibi_bias.py b/keras_nlp/src/layers/modeling/alibi_bias.py
similarity index 97%
rename from keras_nlp/layers/modeling/alibi_bias.py
rename to keras_nlp/src/layers/modeling/alibi_bias.py
index c5f8706f9d..663dfb8cd9 100644
--- a/keras_nlp/layers/modeling/alibi_bias.py
+++ b/keras_nlp/src/layers/modeling/alibi_bias.py
@@ -13,9 +13,9 @@
 # limitations under the License.
 import math
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
 
 
 @keras_nlp_export("keras_nlp.layers.AlibiBias")
diff --git a/keras_nlp/layers/modeling/cached_multi_head_attention.py b/keras_nlp/src/layers/modeling/cached_multi_head_attention.py
similarity index 97%
rename from keras_nlp/layers/modeling/cached_multi_head_attention.py
rename to keras_nlp/src/layers/modeling/cached_multi_head_attention.py
index 3f30cb16ad..13341f4a7d 100644
--- a/keras_nlp/layers/modeling/cached_multi_head_attention.py
+++ b/keras_nlp/src/layers/modeling/cached_multi_head_attention.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
 
 
 @keras_nlp_export("keras_nlp.layers.CachedMultiHeadAttention")
diff --git a/keras_nlp/layers/modeling/f_net_encoder.py b/keras_nlp/src/layers/modeling/f_net_encoder.py
similarity index 97%
rename from keras_nlp/layers/modeling/f_net_encoder.py
rename to keras_nlp/src/layers/modeling/f_net_encoder.py
index 919e3beb08..9cb0906c16 100644
--- a/keras_nlp/layers/modeling/f_net_encoder.py
+++ b/keras_nlp/src/layers/modeling/f_net_encoder.py
@@ -12,10 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.utils.keras_utils import clone_initializer
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.utils.keras_utils import clone_initializer
 
 
 @keras_nlp_export("keras_nlp.layers.FNetEncoder")
diff --git a/keras_nlp/layers/modeling/masked_lm_head.py b/keras_nlp/src/layers/modeling/masked_lm_head.py
similarity index 98%
rename from keras_nlp/layers/modeling/masked_lm_head.py
rename to keras_nlp/src/layers/modeling/masked_lm_head.py
index 9f9397cb70..d5ca42cdf5 100644
--- a/keras_nlp/layers/modeling/masked_lm_head.py
+++ b/keras_nlp/src/layers/modeling/masked_lm_head.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
 
 
 @keras_nlp_export("keras_nlp.layers.MaskedLMHead")
diff --git a/keras_nlp/layers/modeling/position_embedding.py b/keras_nlp/src/layers/modeling/position_embedding.py
similarity index 97%
rename from keras_nlp/layers/modeling/position_embedding.py
rename to keras_nlp/src/layers/modeling/position_embedding.py
index 9f6b314b96..c77cc6c93e 100644
--- a/keras_nlp/layers/modeling/position_embedding.py
+++ b/keras_nlp/src/layers/modeling/position_embedding.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
 
 
 @keras_nlp_export("keras_nlp.layers.PositionEmbedding")
diff --git a/keras_nlp/layers/modeling/reversible_embedding.py b/keras_nlp/src/layers/modeling/reversible_embedding.py
similarity index 97%
rename from keras_nlp/layers/modeling/reversible_embedding.py
rename to keras_nlp/src/layers/modeling/reversible_embedding.py
index df57860c4d..b60c6730de 100644
--- a/keras_nlp/layers/modeling/reversible_embedding.py
+++ b/keras_nlp/src/layers/modeling/reversible_embedding.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
 
 
 @keras_nlp_export("keras_nlp.layers.ReversibleEmbedding")
diff --git a/keras_nlp/layers/modeling/rotary_embedding.py b/keras_nlp/src/layers/modeling/rotary_embedding.py
similarity index 98%
rename from keras_nlp/layers/modeling/rotary_embedding.py
rename to keras_nlp/src/layers/modeling/rotary_embedding.py
index a0e8a23e6d..8e375ba5ad 100644
--- a/keras_nlp/layers/modeling/rotary_embedding.py
+++ b/keras_nlp/src/layers/modeling/rotary_embedding.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
 
 
 @keras_nlp_export("keras_nlp.layers.RotaryEmbedding")
diff --git a/keras_nlp/layers/modeling/sine_position_encoding.py b/keras_nlp/src/layers/modeling/sine_position_encoding.py
similarity index 96%
rename from keras_nlp/layers/modeling/sine_position_encoding.py
rename to keras_nlp/src/layers/modeling/sine_position_encoding.py
index b1cd7fbf42..4ca33274ac 100644
--- a/keras_nlp/layers/modeling/sine_position_encoding.py
+++ b/keras_nlp/src/layers/modeling/sine_position_encoding.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
 
 
 @keras_nlp_export("keras_nlp.layers.SinePositionEncoding")
diff --git a/keras_nlp/layers/modeling/token_and_position_embedding.py b/keras_nlp/src/layers/modeling/token_and_position_embedding.py
similarity index 94%
rename from keras_nlp/layers/modeling/token_and_position_embedding.py
rename to keras_nlp/src/layers/modeling/token_and_position_embedding.py
index 8261cc7f34..ecadada251 100644
--- a/keras_nlp/layers/modeling/token_and_position_embedding.py
+++ b/keras_nlp/src/layers/modeling/token_and_position_embedding.py
@@ -12,11 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.layers.modeling.position_embedding import PositionEmbedding
-from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding
-from keras_nlp.utils.keras_utils import clone_initializer
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.modeling.position_embedding import PositionEmbedding
+from keras_nlp.src.layers.modeling.reversible_embedding import (
+    ReversibleEmbedding,
+)
+from keras_nlp.src.utils.keras_utils import clone_initializer
 
 
 @keras_nlp_export("keras_nlp.layers.TokenAndPositionEmbedding")
diff --git a/keras_nlp/layers/modeling/transformer_decoder.py b/keras_nlp/src/layers/modeling/transformer_decoder.py
similarity index 98%
rename from keras_nlp/layers/modeling/transformer_decoder.py
rename to keras_nlp/src/layers/modeling/transformer_decoder.py
index b8f797f2e2..649f048e70 100644
--- a/keras_nlp/layers/modeling/transformer_decoder.py
+++ b/keras_nlp/src/layers/modeling/transformer_decoder.py
@@ -12,15 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.layers.modeling.cached_multi_head_attention import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.layers.modeling.cached_multi_head_attention import (
     CachedMultiHeadAttention,
 )
-from keras_nlp.utils.keras_utils import clone_initializer
+from keras_nlp.src.utils.keras_utils import clone_initializer
 
-from keras_nlp.layers.modeling.transformer_layer_utils import (  # isort:skip
+from keras_nlp.src.layers.modeling.transformer_layer_utils import (  # isort:skip
     compute_causal_mask,
     merge_padding_and_attention_mask,
 )
diff --git a/keras_nlp/layers/modeling/transformer_encoder.py b/keras_nlp/src/layers/modeling/transformer_encoder.py
similarity index 97%
rename from keras_nlp/layers/modeling/transformer_encoder.py
rename to keras_nlp/src/layers/modeling/transformer_encoder.py
index 20cec4ecf1..d04f027e12 100644
--- a/keras_nlp/layers/modeling/transformer_encoder.py
+++ b/keras_nlp/src/layers/modeling/transformer_encoder.py
@@ -12,11 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.utils.keras_utils import clone_initializer
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.utils.keras_utils import clone_initializer
 
-from keras_nlp.layers.modeling.transformer_layer_utils import (  # isort:skip
+from keras_nlp.src.layers.modeling.transformer_layer_utils import (  # isort:skip
     merge_padding_and_attention_mask,
 )
 
diff --git a/keras_nlp/layers/modeling/transformer_layer_utils.py b/keras_nlp/src/layers/modeling/transformer_layer_utils.py
similarity index 99%
rename from keras_nlp/layers/modeling/transformer_layer_utils.py
rename to keras_nlp/src/layers/modeling/transformer_layer_utils.py
index f375bf1b9d..7e9d97baf3 100644
--- a/keras_nlp/layers/modeling/transformer_layer_utils.py
+++ b/keras_nlp/src/layers/modeling/transformer_layer_utils.py
@@ -14,7 +14,7 @@
 
 from absl import logging
 
-from keras_nlp.backend import ops
+from keras_nlp.src.backend import ops
 
 
 def _check_masks_shapes(inputs, padding_mask, attention_mask):
diff --git a/keras_nlp/layers/preprocessing/__init__.py b/keras_nlp/src/layers/preprocessing/__init__.py
similarity index 100%
rename from keras_nlp/layers/preprocessing/__init__.py
rename to keras_nlp/src/layers/preprocessing/__init__.py
diff --git a/keras_nlp/layers/preprocessing/masked_lm_mask_generator.py b/keras_nlp/src/layers/preprocessing/masked_lm_mask_generator.py
similarity index 97%
rename from keras_nlp/layers/preprocessing/masked_lm_mask_generator.py
rename to keras_nlp/src/layers/preprocessing/masked_lm_mask_generator.py
index b2f950d31b..96ee123345 100644
--- a/keras_nlp/layers/preprocessing/masked_lm_mask_generator.py
+++ b/keras_nlp/src/layers/preprocessing/masked_lm_mask_generator.py
@@ -20,12 +20,12 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.preprocessing_layer import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.preprocessing_layer import (
     PreprocessingLayer,
 )
-from keras_nlp.utils.tensor_utils import assert_tf_text_installed
-from keras_nlp.utils.tensor_utils import convert_to_ragged_batch
+from keras_nlp.src.utils.tensor_utils import assert_tf_text_installed
+from keras_nlp.src.utils.tensor_utils import convert_to_ragged_batch
 
 try:
     import tensorflow_text as tf_text
diff --git a/keras_nlp/layers/preprocessing/multi_segment_packer.py b/keras_nlp/src/layers/preprocessing/multi_segment_packer.py
similarity index 97%
rename from keras_nlp/layers/preprocessing/multi_segment_packer.py
rename to keras_nlp/src/layers/preprocessing/multi_segment_packer.py
index cdda675cbe..a8ed0f745c 100644
--- a/keras_nlp/layers/preprocessing/multi_segment_packer.py
+++ b/keras_nlp/src/layers/preprocessing/multi_segment_packer.py
@@ -20,12 +20,12 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.preprocessing_layer import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.preprocessing_layer import (
     PreprocessingLayer,
 )
-from keras_nlp.utils.tensor_utils import assert_tf_text_installed
-from keras_nlp.utils.tensor_utils import convert_to_ragged_batch
+from keras_nlp.src.utils.tensor_utils import assert_tf_text_installed
+from keras_nlp.src.utils.tensor_utils import convert_to_ragged_batch
 
 try:
     import tensorflow_text as tf_text
diff --git a/keras_nlp/layers/preprocessing/preprocessing_layer.py b/keras_nlp/src/layers/preprocessing/preprocessing_layer.py
similarity index 94%
rename from keras_nlp/layers/preprocessing/preprocessing_layer.py
rename to keras_nlp/src/layers/preprocessing/preprocessing_layer.py
index 95d9405e71..5c06b0dce4 100644
--- a/keras_nlp/layers/preprocessing/preprocessing_layer.py
+++ b/keras_nlp/src/layers/preprocessing/preprocessing_layer.py
@@ -21,9 +21,9 @@
     )
 import tree
 
-from keras_nlp.backend import config
-from keras_nlp.backend import keras
-from keras_nlp.utils.tensor_utils import (
+from keras_nlp.src.backend import config
+from keras_nlp.src.backend import keras
+from keras_nlp.src.utils.tensor_utils import (
     convert_to_backend_tensor_or_python_list,
 )
 
diff --git a/keras_nlp/layers/preprocessing/random_deletion.py b/keras_nlp/src/layers/preprocessing/random_deletion.py
similarity index 97%
rename from keras_nlp/layers/preprocessing/random_deletion.py
rename to keras_nlp/src/layers/preprocessing/random_deletion.py
index a807d97042..362cbacbdf 100644
--- a/keras_nlp/layers/preprocessing/random_deletion.py
+++ b/keras_nlp/src/layers/preprocessing/random_deletion.py
@@ -22,13 +22,13 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.preprocessing_layer import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.preprocessing_layer import (
     PreprocessingLayer,
 )
-from keras_nlp.utils.tensor_utils import convert_to_ragged_batch
-from keras_nlp.utils.tensor_utils import is_int_dtype
-from keras_nlp.utils.tensor_utils import is_string_dtype
+from keras_nlp.src.utils.tensor_utils import convert_to_ragged_batch
+from keras_nlp.src.utils.tensor_utils import is_int_dtype
+from keras_nlp.src.utils.tensor_utils import is_string_dtype
 
 
 @keras_nlp_export("keras_nlp.layers.RandomDeletion")
diff --git a/keras_nlp/layers/preprocessing/random_swap.py b/keras_nlp/src/layers/preprocessing/random_swap.py
similarity index 97%
rename from keras_nlp/layers/preprocessing/random_swap.py
rename to keras_nlp/src/layers/preprocessing/random_swap.py
index ccbae08ec2..bf13fbbb5c 100644
--- a/keras_nlp/layers/preprocessing/random_swap.py
+++ b/keras_nlp/src/layers/preprocessing/random_swap.py
@@ -22,13 +22,13 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.preprocessing_layer import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.preprocessing_layer import (
     PreprocessingLayer,
 )
-from keras_nlp.utils.tensor_utils import convert_to_ragged_batch
-from keras_nlp.utils.tensor_utils import is_int_dtype
-from keras_nlp.utils.tensor_utils import is_string_dtype
+from keras_nlp.src.utils.tensor_utils import convert_to_ragged_batch
+from keras_nlp.src.utils.tensor_utils import is_int_dtype
+from keras_nlp.src.utils.tensor_utils import is_string_dtype
 
 
 @keras_nlp_export("keras_nlp.layers.RandomSwap")
diff --git a/keras_nlp/layers/preprocessing/start_end_packer.py b/keras_nlp/src/layers/preprocessing/start_end_packer.py
similarity index 97%
rename from keras_nlp/layers/preprocessing/start_end_packer.py
rename to keras_nlp/src/layers/preprocessing/start_end_packer.py
index 499ec82153..1a3620fda8 100644
--- a/keras_nlp/layers/preprocessing/start_end_packer.py
+++ b/keras_nlp/src/layers/preprocessing/start_end_packer.py
@@ -20,11 +20,11 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.preprocessing_layer import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.preprocessing_layer import (
     PreprocessingLayer,
 )
-from keras_nlp.utils.tensor_utils import convert_to_ragged_batch
+from keras_nlp.src.utils.tensor_utils import convert_to_ragged_batch
 
 
 @keras_nlp_export("keras_nlp.layers.StartEndPacker")
diff --git a/keras_nlp/metrics/__init__.py b/keras_nlp/src/metrics/__init__.py
similarity index 69%
rename from keras_nlp/metrics/__init__.py
rename to keras_nlp/src/metrics/__init__.py
index e20ad29167..308ad2100c 100644
--- a/keras_nlp/metrics/__init__.py
+++ b/keras_nlp/src/metrics/__init__.py
@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.metrics.bleu import Bleu
-from keras_nlp.metrics.edit_distance import EditDistance
-from keras_nlp.metrics.perplexity import Perplexity
-from keras_nlp.metrics.rouge_l import RougeL
-from keras_nlp.metrics.rouge_n import RougeN
+from keras_nlp.src.metrics.bleu import Bleu
+from keras_nlp.src.metrics.edit_distance import EditDistance
+from keras_nlp.src.metrics.perplexity import Perplexity
+from keras_nlp.src.metrics.rouge_l import RougeL
+from keras_nlp.src.metrics.rouge_n import RougeN
diff --git a/keras_nlp/metrics/bleu.py b/keras_nlp/src/metrics/bleu.py
similarity index 98%
rename from keras_nlp/metrics/bleu.py
rename to keras_nlp/src/metrics/bleu.py
index a73d08ea27..790978f331 100644
--- a/keras_nlp/metrics/bleu.py
+++ b/keras_nlp/src/metrics/bleu.py
@@ -23,11 +23,11 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.utils.tensor_utils import is_float_dtype
-from keras_nlp.utils.tensor_utils import tensor_to_list
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.utils.tensor_utils import is_float_dtype
+from keras_nlp.src.utils.tensor_utils import tensor_to_list
 
 REPLACE_SUBSTRINGS = [
     ("<skipped>", ""),
diff --git a/keras_nlp/metrics/edit_distance.py b/keras_nlp/src/metrics/edit_distance.py
similarity index 97%
rename from keras_nlp/metrics/edit_distance.py
rename to keras_nlp/src/metrics/edit_distance.py
index e2668b16d0..0cd4dddd3a 100644
--- a/keras_nlp/metrics/edit_distance.py
+++ b/keras_nlp/src/metrics/edit_distance.py
@@ -20,9 +20,9 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.utils.tensor_utils import is_float_dtype
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.utils.tensor_utils import is_float_dtype
 
 
 @keras_nlp_export("keras_nlp.metrics.EditDistance")
diff --git a/keras_nlp/metrics/perplexity.py b/keras_nlp/src/metrics/perplexity.py
similarity index 97%
rename from keras_nlp/metrics/perplexity.py
rename to keras_nlp/src/metrics/perplexity.py
index 4a7e626bc9..eb22ca056a 100644
--- a/keras_nlp/metrics/perplexity.py
+++ b/keras_nlp/src/metrics/perplexity.py
@@ -12,10 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.utils.tensor_utils import is_float_dtype
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.utils.tensor_utils import is_float_dtype
 
 
 @keras_nlp_export("keras_nlp.metrics.Perplexity")
diff --git a/keras_nlp/metrics/rouge_base.py b/keras_nlp/src/metrics/rouge_base.py
similarity index 97%
rename from keras_nlp/metrics/rouge_base.py
rename to keras_nlp/src/metrics/rouge_base.py
index 16220e2623..0f108e31bf 100644
--- a/keras_nlp/metrics/rouge_base.py
+++ b/keras_nlp/src/metrics/rouge_base.py
@@ -20,10 +20,10 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.utils.tensor_utils import is_float_dtype
-from keras_nlp.utils.tensor_utils import tensor_to_list
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.utils.tensor_utils import is_float_dtype
+from keras_nlp.src.utils.tensor_utils import tensor_to_list
 
 try:
     from rouge_score import rouge_scorer
diff --git a/keras_nlp/metrics/rouge_l.py b/keras_nlp/src/metrics/rouge_l.py
similarity index 96%
rename from keras_nlp/metrics/rouge_l.py
rename to keras_nlp/src/metrics/rouge_l.py
index 82fcdedade..f5a8ec4c8e 100644
--- a/keras_nlp/metrics/rouge_l.py
+++ b/keras_nlp/src/metrics/rouge_l.py
@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.metrics.rouge_base import RougeBase
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.metrics.rouge_base import RougeBase
 
 
 @keras_nlp_export("keras_nlp.metrics.RougeL")
diff --git a/keras_nlp/metrics/rouge_n.py b/keras_nlp/src/metrics/rouge_n.py
similarity index 97%
rename from keras_nlp/metrics/rouge_n.py
rename to keras_nlp/src/metrics/rouge_n.py
index 8b135d1dd0..8c67b3cf0c 100644
--- a/keras_nlp/metrics/rouge_n.py
+++ b/keras_nlp/src/metrics/rouge_n.py
@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.metrics.rouge_base import RougeBase
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.metrics.rouge_base import RougeBase
 
 
 @keras_nlp_export("keras_nlp.metrics.RougeN")
diff --git a/keras_nlp/src/models/__init__.py b/keras_nlp/src/models/__init__.py
new file mode 100644
index 0000000000..9e122f35ad
--- /dev/null
+++ b/keras_nlp/src/models/__init__.py
@@ -0,0 +1,188 @@
+# Copyright 2023 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from keras_nlp.src.models.albert.albert_backbone import AlbertBackbone
+from keras_nlp.src.models.albert.albert_classifier import AlbertClassifier
+from keras_nlp.src.models.albert.albert_masked_lm import AlbertMaskedLM
+from keras_nlp.src.models.albert.albert_masked_lm_preprocessor import (
+    AlbertMaskedLMPreprocessor,
+)
+from keras_nlp.src.models.albert.albert_preprocessor import AlbertPreprocessor
+from keras_nlp.src.models.albert.albert_tokenizer import AlbertTokenizer
+from keras_nlp.src.models.backbone import Backbone
+from keras_nlp.src.models.bart.bart_backbone import BartBackbone
+from keras_nlp.src.models.bart.bart_preprocessor import BartPreprocessor
+from keras_nlp.src.models.bart.bart_seq_2_seq_lm import BartSeq2SeqLM
+from keras_nlp.src.models.bart.bart_seq_2_seq_lm_preprocessor import (
+    BartSeq2SeqLMPreprocessor,
+)
+from keras_nlp.src.models.bart.bart_tokenizer import BartTokenizer
+from keras_nlp.src.models.bert.bert_backbone import BertBackbone
+from keras_nlp.src.models.bert.bert_classifier import BertClassifier
+from keras_nlp.src.models.bert.bert_masked_lm import BertMaskedLM
+from keras_nlp.src.models.bert.bert_masked_lm_preprocessor import (
+    BertMaskedLMPreprocessor,
+)
+from keras_nlp.src.models.bert.bert_preprocessor import BertPreprocessor
+from keras_nlp.src.models.bert.bert_tokenizer import BertTokenizer
+from keras_nlp.src.models.bloom.bloom_backbone import BloomBackbone
+from keras_nlp.src.models.bloom.bloom_causal_lm import BloomCausalLM
+from keras_nlp.src.models.bloom.bloom_causal_lm_preprocessor import (
+    BloomCausalLMPreprocessor,
+)
+from keras_nlp.src.models.bloom.bloom_preprocessor import BloomPreprocessor
+from keras_nlp.src.models.bloom.bloom_tokenizer import BloomTokenizer
+from keras_nlp.src.models.causal_lm import CausalLM
+from keras_nlp.src.models.classifier import Classifier
+from keras_nlp.src.models.deberta_v3.deberta_v3_backbone import (
+    DebertaV3Backbone,
+)
+from keras_nlp.src.models.deberta_v3.deberta_v3_classifier import (
+    DebertaV3Classifier,
+)
+from keras_nlp.src.models.deberta_v3.deberta_v3_masked_lm import (
+    DebertaV3MaskedLM,
+)
+from keras_nlp.src.models.deberta_v3.deberta_v3_masked_lm_preprocessor import (
+    DebertaV3MaskedLMPreprocessor,
+)
+from keras_nlp.src.models.deberta_v3.deberta_v3_preprocessor import (
+    DebertaV3Preprocessor,
+)
+from keras_nlp.src.models.deberta_v3.deberta_v3_tokenizer import (
+    DebertaV3Tokenizer,
+)
+from keras_nlp.src.models.distil_bert.distil_bert_backbone import (
+    DistilBertBackbone,
+)
+from keras_nlp.src.models.distil_bert.distil_bert_classifier import (
+    DistilBertClassifier,
+)
+from keras_nlp.src.models.distil_bert.distil_bert_masked_lm import (
+    DistilBertMaskedLM,
+)
+from keras_nlp.src.models.distil_bert.distil_bert_masked_lm_preprocessor import (
+    DistilBertMaskedLMPreprocessor,
+)
+from keras_nlp.src.models.distil_bert.distil_bert_preprocessor import (
+    DistilBertPreprocessor,
+)
+from keras_nlp.src.models.distil_bert.distil_bert_tokenizer import (
+    DistilBertTokenizer,
+)
+from keras_nlp.src.models.electra.electra_backbone import ElectraBackbone
+from keras_nlp.src.models.electra.electra_preprocessor import (
+    ElectraPreprocessor,
+)
+from keras_nlp.src.models.electra.electra_tokenizer import ElectraTokenizer
+from keras_nlp.src.models.f_net.f_net_backbone import FNetBackbone
+from keras_nlp.src.models.f_net.f_net_classifier import FNetClassifier
+from keras_nlp.src.models.f_net.f_net_masked_lm import FNetMaskedLM
+from keras_nlp.src.models.f_net.f_net_masked_lm_preprocessor import (
+    FNetMaskedLMPreprocessor,
+)
+from keras_nlp.src.models.f_net.f_net_preprocessor import FNetPreprocessor
+from keras_nlp.src.models.f_net.f_net_tokenizer import FNetTokenizer
+from keras_nlp.src.models.falcon.falcon_backbone import FalconBackbone
+from keras_nlp.src.models.falcon.falcon_tokenizer import FalconTokenizer
+from keras_nlp.src.models.gemma.gemma_backbone import GemmaBackbone
+from keras_nlp.src.models.gemma.gemma_causal_lm import GemmaCausalLM
+from keras_nlp.src.models.gemma.gemma_causal_lm_preprocessor import (
+    GemmaCausalLMPreprocessor,
+)
+from keras_nlp.src.models.gemma.gemma_preprocessor import GemmaPreprocessor
+from keras_nlp.src.models.gemma.gemma_tokenizer import GemmaTokenizer
+from keras_nlp.src.models.gpt2.gpt2_backbone import GPT2Backbone
+from keras_nlp.src.models.gpt2.gpt2_causal_lm import GPT2CausalLM
+from keras_nlp.src.models.gpt2.gpt2_causal_lm_preprocessor import (
+    GPT2CausalLMPreprocessor,
+)
+from keras_nlp.src.models.gpt2.gpt2_preprocessor import GPT2Preprocessor
+from keras_nlp.src.models.gpt2.gpt2_tokenizer import GPT2Tokenizer
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_backbone import GPTNeoXBackbone
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_causal_lm import GPTNeoXCausalLM
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_causal_lm_preprocessor import (
+    GPTNeoXCausalLMPreprocessor,
+)
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_preprocessor import (
+    GPTNeoXPreprocessor,
+)
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_tokenizer import GPTNeoXTokenizer
+from keras_nlp.src.models.llama.llama_backbone import LlamaBackbone
+from keras_nlp.src.models.llama.llama_causal_lm import LlamaCausalLM
+from keras_nlp.src.models.llama.llama_causal_lm_preprocessor import (
+    LlamaCausalLMPreprocessor,
+)
+from keras_nlp.src.models.llama.llama_preprocessor import LlamaPreprocessor
+from keras_nlp.src.models.llama.llama_tokenizer import LlamaTokenizer
+from keras_nlp.src.models.masked_lm import MaskedLM
+from keras_nlp.src.models.mistral.mistral_backbone import MistralBackbone
+from keras_nlp.src.models.mistral.mistral_causal_lm import MistralCausalLM
+from keras_nlp.src.models.mistral.mistral_causal_lm_preprocessor import (
+    MistralCausalLMPreprocessor,
+)
+from keras_nlp.src.models.mistral.mistral_preprocessor import (
+    MistralPreprocessor,
+)
+from keras_nlp.src.models.mistral.mistral_tokenizer import MistralTokenizer
+from keras_nlp.src.models.opt.opt_backbone import OPTBackbone
+from keras_nlp.src.models.opt.opt_causal_lm import OPTCausalLM
+from keras_nlp.src.models.opt.opt_causal_lm_preprocessor import (
+    OPTCausalLMPreprocessor,
+)
+from keras_nlp.src.models.opt.opt_preprocessor import OPTPreprocessor
+from keras_nlp.src.models.opt.opt_tokenizer import OPTTokenizer
+from keras_nlp.src.models.preprocessor import Preprocessor
+from keras_nlp.src.models.roberta.roberta_backbone import RobertaBackbone
+from keras_nlp.src.models.roberta.roberta_classifier import RobertaClassifier
+from keras_nlp.src.models.roberta.roberta_masked_lm import RobertaMaskedLM
+from keras_nlp.src.models.roberta.roberta_masked_lm_preprocessor import (
+    RobertaMaskedLMPreprocessor,
+)
+from keras_nlp.src.models.roberta.roberta_preprocessor import (
+    RobertaPreprocessor,
+)
+from keras_nlp.src.models.roberta.roberta_tokenizer import RobertaTokenizer
+from keras_nlp.src.models.seq_2_seq_lm import Seq2SeqLM
+from keras_nlp.src.models.t5.t5_backbone import T5Backbone
+from keras_nlp.src.models.t5.t5_tokenizer import T5Tokenizer
+from keras_nlp.src.models.task import Task
+from keras_nlp.src.models.whisper.whisper_audio_feature_extractor import (
+    WhisperAudioFeatureExtractor,
+)
+from keras_nlp.src.models.whisper.whisper_backbone import WhisperBackbone
+from keras_nlp.src.models.whisper.whisper_preprocessor import (
+    WhisperPreprocessor,
+)
+from keras_nlp.src.models.whisper.whisper_tokenizer import WhisperTokenizer
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_backbone import (
+    XLMRobertaBackbone,
+)
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_classifier import (
+    XLMRobertaClassifier,
+)
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_masked_lm import (
+    XLMRobertaMaskedLM,
+)
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_masked_lm_preprocessor import (
+    XLMRobertaMaskedLMPreprocessor,
+)
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_preprocessor import (
+    XLMRobertaPreprocessor,
+)
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_tokenizer import (
+    XLMRobertaTokenizer,
+)
+from keras_nlp.src.models.xlnet.xlnet_backbone import XLNetBackbone
+from keras_nlp.src.tokenizers.tokenizer import Tokenizer
diff --git a/keras_nlp/models/albert/__init__.py b/keras_nlp/src/models/albert/__init__.py
similarity index 70%
rename from keras_nlp/models/albert/__init__.py
rename to keras_nlp/src/models/albert/__init__.py
index c0ae8e8fa1..74c528ebf3 100644
--- a/keras_nlp/models/albert/__init__.py
+++ b/keras_nlp/src/models/albert/__init__.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.models.albert.albert_backbone import AlbertBackbone
-from keras_nlp.models.albert.albert_presets import backbone_presets
-from keras_nlp.models.albert.albert_tokenizer import AlbertTokenizer
-from keras_nlp.utils.preset_utils import register_presets
+from keras_nlp.src.models.albert.albert_backbone import AlbertBackbone
+from keras_nlp.src.models.albert.albert_presets import backbone_presets
+from keras_nlp.src.models.albert.albert_tokenizer import AlbertTokenizer
+from keras_nlp.src.utils.preset_utils import register_presets
 
 register_presets(backbone_presets, (AlbertBackbone, AlbertTokenizer))
diff --git a/keras_nlp/models/albert/albert_backbone.py b/keras_nlp/src/models/albert/albert_backbone.py
similarity index 95%
rename from keras_nlp/models/albert/albert_backbone.py
rename to keras_nlp/src/models/albert/albert_backbone.py
index e8acef81d5..abcbbaace4 100644
--- a/keras_nlp/models/albert/albert_backbone.py
+++ b/keras_nlp/src/models/albert/albert_backbone.py
@@ -12,13 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.layers.modeling.position_embedding import PositionEmbedding
-from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding
-from keras_nlp.layers.modeling.transformer_encoder import TransformerEncoder
-from keras_nlp.models.backbone import Backbone
-from keras_nlp.utils.keras_utils import gelu_approximate
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.modeling.position_embedding import PositionEmbedding
+from keras_nlp.src.layers.modeling.reversible_embedding import (
+    ReversibleEmbedding,
+)
+from keras_nlp.src.layers.modeling.transformer_encoder import TransformerEncoder
+from keras_nlp.src.models.backbone import Backbone
+from keras_nlp.src.utils.keras_utils import gelu_approximate
 
 
 def albert_kernel_initializer(stddev=0.02):
diff --git a/keras_nlp/models/albert/albert_classifier.py b/keras_nlp/src/models/albert/albert_classifier.py
similarity index 94%
rename from keras_nlp/models/albert/albert_classifier.py
rename to keras_nlp/src/models/albert/albert_classifier.py
index d732bc1973..1077c327a4 100644
--- a/keras_nlp/models/albert/albert_classifier.py
+++ b/keras_nlp/src/models/albert/albert_classifier.py
@@ -12,12 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.models.albert.albert_backbone import AlbertBackbone
-from keras_nlp.models.albert.albert_backbone import albert_kernel_initializer
-from keras_nlp.models.albert.albert_preprocessor import AlbertPreprocessor
-from keras_nlp.models.classifier import Classifier
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.models.albert.albert_backbone import AlbertBackbone
+from keras_nlp.src.models.albert.albert_backbone import (
+    albert_kernel_initializer,
+)
+from keras_nlp.src.models.albert.albert_preprocessor import AlbertPreprocessor
+from keras_nlp.src.models.classifier import Classifier
 
 
 @keras_nlp_export("keras_nlp.models.AlbertClassifier")
diff --git a/keras_nlp/models/albert/albert_masked_lm.py b/keras_nlp/src/models/albert/albert_masked_lm.py
similarity index 89%
rename from keras_nlp/models/albert/albert_masked_lm.py
rename to keras_nlp/src/models/albert/albert_masked_lm.py
index 752f20c2ff..554867f5bc 100644
--- a/keras_nlp/models/albert/albert_masked_lm.py
+++ b/keras_nlp/src/models/albert/albert_masked_lm.py
@@ -12,16 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.layers.modeling.masked_lm_head import MaskedLMHead
-from keras_nlp.models.albert.albert_backbone import AlbertBackbone
-from keras_nlp.models.albert.albert_backbone import albert_kernel_initializer
-from keras_nlp.models.albert.albert_masked_lm_preprocessor import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.modeling.masked_lm_head import MaskedLMHead
+from keras_nlp.src.models.albert.albert_backbone import AlbertBackbone
+from keras_nlp.src.models.albert.albert_backbone import (
+    albert_kernel_initializer,
+)
+from keras_nlp.src.models.albert.albert_masked_lm_preprocessor import (
     AlbertMaskedLMPreprocessor,
 )
-from keras_nlp.models.masked_lm import MaskedLM
-from keras_nlp.utils.keras_utils import gelu_approximate
+from keras_nlp.src.models.masked_lm import MaskedLM
+from keras_nlp.src.utils.keras_utils import gelu_approximate
 
 
 @keras_nlp_export("keras_nlp.models.AlbertMaskedLM")
diff --git a/keras_nlp/models/albert/albert_masked_lm_preprocessor.py b/keras_nlp/src/models/albert/albert_masked_lm_preprocessor.py
similarity index 96%
rename from keras_nlp/models/albert/albert_masked_lm_preprocessor.py
rename to keras_nlp/src/models/albert/albert_masked_lm_preprocessor.py
index 89cf134465..84395cbae0 100644
--- a/keras_nlp/models/albert/albert_masked_lm_preprocessor.py
+++ b/keras_nlp/src/models/albert/albert_masked_lm_preprocessor.py
@@ -14,12 +14,12 @@
 
 from absl import logging
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.masked_lm_mask_generator import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.masked_lm_mask_generator import (
     MaskedLMMaskGenerator,
 )
-from keras_nlp.models.albert.albert_preprocessor import AlbertPreprocessor
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.models.albert.albert_preprocessor import AlbertPreprocessor
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.AlbertMaskedLMPreprocessor")
diff --git a/keras_nlp/models/albert/albert_preprocessor.py b/keras_nlp/src/models/albert/albert_preprocessor.py
similarity index 95%
rename from keras_nlp/models/albert/albert_preprocessor.py
rename to keras_nlp/src/models/albert/albert_preprocessor.py
index 8e49d4e650..3c1bdfdb50 100644
--- a/keras_nlp/models/albert/albert_preprocessor.py
+++ b/keras_nlp/src/models/albert/albert_preprocessor.py
@@ -12,16 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.multi_segment_packer import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.multi_segment_packer import (
     MultiSegmentPacker,
 )
-from keras_nlp.models.albert.albert_tokenizer import AlbertTokenizer
-from keras_nlp.models.preprocessor import Preprocessor
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.models.albert.albert_tokenizer import AlbertTokenizer
+from keras_nlp.src.models.preprocessor import Preprocessor
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.AlbertPreprocessor")
diff --git a/keras_nlp/models/albert/albert_presets.py b/keras_nlp/src/models/albert/albert_presets.py
similarity index 100%
rename from keras_nlp/models/albert/albert_presets.py
rename to keras_nlp/src/models/albert/albert_presets.py
diff --git a/keras_nlp/models/albert/albert_tokenizer.py b/keras_nlp/src/models/albert/albert_tokenizer.py
similarity index 96%
rename from keras_nlp/models/albert/albert_tokenizer.py
rename to keras_nlp/src/models/albert/albert_tokenizer.py
index 8887893afc..824538e482 100644
--- a/keras_nlp/models/albert/albert_tokenizer.py
+++ b/keras_nlp/src/models/albert/albert_tokenizer.py
@@ -12,8 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.tokenizers.sentence_piece_tokenizer import SentencePieceTokenizer
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.tokenizers.sentence_piece_tokenizer import (
+    SentencePieceTokenizer,
+)
 
 
 @keras_nlp_export("keras_nlp.models.AlbertTokenizer")
diff --git a/keras_nlp/models/backbone.py b/keras_nlp/src/models/backbone.py
similarity index 93%
rename from keras_nlp/models/backbone.py
rename to keras_nlp/src/models/backbone.py
index 4cece5c6cc..b862b6f41d 100644
--- a/keras_nlp/models/backbone.py
+++ b/keras_nlp/src/models/backbone.py
@@ -14,21 +14,21 @@
 
 import os
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import config
-from keras_nlp.backend import keras
-from keras_nlp.utils.preset_utils import CONFIG_FILE
-from keras_nlp.utils.preset_utils import MODEL_WEIGHTS_FILE
-from keras_nlp.utils.preset_utils import check_config_class
-from keras_nlp.utils.preset_utils import get_file
-from keras_nlp.utils.preset_utils import jax_memory_cleanup
-from keras_nlp.utils.preset_utils import list_presets
-from keras_nlp.utils.preset_utils import list_subclasses
-from keras_nlp.utils.preset_utils import load_serialized_object
-from keras_nlp.utils.preset_utils import save_metadata
-from keras_nlp.utils.preset_utils import save_serialized_object
-from keras_nlp.utils.preset_utils import validate_metadata
-from keras_nlp.utils.python_utils import classproperty
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import config
+from keras_nlp.src.backend import keras
+from keras_nlp.src.utils.preset_utils import CONFIG_FILE
+from keras_nlp.src.utils.preset_utils import MODEL_WEIGHTS_FILE
+from keras_nlp.src.utils.preset_utils import check_config_class
+from keras_nlp.src.utils.preset_utils import get_file
+from keras_nlp.src.utils.preset_utils import jax_memory_cleanup
+from keras_nlp.src.utils.preset_utils import list_presets
+from keras_nlp.src.utils.preset_utils import list_subclasses
+from keras_nlp.src.utils.preset_utils import load_serialized_object
+from keras_nlp.src.utils.preset_utils import save_metadata
+from keras_nlp.src.utils.preset_utils import save_serialized_object
+from keras_nlp.src.utils.preset_utils import validate_metadata
+from keras_nlp.src.utils.python_utils import classproperty
 
 
 @keras_nlp_export("keras_nlp.models.Backbone")
diff --git a/keras_nlp/models/bart/__init__.py b/keras_nlp/src/models/bart/__init__.py
similarity index 71%
rename from keras_nlp/models/bart/__init__.py
rename to keras_nlp/src/models/bart/__init__.py
index b14f1f06b6..6c0919c99c 100644
--- a/keras_nlp/models/bart/__init__.py
+++ b/keras_nlp/src/models/bart/__init__.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.models.bart.bart_backbone import BartBackbone
-from keras_nlp.models.bart.bart_presets import backbone_presets
-from keras_nlp.models.bart.bart_tokenizer import BartTokenizer
-from keras_nlp.utils.preset_utils import register_presets
+from keras_nlp.src.models.bart.bart_backbone import BartBackbone
+from keras_nlp.src.models.bart.bart_presets import backbone_presets
+from keras_nlp.src.models.bart.bart_tokenizer import BartTokenizer
+from keras_nlp.src.utils.preset_utils import register_presets
 
 register_presets(backbone_presets, (BartBackbone, BartTokenizer))
diff --git a/keras_nlp/models/bart/bart_backbone.py b/keras_nlp/src/models/bart/bart_backbone.py
similarity index 95%
rename from keras_nlp/models/bart/bart_backbone.py
rename to keras_nlp/src/models/bart/bart_backbone.py
index d10f1cc240..2cd501308e 100644
--- a/keras_nlp/models/bart/bart_backbone.py
+++ b/keras_nlp/src/models/bart/bart_backbone.py
@@ -12,13 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.layers.modeling.position_embedding import PositionEmbedding
-from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding
-from keras_nlp.layers.modeling.transformer_decoder import TransformerDecoder
-from keras_nlp.layers.modeling.transformer_encoder import TransformerEncoder
-from keras_nlp.models.backbone import Backbone
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.modeling.position_embedding import PositionEmbedding
+from keras_nlp.src.layers.modeling.reversible_embedding import (
+    ReversibleEmbedding,
+)
+from keras_nlp.src.layers.modeling.transformer_decoder import TransformerDecoder
+from keras_nlp.src.layers.modeling.transformer_encoder import TransformerEncoder
+from keras_nlp.src.models.backbone import Backbone
 
 
 def bart_kernel_initializer(stddev=0.02):
diff --git a/keras_nlp/models/bart/bart_preprocessor.py b/keras_nlp/src/models/bart/bart_preprocessor.py
similarity index 96%
rename from keras_nlp/models/bart/bart_preprocessor.py
rename to keras_nlp/src/models/bart/bart_preprocessor.py
index 276d73af04..7a94c1f836 100644
--- a/keras_nlp/models/bart/bart_preprocessor.py
+++ b/keras_nlp/src/models/bart/bart_preprocessor.py
@@ -13,14 +13,14 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.start_end_packer import StartEndPacker
-from keras_nlp.models.bart.bart_tokenizer import BartTokenizer
-from keras_nlp.models.preprocessor import Preprocessor
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.start_end_packer import StartEndPacker
+from keras_nlp.src.models.bart.bart_tokenizer import BartTokenizer
+from keras_nlp.src.models.preprocessor import Preprocessor
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.BartPreprocessor")
diff --git a/keras_nlp/models/bart/bart_presets.py b/keras_nlp/src/models/bart/bart_presets.py
similarity index 100%
rename from keras_nlp/models/bart/bart_presets.py
rename to keras_nlp/src/models/bart/bart_presets.py
diff --git a/keras_nlp/models/bart/bart_seq_2_seq_lm.py b/keras_nlp/src/models/bart/bart_seq_2_seq_lm.py
similarity index 98%
rename from keras_nlp/models/bart/bart_seq_2_seq_lm.py
rename to keras_nlp/src/models/bart/bart_seq_2_seq_lm.py
index 23f65522ec..8208c42a5f 100644
--- a/keras_nlp/models/bart/bart_seq_2_seq_lm.py
+++ b/keras_nlp/src/models/bart/bart_seq_2_seq_lm.py
@@ -13,14 +13,14 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import ops
-from keras_nlp.models.bart.bart_backbone import BartBackbone
-from keras_nlp.models.bart.bart_seq_2_seq_lm_preprocessor import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.bart.bart_backbone import BartBackbone
+from keras_nlp.src.models.bart.bart_seq_2_seq_lm_preprocessor import (
     BartSeq2SeqLMPreprocessor,
 )
-from keras_nlp.models.seq_2_seq_lm import Seq2SeqLM
-from keras_nlp.utils.tensor_utils import any_equal
+from keras_nlp.src.models.seq_2_seq_lm import Seq2SeqLM
+from keras_nlp.src.utils.tensor_utils import any_equal
 
 
 @keras_nlp_export("keras_nlp.models.BartSeq2SeqLM")
diff --git a/keras_nlp/models/bart/bart_seq_2_seq_lm_preprocessor.py b/keras_nlp/src/models/bart/bart_seq_2_seq_lm_preprocessor.py
similarity index 97%
rename from keras_nlp/models/bart/bart_seq_2_seq_lm_preprocessor.py
rename to keras_nlp/src/models/bart/bart_seq_2_seq_lm_preprocessor.py
index a96db3f436..8d6a70917f 100644
--- a/keras_nlp/models/bart/bart_seq_2_seq_lm_preprocessor.py
+++ b/keras_nlp/src/models/bart/bart_seq_2_seq_lm_preprocessor.py
@@ -22,13 +22,13 @@
     )
 from absl import logging
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import ops
-from keras_nlp.models.bart.bart_preprocessor import BartPreprocessor
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.bart.bart_preprocessor import BartPreprocessor
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.BartSeq2SeqLMPreprocessor")
diff --git a/keras_nlp/models/bart/bart_tokenizer.py b/keras_nlp/src/models/bart/bart_tokenizer.py
similarity index 97%
rename from keras_nlp/models/bart/bart_tokenizer.py
rename to keras_nlp/src/models/bart/bart_tokenizer.py
index c4e3d1204d..4dad21dc5f 100644
--- a/keras_nlp/models/bart/bart_tokenizer.py
+++ b/keras_nlp/src/models/bart/bart_tokenizer.py
@@ -13,8 +13,8 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.tokenizers.byte_pair_tokenizer import BytePairTokenizer
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.tokenizers.byte_pair_tokenizer import BytePairTokenizer
 
 
 @keras_nlp_export("keras_nlp.models.BartTokenizer")
diff --git a/keras_nlp/models/bert/__init__.py b/keras_nlp/src/models/bert/__init__.py
similarity index 64%
rename from keras_nlp/models/bert/__init__.py
rename to keras_nlp/src/models/bert/__init__.py
index a34b85d0eb..1563af27c4 100644
--- a/keras_nlp/models/bert/__init__.py
+++ b/keras_nlp/src/models/bert/__init__.py
@@ -12,12 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.models.bert.bert_backbone import BertBackbone
-from keras_nlp.models.bert.bert_classifier import BertClassifier
-from keras_nlp.models.bert.bert_presets import backbone_presets
-from keras_nlp.models.bert.bert_presets import classifier_presets
-from keras_nlp.models.bert.bert_tokenizer import BertTokenizer
-from keras_nlp.utils.preset_utils import register_presets
+from keras_nlp.src.models.bert.bert_backbone import BertBackbone
+from keras_nlp.src.models.bert.bert_classifier import BertClassifier
+from keras_nlp.src.models.bert.bert_presets import backbone_presets
+from keras_nlp.src.models.bert.bert_presets import classifier_presets
+from keras_nlp.src.models.bert.bert_tokenizer import BertTokenizer
+from keras_nlp.src.utils.preset_utils import register_presets
 
 register_presets(backbone_presets, (BertBackbone, BertTokenizer))
 register_presets(classifier_presets, (BertClassifier, BertTokenizer))
diff --git a/keras_nlp/models/bert/bert_backbone.py b/keras_nlp/src/models/bert/bert_backbone.py
similarity index 94%
rename from keras_nlp/models/bert/bert_backbone.py
rename to keras_nlp/src/models/bert/bert_backbone.py
index 1f5a02dbb1..21d1d6ce82 100644
--- a/keras_nlp/models/bert/bert_backbone.py
+++ b/keras_nlp/src/models/bert/bert_backbone.py
@@ -12,13 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.layers.modeling.position_embedding import PositionEmbedding
-from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding
-from keras_nlp.layers.modeling.transformer_encoder import TransformerEncoder
-from keras_nlp.models.backbone import Backbone
-from keras_nlp.utils.keras_utils import gelu_approximate
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.modeling.position_embedding import PositionEmbedding
+from keras_nlp.src.layers.modeling.reversible_embedding import (
+    ReversibleEmbedding,
+)
+from keras_nlp.src.layers.modeling.transformer_encoder import TransformerEncoder
+from keras_nlp.src.models.backbone import Backbone
+from keras_nlp.src.utils.keras_utils import gelu_approximate
 
 
 def bert_kernel_initializer(stddev=0.02):
diff --git a/keras_nlp/models/bert/bert_classifier.py b/keras_nlp/src/models/bert/bert_classifier.py
similarity index 94%
rename from keras_nlp/models/bert/bert_classifier.py
rename to keras_nlp/src/models/bert/bert_classifier.py
index 8b4a927af9..bf2d75ec79 100644
--- a/keras_nlp/models/bert/bert_classifier.py
+++ b/keras_nlp/src/models/bert/bert_classifier.py
@@ -12,12 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.models.bert.bert_backbone import BertBackbone
-from keras_nlp.models.bert.bert_backbone import bert_kernel_initializer
-from keras_nlp.models.bert.bert_preprocessor import BertPreprocessor
-from keras_nlp.models.classifier import Classifier
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.models.bert.bert_backbone import BertBackbone
+from keras_nlp.src.models.bert.bert_backbone import bert_kernel_initializer
+from keras_nlp.src.models.bert.bert_preprocessor import BertPreprocessor
+from keras_nlp.src.models.classifier import Classifier
 
 
 @keras_nlp_export("keras_nlp.models.BertClassifier")
diff --git a/keras_nlp/models/bert/bert_masked_lm.py b/keras_nlp/src/models/bert/bert_masked_lm.py
similarity index 90%
rename from keras_nlp/models/bert/bert_masked_lm.py
rename to keras_nlp/src/models/bert/bert_masked_lm.py
index 5966c91cdb..bdbfb9a3cc 100644
--- a/keras_nlp/models/bert/bert_masked_lm.py
+++ b/keras_nlp/src/models/bert/bert_masked_lm.py
@@ -12,15 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.layers.modeling.masked_lm_head import MaskedLMHead
-from keras_nlp.models.bert.bert_backbone import BertBackbone
-from keras_nlp.models.bert.bert_backbone import bert_kernel_initializer
-from keras_nlp.models.bert.bert_masked_lm_preprocessor import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.modeling.masked_lm_head import MaskedLMHead
+from keras_nlp.src.models.bert.bert_backbone import BertBackbone
+from keras_nlp.src.models.bert.bert_backbone import bert_kernel_initializer
+from keras_nlp.src.models.bert.bert_masked_lm_preprocessor import (
     BertMaskedLMPreprocessor,
 )
-from keras_nlp.models.masked_lm import MaskedLM
+from keras_nlp.src.models.masked_lm import MaskedLM
 
 
 @keras_nlp_export("keras_nlp.models.BertMaskedLM")
diff --git a/keras_nlp/models/bert/bert_masked_lm_preprocessor.py b/keras_nlp/src/models/bert/bert_masked_lm_preprocessor.py
similarity index 96%
rename from keras_nlp/models/bert/bert_masked_lm_preprocessor.py
rename to keras_nlp/src/models/bert/bert_masked_lm_preprocessor.py
index cdc61fbac3..c91da343b8 100644
--- a/keras_nlp/models/bert/bert_masked_lm_preprocessor.py
+++ b/keras_nlp/src/models/bert/bert_masked_lm_preprocessor.py
@@ -14,12 +14,12 @@
 
 from absl import logging
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.masked_lm_mask_generator import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.masked_lm_mask_generator import (
     MaskedLMMaskGenerator,
 )
-from keras_nlp.models.bert.bert_preprocessor import BertPreprocessor
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.models.bert.bert_preprocessor import BertPreprocessor
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.BertMaskedLMPreprocessor")
diff --git a/keras_nlp/models/bert/bert_preprocessor.py b/keras_nlp/src/models/bert/bert_preprocessor.py
similarity index 94%
rename from keras_nlp/models/bert/bert_preprocessor.py
rename to keras_nlp/src/models/bert/bert_preprocessor.py
index 2975eae9c6..04dfb4657d 100644
--- a/keras_nlp/models/bert/bert_preprocessor.py
+++ b/keras_nlp/src/models/bert/bert_preprocessor.py
@@ -12,16 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.multi_segment_packer import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.multi_segment_packer import (
     MultiSegmentPacker,
 )
-from keras_nlp.models.bert.bert_tokenizer import BertTokenizer
-from keras_nlp.models.preprocessor import Preprocessor
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.models.bert.bert_tokenizer import BertTokenizer
+from keras_nlp.src.models.preprocessor import Preprocessor
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.BertPreprocessor")
diff --git a/keras_nlp/models/bert/bert_presets.py b/keras_nlp/src/models/bert/bert_presets.py
similarity index 100%
rename from keras_nlp/models/bert/bert_presets.py
rename to keras_nlp/src/models/bert/bert_presets.py
diff --git a/keras_nlp/models/bert/bert_tokenizer.py b/keras_nlp/src/models/bert/bert_tokenizer.py
similarity index 96%
rename from keras_nlp/models/bert/bert_tokenizer.py
rename to keras_nlp/src/models/bert/bert_tokenizer.py
index 819bf2f63f..dcaa66b015 100644
--- a/keras_nlp/models/bert/bert_tokenizer.py
+++ b/keras_nlp/src/models/bert/bert_tokenizer.py
@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.tokenizers.word_piece_tokenizer import WordPieceTokenizer
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.tokenizers.word_piece_tokenizer import WordPieceTokenizer
 
 
 @keras_nlp_export("keras_nlp.models.BertTokenizer")
diff --git a/keras_nlp/models/bloom/__init__.py b/keras_nlp/src/models/bloom/__init__.py
similarity index 70%
rename from keras_nlp/models/bloom/__init__.py
rename to keras_nlp/src/models/bloom/__init__.py
index 2b7ad787b4..fe7e0699fa 100644
--- a/keras_nlp/models/bloom/__init__.py
+++ b/keras_nlp/src/models/bloom/__init__.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.models.bloom.bloom_backbone import BloomBackbone
-from keras_nlp.models.bloom.bloom_presets import backbone_presets
-from keras_nlp.models.bloom.bloom_tokenizer import BloomTokenizer
-from keras_nlp.utils.preset_utils import register_presets
+from keras_nlp.src.models.bloom.bloom_backbone import BloomBackbone
+from keras_nlp.src.models.bloom.bloom_presets import backbone_presets
+from keras_nlp.src.models.bloom.bloom_tokenizer import BloomTokenizer
+from keras_nlp.src.utils.preset_utils import register_presets
 
 register_presets(backbone_presets, (BloomBackbone, BloomTokenizer))
diff --git a/keras_nlp/models/bloom/bloom_attention.py b/keras_nlp/src/models/bloom/bloom_attention.py
similarity index 96%
rename from keras_nlp/models/bloom/bloom_attention.py
rename to keras_nlp/src/models/bloom/bloom_attention.py
index e36c6fac62..c3bd29c373 100644
--- a/keras_nlp/models/bloom/bloom_attention.py
+++ b/keras_nlp/src/models/bloom/bloom_attention.py
@@ -13,10 +13,10 @@
 # limitations under the License.
 import math
 
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.layers.modeling.alibi_bias import AlibiBias
-from keras_nlp.utils.keras_utils import clone_initializer
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.layers.modeling.alibi_bias import AlibiBias
+from keras_nlp.src.utils.keras_utils import clone_initializer
 
 
 class BloomAttention(keras.layers.Layer):
diff --git a/keras_nlp/models/bloom/bloom_backbone.py b/keras_nlp/src/models/bloom/bloom_backbone.py
similarity index 95%
rename from keras_nlp/models/bloom/bloom_backbone.py
rename to keras_nlp/src/models/bloom/bloom_backbone.py
index 4e153b868d..d3ffda1017 100644
--- a/keras_nlp/models/bloom/bloom_backbone.py
+++ b/keras_nlp/src/models/bloom/bloom_backbone.py
@@ -12,11 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding
-from keras_nlp.models.backbone import Backbone
-from keras_nlp.models.bloom.bloom_decoder import BloomDecoder
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.modeling.reversible_embedding import (
+    ReversibleEmbedding,
+)
+from keras_nlp.src.models.backbone import Backbone
+from keras_nlp.src.models.bloom.bloom_decoder import BloomDecoder
 
 
 def _bloom_kernel_initializer(stddev=0.02):
diff --git a/keras_nlp/models/bloom/bloom_causal_lm.py b/keras_nlp/src/models/bloom/bloom_causal_lm.py
similarity index 97%
rename from keras_nlp/models/bloom/bloom_causal_lm.py
rename to keras_nlp/src/models/bloom/bloom_causal_lm.py
index 823b93fc32..886198550f 100644
--- a/keras_nlp/models/bloom/bloom_causal_lm.py
+++ b/keras_nlp/src/models/bloom/bloom_causal_lm.py
@@ -13,14 +13,14 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import ops
-from keras_nlp.models.bloom.bloom_backbone import BloomBackbone
-from keras_nlp.models.bloom.bloom_causal_lm_preprocessor import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.bloom.bloom_backbone import BloomBackbone
+from keras_nlp.src.models.bloom.bloom_causal_lm_preprocessor import (
     BloomCausalLMPreprocessor,
 )
-from keras_nlp.models.causal_lm import CausalLM
-from keras_nlp.utils.tensor_utils import any_equal
+from keras_nlp.src.models.causal_lm import CausalLM
+from keras_nlp.src.utils.tensor_utils import any_equal
 
 
 @keras_nlp_export("keras_nlp.models.BloomCausalLM")
diff --git a/keras_nlp/models/bloom/bloom_causal_lm_preprocessor.py b/keras_nlp/src/models/bloom/bloom_causal_lm_preprocessor.py
similarity index 96%
rename from keras_nlp/models/bloom/bloom_causal_lm_preprocessor.py
rename to keras_nlp/src/models/bloom/bloom_causal_lm_preprocessor.py
index c75bcdb864..97477a99bd 100644
--- a/keras_nlp/models/bloom/bloom_causal_lm_preprocessor.py
+++ b/keras_nlp/src/models/bloom/bloom_causal_lm_preprocessor.py
@@ -21,13 +21,13 @@
     )
 from absl import logging
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import ops
-from keras_nlp.models.bloom.bloom_preprocessor import BloomPreprocessor
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.bloom.bloom_preprocessor import BloomPreprocessor
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.BloomCausalLMPreprocessor")
diff --git a/keras_nlp/models/bloom/bloom_decoder.py b/keras_nlp/src/models/bloom/bloom_decoder.py
similarity index 94%
rename from keras_nlp/models/bloom/bloom_decoder.py
rename to keras_nlp/src/models/bloom/bloom_decoder.py
index c6478e7270..496972646d 100644
--- a/keras_nlp/models/bloom/bloom_decoder.py
+++ b/keras_nlp/src/models/bloom/bloom_decoder.py
@@ -11,18 +11,18 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# from keras_nlp.backend import keras
-# from keras_nlp.backend import ops
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.layers.modeling.transformer_layer_utils import (
+# from keras_nlp.src.backend import keras
+# from keras_nlp.src.backend import ops
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.layers.modeling.transformer_layer_utils import (
     compute_causal_mask,
 )
-from keras_nlp.layers.modeling.transformer_layer_utils import (
+from keras_nlp.src.layers.modeling.transformer_layer_utils import (
     merge_padding_and_attention_mask,
 )
-from keras_nlp.models.bloom.bloom_attention import BloomAttention
-from keras_nlp.utils.keras_utils import clone_initializer
+from keras_nlp.src.models.bloom.bloom_attention import BloomAttention
+from keras_nlp.src.utils.keras_utils import clone_initializer
 
 
 class BloomDecoder(keras.layers.Layer):
diff --git a/keras_nlp/models/bloom/bloom_preprocessor.py b/keras_nlp/src/models/bloom/bloom_preprocessor.py
similarity index 94%
rename from keras_nlp/models/bloom/bloom_preprocessor.py
rename to keras_nlp/src/models/bloom/bloom_preprocessor.py
index dfaac0332d..4639e423f6 100644
--- a/keras_nlp/models/bloom/bloom_preprocessor.py
+++ b/keras_nlp/src/models/bloom/bloom_preprocessor.py
@@ -13,14 +13,14 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.start_end_packer import StartEndPacker
-from keras_nlp.models.bloom.bloom_tokenizer import BloomTokenizer
-from keras_nlp.models.preprocessor import Preprocessor
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.start_end_packer import StartEndPacker
+from keras_nlp.src.models.bloom.bloom_tokenizer import BloomTokenizer
+from keras_nlp.src.models.preprocessor import Preprocessor
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.BloomPreprocessor")
diff --git a/keras_nlp/models/bloom/bloom_presets.py b/keras_nlp/src/models/bloom/bloom_presets.py
similarity index 100%
rename from keras_nlp/models/bloom/bloom_presets.py
rename to keras_nlp/src/models/bloom/bloom_presets.py
diff --git a/keras_nlp/models/bloom/bloom_tokenizer.py b/keras_nlp/src/models/bloom/bloom_tokenizer.py
similarity index 97%
rename from keras_nlp/models/bloom/bloom_tokenizer.py
rename to keras_nlp/src/models/bloom/bloom_tokenizer.py
index 6c6097e4ce..b248f344b9 100644
--- a/keras_nlp/models/bloom/bloom_tokenizer.py
+++ b/keras_nlp/src/models/bloom/bloom_tokenizer.py
@@ -13,8 +13,8 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.tokenizers.byte_pair_tokenizer import BytePairTokenizer
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.tokenizers.byte_pair_tokenizer import BytePairTokenizer
 
 
 @keras_nlp_export("keras_nlp.models.BloomTokenizer")
diff --git a/keras_nlp/models/causal_lm.py b/keras_nlp/src/models/causal_lm.py
similarity index 97%
rename from keras_nlp/models/causal_lm.py
rename to keras_nlp/src/models/causal_lm.py
index 247b2f05e9..3493532e2c 100644
--- a/keras_nlp/models/causal_lm.py
+++ b/keras_nlp/src/models/causal_lm.py
@@ -24,13 +24,13 @@
     )
 import tree
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import config
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.models.task import Task
-from keras_nlp.samplers.serialization import get as get_sampler
-from keras_nlp.utils.tensor_utils import tensor_to_list
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import config
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.task import Task
+from keras_nlp.src.samplers.serialization import get as get_sampler
+from keras_nlp.src.utils.tensor_utils import tensor_to_list
 
 
 @keras_nlp_export("keras_nlp.models.CausalLM")
diff --git a/keras_nlp/models/classifier.py b/keras_nlp/src/models/classifier.py
similarity index 96%
rename from keras_nlp/models/classifier.py
rename to keras_nlp/src/models/classifier.py
index 0e933d4e75..61fb6c9a09 100644
--- a/keras_nlp/models/classifier.py
+++ b/keras_nlp/src/models/classifier.py
@@ -11,10 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import config
-from keras_nlp.backend import keras
-from keras_nlp.models.task import Task
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import config
+from keras_nlp.src.backend import keras
+from keras_nlp.src.models.task import Task
 
 
 @keras_nlp_export("keras_nlp.models.Classifier")
diff --git a/keras_nlp/models/deberta_v3/__init__.py b/keras_nlp/src/models/deberta_v3/__init__.py
similarity index 67%
rename from keras_nlp/models/deberta_v3/__init__.py
rename to keras_nlp/src/models/deberta_v3/__init__.py
index 4a5f6df5cd..b703983725 100644
--- a/keras_nlp/models/deberta_v3/__init__.py
+++ b/keras_nlp/src/models/deberta_v3/__init__.py
@@ -12,9 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.models.deberta_v3.deberta_v3_backbone import DebertaV3Backbone
-from keras_nlp.models.deberta_v3.deberta_v3_presets import backbone_presets
-from keras_nlp.models.deberta_v3.deberta_v3_tokenizer import DebertaV3Tokenizer
-from keras_nlp.utils.preset_utils import register_presets
+from keras_nlp.src.models.deberta_v3.deberta_v3_backbone import (
+    DebertaV3Backbone,
+)
+from keras_nlp.src.models.deberta_v3.deberta_v3_presets import backbone_presets
+from keras_nlp.src.models.deberta_v3.deberta_v3_tokenizer import (
+    DebertaV3Tokenizer,
+)
+from keras_nlp.src.utils.preset_utils import register_presets
 
 register_presets(backbone_presets, (DebertaV3Backbone, DebertaV3Tokenizer))
diff --git a/keras_nlp/models/deberta_v3/deberta_v3_backbone.py b/keras_nlp/src/models/deberta_v3/deberta_v3_backbone.py
similarity index 94%
rename from keras_nlp/models/deberta_v3/deberta_v3_backbone.py
rename to keras_nlp/src/models/deberta_v3/deberta_v3_backbone.py
index 0029013098..2bd227cdeb 100644
--- a/keras_nlp/models/deberta_v3/deberta_v3_backbone.py
+++ b/keras_nlp/src/models/deberta_v3/deberta_v3_backbone.py
@@ -13,14 +13,16 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding
-from keras_nlp.models.backbone import Backbone
-from keras_nlp.models.deberta_v3.disentangled_attention_encoder import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.modeling.reversible_embedding import (
+    ReversibleEmbedding,
+)
+from keras_nlp.src.models.backbone import Backbone
+from keras_nlp.src.models.deberta_v3.disentangled_attention_encoder import (
     DisentangledAttentionEncoder,
 )
-from keras_nlp.models.deberta_v3.relative_embedding import RelativeEmbedding
+from keras_nlp.src.models.deberta_v3.relative_embedding import RelativeEmbedding
 
 
 def deberta_kernel_initializer(stddev=0.02):
diff --git a/keras_nlp/models/deberta_v3/deberta_v3_classifier.py b/keras_nlp/src/models/deberta_v3/deberta_v3_classifier.py
similarity index 95%
rename from keras_nlp/models/deberta_v3/deberta_v3_classifier.py
rename to keras_nlp/src/models/deberta_v3/deberta_v3_classifier.py
index 848e4c342c..4886e2543d 100644
--- a/keras_nlp/models/deberta_v3/deberta_v3_classifier.py
+++ b/keras_nlp/src/models/deberta_v3/deberta_v3_classifier.py
@@ -13,14 +13,16 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.models.classifier import Classifier
-from keras_nlp.models.deberta_v3.deberta_v3_backbone import DebertaV3Backbone
-from keras_nlp.models.deberta_v3.deberta_v3_backbone import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.models.classifier import Classifier
+from keras_nlp.src.models.deberta_v3.deberta_v3_backbone import (
+    DebertaV3Backbone,
+)
+from keras_nlp.src.models.deberta_v3.deberta_v3_backbone import (
     deberta_kernel_initializer,
 )
-from keras_nlp.models.deberta_v3.deberta_v3_preprocessor import (
+from keras_nlp.src.models.deberta_v3.deberta_v3_preprocessor import (
     DebertaV3Preprocessor,
 )
 
diff --git a/keras_nlp/models/deberta_v3/deberta_v3_masked_lm.py b/keras_nlp/src/models/deberta_v3/deberta_v3_masked_lm.py
similarity index 90%
rename from keras_nlp/models/deberta_v3/deberta_v3_masked_lm.py
rename to keras_nlp/src/models/deberta_v3/deberta_v3_masked_lm.py
index bad6f61d1e..07973c0d25 100644
--- a/keras_nlp/models/deberta_v3/deberta_v3_masked_lm.py
+++ b/keras_nlp/src/models/deberta_v3/deberta_v3_masked_lm.py
@@ -13,17 +13,19 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.layers.modeling.masked_lm_head import MaskedLMHead
-from keras_nlp.models.deberta_v3.deberta_v3_backbone import DebertaV3Backbone
-from keras_nlp.models.deberta_v3.deberta_v3_backbone import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.modeling.masked_lm_head import MaskedLMHead
+from keras_nlp.src.models.deberta_v3.deberta_v3_backbone import (
+    DebertaV3Backbone,
+)
+from keras_nlp.src.models.deberta_v3.deberta_v3_backbone import (
     deberta_kernel_initializer,
 )
-from keras_nlp.models.deberta_v3.deberta_v3_masked_lm_preprocessor import (
+from keras_nlp.src.models.deberta_v3.deberta_v3_masked_lm_preprocessor import (
     DebertaV3MaskedLMPreprocessor,
 )
-from keras_nlp.models.masked_lm import MaskedLM
+from keras_nlp.src.models.masked_lm import MaskedLM
 
 
 @keras_nlp_export("keras_nlp.models.DebertaV3MaskedLM")
diff --git a/keras_nlp/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py b/keras_nlp/src/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py
similarity index 96%
rename from keras_nlp/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py
rename to keras_nlp/src/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py
index 519b0b4fca..4decc65219 100644
--- a/keras_nlp/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py
+++ b/keras_nlp/src/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py
@@ -14,14 +14,14 @@
 
 from absl import logging
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.masked_lm_mask_generator import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.masked_lm_mask_generator import (
     MaskedLMMaskGenerator,
 )
-from keras_nlp.models.deberta_v3.deberta_v3_preprocessor import (
+from keras_nlp.src.models.deberta_v3.deberta_v3_preprocessor import (
     DebertaV3Preprocessor,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.DebertaV3MaskedLMPreprocessor")
diff --git a/keras_nlp/models/deberta_v3/deberta_v3_preprocessor.py b/keras_nlp/src/models/deberta_v3/deberta_v3_preprocessor.py
similarity index 95%
rename from keras_nlp/models/deberta_v3/deberta_v3_preprocessor.py
rename to keras_nlp/src/models/deberta_v3/deberta_v3_preprocessor.py
index 67466ef948..b234fe4cba 100644
--- a/keras_nlp/models/deberta_v3/deberta_v3_preprocessor.py
+++ b/keras_nlp/src/models/deberta_v3/deberta_v3_preprocessor.py
@@ -13,16 +13,18 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.multi_segment_packer import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.multi_segment_packer import (
     MultiSegmentPacker,
 )
-from keras_nlp.models.deberta_v3.deberta_v3_tokenizer import DebertaV3Tokenizer
-from keras_nlp.models.preprocessor import Preprocessor
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.models.deberta_v3.deberta_v3_tokenizer import (
+    DebertaV3Tokenizer,
+)
+from keras_nlp.src.models.preprocessor import Preprocessor
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.DebertaV3Preprocessor")
diff --git a/keras_nlp/models/deberta_v3/deberta_v3_presets.py b/keras_nlp/src/models/deberta_v3/deberta_v3_presets.py
similarity index 100%
rename from keras_nlp/models/deberta_v3/deberta_v3_presets.py
rename to keras_nlp/src/models/deberta_v3/deberta_v3_presets.py
diff --git a/keras_nlp/models/deberta_v3/deberta_v3_tokenizer.py b/keras_nlp/src/models/deberta_v3/deberta_v3_tokenizer.py
similarity index 97%
rename from keras_nlp/models/deberta_v3/deberta_v3_tokenizer.py
rename to keras_nlp/src/models/deberta_v3/deberta_v3_tokenizer.py
index c45f83cff2..27be306724 100644
--- a/keras_nlp/models/deberta_v3/deberta_v3_tokenizer.py
+++ b/keras_nlp/src/models/deberta_v3/deberta_v3_tokenizer.py
@@ -21,8 +21,10 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.tokenizers.sentence_piece_tokenizer import SentencePieceTokenizer
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.tokenizers.sentence_piece_tokenizer import (
+    SentencePieceTokenizer,
+)
 
 
 @keras_nlp_export("keras_nlp.models.DebertaV3Tokenizer")
diff --git a/keras_nlp/models/deberta_v3/disentangled_attention_encoder.py b/keras_nlp/src/models/deberta_v3/disentangled_attention_encoder.py
similarity index 97%
rename from keras_nlp/models/deberta_v3/disentangled_attention_encoder.py
rename to keras_nlp/src/models/deberta_v3/disentangled_attention_encoder.py
index be79bc98ff..df0f6fecc9 100644
--- a/keras_nlp/models/deberta_v3/disentangled_attention_encoder.py
+++ b/keras_nlp/src/models/deberta_v3/disentangled_attention_encoder.py
@@ -12,13 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.backend import keras
-from keras_nlp.models.deberta_v3.disentangled_self_attention import (
+from keras_nlp.src.backend import keras
+from keras_nlp.src.models.deberta_v3.disentangled_self_attention import (
     DisentangledSelfAttention,
 )
-from keras_nlp.utils.keras_utils import clone_initializer
+from keras_nlp.src.utils.keras_utils import clone_initializer
 
-from keras_nlp.layers.modeling.transformer_layer_utils import (  # isort:skip
+from keras_nlp.src.layers.modeling.transformer_layer_utils import (  # isort:skip
     merge_padding_and_attention_mask,
 )
 
diff --git a/keras_nlp/models/deberta_v3/disentangled_self_attention.py b/keras_nlp/src/models/deberta_v3/disentangled_self_attention.py
similarity index 98%
rename from keras_nlp/models/deberta_v3/disentangled_self_attention.py
rename to keras_nlp/src/models/deberta_v3/disentangled_self_attention.py
index 48730f2ad6..dbede909de 100644
--- a/keras_nlp/models/deberta_v3/disentangled_self_attention.py
+++ b/keras_nlp/src/models/deberta_v3/disentangled_self_attention.py
@@ -14,9 +14,9 @@
 
 import math
 
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.utils.keras_utils import clone_initializer
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.utils.keras_utils import clone_initializer
 
 
 class DisentangledSelfAttention(keras.layers.Layer):
diff --git a/keras_nlp/models/deberta_v3/relative_embedding.py b/keras_nlp/src/models/deberta_v3/relative_embedding.py
similarity index 97%
rename from keras_nlp/models/deberta_v3/relative_embedding.py
rename to keras_nlp/src/models/deberta_v3/relative_embedding.py
index f727ce0568..5dba945923 100644
--- a/keras_nlp/models/deberta_v3/relative_embedding.py
+++ b/keras_nlp/src/models/deberta_v3/relative_embedding.py
@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
 
 
 class RelativeEmbedding(keras.layers.Layer):
diff --git a/keras_nlp/models/distil_bert/__init__.py b/keras_nlp/src/models/distil_bert/__init__.py
similarity index 68%
rename from keras_nlp/models/distil_bert/__init__.py
rename to keras_nlp/src/models/distil_bert/__init__.py
index beb3742c9e..d809d30315 100644
--- a/keras_nlp/models/distil_bert/__init__.py
+++ b/keras_nlp/src/models/distil_bert/__init__.py
@@ -12,11 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.models.distil_bert.distil_bert_backbone import DistilBertBackbone
-from keras_nlp.models.distil_bert.distil_bert_presets import backbone_presets
-from keras_nlp.models.distil_bert.distil_bert_tokenizer import (
+from keras_nlp.src.models.distil_bert.distil_bert_backbone import (
+    DistilBertBackbone,
+)
+from keras_nlp.src.models.distil_bert.distil_bert_presets import (
+    backbone_presets,
+)
+from keras_nlp.src.models.distil_bert.distil_bert_tokenizer import (
     DistilBertTokenizer,
 )
-from keras_nlp.utils.preset_utils import register_presets
+from keras_nlp.src.utils.preset_utils import register_presets
 
 register_presets(backbone_presets, (DistilBertBackbone, DistilBertTokenizer))
diff --git a/keras_nlp/models/distil_bert/distil_bert_backbone.py b/keras_nlp/src/models/distil_bert/distil_bert_backbone.py
similarity index 95%
rename from keras_nlp/models/distil_bert/distil_bert_backbone.py
rename to keras_nlp/src/models/distil_bert/distil_bert_backbone.py
index 4374707f13..219842244d 100644
--- a/keras_nlp/models/distil_bert/distil_bert_backbone.py
+++ b/keras_nlp/src/models/distil_bert/distil_bert_backbone.py
@@ -13,13 +13,13 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.layers.modeling.token_and_position_embedding import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.modeling.token_and_position_embedding import (
     TokenAndPositionEmbedding,
 )
-from keras_nlp.layers.modeling.transformer_encoder import TransformerEncoder
-from keras_nlp.models.backbone import Backbone
+from keras_nlp.src.layers.modeling.transformer_encoder import TransformerEncoder
+from keras_nlp.src.models.backbone import Backbone
 
 
 def distilbert_kernel_initializer(stddev=0.02):
diff --git a/keras_nlp/models/distil_bert/distil_bert_classifier.py b/keras_nlp/src/models/distil_bert/distil_bert_classifier.py
similarity index 94%
rename from keras_nlp/models/distil_bert/distil_bert_classifier.py
rename to keras_nlp/src/models/distil_bert/distil_bert_classifier.py
index cb6560eade..b2b21203b9 100644
--- a/keras_nlp/models/distil_bert/distil_bert_classifier.py
+++ b/keras_nlp/src/models/distil_bert/distil_bert_classifier.py
@@ -13,14 +13,16 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.models.classifier import Classifier
-from keras_nlp.models.distil_bert.distil_bert_backbone import DistilBertBackbone
-from keras_nlp.models.distil_bert.distil_bert_backbone import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.models.classifier import Classifier
+from keras_nlp.src.models.distil_bert.distil_bert_backbone import (
+    DistilBertBackbone,
+)
+from keras_nlp.src.models.distil_bert.distil_bert_backbone import (
     distilbert_kernel_initializer,
 )
-from keras_nlp.models.distil_bert.distil_bert_preprocessor import (
+from keras_nlp.src.models.distil_bert.distil_bert_preprocessor import (
     DistilBertPreprocessor,
 )
 
diff --git a/keras_nlp/models/distil_bert/distil_bert_masked_lm.py b/keras_nlp/src/models/distil_bert/distil_bert_masked_lm.py
similarity index 90%
rename from keras_nlp/models/distil_bert/distil_bert_masked_lm.py
rename to keras_nlp/src/models/distil_bert/distil_bert_masked_lm.py
index e78a5bb85d..c24b96ce9d 100644
--- a/keras_nlp/models/distil_bert/distil_bert_masked_lm.py
+++ b/keras_nlp/src/models/distil_bert/distil_bert_masked_lm.py
@@ -13,17 +13,19 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.layers.modeling.masked_lm_head import MaskedLMHead
-from keras_nlp.models.distil_bert.distil_bert_backbone import DistilBertBackbone
-from keras_nlp.models.distil_bert.distil_bert_backbone import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.modeling.masked_lm_head import MaskedLMHead
+from keras_nlp.src.models.distil_bert.distil_bert_backbone import (
+    DistilBertBackbone,
+)
+from keras_nlp.src.models.distil_bert.distil_bert_backbone import (
     distilbert_kernel_initializer,
 )
-from keras_nlp.models.distil_bert.distil_bert_masked_lm_preprocessor import (
+from keras_nlp.src.models.distil_bert.distil_bert_masked_lm_preprocessor import (
     DistilBertMaskedLMPreprocessor,
 )
-from keras_nlp.models.masked_lm import MaskedLM
+from keras_nlp.src.models.masked_lm import MaskedLM
 
 
 @keras_nlp_export("keras_nlp.models.DistilBertMaskedLM")
diff --git a/keras_nlp/models/distil_bert/distil_bert_masked_lm_preprocessor.py b/keras_nlp/src/models/distil_bert/distil_bert_masked_lm_preprocessor.py
similarity index 96%
rename from keras_nlp/models/distil_bert/distil_bert_masked_lm_preprocessor.py
rename to keras_nlp/src/models/distil_bert/distil_bert_masked_lm_preprocessor.py
index f1360f58b7..375bbe80a7 100644
--- a/keras_nlp/models/distil_bert/distil_bert_masked_lm_preprocessor.py
+++ b/keras_nlp/src/models/distil_bert/distil_bert_masked_lm_preprocessor.py
@@ -14,14 +14,14 @@
 
 from absl import logging
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.masked_lm_mask_generator import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.masked_lm_mask_generator import (
     MaskedLMMaskGenerator,
 )
-from keras_nlp.models.distil_bert.distil_bert_preprocessor import (
+from keras_nlp.src.models.distil_bert.distil_bert_preprocessor import (
     DistilBertPreprocessor,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.DistilBertMaskedLMPreprocessor")
diff --git a/keras_nlp/models/distil_bert/distil_bert_preprocessor.py b/keras_nlp/src/models/distil_bert/distil_bert_preprocessor.py
similarity index 94%
rename from keras_nlp/models/distil_bert/distil_bert_preprocessor.py
rename to keras_nlp/src/models/distil_bert/distil_bert_preprocessor.py
index 1163f7f029..7b5b8b1840 100644
--- a/keras_nlp/models/distil_bert/distil_bert_preprocessor.py
+++ b/keras_nlp/src/models/distil_bert/distil_bert_preprocessor.py
@@ -13,18 +13,18 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.multi_segment_packer import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.multi_segment_packer import (
     MultiSegmentPacker,
 )
-from keras_nlp.models.distil_bert.distil_bert_tokenizer import (
+from keras_nlp.src.models.distil_bert.distil_bert_tokenizer import (
     DistilBertTokenizer,
 )
-from keras_nlp.models.preprocessor import Preprocessor
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.models.preprocessor import Preprocessor
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.DistilBertPreprocessor")
diff --git a/keras_nlp/models/distil_bert/distil_bert_presets.py b/keras_nlp/src/models/distil_bert/distil_bert_presets.py
similarity index 100%
rename from keras_nlp/models/distil_bert/distil_bert_presets.py
rename to keras_nlp/src/models/distil_bert/distil_bert_presets.py
diff --git a/keras_nlp/models/distil_bert/distil_bert_tokenizer.py b/keras_nlp/src/models/distil_bert/distil_bert_tokenizer.py
similarity index 96%
rename from keras_nlp/models/distil_bert/distil_bert_tokenizer.py
rename to keras_nlp/src/models/distil_bert/distil_bert_tokenizer.py
index c03792f761..a282be3086 100644
--- a/keras_nlp/models/distil_bert/distil_bert_tokenizer.py
+++ b/keras_nlp/src/models/distil_bert/distil_bert_tokenizer.py
@@ -13,8 +13,8 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.tokenizers.word_piece_tokenizer import WordPieceTokenizer
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.tokenizers.word_piece_tokenizer import WordPieceTokenizer
 
 
 @keras_nlp_export("keras_nlp.models.DistilBertTokenizer")
diff --git a/keras_nlp/models/electra/__init__.py b/keras_nlp/src/models/electra/__init__.py
similarity index 69%
rename from keras_nlp/models/electra/__init__.py
rename to keras_nlp/src/models/electra/__init__.py
index 0717b97a72..ee2c2bef22 100644
--- a/keras_nlp/models/electra/__init__.py
+++ b/keras_nlp/src/models/electra/__init__.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.models.electra.electra_backbone import ElectraBackbone
-from keras_nlp.models.electra.electra_presets import backbone_presets
-from keras_nlp.models.electra.electra_tokenizer import ElectraTokenizer
-from keras_nlp.utils.preset_utils import register_presets
+from keras_nlp.src.models.electra.electra_backbone import ElectraBackbone
+from keras_nlp.src.models.electra.electra_presets import backbone_presets
+from keras_nlp.src.models.electra.electra_tokenizer import ElectraTokenizer
+from keras_nlp.src.utils.preset_utils import register_presets
 
 register_presets(backbone_presets, (ElectraBackbone, ElectraTokenizer))
diff --git a/keras_nlp/models/electra/electra_backbone.py b/keras_nlp/src/models/electra/electra_backbone.py
similarity index 95%
rename from keras_nlp/models/electra/electra_backbone.py
rename to keras_nlp/src/models/electra/electra_backbone.py
index 3ee88de826..e18f4e9cd3 100644
--- a/keras_nlp/models/electra/electra_backbone.py
+++ b/keras_nlp/src/models/electra/electra_backbone.py
@@ -12,13 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.layers.modeling.position_embedding import PositionEmbedding
-from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding
-from keras_nlp.layers.modeling.transformer_encoder import TransformerEncoder
-from keras_nlp.models.backbone import Backbone
-from keras_nlp.utils.keras_utils import gelu_approximate
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.modeling.position_embedding import PositionEmbedding
+from keras_nlp.src.layers.modeling.reversible_embedding import (
+    ReversibleEmbedding,
+)
+from keras_nlp.src.layers.modeling.transformer_encoder import TransformerEncoder
+from keras_nlp.src.models.backbone import Backbone
+from keras_nlp.src.utils.keras_utils import gelu_approximate
 
 
 def electra_kernel_initializer(stddev=0.02):
diff --git a/keras_nlp/models/electra/electra_preprocessor.py b/keras_nlp/src/models/electra/electra_preprocessor.py
similarity index 93%
rename from keras_nlp/models/electra/electra_preprocessor.py
rename to keras_nlp/src/models/electra/electra_preprocessor.py
index 2ee3e294d8..84ea2f5b33 100644
--- a/keras_nlp/models/electra/electra_preprocessor.py
+++ b/keras_nlp/src/models/electra/electra_preprocessor.py
@@ -12,16 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.multi_segment_packer import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.multi_segment_packer import (
     MultiSegmentPacker,
 )
-from keras_nlp.models.electra.electra_tokenizer import ElectraTokenizer
-from keras_nlp.models.preprocessor import Preprocessor
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.models.electra.electra_tokenizer import ElectraTokenizer
+from keras_nlp.src.models.preprocessor import Preprocessor
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.ElectraPreprocessor")
diff --git a/keras_nlp/models/electra/electra_presets.py b/keras_nlp/src/models/electra/electra_presets.py
similarity index 100%
rename from keras_nlp/models/electra/electra_presets.py
rename to keras_nlp/src/models/electra/electra_presets.py
diff --git a/keras_nlp/models/electra/electra_tokenizer.py b/keras_nlp/src/models/electra/electra_tokenizer.py
similarity index 97%
rename from keras_nlp/models/electra/electra_tokenizer.py
rename to keras_nlp/src/models/electra/electra_tokenizer.py
index 583b756165..90dbf5f91b 100644
--- a/keras_nlp/models/electra/electra_tokenizer.py
+++ b/keras_nlp/src/models/electra/electra_tokenizer.py
@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.tokenizers import WordPieceTokenizer
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.tokenizers import WordPieceTokenizer
 
 
 @keras_nlp_export("keras_nlp.models.ElectraTokenizer")
diff --git a/keras_nlp/models/f_net/__init__.py b/keras_nlp/src/models/f_net/__init__.py
similarity index 70%
rename from keras_nlp/models/f_net/__init__.py
rename to keras_nlp/src/models/f_net/__init__.py
index 7921ed6ca3..05c2aefa25 100644
--- a/keras_nlp/models/f_net/__init__.py
+++ b/keras_nlp/src/models/f_net/__init__.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.models.f_net.f_net_backbone import FNetBackbone
-from keras_nlp.models.f_net.f_net_presets import backbone_presets
-from keras_nlp.models.f_net.f_net_tokenizer import FNetTokenizer
-from keras_nlp.utils.preset_utils import register_presets
+from keras_nlp.src.models.f_net.f_net_backbone import FNetBackbone
+from keras_nlp.src.models.f_net.f_net_presets import backbone_presets
+from keras_nlp.src.models.f_net.f_net_tokenizer import FNetTokenizer
+from keras_nlp.src.utils.preset_utils import register_presets
 
 register_presets(backbone_presets, (FNetBackbone, FNetTokenizer))
diff --git a/keras_nlp/models/f_net/f_net_backbone.py b/keras_nlp/src/models/f_net/f_net_backbone.py
similarity index 95%
rename from keras_nlp/models/f_net/f_net_backbone.py
rename to keras_nlp/src/models/f_net/f_net_backbone.py
index 91034a4359..449e414769 100644
--- a/keras_nlp/models/f_net/f_net_backbone.py
+++ b/keras_nlp/src/models/f_net/f_net_backbone.py
@@ -13,13 +13,15 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.layers.modeling.f_net_encoder import FNetEncoder
-from keras_nlp.layers.modeling.position_embedding import PositionEmbedding
-from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding
-from keras_nlp.models.backbone import Backbone
-from keras_nlp.utils.keras_utils import gelu_approximate
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.modeling.f_net_encoder import FNetEncoder
+from keras_nlp.src.layers.modeling.position_embedding import PositionEmbedding
+from keras_nlp.src.layers.modeling.reversible_embedding import (
+    ReversibleEmbedding,
+)
+from keras_nlp.src.models.backbone import Backbone
+from keras_nlp.src.utils.keras_utils import gelu_approximate
 
 
 def f_net_kernel_initializer(stddev=0.02):
diff --git a/keras_nlp/models/f_net/f_net_classifier.py b/keras_nlp/src/models/f_net/f_net_classifier.py
similarity index 93%
rename from keras_nlp/models/f_net/f_net_classifier.py
rename to keras_nlp/src/models/f_net/f_net_classifier.py
index d4c7f6260f..84f2d2f4d3 100644
--- a/keras_nlp/models/f_net/f_net_classifier.py
+++ b/keras_nlp/src/models/f_net/f_net_classifier.py
@@ -13,12 +13,12 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.models.classifier import Classifier
-from keras_nlp.models.f_net.f_net_backbone import FNetBackbone
-from keras_nlp.models.f_net.f_net_backbone import f_net_kernel_initializer
-from keras_nlp.models.f_net.f_net_preprocessor import FNetPreprocessor
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.models.classifier import Classifier
+from keras_nlp.src.models.f_net.f_net_backbone import FNetBackbone
+from keras_nlp.src.models.f_net.f_net_backbone import f_net_kernel_initializer
+from keras_nlp.src.models.f_net.f_net_preprocessor import FNetPreprocessor
 
 
 @keras_nlp_export("keras_nlp.models.FNetClassifier")
diff --git a/keras_nlp/models/f_net/f_net_masked_lm.py b/keras_nlp/src/models/f_net/f_net_masked_lm.py
similarity index 90%
rename from keras_nlp/models/f_net/f_net_masked_lm.py
rename to keras_nlp/src/models/f_net/f_net_masked_lm.py
index 8ef197df69..0fff8c1d9d 100644
--- a/keras_nlp/models/f_net/f_net_masked_lm.py
+++ b/keras_nlp/src/models/f_net/f_net_masked_lm.py
@@ -13,15 +13,15 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.layers.modeling.masked_lm_head import MaskedLMHead
-from keras_nlp.models.f_net.f_net_backbone import FNetBackbone
-from keras_nlp.models.f_net.f_net_backbone import f_net_kernel_initializer
-from keras_nlp.models.f_net.f_net_masked_lm_preprocessor import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.modeling.masked_lm_head import MaskedLMHead
+from keras_nlp.src.models.f_net.f_net_backbone import FNetBackbone
+from keras_nlp.src.models.f_net.f_net_backbone import f_net_kernel_initializer
+from keras_nlp.src.models.f_net.f_net_masked_lm_preprocessor import (
     FNetMaskedLMPreprocessor,
 )
-from keras_nlp.models.masked_lm import MaskedLM
+from keras_nlp.src.models.masked_lm import MaskedLM
 
 
 @keras_nlp_export("keras_nlp.models.FNetMaskedLM")
diff --git a/keras_nlp/models/f_net/f_net_masked_lm_preprocessor.py b/keras_nlp/src/models/f_net/f_net_masked_lm_preprocessor.py
similarity index 96%
rename from keras_nlp/models/f_net/f_net_masked_lm_preprocessor.py
rename to keras_nlp/src/models/f_net/f_net_masked_lm_preprocessor.py
index 51b4a4d1e7..822061185c 100644
--- a/keras_nlp/models/f_net/f_net_masked_lm_preprocessor.py
+++ b/keras_nlp/src/models/f_net/f_net_masked_lm_preprocessor.py
@@ -14,12 +14,12 @@
 
 from absl import logging
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.masked_lm_mask_generator import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.masked_lm_mask_generator import (
     MaskedLMMaskGenerator,
 )
-from keras_nlp.models.f_net.f_net_preprocessor import FNetPreprocessor
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.models.f_net.f_net_preprocessor import FNetPreprocessor
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.FNetMaskedLMPreprocessor")
diff --git a/keras_nlp/models/f_net/f_net_preprocessor.py b/keras_nlp/src/models/f_net/f_net_preprocessor.py
similarity index 94%
rename from keras_nlp/models/f_net/f_net_preprocessor.py
rename to keras_nlp/src/models/f_net/f_net_preprocessor.py
index dfe1b71cfe..26ca75a2fb 100644
--- a/keras_nlp/models/f_net/f_net_preprocessor.py
+++ b/keras_nlp/src/models/f_net/f_net_preprocessor.py
@@ -13,16 +13,16 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.multi_segment_packer import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.multi_segment_packer import (
     MultiSegmentPacker,
 )
-from keras_nlp.models.f_net.f_net_tokenizer import FNetTokenizer
-from keras_nlp.models.preprocessor import Preprocessor
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.models.f_net.f_net_tokenizer import FNetTokenizer
+from keras_nlp.src.models.preprocessor import Preprocessor
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.FNetPreprocessor")
diff --git a/keras_nlp/models/f_net/f_net_presets.py b/keras_nlp/src/models/f_net/f_net_presets.py
similarity index 100%
rename from keras_nlp/models/f_net/f_net_presets.py
rename to keras_nlp/src/models/f_net/f_net_presets.py
diff --git a/keras_nlp/models/f_net/f_net_tokenizer.py b/keras_nlp/src/models/f_net/f_net_tokenizer.py
similarity index 95%
rename from keras_nlp/models/f_net/f_net_tokenizer.py
rename to keras_nlp/src/models/f_net/f_net_tokenizer.py
index 12a055b16c..0437fd8455 100644
--- a/keras_nlp/models/f_net/f_net_tokenizer.py
+++ b/keras_nlp/src/models/f_net/f_net_tokenizer.py
@@ -13,8 +13,10 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.tokenizers.sentence_piece_tokenizer import SentencePieceTokenizer
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.tokenizers.sentence_piece_tokenizer import (
+    SentencePieceTokenizer,
+)
 
 
 @keras_nlp_export("keras_nlp.models.FNetTokenizer")
diff --git a/keras_nlp/models/falcon/__init__.py b/keras_nlp/src/models/falcon/__init__.py
similarity index 70%
rename from keras_nlp/models/falcon/__init__.py
rename to keras_nlp/src/models/falcon/__init__.py
index cfc0b821cb..414c8650a9 100644
--- a/keras_nlp/models/falcon/__init__.py
+++ b/keras_nlp/src/models/falcon/__init__.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.models.falcon.falcon_backbone import FalconBackbone
-from keras_nlp.models.falcon.falcon_presets import backbone_presets
-from keras_nlp.models.falcon.falcon_tokenizer import FalconTokenizer
-from keras_nlp.utils.preset_utils import register_presets
+from keras_nlp.src.models.falcon.falcon_backbone import FalconBackbone
+from keras_nlp.src.models.falcon.falcon_presets import backbone_presets
+from keras_nlp.src.models.falcon.falcon_tokenizer import FalconTokenizer
+from keras_nlp.src.utils.preset_utils import register_presets
 
 register_presets(backbone_presets, (FalconBackbone, FalconTokenizer))
diff --git a/keras_nlp/models/falcon/falcon_attention.py b/keras_nlp/src/models/falcon/falcon_attention.py
similarity index 98%
rename from keras_nlp/models/falcon/falcon_attention.py
rename to keras_nlp/src/models/falcon/falcon_attention.py
index 0358ade54b..e725826969 100644
--- a/keras_nlp/models/falcon/falcon_attention.py
+++ b/keras_nlp/src/models/falcon/falcon_attention.py
@@ -13,8 +13,8 @@
 # limitations under the License.
 import math
 
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
 
 
 class FalconAttention(keras.layers.Layer):
diff --git a/keras_nlp/models/falcon/falcon_backbone.py b/keras_nlp/src/models/falcon/falcon_backbone.py
similarity index 94%
rename from keras_nlp/models/falcon/falcon_backbone.py
rename to keras_nlp/src/models/falcon/falcon_backbone.py
index 5a3a0fccda..53f33eccef 100644
--- a/keras_nlp/models/falcon/falcon_backbone.py
+++ b/keras_nlp/src/models/falcon/falcon_backbone.py
@@ -11,11 +11,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding
-from keras_nlp.models.backbone import Backbone
-from keras_nlp.models.falcon.falcon_transformer_decoder import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.modeling.reversible_embedding import (
+    ReversibleEmbedding,
+)
+from keras_nlp.src.models.backbone import Backbone
+from keras_nlp.src.models.falcon.falcon_transformer_decoder import (
     FalconTransformerDecoder,
 )
 
diff --git a/keras_nlp/models/falcon/falcon_causal_lm_preprocessor.py b/keras_nlp/src/models/falcon/falcon_causal_lm_preprocessor.py
similarity index 96%
rename from keras_nlp/models/falcon/falcon_causal_lm_preprocessor.py
rename to keras_nlp/src/models/falcon/falcon_causal_lm_preprocessor.py
index 83e01e1cf8..6ce531f1dd 100644
--- a/keras_nlp/models/falcon/falcon_causal_lm_preprocessor.py
+++ b/keras_nlp/src/models/falcon/falcon_causal_lm_preprocessor.py
@@ -21,13 +21,13 @@
     )
 from absl import logging
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import ops
-from keras_nlp.models.falcon.falcon_preprocessor import FalconPreprocessor
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.falcon.falcon_preprocessor import FalconPreprocessor
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.FalconCausalLMPreprocessor")
diff --git a/keras_nlp/models/falcon/falcon_preprocessor.py b/keras_nlp/src/models/falcon/falcon_preprocessor.py
similarity index 94%
rename from keras_nlp/models/falcon/falcon_preprocessor.py
rename to keras_nlp/src/models/falcon/falcon_preprocessor.py
index 8a14f3c255..fb3569ef22 100644
--- a/keras_nlp/models/falcon/falcon_preprocessor.py
+++ b/keras_nlp/src/models/falcon/falcon_preprocessor.py
@@ -13,14 +13,14 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.start_end_packer import StartEndPacker
-from keras_nlp.models.falcon.falcon_tokenizer import FalconTokenizer
-from keras_nlp.models.preprocessor import Preprocessor
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.start_end_packer import StartEndPacker
+from keras_nlp.src.models.falcon.falcon_tokenizer import FalconTokenizer
+from keras_nlp.src.models.preprocessor import Preprocessor
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.FalconPreprocessor")
diff --git a/keras_nlp/models/falcon/falcon_presets.py b/keras_nlp/src/models/falcon/falcon_presets.py
similarity index 100%
rename from keras_nlp/models/falcon/falcon_presets.py
rename to keras_nlp/src/models/falcon/falcon_presets.py
diff --git a/keras_nlp/models/falcon/falcon_tokenizer.py b/keras_nlp/src/models/falcon/falcon_tokenizer.py
similarity index 96%
rename from keras_nlp/models/falcon/falcon_tokenizer.py
rename to keras_nlp/src/models/falcon/falcon_tokenizer.py
index 80d7334fe7..7d00459008 100644
--- a/keras_nlp/models/falcon/falcon_tokenizer.py
+++ b/keras_nlp/src/models/falcon/falcon_tokenizer.py
@@ -13,8 +13,8 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.tokenizers.byte_pair_tokenizer import BytePairTokenizer
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.tokenizers.byte_pair_tokenizer import BytePairTokenizer
 
 
 @keras_nlp_export("keras_nlp.models.FalconTokenizer")
diff --git a/keras_nlp/models/falcon/falcon_transformer_decoder.py b/keras_nlp/src/models/falcon/falcon_transformer_decoder.py
similarity index 96%
rename from keras_nlp/models/falcon/falcon_transformer_decoder.py
rename to keras_nlp/src/models/falcon/falcon_transformer_decoder.py
index 3b29cedd7e..ffdb6303e3 100644
--- a/keras_nlp/models/falcon/falcon_transformer_decoder.py
+++ b/keras_nlp/src/models/falcon/falcon_transformer_decoder.py
@@ -13,15 +13,15 @@
 # limitations under the License.
 import math
 
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.layers.modeling.transformer_layer_utils import (
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.layers.modeling.transformer_layer_utils import (
     compute_causal_mask,
 )
-from keras_nlp.layers.modeling.transformer_layer_utils import (
+from keras_nlp.src.layers.modeling.transformer_layer_utils import (
     merge_padding_and_attention_mask,
 )
-from keras_nlp.models.falcon.falcon_attention import FalconAttention
+from keras_nlp.src.models.falcon.falcon_attention import FalconAttention
 
 
 class FalconTransformerDecoder(keras.layers.Layer):
diff --git a/keras_nlp/models/gemma/__init__.py b/keras_nlp/src/models/gemma/__init__.py
similarity index 70%
rename from keras_nlp/models/gemma/__init__.py
rename to keras_nlp/src/models/gemma/__init__.py
index b390926a21..f60483b813 100644
--- a/keras_nlp/models/gemma/__init__.py
+++ b/keras_nlp/src/models/gemma/__init__.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.models.gemma.gemma_backbone import GemmaBackbone
-from keras_nlp.models.gemma.gemma_presets import backbone_presets
-from keras_nlp.models.gemma.gemma_tokenizer import GemmaTokenizer
-from keras_nlp.utils.preset_utils import register_presets
+from keras_nlp.src.models.gemma.gemma_backbone import GemmaBackbone
+from keras_nlp.src.models.gemma.gemma_presets import backbone_presets
+from keras_nlp.src.models.gemma.gemma_tokenizer import GemmaTokenizer
+from keras_nlp.src.utils.preset_utils import register_presets
 
 register_presets(backbone_presets, (GemmaBackbone, GemmaTokenizer))
diff --git a/keras_nlp/models/gemma/gemma_attention.py b/keras_nlp/src/models/gemma/gemma_attention.py
similarity index 96%
rename from keras_nlp/models/gemma/gemma_attention.py
rename to keras_nlp/src/models/gemma/gemma_attention.py
index 4b391264a2..7c5593b01f 100644
--- a/keras_nlp/models/gemma/gemma_attention.py
+++ b/keras_nlp/src/models/gemma/gemma_attention.py
@@ -13,10 +13,10 @@
 # limitations under the License.
 import numpy as np
 
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.layers.modeling.rotary_embedding import RotaryEmbedding
-from keras_nlp.utils.keras_utils import clone_initializer
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.layers.modeling.rotary_embedding import RotaryEmbedding
+from keras_nlp.src.utils.keras_utils import clone_initializer
 
 
 class CachedGemmaAttention(keras.layers.Layer):
diff --git a/keras_nlp/models/gemma/gemma_backbone.py b/keras_nlp/src/models/gemma/gemma_backbone.py
similarity index 95%
rename from keras_nlp/models/gemma/gemma_backbone.py
rename to keras_nlp/src/models/gemma/gemma_backbone.py
index a7973f9dec..50e6fea2c0 100644
--- a/keras_nlp/models/gemma/gemma_backbone.py
+++ b/keras_nlp/src/models/gemma/gemma_backbone.py
@@ -13,14 +13,16 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import config
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding
-from keras_nlp.models.backbone import Backbone
-from keras_nlp.models.gemma.gemma_decoder_block import GemmaDecoderBlock
-from keras_nlp.models.gemma.rms_normalization import RMSNormalization
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import config
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.layers.modeling.reversible_embedding import (
+    ReversibleEmbedding,
+)
+from keras_nlp.src.models.backbone import Backbone
+from keras_nlp.src.models.gemma.gemma_decoder_block import GemmaDecoderBlock
+from keras_nlp.src.models.gemma.rms_normalization import RMSNormalization
 
 
 @keras_nlp_export("keras_nlp.models.GemmaBackbone")
diff --git a/keras_nlp/models/gemma/gemma_causal_lm.py b/keras_nlp/src/models/gemma/gemma_causal_lm.py
similarity index 97%
rename from keras_nlp/models/gemma/gemma_causal_lm.py
rename to keras_nlp/src/models/gemma/gemma_causal_lm.py
index 26e9aad8a2..ee0eec168d 100644
--- a/keras_nlp/models/gemma/gemma_causal_lm.py
+++ b/keras_nlp/src/models/gemma/gemma_causal_lm.py
@@ -13,15 +13,15 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.models.causal_lm import CausalLM
-from keras_nlp.models.gemma.gemma_backbone import GemmaBackbone
-from keras_nlp.models.gemma.gemma_causal_lm_preprocessor import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.causal_lm import CausalLM
+from keras_nlp.src.models.gemma.gemma_backbone import GemmaBackbone
+from keras_nlp.src.models.gemma.gemma_causal_lm_preprocessor import (
     GemmaCausalLMPreprocessor,
 )
-from keras_nlp.utils.tensor_utils import any_equal
+from keras_nlp.src.utils.tensor_utils import any_equal
 
 
 @keras_nlp_export("keras_nlp.models.GemmaCausalLM")
diff --git a/keras_nlp/models/gemma/gemma_causal_lm_preprocessor.py b/keras_nlp/src/models/gemma/gemma_causal_lm_preprocessor.py
similarity index 96%
rename from keras_nlp/models/gemma/gemma_causal_lm_preprocessor.py
rename to keras_nlp/src/models/gemma/gemma_causal_lm_preprocessor.py
index a75bb45379..a17aee2767 100644
--- a/keras_nlp/models/gemma/gemma_causal_lm_preprocessor.py
+++ b/keras_nlp/src/models/gemma/gemma_causal_lm_preprocessor.py
@@ -21,13 +21,13 @@
     )
 from absl import logging
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import ops
-from keras_nlp.models.gemma.gemma_preprocessor import GemmaPreprocessor
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.gemma.gemma_preprocessor import GemmaPreprocessor
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.GemmaCausalLMPreprocessor")
diff --git a/keras_nlp/models/gemma/gemma_decoder_block.py b/keras_nlp/src/models/gemma/gemma_decoder_block.py
similarity index 94%
rename from keras_nlp/models/gemma/gemma_decoder_block.py
rename to keras_nlp/src/models/gemma/gemma_decoder_block.py
index 0a91655fc4..5f0375b702 100644
--- a/keras_nlp/models/gemma/gemma_decoder_block.py
+++ b/keras_nlp/src/models/gemma/gemma_decoder_block.py
@@ -11,16 +11,16 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.layers.modeling.transformer_layer_utils import (
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.layers.modeling.transformer_layer_utils import (
     compute_causal_mask,
 )
-from keras_nlp.layers.modeling.transformer_layer_utils import (
+from keras_nlp.src.layers.modeling.transformer_layer_utils import (
     merge_padding_and_attention_mask,
 )
-from keras_nlp.models.gemma.gemma_attention import CachedGemmaAttention
-from keras_nlp.models.gemma.rms_normalization import RMSNormalization
+from keras_nlp.src.models.gemma.gemma_attention import CachedGemmaAttention
+from keras_nlp.src.models.gemma.rms_normalization import RMSNormalization
 
 
 class GemmaDecoderBlock(keras.layers.Layer):
diff --git a/keras_nlp/models/gemma/gemma_preprocessor.py b/keras_nlp/src/models/gemma/gemma_preprocessor.py
similarity index 94%
rename from keras_nlp/models/gemma/gemma_preprocessor.py
rename to keras_nlp/src/models/gemma/gemma_preprocessor.py
index 86db9a4e81..874a2fb09b 100644
--- a/keras_nlp/models/gemma/gemma_preprocessor.py
+++ b/keras_nlp/src/models/gemma/gemma_preprocessor.py
@@ -13,14 +13,14 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.start_end_packer import StartEndPacker
-from keras_nlp.models.gemma.gemma_tokenizer import GemmaTokenizer
-from keras_nlp.models.preprocessor import Preprocessor
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.start_end_packer import StartEndPacker
+from keras_nlp.src.models.gemma.gemma_tokenizer import GemmaTokenizer
+from keras_nlp.src.models.preprocessor import Preprocessor
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.GemmaPreprocessor")
diff --git a/keras_nlp/models/gemma/gemma_presets.py b/keras_nlp/src/models/gemma/gemma_presets.py
similarity index 100%
rename from keras_nlp/models/gemma/gemma_presets.py
rename to keras_nlp/src/models/gemma/gemma_presets.py
diff --git a/keras_nlp/models/gemma/gemma_tokenizer.py b/keras_nlp/src/models/gemma/gemma_tokenizer.py
similarity index 96%
rename from keras_nlp/models/gemma/gemma_tokenizer.py
rename to keras_nlp/src/models/gemma/gemma_tokenizer.py
index 7722d35f35..e87dee1b2e 100644
--- a/keras_nlp/models/gemma/gemma_tokenizer.py
+++ b/keras_nlp/src/models/gemma/gemma_tokenizer.py
@@ -12,8 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.tokenizers.sentence_piece_tokenizer import SentencePieceTokenizer
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.tokenizers.sentence_piece_tokenizer import (
+    SentencePieceTokenizer,
+)
 
 
 @keras_nlp_export("keras_nlp.models.GemmaTokenizer")
diff --git a/keras_nlp/models/gemma/rms_normalization.py b/keras_nlp/src/models/gemma/rms_normalization.py
similarity index 94%
rename from keras_nlp/models/gemma/rms_normalization.py
rename to keras_nlp/src/models/gemma/rms_normalization.py
index c3e4296020..8b1dbfc212 100644
--- a/keras_nlp/models/gemma/rms_normalization.py
+++ b/keras_nlp/src/models/gemma/rms_normalization.py
@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
 
 
 class RMSNormalization(keras.layers.Layer):
diff --git a/keras_nlp/models/gpt2/__init__.py b/keras_nlp/src/models/gpt2/__init__.py
similarity index 71%
rename from keras_nlp/models/gpt2/__init__.py
rename to keras_nlp/src/models/gpt2/__init__.py
index ad86022a86..b8fc50e9a2 100644
--- a/keras_nlp/models/gpt2/__init__.py
+++ b/keras_nlp/src/models/gpt2/__init__.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.models.gpt2.gpt2_backbone import GPT2Backbone
-from keras_nlp.models.gpt2.gpt2_presets import backbone_presets
-from keras_nlp.models.gpt2.gpt2_tokenizer import GPT2Tokenizer
-from keras_nlp.utils.preset_utils import register_presets
+from keras_nlp.src.models.gpt2.gpt2_backbone import GPT2Backbone
+from keras_nlp.src.models.gpt2.gpt2_presets import backbone_presets
+from keras_nlp.src.models.gpt2.gpt2_tokenizer import GPT2Tokenizer
+from keras_nlp.src.utils.preset_utils import register_presets
 
 register_presets(backbone_presets, (GPT2Backbone, GPT2Tokenizer))
diff --git a/keras_nlp/models/gpt2/gpt2_backbone.py b/keras_nlp/src/models/gpt2/gpt2_backbone.py
similarity index 93%
rename from keras_nlp/models/gpt2/gpt2_backbone.py
rename to keras_nlp/src/models/gpt2/gpt2_backbone.py
index 49929e4091..d9832b8ab6 100644
--- a/keras_nlp/models/gpt2/gpt2_backbone.py
+++ b/keras_nlp/src/models/gpt2/gpt2_backbone.py
@@ -13,13 +13,15 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.layers.modeling.position_embedding import PositionEmbedding
-from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding
-from keras_nlp.layers.modeling.transformer_decoder import TransformerDecoder
-from keras_nlp.models.backbone import Backbone
-from keras_nlp.utils.keras_utils import gelu_approximate
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.modeling.position_embedding import PositionEmbedding
+from keras_nlp.src.layers.modeling.reversible_embedding import (
+    ReversibleEmbedding,
+)
+from keras_nlp.src.layers.modeling.transformer_decoder import TransformerDecoder
+from keras_nlp.src.models.backbone import Backbone
+from keras_nlp.src.utils.keras_utils import gelu_approximate
 
 
 def _gpt_2_kernel_initializer(stddev=0.02):
diff --git a/keras_nlp/models/gpt2/gpt2_causal_lm.py b/keras_nlp/src/models/gpt2/gpt2_causal_lm.py
similarity index 97%
rename from keras_nlp/models/gpt2/gpt2_causal_lm.py
rename to keras_nlp/src/models/gpt2/gpt2_causal_lm.py
index bdaeabeb45..777b3bf0d4 100644
--- a/keras_nlp/models/gpt2/gpt2_causal_lm.py
+++ b/keras_nlp/src/models/gpt2/gpt2_causal_lm.py
@@ -13,15 +13,15 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.models.causal_lm import CausalLM
-from keras_nlp.models.gpt2.gpt2_backbone import GPT2Backbone
-from keras_nlp.models.gpt2.gpt2_causal_lm_preprocessor import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.causal_lm import CausalLM
+from keras_nlp.src.models.gpt2.gpt2_backbone import GPT2Backbone
+from keras_nlp.src.models.gpt2.gpt2_causal_lm_preprocessor import (
     GPT2CausalLMPreprocessor,
 )
-from keras_nlp.utils.tensor_utils import any_equal
+from keras_nlp.src.utils.tensor_utils import any_equal
 
 
 @keras_nlp_export("keras_nlp.models.GPT2CausalLM")
diff --git a/keras_nlp/models/gpt2/gpt2_causal_lm_preprocessor.py b/keras_nlp/src/models/gpt2/gpt2_causal_lm_preprocessor.py
similarity index 96%
rename from keras_nlp/models/gpt2/gpt2_causal_lm_preprocessor.py
rename to keras_nlp/src/models/gpt2/gpt2_causal_lm_preprocessor.py
index b50dd944a7..f26120c84b 100644
--- a/keras_nlp/models/gpt2/gpt2_causal_lm_preprocessor.py
+++ b/keras_nlp/src/models/gpt2/gpt2_causal_lm_preprocessor.py
@@ -21,13 +21,13 @@
     )
 from absl import logging
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import ops
-from keras_nlp.models.gpt2.gpt2_preprocessor import GPT2Preprocessor
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.gpt2.gpt2_preprocessor import GPT2Preprocessor
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.GPT2CausalLMPreprocessor")
diff --git a/keras_nlp/models/gpt2/gpt2_preprocessor.py b/keras_nlp/src/models/gpt2/gpt2_preprocessor.py
similarity index 94%
rename from keras_nlp/models/gpt2/gpt2_preprocessor.py
rename to keras_nlp/src/models/gpt2/gpt2_preprocessor.py
index 4641a32020..3ce5c46b31 100644
--- a/keras_nlp/models/gpt2/gpt2_preprocessor.py
+++ b/keras_nlp/src/models/gpt2/gpt2_preprocessor.py
@@ -13,14 +13,14 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.start_end_packer import StartEndPacker
-from keras_nlp.models.gpt2.gpt2_tokenizer import GPT2Tokenizer
-from keras_nlp.models.preprocessor import Preprocessor
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.start_end_packer import StartEndPacker
+from keras_nlp.src.models.gpt2.gpt2_tokenizer import GPT2Tokenizer
+from keras_nlp.src.models.preprocessor import Preprocessor
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.GPT2Preprocessor")
diff --git a/keras_nlp/models/gpt2/gpt2_presets.py b/keras_nlp/src/models/gpt2/gpt2_presets.py
similarity index 100%
rename from keras_nlp/models/gpt2/gpt2_presets.py
rename to keras_nlp/src/models/gpt2/gpt2_presets.py
diff --git a/keras_nlp/models/gpt2/gpt2_tokenizer.py b/keras_nlp/src/models/gpt2/gpt2_tokenizer.py
similarity index 96%
rename from keras_nlp/models/gpt2/gpt2_tokenizer.py
rename to keras_nlp/src/models/gpt2/gpt2_tokenizer.py
index 4a585c3176..51e86ed606 100644
--- a/keras_nlp/models/gpt2/gpt2_tokenizer.py
+++ b/keras_nlp/src/models/gpt2/gpt2_tokenizer.py
@@ -13,8 +13,8 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.tokenizers.byte_pair_tokenizer import BytePairTokenizer
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.tokenizers.byte_pair_tokenizer import BytePairTokenizer
 
 
 @keras_nlp_export("keras_nlp.models.GPT2Tokenizer")
diff --git a/keras_nlp/models/gpt_neo_x/__init__.py b/keras_nlp/src/models/gpt_neo_x/__init__.py
similarity index 100%
rename from keras_nlp/models/gpt_neo_x/__init__.py
rename to keras_nlp/src/models/gpt_neo_x/__init__.py
diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_attention.py b/keras_nlp/src/models/gpt_neo_x/gpt_neo_x_attention.py
similarity index 97%
rename from keras_nlp/models/gpt_neo_x/gpt_neo_x_attention.py
rename to keras_nlp/src/models/gpt_neo_x/gpt_neo_x_attention.py
index dee8addf14..485a4ea232 100644
--- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_attention.py
+++ b/keras_nlp/src/models/gpt_neo_x/gpt_neo_x_attention.py
@@ -12,10 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.layers.modeling.rotary_embedding import RotaryEmbedding
-from keras_nlp.utils.keras_utils import clone_initializer
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.layers.modeling.rotary_embedding import RotaryEmbedding
+from keras_nlp.src.utils.keras_utils import clone_initializer
 
 
 class GPTNeoXAttention(keras.layers.Layer):
diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_backbone.py b/keras_nlp/src/models/gpt_neo_x/gpt_neo_x_backbone.py
similarity index 94%
rename from keras_nlp/models/gpt_neo_x/gpt_neo_x_backbone.py
rename to keras_nlp/src/models/gpt_neo_x/gpt_neo_x_backbone.py
index 415fa56af2..58ff85f709 100644
--- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_backbone.py
+++ b/keras_nlp/src/models/gpt_neo_x/gpt_neo_x_backbone.py
@@ -12,12 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding
-from keras_nlp.models.backbone import Backbone
-from keras_nlp.models.gpt_neo_x.gpt_neo_x_decoder import GPTNeoXDecoder
-from keras_nlp.utils.keras_utils import gelu_approximate
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.modeling.reversible_embedding import (
+    ReversibleEmbedding,
+)
+from keras_nlp.src.models.backbone import Backbone
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_decoder import GPTNeoXDecoder
+from keras_nlp.src.utils.keras_utils import gelu_approximate
 
 
 def _gpt_neo_x_kernel_initializer(stddev=0.02):
diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm.py b/keras_nlp/src/models/gpt_neo_x/gpt_neo_x_causal_lm.py
similarity index 95%
rename from keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm.py
rename to keras_nlp/src/models/gpt_neo_x/gpt_neo_x_causal_lm.py
index 0c1a5a583e..79cc6047be 100644
--- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm.py
+++ b/keras_nlp/src/models/gpt_neo_x/gpt_neo_x_causal_lm.py
@@ -12,14 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import ops
-from keras_nlp.models.causal_lm import CausalLM
-from keras_nlp.models.gpt_neo_x.gpt_neo_x_backbone import GPTNeoXBackbone
-from keras_nlp.models.gpt_neo_x.gpt_neo_x_causal_lm_preprocessor import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.causal_lm import CausalLM
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_backbone import GPTNeoXBackbone
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_causal_lm_preprocessor import (
     GPTNeoXCausalLMPreprocessor,
 )
-from keras_nlp.utils.tensor_utils import any_equal
+from keras_nlp.src.utils.tensor_utils import any_equal
 
 
 @keras_nlp_export("keras_nlp.models.GPTNeoXCausalLM")
diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py b/keras_nlp/src/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py
similarity index 95%
rename from keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py
rename to keras_nlp/src/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py
index cdc68d980c..f01b1082fb 100644
--- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py
+++ b/keras_nlp/src/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py
@@ -21,15 +21,15 @@
     )
 from absl import logging
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import ops
-from keras_nlp.models.gpt_neo_x.gpt_neo_x_preprocessor import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_preprocessor import (
     GPTNeoXPreprocessor,
 )
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.GPTNeoXCausalLMPreprocessor")
diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_decoder.py b/keras_nlp/src/models/gpt_neo_x/gpt_neo_x_decoder.py
similarity index 96%
rename from keras_nlp/models/gpt_neo_x/gpt_neo_x_decoder.py
rename to keras_nlp/src/models/gpt_neo_x/gpt_neo_x_decoder.py
index 0a7bad7cd9..cb6cf2e271 100644
--- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_decoder.py
+++ b/keras_nlp/src/models/gpt_neo_x/gpt_neo_x_decoder.py
@@ -12,16 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.layers.modeling.transformer_layer_utils import (
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.layers.modeling.transformer_layer_utils import (
     compute_causal_mask,
 )
-from keras_nlp.layers.modeling.transformer_layer_utils import (
+from keras_nlp.src.layers.modeling.transformer_layer_utils import (
     merge_padding_and_attention_mask,
 )
-from keras_nlp.models.gpt_neo_x.gpt_neo_x_attention import GPTNeoXAttention
-from keras_nlp.utils.keras_utils import clone_initializer
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_attention import GPTNeoXAttention
+from keras_nlp.src.utils.keras_utils import clone_initializer
 
 
 class GPTNeoXDecoder(keras.layers.Layer):
diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_preprocessor.py b/keras_nlp/src/models/gpt_neo_x/gpt_neo_x_preprocessor.py
similarity index 93%
rename from keras_nlp/models/gpt_neo_x/gpt_neo_x_preprocessor.py
rename to keras_nlp/src/models/gpt_neo_x/gpt_neo_x_preprocessor.py
index d5406a6a3d..2443e7c241 100644
--- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_preprocessor.py
+++ b/keras_nlp/src/models/gpt_neo_x/gpt_neo_x_preprocessor.py
@@ -12,14 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.start_end_packer import StartEndPacker
-from keras_nlp.models.gpt_neo_x.gpt_neo_x_tokenizer import GPTNeoXTokenizer
-from keras_nlp.models.preprocessor import Preprocessor
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.start_end_packer import StartEndPacker
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_tokenizer import GPTNeoXTokenizer
+from keras_nlp.src.models.preprocessor import Preprocessor
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.GPTNeoXPreprocessor")
diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_tokenizer.py b/keras_nlp/src/models/gpt_neo_x/gpt_neo_x_tokenizer.py
similarity index 96%
rename from keras_nlp/models/gpt_neo_x/gpt_neo_x_tokenizer.py
rename to keras_nlp/src/models/gpt_neo_x/gpt_neo_x_tokenizer.py
index d109c5849d..e270ae8579 100644
--- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_tokenizer.py
+++ b/keras_nlp/src/models/gpt_neo_x/gpt_neo_x_tokenizer.py
@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.tokenizers.byte_pair_tokenizer import BytePairTokenizer
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.tokenizers.byte_pair_tokenizer import BytePairTokenizer
 
 
 @keras_nlp_export("keras_nlp.models.GPTNeoXTokenizer")
diff --git a/keras_nlp/models/llama/__init__.py b/keras_nlp/src/models/llama/__init__.py
similarity index 70%
rename from keras_nlp/models/llama/__init__.py
rename to keras_nlp/src/models/llama/__init__.py
index 3a57fccd42..6513d69cf4 100644
--- a/keras_nlp/models/llama/__init__.py
+++ b/keras_nlp/src/models/llama/__init__.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.models.llama.llama_backbone import LlamaBackbone
-from keras_nlp.models.llama.llama_presets import backbone_presets
-from keras_nlp.models.llama.llama_tokenizer import LlamaTokenizer
-from keras_nlp.utils.preset_utils import register_presets
+from keras_nlp.src.models.llama.llama_backbone import LlamaBackbone
+from keras_nlp.src.models.llama.llama_presets import backbone_presets
+from keras_nlp.src.models.llama.llama_tokenizer import LlamaTokenizer
+from keras_nlp.src.utils.preset_utils import register_presets
 
 register_presets(backbone_presets, (LlamaBackbone, LlamaTokenizer))
diff --git a/keras_nlp/models/llama/llama_attention.py b/keras_nlp/src/models/llama/llama_attention.py
similarity index 97%
rename from keras_nlp/models/llama/llama_attention.py
rename to keras_nlp/src/models/llama/llama_attention.py
index 415af7f02d..7cc130879a 100644
--- a/keras_nlp/models/llama/llama_attention.py
+++ b/keras_nlp/src/models/llama/llama_attention.py
@@ -11,10 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.layers.modeling.rotary_embedding import RotaryEmbedding
-from keras_nlp.utils.keras_utils import clone_initializer
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.layers.modeling.rotary_embedding import RotaryEmbedding
+from keras_nlp.src.utils.keras_utils import clone_initializer
 
 
 class LlamaAttention(keras.layers.Layer):
diff --git a/keras_nlp/models/llama/llama_backbone.py b/keras_nlp/src/models/llama/llama_backbone.py
similarity index 94%
rename from keras_nlp/models/llama/llama_backbone.py
rename to keras_nlp/src/models/llama/llama_backbone.py
index e586fa97f5..ec2d93ab6e 100644
--- a/keras_nlp/models/llama/llama_backbone.py
+++ b/keras_nlp/src/models/llama/llama_backbone.py
@@ -11,13 +11,15 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding
-from keras_nlp.models.backbone import Backbone
-from keras_nlp.models.llama.llama_decoder import LlamaTransformerDecoder
-from keras_nlp.models.llama.llama_layernorm import LlamaLayerNorm
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.layers.modeling.reversible_embedding import (
+    ReversibleEmbedding,
+)
+from keras_nlp.src.models.backbone import Backbone
+from keras_nlp.src.models.llama.llama_decoder import LlamaTransformerDecoder
+from keras_nlp.src.models.llama.llama_layernorm import LlamaLayerNorm
 
 
 def _llama_kernel_initializer(stddev=0.02):
diff --git a/keras_nlp/models/llama/llama_causal_lm.py b/keras_nlp/src/models/llama/llama_causal_lm.py
similarity index 96%
rename from keras_nlp/models/llama/llama_causal_lm.py
rename to keras_nlp/src/models/llama/llama_causal_lm.py
index 8bc629281c..7dc179347c 100644
--- a/keras_nlp/models/llama/llama_causal_lm.py
+++ b/keras_nlp/src/models/llama/llama_causal_lm.py
@@ -11,16 +11,16 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.models.causal_lm import CausalLM
-from keras_nlp.models.llama.llama_backbone import LlamaBackbone
-from keras_nlp.models.llama.llama_causal_lm_preprocessor import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.causal_lm import CausalLM
+from keras_nlp.src.models.llama.llama_backbone import LlamaBackbone
+from keras_nlp.src.models.llama.llama_causal_lm_preprocessor import (
     LlamaCausalLMPreprocessor,
 )
-from keras_nlp.utils.python_utils import classproperty
-from keras_nlp.utils.tensor_utils import any_equal
+from keras_nlp.src.utils.python_utils import classproperty
+from keras_nlp.src.utils.tensor_utils import any_equal
 
 
 @keras_nlp_export("keras_nlp.models.LlamaCausalLM")
diff --git a/keras_nlp/models/llama/llama_causal_lm_preprocessor.py b/keras_nlp/src/models/llama/llama_causal_lm_preprocessor.py
similarity index 96%
rename from keras_nlp/models/llama/llama_causal_lm_preprocessor.py
rename to keras_nlp/src/models/llama/llama_causal_lm_preprocessor.py
index 50378119d4..32fd4d8a41 100644
--- a/keras_nlp/models/llama/llama_causal_lm_preprocessor.py
+++ b/keras_nlp/src/models/llama/llama_causal_lm_preprocessor.py
@@ -21,13 +21,13 @@
     )
 from absl import logging
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import ops
-from keras_nlp.models.llama.llama_preprocessor import LlamaPreprocessor
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.llama.llama_preprocessor import LlamaPreprocessor
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.LlamaCausalLMPreprocessor")
diff --git a/keras_nlp/models/llama/llama_decoder.py b/keras_nlp/src/models/llama/llama_decoder.py
similarity index 95%
rename from keras_nlp/models/llama/llama_decoder.py
rename to keras_nlp/src/models/llama/llama_decoder.py
index 7b4ad5f75d..0cfbc454b2 100644
--- a/keras_nlp/models/llama/llama_decoder.py
+++ b/keras_nlp/src/models/llama/llama_decoder.py
@@ -11,17 +11,17 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.layers.modeling.transformer_layer_utils import (
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.layers.modeling.transformer_layer_utils import (
     compute_causal_mask,
 )
-from keras_nlp.layers.modeling.transformer_layer_utils import (
+from keras_nlp.src.layers.modeling.transformer_layer_utils import (
     merge_padding_and_attention_mask,
 )
-from keras_nlp.models.llama.llama_attention import LlamaAttention
-from keras_nlp.models.llama.llama_layernorm import LlamaLayerNorm
-from keras_nlp.utils.keras_utils import clone_initializer
+from keras_nlp.src.models.llama.llama_attention import LlamaAttention
+from keras_nlp.src.models.llama.llama_layernorm import LlamaLayerNorm
+from keras_nlp.src.utils.keras_utils import clone_initializer
 
 
 class LlamaTransformerDecoder(keras.layers.Layer):
diff --git a/keras_nlp/models/llama/llama_layernorm.py b/keras_nlp/src/models/llama/llama_layernorm.py
similarity index 95%
rename from keras_nlp/models/llama/llama_layernorm.py
rename to keras_nlp/src/models/llama/llama_layernorm.py
index 8d19c9bc19..a5481dc161 100644
--- a/keras_nlp/models/llama/llama_layernorm.py
+++ b/keras_nlp/src/models/llama/llama_layernorm.py
@@ -11,8 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
 
 
 # TODO: Deprecate this in favor of
diff --git a/keras_nlp/models/llama/llama_preprocessor.py b/keras_nlp/src/models/llama/llama_preprocessor.py
similarity index 94%
rename from keras_nlp/models/llama/llama_preprocessor.py
rename to keras_nlp/src/models/llama/llama_preprocessor.py
index f3aaa208a8..d957fde62b 100644
--- a/keras_nlp/models/llama/llama_preprocessor.py
+++ b/keras_nlp/src/models/llama/llama_preprocessor.py
@@ -11,14 +11,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.start_end_packer import StartEndPacker
-from keras_nlp.models.llama.llama_tokenizer import LlamaTokenizer
-from keras_nlp.models.preprocessor import Preprocessor
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.start_end_packer import StartEndPacker
+from keras_nlp.src.models.llama.llama_tokenizer import LlamaTokenizer
+from keras_nlp.src.models.preprocessor import Preprocessor
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.LlamaPreprocessor")
diff --git a/keras_nlp/models/llama/llama_presets.py b/keras_nlp/src/models/llama/llama_presets.py
similarity index 100%
rename from keras_nlp/models/llama/llama_presets.py
rename to keras_nlp/src/models/llama/llama_presets.py
diff --git a/keras_nlp/models/llama/llama_tokenizer.py b/keras_nlp/src/models/llama/llama_tokenizer.py
similarity index 92%
rename from keras_nlp/models/llama/llama_tokenizer.py
rename to keras_nlp/src/models/llama/llama_tokenizer.py
index 07b0f21037..74b2e89987 100644
--- a/keras_nlp/models/llama/llama_tokenizer.py
+++ b/keras_nlp/src/models/llama/llama_tokenizer.py
@@ -13,10 +13,12 @@
 # limitations under the License.
 import copy
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.models.llama.llama_presets import backbone_presets
-from keras_nlp.tokenizers.sentence_piece_tokenizer import SentencePieceTokenizer
-from keras_nlp.utils.python_utils import classproperty
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.models.llama.llama_presets import backbone_presets
+from keras_nlp.src.tokenizers.sentence_piece_tokenizer import (
+    SentencePieceTokenizer,
+)
+from keras_nlp.src.utils.python_utils import classproperty
 
 
 @keras_nlp_export("keras_nlp.models.LlamaTokenizer")
diff --git a/keras_nlp/models/masked_lm.py b/keras_nlp/src/models/masked_lm.py
similarity index 95%
rename from keras_nlp/models/masked_lm.py
rename to keras_nlp/src/models/masked_lm.py
index db88a56054..2323ba9da7 100644
--- a/keras_nlp/models/masked_lm.py
+++ b/keras_nlp/src/models/masked_lm.py
@@ -11,10 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import config
-from keras_nlp.backend import keras
-from keras_nlp.models.task import Task
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import config
+from keras_nlp.src.backend import keras
+from keras_nlp.src.models.task import Task
 
 
 @keras_nlp_export("keras_nlp.models.MaskedLM")
diff --git a/keras_nlp/models/mistral/__init__.py b/keras_nlp/src/models/mistral/__init__.py
similarity index 69%
rename from keras_nlp/models/mistral/__init__.py
rename to keras_nlp/src/models/mistral/__init__.py
index 2593ab0201..49043ee95e 100644
--- a/keras_nlp/models/mistral/__init__.py
+++ b/keras_nlp/src/models/mistral/__init__.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.models.mistral.mistral_backbone import MistralBackbone
-from keras_nlp.models.mistral.mistral_presets import backbone_presets
-from keras_nlp.models.mistral.mistral_tokenizer import MistralTokenizer
-from keras_nlp.utils.preset_utils import register_presets
+from keras_nlp.src.models.mistral.mistral_backbone import MistralBackbone
+from keras_nlp.src.models.mistral.mistral_presets import backbone_presets
+from keras_nlp.src.models.mistral.mistral_tokenizer import MistralTokenizer
+from keras_nlp.src.utils.preset_utils import register_presets
 
 register_presets(backbone_presets, (MistralBackbone, MistralTokenizer))
diff --git a/keras_nlp/models/mistral/mistral_attention.py b/keras_nlp/src/models/mistral/mistral_attention.py
similarity index 97%
rename from keras_nlp/models/mistral/mistral_attention.py
rename to keras_nlp/src/models/mistral/mistral_attention.py
index 1244132b2d..65d96e8f1b 100644
--- a/keras_nlp/models/mistral/mistral_attention.py
+++ b/keras_nlp/src/models/mistral/mistral_attention.py
@@ -11,10 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.layers.modeling.rotary_embedding import RotaryEmbedding
-from keras_nlp.utils.keras_utils import clone_initializer
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.layers.modeling.rotary_embedding import RotaryEmbedding
+from keras_nlp.src.utils.keras_utils import clone_initializer
 
 
 # This is just a self-attention layer in Mistral. But it can be generalized
diff --git a/keras_nlp/models/mistral/mistral_backbone.py b/keras_nlp/src/models/mistral/mistral_backbone.py
similarity index 94%
rename from keras_nlp/models/mistral/mistral_backbone.py
rename to keras_nlp/src/models/mistral/mistral_backbone.py
index 1ee2dfce66..9115d363e6 100644
--- a/keras_nlp/models/mistral/mistral_backbone.py
+++ b/keras_nlp/src/models/mistral/mistral_backbone.py
@@ -12,15 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding
-from keras_nlp.models.backbone import Backbone
-from keras_nlp.models.mistral.mistral_layer_norm import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.layers.modeling.reversible_embedding import (
+    ReversibleEmbedding,
+)
+from keras_nlp.src.models.backbone import Backbone
+from keras_nlp.src.models.mistral.mistral_layer_norm import (
     MistralLayerNormalization,
 )
-from keras_nlp.models.mistral.mistral_transformer_decoder import (
+from keras_nlp.src.models.mistral.mistral_transformer_decoder import (
     MistralTransformerDecoder,
 )
 
diff --git a/keras_nlp/models/mistral/mistral_causal_lm.py b/keras_nlp/src/models/mistral/mistral_causal_lm.py
similarity index 97%
rename from keras_nlp/models/mistral/mistral_causal_lm.py
rename to keras_nlp/src/models/mistral/mistral_causal_lm.py
index b56947ebfc..8c579d3799 100644
--- a/keras_nlp/models/mistral/mistral_causal_lm.py
+++ b/keras_nlp/src/models/mistral/mistral_causal_lm.py
@@ -12,15 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.models.causal_lm import CausalLM
-from keras_nlp.models.mistral.mistral_backbone import MistralBackbone
-from keras_nlp.models.mistral.mistral_causal_lm_preprocessor import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.causal_lm import CausalLM
+from keras_nlp.src.models.mistral.mistral_backbone import MistralBackbone
+from keras_nlp.src.models.mistral.mistral_causal_lm_preprocessor import (
     MistralCausalLMPreprocessor,
 )
-from keras_nlp.utils.tensor_utils import any_equal
+from keras_nlp.src.utils.tensor_utils import any_equal
 
 
 @keras_nlp_export("keras_nlp.models.MistralCausalLM")
diff --git a/keras_nlp/models/mistral/mistral_causal_lm_preprocessor.py b/keras_nlp/src/models/mistral/mistral_causal_lm_preprocessor.py
similarity index 96%
rename from keras_nlp/models/mistral/mistral_causal_lm_preprocessor.py
rename to keras_nlp/src/models/mistral/mistral_causal_lm_preprocessor.py
index 280355d6e0..99d395f460 100644
--- a/keras_nlp/models/mistral/mistral_causal_lm_preprocessor.py
+++ b/keras_nlp/src/models/mistral/mistral_causal_lm_preprocessor.py
@@ -21,13 +21,15 @@
     )
 from absl import logging
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import ops
-from keras_nlp.models.mistral.mistral_preprocessor import MistralPreprocessor
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.mistral.mistral_preprocessor import (
+    MistralPreprocessor,
+)
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.MistralCausalLMPreprocessor")
diff --git a/keras_nlp/models/mistral/mistral_layer_norm.py b/keras_nlp/src/models/mistral/mistral_layer_norm.py
similarity index 95%
rename from keras_nlp/models/mistral/mistral_layer_norm.py
rename to keras_nlp/src/models/mistral/mistral_layer_norm.py
index 6801a4679b..ca0cdfbd08 100644
--- a/keras_nlp/models/mistral/mistral_layer_norm.py
+++ b/keras_nlp/src/models/mistral/mistral_layer_norm.py
@@ -11,8 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
 
 
 # TODO: Deprecate this in favor of
diff --git a/keras_nlp/models/mistral/mistral_preprocessor.py b/keras_nlp/src/models/mistral/mistral_preprocessor.py
similarity index 94%
rename from keras_nlp/models/mistral/mistral_preprocessor.py
rename to keras_nlp/src/models/mistral/mistral_preprocessor.py
index 3df849fd06..40dfe3d54d 100644
--- a/keras_nlp/models/mistral/mistral_preprocessor.py
+++ b/keras_nlp/src/models/mistral/mistral_preprocessor.py
@@ -12,14 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.start_end_packer import StartEndPacker
-from keras_nlp.models.mistral.mistral_tokenizer import MistralTokenizer
-from keras_nlp.models.preprocessor import Preprocessor
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.start_end_packer import StartEndPacker
+from keras_nlp.src.models.mistral.mistral_tokenizer import MistralTokenizer
+from keras_nlp.src.models.preprocessor import Preprocessor
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.MistralPreprocessor")
diff --git a/keras_nlp/models/mistral/mistral_presets.py b/keras_nlp/src/models/mistral/mistral_presets.py
similarity index 100%
rename from keras_nlp/models/mistral/mistral_presets.py
rename to keras_nlp/src/models/mistral/mistral_presets.py
diff --git a/keras_nlp/models/mistral/mistral_tokenizer.py b/keras_nlp/src/models/mistral/mistral_tokenizer.py
similarity index 95%
rename from keras_nlp/models/mistral/mistral_tokenizer.py
rename to keras_nlp/src/models/mistral/mistral_tokenizer.py
index e91a20df88..ca22dfcfa2 100644
--- a/keras_nlp/models/mistral/mistral_tokenizer.py
+++ b/keras_nlp/src/models/mistral/mistral_tokenizer.py
@@ -12,8 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.tokenizers.sentence_piece_tokenizer import SentencePieceTokenizer
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.tokenizers.sentence_piece_tokenizer import (
+    SentencePieceTokenizer,
+)
 
 
 @keras_nlp_export("keras_nlp.models.MistralTokenizer")
diff --git a/keras_nlp/models/mistral/mistral_transformer_decoder.py b/keras_nlp/src/models/mistral/mistral_transformer_decoder.py
similarity index 95%
rename from keras_nlp/models/mistral/mistral_transformer_decoder.py
rename to keras_nlp/src/models/mistral/mistral_transformer_decoder.py
index 3ef91d3066..8f8c264191 100644
--- a/keras_nlp/models/mistral/mistral_transformer_decoder.py
+++ b/keras_nlp/src/models/mistral/mistral_transformer_decoder.py
@@ -11,19 +11,21 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.layers.modeling.transformer_layer_utils import (
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.layers.modeling.transformer_layer_utils import (
     compute_causal_mask,
 )
-from keras_nlp.layers.modeling.transformer_layer_utils import (
+from keras_nlp.src.layers.modeling.transformer_layer_utils import (
     merge_padding_and_attention_mask,
 )
-from keras_nlp.models.mistral.mistral_attention import CachedMistralAttention
-from keras_nlp.models.mistral.mistral_layer_norm import (
+from keras_nlp.src.models.mistral.mistral_attention import (
+    CachedMistralAttention,
+)
+from keras_nlp.src.models.mistral.mistral_layer_norm import (
     MistralLayerNormalization,
 )
-from keras_nlp.utils.keras_utils import clone_initializer
+from keras_nlp.src.utils.keras_utils import clone_initializer
 
 
 class MistralTransformerDecoder(keras.layers.Layer):
diff --git a/keras_nlp/models/opt/__init__.py b/keras_nlp/src/models/opt/__init__.py
similarity index 71%
rename from keras_nlp/models/opt/__init__.py
rename to keras_nlp/src/models/opt/__init__.py
index a81c5f3bc1..3cf0bb22b6 100644
--- a/keras_nlp/models/opt/__init__.py
+++ b/keras_nlp/src/models/opt/__init__.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.models.opt.opt_backbone import OPTBackbone
-from keras_nlp.models.opt.opt_presets import backbone_presets
-from keras_nlp.models.opt.opt_tokenizer import OPTTokenizer
-from keras_nlp.utils.preset_utils import register_presets
+from keras_nlp.src.models.opt.opt_backbone import OPTBackbone
+from keras_nlp.src.models.opt.opt_presets import backbone_presets
+from keras_nlp.src.models.opt.opt_tokenizer import OPTTokenizer
+from keras_nlp.src.utils.preset_utils import register_presets
 
 register_presets(backbone_presets, (OPTBackbone, OPTTokenizer))
diff --git a/keras_nlp/models/opt/opt_backbone.py b/keras_nlp/src/models/opt/opt_backbone.py
similarity index 95%
rename from keras_nlp/models/opt/opt_backbone.py
rename to keras_nlp/src/models/opt/opt_backbone.py
index 16bd89c4d5..777b31ad20 100644
--- a/keras_nlp/models/opt/opt_backbone.py
+++ b/keras_nlp/src/models/opt/opt_backbone.py
@@ -13,13 +13,13 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.layers.modeling.token_and_position_embedding import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.modeling.token_and_position_embedding import (
     TokenAndPositionEmbedding,
 )
-from keras_nlp.layers.modeling.transformer_decoder import TransformerDecoder
-from keras_nlp.models.backbone import Backbone
+from keras_nlp.src.layers.modeling.transformer_decoder import TransformerDecoder
+from keras_nlp.src.models.backbone import Backbone
 
 
 def opt_kernel_initializer(stddev=0.02):
diff --git a/keras_nlp/models/opt/opt_causal_lm.py b/keras_nlp/src/models/opt/opt_causal_lm.py
similarity index 97%
rename from keras_nlp/models/opt/opt_causal_lm.py
rename to keras_nlp/src/models/opt/opt_causal_lm.py
index 5ec6bf416a..03fe1bb79b 100644
--- a/keras_nlp/models/opt/opt_causal_lm.py
+++ b/keras_nlp/src/models/opt/opt_causal_lm.py
@@ -13,14 +13,14 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import ops
-from keras_nlp.models.causal_lm import CausalLM
-from keras_nlp.models.opt.opt_backbone import OPTBackbone
-from keras_nlp.models.opt.opt_causal_lm_preprocessor import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.causal_lm import CausalLM
+from keras_nlp.src.models.opt.opt_backbone import OPTBackbone
+from keras_nlp.src.models.opt.opt_causal_lm_preprocessor import (
     OPTCausalLMPreprocessor,
 )
-from keras_nlp.utils.tensor_utils import any_equal
+from keras_nlp.src.utils.tensor_utils import any_equal
 
 
 @keras_nlp_export("keras_nlp.models.OPTCausalLM")
diff --git a/keras_nlp/models/opt/opt_causal_lm_preprocessor.py b/keras_nlp/src/models/opt/opt_causal_lm_preprocessor.py
similarity index 96%
rename from keras_nlp/models/opt/opt_causal_lm_preprocessor.py
rename to keras_nlp/src/models/opt/opt_causal_lm_preprocessor.py
index 3128791da5..53f9245007 100644
--- a/keras_nlp/models/opt/opt_causal_lm_preprocessor.py
+++ b/keras_nlp/src/models/opt/opt_causal_lm_preprocessor.py
@@ -21,13 +21,13 @@
     )
 from absl import logging
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import ops
-from keras_nlp.models.opt.opt_preprocessor import OPTPreprocessor
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.opt.opt_preprocessor import OPTPreprocessor
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.OPTCausalLMPreprocessor")
diff --git a/keras_nlp/models/opt/opt_preprocessor.py b/keras_nlp/src/models/opt/opt_preprocessor.py
similarity index 94%
rename from keras_nlp/models/opt/opt_preprocessor.py
rename to keras_nlp/src/models/opt/opt_preprocessor.py
index 9eae445aec..c03c2c8348 100644
--- a/keras_nlp/models/opt/opt_preprocessor.py
+++ b/keras_nlp/src/models/opt/opt_preprocessor.py
@@ -13,14 +13,14 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.start_end_packer import StartEndPacker
-from keras_nlp.models.opt.opt_tokenizer import OPTTokenizer
-from keras_nlp.models.preprocessor import Preprocessor
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.start_end_packer import StartEndPacker
+from keras_nlp.src.models.opt.opt_tokenizer import OPTTokenizer
+from keras_nlp.src.models.preprocessor import Preprocessor
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.OPTPreprocessor")
diff --git a/keras_nlp/models/opt/opt_presets.py b/keras_nlp/src/models/opt/opt_presets.py
similarity index 100%
rename from keras_nlp/models/opt/opt_presets.py
rename to keras_nlp/src/models/opt/opt_presets.py
diff --git a/keras_nlp/models/opt/opt_tokenizer.py b/keras_nlp/src/models/opt/opt_tokenizer.py
similarity index 97%
rename from keras_nlp/models/opt/opt_tokenizer.py
rename to keras_nlp/src/models/opt/opt_tokenizer.py
index addcd0c01f..ee8a80ce87 100644
--- a/keras_nlp/models/opt/opt_tokenizer.py
+++ b/keras_nlp/src/models/opt/opt_tokenizer.py
@@ -13,8 +13,8 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.tokenizers.byte_pair_tokenizer import BytePairTokenizer
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.tokenizers.byte_pair_tokenizer import BytePairTokenizer
 
 
 @keras_nlp_export("keras_nlp.models.OPTTokenizer")
diff --git a/keras_nlp/models/preprocessor.py b/keras_nlp/src/models/preprocessor.py
similarity index 89%
rename from keras_nlp/models/preprocessor.py
rename to keras_nlp/src/models/preprocessor.py
index 5dd049e309..10d8e93ce2 100644
--- a/keras_nlp/models/preprocessor.py
+++ b/keras_nlp/src/models/preprocessor.py
@@ -12,21 +12,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.layers.preprocessing.preprocessing_layer import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.preprocessing.preprocessing_layer import (
     PreprocessingLayer,
 )
-from keras_nlp.utils.preset_utils import PREPROCESSOR_CONFIG_FILE
-from keras_nlp.utils.preset_utils import TOKENIZER_CONFIG_FILE
-from keras_nlp.utils.preset_utils import check_config_class
-from keras_nlp.utils.preset_utils import check_file_exists
-from keras_nlp.utils.preset_utils import list_presets
-from keras_nlp.utils.preset_utils import list_subclasses
-from keras_nlp.utils.preset_utils import load_serialized_object
-from keras_nlp.utils.preset_utils import save_serialized_object
-from keras_nlp.utils.preset_utils import validate_metadata
-from keras_nlp.utils.python_utils import classproperty
+from keras_nlp.src.utils.preset_utils import PREPROCESSOR_CONFIG_FILE
+from keras_nlp.src.utils.preset_utils import TOKENIZER_CONFIG_FILE
+from keras_nlp.src.utils.preset_utils import check_config_class
+from keras_nlp.src.utils.preset_utils import check_file_exists
+from keras_nlp.src.utils.preset_utils import list_presets
+from keras_nlp.src.utils.preset_utils import list_subclasses
+from keras_nlp.src.utils.preset_utils import load_serialized_object
+from keras_nlp.src.utils.preset_utils import save_serialized_object
+from keras_nlp.src.utils.preset_utils import validate_metadata
+from keras_nlp.src.utils.python_utils import classproperty
 
 
 @keras_nlp_export("keras_nlp.models.Preprocessor")
diff --git a/keras_nlp/models/roberta/__init__.py b/keras_nlp/src/models/roberta/__init__.py
similarity index 69%
rename from keras_nlp/models/roberta/__init__.py
rename to keras_nlp/src/models/roberta/__init__.py
index 1fe69366c7..57a9d5ea6e 100644
--- a/keras_nlp/models/roberta/__init__.py
+++ b/keras_nlp/src/models/roberta/__init__.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.models.roberta.roberta_backbone import RobertaBackbone
-from keras_nlp.models.roberta.roberta_presets import backbone_presets
-from keras_nlp.models.roberta.roberta_tokenizer import RobertaTokenizer
-from keras_nlp.utils.preset_utils import register_presets
+from keras_nlp.src.models.roberta.roberta_backbone import RobertaBackbone
+from keras_nlp.src.models.roberta.roberta_presets import backbone_presets
+from keras_nlp.src.models.roberta.roberta_tokenizer import RobertaTokenizer
+from keras_nlp.src.utils.preset_utils import register_presets
 
 register_presets(backbone_presets, (RobertaBackbone, RobertaTokenizer))
diff --git a/keras_nlp/models/roberta/roberta_backbone.py b/keras_nlp/src/models/roberta/roberta_backbone.py
similarity index 95%
rename from keras_nlp/models/roberta/roberta_backbone.py
rename to keras_nlp/src/models/roberta/roberta_backbone.py
index 6c25f2ae6f..9673f02427 100644
--- a/keras_nlp/models/roberta/roberta_backbone.py
+++ b/keras_nlp/src/models/roberta/roberta_backbone.py
@@ -13,13 +13,13 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.layers.modeling.token_and_position_embedding import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.modeling.token_and_position_embedding import (
     TokenAndPositionEmbedding,
 )
-from keras_nlp.layers.modeling.transformer_encoder import TransformerEncoder
-from keras_nlp.models.backbone import Backbone
+from keras_nlp.src.layers.modeling.transformer_encoder import TransformerEncoder
+from keras_nlp.src.models.backbone import Backbone
 
 
 def roberta_kernel_initializer(stddev=0.02):
diff --git a/keras_nlp/models/roberta/roberta_classifier.py b/keras_nlp/src/models/roberta/roberta_classifier.py
similarity index 94%
rename from keras_nlp/models/roberta/roberta_classifier.py
rename to keras_nlp/src/models/roberta/roberta_classifier.py
index ba6d0e4902..c5bbb108f5 100644
--- a/keras_nlp/models/roberta/roberta_classifier.py
+++ b/keras_nlp/src/models/roberta/roberta_classifier.py
@@ -13,12 +13,16 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.models.classifier import Classifier
-from keras_nlp.models.roberta.roberta_backbone import RobertaBackbone
-from keras_nlp.models.roberta.roberta_backbone import roberta_kernel_initializer
-from keras_nlp.models.roberta.roberta_preprocessor import RobertaPreprocessor
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.models.classifier import Classifier
+from keras_nlp.src.models.roberta.roberta_backbone import RobertaBackbone
+from keras_nlp.src.models.roberta.roberta_backbone import (
+    roberta_kernel_initializer,
+)
+from keras_nlp.src.models.roberta.roberta_preprocessor import (
+    RobertaPreprocessor,
+)
 
 
 @keras_nlp_export("keras_nlp.models.RobertaClassifier")
diff --git a/keras_nlp/models/roberta/roberta_masked_lm.py b/keras_nlp/src/models/roberta/roberta_masked_lm.py
similarity index 90%
rename from keras_nlp/models/roberta/roberta_masked_lm.py
rename to keras_nlp/src/models/roberta/roberta_masked_lm.py
index 1429b367bd..fbcf7fc23e 100644
--- a/keras_nlp/models/roberta/roberta_masked_lm.py
+++ b/keras_nlp/src/models/roberta/roberta_masked_lm.py
@@ -13,13 +13,15 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.layers.modeling.masked_lm_head import MaskedLMHead
-from keras_nlp.models.masked_lm import MaskedLM
-from keras_nlp.models.roberta.roberta_backbone import RobertaBackbone
-from keras_nlp.models.roberta.roberta_backbone import roberta_kernel_initializer
-from keras_nlp.models.roberta.roberta_masked_lm_preprocessor import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.modeling.masked_lm_head import MaskedLMHead
+from keras_nlp.src.models.masked_lm import MaskedLM
+from keras_nlp.src.models.roberta.roberta_backbone import RobertaBackbone
+from keras_nlp.src.models.roberta.roberta_backbone import (
+    roberta_kernel_initializer,
+)
+from keras_nlp.src.models.roberta.roberta_masked_lm_preprocessor import (
     RobertaMaskedLMPreprocessor,
 )
 
diff --git a/keras_nlp/models/roberta/roberta_masked_lm_preprocessor.py b/keras_nlp/src/models/roberta/roberta_masked_lm_preprocessor.py
similarity index 96%
rename from keras_nlp/models/roberta/roberta_masked_lm_preprocessor.py
rename to keras_nlp/src/models/roberta/roberta_masked_lm_preprocessor.py
index c69c300dc8..acd9fd6e08 100644
--- a/keras_nlp/models/roberta/roberta_masked_lm_preprocessor.py
+++ b/keras_nlp/src/models/roberta/roberta_masked_lm_preprocessor.py
@@ -14,12 +14,14 @@
 
 from absl import logging
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.masked_lm_mask_generator import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.masked_lm_mask_generator import (
     MaskedLMMaskGenerator,
 )
-from keras_nlp.models.roberta.roberta_preprocessor import RobertaPreprocessor
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.models.roberta.roberta_preprocessor import (
+    RobertaPreprocessor,
+)
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.RobertaMaskedLMPreprocessor")
diff --git a/keras_nlp/models/roberta/roberta_preprocessor.py b/keras_nlp/src/models/roberta/roberta_preprocessor.py
similarity index 94%
rename from keras_nlp/models/roberta/roberta_preprocessor.py
rename to keras_nlp/src/models/roberta/roberta_preprocessor.py
index f0e027a910..66d8b502f7 100644
--- a/keras_nlp/models/roberta/roberta_preprocessor.py
+++ b/keras_nlp/src/models/roberta/roberta_preprocessor.py
@@ -13,16 +13,16 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.multi_segment_packer import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.multi_segment_packer import (
     MultiSegmentPacker,
 )
-from keras_nlp.models.preprocessor import Preprocessor
-from keras_nlp.models.roberta.roberta_tokenizer import RobertaTokenizer
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.models.preprocessor import Preprocessor
+from keras_nlp.src.models.roberta.roberta_tokenizer import RobertaTokenizer
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.RobertaPreprocessor")
diff --git a/keras_nlp/models/roberta/roberta_presets.py b/keras_nlp/src/models/roberta/roberta_presets.py
similarity index 100%
rename from keras_nlp/models/roberta/roberta_presets.py
rename to keras_nlp/src/models/roberta/roberta_presets.py
diff --git a/keras_nlp/models/roberta/roberta_tokenizer.py b/keras_nlp/src/models/roberta/roberta_tokenizer.py
similarity index 97%
rename from keras_nlp/models/roberta/roberta_tokenizer.py
rename to keras_nlp/src/models/roberta/roberta_tokenizer.py
index acf7f0aef9..1178a06231 100644
--- a/keras_nlp/models/roberta/roberta_tokenizer.py
+++ b/keras_nlp/src/models/roberta/roberta_tokenizer.py
@@ -13,8 +13,8 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.tokenizers.byte_pair_tokenizer import BytePairTokenizer
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.tokenizers.byte_pair_tokenizer import BytePairTokenizer
 
 
 @keras_nlp_export("keras_nlp.models.RobertaTokenizer")
diff --git a/keras_nlp/models/seq_2_seq_lm.py b/keras_nlp/src/models/seq_2_seq_lm.py
similarity index 95%
rename from keras_nlp/models/seq_2_seq_lm.py
rename to keras_nlp/src/models/seq_2_seq_lm.py
index 7c2ee9caa7..80ed86e993 100644
--- a/keras_nlp/models/seq_2_seq_lm.py
+++ b/keras_nlp/src/models/seq_2_seq_lm.py
@@ -11,8 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.models.causal_lm import CausalLM
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.models.causal_lm import CausalLM
 
 
 @keras_nlp_export("keras_nlp.models.Seq2SeqLM")
diff --git a/keras_nlp/models/t5/__init__.py b/keras_nlp/src/models/t5/__init__.py
similarity index 72%
rename from keras_nlp/models/t5/__init__.py
rename to keras_nlp/src/models/t5/__init__.py
index 10064e442b..07cb095782 100644
--- a/keras_nlp/models/t5/__init__.py
+++ b/keras_nlp/src/models/t5/__init__.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.models.t5.t5_backbone import T5Backbone
-from keras_nlp.models.t5.t5_presets import backbone_presets
-from keras_nlp.models.t5.t5_tokenizer import T5Tokenizer
-from keras_nlp.utils.preset_utils import register_presets
+from keras_nlp.src.models.t5.t5_backbone import T5Backbone
+from keras_nlp.src.models.t5.t5_presets import backbone_presets
+from keras_nlp.src.models.t5.t5_tokenizer import T5Tokenizer
+from keras_nlp.src.utils.preset_utils import register_presets
 
 register_presets(backbone_presets, (T5Backbone, T5Tokenizer))
diff --git a/keras_nlp/models/t5/t5_backbone.py b/keras_nlp/src/models/t5/t5_backbone.py
similarity index 96%
rename from keras_nlp/models/t5/t5_backbone.py
rename to keras_nlp/src/models/t5/t5_backbone.py
index 67d3b2a6a9..7db1b0f8ff 100644
--- a/keras_nlp/models/t5/t5_backbone.py
+++ b/keras_nlp/src/models/t5/t5_backbone.py
@@ -12,12 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding
-from keras_nlp.models.backbone import Backbone
-from keras_nlp.models.t5.t5_layer_norm import T5LayerNorm
-from keras_nlp.models.t5.t5_transformer_layer import T5TransformerLayer
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.modeling.reversible_embedding import (
+    ReversibleEmbedding,
+)
+from keras_nlp.src.models.backbone import Backbone
+from keras_nlp.src.models.t5.t5_layer_norm import T5LayerNorm
+from keras_nlp.src.models.t5.t5_transformer_layer import T5TransformerLayer
 
 
 @keras_nlp_export("keras_nlp.models.T5Backbone")
diff --git a/keras_nlp/models/t5/t5_layer_norm.py b/keras_nlp/src/models/t5/t5_layer_norm.py
similarity index 93%
rename from keras_nlp/models/t5/t5_layer_norm.py
rename to keras_nlp/src/models/t5/t5_layer_norm.py
index b4f157c004..5abae8c70a 100644
--- a/keras_nlp/models/t5/t5_layer_norm.py
+++ b/keras_nlp/src/models/t5/t5_layer_norm.py
@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
 
 
 class T5LayerNorm(keras.layers.Layer):
diff --git a/keras_nlp/models/t5/t5_multi_head_attention.py b/keras_nlp/src/models/t5/t5_multi_head_attention.py
similarity index 99%
rename from keras_nlp/models/t5/t5_multi_head_attention.py
rename to keras_nlp/src/models/t5/t5_multi_head_attention.py
index 2e3647b1d5..a6623aedd4 100644
--- a/keras_nlp/models/t5/t5_multi_head_attention.py
+++ b/keras_nlp/src/models/t5/t5_multi_head_attention.py
@@ -14,8 +14,8 @@
 
 import numpy as np
 
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
 
 
 class T5MultiHeadAttention(keras.layers.Layer):
diff --git a/keras_nlp/models/t5/t5_presets.py b/keras_nlp/src/models/t5/t5_presets.py
similarity index 100%
rename from keras_nlp/models/t5/t5_presets.py
rename to keras_nlp/src/models/t5/t5_presets.py
diff --git a/keras_nlp/models/t5/t5_tokenizer.py b/keras_nlp/src/models/t5/t5_tokenizer.py
similarity index 95%
rename from keras_nlp/models/t5/t5_tokenizer.py
rename to keras_nlp/src/models/t5/t5_tokenizer.py
index ec5f0bf324..411a343f98 100644
--- a/keras_nlp/models/t5/t5_tokenizer.py
+++ b/keras_nlp/src/models/t5/t5_tokenizer.py
@@ -12,8 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.tokenizers.sentence_piece_tokenizer import SentencePieceTokenizer
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.tokenizers.sentence_piece_tokenizer import (
+    SentencePieceTokenizer,
+)
 
 
 @keras_nlp_export("keras_nlp.models.T5Tokenizer")
diff --git a/keras_nlp/models/t5/t5_transformer_layer.py b/keras_nlp/src/models/t5/t5_transformer_layer.py
similarity index 95%
rename from keras_nlp/models/t5/t5_transformer_layer.py
rename to keras_nlp/src/models/t5/t5_transformer_layer.py
index ddff7a164a..3f118b6354 100644
--- a/keras_nlp/models/t5/t5_transformer_layer.py
+++ b/keras_nlp/src/models/t5/t5_transformer_layer.py
@@ -12,13 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.layers.modeling.transformer_layer_utils import (
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.layers.modeling.transformer_layer_utils import (
     compute_causal_mask,
 )
-from keras_nlp.models.t5.t5_layer_norm import T5LayerNorm
-from keras_nlp.models.t5.t5_multi_head_attention import T5MultiHeadAttention
+from keras_nlp.src.models.t5.t5_layer_norm import T5LayerNorm
+from keras_nlp.src.models.t5.t5_multi_head_attention import T5MultiHeadAttention
 
 
 class T5TransformerLayer(keras.layers.Layer):
diff --git a/keras_nlp/models/task.py b/keras_nlp/src/models/task.py
similarity index 93%
rename from keras_nlp/models/task.py
rename to keras_nlp/src/models/task.py
index bb4e389e32..a2fcd9e456 100644
--- a/keras_nlp/models/task.py
+++ b/keras_nlp/src/models/task.py
@@ -18,25 +18,25 @@
 from rich import markup
 from rich import table as rich_table
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import config
-from keras_nlp.backend import keras
-from keras_nlp.utils.keras_utils import print_msg
-from keras_nlp.utils.pipeline_model import PipelineModel
-from keras_nlp.utils.preset_utils import CONFIG_FILE
-from keras_nlp.utils.preset_utils import MODEL_WEIGHTS_FILE
-from keras_nlp.utils.preset_utils import TASK_CONFIG_FILE
-from keras_nlp.utils.preset_utils import TASK_WEIGHTS_FILE
-from keras_nlp.utils.preset_utils import check_config_class
-from keras_nlp.utils.preset_utils import check_file_exists
-from keras_nlp.utils.preset_utils import get_file
-from keras_nlp.utils.preset_utils import jax_memory_cleanup
-from keras_nlp.utils.preset_utils import list_presets
-from keras_nlp.utils.preset_utils import list_subclasses
-from keras_nlp.utils.preset_utils import load_serialized_object
-from keras_nlp.utils.preset_utils import save_serialized_object
-from keras_nlp.utils.preset_utils import validate_metadata
-from keras_nlp.utils.python_utils import classproperty
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import config
+from keras_nlp.src.backend import keras
+from keras_nlp.src.utils.keras_utils import print_msg
+from keras_nlp.src.utils.pipeline_model import PipelineModel
+from keras_nlp.src.utils.preset_utils import CONFIG_FILE
+from keras_nlp.src.utils.preset_utils import MODEL_WEIGHTS_FILE
+from keras_nlp.src.utils.preset_utils import TASK_CONFIG_FILE
+from keras_nlp.src.utils.preset_utils import TASK_WEIGHTS_FILE
+from keras_nlp.src.utils.preset_utils import check_config_class
+from keras_nlp.src.utils.preset_utils import check_file_exists
+from keras_nlp.src.utils.preset_utils import get_file
+from keras_nlp.src.utils.preset_utils import jax_memory_cleanup
+from keras_nlp.src.utils.preset_utils import list_presets
+from keras_nlp.src.utils.preset_utils import list_subclasses
+from keras_nlp.src.utils.preset_utils import load_serialized_object
+from keras_nlp.src.utils.preset_utils import save_serialized_object
+from keras_nlp.src.utils.preset_utils import validate_metadata
+from keras_nlp.src.utils.python_utils import classproperty
 
 
 @keras_nlp_export("keras_nlp.models.Task")
diff --git a/keras_nlp/models/whisper/__init__.py b/keras_nlp/src/models/whisper/__init__.py
similarity index 69%
rename from keras_nlp/models/whisper/__init__.py
rename to keras_nlp/src/models/whisper/__init__.py
index 328814046c..254c99cb54 100644
--- a/keras_nlp/models/whisper/__init__.py
+++ b/keras_nlp/src/models/whisper/__init__.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.models.whisper.whisper_backbone import WhisperBackbone
-from keras_nlp.models.whisper.whisper_presets import backbone_presets
-from keras_nlp.models.whisper.whisper_tokenizer import WhisperTokenizer
-from keras_nlp.utils.preset_utils import register_presets
+from keras_nlp.src.models.whisper.whisper_backbone import WhisperBackbone
+from keras_nlp.src.models.whisper.whisper_presets import backbone_presets
+from keras_nlp.src.models.whisper.whisper_tokenizer import WhisperTokenizer
+from keras_nlp.src.utils.preset_utils import register_presets
 
 register_presets(backbone_presets, (WhisperBackbone, WhisperTokenizer))
diff --git a/keras_nlp/models/whisper/whisper_audio_feature_extractor.py b/keras_nlp/src/models/whisper/whisper_audio_feature_extractor.py
similarity index 98%
rename from keras_nlp/models/whisper/whisper_audio_feature_extractor.py
rename to keras_nlp/src/models/whisper/whisper_audio_feature_extractor.py
index 20ffcf8371..aec7dd3c07 100644
--- a/keras_nlp/models/whisper/whisper_audio_feature_extractor.py
+++ b/keras_nlp/src/models/whisper/whisper_audio_feature_extractor.py
@@ -23,8 +23,8 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.preprocessing_layer import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.preprocessing_layer import (
     PreprocessingLayer,
 )
 
diff --git a/keras_nlp/models/whisper/whisper_backbone.py b/keras_nlp/src/models/whisper/whisper_backbone.py
similarity index 95%
rename from keras_nlp/models/whisper/whisper_backbone.py
rename to keras_nlp/src/models/whisper/whisper_backbone.py
index 3f5a017f14..3ef73f58d8 100644
--- a/keras_nlp/models/whisper/whisper_backbone.py
+++ b/keras_nlp/src/models/whisper/whisper_backbone.py
@@ -13,17 +13,17 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.layers.modeling.position_embedding import PositionEmbedding
-from keras_nlp.layers.modeling.token_and_position_embedding import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.layers.modeling.position_embedding import PositionEmbedding
+from keras_nlp.src.layers.modeling.token_and_position_embedding import (
     TokenAndPositionEmbedding,
 )
-from keras_nlp.models.backbone import Backbone
-from keras_nlp.models.whisper.whisper_decoder import WhisperDecoder
-from keras_nlp.models.whisper.whisper_encoder import WhisperEncoder
-from keras_nlp.utils.tensor_utils import assert_tf_backend
+from keras_nlp.src.models.backbone import Backbone
+from keras_nlp.src.models.whisper.whisper_decoder import WhisperDecoder
+from keras_nlp.src.models.whisper.whisper_encoder import WhisperEncoder
+from keras_nlp.src.utils.tensor_utils import assert_tf_backend
 
 
 def whisper_kernel_initializer(stddev=0.02):
diff --git a/keras_nlp/models/whisper/whisper_cached_multi_head_attention.py b/keras_nlp/src/models/whisper/whisper_cached_multi_head_attention.py
similarity index 97%
rename from keras_nlp/models/whisper/whisper_cached_multi_head_attention.py
rename to keras_nlp/src/models/whisper/whisper_cached_multi_head_attention.py
index 01ad18ba4b..e5a3225a07 100644
--- a/keras_nlp/models/whisper/whisper_cached_multi_head_attention.py
+++ b/keras_nlp/src/models/whisper/whisper_cached_multi_head_attention.py
@@ -16,8 +16,8 @@
 import collections
 import string
 
-import keras_nlp
-from keras_nlp.backend import keras
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers import CachedMultiHeadAttention
 
 
 def _index_to_einsum_variable(i):
@@ -62,9 +62,7 @@ def _get_output_shape(output_rank, known_last_dims):
 
 
 @keras.saving.register_keras_serializable(package="keras_nlp")
-class WhisperCachedMultiHeadAttention(
-    keras_nlp.layers.CachedMultiHeadAttention
-):
+class WhisperCachedMultiHeadAttention(CachedMultiHeadAttention):
     """Whisper Cached Multi-Head Attention layer.
 
     Inherits from `keras_nlp.layers.CachedMultiHeadAttention`, and overrides the
diff --git a/keras_nlp/models/whisper/whisper_decoder.py b/keras_nlp/src/models/whisper/whisper_decoder.py
similarity index 95%
rename from keras_nlp/models/whisper/whisper_decoder.py
rename to keras_nlp/src/models/whisper/whisper_decoder.py
index c41a870a42..5bac97f27b 100644
--- a/keras_nlp/models/whisper/whisper_decoder.py
+++ b/keras_nlp/src/models/whisper/whisper_decoder.py
@@ -14,12 +14,12 @@
 """Whisper decoder block."""
 
 
-from keras_nlp.backend import keras
-from keras_nlp.layers.modeling.transformer_decoder import TransformerDecoder
-from keras_nlp.models.whisper.whisper_cached_multi_head_attention import (
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.modeling.transformer_decoder import TransformerDecoder
+from keras_nlp.src.models.whisper.whisper_cached_multi_head_attention import (
     WhisperCachedMultiHeadAttention,
 )
-from keras_nlp.utils.keras_utils import clone_initializer
+from keras_nlp.src.utils.keras_utils import clone_initializer
 
 
 @keras.saving.register_keras_serializable(package="keras_nlp")
diff --git a/keras_nlp/models/whisper/whisper_encoder.py b/keras_nlp/src/models/whisper/whisper_encoder.py
similarity index 93%
rename from keras_nlp/models/whisper/whisper_encoder.py
rename to keras_nlp/src/models/whisper/whisper_encoder.py
index 9d5b41d0d2..803cce49c5 100644
--- a/keras_nlp/models/whisper/whisper_encoder.py
+++ b/keras_nlp/src/models/whisper/whisper_encoder.py
@@ -14,12 +14,12 @@
 """Whisper encoder block."""
 
 
-from keras_nlp.backend import keras
-from keras_nlp.layers.modeling.transformer_encoder import TransformerEncoder
-from keras_nlp.models.whisper.whisper_cached_multi_head_attention import (
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.modeling.transformer_encoder import TransformerEncoder
+from keras_nlp.src.models.whisper.whisper_cached_multi_head_attention import (
     WhisperCachedMultiHeadAttention,
 )
-from keras_nlp.utils.keras_utils import clone_initializer
+from keras_nlp.src.utils.keras_utils import clone_initializer
 
 
 @keras.saving.register_keras_serializable(package="keras_nlp")
diff --git a/keras_nlp/models/whisper/whisper_preprocessor.py b/keras_nlp/src/models/whisper/whisper_preprocessor.py
similarity index 95%
rename from keras_nlp/models/whisper/whisper_preprocessor.py
rename to keras_nlp/src/models/whisper/whisper_preprocessor.py
index a78ea96639..432fa4aebf 100644
--- a/keras_nlp/models/whisper/whisper_preprocessor.py
+++ b/keras_nlp/src/models/whisper/whisper_preprocessor.py
@@ -15,18 +15,18 @@
 
 from absl import logging
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.layers.preprocessing.start_end_packer import StartEndPacker
-from keras_nlp.models.preprocessor import Preprocessor
-from keras_nlp.models.whisper.whisper_audio_feature_extractor import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.preprocessing.start_end_packer import StartEndPacker
+from keras_nlp.src.models.preprocessor import Preprocessor
+from keras_nlp.src.models.whisper.whisper_audio_feature_extractor import (
     WhisperAudioFeatureExtractor,
 )
-from keras_nlp.models.whisper.whisper_tokenizer import WhisperTokenizer
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.models.whisper.whisper_tokenizer import WhisperTokenizer
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.WhisperPreprocessor")
diff --git a/keras_nlp/models/whisper/whisper_presets.py b/keras_nlp/src/models/whisper/whisper_presets.py
similarity index 100%
rename from keras_nlp/models/whisper/whisper_presets.py
rename to keras_nlp/src/models/whisper/whisper_presets.py
diff --git a/keras_nlp/models/whisper/whisper_tokenizer.py b/keras_nlp/src/models/whisper/whisper_tokenizer.py
similarity index 97%
rename from keras_nlp/models/whisper/whisper_tokenizer.py
rename to keras_nlp/src/models/whisper/whisper_tokenizer.py
index f14fd1ee98..e6f2a7d443 100644
--- a/keras_nlp/models/whisper/whisper_tokenizer.py
+++ b/keras_nlp/src/models/whisper/whisper_tokenizer.py
@@ -14,8 +14,8 @@
 
 import json
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.tokenizers.byte_pair_tokenizer import BytePairTokenizer
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.tokenizers.byte_pair_tokenizer import BytePairTokenizer
 
 
 def _load_dict(dict_or_path):
diff --git a/keras_nlp/models/xlm_roberta/__init__.py b/keras_nlp/src/models/xlm_roberta/__init__.py
similarity index 68%
rename from keras_nlp/models/xlm_roberta/__init__.py
rename to keras_nlp/src/models/xlm_roberta/__init__.py
index f8be3006ad..01f5e68228 100644
--- a/keras_nlp/models/xlm_roberta/__init__.py
+++ b/keras_nlp/src/models/xlm_roberta/__init__.py
@@ -12,11 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.models.xlm_roberta.xlm_roberta_backbone import XLMRobertaBackbone
-from keras_nlp.models.xlm_roberta.xlm_roberta_presets import backbone_presets
-from keras_nlp.models.xlm_roberta.xlm_roberta_tokenizer import (
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_backbone import (
+    XLMRobertaBackbone,
+)
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_presets import (
+    backbone_presets,
+)
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_tokenizer import (
     XLMRobertaTokenizer,
 )
-from keras_nlp.utils.preset_utils import register_presets
+from keras_nlp.src.utils.preset_utils import register_presets
 
 register_presets(backbone_presets, (XLMRobertaBackbone, XLMRobertaTokenizer))
diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_backbone.py b/keras_nlp/src/models/xlm_roberta/xlm_roberta_backbone.py
similarity index 96%
rename from keras_nlp/models/xlm_roberta/xlm_roberta_backbone.py
rename to keras_nlp/src/models/xlm_roberta/xlm_roberta_backbone.py
index 28d7f18728..f4f7e56553 100644
--- a/keras_nlp/models/xlm_roberta/xlm_roberta_backbone.py
+++ b/keras_nlp/src/models/xlm_roberta/xlm_roberta_backbone.py
@@ -13,8 +13,8 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.models.roberta import roberta_backbone
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.models.roberta import roberta_backbone
 
 
 @keras_nlp_export("keras_nlp.models.XLMRobertaBackbone")
diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_classifier.py b/keras_nlp/src/models/xlm_roberta/xlm_roberta_classifier.py
similarity index 94%
rename from keras_nlp/models/xlm_roberta/xlm_roberta_classifier.py
rename to keras_nlp/src/models/xlm_roberta/xlm_roberta_classifier.py
index bf078862a9..59e12e6362 100644
--- a/keras_nlp/models/xlm_roberta/xlm_roberta_classifier.py
+++ b/keras_nlp/src/models/xlm_roberta/xlm_roberta_classifier.py
@@ -13,12 +13,16 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.models.classifier import Classifier
-from keras_nlp.models.roberta.roberta_backbone import roberta_kernel_initializer
-from keras_nlp.models.xlm_roberta.xlm_roberta_backbone import XLMRobertaBackbone
-from keras_nlp.models.xlm_roberta.xlm_roberta_preprocessor import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.models.classifier import Classifier
+from keras_nlp.src.models.roberta.roberta_backbone import (
+    roberta_kernel_initializer,
+)
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_backbone import (
+    XLMRobertaBackbone,
+)
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_preprocessor import (
     XLMRobertaPreprocessor,
 )
 
diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm.py b/keras_nlp/src/models/xlm_roberta/xlm_roberta_masked_lm.py
similarity index 90%
rename from keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm.py
rename to keras_nlp/src/models/xlm_roberta/xlm_roberta_masked_lm.py
index 9abbbe5dc8..fc2302b0b5 100644
--- a/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm.py
+++ b/keras_nlp/src/models/xlm_roberta/xlm_roberta_masked_lm.py
@@ -13,13 +13,17 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.layers.modeling.masked_lm_head import MaskedLMHead
-from keras_nlp.models.masked_lm import MaskedLM
-from keras_nlp.models.roberta.roberta_backbone import roberta_kernel_initializer
-from keras_nlp.models.xlm_roberta.xlm_roberta_backbone import XLMRobertaBackbone
-from keras_nlp.models.xlm_roberta.xlm_roberta_masked_lm_preprocessor import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.modeling.masked_lm_head import MaskedLMHead
+from keras_nlp.src.models.masked_lm import MaskedLM
+from keras_nlp.src.models.roberta.roberta_backbone import (
+    roberta_kernel_initializer,
+)
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_backbone import (
+    XLMRobertaBackbone,
+)
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_masked_lm_preprocessor import (
     XLMRobertaMaskedLMPreprocessor,
 )
 
diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py b/keras_nlp/src/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py
similarity index 96%
rename from keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py
rename to keras_nlp/src/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py
index a26905e9e3..d0f6c24c13 100644
--- a/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py
+++ b/keras_nlp/src/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py
@@ -14,14 +14,14 @@
 
 from absl import logging
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.masked_lm_mask_generator import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.masked_lm_mask_generator import (
     MaskedLMMaskGenerator,
 )
-from keras_nlp.models.xlm_roberta.xlm_roberta_preprocessor import (
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_preprocessor import (
     XLMRobertaPreprocessor,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.XLMRobertaMaskedLMPreprocessor")
diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_preprocessor.py b/keras_nlp/src/models/xlm_roberta/xlm_roberta_preprocessor.py
similarity index 95%
rename from keras_nlp/models/xlm_roberta/xlm_roberta_preprocessor.py
rename to keras_nlp/src/models/xlm_roberta/xlm_roberta_preprocessor.py
index 7901e35720..9d8c6686e1 100644
--- a/keras_nlp/models/xlm_roberta/xlm_roberta_preprocessor.py
+++ b/keras_nlp/src/models/xlm_roberta/xlm_roberta_preprocessor.py
@@ -13,18 +13,18 @@
 # limitations under the License.
 
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.multi_segment_packer import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.multi_segment_packer import (
     MultiSegmentPacker,
 )
-from keras_nlp.models.preprocessor import Preprocessor
-from keras_nlp.models.xlm_roberta.xlm_roberta_tokenizer import (
+from keras_nlp.src.models.preprocessor import Preprocessor
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_tokenizer import (
     XLMRobertaTokenizer,
 )
-from keras_nlp.utils.keras_utils import (
+from keras_nlp.src.utils.keras_utils import (
     convert_inputs_to_list_of_tensor_segments,
 )
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 @keras_nlp_export("keras_nlp.models.XLMRobertaPreprocessor")
diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_presets.py b/keras_nlp/src/models/xlm_roberta/xlm_roberta_presets.py
similarity index 100%
rename from keras_nlp/models/xlm_roberta/xlm_roberta_presets.py
rename to keras_nlp/src/models/xlm_roberta/xlm_roberta_presets.py
diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_tokenizer.py b/keras_nlp/src/models/xlm_roberta/xlm_roberta_tokenizer.py
similarity index 97%
rename from keras_nlp/models/xlm_roberta/xlm_roberta_tokenizer.py
rename to keras_nlp/src/models/xlm_roberta/xlm_roberta_tokenizer.py
index b413e9cafa..2cc809832a 100644
--- a/keras_nlp/models/xlm_roberta/xlm_roberta_tokenizer.py
+++ b/keras_nlp/src/models/xlm_roberta/xlm_roberta_tokenizer.py
@@ -21,9 +21,11 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.tokenizers.sentence_piece_tokenizer import SentencePieceTokenizer
-from keras_nlp.utils.tensor_utils import tensor_to_list
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.tokenizers.sentence_piece_tokenizer import (
+    SentencePieceTokenizer,
+)
+from keras_nlp.src.utils.tensor_utils import tensor_to_list
 
 
 @keras_nlp_export("keras_nlp.models.XLMRobertaTokenizer")
diff --git a/keras_nlp/models/xlnet/__init__.py b/keras_nlp/src/models/xlnet/__init__.py
similarity index 100%
rename from keras_nlp/models/xlnet/__init__.py
rename to keras_nlp/src/models/xlnet/__init__.py
diff --git a/keras_nlp/models/xlnet/relative_attention.py b/keras_nlp/src/models/xlnet/relative_attention.py
similarity index 99%
rename from keras_nlp/models/xlnet/relative_attention.py
rename to keras_nlp/src/models/xlnet/relative_attention.py
index 6ae56d1450..09f004dd60 100644
--- a/keras_nlp/models/xlnet/relative_attention.py
+++ b/keras_nlp/src/models/xlnet/relative_attention.py
@@ -15,8 +15,8 @@
 import math
 import string
 
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
 
 _CHR_IDX = string.ascii_lowercase
 
diff --git a/keras_nlp/models/xlnet/xlnet_backbone.py b/keras_nlp/src/models/xlnet/xlnet_backbone.py
similarity index 94%
rename from keras_nlp/models/xlnet/xlnet_backbone.py
rename to keras_nlp/src/models/xlnet/xlnet_backbone.py
index 03ea607d9e..d69df6bc42 100644
--- a/keras_nlp/models/xlnet/xlnet_backbone.py
+++ b/keras_nlp/src/models/xlnet/xlnet_backbone.py
@@ -12,15 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.models.backbone import Backbone
-from keras_nlp.models.xlnet.xlnet_content_and_query_embedding import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.models.backbone import Backbone
+from keras_nlp.src.models.xlnet.xlnet_content_and_query_embedding import (
     ContentAndQueryEmbedding,
 )
-from keras_nlp.models.xlnet.xlnet_encoder import XLNetAttentionMaskLayer
-from keras_nlp.models.xlnet.xlnet_encoder import XLNetEncoder
-from keras_nlp.models.xlnet.xlnet_encoder import XLNetSegmentMatrixLayer
+from keras_nlp.src.models.xlnet.xlnet_encoder import XLNetAttentionMaskLayer
+from keras_nlp.src.models.xlnet.xlnet_encoder import XLNetEncoder
+from keras_nlp.src.models.xlnet.xlnet_encoder import XLNetSegmentMatrixLayer
 
 
 @keras_nlp_export("keras_nlp.models.XLNetBackbone")
@@ -68,7 +68,7 @@ class XLNetBackbone(Backbone):
     Example:
     ```python
     import numpy as np
-    from keras_nlp.models import XLNetBackbone
+    from keras_nlp.src.models import XLNetBackbone
 
     input_data = {
         "token_ids": np.array(
diff --git a/keras_nlp/models/xlnet/xlnet_content_and_query_embedding.py b/keras_nlp/src/models/xlnet/xlnet_content_and_query_embedding.py
similarity index 98%
rename from keras_nlp/models/xlnet/xlnet_content_and_query_embedding.py
rename to keras_nlp/src/models/xlnet/xlnet_content_and_query_embedding.py
index 2de2c31780..795ca0416a 100644
--- a/keras_nlp/models/xlnet/xlnet_content_and_query_embedding.py
+++ b/keras_nlp/src/models/xlnet/xlnet_content_and_query_embedding.py
@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
 
 
 class ContentAndQueryEmbedding(keras.layers.Layer):
diff --git a/keras_nlp/models/xlnet/xlnet_encoder.py b/keras_nlp/src/models/xlnet/xlnet_encoder.py
similarity index 98%
rename from keras_nlp/models/xlnet/xlnet_encoder.py
rename to keras_nlp/src/models/xlnet/xlnet_encoder.py
index 6aa44b150e..069e55e423 100644
--- a/keras_nlp/models/xlnet/xlnet_encoder.py
+++ b/keras_nlp/src/models/xlnet/xlnet_encoder.py
@@ -12,9 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.models.xlnet.relative_attention import TwoStreamRelativeAttention
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.xlnet.relative_attention import (
+    TwoStreamRelativeAttention,
+)
 
 
 def xlnet_kernel_initializer(stddev=0.02):
diff --git a/keras_nlp/src/samplers/__init__.py b/keras_nlp/src/samplers/__init__.py
new file mode 100644
index 0000000000..f6ce74f3b9
--- /dev/null
+++ b/keras_nlp/src/samplers/__init__.py
@@ -0,0 +1,25 @@
+# Copyright 2023 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from keras_nlp.src.backend import keras
+from keras_nlp.src.samplers.beam_sampler import BeamSampler
+from keras_nlp.src.samplers.contrastive_sampler import ContrastiveSampler
+from keras_nlp.src.samplers.greedy_sampler import GreedySampler
+from keras_nlp.src.samplers.random_sampler import RandomSampler
+from keras_nlp.src.samplers.sampler import Sampler
+from keras_nlp.src.samplers.serialization import deserialize
+from keras_nlp.src.samplers.serialization import get
+from keras_nlp.src.samplers.serialization import serialize
+from keras_nlp.src.samplers.top_k_sampler import TopKSampler
+from keras_nlp.src.samplers.top_p_sampler import TopPSampler
diff --git a/keras_nlp/samplers/beam_sampler.py b/keras_nlp/src/samplers/beam_sampler.py
similarity index 97%
rename from keras_nlp/samplers/beam_sampler.py
rename to keras_nlp/src/samplers/beam_sampler.py
index 4f1f0d34d2..9cf0b67583 100644
--- a/keras_nlp/samplers/beam_sampler.py
+++ b/keras_nlp/src/samplers/beam_sampler.py
@@ -21,10 +21,10 @@
     )
 import tree
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import ops
-from keras_nlp.samplers.sampler import Sampler
-from keras_nlp.utils.tensor_utils import any_equal
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import ops
+from keras_nlp.src.samplers.sampler import Sampler
+from keras_nlp.src.utils.tensor_utils import any_equal
 
 
 @keras_nlp_export("keras_nlp.samplers.BeamSampler")
diff --git a/keras_nlp/samplers/contrastive_sampler.py b/keras_nlp/src/samplers/contrastive_sampler.py
similarity index 97%
rename from keras_nlp/samplers/contrastive_sampler.py
rename to keras_nlp/src/samplers/contrastive_sampler.py
index 24f983fd0b..b4c4340de7 100644
--- a/keras_nlp/samplers/contrastive_sampler.py
+++ b/keras_nlp/src/samplers/contrastive_sampler.py
@@ -14,10 +14,10 @@
 
 import tree
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import ops
-from keras_nlp.samplers.sampler import Sampler
-from keras_nlp.utils.tensor_utils import any_equal
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import ops
+from keras_nlp.src.samplers.sampler import Sampler
+from keras_nlp.src.utils.tensor_utils import any_equal
 
 
 @keras_nlp_export("keras_nlp.samplers.ContrastiveSampler")
diff --git a/keras_nlp/samplers/greedy_sampler.py b/keras_nlp/src/samplers/greedy_sampler.py
similarity index 90%
rename from keras_nlp/samplers/greedy_sampler.py
rename to keras_nlp/src/samplers/greedy_sampler.py
index ee8a6ecc2d..20257197a2 100644
--- a/keras_nlp/samplers/greedy_sampler.py
+++ b/keras_nlp/src/samplers/greedy_sampler.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import ops
-from keras_nlp.samplers.sampler import Sampler
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import ops
+from keras_nlp.src.samplers.sampler import Sampler
 
 
 @keras_nlp_export("keras_nlp.samplers.GreedySampler")
diff --git a/keras_nlp/samplers/random_sampler.py b/keras_nlp/src/samplers/random_sampler.py
similarity index 92%
rename from keras_nlp/samplers/random_sampler.py
rename to keras_nlp/src/samplers/random_sampler.py
index 1ff39c9f9b..eccb31dd36 100644
--- a/keras_nlp/samplers/random_sampler.py
+++ b/keras_nlp/src/samplers/random_sampler.py
@@ -12,10 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import ops
-from keras_nlp.backend import random
-from keras_nlp.samplers.sampler import Sampler
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import ops
+from keras_nlp.src.backend import random
+from keras_nlp.src.samplers.sampler import Sampler
 
 
 @keras_nlp_export("keras_nlp.samplers.RandomSampler")
diff --git a/keras_nlp/samplers/sampler.py b/keras_nlp/src/samplers/sampler.py
similarity index 96%
rename from keras_nlp/samplers/sampler.py
rename to keras_nlp/src/samplers/sampler.py
index d3680a3d51..27ab80236d 100644
--- a/keras_nlp/samplers/sampler.py
+++ b/keras_nlp/src/samplers/sampler.py
@@ -12,12 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import config
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.backend import random
-from keras_nlp.utils.tensor_utils import any_equal
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import config
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.backend import random
+from keras_nlp.src.utils.tensor_utils import any_equal
 
 
 @keras_nlp_export("keras_nlp.samplers.Sampler")
diff --git a/keras_nlp/samplers/serialization.py b/keras_nlp/src/samplers/serialization.py
similarity index 85%
rename from keras_nlp/samplers/serialization.py
rename to keras_nlp/src/samplers/serialization.py
index 9c82a51957..5d294d68ea 100644
--- a/keras_nlp/samplers/serialization.py
+++ b/keras_nlp/src/samplers/serialization.py
@@ -12,14 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import keras
-from keras_nlp.samplers.beam_sampler import BeamSampler
-from keras_nlp.samplers.contrastive_sampler import ContrastiveSampler
-from keras_nlp.samplers.greedy_sampler import GreedySampler
-from keras_nlp.samplers.random_sampler import RandomSampler
-from keras_nlp.samplers.top_k_sampler import TopKSampler
-from keras_nlp.samplers.top_p_sampler import TopPSampler
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import keras
+from keras_nlp.src.samplers.beam_sampler import BeamSampler
+from keras_nlp.src.samplers.contrastive_sampler import ContrastiveSampler
+from keras_nlp.src.samplers.greedy_sampler import GreedySampler
+from keras_nlp.src.samplers.random_sampler import RandomSampler
+from keras_nlp.src.samplers.top_k_sampler import TopKSampler
+from keras_nlp.src.samplers.top_p_sampler import TopPSampler
 
 
 @keras_nlp_export("keras_nlp.samplers.serialize")
diff --git a/keras_nlp/samplers/top_k_sampler.py b/keras_nlp/src/samplers/top_k_sampler.py
similarity index 93%
rename from keras_nlp/samplers/top_k_sampler.py
rename to keras_nlp/src/samplers/top_k_sampler.py
index 513dd738c7..6326b5894f 100644
--- a/keras_nlp/samplers/top_k_sampler.py
+++ b/keras_nlp/src/samplers/top_k_sampler.py
@@ -12,10 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import ops
-from keras_nlp.backend import random
-from keras_nlp.samplers.sampler import Sampler
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import ops
+from keras_nlp.src.backend import random
+from keras_nlp.src.samplers.sampler import Sampler
 
 
 @keras_nlp_export("keras_nlp.samplers.TopKSampler")
diff --git a/keras_nlp/samplers/top_p_sampler.py b/keras_nlp/src/samplers/top_p_sampler.py
similarity index 95%
rename from keras_nlp/samplers/top_p_sampler.py
rename to keras_nlp/src/samplers/top_p_sampler.py
index 326f5797a6..86e1ce41ef 100644
--- a/keras_nlp/samplers/top_p_sampler.py
+++ b/keras_nlp/src/samplers/top_p_sampler.py
@@ -12,10 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import ops
-from keras_nlp.backend import random
-from keras_nlp.samplers.sampler import Sampler
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import ops
+from keras_nlp.src.backend import random
+from keras_nlp.src.samplers.sampler import Sampler
 
 
 @keras_nlp_export("keras_nlp.samplers.TopPSampler")
diff --git a/keras_nlp/tokenizers/__init__.py b/keras_nlp/src/tokenizers/__init__.py
similarity index 54%
rename from keras_nlp/tokenizers/__init__.py
rename to keras_nlp/src/tokenizers/__init__.py
index 41e0007c97..c15d73e0b6 100644
--- a/keras_nlp/tokenizers/__init__.py
+++ b/keras_nlp/src/tokenizers/__init__.py
@@ -12,17 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.tokenizers.byte_pair_tokenizer import BytePairTokenizer
-from keras_nlp.tokenizers.byte_tokenizer import ByteTokenizer
-from keras_nlp.tokenizers.sentence_piece_tokenizer import SentencePieceTokenizer
-from keras_nlp.tokenizers.sentence_piece_tokenizer_trainer import (
+from keras_nlp.src.tokenizers.byte_pair_tokenizer import BytePairTokenizer
+from keras_nlp.src.tokenizers.byte_tokenizer import ByteTokenizer
+from keras_nlp.src.tokenizers.sentence_piece_tokenizer import (
+    SentencePieceTokenizer,
+)
+from keras_nlp.src.tokenizers.sentence_piece_tokenizer_trainer import (
     compute_sentence_piece_proto,
 )
-from keras_nlp.tokenizers.tokenizer import Tokenizer
-from keras_nlp.tokenizers.unicode_codepoint_tokenizer import (
+from keras_nlp.src.tokenizers.tokenizer import Tokenizer
+from keras_nlp.src.tokenizers.unicode_codepoint_tokenizer import (
     UnicodeCodepointTokenizer,
 )
-from keras_nlp.tokenizers.word_piece_tokenizer import WordPieceTokenizer
-from keras_nlp.tokenizers.word_piece_tokenizer_trainer import (
+from keras_nlp.src.tokenizers.word_piece_tokenizer import WordPieceTokenizer
+from keras_nlp.src.tokenizers.word_piece_tokenizer_trainer import (
     compute_word_piece_vocabulary,
 )
diff --git a/keras_nlp/tokenizers/byte_pair_tokenizer.py b/keras_nlp/src/tokenizers/byte_pair_tokenizer.py
similarity index 98%
rename from keras_nlp/tokenizers/byte_pair_tokenizer.py
rename to keras_nlp/src/tokenizers/byte_pair_tokenizer.py
index bb85bfc09d..c6f1270139 100644
--- a/keras_nlp/tokenizers/byte_pair_tokenizer.py
+++ b/keras_nlp/src/tokenizers/byte_pair_tokenizer.py
@@ -34,12 +34,12 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.tokenizers import tokenizer
-from keras_nlp.utils.tensor_utils import assert_tf_text_installed
-from keras_nlp.utils.tensor_utils import convert_to_ragged_batch
-from keras_nlp.utils.tensor_utils import is_int_dtype
-from keras_nlp.utils.tensor_utils import is_string_dtype
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.tokenizers import tokenizer
+from keras_nlp.src.utils.tensor_utils import assert_tf_text_installed
+from keras_nlp.src.utils.tensor_utils import convert_to_ragged_batch
+from keras_nlp.src.utils.tensor_utils import is_int_dtype
+from keras_nlp.src.utils.tensor_utils import is_string_dtype
 
 try:
     import tensorflow_text as tf_text
diff --git a/keras_nlp/tokenizers/byte_tokenizer.py b/keras_nlp/src/tokenizers/byte_tokenizer.py
similarity index 97%
rename from keras_nlp/tokenizers/byte_tokenizer.py
rename to keras_nlp/src/tokenizers/byte_tokenizer.py
index a68ba366e5..128aac2778 100644
--- a/keras_nlp/tokenizers/byte_tokenizer.py
+++ b/keras_nlp/src/tokenizers/byte_tokenizer.py
@@ -22,11 +22,11 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.tokenizers import tokenizer
-from keras_nlp.utils.tensor_utils import assert_tf_text_installed
-from keras_nlp.utils.tensor_utils import convert_to_ragged_batch
-from keras_nlp.utils.tensor_utils import is_int_dtype
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.tokenizers import tokenizer
+from keras_nlp.src.utils.tensor_utils import assert_tf_text_installed
+from keras_nlp.src.utils.tensor_utils import convert_to_ragged_batch
+from keras_nlp.src.utils.tensor_utils import is_int_dtype
 
 try:
     import tensorflow_text as tf_text
diff --git a/keras_nlp/tokenizers/sentence_piece_tokenizer.py b/keras_nlp/src/tokenizers/sentence_piece_tokenizer.py
similarity index 95%
rename from keras_nlp/tokenizers/sentence_piece_tokenizer.py
rename to keras_nlp/src/tokenizers/sentence_piece_tokenizer.py
index aac0f32073..92c818db3c 100644
--- a/keras_nlp/tokenizers/sentence_piece_tokenizer.py
+++ b/keras_nlp/src/tokenizers/sentence_piece_tokenizer.py
@@ -26,13 +26,13 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.tokenizers import tokenizer
-from keras_nlp.utils.tensor_utils import assert_tf_text_installed
-from keras_nlp.utils.tensor_utils import convert_to_ragged_batch
-from keras_nlp.utils.tensor_utils import is_int_dtype
-from keras_nlp.utils.tensor_utils import is_string_dtype
-from keras_nlp.utils.tensor_utils import tensor_to_list
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.tokenizers import tokenizer
+from keras_nlp.src.utils.tensor_utils import assert_tf_text_installed
+from keras_nlp.src.utils.tensor_utils import convert_to_ragged_batch
+from keras_nlp.src.utils.tensor_utils import is_int_dtype
+from keras_nlp.src.utils.tensor_utils import is_string_dtype
+from keras_nlp.src.utils.tensor_utils import tensor_to_list
 
 try:
     import tensorflow_text as tf_text
diff --git a/keras_nlp/tokenizers/sentence_piece_tokenizer_trainer.py b/keras_nlp/src/tokenizers/sentence_piece_tokenizer_trainer.py
similarity index 99%
rename from keras_nlp/tokenizers/sentence_piece_tokenizer_trainer.py
rename to keras_nlp/src/tokenizers/sentence_piece_tokenizer_trainer.py
index ab2e3f898d..9ed6dc9b71 100644
--- a/keras_nlp/tokenizers/sentence_piece_tokenizer_trainer.py
+++ b/keras_nlp/src/tokenizers/sentence_piece_tokenizer_trainer.py
@@ -27,7 +27,7 @@
 except ImportError:
     spm = None
 
-from keras_nlp.api_export import keras_nlp_export
+from keras_nlp.src.api_export import keras_nlp_export
 
 
 @keras_nlp_export("keras_nlp.tokenizers.compute_sentence_piece_proto")
diff --git a/keras_nlp/tokenizers/tokenizer.py b/keras_nlp/src/tokenizers/tokenizer.py
similarity index 90%
rename from keras_nlp/tokenizers/tokenizer.py
rename to keras_nlp/src/tokenizers/tokenizer.py
index 5c96645413..7b918f5a99 100644
--- a/keras_nlp/tokenizers/tokenizer.py
+++ b/keras_nlp/src/tokenizers/tokenizer.py
@@ -13,21 +13,21 @@
 # limitations under the License.
 import os
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.layers.preprocessing.preprocessing_layer import (
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.layers.preprocessing.preprocessing_layer import (
     PreprocessingLayer,
 )
-from keras_nlp.utils.preset_utils import TOKENIZER_ASSET_DIR
-from keras_nlp.utils.preset_utils import TOKENIZER_CONFIG_FILE
-from keras_nlp.utils.preset_utils import check_config_class
-from keras_nlp.utils.preset_utils import get_file
-from keras_nlp.utils.preset_utils import list_presets
-from keras_nlp.utils.preset_utils import list_subclasses
-from keras_nlp.utils.preset_utils import load_serialized_object
-from keras_nlp.utils.preset_utils import save_serialized_object
-from keras_nlp.utils.preset_utils import save_tokenizer_assets
-from keras_nlp.utils.preset_utils import validate_metadata
-from keras_nlp.utils.python_utils import classproperty
+from keras_nlp.src.utils.preset_utils import TOKENIZER_ASSET_DIR
+from keras_nlp.src.utils.preset_utils import TOKENIZER_CONFIG_FILE
+from keras_nlp.src.utils.preset_utils import check_config_class
+from keras_nlp.src.utils.preset_utils import get_file
+from keras_nlp.src.utils.preset_utils import list_presets
+from keras_nlp.src.utils.preset_utils import list_subclasses
+from keras_nlp.src.utils.preset_utils import load_serialized_object
+from keras_nlp.src.utils.preset_utils import save_serialized_object
+from keras_nlp.src.utils.preset_utils import save_tokenizer_assets
+from keras_nlp.src.utils.preset_utils import validate_metadata
+from keras_nlp.src.utils.python_utils import classproperty
 
 
 @keras_nlp_export(
diff --git a/keras_nlp/tokenizers/unicode_codepoint_tokenizer.py b/keras_nlp/src/tokenizers/unicode_codepoint_tokenizer.py
similarity index 97%
rename from keras_nlp/tokenizers/unicode_codepoint_tokenizer.py
rename to keras_nlp/src/tokenizers/unicode_codepoint_tokenizer.py
index 578e03bca7..8403f6fb5a 100644
--- a/keras_nlp/tokenizers/unicode_codepoint_tokenizer.py
+++ b/keras_nlp/src/tokenizers/unicode_codepoint_tokenizer.py
@@ -14,11 +14,11 @@
 
 import tensorflow as tf
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.tokenizers import tokenizer
-from keras_nlp.utils.tensor_utils import assert_tf_text_installed
-from keras_nlp.utils.tensor_utils import convert_to_ragged_batch
-from keras_nlp.utils.tensor_utils import is_int_dtype
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.tokenizers import tokenizer
+from keras_nlp.src.utils.tensor_utils import assert_tf_text_installed
+from keras_nlp.src.utils.tensor_utils import convert_to_ragged_batch
+from keras_nlp.src.utils.tensor_utils import is_int_dtype
 
 try:
     import tensorflow_text as tf_text
diff --git a/keras_nlp/tokenizers/word_piece_tokenizer.py b/keras_nlp/src/tokenizers/word_piece_tokenizer.py
similarity index 98%
rename from keras_nlp/tokenizers/word_piece_tokenizer.py
rename to keras_nlp/src/tokenizers/word_piece_tokenizer.py
index 75c66fb35e..918e17dc02 100644
--- a/keras_nlp/tokenizers/word_piece_tokenizer.py
+++ b/keras_nlp/src/tokenizers/word_piece_tokenizer.py
@@ -26,12 +26,12 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.tokenizers import tokenizer
-from keras_nlp.utils.tensor_utils import assert_tf_text_installed
-from keras_nlp.utils.tensor_utils import convert_to_ragged_batch
-from keras_nlp.utils.tensor_utils import is_int_dtype
-from keras_nlp.utils.tensor_utils import is_string_dtype
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.tokenizers import tokenizer
+from keras_nlp.src.utils.tensor_utils import assert_tf_text_installed
+from keras_nlp.src.utils.tensor_utils import convert_to_ragged_batch
+from keras_nlp.src.utils.tensor_utils import is_int_dtype
+from keras_nlp.src.utils.tensor_utils import is_string_dtype
 
 try:
     import tensorflow_text as tf_text
diff --git a/keras_nlp/tokenizers/word_piece_tokenizer_trainer.py b/keras_nlp/src/tokenizers/word_piece_tokenizer_trainer.py
similarity index 97%
rename from keras_nlp/tokenizers/word_piece_tokenizer_trainer.py
rename to keras_nlp/src/tokenizers/word_piece_tokenizer_trainer.py
index 52eb7823eb..085c373acd 100644
--- a/keras_nlp/tokenizers/word_piece_tokenizer_trainer.py
+++ b/keras_nlp/src/tokenizers/word_piece_tokenizer_trainer.py
@@ -20,9 +20,9 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.tokenizers.word_piece_tokenizer import pretokenize
-from keras_nlp.utils.tensor_utils import assert_tf_text_installed
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.tokenizers.word_piece_tokenizer import pretokenize
+from keras_nlp.src.utils.tensor_utils import assert_tf_text_installed
 
 try:
     from tensorflow_text.tools.wordpiece_vocab import (
diff --git a/keras_nlp/utils/__init__.py b/keras_nlp/src/utils/__init__.py
similarity index 100%
rename from keras_nlp/utils/__init__.py
rename to keras_nlp/src/utils/__init__.py
diff --git a/keras_nlp/utils/keras_utils.py b/keras_nlp/src/utils/keras_utils.py
similarity index 98%
rename from keras_nlp/utils/keras_utils.py
rename to keras_nlp/src/utils/keras_utils.py
index c550837b2a..60d3311aea 100644
--- a/keras_nlp/utils/keras_utils.py
+++ b/keras_nlp/src/utils/keras_utils.py
@@ -23,8 +23,8 @@
     )
 from absl import logging
 
-from keras_nlp.backend import keras
-from keras_nlp.utils.tensor_utils import is_tensor_type
+from keras_nlp.src.backend import keras
+from keras_nlp.src.utils.tensor_utils import is_tensor_type
 
 
 def clone_initializer(initializer):
diff --git a/keras_nlp/utils/pipeline_model.py b/keras_nlp/src/utils/pipeline_model.py
similarity index 97%
rename from keras_nlp/utils/pipeline_model.py
rename to keras_nlp/src/utils/pipeline_model.py
index d7250cfdd1..4aae1837a1 100644
--- a/keras_nlp/utils/pipeline_model.py
+++ b/keras_nlp/src/utils/pipeline_model.py
@@ -24,10 +24,10 @@
     )
 import tree
 
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
-from keras_nlp.utils.tensor_utils import is_tensor_type
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.utils.tensor_utils import is_tensor_type
 
 
 def _convert_inputs_to_dataset(
diff --git a/keras_nlp/utils/preset_utils.py b/keras_nlp/src/utils/preset_utils.py
similarity index 98%
rename from keras_nlp/utils/preset_utils.py
rename to keras_nlp/src/utils/preset_utils.py
index 7affc663c8..7c1954964a 100644
--- a/keras_nlp/utils/preset_utils.py
+++ b/keras_nlp/src/utils/preset_utils.py
@@ -22,9 +22,9 @@
 from absl import logging
 from packaging.version import parse
 
-from keras_nlp.api_export import keras_nlp_export
-from keras_nlp.backend import config as backend_config
-from keras_nlp.backend import keras
+from keras_nlp.src.api_export import keras_nlp_export
+from keras_nlp.src.backend import config as backend_config
+from keras_nlp.src.backend import keras
 
 try:
     import kagglehub
@@ -203,7 +203,7 @@ def check_file_exists(preset, path):
 def get_tokenizer(layer):
     """Get the tokenizer from any KerasNLP model or layer."""
     # Avoid circular import.
-    from keras_nlp.tokenizers.tokenizer import Tokenizer
+    from keras_nlp.src.tokenizers.tokenizer import Tokenizer
 
     if isinstance(layer, Tokenizer):
         return layer
@@ -260,7 +260,7 @@ def save_serialized_object(
 
 
 def save_metadata(layer, preset):
-    from keras_nlp import __version__ as keras_nlp_version
+    from keras_nlp.src import __version__ as keras_nlp_version
 
     keras_version = keras.version() if hasattr(keras, "version") else None
     metadata = {
diff --git a/keras_nlp/utils/python_utils.py b/keras_nlp/src/utils/python_utils.py
similarity index 100%
rename from keras_nlp/utils/python_utils.py
rename to keras_nlp/src/utils/python_utils.py
diff --git a/keras_nlp/utils/tensor_utils.py b/keras_nlp/src/utils/tensor_utils.py
similarity index 98%
rename from keras_nlp/utils/tensor_utils.py
rename to keras_nlp/src/utils/tensor_utils.py
index af8bc605a7..28a4b47a19 100644
--- a/keras_nlp/utils/tensor_utils.py
+++ b/keras_nlp/src/utils/tensor_utils.py
@@ -20,9 +20,9 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.backend import config
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
+from keras_nlp.src.backend import config
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
 
 try:
     import tensorflow_text as tf_text
diff --git a/keras_nlp/version_utils.py b/keras_nlp/src/version_utils.py
similarity index 93%
rename from keras_nlp/version_utils.py
rename to keras_nlp/src/version_utils.py
index 8d719dc4da..a2128273e1 100644
--- a/keras_nlp/version_utils.py
+++ b/keras_nlp/src/version_utils.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.api_export import keras_nlp_export
+from keras_nlp.src.api_export import keras_nlp_export
 
 # Unique source of truth for the version number.
 __version__ = "0.10.0"
diff --git a/pip_build.py b/pip_build.py
index 0c83cbb436..159659a390 100644
--- a/pip_build.py
+++ b/pip_build.py
@@ -27,23 +27,19 @@
 python3 pip_build.py --install
 ```
 """
+
 import argparse
 import datetime
 import glob
 import os
 import pathlib
+import re
 import shutil
 
-import namex
-
 package = "keras_nlp"
 build_directory = "tmp_build_dir"
 dist_directory = "dist"
-to_copy = [
-    "setup.py",
-    "setup.cfg",
-    "README.md",
-]
+to_copy = ["setup.py", "setup.cfg", "README.md"]
 
 
 def ignore_files(_, filenames):
@@ -62,30 +58,23 @@ def export_version_string(version, is_nightly=False):
             setup_contents = setup_contents.replace(
                 'name="keras-nlp"', 'name="keras-nlp-nightly"'
             )
-            setup_contents = setup_contents.replace(
-                '"tensorflow-text', '"tf-nightly", "tensorflow-text-nightly'
-            )
             f.write(setup_contents)
 
-    # Overwrite the version string with our package version.
+    # Make sure to export the __version__ string
     with open(os.path.join(package, "src", "version_utils.py")) as f:
-        version_contents = f.readlines()
-    with open(os.path.join(package, "src", "version_utils.py"), "w") as f:
-        for line in version_contents:
-            if line.startswith("__version__"):
-                f.write(f'__version__ = "{version}"\n')
-            else:
-                f.write(line)
-    # Make sure to export the __version__ string.
-    with open(os.path.join(package, "__init__.py")) as f:
         init_contents = f.read()
-    with open(os.path.join(package, "__init__.py"), "w") as f:
+    with open(os.path.join(package, "src", "version_utils.py"), "w") as f:
+        init_contents = re.sub(
+            "\n__version__ = .*\n",
+            f'\n__version__ = "{version}"\n',
+            init_contents,
+        )
         f.write(init_contents)
-        f.write("from keras_nlp.src.version_utils import __version__\n")
 
 
 def copy_source_to_build_directory(root_path):
-    # Copy sources (`keras_nlp/` directory and setup files) to build dir
+    # Copy sources (`keras_nlp/` directory and setup files) to build
+    # directory
     os.chdir(root_path)
     os.mkdir(build_directory)
     shutil.copytree(
@@ -96,16 +85,25 @@ def copy_source_to_build_directory(root_path):
     os.chdir(build_directory)
 
 
-def run_namex_conversion():
-    # Restructure the codebase so that source files live in `keras_nlp/src`
-    namex.convert_codebase(package, code_directory="src")
+def build(root_path, is_nightly=False):
+    if os.path.exists(build_directory):
+        raise ValueError(f"Directory already exists: {build_directory}")
+
+    try:
+        copy_source_to_build_directory(root_path)
+        print(os.getcwd())
+
+        from keras_nlp.src.version_utils import __version__  # noqa: E402
 
-    # Generate API __init__.py files in `keras_nlp/`
-    namex.generate_api_files(package, code_directory="src", verbose=True)
+        export_version_string(__version__, is_nightly)
+        return build_and_save_output(root_path, __version__)
+    finally:
+        # Clean up: remove the build directory (no longer needed)
+        shutil.rmtree(build_directory)
 
 
 def build_and_save_output(root_path, __version__):
-    """Build the package."""
+    # Build the package
     os.system("python3 -m build")
 
     # Save the dist files generated by the build process
@@ -129,25 +127,8 @@ def build_and_save_output(root_path, __version__):
     return whl_path
 
 
-def build(root_path, is_nightly=False):
-    if os.path.exists(build_directory):
-        raise ValueError(f"Directory already exists: {build_directory}")
-
-    try:
-        copy_source_to_build_directory(root_path)
-        run_namex_conversion()
-        # Make sure to export the __version__ string
-        from keras_nlp.src import __version__  # noqa: E402
-
-        export_version_string(__version__, is_nightly)
-        return build_and_save_output(root_path, __version__)
-    finally:
-        # Clean up: remove the build directory (no longer needed)
-        shutil.rmtree(build_directory)
-
-
 def install_whl(whl_fpath):
-    print("Installing wheel file.")
+    print(f"Installing wheel file: {whl_fpath}")
     os.system(f"pip3 install {whl_fpath} --force-reinstall --no-dependencies")
 
 
diff --git a/shell/api_gen.sh b/shell/api_gen.sh
new file mode 100755
index 0000000000..389874b890
--- /dev/null
+++ b/shell/api_gen.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+set -Eeuo pipefail
+
+base_dir=$(dirname $(dirname $0))
+
+echo "Generating api directory with public APIs..."
+# Generate API Files
+python3 "${base_dir}"/api_gen.py
+
+echo "Formatting api directory..."
+# Format API Files
+bash "${base_dir}"/shell/format.sh


Test patch:
diff --git a/keras_nlp/conftest.py b/conftest.py
similarity index 97%
rename from keras_nlp/conftest.py
rename to conftest.py
index 819ba80e86..77fe5217fb 100644
--- a/keras_nlp/conftest.py
+++ b/conftest.py
@@ -24,8 +24,8 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.backend import config as backend_config
-from keras_nlp.backend import keras
+from keras_nlp.src.backend import config as backend_config
+from keras_nlp.src.backend import keras
 
 
 def pytest_addoption(parser):
diff --git a/keras_nlp/layers/modeling/alibi_bias_test.py b/keras_nlp/src/layers/modeling/alibi_bias_test.py
similarity index 96%
rename from keras_nlp/layers/modeling/alibi_bias_test.py
rename to keras_nlp/src/layers/modeling/alibi_bias_test.py
index 69a48f29e1..bba5be715c 100644
--- a/keras_nlp/layers/modeling/alibi_bias_test.py
+++ b/keras_nlp/src/layers/modeling/alibi_bias_test.py
@@ -12,11 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.backend import random
-from keras_nlp.layers.modeling.alibi_bias import AlibiBias
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.backend import random
+from keras_nlp.src.layers.modeling.alibi_bias import AlibiBias
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class AlibiBiasTest(TestCase):
diff --git a/keras_nlp/layers/modeling/cached_multi_head_attention_test.py b/keras_nlp/src/layers/modeling/cached_multi_head_attention_test.py
similarity index 92%
rename from keras_nlp/layers/modeling/cached_multi_head_attention_test.py
rename to keras_nlp/src/layers/modeling/cached_multi_head_attention_test.py
index 6bf4311423..c44096593b 100644
--- a/keras_nlp/layers/modeling/cached_multi_head_attention_test.py
+++ b/keras_nlp/src/layers/modeling/cached_multi_head_attention_test.py
@@ -12,13 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.backend import config
-from keras_nlp.backend import ops
-from keras_nlp.backend import random
-from keras_nlp.layers.modeling.cached_multi_head_attention import (
+from keras_nlp.src.backend import config
+from keras_nlp.src.backend import ops
+from keras_nlp.src.backend import random
+from keras_nlp.src.layers.modeling.cached_multi_head_attention import (
     CachedMultiHeadAttention,
 )
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class CachedMultiHeadAttentionTest(TestCase):
diff --git a/keras_nlp/layers/modeling/f_net_encoder_test.py b/keras_nlp/src/layers/modeling/f_net_encoder_test.py
similarity index 90%
rename from keras_nlp/layers/modeling/f_net_encoder_test.py
rename to keras_nlp/src/layers/modeling/f_net_encoder_test.py
index 47c189ef11..c6d086a1a7 100644
--- a/keras_nlp/layers/modeling/f_net_encoder_test.py
+++ b/keras_nlp/src/layers/modeling/f_net_encoder_test.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.backend import random
-from keras_nlp.layers.modeling.f_net_encoder import FNetEncoder
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import random
+from keras_nlp.src.layers.modeling.f_net_encoder import FNetEncoder
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class FNetEncoderTest(TestCase):
diff --git a/keras_nlp/layers/modeling/masked_lm_head_test.py b/keras_nlp/src/layers/modeling/masked_lm_head_test.py
similarity index 90%
rename from keras_nlp/layers/modeling/masked_lm_head_test.py
rename to keras_nlp/src/layers/modeling/masked_lm_head_test.py
index 69a6288911..b4f8816755 100644
--- a/keras_nlp/layers/modeling/masked_lm_head_test.py
+++ b/keras_nlp/src/layers/modeling/masked_lm_head_test.py
@@ -12,10 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.backend import random
-from keras_nlp.layers.modeling.masked_lm_head import MaskedLMHead
-from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import random
+from keras_nlp.src.layers.modeling.masked_lm_head import MaskedLMHead
+from keras_nlp.src.layers.modeling.reversible_embedding import (
+    ReversibleEmbedding,
+)
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class MaskedLMHeadTest(TestCase):
diff --git a/keras_nlp/layers/modeling/position_embedding_test.py b/keras_nlp/src/layers/modeling/position_embedding_test.py
similarity index 96%
rename from keras_nlp/layers/modeling/position_embedding_test.py
rename to keras_nlp/src/layers/modeling/position_embedding_test.py
index 549411e0b8..22765ce525 100644
--- a/keras_nlp/layers/modeling/position_embedding_test.py
+++ b/keras_nlp/src/layers/modeling/position_embedding_test.py
@@ -14,11 +14,11 @@
 
 import numpy as np
 
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.backend import random
-from keras_nlp.layers.modeling.position_embedding import PositionEmbedding
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.backend import random
+from keras_nlp.src.layers.modeling.position_embedding import PositionEmbedding
+from keras_nlp.src.tests.test_case import TestCase
 
 
 def custom_init(shape, dtype=None):
diff --git a/keras_nlp/layers/modeling/reversible_embedding_test.py b/keras_nlp/src/layers/modeling/reversible_embedding_test.py
similarity index 92%
rename from keras_nlp/layers/modeling/reversible_embedding_test.py
rename to keras_nlp/src/layers/modeling/reversible_embedding_test.py
index 62ddd49d77..b3153b9e17 100644
--- a/keras_nlp/layers/modeling/reversible_embedding_test.py
+++ b/keras_nlp/src/layers/modeling/reversible_embedding_test.py
@@ -17,12 +17,14 @@
 import numpy as np
 from absl.testing import parameterized
 
-from keras_nlp.backend import config
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.backend import random
-from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import config
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.backend import random
+from keras_nlp.src.layers.modeling.reversible_embedding import (
+    ReversibleEmbedding,
+)
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class ReversibleEmbeddingTest(TestCase):
diff --git a/keras_nlp/layers/modeling/rotary_embedding_test.py b/keras_nlp/src/layers/modeling/rotary_embedding_test.py
similarity index 96%
rename from keras_nlp/layers/modeling/rotary_embedding_test.py
rename to keras_nlp/src/layers/modeling/rotary_embedding_test.py
index 28ebf44929..3ce7763236 100644
--- a/keras_nlp/layers/modeling/rotary_embedding_test.py
+++ b/keras_nlp/src/layers/modeling/rotary_embedding_test.py
@@ -13,11 +13,11 @@
 # limitations under the License.
 import numpy as np
 
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.backend import random
-from keras_nlp.layers.modeling.rotary_embedding import RotaryEmbedding
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.backend import random
+from keras_nlp.src.layers.modeling.rotary_embedding import RotaryEmbedding
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class RotaryEmbeddingTest(TestCase):
diff --git a/keras_nlp/layers/modeling/sine_position_encoding_test.py b/keras_nlp/src/layers/modeling/sine_position_encoding_test.py
similarity index 94%
rename from keras_nlp/layers/modeling/sine_position_encoding_test.py
rename to keras_nlp/src/layers/modeling/sine_position_encoding_test.py
index e7ea4b1b05..7ce040c375 100644
--- a/keras_nlp/layers/modeling/sine_position_encoding_test.py
+++ b/keras_nlp/src/layers/modeling/sine_position_encoding_test.py
@@ -12,13 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.backend import random
-from keras_nlp.layers.modeling.sine_position_encoding import (
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.backend import random
+from keras_nlp.src.layers.modeling.sine_position_encoding import (
     SinePositionEncoding,
 )
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class SinePositionEncodingTest(TestCase):
diff --git a/keras_nlp/layers/modeling/token_and_position_embedding_test.py b/keras_nlp/src/layers/modeling/token_and_position_embedding_test.py
similarity index 88%
rename from keras_nlp/layers/modeling/token_and_position_embedding_test.py
rename to keras_nlp/src/layers/modeling/token_and_position_embedding_test.py
index 122d74e13d..3e2219a6a1 100644
--- a/keras_nlp/layers/modeling/token_and_position_embedding_test.py
+++ b/keras_nlp/src/layers/modeling/token_and_position_embedding_test.py
@@ -14,12 +14,12 @@
 
 import numpy as np
 
-from keras_nlp.backend import ops
-from keras_nlp.backend import random
-from keras_nlp.layers.modeling.token_and_position_embedding import (
+from keras_nlp.src.backend import ops
+from keras_nlp.src.backend import random
+from keras_nlp.src.layers.modeling.token_and_position_embedding import (
     TokenAndPositionEmbedding,
 )
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class TokenAndPositionEmbeddingTest(TestCase):
diff --git a/keras_nlp/layers/modeling/transformer_decoder_test.py b/keras_nlp/src/layers/modeling/transformer_decoder_test.py
similarity index 97%
rename from keras_nlp/layers/modeling/transformer_decoder_test.py
rename to keras_nlp/src/layers/modeling/transformer_decoder_test.py
index aa16d9ae5a..f5c9a17c7e 100644
--- a/keras_nlp/layers/modeling/transformer_decoder_test.py
+++ b/keras_nlp/src/layers/modeling/transformer_decoder_test.py
@@ -14,10 +14,10 @@
 
 from absl.testing import parameterized
 
-from keras_nlp.backend import ops
-from keras_nlp.backend import random
-from keras_nlp.layers.modeling.transformer_decoder import TransformerDecoder
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.backend import random
+from keras_nlp.src.layers.modeling.transformer_decoder import TransformerDecoder
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class TransformerDecoderTest(TestCase):
diff --git a/keras_nlp/layers/modeling/transformer_encoder_test.py b/keras_nlp/src/layers/modeling/transformer_encoder_test.py
similarity index 92%
rename from keras_nlp/layers/modeling/transformer_encoder_test.py
rename to keras_nlp/src/layers/modeling/transformer_encoder_test.py
index edcfdfc470..99edf9b0ff 100644
--- a/keras_nlp/layers/modeling/transformer_encoder_test.py
+++ b/keras_nlp/src/layers/modeling/transformer_encoder_test.py
@@ -14,11 +14,11 @@
 
 from absl.testing import parameterized
 
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.backend import random
-from keras_nlp.layers.modeling.transformer_encoder import TransformerEncoder
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.backend import random
+from keras_nlp.src.layers.modeling.transformer_encoder import TransformerEncoder
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class TransformerEncoderTest(TestCase):
diff --git a/keras_nlp/layers/modeling/transformer_layer_utils_test.py b/keras_nlp/src/layers/modeling/transformer_layer_utils_test.py
similarity index 90%
rename from keras_nlp/layers/modeling/transformer_layer_utils_test.py
rename to keras_nlp/src/layers/modeling/transformer_layer_utils_test.py
index 57df677e94..488f3c6fd4 100644
--- a/keras_nlp/layers/modeling/transformer_layer_utils_test.py
+++ b/keras_nlp/src/layers/modeling/transformer_layer_utils_test.py
@@ -12,10 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import keras_nlp.layers.modeling.transformer_layer_utils as utils
-from keras_nlp.backend import ops
-from keras_nlp.backend import random
-from keras_nlp.tests.test_case import TestCase
+import keras_nlp.src.layers.modeling.transformer_layer_utils as utils
+from keras_nlp.src.backend import ops
+from keras_nlp.src.backend import random
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class TransformerLayerUtilsTest(TestCase):
diff --git a/keras_nlp/layers/preprocessing/masked_lm_mask_generator_test.py b/keras_nlp/src/layers/preprocessing/masked_lm_mask_generator_test.py
similarity index 97%
rename from keras_nlp/layers/preprocessing/masked_lm_mask_generator_test.py
rename to keras_nlp/src/layers/preprocessing/masked_lm_mask_generator_test.py
index b3894999bf..e42321f67e 100644
--- a/keras_nlp/layers/preprocessing/masked_lm_mask_generator_test.py
+++ b/keras_nlp/src/layers/preprocessing/masked_lm_mask_generator_test.py
@@ -20,11 +20,11 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.backend import ops
-from keras_nlp.layers.preprocessing.masked_lm_mask_generator import (
+from keras_nlp.src.backend import ops
+from keras_nlp.src.layers.preprocessing.masked_lm_mask_generator import (
     MaskedLMMaskGenerator,
 )
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class MaskedLMMaskGeneratorTest(TestCase):
diff --git a/keras_nlp/layers/preprocessing/multi_segment_packer_test.py b/keras_nlp/src/layers/preprocessing/multi_segment_packer_test.py
similarity index 98%
rename from keras_nlp/layers/preprocessing/multi_segment_packer_test.py
rename to keras_nlp/src/layers/preprocessing/multi_segment_packer_test.py
index 6c17495066..36ad013f36 100644
--- a/keras_nlp/layers/preprocessing/multi_segment_packer_test.py
+++ b/keras_nlp/src/layers/preprocessing/multi_segment_packer_test.py
@@ -14,10 +14,10 @@
 
 import numpy as np
 
-from keras_nlp.layers.preprocessing.multi_segment_packer import (
+from keras_nlp.src.layers.preprocessing.multi_segment_packer import (
     MultiSegmentPacker,
 )
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class MultiSegmentPackerTest(TestCase):
diff --git a/keras_nlp/layers/preprocessing/random_deletion_test.py b/keras_nlp/src/layers/preprocessing/random_deletion_test.py
similarity index 97%
rename from keras_nlp/layers/preprocessing/random_deletion_test.py
rename to keras_nlp/src/layers/preprocessing/random_deletion_test.py
index ea834c7401..3fd23c1c06 100644
--- a/keras_nlp/layers/preprocessing/random_deletion_test.py
+++ b/keras_nlp/src/layers/preprocessing/random_deletion_test.py
@@ -20,9 +20,9 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.backend import keras
-from keras_nlp.layers.preprocessing.random_deletion import RandomDeletion
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.preprocessing.random_deletion import RandomDeletion
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class RandomDeletionTest(TestCase):
diff --git a/keras_nlp/layers/preprocessing/random_swap_test.py b/keras_nlp/src/layers/preprocessing/random_swap_test.py
similarity index 97%
rename from keras_nlp/layers/preprocessing/random_swap_test.py
rename to keras_nlp/src/layers/preprocessing/random_swap_test.py
index b4628ad0c8..01290793a9 100644
--- a/keras_nlp/layers/preprocessing/random_swap_test.py
+++ b/keras_nlp/src/layers/preprocessing/random_swap_test.py
@@ -20,9 +20,9 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.backend import keras
-from keras_nlp.layers.preprocessing.random_swap import RandomSwap
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import keras
+from keras_nlp.src.layers.preprocessing.random_swap import RandomSwap
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class RandomSwapTest(TestCase):
diff --git a/keras_nlp/layers/preprocessing/start_end_packer_test.py b/keras_nlp/src/layers/preprocessing/start_end_packer_test.py
similarity index 98%
rename from keras_nlp/layers/preprocessing/start_end_packer_test.py
rename to keras_nlp/src/layers/preprocessing/start_end_packer_test.py
index 017fce5ed0..7011f30e9c 100644
--- a/keras_nlp/layers/preprocessing/start_end_packer_test.py
+++ b/keras_nlp/src/layers/preprocessing/start_end_packer_test.py
@@ -23,8 +23,8 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.layers.preprocessing.start_end_packer import StartEndPacker
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.layers.preprocessing.start_end_packer import StartEndPacker
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class StartEndPackerTest(TestCase):
diff --git a/keras_nlp/metrics/bleu_test.py b/keras_nlp/src/metrics/bleu_test.py
similarity index 96%
rename from keras_nlp/metrics/bleu_test.py
rename to keras_nlp/src/metrics/bleu_test.py
index 58dd2a3d85..d8f5883a4c 100644
--- a/keras_nlp/metrics/bleu_test.py
+++ b/keras_nlp/src/metrics/bleu_test.py
@@ -22,10 +22,10 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.backend import keras
-from keras_nlp.metrics.bleu import Bleu
-from keras_nlp.tests.test_case import TestCase
-from keras_nlp.tokenizers.byte_tokenizer import ByteTokenizer
+from keras_nlp.src.backend import keras
+from keras_nlp.src.metrics.bleu import Bleu
+from keras_nlp.src.tests.test_case import TestCase
+from keras_nlp.src.tokenizers.byte_tokenizer import ByteTokenizer
 
 
 class BleuTest(TestCase):
diff --git a/keras_nlp/metrics/edit_distance_test.py b/keras_nlp/src/metrics/edit_distance_test.py
similarity index 98%
rename from keras_nlp/metrics/edit_distance_test.py
rename to keras_nlp/src/metrics/edit_distance_test.py
index ae17b7710c..4e3e4712a1 100644
--- a/keras_nlp/metrics/edit_distance_test.py
+++ b/keras_nlp/src/metrics/edit_distance_test.py
@@ -22,9 +22,9 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.backend import keras
-from keras_nlp.metrics.edit_distance import EditDistance
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import keras
+from keras_nlp.src.metrics.edit_distance import EditDistance
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class EditDistanceTest(TestCase):
diff --git a/keras_nlp/metrics/perplexity_test.py b/keras_nlp/src/metrics/perplexity_test.py
similarity index 98%
rename from keras_nlp/metrics/perplexity_test.py
rename to keras_nlp/src/metrics/perplexity_test.py
index 7b54e8d260..00a9ddf2a6 100644
--- a/keras_nlp/metrics/perplexity_test.py
+++ b/keras_nlp/src/metrics/perplexity_test.py
@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.backend import ops
-from keras_nlp.metrics.perplexity import Perplexity
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.metrics.perplexity import Perplexity
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class PerplexityTest(TestCase):
diff --git a/keras_nlp/metrics/rouge_l_test.py b/keras_nlp/src/metrics/rouge_l_test.py
similarity index 97%
rename from keras_nlp/metrics/rouge_l_test.py
rename to keras_nlp/src/metrics/rouge_l_test.py
index 4211c29f51..9f3f03d1bf 100644
--- a/keras_nlp/metrics/rouge_l_test.py
+++ b/keras_nlp/src/metrics/rouge_l_test.py
@@ -20,8 +20,8 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.metrics.rouge_l import RougeL
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.metrics.rouge_l import RougeL
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class RougeLTest(TestCase):
diff --git a/keras_nlp/metrics/rouge_n_test.py b/keras_nlp/src/metrics/rouge_n_test.py
similarity index 97%
rename from keras_nlp/metrics/rouge_n_test.py
rename to keras_nlp/src/metrics/rouge_n_test.py
index 1d5da61d4d..28d11d05b2 100644
--- a/keras_nlp/metrics/rouge_n_test.py
+++ b/keras_nlp/src/metrics/rouge_n_test.py
@@ -22,9 +22,9 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.backend import keras
-from keras_nlp.metrics.rouge_n import RougeN
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import keras
+from keras_nlp.src.metrics.rouge_n import RougeN
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class RougeNTest(TestCase):
diff --git a/keras_nlp/models/albert/albert_backbone_test.py b/keras_nlp/src/models/albert/albert_backbone_test.py
similarity index 95%
rename from keras_nlp/models/albert/albert_backbone_test.py
rename to keras_nlp/src/models/albert/albert_backbone_test.py
index f1211e0fa3..60d73fb2cd 100644
--- a/keras_nlp/models/albert/albert_backbone_test.py
+++ b/keras_nlp/src/models/albert/albert_backbone_test.py
@@ -14,9 +14,9 @@
 
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.albert.albert_backbone import AlbertBackbone
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.albert.albert_backbone import AlbertBackbone
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class AlbertBackboneTest(TestCase):
diff --git a/keras_nlp/models/albert/albert_classifier_test.py b/keras_nlp/src/models/albert/albert_classifier_test.py
similarity index 87%
rename from keras_nlp/models/albert/albert_classifier_test.py
rename to keras_nlp/src/models/albert/albert_classifier_test.py
index 5a60ff998b..e15a2826a2 100644
--- a/keras_nlp/models/albert/albert_classifier_test.py
+++ b/keras_nlp/src/models/albert/albert_classifier_test.py
@@ -16,11 +16,11 @@
 
 import pytest
 
-from keras_nlp.models.albert.albert_backbone import AlbertBackbone
-from keras_nlp.models.albert.albert_classifier import AlbertClassifier
-from keras_nlp.models.albert.albert_preprocessor import AlbertPreprocessor
-from keras_nlp.models.albert.albert_tokenizer import AlbertTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.albert.albert_backbone import AlbertBackbone
+from keras_nlp.src.models.albert.albert_classifier import AlbertClassifier
+from keras_nlp.src.models.albert.albert_preprocessor import AlbertPreprocessor
+from keras_nlp.src.models.albert.albert_tokenizer import AlbertTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class AlbertClassifierTest(TestCase):
diff --git a/keras_nlp/models/albert/albert_masked_lm_preprocessor_test.py b/keras_nlp/src/models/albert/albert_masked_lm_preprocessor_test.py
similarity index 93%
rename from keras_nlp/models/albert/albert_masked_lm_preprocessor_test.py
rename to keras_nlp/src/models/albert/albert_masked_lm_preprocessor_test.py
index b9bf693c17..70ef70d08f 100644
--- a/keras_nlp/models/albert/albert_masked_lm_preprocessor_test.py
+++ b/keras_nlp/src/models/albert/albert_masked_lm_preprocessor_test.py
@@ -16,11 +16,11 @@
 
 import pytest
 
-from keras_nlp.models.albert.albert_masked_lm_preprocessor import (
+from keras_nlp.src.models.albert.albert_masked_lm_preprocessor import (
     AlbertMaskedLMPreprocessor,
 )
-from keras_nlp.models.albert.albert_tokenizer import AlbertTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.albert.albert_tokenizer import AlbertTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class AlbertMaskedLMPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/albert/albert_masked_lm_test.py b/keras_nlp/src/models/albert/albert_masked_lm_test.py
similarity index 88%
rename from keras_nlp/models/albert/albert_masked_lm_test.py
rename to keras_nlp/src/models/albert/albert_masked_lm_test.py
index 49e6a595cd..954fc32b4a 100644
--- a/keras_nlp/models/albert/albert_masked_lm_test.py
+++ b/keras_nlp/src/models/albert/albert_masked_lm_test.py
@@ -16,13 +16,13 @@
 
 import pytest
 
-from keras_nlp.models.albert.albert_backbone import AlbertBackbone
-from keras_nlp.models.albert.albert_masked_lm import AlbertMaskedLM
-from keras_nlp.models.albert.albert_masked_lm_preprocessor import (
+from keras_nlp.src.models.albert.albert_backbone import AlbertBackbone
+from keras_nlp.src.models.albert.albert_masked_lm import AlbertMaskedLM
+from keras_nlp.src.models.albert.albert_masked_lm_preprocessor import (
     AlbertMaskedLMPreprocessor,
 )
-from keras_nlp.models.albert.albert_tokenizer import AlbertTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.albert.albert_tokenizer import AlbertTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class AlbertMaskedLMTest(TestCase):
diff --git a/keras_nlp/models/albert/albert_preprocessor_test.py b/keras_nlp/src/models/albert/albert_preprocessor_test.py
similarity index 91%
rename from keras_nlp/models/albert/albert_preprocessor_test.py
rename to keras_nlp/src/models/albert/albert_preprocessor_test.py
index ad5da8a47b..b13234d53a 100644
--- a/keras_nlp/models/albert/albert_preprocessor_test.py
+++ b/keras_nlp/src/models/albert/albert_preprocessor_test.py
@@ -16,9 +16,9 @@
 
 import pytest
 
-from keras_nlp.models.albert.albert_preprocessor import AlbertPreprocessor
-from keras_nlp.models.albert.albert_tokenizer import AlbertTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.albert.albert_preprocessor import AlbertPreprocessor
+from keras_nlp.src.models.albert.albert_tokenizer import AlbertTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class AlbertPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/albert/albert_tokenizer_test.py b/keras_nlp/src/models/albert/albert_tokenizer_test.py
similarity index 94%
rename from keras_nlp/models/albert/albert_tokenizer_test.py
rename to keras_nlp/src/models/albert/albert_tokenizer_test.py
index 74ad0604dc..a6805ee085 100644
--- a/keras_nlp/models/albert/albert_tokenizer_test.py
+++ b/keras_nlp/src/models/albert/albert_tokenizer_test.py
@@ -16,8 +16,8 @@
 
 import pytest
 
-from keras_nlp.models.albert.albert_tokenizer import AlbertTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.albert.albert_tokenizer import AlbertTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class AlbertTokenizerTest(TestCase):
diff --git a/keras_nlp/models/backbone_test.py b/keras_nlp/src/models/backbone_test.py
similarity index 85%
rename from keras_nlp/models/backbone_test.py
rename to keras_nlp/src/models/backbone_test.py
index 5f37c0a75b..bc677ab966 100644
--- a/keras_nlp/models/backbone_test.py
+++ b/keras_nlp/src/models/backbone_test.py
@@ -16,15 +16,15 @@
 import numpy as np
 import pytest
 
-from keras_nlp.models.backbone import Backbone
-from keras_nlp.models.bert.bert_backbone import BertBackbone
-from keras_nlp.models.gpt2.gpt2_backbone import GPT2Backbone
-from keras_nlp.tests.test_case import TestCase
-from keras_nlp.utils.preset_utils import CONFIG_FILE
-from keras_nlp.utils.preset_utils import METADATA_FILE
-from keras_nlp.utils.preset_utils import MODEL_WEIGHTS_FILE
-from keras_nlp.utils.preset_utils import check_config_class
-from keras_nlp.utils.preset_utils import load_config
+from keras_nlp.src.models.backbone import Backbone
+from keras_nlp.src.models.bert.bert_backbone import BertBackbone
+from keras_nlp.src.models.gpt2.gpt2_backbone import GPT2Backbone
+from keras_nlp.src.tests.test_case import TestCase
+from keras_nlp.src.utils.preset_utils import CONFIG_FILE
+from keras_nlp.src.utils.preset_utils import METADATA_FILE
+from keras_nlp.src.utils.preset_utils import MODEL_WEIGHTS_FILE
+from keras_nlp.src.utils.preset_utils import check_config_class
+from keras_nlp.src.utils.preset_utils import load_config
 
 
 class TestTask(TestCase):
diff --git a/keras_nlp/models/bart/bart_backbone_test.py b/keras_nlp/src/models/bart/bart_backbone_test.py
similarity index 95%
rename from keras_nlp/models/bart/bart_backbone_test.py
rename to keras_nlp/src/models/bart/bart_backbone_test.py
index fe4b6af52a..53e131563a 100644
--- a/keras_nlp/models/bart/bart_backbone_test.py
+++ b/keras_nlp/src/models/bart/bart_backbone_test.py
@@ -14,9 +14,9 @@
 
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.bart.bart_backbone import BartBackbone
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.bart.bart_backbone import BartBackbone
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class BartBackboneTest(TestCase):
diff --git a/keras_nlp/models/bart/bart_preprocessor_test.py b/keras_nlp/src/models/bart/bart_preprocessor_test.py
similarity index 94%
rename from keras_nlp/models/bart/bart_preprocessor_test.py
rename to keras_nlp/src/models/bart/bart_preprocessor_test.py
index 8f4ccf7930..1679d4f55e 100644
--- a/keras_nlp/models/bart/bart_preprocessor_test.py
+++ b/keras_nlp/src/models/bart/bart_preprocessor_test.py
@@ -22,9 +22,9 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.models.bart.bart_preprocessor import BartPreprocessor
-from keras_nlp.models.bart.bart_tokenizer import BartTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.bart.bart_preprocessor import BartPreprocessor
+from keras_nlp.src.models.bart.bart_tokenizer import BartTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class BartPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/bart/bart_seq_2_seq_lm_preprocessor_test.py b/keras_nlp/src/models/bart/bart_seq_2_seq_lm_preprocessor_test.py
similarity index 94%
rename from keras_nlp/models/bart/bart_seq_2_seq_lm_preprocessor_test.py
rename to keras_nlp/src/models/bart/bart_seq_2_seq_lm_preprocessor_test.py
index 2f40e69722..1cc9aef070 100644
--- a/keras_nlp/models/bart/bart_seq_2_seq_lm_preprocessor_test.py
+++ b/keras_nlp/src/models/bart/bart_seq_2_seq_lm_preprocessor_test.py
@@ -14,11 +14,11 @@
 
 import pytest
 
-from keras_nlp.models.bart.bart_seq_2_seq_lm_preprocessor import (
+from keras_nlp.src.models.bart.bart_seq_2_seq_lm_preprocessor import (
     BartSeq2SeqLMPreprocessor,
 )
-from keras_nlp.models.bart.bart_tokenizer import BartTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.bart.bart_tokenizer import BartTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class BartSeq2SeqLMPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/bart/bart_seq_2_seq_lm_test.py b/keras_nlp/src/models/bart/bart_seq_2_seq_lm_test.py
similarity index 94%
rename from keras_nlp/models/bart/bart_seq_2_seq_lm_test.py
rename to keras_nlp/src/models/bart/bart_seq_2_seq_lm_test.py
index 2edd6b3a94..0f603c3b78 100644
--- a/keras_nlp/models/bart/bart_seq_2_seq_lm_test.py
+++ b/keras_nlp/src/models/bart/bart_seq_2_seq_lm_test.py
@@ -16,14 +16,14 @@
 
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.bart.bart_backbone import BartBackbone
-from keras_nlp.models.bart.bart_seq_2_seq_lm import BartSeq2SeqLM
-from keras_nlp.models.bart.bart_seq_2_seq_lm_preprocessor import (
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.bart.bart_backbone import BartBackbone
+from keras_nlp.src.models.bart.bart_seq_2_seq_lm import BartSeq2SeqLM
+from keras_nlp.src.models.bart.bart_seq_2_seq_lm_preprocessor import (
     BartSeq2SeqLMPreprocessor,
 )
-from keras_nlp.models.bart.bart_tokenizer import BartTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.bart.bart_tokenizer import BartTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class BartSeq2SeqLMTest(TestCase):
diff --git a/keras_nlp/models/bart/bart_tokenizer_test.py b/keras_nlp/src/models/bart/bart_tokenizer_test.py
similarity index 95%
rename from keras_nlp/models/bart/bart_tokenizer_test.py
rename to keras_nlp/src/models/bart/bart_tokenizer_test.py
index 5a0015357b..c11c49adf5 100644
--- a/keras_nlp/models/bart/bart_tokenizer_test.py
+++ b/keras_nlp/src/models/bart/bart_tokenizer_test.py
@@ -14,8 +14,8 @@
 
 import pytest
 
-from keras_nlp.models.bart.bart_tokenizer import BartTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.bart.bart_tokenizer import BartTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class BartTokenizerTest(TestCase):
diff --git a/keras_nlp/models/bert/bert_backbone_test.py b/keras_nlp/src/models/bert/bert_backbone_test.py
similarity index 94%
rename from keras_nlp/models/bert/bert_backbone_test.py
rename to keras_nlp/src/models/bert/bert_backbone_test.py
index c1039114f5..4793f26fc2 100644
--- a/keras_nlp/models/bert/bert_backbone_test.py
+++ b/keras_nlp/src/models/bert/bert_backbone_test.py
@@ -14,9 +14,9 @@
 
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.bert.bert_backbone import BertBackbone
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.bert.bert_backbone import BertBackbone
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class BertBackboneTest(TestCase):
diff --git a/keras_nlp/models/bert/bert_classifier_test.py b/keras_nlp/src/models/bert/bert_classifier_test.py
similarity index 88%
rename from keras_nlp/models/bert/bert_classifier_test.py
rename to keras_nlp/src/models/bert/bert_classifier_test.py
index 53da9d4fab..ef4d81aa98 100644
--- a/keras_nlp/models/bert/bert_classifier_test.py
+++ b/keras_nlp/src/models/bert/bert_classifier_test.py
@@ -14,11 +14,11 @@
 
 import pytest
 
-from keras_nlp.models.bert.bert_backbone import BertBackbone
-from keras_nlp.models.bert.bert_classifier import BertClassifier
-from keras_nlp.models.bert.bert_preprocessor import BertPreprocessor
-from keras_nlp.models.bert.bert_tokenizer import BertTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.bert.bert_backbone import BertBackbone
+from keras_nlp.src.models.bert.bert_classifier import BertClassifier
+from keras_nlp.src.models.bert.bert_preprocessor import BertPreprocessor
+from keras_nlp.src.models.bert.bert_tokenizer import BertTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class BertClassifierTest(TestCase):
diff --git a/keras_nlp/models/bert/bert_masked_lm_preprocessor_test.py b/keras_nlp/src/models/bert/bert_masked_lm_preprocessor_test.py
similarity index 94%
rename from keras_nlp/models/bert/bert_masked_lm_preprocessor_test.py
rename to keras_nlp/src/models/bert/bert_masked_lm_preprocessor_test.py
index 479d9e879b..4b898f7bff 100644
--- a/keras_nlp/models/bert/bert_masked_lm_preprocessor_test.py
+++ b/keras_nlp/src/models/bert/bert_masked_lm_preprocessor_test.py
@@ -14,11 +14,11 @@
 
 import pytest
 
-from keras_nlp.models.bert.bert_masked_lm_preprocessor import (
+from keras_nlp.src.models.bert.bert_masked_lm_preprocessor import (
     BertMaskedLMPreprocessor,
 )
-from keras_nlp.models.bert.bert_tokenizer import BertTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.bert.bert_tokenizer import BertTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class BertMaskedLMPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/bert/bert_masked_lm_test.py b/keras_nlp/src/models/bert/bert_masked_lm_test.py
similarity index 88%
rename from keras_nlp/models/bert/bert_masked_lm_test.py
rename to keras_nlp/src/models/bert/bert_masked_lm_test.py
index dd6f41b0b7..de8d8e651b 100644
--- a/keras_nlp/models/bert/bert_masked_lm_test.py
+++ b/keras_nlp/src/models/bert/bert_masked_lm_test.py
@@ -14,13 +14,13 @@
 
 import pytest
 
-from keras_nlp.models.bert.bert_backbone import BertBackbone
-from keras_nlp.models.bert.bert_masked_lm import BertMaskedLM
-from keras_nlp.models.bert.bert_masked_lm_preprocessor import (
+from keras_nlp.src.models.bert.bert_backbone import BertBackbone
+from keras_nlp.src.models.bert.bert_masked_lm import BertMaskedLM
+from keras_nlp.src.models.bert.bert_masked_lm_preprocessor import (
     BertMaskedLMPreprocessor,
 )
-from keras_nlp.models.bert.bert_tokenizer import BertTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.bert.bert_tokenizer import BertTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class BertMaskedLMTest(TestCase):
diff --git a/keras_nlp/models/bert/bert_preprocessor_test.py b/keras_nlp/src/models/bert/bert_preprocessor_test.py
similarity index 92%
rename from keras_nlp/models/bert/bert_preprocessor_test.py
rename to keras_nlp/src/models/bert/bert_preprocessor_test.py
index c109d1006d..0ab5544693 100644
--- a/keras_nlp/models/bert/bert_preprocessor_test.py
+++ b/keras_nlp/src/models/bert/bert_preprocessor_test.py
@@ -14,9 +14,9 @@
 
 import pytest
 
-from keras_nlp.models.bert.bert_preprocessor import BertPreprocessor
-from keras_nlp.models.bert.bert_tokenizer import BertTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.bert.bert_preprocessor import BertPreprocessor
+from keras_nlp.src.models.bert.bert_tokenizer import BertTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class BertPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/bert/bert_tokenizer_test.py b/keras_nlp/src/models/bert/bert_tokenizer_test.py
similarity index 95%
rename from keras_nlp/models/bert/bert_tokenizer_test.py
rename to keras_nlp/src/models/bert/bert_tokenizer_test.py
index 78b4417e13..0ee192b559 100644
--- a/keras_nlp/models/bert/bert_tokenizer_test.py
+++ b/keras_nlp/src/models/bert/bert_tokenizer_test.py
@@ -14,8 +14,8 @@
 
 import pytest
 
-from keras_nlp.models.bert.bert_tokenizer import BertTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.bert.bert_tokenizer import BertTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class BertTokenizerTest(TestCase):
diff --git a/keras_nlp/models/bloom/bloom_backbone_test.py b/keras_nlp/src/models/bloom/bloom_backbone_test.py
similarity index 93%
rename from keras_nlp/models/bloom/bloom_backbone_test.py
rename to keras_nlp/src/models/bloom/bloom_backbone_test.py
index 47ff7ec4cc..0eab635fc5 100644
--- a/keras_nlp/models/bloom/bloom_backbone_test.py
+++ b/keras_nlp/src/models/bloom/bloom_backbone_test.py
@@ -14,9 +14,9 @@
 
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.bloom.bloom_backbone import BloomBackbone
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.bloom.bloom_backbone import BloomBackbone
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class BloomBackboneTest(TestCase):
diff --git a/keras_nlp/models/bloom/bloom_causal_lm_preprocessor_test.py b/keras_nlp/src/models/bloom/bloom_causal_lm_preprocessor_test.py
similarity index 94%
rename from keras_nlp/models/bloom/bloom_causal_lm_preprocessor_test.py
rename to keras_nlp/src/models/bloom/bloom_causal_lm_preprocessor_test.py
index a281519340..f0c197c578 100644
--- a/keras_nlp/models/bloom/bloom_causal_lm_preprocessor_test.py
+++ b/keras_nlp/src/models/bloom/bloom_causal_lm_preprocessor_test.py
@@ -14,11 +14,11 @@
 
 import pytest
 
-from keras_nlp.models.bloom.bloom_causal_lm_preprocessor import (
+from keras_nlp.src.models.bloom.bloom_causal_lm_preprocessor import (
     BloomCausalLMPreprocessor,
 )
-from keras_nlp.models.bloom.bloom_tokenizer import BloomTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.bloom.bloom_tokenizer import BloomTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class BloomCausalLMPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/bloom/bloom_causal_lm_test.py b/keras_nlp/src/models/bloom/bloom_causal_lm_test.py
similarity index 94%
rename from keras_nlp/models/bloom/bloom_causal_lm_test.py
rename to keras_nlp/src/models/bloom/bloom_causal_lm_test.py
index 431a734da5..ef9f7e552b 100644
--- a/keras_nlp/models/bloom/bloom_causal_lm_test.py
+++ b/keras_nlp/src/models/bloom/bloom_causal_lm_test.py
@@ -16,14 +16,14 @@
 
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.bloom.bloom_backbone import BloomBackbone
-from keras_nlp.models.bloom.bloom_causal_lm import BloomCausalLM
-from keras_nlp.models.bloom.bloom_causal_lm_preprocessor import (
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.bloom.bloom_backbone import BloomBackbone
+from keras_nlp.src.models.bloom.bloom_causal_lm import BloomCausalLM
+from keras_nlp.src.models.bloom.bloom_causal_lm_preprocessor import (
     BloomCausalLMPreprocessor,
 )
-from keras_nlp.models.bloom.bloom_tokenizer import BloomTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.bloom.bloom_tokenizer import BloomTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class BloomCausalLMTest(TestCase):
diff --git a/keras_nlp/models/bloom/bloom_preprocessor_test.py b/keras_nlp/src/models/bloom/bloom_preprocessor_test.py
similarity index 93%
rename from keras_nlp/models/bloom/bloom_preprocessor_test.py
rename to keras_nlp/src/models/bloom/bloom_preprocessor_test.py
index 938113ef4b..d0f486aedc 100644
--- a/keras_nlp/models/bloom/bloom_preprocessor_test.py
+++ b/keras_nlp/src/models/bloom/bloom_preprocessor_test.py
@@ -14,9 +14,9 @@
 
 import pytest
 
-from keras_nlp.models.bloom.bloom_preprocessor import BloomPreprocessor
-from keras_nlp.models.bloom.bloom_tokenizer import BloomTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.bloom.bloom_preprocessor import BloomPreprocessor
+from keras_nlp.src.models.bloom.bloom_tokenizer import BloomTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class BloomPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/bloom/bloom_tokenizer_test.py b/keras_nlp/src/models/bloom/bloom_tokenizer_test.py
similarity index 94%
rename from keras_nlp/models/bloom/bloom_tokenizer_test.py
rename to keras_nlp/src/models/bloom/bloom_tokenizer_test.py
index 9ae9c0cc00..469d9e9637 100644
--- a/keras_nlp/models/bloom/bloom_tokenizer_test.py
+++ b/keras_nlp/src/models/bloom/bloom_tokenizer_test.py
@@ -14,8 +14,8 @@
 
 import pytest
 
-from keras_nlp.models.bloom.bloom_tokenizer import BloomTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.bloom.bloom_tokenizer import BloomTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class BloomTokenizerTest(TestCase):
diff --git a/keras_nlp/models/deberta_v3/deberta_v3_backbone_test.py b/keras_nlp/src/models/deberta_v3/deberta_v3_backbone_test.py
similarity index 93%
rename from keras_nlp/models/deberta_v3/deberta_v3_backbone_test.py
rename to keras_nlp/src/models/deberta_v3/deberta_v3_backbone_test.py
index 3559002864..4db119f9b7 100644
--- a/keras_nlp/models/deberta_v3/deberta_v3_backbone_test.py
+++ b/keras_nlp/src/models/deberta_v3/deberta_v3_backbone_test.py
@@ -14,9 +14,11 @@
 
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.deberta_v3.deberta_v3_backbone import DebertaV3Backbone
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.deberta_v3.deberta_v3_backbone import (
+    DebertaV3Backbone,
+)
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class DebertaV3BackboneTest(TestCase):
diff --git a/keras_nlp/models/deberta_v3/deberta_v3_classifier_test.py b/keras_nlp/src/models/deberta_v3/deberta_v3_classifier_test.py
similarity index 87%
rename from keras_nlp/models/deberta_v3/deberta_v3_classifier_test.py
rename to keras_nlp/src/models/deberta_v3/deberta_v3_classifier_test.py
index 7d4f61e045..fb17e09ec5 100644
--- a/keras_nlp/models/deberta_v3/deberta_v3_classifier_test.py
+++ b/keras_nlp/src/models/deberta_v3/deberta_v3_classifier_test.py
@@ -16,15 +16,19 @@
 
 import pytest
 
-from keras_nlp.models.deberta_v3.deberta_v3_backbone import DebertaV3Backbone
-from keras_nlp.models.deberta_v3.deberta_v3_classifier import (
+from keras_nlp.src.models.deberta_v3.deberta_v3_backbone import (
+    DebertaV3Backbone,
+)
+from keras_nlp.src.models.deberta_v3.deberta_v3_classifier import (
     DebertaV3Classifier,
 )
-from keras_nlp.models.deberta_v3.deberta_v3_preprocessor import (
+from keras_nlp.src.models.deberta_v3.deberta_v3_preprocessor import (
     DebertaV3Preprocessor,
 )
-from keras_nlp.models.deberta_v3.deberta_v3_tokenizer import DebertaV3Tokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.deberta_v3.deberta_v3_tokenizer import (
+    DebertaV3Tokenizer,
+)
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class DebertaV3ClassifierTest(TestCase):
diff --git a/keras_nlp/models/deberta_v3/deberta_v3_masked_lm_preprocessor_test.py b/keras_nlp/src/models/deberta_v3/deberta_v3_masked_lm_preprocessor_test.py
similarity index 92%
rename from keras_nlp/models/deberta_v3/deberta_v3_masked_lm_preprocessor_test.py
rename to keras_nlp/src/models/deberta_v3/deberta_v3_masked_lm_preprocessor_test.py
index f041a6f7ff..4df5a60f2b 100644
--- a/keras_nlp/models/deberta_v3/deberta_v3_masked_lm_preprocessor_test.py
+++ b/keras_nlp/src/models/deberta_v3/deberta_v3_masked_lm_preprocessor_test.py
@@ -16,11 +16,13 @@
 
 import pytest
 
-from keras_nlp.models.deberta_v3.deberta_v3_masked_lm_preprocessor import (
+from keras_nlp.src.models.deberta_v3.deberta_v3_masked_lm_preprocessor import (
     DebertaV3MaskedLMPreprocessor,
 )
-from keras_nlp.models.deberta_v3.deberta_v3_tokenizer import DebertaV3Tokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.deberta_v3.deberta_v3_tokenizer import (
+    DebertaV3Tokenizer,
+)
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class DebertaV3MaskedLMPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/deberta_v3/deberta_v3_masked_lm_test.py b/keras_nlp/src/models/deberta_v3/deberta_v3_masked_lm_test.py
similarity index 86%
rename from keras_nlp/models/deberta_v3/deberta_v3_masked_lm_test.py
rename to keras_nlp/src/models/deberta_v3/deberta_v3_masked_lm_test.py
index b103f390f6..8a1671b586 100644
--- a/keras_nlp/models/deberta_v3/deberta_v3_masked_lm_test.py
+++ b/keras_nlp/src/models/deberta_v3/deberta_v3_masked_lm_test.py
@@ -16,13 +16,19 @@
 
 import pytest
 
-from keras_nlp.models.deberta_v3.deberta_v3_backbone import DebertaV3Backbone
-from keras_nlp.models.deberta_v3.deberta_v3_masked_lm import DebertaV3MaskedLM
-from keras_nlp.models.deberta_v3.deberta_v3_masked_lm_preprocessor import (
+from keras_nlp.src.models.deberta_v3.deberta_v3_backbone import (
+    DebertaV3Backbone,
+)
+from keras_nlp.src.models.deberta_v3.deberta_v3_masked_lm import (
+    DebertaV3MaskedLM,
+)
+from keras_nlp.src.models.deberta_v3.deberta_v3_masked_lm_preprocessor import (
     DebertaV3MaskedLMPreprocessor,
 )
-from keras_nlp.models.deberta_v3.deberta_v3_tokenizer import DebertaV3Tokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.deberta_v3.deberta_v3_tokenizer import (
+    DebertaV3Tokenizer,
+)
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class DebertaV3MaskedLMTest(TestCase):
diff --git a/keras_nlp/models/deberta_v3/deberta_v3_preprocessor_test.py b/keras_nlp/src/models/deberta_v3/deberta_v3_preprocessor_test.py
similarity index 91%
rename from keras_nlp/models/deberta_v3/deberta_v3_preprocessor_test.py
rename to keras_nlp/src/models/deberta_v3/deberta_v3_preprocessor_test.py
index a9e2a59c29..0c83f6dd65 100644
--- a/keras_nlp/models/deberta_v3/deberta_v3_preprocessor_test.py
+++ b/keras_nlp/src/models/deberta_v3/deberta_v3_preprocessor_test.py
@@ -16,11 +16,13 @@
 
 import pytest
 
-from keras_nlp.models.deberta_v3.deberta_v3_preprocessor import (
+from keras_nlp.src.models.deberta_v3.deberta_v3_preprocessor import (
     DebertaV3Preprocessor,
 )
-from keras_nlp.models.deberta_v3.deberta_v3_tokenizer import DebertaV3Tokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.deberta_v3.deberta_v3_tokenizer import (
+    DebertaV3Tokenizer,
+)
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class DebertaV3PreprocessorTest(TestCase):
diff --git a/keras_nlp/models/deberta_v3/deberta_v3_tokenizer_test.py b/keras_nlp/src/models/deberta_v3/deberta_v3_tokenizer_test.py
similarity index 94%
rename from keras_nlp/models/deberta_v3/deberta_v3_tokenizer_test.py
rename to keras_nlp/src/models/deberta_v3/deberta_v3_tokenizer_test.py
index 3c17cfa397..003a99fc02 100644
--- a/keras_nlp/models/deberta_v3/deberta_v3_tokenizer_test.py
+++ b/keras_nlp/src/models/deberta_v3/deberta_v3_tokenizer_test.py
@@ -16,8 +16,10 @@
 
 import pytest
 
-from keras_nlp.models.deberta_v3.deberta_v3_tokenizer import DebertaV3Tokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.deberta_v3.deberta_v3_tokenizer import (
+    DebertaV3Tokenizer,
+)
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class DebertaV3TokenizerTest(TestCase):
diff --git a/keras_nlp/models/distil_bert/distil_bert_backbone_test.py b/keras_nlp/src/models/distil_bert/distil_bert_backbone_test.py
similarity index 93%
rename from keras_nlp/models/distil_bert/distil_bert_backbone_test.py
rename to keras_nlp/src/models/distil_bert/distil_bert_backbone_test.py
index 8790f87e93..4e8ca9b06b 100644
--- a/keras_nlp/models/distil_bert/distil_bert_backbone_test.py
+++ b/keras_nlp/src/models/distil_bert/distil_bert_backbone_test.py
@@ -14,9 +14,11 @@
 
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.distil_bert.distil_bert_backbone import DistilBertBackbone
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.distil_bert.distil_bert_backbone import (
+    DistilBertBackbone,
+)
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class DistilBertBackboneTest(TestCase):
diff --git a/keras_nlp/models/distil_bert/distil_bert_classifier_test.py b/keras_nlp/src/models/distil_bert/distil_bert_classifier_test.py
similarity index 87%
rename from keras_nlp/models/distil_bert/distil_bert_classifier_test.py
rename to keras_nlp/src/models/distil_bert/distil_bert_classifier_test.py
index d25f176894..dcb99b0b8a 100644
--- a/keras_nlp/models/distil_bert/distil_bert_classifier_test.py
+++ b/keras_nlp/src/models/distil_bert/distil_bert_classifier_test.py
@@ -14,17 +14,19 @@
 
 import pytest
 
-from keras_nlp.models.distil_bert.distil_bert_backbone import DistilBertBackbone
-from keras_nlp.models.distil_bert.distil_bert_classifier import (
+from keras_nlp.src.models.distil_bert.distil_bert_backbone import (
+    DistilBertBackbone,
+)
+from keras_nlp.src.models.distil_bert.distil_bert_classifier import (
     DistilBertClassifier,
 )
-from keras_nlp.models.distil_bert.distil_bert_preprocessor import (
+from keras_nlp.src.models.distil_bert.distil_bert_preprocessor import (
     DistilBertPreprocessor,
 )
-from keras_nlp.models.distil_bert.distil_bert_tokenizer import (
+from keras_nlp.src.models.distil_bert.distil_bert_tokenizer import (
     DistilBertTokenizer,
 )
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class DistilBertClassifierTest(TestCase):
diff --git a/keras_nlp/models/distil_bert/distil_bert_masked_lm_preprocessor_test.py b/keras_nlp/src/models/distil_bert/distil_bert_masked_lm_preprocessor_test.py
similarity index 93%
rename from keras_nlp/models/distil_bert/distil_bert_masked_lm_preprocessor_test.py
rename to keras_nlp/src/models/distil_bert/distil_bert_masked_lm_preprocessor_test.py
index 85ee5bdd43..a49390572e 100644
--- a/keras_nlp/models/distil_bert/distil_bert_masked_lm_preprocessor_test.py
+++ b/keras_nlp/src/models/distil_bert/distil_bert_masked_lm_preprocessor_test.py
@@ -14,13 +14,13 @@
 
 import pytest
 
-from keras_nlp.models.distil_bert.distil_bert_masked_lm_preprocessor import (
+from keras_nlp.src.models.distil_bert.distil_bert_masked_lm_preprocessor import (
     DistilBertMaskedLMPreprocessor,
 )
-from keras_nlp.models.distil_bert.distil_bert_tokenizer import (
+from keras_nlp.src.models.distil_bert.distil_bert_tokenizer import (
     DistilBertTokenizer,
 )
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class DistilBertMaskedLMPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/distil_bert/distil_bert_masked_lm_test.py b/keras_nlp/src/models/distil_bert/distil_bert_masked_lm_test.py
similarity index 87%
rename from keras_nlp/models/distil_bert/distil_bert_masked_lm_test.py
rename to keras_nlp/src/models/distil_bert/distil_bert_masked_lm_test.py
index 4aa8327ae7..482f703fca 100644
--- a/keras_nlp/models/distil_bert/distil_bert_masked_lm_test.py
+++ b/keras_nlp/src/models/distil_bert/distil_bert_masked_lm_test.py
@@ -14,17 +14,19 @@
 
 import pytest
 
-from keras_nlp.models.distil_bert.distil_bert_backbone import DistilBertBackbone
-from keras_nlp.models.distil_bert.distil_bert_masked_lm import (
+from keras_nlp.src.models.distil_bert.distil_bert_backbone import (
+    DistilBertBackbone,
+)
+from keras_nlp.src.models.distil_bert.distil_bert_masked_lm import (
     DistilBertMaskedLM,
 )
-from keras_nlp.models.distil_bert.distil_bert_masked_lm_preprocessor import (
+from keras_nlp.src.models.distil_bert.distil_bert_masked_lm_preprocessor import (
     DistilBertMaskedLMPreprocessor,
 )
-from keras_nlp.models.distil_bert.distil_bert_tokenizer import (
+from keras_nlp.src.models.distil_bert.distil_bert_tokenizer import (
     DistilBertTokenizer,
 )
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class DistilBertMaskedLMTest(TestCase):
diff --git a/keras_nlp/models/distil_bert/distil_bert_preprocessor_test.py b/keras_nlp/src/models/distil_bert/distil_bert_preprocessor_test.py
similarity index 92%
rename from keras_nlp/models/distil_bert/distil_bert_preprocessor_test.py
rename to keras_nlp/src/models/distil_bert/distil_bert_preprocessor_test.py
index f58b42cd39..1cc5e65639 100644
--- a/keras_nlp/models/distil_bert/distil_bert_preprocessor_test.py
+++ b/keras_nlp/src/models/distil_bert/distil_bert_preprocessor_test.py
@@ -14,13 +14,13 @@
 
 import pytest
 
-from keras_nlp.models.distil_bert.distil_bert_preprocessor import (
+from keras_nlp.src.models.distil_bert.distil_bert_preprocessor import (
     DistilBertPreprocessor,
 )
-from keras_nlp.models.distil_bert.distil_bert_tokenizer import (
+from keras_nlp.src.models.distil_bert.distil_bert_tokenizer import (
     DistilBertTokenizer,
 )
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class DistilBertPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/distil_bert/distil_bert_tokenizer_test.py b/keras_nlp/src/models/distil_bert/distil_bert_tokenizer_test.py
similarity index 95%
rename from keras_nlp/models/distil_bert/distil_bert_tokenizer_test.py
rename to keras_nlp/src/models/distil_bert/distil_bert_tokenizer_test.py
index 42bfde39e5..ac05fc02a6 100644
--- a/keras_nlp/models/distil_bert/distil_bert_tokenizer_test.py
+++ b/keras_nlp/src/models/distil_bert/distil_bert_tokenizer_test.py
@@ -14,10 +14,10 @@
 
 import pytest
 
-from keras_nlp.models.distil_bert.distil_bert_tokenizer import (
+from keras_nlp.src.models.distil_bert.distil_bert_tokenizer import (
     DistilBertTokenizer,
 )
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class DistilBertTokenizerTest(TestCase):
diff --git a/keras_nlp/models/electra/electra_backbone_test.py b/keras_nlp/src/models/electra/electra_backbone_test.py
similarity index 94%
rename from keras_nlp/models/electra/electra_backbone_test.py
rename to keras_nlp/src/models/electra/electra_backbone_test.py
index c51bb579e1..929e900643 100644
--- a/keras_nlp/models/electra/electra_backbone_test.py
+++ b/keras_nlp/src/models/electra/electra_backbone_test.py
@@ -14,9 +14,9 @@
 
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.electra.electra_backbone import ElectraBackbone
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.electra.electra_backbone import ElectraBackbone
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class ElectraBackboneTest(TestCase):
diff --git a/keras_nlp/models/electra/electra_preprocessor_test.py b/keras_nlp/src/models/electra/electra_preprocessor_test.py
similarity index 91%
rename from keras_nlp/models/electra/electra_preprocessor_test.py
rename to keras_nlp/src/models/electra/electra_preprocessor_test.py
index 5dd48fe40a..f880a9a302 100644
--- a/keras_nlp/models/electra/electra_preprocessor_test.py
+++ b/keras_nlp/src/models/electra/electra_preprocessor_test.py
@@ -14,9 +14,11 @@
 
 import pytest
 
-from keras_nlp.models.electra.electra_preprocessor import ElectraPreprocessor
-from keras_nlp.models.electra.electra_tokenizer import ElectraTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.electra.electra_preprocessor import (
+    ElectraPreprocessor,
+)
+from keras_nlp.src.models.electra.electra_tokenizer import ElectraTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class ElectraPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/electra/electra_tokenizer_test.py b/keras_nlp/src/models/electra/electra_tokenizer_test.py
similarity index 95%
rename from keras_nlp/models/electra/electra_tokenizer_test.py
rename to keras_nlp/src/models/electra/electra_tokenizer_test.py
index 9126ddc8a7..a1aabf96b1 100644
--- a/keras_nlp/models/electra/electra_tokenizer_test.py
+++ b/keras_nlp/src/models/electra/electra_tokenizer_test.py
@@ -14,8 +14,8 @@
 
 import pytest
 
-from keras_nlp.models.electra.electra_tokenizer import ElectraTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.electra.electra_tokenizer import ElectraTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class ElectraTokenizerTest(TestCase):
diff --git a/keras_nlp/models/f_net/f_net_backbone_test.py b/keras_nlp/src/models/f_net/f_net_backbone_test.py
similarity index 94%
rename from keras_nlp/models/f_net/f_net_backbone_test.py
rename to keras_nlp/src/models/f_net/f_net_backbone_test.py
index 25dfaf799a..4b93956b1e 100644
--- a/keras_nlp/models/f_net/f_net_backbone_test.py
+++ b/keras_nlp/src/models/f_net/f_net_backbone_test.py
@@ -14,9 +14,9 @@
 
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.f_net.f_net_backbone import FNetBackbone
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.f_net.f_net_backbone import FNetBackbone
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class FNetBackboneTest(TestCase):
diff --git a/keras_nlp/models/f_net/f_net_classifier_test.py b/keras_nlp/src/models/f_net/f_net_classifier_test.py
similarity index 88%
rename from keras_nlp/models/f_net/f_net_classifier_test.py
rename to keras_nlp/src/models/f_net/f_net_classifier_test.py
index 4defce4a71..25770ed90a 100644
--- a/keras_nlp/models/f_net/f_net_classifier_test.py
+++ b/keras_nlp/src/models/f_net/f_net_classifier_test.py
@@ -16,11 +16,11 @@
 
 import pytest
 
-from keras_nlp.models.f_net.f_net_backbone import FNetBackbone
-from keras_nlp.models.f_net.f_net_classifier import FNetClassifier
-from keras_nlp.models.f_net.f_net_preprocessor import FNetPreprocessor
-from keras_nlp.models.f_net.f_net_tokenizer import FNetTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.f_net.f_net_backbone import FNetBackbone
+from keras_nlp.src.models.f_net.f_net_classifier import FNetClassifier
+from keras_nlp.src.models.f_net.f_net_preprocessor import FNetPreprocessor
+from keras_nlp.src.models.f_net.f_net_tokenizer import FNetTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class FNetClassifierTest(TestCase):
diff --git a/keras_nlp/models/f_net/f_net_masked_lm_preprocessor_test.py b/keras_nlp/src/models/f_net/f_net_masked_lm_preprocessor_test.py
similarity index 93%
rename from keras_nlp/models/f_net/f_net_masked_lm_preprocessor_test.py
rename to keras_nlp/src/models/f_net/f_net_masked_lm_preprocessor_test.py
index 7d2ecc0f17..e1ee5443c3 100644
--- a/keras_nlp/models/f_net/f_net_masked_lm_preprocessor_test.py
+++ b/keras_nlp/src/models/f_net/f_net_masked_lm_preprocessor_test.py
@@ -16,11 +16,11 @@
 
 import pytest
 
-from keras_nlp.models.f_net.f_net_masked_lm_preprocessor import (
+from keras_nlp.src.models.f_net.f_net_masked_lm_preprocessor import (
     FNetMaskedLMPreprocessor,
 )
-from keras_nlp.models.f_net.f_net_tokenizer import FNetTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.f_net.f_net_tokenizer import FNetTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class FNetMaskedLMPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/f_net/f_net_masked_lm_test.py b/keras_nlp/src/models/f_net/f_net_masked_lm_test.py
similarity index 88%
rename from keras_nlp/models/f_net/f_net_masked_lm_test.py
rename to keras_nlp/src/models/f_net/f_net_masked_lm_test.py
index 1db6b361ed..9ab171a51d 100644
--- a/keras_nlp/models/f_net/f_net_masked_lm_test.py
+++ b/keras_nlp/src/models/f_net/f_net_masked_lm_test.py
@@ -16,13 +16,13 @@
 
 import pytest
 
-from keras_nlp.models.f_net.f_net_backbone import FNetBackbone
-from keras_nlp.models.f_net.f_net_masked_lm import FNetMaskedLM
-from keras_nlp.models.f_net.f_net_masked_lm_preprocessor import (
+from keras_nlp.src.models.f_net.f_net_backbone import FNetBackbone
+from keras_nlp.src.models.f_net.f_net_masked_lm import FNetMaskedLM
+from keras_nlp.src.models.f_net.f_net_masked_lm_preprocessor import (
     FNetMaskedLMPreprocessor,
 )
-from keras_nlp.models.f_net.f_net_tokenizer import FNetTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.f_net.f_net_tokenizer import FNetTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class FNetMaskedLMTest(TestCase):
diff --git a/keras_nlp/models/f_net/f_net_preprocessor_test.py b/keras_nlp/src/models/f_net/f_net_preprocessor_test.py
similarity index 91%
rename from keras_nlp/models/f_net/f_net_preprocessor_test.py
rename to keras_nlp/src/models/f_net/f_net_preprocessor_test.py
index c9096ac59f..53dc7c1235 100644
--- a/keras_nlp/models/f_net/f_net_preprocessor_test.py
+++ b/keras_nlp/src/models/f_net/f_net_preprocessor_test.py
@@ -16,9 +16,9 @@
 
 import pytest
 
-from keras_nlp.models.f_net.f_net_preprocessor import FNetPreprocessor
-from keras_nlp.models.f_net.f_net_tokenizer import FNetTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.f_net.f_net_preprocessor import FNetPreprocessor
+from keras_nlp.src.models.f_net.f_net_tokenizer import FNetTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class FNetPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/f_net/f_net_tokenizer_test.py b/keras_nlp/src/models/f_net/f_net_tokenizer_test.py
similarity index 94%
rename from keras_nlp/models/f_net/f_net_tokenizer_test.py
rename to keras_nlp/src/models/f_net/f_net_tokenizer_test.py
index 3dde34e849..5fb2b1bc5a 100644
--- a/keras_nlp/models/f_net/f_net_tokenizer_test.py
+++ b/keras_nlp/src/models/f_net/f_net_tokenizer_test.py
@@ -16,8 +16,8 @@
 
 import pytest
 
-from keras_nlp.models.f_net.f_net_tokenizer import FNetTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.f_net.f_net_tokenizer import FNetTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class FNetTokenizerTest(TestCase):
diff --git a/keras_nlp/models/falcon/falcon_backbone_test.py b/keras_nlp/src/models/falcon/falcon_backbone_test.py
similarity index 90%
rename from keras_nlp/models/falcon/falcon_backbone_test.py
rename to keras_nlp/src/models/falcon/falcon_backbone_test.py
index 140ce7e7bf..3cce714d1f 100644
--- a/keras_nlp/models/falcon/falcon_backbone_test.py
+++ b/keras_nlp/src/models/falcon/falcon_backbone_test.py
@@ -13,9 +13,9 @@
 # limitations under the License.
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.falcon.falcon_backbone import FalconBackbone
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.falcon.falcon_backbone import FalconBackbone
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class FalconBackboneTest(TestCase):
diff --git a/keras_nlp/models/falcon/falcon_causal_lm_preprocessor_test.py b/keras_nlp/src/models/falcon/falcon_causal_lm_preprocessor_test.py
similarity index 94%
rename from keras_nlp/models/falcon/falcon_causal_lm_preprocessor_test.py
rename to keras_nlp/src/models/falcon/falcon_causal_lm_preprocessor_test.py
index 5e812259e2..f768544a7f 100644
--- a/keras_nlp/models/falcon/falcon_causal_lm_preprocessor_test.py
+++ b/keras_nlp/src/models/falcon/falcon_causal_lm_preprocessor_test.py
@@ -14,11 +14,11 @@
 
 import pytest
 
-from keras_nlp.models.falcon.falcon_causal_lm_preprocessor import (
+from keras_nlp.src.models.falcon.falcon_causal_lm_preprocessor import (
     FalconCausalLMPreprocessor,
 )
-from keras_nlp.models.falcon.falcon_tokenizer import FalconTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.falcon.falcon_tokenizer import FalconTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class FalconCausalLMPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/falcon/falcon_preprocessor_test.py b/keras_nlp/src/models/falcon/falcon_preprocessor_test.py
similarity index 93%
rename from keras_nlp/models/falcon/falcon_preprocessor_test.py
rename to keras_nlp/src/models/falcon/falcon_preprocessor_test.py
index 7676062287..075710b23b 100644
--- a/keras_nlp/models/falcon/falcon_preprocessor_test.py
+++ b/keras_nlp/src/models/falcon/falcon_preprocessor_test.py
@@ -14,9 +14,9 @@
 
 import pytest
 
-from keras_nlp.models.falcon.falcon_preprocessor import FalconPreprocessor
-from keras_nlp.models.falcon.falcon_tokenizer import FalconTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.falcon.falcon_preprocessor import FalconPreprocessor
+from keras_nlp.src.models.falcon.falcon_tokenizer import FalconTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class FalconPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/falcon/falcon_tokenizer_test.py b/keras_nlp/src/models/falcon/falcon_tokenizer_test.py
similarity index 94%
rename from keras_nlp/models/falcon/falcon_tokenizer_test.py
rename to keras_nlp/src/models/falcon/falcon_tokenizer_test.py
index 735bcac4b6..51d30b975d 100644
--- a/keras_nlp/models/falcon/falcon_tokenizer_test.py
+++ b/keras_nlp/src/models/falcon/falcon_tokenizer_test.py
@@ -13,8 +13,8 @@
 # limitations under the License.
 import pytest
 
-from keras_nlp.models.falcon.falcon_tokenizer import FalconTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.falcon.falcon_tokenizer import FalconTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class FalconTokenizerTest(TestCase):
diff --git a/keras_nlp/models/gemma/gemma_backbone_test.py b/keras_nlp/src/models/gemma/gemma_backbone_test.py
similarity index 96%
rename from keras_nlp/models/gemma/gemma_backbone_test.py
rename to keras_nlp/src/models/gemma/gemma_backbone_test.py
index 7b02de2b7a..ef7d16a508 100644
--- a/keras_nlp/models/gemma/gemma_backbone_test.py
+++ b/keras_nlp/src/models/gemma/gemma_backbone_test.py
@@ -13,10 +13,10 @@
 # limitations under the License.
 import pytest
 
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.models.gemma.gemma_backbone import GemmaBackbone
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.gemma.gemma_backbone import GemmaBackbone
+from keras_nlp.src.tests.test_case import TestCase
 
 
 @pytest.mark.keras_3_only
diff --git a/keras_nlp/models/gemma/gemma_causal_lm_preprocessor_test.py b/keras_nlp/src/models/gemma/gemma_causal_lm_preprocessor_test.py
similarity index 94%
rename from keras_nlp/models/gemma/gemma_causal_lm_preprocessor_test.py
rename to keras_nlp/src/models/gemma/gemma_causal_lm_preprocessor_test.py
index c3305afe91..0bc1db32d2 100644
--- a/keras_nlp/models/gemma/gemma_causal_lm_preprocessor_test.py
+++ b/keras_nlp/src/models/gemma/gemma_causal_lm_preprocessor_test.py
@@ -16,11 +16,11 @@
 
 import pytest
 
-from keras_nlp.models.gemma.gemma_causal_lm_preprocessor import (
+from keras_nlp.src.models.gemma.gemma_causal_lm_preprocessor import (
     GemmaCausalLMPreprocessor,
 )
-from keras_nlp.models.gemma.gemma_tokenizer import GemmaTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.gemma.gemma_tokenizer import GemmaTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 @pytest.mark.keras_3_only
diff --git a/keras_nlp/models/gemma/gemma_causal_lm_test.py b/keras_nlp/src/models/gemma/gemma_causal_lm_test.py
similarity index 96%
rename from keras_nlp/models/gemma/gemma_causal_lm_test.py
rename to keras_nlp/src/models/gemma/gemma_causal_lm_test.py
index 8f681df52f..c0279574e6 100644
--- a/keras_nlp/models/gemma/gemma_causal_lm_test.py
+++ b/keras_nlp/src/models/gemma/gemma_causal_lm_test.py
@@ -17,15 +17,15 @@
 
 import pytest
 
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.models.gemma.gemma_backbone import GemmaBackbone
-from keras_nlp.models.gemma.gemma_causal_lm import GemmaCausalLM
-from keras_nlp.models.gemma.gemma_causal_lm_preprocessor import (
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.gemma.gemma_backbone import GemmaBackbone
+from keras_nlp.src.models.gemma.gemma_causal_lm import GemmaCausalLM
+from keras_nlp.src.models.gemma.gemma_causal_lm_preprocessor import (
     GemmaCausalLMPreprocessor,
 )
-from keras_nlp.models.gemma.gemma_tokenizer import GemmaTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.gemma.gemma_tokenizer import GemmaTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 @pytest.mark.keras_3_only
diff --git a/keras_nlp/models/gemma/gemma_lora_test.py b/keras_nlp/src/models/gemma/gemma_lora_test.py
similarity index 96%
rename from keras_nlp/models/gemma/gemma_lora_test.py
rename to keras_nlp/src/models/gemma/gemma_lora_test.py
index 1cbbdfa67f..edd188c6a5 100644
--- a/keras_nlp/models/gemma/gemma_lora_test.py
+++ b/keras_nlp/src/models/gemma/gemma_lora_test.py
@@ -16,8 +16,8 @@
 import numpy as np
 import pytest
 
-from keras_nlp.models.gemma.gemma_backbone import GemmaBackbone
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.gemma.gemma_backbone import GemmaBackbone
+from keras_nlp.src.tests.test_case import TestCase
 
 
 @pytest.mark.keras_3_only
diff --git a/keras_nlp/models/gemma/gemma_preprocessor_test.py b/keras_nlp/src/models/gemma/gemma_preprocessor_test.py
similarity index 92%
rename from keras_nlp/models/gemma/gemma_preprocessor_test.py
rename to keras_nlp/src/models/gemma/gemma_preprocessor_test.py
index 0cb427af03..8f2f4b81f0 100644
--- a/keras_nlp/models/gemma/gemma_preprocessor_test.py
+++ b/keras_nlp/src/models/gemma/gemma_preprocessor_test.py
@@ -16,9 +16,9 @@
 
 import pytest
 
-from keras_nlp.models.gemma.gemma_preprocessor import GemmaPreprocessor
-from keras_nlp.models.gemma.gemma_tokenizer import GemmaTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.gemma.gemma_preprocessor import GemmaPreprocessor
+from keras_nlp.src.models.gemma.gemma_tokenizer import GemmaTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 @pytest.mark.keras_3_only
diff --git a/keras_nlp/models/gemma/gemma_tokenizer_test.py b/keras_nlp/src/models/gemma/gemma_tokenizer_test.py
similarity index 94%
rename from keras_nlp/models/gemma/gemma_tokenizer_test.py
rename to keras_nlp/src/models/gemma/gemma_tokenizer_test.py
index 65569c8174..60bb3e9a97 100644
--- a/keras_nlp/models/gemma/gemma_tokenizer_test.py
+++ b/keras_nlp/src/models/gemma/gemma_tokenizer_test.py
@@ -16,8 +16,8 @@
 
 import pytest
 
-from keras_nlp.models.gemma.gemma_tokenizer import GemmaTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.gemma.gemma_tokenizer import GemmaTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 @pytest.mark.keras_3_only
diff --git a/keras_nlp/models/gpt2/gpt2_backbone_test.py b/keras_nlp/src/models/gpt2/gpt2_backbone_test.py
similarity index 93%
rename from keras_nlp/models/gpt2/gpt2_backbone_test.py
rename to keras_nlp/src/models/gpt2/gpt2_backbone_test.py
index d29bc68565..93606d96af 100644
--- a/keras_nlp/models/gpt2/gpt2_backbone_test.py
+++ b/keras_nlp/src/models/gpt2/gpt2_backbone_test.py
@@ -14,9 +14,9 @@
 
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.gpt2.gpt2_backbone import GPT2Backbone
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.gpt2.gpt2_backbone import GPT2Backbone
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class GPT2BackboneTest(TestCase):
diff --git a/keras_nlp/models/gpt2/gpt2_causal_lm_preprocessor_test.py b/keras_nlp/src/models/gpt2/gpt2_causal_lm_preprocessor_test.py
similarity index 94%
rename from keras_nlp/models/gpt2/gpt2_causal_lm_preprocessor_test.py
rename to keras_nlp/src/models/gpt2/gpt2_causal_lm_preprocessor_test.py
index 0623d983a9..26caaf9fe1 100644
--- a/keras_nlp/models/gpt2/gpt2_causal_lm_preprocessor_test.py
+++ b/keras_nlp/src/models/gpt2/gpt2_causal_lm_preprocessor_test.py
@@ -14,11 +14,11 @@
 
 import pytest
 
-from keras_nlp.models.gpt2.gpt2_causal_lm_preprocessor import (
+from keras_nlp.src.models.gpt2.gpt2_causal_lm_preprocessor import (
     GPT2CausalLMPreprocessor,
 )
-from keras_nlp.models.gpt2.gpt2_tokenizer import GPT2Tokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.gpt2.gpt2_tokenizer import GPT2Tokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class GPT2CausalLMPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/gpt2/gpt2_causal_lm_test.py b/keras_nlp/src/models/gpt2/gpt2_causal_lm_test.py
similarity index 95%
rename from keras_nlp/models/gpt2/gpt2_causal_lm_test.py
rename to keras_nlp/src/models/gpt2/gpt2_causal_lm_test.py
index 2d2f0e8a0a..f8cef2a15a 100644
--- a/keras_nlp/models/gpt2/gpt2_causal_lm_test.py
+++ b/keras_nlp/src/models/gpt2/gpt2_causal_lm_test.py
@@ -16,14 +16,14 @@
 
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.gpt2.gpt2_backbone import GPT2Backbone
-from keras_nlp.models.gpt2.gpt2_causal_lm import GPT2CausalLM
-from keras_nlp.models.gpt2.gpt2_causal_lm_preprocessor import (
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.gpt2.gpt2_backbone import GPT2Backbone
+from keras_nlp.src.models.gpt2.gpt2_causal_lm import GPT2CausalLM
+from keras_nlp.src.models.gpt2.gpt2_causal_lm_preprocessor import (
     GPT2CausalLMPreprocessor,
 )
-from keras_nlp.models.gpt2.gpt2_tokenizer import GPT2Tokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.gpt2.gpt2_tokenizer import GPT2Tokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class GPT2CausalLMTest(TestCase):
diff --git a/keras_nlp/models/gpt2/gpt2_preprocessor_test.py b/keras_nlp/src/models/gpt2/gpt2_preprocessor_test.py
similarity index 93%
rename from keras_nlp/models/gpt2/gpt2_preprocessor_test.py
rename to keras_nlp/src/models/gpt2/gpt2_preprocessor_test.py
index 35129c200d..4f038fb66c 100644
--- a/keras_nlp/models/gpt2/gpt2_preprocessor_test.py
+++ b/keras_nlp/src/models/gpt2/gpt2_preprocessor_test.py
@@ -14,9 +14,9 @@
 
 import pytest
 
-from keras_nlp.models.gpt2.gpt2_preprocessor import GPT2Preprocessor
-from keras_nlp.models.gpt2.gpt2_tokenizer import GPT2Tokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.gpt2.gpt2_preprocessor import GPT2Preprocessor
+from keras_nlp.src.models.gpt2.gpt2_tokenizer import GPT2Tokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class GPT2PreprocessorTest(TestCase):
diff --git a/keras_nlp/models/gpt2/gpt2_tokenizer_test.py b/keras_nlp/src/models/gpt2/gpt2_tokenizer_test.py
similarity index 94%
rename from keras_nlp/models/gpt2/gpt2_tokenizer_test.py
rename to keras_nlp/src/models/gpt2/gpt2_tokenizer_test.py
index 026392fd25..fa01285c9f 100644
--- a/keras_nlp/models/gpt2/gpt2_tokenizer_test.py
+++ b/keras_nlp/src/models/gpt2/gpt2_tokenizer_test.py
@@ -14,8 +14,8 @@
 
 import pytest
 
-from keras_nlp.models.gpt2.gpt2_tokenizer import GPT2Tokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.gpt2.gpt2_tokenizer import GPT2Tokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class GPT2TokenizerTest(TestCase):
diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_backbone_test.py b/keras_nlp/src/models/gpt_neo_x/gpt_neo_x_backbone_test.py
similarity index 90%
rename from keras_nlp/models/gpt_neo_x/gpt_neo_x_backbone_test.py
rename to keras_nlp/src/models/gpt_neo_x/gpt_neo_x_backbone_test.py
index f207f4f19e..bd5dedd334 100644
--- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_backbone_test.py
+++ b/keras_nlp/src/models/gpt_neo_x/gpt_neo_x_backbone_test.py
@@ -14,9 +14,9 @@
 
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.gpt_neo_x.gpt_neo_x_backbone import GPTNeoXBackbone
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_backbone import GPTNeoXBackbone
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class GPTNeoXBackboneTest(TestCase):
diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor_test.py b/keras_nlp/src/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor_test.py
similarity index 94%
rename from keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor_test.py
rename to keras_nlp/src/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor_test.py
index 0937bb6dd1..76ff473605 100644
--- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor_test.py
+++ b/keras_nlp/src/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor_test.py
@@ -20,11 +20,11 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.models.gpt_neo_x.gpt_neo_x_causal_lm_preprocessor import (
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_causal_lm_preprocessor import (
     GPTNeoXCausalLMPreprocessor,
 )
-from keras_nlp.models.gpt_neo_x.gpt_neo_x_tokenizer import GPTNeoXTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_tokenizer import GPTNeoXTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class GPTNeoXCausalLMPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_test.py b/keras_nlp/src/models/gpt_neo_x/gpt_neo_x_causal_lm_test.py
similarity index 91%
rename from keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_test.py
rename to keras_nlp/src/models/gpt_neo_x/gpt_neo_x_causal_lm_test.py
index 3bf0197e4a..4482c60e91 100644
--- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_test.py
+++ b/keras_nlp/src/models/gpt_neo_x/gpt_neo_x_causal_lm_test.py
@@ -16,14 +16,14 @@
 
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.gpt_neo_x.gpt_neo_x_backbone import GPTNeoXBackbone
-from keras_nlp.models.gpt_neo_x.gpt_neo_x_causal_lm import GPTNeoXCausalLM
-from keras_nlp.models.gpt_neo_x.gpt_neo_x_causal_lm_preprocessor import (
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_backbone import GPTNeoXBackbone
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_causal_lm import GPTNeoXCausalLM
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_causal_lm_preprocessor import (
     GPTNeoXCausalLMPreprocessor,
 )
-from keras_nlp.models.gpt_neo_x.gpt_neo_x_tokenizer import GPTNeoXTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_tokenizer import GPTNeoXTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class GPTNeoXCausalLMTest(TestCase):
diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_preprocessor_test.py b/keras_nlp/src/models/gpt_neo_x/gpt_neo_x_preprocessor_test.py
similarity index 92%
rename from keras_nlp/models/gpt_neo_x/gpt_neo_x_preprocessor_test.py
rename to keras_nlp/src/models/gpt_neo_x/gpt_neo_x_preprocessor_test.py
index 92ea191596..1d21c1de7c 100644
--- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_preprocessor_test.py
+++ b/keras_nlp/src/models/gpt_neo_x/gpt_neo_x_preprocessor_test.py
@@ -12,11 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.models.gpt_neo_x.gpt_neo_x_preprocessor import (
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_preprocessor import (
     GPTNeoXPreprocessor,
 )
-from keras_nlp.models.gpt_neo_x.gpt_neo_x_tokenizer import GPTNeoXTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_tokenizer import GPTNeoXTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class GPTNeoXPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_tokenizer_test.py b/keras_nlp/src/models/gpt_neo_x/gpt_neo_x_tokenizer_test.py
similarity index 92%
rename from keras_nlp/models/gpt_neo_x/gpt_neo_x_tokenizer_test.py
rename to keras_nlp/src/models/gpt_neo_x/gpt_neo_x_tokenizer_test.py
index c23b7dd44d..b62f21cff9 100644
--- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_tokenizer_test.py
+++ b/keras_nlp/src/models/gpt_neo_x/gpt_neo_x_tokenizer_test.py
@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.models.gpt_neo_x.gpt_neo_x_tokenizer import GPTNeoXTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.gpt_neo_x.gpt_neo_x_tokenizer import GPTNeoXTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class GPTNeoXTokenizerTest(TestCase):
diff --git a/keras_nlp/models/llama/llama_backbone_test.py b/keras_nlp/src/models/llama/llama_backbone_test.py
similarity index 94%
rename from keras_nlp/models/llama/llama_backbone_test.py
rename to keras_nlp/src/models/llama/llama_backbone_test.py
index b641a0152e..6e3ea8b1a6 100644
--- a/keras_nlp/models/llama/llama_backbone_test.py
+++ b/keras_nlp/src/models/llama/llama_backbone_test.py
@@ -14,9 +14,9 @@
 
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.llama.llama_backbone import LlamaBackbone
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.llama.llama_backbone import LlamaBackbone
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class LlamaTest(TestCase):
diff --git a/keras_nlp/models/llama/llama_causal_lm_preprocessor_test.py b/keras_nlp/src/models/llama/llama_causal_lm_preprocessor_test.py
similarity index 94%
rename from keras_nlp/models/llama/llama_causal_lm_preprocessor_test.py
rename to keras_nlp/src/models/llama/llama_causal_lm_preprocessor_test.py
index aa4d155c8c..36320cec8e 100644
--- a/keras_nlp/models/llama/llama_causal_lm_preprocessor_test.py
+++ b/keras_nlp/src/models/llama/llama_causal_lm_preprocessor_test.py
@@ -16,11 +16,11 @@
 
 import pytest
 
-from keras_nlp.models.llama.llama_causal_lm_preprocessor import (
+from keras_nlp.src.models.llama.llama_causal_lm_preprocessor import (
     LlamaCausalLMPreprocessor,
 )
-from keras_nlp.models.llama.llama_tokenizer import LlamaTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.llama.llama_tokenizer import LlamaTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class LlamaCausalLMPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/llama/llama_causal_lm_test.py b/keras_nlp/src/models/llama/llama_causal_lm_test.py
similarity index 95%
rename from keras_nlp/models/llama/llama_causal_lm_test.py
rename to keras_nlp/src/models/llama/llama_causal_lm_test.py
index 7a449ad3ed..81c6a44447 100644
--- a/keras_nlp/models/llama/llama_causal_lm_test.py
+++ b/keras_nlp/src/models/llama/llama_causal_lm_test.py
@@ -17,14 +17,14 @@
 
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.llama.llama_backbone import LlamaBackbone
-from keras_nlp.models.llama.llama_causal_lm import LlamaCausalLM
-from keras_nlp.models.llama.llama_causal_lm_preprocessor import (
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.llama.llama_backbone import LlamaBackbone
+from keras_nlp.src.models.llama.llama_causal_lm import LlamaCausalLM
+from keras_nlp.src.models.llama.llama_causal_lm_preprocessor import (
     LlamaCausalLMPreprocessor,
 )
-from keras_nlp.models.llama.llama_tokenizer import LlamaTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.llama.llama_tokenizer import LlamaTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class LlamaCausalLMTest(TestCase):
diff --git a/keras_nlp/models/llama/llama_preprocessor_test.py b/keras_nlp/src/models/llama/llama_preprocessor_test.py
similarity index 91%
rename from keras_nlp/models/llama/llama_preprocessor_test.py
rename to keras_nlp/src/models/llama/llama_preprocessor_test.py
index 52a559aa2e..308ff10b1b 100644
--- a/keras_nlp/models/llama/llama_preprocessor_test.py
+++ b/keras_nlp/src/models/llama/llama_preprocessor_test.py
@@ -16,9 +16,9 @@
 
 import pytest
 
-from keras_nlp.models.llama.llama_preprocessor import LlamaPreprocessor
-from keras_nlp.models.llama.llama_tokenizer import LlamaTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.llama.llama_preprocessor import LlamaPreprocessor
+from keras_nlp.src.models.llama.llama_tokenizer import LlamaTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class LlamaPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/llama/llama_tokenizer_test.py b/keras_nlp/src/models/llama/llama_tokenizer_test.py
similarity index 94%
rename from keras_nlp/models/llama/llama_tokenizer_test.py
rename to keras_nlp/src/models/llama/llama_tokenizer_test.py
index 51687731e5..b07c39cef5 100644
--- a/keras_nlp/models/llama/llama_tokenizer_test.py
+++ b/keras_nlp/src/models/llama/llama_tokenizer_test.py
@@ -16,8 +16,8 @@
 
 import pytest
 
-from keras_nlp.models.llama.llama_tokenizer import LlamaTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.llama.llama_tokenizer import LlamaTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class LlamaTokenizerTest(TestCase):
diff --git a/keras_nlp/models/mistral/mistral_backbone_test.py b/keras_nlp/src/models/mistral/mistral_backbone_test.py
similarity index 94%
rename from keras_nlp/models/mistral/mistral_backbone_test.py
rename to keras_nlp/src/models/mistral/mistral_backbone_test.py
index fbfcb91124..06acd09b4a 100644
--- a/keras_nlp/models/mistral/mistral_backbone_test.py
+++ b/keras_nlp/src/models/mistral/mistral_backbone_test.py
@@ -13,9 +13,9 @@
 # limitations under the License.
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.mistral.mistral_backbone import MistralBackbone
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.mistral.mistral_backbone import MistralBackbone
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class MistralBackboneTest(TestCase):
diff --git a/keras_nlp/models/mistral/mistral_causal_lm_preprocessor_test.py b/keras_nlp/src/models/mistral/mistral_causal_lm_preprocessor_test.py
similarity index 94%
rename from keras_nlp/models/mistral/mistral_causal_lm_preprocessor_test.py
rename to keras_nlp/src/models/mistral/mistral_causal_lm_preprocessor_test.py
index dbacce37ce..28a7abf597 100644
--- a/keras_nlp/models/mistral/mistral_causal_lm_preprocessor_test.py
+++ b/keras_nlp/src/models/mistral/mistral_causal_lm_preprocessor_test.py
@@ -16,11 +16,11 @@
 
 import pytest
 
-from keras_nlp.models.mistral.mistral_causal_lm_preprocessor import (
+from keras_nlp.src.models.mistral.mistral_causal_lm_preprocessor import (
     MistralCausalLMPreprocessor,
 )
-from keras_nlp.models.mistral.mistral_tokenizer import MistralTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.mistral.mistral_tokenizer import MistralTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class MistralCausalLMPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/mistral/mistral_causal_lm_test.py b/keras_nlp/src/models/mistral/mistral_causal_lm_test.py
similarity index 95%
rename from keras_nlp/models/mistral/mistral_causal_lm_test.py
rename to keras_nlp/src/models/mistral/mistral_causal_lm_test.py
index a5f1ac9d89..4194fef970 100644
--- a/keras_nlp/models/mistral/mistral_causal_lm_test.py
+++ b/keras_nlp/src/models/mistral/mistral_causal_lm_test.py
@@ -17,14 +17,14 @@
 
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.mistral.mistral_backbone import MistralBackbone
-from keras_nlp.models.mistral.mistral_causal_lm import MistralCausalLM
-from keras_nlp.models.mistral.mistral_causal_lm_preprocessor import (
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.mistral.mistral_backbone import MistralBackbone
+from keras_nlp.src.models.mistral.mistral_causal_lm import MistralCausalLM
+from keras_nlp.src.models.mistral.mistral_causal_lm_preprocessor import (
     MistralCausalLMPreprocessor,
 )
-from keras_nlp.models.mistral.mistral_tokenizer import MistralTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.mistral.mistral_tokenizer import MistralTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class MistralCausalLMTest(TestCase):
diff --git a/keras_nlp/models/mistral/mistral_preprocessor_test.py b/keras_nlp/src/models/mistral/mistral_preprocessor_test.py
similarity index 91%
rename from keras_nlp/models/mistral/mistral_preprocessor_test.py
rename to keras_nlp/src/models/mistral/mistral_preprocessor_test.py
index e3ddd38f6f..5bc6a8606d 100644
--- a/keras_nlp/models/mistral/mistral_preprocessor_test.py
+++ b/keras_nlp/src/models/mistral/mistral_preprocessor_test.py
@@ -16,9 +16,11 @@
 
 import pytest
 
-from keras_nlp.models.mistral.mistral_preprocessor import MistralPreprocessor
-from keras_nlp.models.mistral.mistral_tokenizer import MistralTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.mistral.mistral_preprocessor import (
+    MistralPreprocessor,
+)
+from keras_nlp.src.models.mistral.mistral_tokenizer import MistralTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class MistralPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/mistral/mistral_tokenizer_test.py b/keras_nlp/src/models/mistral/mistral_tokenizer_test.py
similarity index 94%
rename from keras_nlp/models/mistral/mistral_tokenizer_test.py
rename to keras_nlp/src/models/mistral/mistral_tokenizer_test.py
index 6b700bf711..4da849b05a 100644
--- a/keras_nlp/models/mistral/mistral_tokenizer_test.py
+++ b/keras_nlp/src/models/mistral/mistral_tokenizer_test.py
@@ -16,8 +16,8 @@
 
 import pytest
 
-from keras_nlp.models.mistral.mistral_tokenizer import MistralTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.mistral.mistral_tokenizer import MistralTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class MistralTokenizerTest(TestCase):
diff --git a/keras_nlp/models/opt/opt_backbone_test.py b/keras_nlp/src/models/opt/opt_backbone_test.py
similarity index 93%
rename from keras_nlp/models/opt/opt_backbone_test.py
rename to keras_nlp/src/models/opt/opt_backbone_test.py
index 445bdaebad..e1481bb5dd 100644
--- a/keras_nlp/models/opt/opt_backbone_test.py
+++ b/keras_nlp/src/models/opt/opt_backbone_test.py
@@ -14,9 +14,9 @@
 
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.opt.opt_backbone import OPTBackbone
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.opt.opt_backbone import OPTBackbone
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class OPTBackboneTest(TestCase):
diff --git a/keras_nlp/models/opt/opt_causal_lm_preprocessor_test.py b/keras_nlp/src/models/opt/opt_causal_lm_preprocessor_test.py
similarity index 95%
rename from keras_nlp/models/opt/opt_causal_lm_preprocessor_test.py
rename to keras_nlp/src/models/opt/opt_causal_lm_preprocessor_test.py
index e04436f092..e7d62e3e49 100644
--- a/keras_nlp/models/opt/opt_causal_lm_preprocessor_test.py
+++ b/keras_nlp/src/models/opt/opt_causal_lm_preprocessor_test.py
@@ -14,11 +14,11 @@
 
 import pytest
 
-from keras_nlp.models.opt.opt_causal_lm_preprocessor import (
+from keras_nlp.src.models.opt.opt_causal_lm_preprocessor import (
     OPTCausalLMPreprocessor,
 )
-from keras_nlp.models.opt.opt_tokenizer import OPTTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.opt.opt_tokenizer import OPTTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class OPTCausalLMPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/opt/opt_causal_lm_test.py b/keras_nlp/src/models/opt/opt_causal_lm_test.py
similarity index 93%
rename from keras_nlp/models/opt/opt_causal_lm_test.py
rename to keras_nlp/src/models/opt/opt_causal_lm_test.py
index 82c3614a73..80fbbbc97e 100644
--- a/keras_nlp/models/opt/opt_causal_lm_test.py
+++ b/keras_nlp/src/models/opt/opt_causal_lm_test.py
@@ -16,14 +16,14 @@
 
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.opt.opt_backbone import OPTBackbone
-from keras_nlp.models.opt.opt_causal_lm import OPTCausalLM
-from keras_nlp.models.opt.opt_causal_lm_preprocessor import (
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.opt.opt_backbone import OPTBackbone
+from keras_nlp.src.models.opt.opt_causal_lm import OPTCausalLM
+from keras_nlp.src.models.opt.opt_causal_lm_preprocessor import (
     OPTCausalLMPreprocessor,
 )
-from keras_nlp.models.opt.opt_tokenizer import OPTTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.opt.opt_tokenizer import OPTTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class OPTCausalLMTest(TestCase):
diff --git a/keras_nlp/models/opt/opt_preprocessor_test.py b/keras_nlp/src/models/opt/opt_preprocessor_test.py
similarity index 93%
rename from keras_nlp/models/opt/opt_preprocessor_test.py
rename to keras_nlp/src/models/opt/opt_preprocessor_test.py
index 901efc7bee..371ceb1fd8 100644
--- a/keras_nlp/models/opt/opt_preprocessor_test.py
+++ b/keras_nlp/src/models/opt/opt_preprocessor_test.py
@@ -14,9 +14,9 @@
 
 import pytest
 
-from keras_nlp.models.opt.opt_preprocessor import OPTPreprocessor
-from keras_nlp.models.opt.opt_tokenizer import OPTTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.opt.opt_preprocessor import OPTPreprocessor
+from keras_nlp.src.models.opt.opt_tokenizer import OPTTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class OPTPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/opt/opt_tokenizer_test.py b/keras_nlp/src/models/opt/opt_tokenizer_test.py
similarity index 94%
rename from keras_nlp/models/opt/opt_tokenizer_test.py
rename to keras_nlp/src/models/opt/opt_tokenizer_test.py
index 4b52ef1aed..5f9b70fb67 100644
--- a/keras_nlp/models/opt/opt_tokenizer_test.py
+++ b/keras_nlp/src/models/opt/opt_tokenizer_test.py
@@ -14,8 +14,8 @@
 
 import pytest
 
-from keras_nlp.models.opt.opt_tokenizer import OPTTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.opt.opt_tokenizer import OPTTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class OPTTokenizerTest(TestCase):
diff --git a/keras_nlp/models/preprocessor_test.py b/keras_nlp/src/models/preprocessor_test.py
similarity index 82%
rename from keras_nlp/models/preprocessor_test.py
rename to keras_nlp/src/models/preprocessor_test.py
index 8ecc5665b2..71c78a3bd3 100644
--- a/keras_nlp/models/preprocessor_test.py
+++ b/keras_nlp/src/models/preprocessor_test.py
@@ -16,19 +16,21 @@
 import pytest
 from absl.testing import parameterized
 
-from keras_nlp.models.albert.albert_preprocessor import AlbertPreprocessor
-from keras_nlp.models.bert.bert_masked_lm_preprocessor import (
+from keras_nlp.src.models.albert.albert_preprocessor import AlbertPreprocessor
+from keras_nlp.src.models.bert.bert_masked_lm_preprocessor import (
     BertMaskedLMPreprocessor,
 )
-from keras_nlp.models.bert.bert_preprocessor import BertPreprocessor
-from keras_nlp.models.gpt2.gpt2_preprocessor import GPT2Preprocessor
-from keras_nlp.models.preprocessor import Preprocessor
-from keras_nlp.models.roberta.roberta_preprocessor import RobertaPreprocessor
-from keras_nlp.tests.test_case import TestCase
-from keras_nlp.utils.preset_utils import METADATA_FILE
-from keras_nlp.utils.preset_utils import PREPROCESSOR_CONFIG_FILE
-from keras_nlp.utils.preset_utils import TOKENIZER_ASSET_DIR
-from keras_nlp.utils.preset_utils import check_config_class
+from keras_nlp.src.models.bert.bert_preprocessor import BertPreprocessor
+from keras_nlp.src.models.gpt2.gpt2_preprocessor import GPT2Preprocessor
+from keras_nlp.src.models.preprocessor import Preprocessor
+from keras_nlp.src.models.roberta.roberta_preprocessor import (
+    RobertaPreprocessor,
+)
+from keras_nlp.src.tests.test_case import TestCase
+from keras_nlp.src.utils.preset_utils import METADATA_FILE
+from keras_nlp.src.utils.preset_utils import PREPROCESSOR_CONFIG_FILE
+from keras_nlp.src.utils.preset_utils import TOKENIZER_ASSET_DIR
+from keras_nlp.src.utils.preset_utils import check_config_class
 
 
 class TestTask(TestCase):
diff --git a/keras_nlp/models/roberta/roberta_backbone_test.py b/keras_nlp/src/models/roberta/roberta_backbone_test.py
similarity index 93%
rename from keras_nlp/models/roberta/roberta_backbone_test.py
rename to keras_nlp/src/models/roberta/roberta_backbone_test.py
index fe85e183a8..324b98f9e9 100644
--- a/keras_nlp/models/roberta/roberta_backbone_test.py
+++ b/keras_nlp/src/models/roberta/roberta_backbone_test.py
@@ -14,9 +14,9 @@
 
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.roberta.roberta_backbone import RobertaBackbone
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.roberta.roberta_backbone import RobertaBackbone
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class RobertaBackboneTest(TestCase):
diff --git a/keras_nlp/models/roberta/roberta_classifier_test.py b/keras_nlp/src/models/roberta/roberta_classifier_test.py
similarity index 88%
rename from keras_nlp/models/roberta/roberta_classifier_test.py
rename to keras_nlp/src/models/roberta/roberta_classifier_test.py
index e85d2a3703..d783bffee6 100644
--- a/keras_nlp/models/roberta/roberta_classifier_test.py
+++ b/keras_nlp/src/models/roberta/roberta_classifier_test.py
@@ -14,11 +14,13 @@
 
 import pytest
 
-from keras_nlp.models.roberta.roberta_backbone import RobertaBackbone
-from keras_nlp.models.roberta.roberta_classifier import RobertaClassifier
-from keras_nlp.models.roberta.roberta_preprocessor import RobertaPreprocessor
-from keras_nlp.models.roberta.roberta_tokenizer import RobertaTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.roberta.roberta_backbone import RobertaBackbone
+from keras_nlp.src.models.roberta.roberta_classifier import RobertaClassifier
+from keras_nlp.src.models.roberta.roberta_preprocessor import (
+    RobertaPreprocessor,
+)
+from keras_nlp.src.models.roberta.roberta_tokenizer import RobertaTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class RobertaClassifierTest(TestCase):
diff --git a/keras_nlp/models/roberta/roberta_masked_lm_preprocessor_test.py b/keras_nlp/src/models/roberta/roberta_masked_lm_preprocessor_test.py
similarity index 94%
rename from keras_nlp/models/roberta/roberta_masked_lm_preprocessor_test.py
rename to keras_nlp/src/models/roberta/roberta_masked_lm_preprocessor_test.py
index a842e99f5d..73acabe54c 100644
--- a/keras_nlp/models/roberta/roberta_masked_lm_preprocessor_test.py
+++ b/keras_nlp/src/models/roberta/roberta_masked_lm_preprocessor_test.py
@@ -14,11 +14,11 @@
 
 import pytest
 
-from keras_nlp.models.roberta.roberta_masked_lm_preprocessor import (
+from keras_nlp.src.models.roberta.roberta_masked_lm_preprocessor import (
     RobertaMaskedLMPreprocessor,
 )
-from keras_nlp.models.roberta.roberta_tokenizer import RobertaTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.roberta.roberta_tokenizer import RobertaTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class RobertaMaskedLMPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/roberta/roberta_masked_lm_test.py b/keras_nlp/src/models/roberta/roberta_masked_lm_test.py
similarity index 89%
rename from keras_nlp/models/roberta/roberta_masked_lm_test.py
rename to keras_nlp/src/models/roberta/roberta_masked_lm_test.py
index 6d5ba7eb8d..86abd7e046 100644
--- a/keras_nlp/models/roberta/roberta_masked_lm_test.py
+++ b/keras_nlp/src/models/roberta/roberta_masked_lm_test.py
@@ -14,13 +14,13 @@
 
 import pytest
 
-from keras_nlp.models.roberta.roberta_backbone import RobertaBackbone
-from keras_nlp.models.roberta.roberta_masked_lm import RobertaMaskedLM
-from keras_nlp.models.roberta.roberta_masked_lm_preprocessor import (
+from keras_nlp.src.models.roberta.roberta_backbone import RobertaBackbone
+from keras_nlp.src.models.roberta.roberta_masked_lm import RobertaMaskedLM
+from keras_nlp.src.models.roberta.roberta_masked_lm_preprocessor import (
     RobertaMaskedLMPreprocessor,
 )
-from keras_nlp.models.roberta.roberta_tokenizer import RobertaTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.roberta.roberta_tokenizer import RobertaTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class RobertaMaskedLMTest(TestCase):
diff --git a/keras_nlp/models/roberta/roberta_preprocessor_test.py b/keras_nlp/src/models/roberta/roberta_preprocessor_test.py
similarity index 92%
rename from keras_nlp/models/roberta/roberta_preprocessor_test.py
rename to keras_nlp/src/models/roberta/roberta_preprocessor_test.py
index 699742ea08..7f5fa4d518 100644
--- a/keras_nlp/models/roberta/roberta_preprocessor_test.py
+++ b/keras_nlp/src/models/roberta/roberta_preprocessor_test.py
@@ -14,9 +14,11 @@
 
 import pytest
 
-from keras_nlp.models.roberta.roberta_preprocessor import RobertaPreprocessor
-from keras_nlp.models.roberta.roberta_tokenizer import RobertaTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.roberta.roberta_preprocessor import (
+    RobertaPreprocessor,
+)
+from keras_nlp.src.models.roberta.roberta_tokenizer import RobertaTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class RobertaPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/roberta/roberta_tokenizer_test.py b/keras_nlp/src/models/roberta/roberta_tokenizer_test.py
similarity index 95%
rename from keras_nlp/models/roberta/roberta_tokenizer_test.py
rename to keras_nlp/src/models/roberta/roberta_tokenizer_test.py
index 3b2305608d..2751f6ad04 100644
--- a/keras_nlp/models/roberta/roberta_tokenizer_test.py
+++ b/keras_nlp/src/models/roberta/roberta_tokenizer_test.py
@@ -14,8 +14,8 @@
 
 import pytest
 
-from keras_nlp.models.roberta.roberta_tokenizer import RobertaTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.roberta.roberta_tokenizer import RobertaTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class RobertaTokenizerTest(TestCase):
diff --git a/keras_nlp/models/t5/t5_backbone_test.py b/keras_nlp/src/models/t5/t5_backbone_test.py
similarity index 93%
rename from keras_nlp/models/t5/t5_backbone_test.py
rename to keras_nlp/src/models/t5/t5_backbone_test.py
index bb672afa2c..9bcb4f5054 100644
--- a/keras_nlp/models/t5/t5_backbone_test.py
+++ b/keras_nlp/src/models/t5/t5_backbone_test.py
@@ -14,10 +14,10 @@
 
 import pytest
 
-from keras_nlp.backend import config as backend_config
-from keras_nlp.backend import ops
-from keras_nlp.models.t5.t5_backbone import T5Backbone
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import config as backend_config
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.t5.t5_backbone import T5Backbone
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class T5BackboneTest(TestCase):
diff --git a/keras_nlp/models/t5/t5_tokenizer_test.py b/keras_nlp/src/models/t5/t5_tokenizer_test.py
similarity index 94%
rename from keras_nlp/models/t5/t5_tokenizer_test.py
rename to keras_nlp/src/models/t5/t5_tokenizer_test.py
index a7558e8b13..7c91217b8f 100644
--- a/keras_nlp/models/t5/t5_tokenizer_test.py
+++ b/keras_nlp/src/models/t5/t5_tokenizer_test.py
@@ -16,8 +16,8 @@
 
 import pytest
 
-from keras_nlp.models.t5.t5_tokenizer import T5Tokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.t5.t5_tokenizer import T5Tokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class T5TokenizerTest(TestCase):
diff --git a/keras_nlp/models/task_test.py b/keras_nlp/src/models/task_test.py
similarity index 84%
rename from keras_nlp/models/task_test.py
rename to keras_nlp/src/models/task_test.py
index 6c6aa2543b..0101403c41 100644
--- a/keras_nlp/models/task_test.py
+++ b/keras_nlp/src/models/task_test.py
@@ -16,22 +16,22 @@
 
 import pytest
 
-from keras_nlp.backend import keras
-from keras_nlp.models import CausalLM
-from keras_nlp.models import Preprocessor
-from keras_nlp.models import Task
-from keras_nlp.models import Tokenizer
-from keras_nlp.models.bert.bert_classifier import BertClassifier
-from keras_nlp.models.classifier import Classifier
-from keras_nlp.models.gpt2.gpt2_causal_lm import GPT2CausalLM
-from keras_nlp.tests.test_case import TestCase
-from keras_nlp.utils.preset_utils import CONFIG_FILE
-from keras_nlp.utils.preset_utils import METADATA_FILE
-from keras_nlp.utils.preset_utils import MODEL_WEIGHTS_FILE
-from keras_nlp.utils.preset_utils import TASK_CONFIG_FILE
-from keras_nlp.utils.preset_utils import TASK_WEIGHTS_FILE
-from keras_nlp.utils.preset_utils import check_config_class
-from keras_nlp.utils.preset_utils import load_config
+from keras_nlp.src.backend import keras
+from keras_nlp.src.models import CausalLM
+from keras_nlp.src.models import Preprocessor
+from keras_nlp.src.models import Task
+from keras_nlp.src.models import Tokenizer
+from keras_nlp.src.models.bert.bert_classifier import BertClassifier
+from keras_nlp.src.models.classifier import Classifier
+from keras_nlp.src.models.gpt2.gpt2_causal_lm import GPT2CausalLM
+from keras_nlp.src.tests.test_case import TestCase
+from keras_nlp.src.utils.preset_utils import CONFIG_FILE
+from keras_nlp.src.utils.preset_utils import METADATA_FILE
+from keras_nlp.src.utils.preset_utils import MODEL_WEIGHTS_FILE
+from keras_nlp.src.utils.preset_utils import TASK_CONFIG_FILE
+from keras_nlp.src.utils.preset_utils import TASK_WEIGHTS_FILE
+from keras_nlp.src.utils.preset_utils import check_config_class
+from keras_nlp.src.utils.preset_utils import load_config
 
 
 class SimpleTokenizer(Tokenizer):
diff --git a/keras_nlp/models/whisper/whisper_audio_feature_extractor_test.py b/keras_nlp/src/models/whisper/whisper_audio_feature_extractor_test.py
similarity index 94%
rename from keras_nlp/models/whisper/whisper_audio_feature_extractor_test.py
rename to keras_nlp/src/models/whisper/whisper_audio_feature_extractor_test.py
index b853e09130..3ad6178572 100644
--- a/keras_nlp/models/whisper/whisper_audio_feature_extractor_test.py
+++ b/keras_nlp/src/models/whisper/whisper_audio_feature_extractor_test.py
@@ -20,10 +20,10 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.models.whisper.whisper_audio_feature_extractor import (
+from keras_nlp.src.models.whisper.whisper_audio_feature_extractor import (
     WhisperAudioFeatureExtractor,
 )
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class WhisperAudioFeatureExtractorTest(TestCase):
diff --git a/keras_nlp/models/whisper/whisper_backbone_test.py b/keras_nlp/src/models/whisper/whisper_backbone_test.py
similarity index 95%
rename from keras_nlp/models/whisper/whisper_backbone_test.py
rename to keras_nlp/src/models/whisper/whisper_backbone_test.py
index 63d57615af..65e3fc1008 100644
--- a/keras_nlp/models/whisper/whisper_backbone_test.py
+++ b/keras_nlp/src/models/whisper/whisper_backbone_test.py
@@ -14,9 +14,9 @@
 
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.whisper.whisper_backbone import WhisperBackbone
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.whisper.whisper_backbone import WhisperBackbone
+from keras_nlp.src.tests.test_case import TestCase
 
 
 @pytest.mark.tf_only
diff --git a/keras_nlp/models/whisper/whisper_preprocessor_test.py b/keras_nlp/src/models/whisper/whisper_preprocessor_test.py
similarity index 90%
rename from keras_nlp/models/whisper/whisper_preprocessor_test.py
rename to keras_nlp/src/models/whisper/whisper_preprocessor_test.py
index 8517a6c102..2d763e2b67 100644
--- a/keras_nlp/models/whisper/whisper_preprocessor_test.py
+++ b/keras_nlp/src/models/whisper/whisper_preprocessor_test.py
@@ -14,12 +14,14 @@
 
 import numpy as np
 
-from keras_nlp.models.whisper.whisper_audio_feature_extractor import (
+from keras_nlp.src.models.whisper.whisper_audio_feature_extractor import (
     WhisperAudioFeatureExtractor,
 )
-from keras_nlp.models.whisper.whisper_preprocessor import WhisperPreprocessor
-from keras_nlp.models.whisper.whisper_tokenizer import WhisperTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.whisper.whisper_preprocessor import (
+    WhisperPreprocessor,
+)
+from keras_nlp.src.models.whisper.whisper_tokenizer import WhisperTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class WhisperPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/whisper/whisper_tokenizer_test.py b/keras_nlp/src/models/whisper/whisper_tokenizer_test.py
similarity index 96%
rename from keras_nlp/models/whisper/whisper_tokenizer_test.py
rename to keras_nlp/src/models/whisper/whisper_tokenizer_test.py
index 16fab2e34a..8d5545e953 100644
--- a/keras_nlp/models/whisper/whisper_tokenizer_test.py
+++ b/keras_nlp/src/models/whisper/whisper_tokenizer_test.py
@@ -14,8 +14,8 @@
 
 import pytest
 
-from keras_nlp.models.whisper.whisper_tokenizer import WhisperTokenizer
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.models.whisper.whisper_tokenizer import WhisperTokenizer
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class WhisperTokenizerTest(TestCase):
diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_backbone_test.py b/keras_nlp/src/models/xlm_roberta/xlm_roberta_backbone_test.py
similarity index 93%
rename from keras_nlp/models/xlm_roberta/xlm_roberta_backbone_test.py
rename to keras_nlp/src/models/xlm_roberta/xlm_roberta_backbone_test.py
index e92aaea0ef..809e55af76 100644
--- a/keras_nlp/models/xlm_roberta/xlm_roberta_backbone_test.py
+++ b/keras_nlp/src/models/xlm_roberta/xlm_roberta_backbone_test.py
@@ -14,9 +14,11 @@
 
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.xlm_roberta.xlm_roberta_backbone import XLMRobertaBackbone
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_backbone import (
+    XLMRobertaBackbone,
+)
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class XLMRobertaBackboneTest(TestCase):
diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_classifier_test.py b/keras_nlp/src/models/xlm_roberta/xlm_roberta_classifier_test.py
similarity index 88%
rename from keras_nlp/models/xlm_roberta/xlm_roberta_classifier_test.py
rename to keras_nlp/src/models/xlm_roberta/xlm_roberta_classifier_test.py
index bfa6500247..799b2b7a18 100644
--- a/keras_nlp/models/xlm_roberta/xlm_roberta_classifier_test.py
+++ b/keras_nlp/src/models/xlm_roberta/xlm_roberta_classifier_test.py
@@ -16,17 +16,19 @@
 
 import pytest
 
-from keras_nlp.models.xlm_roberta.xlm_roberta_backbone import XLMRobertaBackbone
-from keras_nlp.models.xlm_roberta.xlm_roberta_classifier import (
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_backbone import (
+    XLMRobertaBackbone,
+)
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_classifier import (
     XLMRobertaClassifier,
 )
-from keras_nlp.models.xlm_roberta.xlm_roberta_preprocessor import (
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_preprocessor import (
     XLMRobertaPreprocessor,
 )
-from keras_nlp.models.xlm_roberta.xlm_roberta_tokenizer import (
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_tokenizer import (
     XLMRobertaTokenizer,
 )
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class XLMRobertaClassifierTest(TestCase):
diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor_test.py b/keras_nlp/src/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor_test.py
similarity index 93%
rename from keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor_test.py
rename to keras_nlp/src/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor_test.py
index 6d77e71319..89923a1b55 100644
--- a/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor_test.py
+++ b/keras_nlp/src/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor_test.py
@@ -16,13 +16,13 @@
 
 import pytest
 
-from keras_nlp.models.xlm_roberta.xlm_roberta_masked_lm_preprocessor import (
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_masked_lm_preprocessor import (
     XLMRobertaMaskedLMPreprocessor,
 )
-from keras_nlp.models.xlm_roberta.xlm_roberta_tokenizer import (
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_tokenizer import (
     XLMRobertaTokenizer,
 )
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class XLMRobertaMaskedLMPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm_test.py b/keras_nlp/src/models/xlm_roberta/xlm_roberta_masked_lm_test.py
similarity index 88%
rename from keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm_test.py
rename to keras_nlp/src/models/xlm_roberta/xlm_roberta_masked_lm_test.py
index d9a1ce68f1..e514be59fa 100644
--- a/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm_test.py
+++ b/keras_nlp/src/models/xlm_roberta/xlm_roberta_masked_lm_test.py
@@ -16,17 +16,19 @@
 
 import pytest
 
-from keras_nlp.models.xlm_roberta.xlm_roberta_backbone import XLMRobertaBackbone
-from keras_nlp.models.xlm_roberta.xlm_roberta_masked_lm import (
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_backbone import (
+    XLMRobertaBackbone,
+)
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_masked_lm import (
     XLMRobertaMaskedLM,
 )
-from keras_nlp.models.xlm_roberta.xlm_roberta_masked_lm_preprocessor import (
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_masked_lm_preprocessor import (
     XLMRobertaMaskedLMPreprocessor,
 )
-from keras_nlp.models.xlm_roberta.xlm_roberta_tokenizer import (
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_tokenizer import (
     XLMRobertaTokenizer,
 )
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class XLMRobertaMaskedLMTest(TestCase):
diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_preprocessor_test.py b/keras_nlp/src/models/xlm_roberta/xlm_roberta_preprocessor_test.py
similarity index 92%
rename from keras_nlp/models/xlm_roberta/xlm_roberta_preprocessor_test.py
rename to keras_nlp/src/models/xlm_roberta/xlm_roberta_preprocessor_test.py
index 85c76fa282..55ede36b63 100644
--- a/keras_nlp/models/xlm_roberta/xlm_roberta_preprocessor_test.py
+++ b/keras_nlp/src/models/xlm_roberta/xlm_roberta_preprocessor_test.py
@@ -16,13 +16,13 @@
 
 import pytest
 
-from keras_nlp.models.xlm_roberta.xlm_roberta_preprocessor import (
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_preprocessor import (
     XLMRobertaPreprocessor,
 )
-from keras_nlp.models.xlm_roberta.xlm_roberta_tokenizer import (
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_tokenizer import (
     XLMRobertaTokenizer,
 )
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class XLMRobertaPreprocessorTest(TestCase):
diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_tokenizer_test.py b/keras_nlp/src/models/xlm_roberta/xlm_roberta_tokenizer_test.py
similarity index 93%
rename from keras_nlp/models/xlm_roberta/xlm_roberta_tokenizer_test.py
rename to keras_nlp/src/models/xlm_roberta/xlm_roberta_tokenizer_test.py
index 2057eff9eb..d9b64a2c3c 100644
--- a/keras_nlp/models/xlm_roberta/xlm_roberta_tokenizer_test.py
+++ b/keras_nlp/src/models/xlm_roberta/xlm_roberta_tokenizer_test.py
@@ -16,10 +16,10 @@
 
 import pytest
 
-from keras_nlp.models.xlm_roberta.xlm_roberta_tokenizer import (
+from keras_nlp.src.models.xlm_roberta.xlm_roberta_tokenizer import (
     XLMRobertaTokenizer,
 )
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class XLMRobertaTokenizerTest(TestCase):
diff --git a/keras_nlp/models/xlnet/xlnet_backbone_test.py b/keras_nlp/src/models/xlnet/xlnet_backbone_test.py
similarity index 90%
rename from keras_nlp/models/xlnet/xlnet_backbone_test.py
rename to keras_nlp/src/models/xlnet/xlnet_backbone_test.py
index f2faf4cdd9..c5fbc5ec35 100644
--- a/keras_nlp/models/xlnet/xlnet_backbone_test.py
+++ b/keras_nlp/src/models/xlnet/xlnet_backbone_test.py
@@ -14,9 +14,9 @@
 
 import pytest
 
-from keras_nlp.backend import ops
-from keras_nlp.models.xlnet.xlnet_backbone import XLNetBackbone
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.models.xlnet.xlnet_backbone import XLNetBackbone
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class XLNetTest(TestCase):
diff --git a/keras_nlp/samplers/beam_sampler_test.py b/keras_nlp/src/samplers/beam_sampler_test.py
similarity index 97%
rename from keras_nlp/samplers/beam_sampler_test.py
rename to keras_nlp/src/samplers/beam_sampler_test.py
index 9293355a80..1302eb9c84 100644
--- a/keras_nlp/samplers/beam_sampler_test.py
+++ b/keras_nlp/src/samplers/beam_sampler_test.py
@@ -23,9 +23,9 @@
     )
 from absl.testing import parameterized
 
-from keras_nlp.backend import ops
-from keras_nlp.samplers.beam_sampler import BeamSampler
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.samplers.beam_sampler import BeamSampler
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class BeamSamplerTest(TestCase):
diff --git a/keras_nlp/samplers/contrastive_sampler_test.py b/keras_nlp/src/samplers/contrastive_sampler_test.py
similarity index 97%
rename from keras_nlp/samplers/contrastive_sampler_test.py
rename to keras_nlp/src/samplers/contrastive_sampler_test.py
index 4ceb7e91b5..3e73928c4b 100644
--- a/keras_nlp/samplers/contrastive_sampler_test.py
+++ b/keras_nlp/src/samplers/contrastive_sampler_test.py
@@ -24,9 +24,9 @@
     )
 from absl.testing import parameterized
 
-from keras_nlp.backend import ops
-from keras_nlp.samplers.contrastive_sampler import ContrastiveSampler
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.samplers.contrastive_sampler import ContrastiveSampler
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class ContrastiveSamplerTest(TestCase):
diff --git a/keras_nlp/samplers/greedy_sampler_test.py b/keras_nlp/src/samplers/greedy_sampler_test.py
similarity index 97%
rename from keras_nlp/samplers/greedy_sampler_test.py
rename to keras_nlp/src/samplers/greedy_sampler_test.py
index e9cc5c60e5..79be269591 100644
--- a/keras_nlp/samplers/greedy_sampler_test.py
+++ b/keras_nlp/src/samplers/greedy_sampler_test.py
@@ -23,9 +23,9 @@
     )
 from absl.testing import parameterized
 
-from keras_nlp.backend import ops
-from keras_nlp.samplers.greedy_sampler import GreedySampler
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.samplers.greedy_sampler import GreedySampler
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class GreedySamplerTest(TestCase):
diff --git a/keras_nlp/samplers/random_sampler_test.py b/keras_nlp/src/samplers/random_sampler_test.py
similarity index 96%
rename from keras_nlp/samplers/random_sampler_test.py
rename to keras_nlp/src/samplers/random_sampler_test.py
index ade2458364..f8dd41ea17 100644
--- a/keras_nlp/samplers/random_sampler_test.py
+++ b/keras_nlp/src/samplers/random_sampler_test.py
@@ -24,9 +24,9 @@
     )
 from absl.testing import parameterized
 
-from keras_nlp.backend import ops
-from keras_nlp.samplers.random_sampler import RandomSampler
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.samplers.random_sampler import RandomSampler
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class RandomSamplerTest(TestCase):
diff --git a/keras_nlp/samplers/serialization_test.py b/keras_nlp/src/samplers/serialization_test.py
similarity index 83%
rename from keras_nlp/samplers/serialization_test.py
rename to keras_nlp/src/samplers/serialization_test.py
index 07dc328790..45a7c44fe5 100644
--- a/keras_nlp/samplers/serialization_test.py
+++ b/keras_nlp/src/samplers/serialization_test.py
@@ -12,11 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.samplers.serialization import deserialize
-from keras_nlp.samplers.serialization import get
-from keras_nlp.samplers.serialization import serialize
-from keras_nlp.samplers.top_k_sampler import TopKSampler
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.samplers.serialization import deserialize
+from keras_nlp.src.samplers.serialization import get
+from keras_nlp.src.samplers.serialization import serialize
+from keras_nlp.src.samplers.top_k_sampler import TopKSampler
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class SerializationTest(TestCase):
diff --git a/keras_nlp/samplers/top_k_sampler_test.py b/keras_nlp/src/samplers/top_k_sampler_test.py
similarity index 97%
rename from keras_nlp/samplers/top_k_sampler_test.py
rename to keras_nlp/src/samplers/top_k_sampler_test.py
index a35c0993ff..d3fd979411 100644
--- a/keras_nlp/samplers/top_k_sampler_test.py
+++ b/keras_nlp/src/samplers/top_k_sampler_test.py
@@ -23,9 +23,9 @@
     )
 from absl.testing import parameterized
 
-from keras_nlp.backend import ops
-from keras_nlp.samplers.top_k_sampler import TopKSampler
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.samplers.top_k_sampler import TopKSampler
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class TopKSamplerTest(TestCase):
diff --git a/keras_nlp/samplers/top_p_sampler_test.py b/keras_nlp/src/samplers/top_p_sampler_test.py
similarity index 97%
rename from keras_nlp/samplers/top_p_sampler_test.py
rename to keras_nlp/src/samplers/top_p_sampler_test.py
index f3a9d8e0eb..1d7c5e3d09 100644
--- a/keras_nlp/samplers/top_p_sampler_test.py
+++ b/keras_nlp/src/samplers/top_p_sampler_test.py
@@ -24,9 +24,9 @@
     )
 from absl.testing import parameterized
 
-from keras_nlp.backend import ops
-from keras_nlp.samplers.top_p_sampler import TopPSampler
-from keras_nlp.tests.test_case import TestCase
+from keras_nlp.src.backend import ops
+from keras_nlp.src.samplers.top_p_sampler import TopPSampler
+from keras_nlp.src.tests.test_case import TestCase
 
 
 class TopPSamplerTest(TestCase):
diff --git a/keras_nlp/tests/__init__.py b/keras_nlp/src/tests/__init__.py
similarity index 100%
rename from keras_nlp/tests/__init__.py
rename to keras_nlp/src/tests/__init__.py
diff --git a/keras_nlp/tests/doc_tests/__init__.py b/keras_nlp/src/tests/doc_tests/__init__.py
similarity index 100%
rename from keras_nlp/tests/doc_tests/__init__.py
rename to keras_nlp/src/tests/doc_tests/__init__.py
diff --git a/keras_nlp/tests/doc_tests/docstring_lib.py b/keras_nlp/src/tests/doc_tests/docstring_lib.py
similarity index 100%
rename from keras_nlp/tests/doc_tests/docstring_lib.py
rename to keras_nlp/src/tests/doc_tests/docstring_lib.py
diff --git a/keras_nlp/tests/doc_tests/docstring_test.py b/keras_nlp/src/tests/doc_tests/docstring_test.py
similarity index 94%
rename from keras_nlp/tests/doc_tests/docstring_test.py
rename to keras_nlp/src/tests/doc_tests/docstring_test.py
index 4698572446..397ab39f1f 100644
--- a/keras_nlp/tests/doc_tests/docstring_test.py
+++ b/keras_nlp/src/tests/doc_tests/docstring_test.py
@@ -31,10 +31,10 @@
     )
 
 import keras_nlp
-from keras_nlp.backend import keras
-from keras_nlp.tests.doc_tests import docstring_lib
-from keras_nlp.tests.doc_tests import fenced_docstring_lib
-from keras_nlp.tests.doc_tests.fenced_docstring_lib import (
+from keras_nlp.src.backend import keras
+from keras_nlp.src.tests.doc_tests import docstring_lib
+from keras_nlp.src.tests.doc_tests import fenced_docstring_lib
+from keras_nlp.src.tests.doc_tests.fenced_docstring_lib import (
     astor,  # For checking conditional import.
 )
 
@@ -56,6 +56,7 @@ def docstring_module(pytestconfig):
 
 
 @pytest.mark.tf_only
+@pytest.mark.keras_3_only
 def test_docstrings(docstring_module):
     keras_nlp_modules = find_modules()
     # As of this writing, it doesn't seem like pytest support load_tests
@@ -95,6 +96,7 @@ def test_docstrings(docstring_module):
 
 
 @pytest.mark.tf_only
+@pytest.mark.keras_3_only
 @pytest.mark.extra_large
 @pytest.mark.skipif(
     astor is None,
diff --git a/keras_nlp/tests/doc_tests/fenced_docstring_lib.py b/keras_nlp/src/tests/doc_tests/fenced_docstring_lib.py
similarity index 100%
rename from keras_nlp/tests/doc_tests/fenced_docstring_lib.py
rename to keras_nlp/src/tests/doc_tests/fenced_docstring_lib.py
diff --git a/keras_nlp/tests/test_case.py b/keras_nlp/src/tests/test_case.py
similarity index 98%
rename from keras_nlp/tests/test_case.py
rename to keras_nlp/src/tests/test_case.py
index b4f1ad3842..7f48bcce59 100644
--- a/keras_nlp/tests/test_case.py
+++ b/keras_nlp/src/tests/test_case.py
@@ -27,12 +27,12 @@
 import tree
 from absl.testing import parameterized
 
-from keras_nlp.backend import config
-from keras_nlp.backend import keras
-from keras_nlp.backend import ops
-from keras_nlp.tokenizers.tokenizer import Tokenizer
-from keras_nlp.utils.tensor_utils import is_float_dtype
-from keras_nlp.utils.tensor_utils import standardize_dtype
+from keras_nlp.src.backend import config
+from keras_nlp.src.backend import keras
+from keras_nlp.src.backend import ops
+from keras_nlp.src.tokenizers.tokenizer import Tokenizer
+from keras_nlp.src.utils.tensor_utils import is_float_dtype
+from keras_nlp.src.utils.tensor_utils import standardize_dtype
 
 
 def convert_to_comparible_type(x):
diff --git a/keras_nlp/tests/test_data/albert_test_vocab.spm b/keras_nlp/src/tests/test_data/albert_test_vocab.spm
similarity index 100%
rename from keras_nlp/tests/test_data/albert_test_vocab.spm
rename to keras_nlp/src/tests/test_data/albert_test_vocab.spm
diff --git a/keras_nlp/tests/test_data/deberta_v3_test_vocab.spm b/keras_nlp/src/tests/test_data/deberta_v3_test_vocab.spm
similarity index 100%
rename from keras_nlp/tests/test_data/deberta_v3_test_vocab.spm
rename to keras_nlp/src/tests/test_data/deberta_v3_test_vocab.spm
diff --git a/keras_nlp/tests/test_data/f_net_test_vocab.spm b/keras_nlp/src/tests/test_data/f_net_test_vocab.spm
similarity index 100%
rename from keras_nlp/tests/test_data/f_net_test_vocab.spm
rename to keras_nlp/src/tests/test_data/f_net_test_vocab.spm
diff --git a/keras_nlp/tests/test_data/gemma_test_vocab.spm b/keras_nlp/src/tests/test_data/gemma_test_vocab.spm
similarity index 100%
rename from keras_nlp/tests/test_data/gemma_test_vocab.spm
rename to keras_nlp/src/tests/test_data/gemma_test_vocab.spm
diff --git a/keras_nlp/tests/test_data/llama_test_vocab.spm b/keras_nlp/src/tests/test_data/llama_test_vocab.spm
similarity index 100%
rename from keras_nlp/tests/test_data/llama_test_vocab.spm
rename to keras_nlp/src/tests/test_data/llama_test_vocab.spm
diff --git a/keras_nlp/tests/test_data/mistral_test_vocab.spm b/keras_nlp/src/tests/test_data/mistral_test_vocab.spm
similarity index 100%
rename from keras_nlp/tests/test_data/mistral_test_vocab.spm
rename to keras_nlp/src/tests/test_data/mistral_test_vocab.spm
diff --git a/keras_nlp/tests/test_data/no_special_token_vocab.spm b/keras_nlp/src/tests/test_data/no_special_token_vocab.spm
similarity index 100%
rename from keras_nlp/tests/test_data/no_special_token_vocab.spm
rename to keras_nlp/src/tests/test_data/no_special_token_vocab.spm
diff --git a/keras_nlp/tests/test_data/t5_test_vocab.spm b/keras_nlp/src/tests/test_data/t5_test_vocab.spm
similarity index 100%
rename from keras_nlp/tests/test_data/t5_test_vocab.spm
rename to keras_nlp/src/tests/test_data/t5_test_vocab.spm
diff --git a/keras_nlp/tests/test_data/tokenizer_test_vocab.spm b/keras_nlp/src/tests/test_data/tokenizer_test_vocab.spm
similarity index 100%
rename from keras_nlp/tests/test_data/tokenizer_test_vocab.spm
rename to keras_nlp/src/tests/test_data/tokenizer_test_vocab.spm
diff --git a/keras_nlp/tests/test_data/xlm_roberta_test_vocab.spm b/keras_nlp/src/tests/test_data/xlm_roberta_test_vocab.spm
similarity index 100%
rename from keras_nlp/tests/test_data/xlm_roberta_test_vocab.spm
rename to keras_nlp/src/tests/test_data/xlm_roberta_test_vocab.spm
diff --git a/keras_nlp/tokenizers/byte_pair_tokenizer_test.py b/keras_nlp/src/tokenizers/byte_pair_tokenizer_test.py
similarity index 97%
rename from keras_nlp/tokenizers/byte_pair_tokenizer_test.py
rename to keras_nlp/src/tokenizers/byte_pair_tokenizer_test.py
index 00f8f9b87f..cb140ca961 100644
--- a/keras_nlp/tokenizers/byte_pair_tokenizer_test.py
+++ b/keras_nlp/src/tokenizers/byte_pair_tokenizer_test.py
@@ -15,9 +15,9 @@
 import pytest
 import tensorflow as tf
 
-from keras_nlp.backend import keras
-from keras_nlp.tests.test_case import TestCase
-from keras_nlp.tokenizers.byte_pair_tokenizer import BytePairTokenizer
+from keras_nlp.src.backend import keras
+from keras_nlp.src.tests.test_case import TestCase
+from keras_nlp.src.tokenizers.byte_pair_tokenizer import BytePairTokenizer
 
 VOCAB_PATH = keras.utils.get_file(
     None,
diff --git a/keras_nlp/tokenizers/byte_tokenizer_test.py b/keras_nlp/src/tokenizers/byte_tokenizer_test.py
similarity index 98%
rename from keras_nlp/tokenizers/byte_tokenizer_test.py
rename to keras_nlp/src/tokenizers/byte_tokenizer_test.py
index 577db7b0d3..b2444f36a0 100644
--- a/keras_nlp/tokenizers/byte_tokenizer_test.py
+++ b/keras_nlp/src/tokenizers/byte_tokenizer_test.py
@@ -20,8 +20,8 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.tests.test_case import TestCase
-from keras_nlp.tokenizers.byte_tokenizer import ByteTokenizer
+from keras_nlp.src.tests.test_case import TestCase
+from keras_nlp.src.tokenizers.byte_tokenizer import ByteTokenizer
 
 
 class ByteTokenizerTest(TestCase):
diff --git a/keras_nlp/tokenizers/sentence_piece_tokenizer_test.py b/keras_nlp/src/tokenizers/sentence_piece_tokenizer_test.py
similarity index 97%
rename from keras_nlp/tokenizers/sentence_piece_tokenizer_test.py
rename to keras_nlp/src/tokenizers/sentence_piece_tokenizer_test.py
index dd4f57cabd..9d6526ce20 100644
--- a/keras_nlp/tokenizers/sentence_piece_tokenizer_test.py
+++ b/keras_nlp/src/tokenizers/sentence_piece_tokenizer_test.py
@@ -22,8 +22,10 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.tests.test_case import TestCase
-from keras_nlp.tokenizers.sentence_piece_tokenizer import SentencePieceTokenizer
+from keras_nlp.src.tests.test_case import TestCase
+from keras_nlp.src.tokenizers.sentence_piece_tokenizer import (
+    SentencePieceTokenizer,
+)
 
 
 class SentencePieceTokenizerTest(TestCase):
diff --git a/keras_nlp/tokenizers/sentence_piece_tokenizer_trainer_test.py b/keras_nlp/src/tokenizers/sentence_piece_tokenizer_trainer_test.py
similarity index 92%
rename from keras_nlp/tokenizers/sentence_piece_tokenizer_trainer_test.py
rename to keras_nlp/src/tokenizers/sentence_piece_tokenizer_trainer_test.py
index 7adbfe1033..f1d6867793 100644
--- a/keras_nlp/tokenizers/sentence_piece_tokenizer_trainer_test.py
+++ b/keras_nlp/src/tokenizers/sentence_piece_tokenizer_trainer_test.py
@@ -15,6 +15,8 @@
 import os
 import re
 
+import pytest
+
 try:
     import tensorflow as tf
 except ImportError:
@@ -23,13 +25,17 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.tests.test_case import TestCase
-from keras_nlp.tokenizers.sentence_piece_tokenizer import SentencePieceTokenizer
-from keras_nlp.tokenizers.sentence_piece_tokenizer_trainer import (
+from keras_nlp.src.tests.test_case import TestCase
+from keras_nlp.src.tokenizers.sentence_piece_tokenizer import (
+    SentencePieceTokenizer,
+)
+from keras_nlp.src.tokenizers.sentence_piece_tokenizer_trainer import (
     compute_sentence_piece_proto,
 )
 
 
+# TODO: Segfault on Keras2 Test when calling compute_sentence_piece_proto
+@pytest.mark.keras_3_only
 class SentencePieceTokenizerTrainerTest(TestCase):
     def test_dataset_input(self):
         test_text = ["Ninjas and Samurais"]
diff --git a/keras_nlp/tokenizers/tokenizer_test.py b/keras_nlp/src/tokenizers/tokenizer_test.py
similarity index 87%
rename from keras_nlp/tokenizers/tokenizer_test.py
rename to keras_nlp/src/tokenizers/tokenizer_test.py
index 5bb7c5d565..38283a0d5b 100644
--- a/keras_nlp/tokenizers/tokenizer_test.py
+++ b/keras_nlp/src/tokenizers/tokenizer_test.py
@@ -25,16 +25,16 @@
     )
 from absl.testing import parameterized
 
-from keras_nlp.models.albert.albert_tokenizer import AlbertTokenizer
-from keras_nlp.models.bert.bert_tokenizer import BertTokenizer
-from keras_nlp.models.gpt2.gpt2_tokenizer import GPT2Tokenizer
-from keras_nlp.models.roberta.roberta_tokenizer import RobertaTokenizer
-from keras_nlp.tests.test_case import TestCase
-from keras_nlp.tokenizers.tokenizer import Tokenizer
-from keras_nlp.utils.preset_utils import METADATA_FILE
-from keras_nlp.utils.preset_utils import TOKENIZER_ASSET_DIR
-from keras_nlp.utils.preset_utils import TOKENIZER_CONFIG_FILE
-from keras_nlp.utils.preset_utils import check_config_class
+from keras_nlp.src.models.albert.albert_tokenizer import AlbertTokenizer
+from keras_nlp.src.models.bert.bert_tokenizer import BertTokenizer
+from keras_nlp.src.models.gpt2.gpt2_tokenizer import GPT2Tokenizer
+from keras_nlp.src.models.roberta.roberta_tokenizer import RobertaTokenizer
+from keras_nlp.src.tests.test_case import TestCase
+from keras_nlp.src.tokenizers.tokenizer import Tokenizer
+from keras_nlp.src.utils.preset_utils import METADATA_FILE
+from keras_nlp.src.utils.preset_utils import TOKENIZER_ASSET_DIR
+from keras_nlp.src.utils.preset_utils import TOKENIZER_CONFIG_FILE
+from keras_nlp.src.utils.preset_utils import check_config_class
 
 
 class SimpleTokenizer(Tokenizer):
diff --git a/keras_nlp/tokenizers/unicode_codepoint_tokenizer_test.py b/keras_nlp/src/tokenizers/unicode_codepoint_tokenizer_test.py
similarity index 98%
rename from keras_nlp/tokenizers/unicode_codepoint_tokenizer_test.py
rename to keras_nlp/src/tokenizers/unicode_codepoint_tokenizer_test.py
index 541f5a5489..773733bb5b 100644
--- a/keras_nlp/tokenizers/unicode_codepoint_tokenizer_test.py
+++ b/keras_nlp/src/tokenizers/unicode_codepoint_tokenizer_test.py
@@ -20,8 +20,8 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.tests.test_case import TestCase
-from keras_nlp.tokenizers.unicode_codepoint_tokenizer import (
+from keras_nlp.src.tests.test_case import TestCase
+from keras_nlp.src.tokenizers.unicode_codepoint_tokenizer import (
     UnicodeCodepointTokenizer,
 )
 
diff --git a/keras_nlp/tokenizers/word_piece_tokenizer_test.py b/keras_nlp/src/tokenizers/word_piece_tokenizer_test.py
similarity index 98%
rename from keras_nlp/tokenizers/word_piece_tokenizer_test.py
rename to keras_nlp/src/tokenizers/word_piece_tokenizer_test.py
index b89cc786f4..2c3d200931 100644
--- a/keras_nlp/tokenizers/word_piece_tokenizer_test.py
+++ b/keras_nlp/src/tokenizers/word_piece_tokenizer_test.py
@@ -22,8 +22,8 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.tests.test_case import TestCase
-from keras_nlp.tokenizers.word_piece_tokenizer import WordPieceTokenizer
+from keras_nlp.src.tests.test_case import TestCase
+from keras_nlp.src.tokenizers.word_piece_tokenizer import WordPieceTokenizer
 
 
 class WordPieceTokenizerTest(TestCase):
diff --git a/keras_nlp/tokenizers/word_piece_tokenizer_trainer_test.py b/keras_nlp/src/tokenizers/word_piece_tokenizer_trainer_test.py
similarity index 98%
rename from keras_nlp/tokenizers/word_piece_tokenizer_trainer_test.py
rename to keras_nlp/src/tokenizers/word_piece_tokenizer_trainer_test.py
index c0fa2fe62d..c236d8b6b2 100644
--- a/keras_nlp/tokenizers/word_piece_tokenizer_trainer_test.py
+++ b/keras_nlp/src/tokenizers/word_piece_tokenizer_trainer_test.py
@@ -22,8 +22,8 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.tests.test_case import TestCase
-from keras_nlp.tokenizers.word_piece_tokenizer_trainer import (
+from keras_nlp.src.tests.test_case import TestCase
+from keras_nlp.src.tokenizers.word_piece_tokenizer_trainer import (
     compute_word_piece_vocabulary,
 )
 
diff --git a/keras_nlp/utils/keras_utils_test.py b/keras_nlp/src/utils/keras_utils_test.py
similarity index 91%
rename from keras_nlp/utils/keras_utils_test.py
rename to keras_nlp/src/utils/keras_utils_test.py
index 3dc09f702f..ba340a13a7 100644
--- a/keras_nlp/utils/keras_utils_test.py
+++ b/keras_nlp/src/utils/keras_utils_test.py
@@ -20,10 +20,10 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.backend import keras
-from keras_nlp.tests.test_case import TestCase
-from keras_nlp.utils.keras_utils import clone_initializer
-from keras_nlp.utils.keras_utils import pack_x_y_sample_weight
+from keras_nlp.src.backend import keras
+from keras_nlp.src.tests.test_case import TestCase
+from keras_nlp.src.utils.keras_utils import clone_initializer
+from keras_nlp.src.utils.keras_utils import pack_x_y_sample_weight
 
 
 class CloneInitializerTest(TestCase):
diff --git a/keras_nlp/utils/pipeline_model_test.py b/keras_nlp/src/utils/pipeline_model_test.py
similarity index 99%
rename from keras_nlp/utils/pipeline_model_test.py
rename to keras_nlp/src/utils/pipeline_model_test.py
index 612e17f1c1..ed3f8f0ab2 100644
--- a/keras_nlp/utils/pipeline_model_test.py
+++ b/keras_nlp/src/utils/pipeline_model_test.py
@@ -24,9 +24,9 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.backend import keras
-from keras_nlp.tests.test_case import TestCase
-from keras_nlp.utils.pipeline_model import PipelineModel
+from keras_nlp.src.backend import keras
+from keras_nlp.src.tests.test_case import TestCase
+from keras_nlp.src.utils.pipeline_model import PipelineModel
 
 
 class NoopPipeline(PipelineModel):
diff --git a/keras_nlp/utils/preset_utils_test.py b/keras_nlp/src/utils/preset_utils_test.py
similarity index 89%
rename from keras_nlp/utils/preset_utils_test.py
rename to keras_nlp/src/utils/preset_utils_test.py
index 3f985cf273..95c0cc9256 100644
--- a/keras_nlp/utils/preset_utils_test.py
+++ b/keras_nlp/src/utils/preset_utils_test.py
@@ -18,15 +18,15 @@
 import pytest
 from absl.testing import parameterized
 
-from keras_nlp import upload_preset
-from keras_nlp.models import AlbertClassifier
-from keras_nlp.models import BertBackbone
-from keras_nlp.models import BertTokenizer
-from keras_nlp.tests.test_case import TestCase
-from keras_nlp.utils.preset_utils import CONFIG_FILE
-from keras_nlp.utils.preset_utils import METADATA_FILE
-from keras_nlp.utils.preset_utils import TOKENIZER_CONFIG_FILE
-from keras_nlp.utils.preset_utils import validate_metadata
+from keras_nlp.src import upload_preset
+from keras_nlp.src.models import AlbertClassifier
+from keras_nlp.src.models import BertBackbone
+from keras_nlp.src.models import BertTokenizer
+from keras_nlp.src.tests.test_case import TestCase
+from keras_nlp.src.utils.preset_utils import CONFIG_FILE
+from keras_nlp.src.utils.preset_utils import METADATA_FILE
+from keras_nlp.src.utils.preset_utils import TOKENIZER_CONFIG_FILE
+from keras_nlp.src.utils.preset_utils import validate_metadata
 
 
 class PresetUtilsTest(TestCase):
diff --git a/keras_nlp/utils/python_utils_test.py b/keras_nlp/src/utils/python_utils_test.py
similarity index 88%
rename from keras_nlp/utils/python_utils_test.py
rename to keras_nlp/src/utils/python_utils_test.py
index 7d08ca091f..6f10b1d3d4 100644
--- a/keras_nlp/utils/python_utils_test.py
+++ b/keras_nlp/src/utils/python_utils_test.py
@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from keras_nlp.tests.test_case import TestCase
-from keras_nlp.utils.python_utils import classproperty
+from keras_nlp.src.tests.test_case import TestCase
+from keras_nlp.src.utils.python_utils import classproperty
 
 
 class ClassPropertyTest(TestCase):
diff --git a/keras_nlp/utils/tensor_utils_test.py b/keras_nlp/src/utils/tensor_utils_test.py
similarity index 95%
rename from keras_nlp/utils/tensor_utils_test.py
rename to keras_nlp/src/utils/tensor_utils_test.py
index e1a081aebe..6b22ee818b 100644
--- a/keras_nlp/utils/tensor_utils_test.py
+++ b/keras_nlp/src/utils/tensor_utils_test.py
@@ -22,11 +22,11 @@
         "The TensorFlow package is required for data preprocessing with any backend."
     )
 
-from keras_nlp.backend import ops
-from keras_nlp.tests.test_case import TestCase
-from keras_nlp.utils.tensor_utils import any_equal
-from keras_nlp.utils.tensor_utils import convert_to_ragged_batch
-from keras_nlp.utils.tensor_utils import tensor_to_list
+from keras_nlp.src.backend import ops
+from keras_nlp.src.tests.test_case import TestCase
+from keras_nlp.src.utils.tensor_utils import any_equal
+from keras_nlp.src.utils.tensor_utils import convert_to_ragged_batch
+from keras_nlp.src.utils.tensor_utils import tensor_to_list
 
 
 class TensorToListTest(TestCase):