Repository: keras-team/keras-nlp
Pull request #1357 (2023-12-08 05:18:59)

ElectraTokenizer 
I have added the electratokenizer . [notebook](https://colab.research.google.com/drive/1DoQpJGE11wsihHmR98CZ7r26TW1ajWfc?usp=sharing) demonstrating that tokens match 

mattdangerw: Please check the tests! Looks like some legitimate failures.
mattdangerw: I'll be out next week, so tagged @tirthasheshpatel and @nkovela1 to take a look.

Patch:
diff --git a/keras_nlp/models/__init__.py b/keras_nlp/models/__init__.py
index 858be70e..23500c74 100644
--- a/keras_nlp/models/__init__.py
+++ b/keras_nlp/models/__init__.py
@@ -64,6 +64,7 @@ from keras_nlp.models.distil_bert.distil_bert_tokenizer import (
     DistilBertTokenizer,
 )
 from keras_nlp.models.electra.electra_backbone import ElectraBackbone
+from keras_nlp.models.electra.electra_tokenizer import ElectraTokenizer
 from keras_nlp.models.f_net.f_net_backbone import FNetBackbone
 from keras_nlp.models.f_net.f_net_classifier import FNetClassifier
 from keras_nlp.models.f_net.f_net_masked_lm import FNetMaskedLM
diff --git a/keras_nlp/models/electra/electra_tokenizer.py b/keras_nlp/models/electra/electra_tokenizer.py
new file mode 100644
index 00000000..c6ec29c4
--- /dev/null
+++ b/keras_nlp/models/electra/electra_tokenizer.py
@@ -0,0 +1,79 @@
+# Copyright 2023 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from keras_nlp.api_export import keras_nlp_export
+from keras_nlp.tokenizers import WordPieceTokenizer
+
+
+@keras_nlp_export("keras_nlp.models.ElectraTokenizer")
+class ElectraTokenizer(WordPieceTokenizer):
+    """A ELECTRA tokenizer using WordPiece subword segmentation.
+
+    This tokenizer class will tokenize raw strings into integer sequences and
+    is based on `keras_nlp.tokenizers.WordPieceTokenizer`.
+
+    If input is a batch of strings (rank > 0), the layer will output a
+    `tf.RaggedTensor` where the last dimension of the output is ragged.
+
+    If input is a scalar string (rank == 0), the layer will output a dense
+    `tf.Tensor` with static shape `[None]`.
+
+    Args:
+        vocabulary: A list of strings or a string filename path. If
+            passing a list, each element of the list should be a single word
+            piece token string. If passing a filename, the file should be a
+            plain text file containing a single word piece token per line.
+        lowercase: If `True`, the input text will be first lowered before
+            tokenization.
+
+    Examples:
+    ```python
+    # Custom Vocabulary.
+    vocab = ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"]
+    vocab += ["The", "quick", "brown", "fox", "jumped", "."]
+
+    # Instantiate the tokenizer.
+    tokenizer = keras_nlp.models.ElectraTokenizer(vocabulary=vocab)
+
+    # Unbatched input.
+    tokenizer("The quick brown fox jumped.")
+
+    # Batched input.
+    tokenizer(["The quick brown fox jumped.", "The fox slept."])
+
+    # Detokenization.
+    tokenizer.detokenize(tokenizer("The quick brown fox jumped."))
+    ```
+    """
+
+    def __init__(self, vocabulary, lowercase=False, **kwargs):
+        super().__init__(vocabulary=vocabulary, lowercase=lowercase, **kwargs)
+
+        # Check for special tokens
+        cls_token = "[CLS]"
+        sep_token = "[SEP]"
+        pad_token = "[PAD]"
+        mask_token = "[MASK]"
+
+        for token in [cls_token, pad_token, sep_token, mask_token]:
+            if token not in self.get_vocabulary():
+                raise ValueError(
+                    f"Cannot find token `'{token}'` in the provided "
+                    f"`vocabulary`. Please provide `'{token}'` in your "
+                    "`vocabulary` or use a pretrained `vocabulary` name."
+                )
+        self.cls_token_id = self.token_to_id(cls_token)
+        self.sep_token_id = self.token_to_id(sep_token)
+        self.pad_token_id = self.token_to_id(pad_token)
+        self.mask_token_id = self.token_to_id(mask_token)


Test patch:
diff --git a/keras_nlp/models/electra/electra_tokenizer_test.py b/keras_nlp/models/electra/electra_tokenizer_test.py
new file mode 100644
index 00000000..2e06fb90
--- /dev/null
+++ b/keras_nlp/models/electra/electra_tokenizer_test.py
@@ -0,0 +1,42 @@
+# Copyright 2023 The KerasNLP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from keras_nlp.models.electra.electra_tokenizer import ElectraTokenizer
+from keras_nlp.tests.test_case import TestCase
+
+
+class ElectraTokenizerTest(TestCase):
+    def setUp(self):
+        self.vocab = ["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"]
+        self.vocab += ["THE", "QUICK", "BROWN", "FOX"]
+        self.vocab += ["the", "quick", "brown", "fox"]
+        self.init_kwargs = {"vocabulary": self.vocab}
+        self.input_data = ["THE QUICK BROWN FOX", "THE FOX"]
+
+    def test_tokenizer_basics(self):
+        self.run_preprocessing_layer_test(
+            cls=ElectraTokenizer,
+            init_kwargs=self.init_kwargs,
+            input_data=self.input_data,
+            expected_output=[[5, 6, 7, 8], [5, 8]],
+        )
+
+    def test_lowercase(self):
+        tokenizer = ElectraTokenizer(vocabulary=self.vocab, lowercase=True)
+        output = tokenizer(self.input_data)
+        self.assertAllEqual(output, [[9, 10, 11, 12], [9, 12]])
+
+    def test_errors_missing_special_tokens(self):
+        with self.assertRaises(ValueError):
+            ElectraTokenizer(vocabulary=["a", "b", "c"])