Repository: keras-team/keras-nlp

Issue:
Documented `id_to_token` doesn't exist for UnicodeCodepointTokenizer
**Describe the bug**
<!---
A clear and concise description of what the bug is.
-->
The Keras.io docs for UnicodeCodepointTokenizer  at https://keras.io/api/keras_nlp/tokenizers/unicode_codepoint_tokenizer/ show methods for that tokenizer that don't exist, and it's impeding my trials of word-based vs character-based tokenizing. In particular missing `id_to_token` means I can't drop in UnicodeCodepointTokenizer where I used to have a BERT-based tokenizer and see the debug output.

I know it's trivial to write `id_to_token` when the tokens are code-points, but it's one more thing I have to worry about when comparing these. Especially considering that it is documented as usable.

**To Reproduce**
<!---
Provide a link to a [Colab Notebook](https://research.google.com/colaboratory/), which reproduces the bug.
-->
```
tokenizer=keras_nlp.tokenizers.UnicodeCodepointTokenizer()
sample_tokens_int = [
    token.tolist() for word in sample_words for token in tokenizer("Hi")
]
# below gives error
# NotImplementedError: No implementation of `id_to_token()` was found for UnicodeCodepointTokenizer.
print(tokenizer.id_to_token(sample_tokens_int[0]))
```

**Expected behavior**
<!---
A clear and concise description of what you expected to happen.
-->
printing the letter "H" in this example

**Additional context**
<!---
Add any other context about the problem here.
-->
Documentation at https://keras.io/api/keras_nlp/tokenizers/unicode_codepoint_tokenizer/ says id_to_token exists for this class, and it makes sense for all tokenizers to implement these

**Would you like to help us fix it?**
Not sure I can

Discussion:
SamanehSaadat: @mattdangerw Thanks for the review and the tip! Updated the description.

Patch:
diff --git a/keras_nlp/src/tokenizers/byte_tokenizer.py b/keras_nlp/src/tokenizers/byte_tokenizer.py
index 128aac27..deffb7ed 100644
--- a/keras_nlp/src/tokenizers/byte_tokenizer.py
+++ b/keras_nlp/src/tokenizers/byte_tokenizer.py
@@ -209,6 +209,12 @@ class ByteTokenizer(tokenizer.Tokenizer):
         """Get the integer size of the tokenizer vocabulary."""
         return 256
 
+    def get_vocabulary(self):
+        vocab = {}
+        for i in range(self.vocabulary_size()):
+            vocab[chr(i)] = i
+        return vocab
+
     def tokenize(self, inputs):
         if not isinstance(inputs, (tf.Tensor, tf.RaggedTensor)):
             inputs = tf.convert_to_tensor(inputs)
@@ -264,6 +270,24 @@ class ByteTokenizer(tokenizer.Tokenizer):
             outputs = tf.squeeze(outputs, 0)
         return outputs
 
+    def id_to_token(self, id):
+        """Convert an integer id to a string token."""
+        if id >= self.vocabulary_size() or id < 0:
+            raise ValueError(
+                f"`id` must be in range [0, {self.vocabulary_size() - 1}]. "
+                f"Received: {id}"
+            )
+        return chr(id)
+
+    def token_to_id(self, token):
+        """Convert a string token to an integer id."""
+        id = ord(token)
+        if id >= self.vocabulary_size():
+            raise ValueError(
+                f"Token {token} is not supported by `ByteTokenizer`."
+            )
+        return id
+
     def get_config(self):
         config = super().get_config()
         config.update(
diff --git a/keras_nlp/src/tokenizers/unicode_codepoint_tokenizer.py b/keras_nlp/src/tokenizers/unicode_codepoint_tokenizer.py
index 8403f6fb..8afaf278 100644
--- a/keras_nlp/src/tokenizers/unicode_codepoint_tokenizer.py
+++ b/keras_nlp/src/tokenizers/unicode_codepoint_tokenizer.py
@@ -280,6 +280,12 @@ class UnicodeCodepointTokenizer(tokenizer.Tokenizer):
         size was provided"""
         return self._vocabulary_size
 
+    def get_vocabulary(self):
+        vocab = {}
+        for i in range(self.vocabulary_size()):
+            vocab[chr(i)] = i
+        return vocab
+
     def tokenize(self, inputs):
         if not isinstance(inputs, (tf.Tensor, tf.RaggedTensor)):
             inputs = tf.convert_to_tensor(inputs)
@@ -331,3 +337,21 @@ class UnicodeCodepointTokenizer(tokenizer.Tokenizer):
         if unbatched:
             outputs = tf.squeeze(outputs, 0)
         return outputs
+
+    def id_to_token(self, id):
+        """Convert an integer id to a string token."""
+        if id >= self.vocabulary_size() or id < 0:
+            raise ValueError(
+                f"`id` must be in range [0, {self.vocabulary_size() - 1}]. "
+                f"Received: {id}"
+            )
+        return chr(id)
+
+    def token_to_id(self, token):
+        """Convert a string token to an integer id."""
+        id = ord(token)
+        if id >= self.vocabulary_size():
+            raise ValueError(
+                f"Token {token} is not supported by `UnicodeCodepointTokenizer`."
+            )
+        return id


Test patch:
diff --git a/keras_nlp/src/tokenizers/byte_tokenizer_test.py b/keras_nlp/src/tokenizers/byte_tokenizer_test.py
index 0ae09688..efa323d4 100644
--- a/keras_nlp/src/tokenizers/byte_tokenizer_test.py
+++ b/keras_nlp/src/tokenizers/byte_tokenizer_test.py
@@ -222,3 +222,17 @@ class ByteTokenizerTest(TestCase):
             tokenizer(input_data),
             cloned_tokenizer(input_data),
         )
+
+    def test_token_to_id(self):
+        input_tokens = ["f", "u", "n"]
+        expected_ids = [102, 117, 110]
+        tokenizer = ByteTokenizer()
+        ids = [tokenizer.token_to_id(t) for t in input_tokens]
+        self.assertAllEqual(ids, expected_ids)
+
+    def test_id_to_token(self):
+        input_ids = [102, 117, 110]
+        expected_tokens = ["f", "u", "n"]
+        tokenizer = ByteTokenizer()
+        tokens = [tokenizer.id_to_token(i) for i in input_ids]
+        self.assertAllEqual(tokens, expected_tokens)
diff --git a/keras_nlp/src/tokenizers/unicode_codepoint_tokenizer_test.py b/keras_nlp/src/tokenizers/unicode_codepoint_tokenizer_test.py
index 4f324da1..a759db9e 100644
--- a/keras_nlp/src/tokenizers/unicode_codepoint_tokenizer_test.py
+++ b/keras_nlp/src/tokenizers/unicode_codepoint_tokenizer_test.py
@@ -280,3 +280,17 @@ class UnicodeCodepointTokenizerTest(TestCase):
             tokenizer(input_data),
             cloned_tokenizer(input_data),
         )
+
+    def test_token_to_id(self):
+        input_tokens = ["ب", "و", "خ"]
+        expected_ids = [1576, 1608, 1582]
+        tokenizer = UnicodeCodepointTokenizer(vocabulary_size=2000)
+        ids = [tokenizer.token_to_id(t) for t in input_tokens]
+        self.assertAllEqual(ids, expected_ids)
+
+    def test_id_to_token(self):
+        input_ids = [1576, 1608, 1582]
+        expected_tokens = ["ب", "و", "خ"]
+        tokenizer = UnicodeCodepointTokenizer(vocabulary_size=2000)
+        tokens = [tokenizer.id_to_token(i) for i in input_ids]
+        self.assertAllEqual(tokens, expected_tokens)