Repository: google-research/text-to-text-transfer-transformer

Issue:
Special tokens
**Hi All!**

I was asking if there is a way to extend the set of special tokens that during the pre-training and the fine-tuning, I do not want the model mask.
For example in hugging face I can do the following: 
tokenizer.add_special_tokens({'additional_special_tokens': ['<special_token1>','<special_token2>'})

I also have another question, if, during the fine-tuning, I'm going to use "extra_id_X" token in the input sample, is T5 able to manage such a token like a special one?

**Thanks in adavance**



Discussion:

Resolving patch:
diff --git a/t5/seqio/vocabularies.py b/t5/seqio/vocabularies.py
--- a/t5/seqio/vocabularies.py
+++ b/t5/seqio/vocabularies.py
@@ -22,6 +22,7 @@
 import tensorflow.compat.v2 as tf
 import tensorflow_text as tf_text
 
+from sentencepiece import sentencepiece_model_pb2
 import sentencepiece as sentencepiece_processor
 
 
@@ -148,6 +149,14 @@ class SentencePieceVocabulary(Vocabulary):
   Assumes the model was built using flags to reserve ID=0 for padding, ID=1 for
   EOS, and ID=2 for UNK.
 
+  If using extra ids, you can represent them in string-form as `<extra_id_0>`,
+  `<extra_id_1>`, etc. They will be indexed starting from the end of the
+  vocabulary to match how the masking preprocessors are set up.
+
+  IMPORTANT NOTE: these placeholders only work properly when they are used at
+  word starts (e.g., "I like peanut butter and <extra_id_0> sandwiches." or
+  "I like peanut butter and <extra_id_0>ly sandwiches" are both okay, but
+  "I like peanut butter and jel<extra_id_0> sandwiches is not.").
   """
 
   def __init__(self, sentencepiece_model_file, extra_ids=None):
@@ -170,8 +179,17 @@ def _load_model(self):
     # Handle cases where SP can't load the file, but gfile can.
     with tf.io.gfile.GFile(self._sentencepiece_model_file, "rb") as f:
       self._sp_model = f.read()
+      # Add placeholde strings for extra IDs.
+      if self._extra_ids:
+        model = sentencepiece_model_pb2.ModelProto.FromString(self._sp_model)
+        # We name them in reverse order to match their use in span corruption.
+        for i in reversed(range(self._extra_ids)):
+          model.pieces.add(
+              piece=f"▁<extra_id_{i}>", score=0.0,
+              type=
+              sentencepiece_model_pb2.ModelProto.SentencePiece.USER_DEFINED)
+        self._sp_model = model.SerializeToString()
     # Load Python tokenizer and ensure the EOS and PAD IDs are correct.
-    # TODO(adarob): Add support for arbitrary EOS and PAD IDs.
     self._tokenizer = sentencepiece_processor.SentencePieceProcessor()
     self._tokenizer.LoadFromSerializedProto(self._sp_model)
     if self._tokenizer.pad_id() != 0:
@@ -207,6 +225,9 @@ def tf_tokenizer(self):
     """Instantiate and return a TF tokenizer."""
     return tf_text.SentencepieceTokenizer(model=self.sp_model)
 
+  @property
+  def vocab_size(self):
+    return self._base_vocab_size
 
   @property
   def _base_vocab_size(self):


Test patch:
diff --git a/t5/seqio/vocabularies_test.py b/t5/seqio/vocabularies_test.py
--- a/t5/seqio/vocabularies_test.py
+++ b/t5/seqio/vocabularies_test.py
@@ -151,9 +151,14 @@ def test_extra_ids(self):
     vocab = test_utils.sentencepiece_vocab(extra_ids=10)
     self.assertEqual(36, vocab.vocab_size)
     self.assertEqual("v", vocab.decode([25]))
-    self.assertEqual(self.UNK_STRING, vocab.decode([35]))
-    self.assertEqual(self.UNK_STRING, _decode_tf(vocab, [35]))
-
+    test_string = "<extra_id_0> <extra_id_1> v <extra_id_9>"
+    test_tokens = (35, 34, 3, 25, 26)
+    self.assertEqual(test_string, vocab.decode(test_tokens))
+    self.assertEqual(test_string, _decode_tf(vocab, test_tokens))
+    self.assertSequenceEqual(test_tokens, vocab.encode(test_string))
+    self.assertSequenceEqual(
+        test_tokens,
+        tuple(vocab.encode_tf(test_string).numpy()))
 
   def test_equal(self):
     vocab1 = test_utils.sentencepiece_vocab()