Tokenization updates (#8)

- Tokenization updates (49fb434703c73a1a8b8e30f7d3aac46daa0b6016)
- Update tokenizer_config.json (f5839f9de43de6cc57bf3d80e46f0cd82d5c694b)
- Update tokenization_xgen.py (35686226b2c0fc4f9a5b19f9ca36686e5ef187f3)
- Update tokenization_xgen.py (db5edb9790bf97938c381f2208b149c2ff2ee01f)
- Update tokenization_xgen.py (5017b1efccce33443a00f8f9b7a34c1db53116ce)


Co-authored-by: Singer <psinger@users.noreply.huggingface.co>

tokenization_xgen.py

@@ -149,20 +149,22 @@ class XgenTokenizer(PreTrainedTokenizer):
     def _convert_token_to_id(self, token):
         """Converts a token (str) in an id using the vocab."""
         if isinstance(token, str):
-            ids = self._tokenize(token)
-            return ids[0]
-        return token
+            return self.encoder.encode_single_token(token)
+        else:
+            return token
 
     def _convert_id_to_token(self, index):
         """Converts an index (integer) in a token (str) using the vocab."""
         return self.encoder.decode_single_token_bytes(index)
 
     def _decode(self, token_ids: List[int], skip_special_tokens: bool = False, **kwargs):
+        if skip_special_tokens:
+            token_ids = [t for t in token_ids if t not in self.all_special_ids]
         return self.encoder.decode(token_ids)
 
     def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None) -> List[int]:
         """Build model inputs from a sequence by appending eos_token_id."""
-        eos_token_id = [50256] if self.add_eos_token else []
+        eos_token_id = [self.eos_token_id] if self.add_eos_token else []
 
         output = token_ids_0 + eos_token_id
 
@@ -218,11 +220,15 @@ class XgenTokenizer(PreTrainedTokenizer):
         Returns:
             `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
         """
-        eos_token_id = [50256] if self.add_eos_token else []
+        eos_token_id = [self.eos_token_id] if self.add_eos_token else []
 
         output = [0] * len(token_ids_0 + eos_token_id)
 
         if token_ids_1 is not None:
             output += [1] * len(token_ids_1 + eos_token_id)
 
-        return output
+        return output
+
+    # has no vocab file
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None):
+        return ()

tokenizer_config.json

@@ -2,6 +2,7 @@
   "add_eos_token": false,
   "add_special_tokens": true,
   "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": null,
   "tokenizer_class": "XgenTokenizer",