Upload 2 files

tokenization_arcade100k.py

@@ -113,13 +113,13 @@ class Arcade100kTokenizer(PreTrainedTokenizer):
         self._tiktoken_config = _arcade100k(vocab_file)
         self.tokenizer = tiktoken.Encoding(**self._tiktoken_config)
 
-        # TODO: Remove this assertion
-        #assert (
-        #    len(self.tokenizer._mergeable_ranks)
-        #    + len(self.tokenizer._special_tokens)
-        #    + 1
-        #    == self.tokenizer.n_vocab
-        #), f"{len(self.tokenizer._mergeable_ranks) + len(self.tokenizer._special_tokens)} != {self.tokenizer.n_vocab} in encoding"
+        # # TODO: Remove this assertion
+        assert (
+            len(self.tokenizer._mergeable_ranks)
+            + len(self.tokenizer._special_tokens)
+            + 1
+            == self.tokenizer.n_vocab
+        ), f"{len(self.tokenizer._mergeable_ranks) + len(self.tokenizer._special_tokens)} != {self.tokenizer.n_vocab} in encoding"
 
         self.decoder = {i: n for n, i in self.tokenizer._mergeable_ranks.items()}
         self.decoder.update({i: n for n, i in self.tokenizer._special_tokens.items()})