update

vocab/gpt_35_turbo/aaa.py

@@ -0,0 +1,38 @@
+"""
+gpt_35_turbo decode UnicodeDecodeError 99413 b' \xe6\xb5'
+gpt_35_turbo decode UnicodeDecodeError 99742 b'\x8c\xa8'
+gpt_35_turbo decode UnicodeDecodeError 99834 b'\xad\x90'
+gpt_35_turbo decode UnicodeDecodeError 100112 b'\xe0\xae\xbf\xe0\xae'
+gpt_35_turbo decode KeyError 100256
+gpt_35_turbo decode KeyError 100261
+gpt_35_turbo decode KeyError 100262
+gpt_35_turbo decode KeyError 100263
+"""
+
+
+
+import json
+import tiktoken
+
+
+tokenizer = tiktoken.encoding_for_model('gpt-3.5-turbo')
+
+
+for token_id in [100263, 99834]:  # special_tokens: 200257-100260 100276
+    try:
+        tokenizer.decode_tokens_bytes([token_id])
+    except:
+        pass
+
+    try:
+        tokenizer.decode_single_token_bytes(token_id)
+    except:
+        pass
+
+    try:
+        tokenizer.decode_bytes([token_id])
+    except:
+        pass
+
+
+

vocab/gpt_4/__init__.py

@@ -2,6 +2,7 @@
 
 import tiktoken
 from tiktoken import Encoding
+from utils.log_util import logger
 
 tokenizer = tiktoken.encoding_for_model('gpt-4')
 tokenizer.vocab_size = tokenizer.n_vocab