fix PyO3PanicException

tokenizer/tiktoken_patch.py

@@ -1,7 +1,7 @@
-
 from tiktoken import Encoding
 from utils.log_util import logger
 
+
 def decode(self, tokens, errors="replace", skip_special_tokens=False):
     """
     默认的decode，可能会报错，详见 decode_test.py
@@ -17,9 +17,13 @@ def decode(self, tokens, errors="replace", skip_special_tokens=False):
         "namereplace"
     """
     try:
+        print(tokens)
         decode_str = self._core_bpe.decode_bytes(tokens).decode("utf-8", errors=errors)
-    except Exception as e:
-        logger.error(f"{e} -> return 'null'")
+    except Exception as e:  # 捕捉不到 PyO3PanicException
+        logger.error(f"{e} for {tokens} -> return 'null'")
+        decode_str = "null"
+    except:
+        logger.error(f"unknow exception for {tokens} -> return 'null'")
         decode_str = "null"
     return decode_str
 
@@ -30,11 +34,14 @@ def convert_ids_to_tokens(self, tokens, skip_special_tokens=False):
     """
     try:
         return self.decode_tokens_bytes(tokens)
-    except Exception as e:
+    except Exception as e:  # 捕捉不到 PyO3PanicException
         # 什么要返回None？见zh_util.py
         # 16个空闲id, 100256 100261-100275
-        logger.error(e)
+        logger.error(f"{e} for {tokens} -> return None")
         return [None for _ in tokens]
+    except:
+        logger.error(f"unknow exception for {tokens} -> return None")
+        return
 
 
 def get_vocab(self, token_type="str"):
@@ -52,7 +59,6 @@ def get_vocab(self, token_type="str"):
                 continue
             # token_str = token_byte.decode("utf-8")
             vocab[token_byte] = i
-
         except UnicodeDecodeError:  # 773 UnicodeDecodeError
             unicode_decode_error_list.append((i, str(token_byte)))
             vocab[token_byte] = i