xu-song commited on
Commit
ae282a4
1 Parent(s): a1b0cd0
Files changed (1) hide show
  1. util.py +2 -1
util.py CHANGED
@@ -42,7 +42,8 @@ def tokenize(text, tokenizer_type, color_num=5):
42
  token_bytes = bytes(token_str, "utf-8")
43
  # json_dumps = json.dumps(token_str)
44
  else:
45
- return
 
46
 
47
  # ⭐
48
  table.append(
 
42
  token_bytes = bytes(token_str, "utf-8")
43
  # json_dumps = json.dumps(token_str)
44
  else:
45
+ logger.error(f"typeError for token {token_id} with {type(token)} " + json.dumps({"text": text, "tokenizer_type": tokenizer_type}, ensure_ascii=False))
46
+ continue
47
 
48
  # ⭐
49
  table.append(