update
Browse files
util.py
CHANGED
@@ -42,7 +42,8 @@ def tokenize(text, tokenizer_type, color_num=5):
|
|
42 |
token_bytes = bytes(token_str, "utf-8")
|
43 |
# json_dumps = json.dumps(token_str)
|
44 |
else:
|
45 |
-
|
|
|
46 |
|
47 |
# ⭐
|
48 |
table.append(
|
|
|
42 |
token_bytes = bytes(token_str, "utf-8")
|
43 |
# json_dumps = json.dumps(token_str)
|
44 |
else:
|
45 |
+
logger.error(f"typeError for token {token_id} with {type(token)} " + json.dumps({"text": text, "tokenizer_type": tokenizer_type}, ensure_ascii=False))
|
46 |
+
continue
|
47 |
|
48 |
# ⭐
|
49 |
table.append(
|