tokenizer-arena / vocab /bloom /test_oov_bloom.py
xu-song's picture
update
751936e
raw
history blame
755 Bytes
import difflib
import webbrowser
from transformers import AutoTokenizer
from data_sample.oov_base import space_tokens, jd_vocab_tokens, docs
tokenizer = AutoTokenizer.from_pretrained("tokenizer")
def test_oov():
d = difflib.HtmlDiff(wrapcolumn=50)
raw_lines = []
decode_lines = []
for line in space_tokens + jd_vocab_tokens + docs:
tokens = tokenizer.encode(line)
decode_line = tokenizer.decode(tokens)
if line != decode_line:
raw_lines.append(line)
decode_lines.append(decode_line)
q = d.make_file(raw_lines, decode_lines)
with open('diff.html', 'w', encoding="utf-8") as f_new:
f_new.write(q)
webbrowser.open('diff.html')
if __name__ == "__main__":
test_oov()