"""
超牛逼，没有OOV
"""

from tokenizers import Tokenizer
from data_sample.oov_base import space_tokens, jd_vocab_tokens, docs

tokenizer = Tokenizer.from_file("20B_tokenizer.json")

def test_oov():
    for line in space_tokens + jd_vocab_tokens + docs:
        tokens = tokenizer.encode(line)
        decode_line = tokenizer.decode(tokens.ids)
        if line != decode_line:
            print("原句：", line)
            print("解码:", decode_line)


if __name__ == "__main__":
    test_oov()