bertweet-base-tweetner7-continuous / tokenizer_config.json
asahi417's picture
add tokenizer
d1642f1
raw history blame
No virus
535 Bytes
{"normalization": false, "bos_token": "<s>", "eos_token": "</s>", "sep_token": "</s>", "cls_token": "<s>", "unk_token": "<unk>", "pad_token": "<pad>", "mask_token": "<mask>", "model_max_length": 128, "name_or_path": "cner_output/model/baseline_2021/bertweet_base_continuous/best_model", "special_tokens_map_file": "/home/c.c2042013/.cache/huggingface/transformers/bb0c2777b6eb6a05be4dc2b2adf35e25a8a273a43015576079f4b979bc928f57.0dc5b1041f62041ebbd23b1297f2f573769d5c97d8b7c28180ec86b8f6185aa8", "tokenizer_class": "BertweetTokenizer"}