system HF staff commited on
Commit
076be04
1 Parent(s): 2b1c87a

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -1 +1 @@
1
- {"bos_token": "[SEP]", "eos_token": "[SEP]", "unk_token": "[UNK]", "sep_token": "[SEP]", "cls_token": "[SEP]", "pad_token": "[PAD]", "mask_token": "[MASK]", "xprophetnet_tokenizer": true}
1
+ {"bos_token": "[SEP]", "eos_token": "[SEP]", "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "mask_token": "[MASK]", "cls_token": "[SEP]", "xprophetnet_tokenizer": true, "special_tokens_map_file": "/home/patrick/.cache/torch/transformers/556e0f8e88bd0068c08bfb686c44642a49cd93c6f73d09d48746de507490238a.39e4687b4b4265f2238315b221e397dae42a730321e6e3dbd2ef3ce2b94ac2c3", "tokenizer_file": null}