system HF staff commited on
Commit
8d77a37
1 Parent(s): 6966ba3

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -1 +1 @@
1
- {"bos_token": "[SEP]", "eos_token": "[SEP]", "unk_token": "[UNK]", "sep_token": "[SEP]", "cls_token": "[SEP]", "pad_token": "[PAD]", "mask_token": "[MASK]", "xprophetnet_tokenizer": true}
1
+ {"bos_token": "[SEP]", "eos_token": "[SEP]", "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "mask_token": "[MASK]", "cls_token": "[SEP]", "xprophetnet_tokenizer": true, "special_tokens_map_file": "/home/patrick/.cache/torch/transformers/f4df25e53eda78996e6da75d992659a82abde574b14ba2633a50a13767478bcc.39e4687b4b4265f2238315b221e397dae42a730321e6e3dbd2ef3ce2b94ac2c3", "tokenizer_file": null}