pacovaldez commited on
Commit
f802dec
1 Parent(s): c40ce8f

set regular config

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -17
tokenizer_config.json CHANGED
@@ -1,17 +1 @@
1
- {
2
- "cls_token": "[CLS]",
3
- "do_basic_tokenize": true,
4
- "do_lower_case": false,
5
- "mask_token": "[MASK]",
6
- "max_len": 512,
7
- "model_max_length": 512,
8
- "name_or_path": "deepset/bert-base-cased-squad2",
9
- "never_split": null,
10
- "pad_token": "[PAD]",
11
- "sep_token": "[SEP]",
12
- "special_tokens_map_file": "/root/.cache/huggingface/hub/models--deepset--bert-base-cased-squad2/snapshots/68ffb81c17a0238f5409cc417cadb3d013378055/special_tokens_map.json",
13
- "strip_accents": null,
14
- "tokenize_chinese_chars": true,
15
- "tokenizer_class": "BertTokenizer",
16
- "unk_token": "[UNK]"
17
- }
 
1
+ {"do_lower_case": false, "max_len": 512, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}