SaulLu commited on
Commit
6513312
1 Parent(s): 32f308b

add modified tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +7 -1
tokenizer.json CHANGED
@@ -49,7 +49,13 @@
49
  "normalized": false
50
  }
51
  ],
52
- "normalizer": null,
 
 
 
 
 
 
53
  "pre_tokenizer": { "type": "BertPreTokenizer" },
54
  "post_processor": {
55
  "type": "TemplateProcessing",
49
  "normalized": false
50
  }
51
  ],
52
+ "normalizer": {
53
+ "type": "BertNormalizer",
54
+ "clean_text": true,
55
+ "handle_chinese_chars": false,
56
+ "strip_accents": null,
57
+ "lowercase": false
58
+ },
59
  "pre_tokenizer": { "type": "BertPreTokenizer" },
60
  "post_processor": {
61
  "type": "TemplateProcessing",