lhy
/

character-level-tokenizer

Model card Files Files and versions Community

lhy commited on Aug 5, 2023

Commit

9e262c9

•

1 Parent(s): 5189f82

Upload tokenizer

Files changed (2) hide show

tokenizer.json +1 -1
tokenizer_config.json +1 -1

tokenizer.json CHANGED Viewed

@@ -53,7 +53,7 @@
     "type": "BertNormalizer",
     "clean_text": true,
     "handle_chinese_chars": false,
-    "strip_accents": false,
     "lowercase": false
   },
   "pre_tokenizer": {

     "type": "BertNormalizer",
     "clean_text": true,
     "handle_chinese_chars": false,
+    "strip_accents": true,
     "lowercase": false
   },
   "pre_tokenizer": {

tokenizer_config.json CHANGED Viewed

@@ -9,7 +9,7 @@
   "never_split": null,
   "pad_token": "[PAD]",
   "sep_token": "[SEP]",
-  "strip_accents": false,
   "tokenize_chinese_chars": false,
   "tokenizer_class": "BertTokenizer",
   "unk_token": "[UNK]"

   "never_split": null,
   "pad_token": "[PAD]",
   "sep_token": "[SEP]",
+  "strip_accents": true,
   "tokenize_chinese_chars": false,
   "tokenizer_class": "BertTokenizer",
   "unk_token": "[UNK]"