pszemraj commited on
Commit
2554ce0
1 Parent(s): c357007

Upload tokenizer

Browse files
special_tokens_map.json CHANGED
@@ -23,8 +23,8 @@
23
  "mask_token": {
24
  "content": "<mask>",
25
  "lstrip": false,
26
- "normalized": false,
27
- "rstrip": false,
28
  "single_word": false
29
  },
30
  "pad_token": {
 
23
  "mask_token": {
24
  "content": "<mask>",
25
  "lstrip": false,
26
+ "normalized": true,
27
+ "rstrip": true,
28
  "single_word": false
29
  },
30
  "pad_token": {
tokenizer.json CHANGED
@@ -87,8 +87,8 @@
87
  "content": "<mask>",
88
  "single_word": false,
89
  "lstrip": false,
90
- "rstrip": false,
91
- "normalized": false,
92
  "special": true
93
  }
94
  ],
 
87
  "content": "<mask>",
88
  "single_word": false,
89
  "lstrip": false,
90
+ "rstrip": true,
91
+ "normalized": true,
92
  "special": true
93
  }
94
  ],
tokenizer_config.json CHANGED
@@ -68,8 +68,8 @@
68
  "65003": {
69
  "content": "<mask>",
70
  "lstrip": false,
71
- "normalized": false,
72
- "rstrip": false,
73
  "single_word": false,
74
  "special": true
75
  }
 
68
  "65003": {
69
  "content": "<mask>",
70
  "lstrip": false,
71
+ "normalized": true,
72
+ "rstrip": true,
73
  "single_word": false,
74
  "special": true
75
  }