pszemraj commited on
Commit
66716bb
1 Parent(s): 2554ce0

fix lstrip on mask token

Browse files
special_tokens_map.json CHANGED
@@ -22,9 +22,9 @@
22
  },
23
  "mask_token": {
24
  "content": "<mask>",
25
- "lstrip": false,
26
  "normalized": true,
27
- "rstrip": true,
28
  "single_word": false
29
  },
30
  "pad_token": {
 
22
  },
23
  "mask_token": {
24
  "content": "<mask>",
25
+ "lstrip": true,
26
  "normalized": true,
27
+ "rstrip": false,
28
  "single_word": false
29
  },
30
  "pad_token": {
tokenizer.json CHANGED
@@ -86,8 +86,8 @@
86
  "id": 65003,
87
  "content": "<mask>",
88
  "single_word": false,
89
- "lstrip": false,
90
- "rstrip": true,
91
  "normalized": true,
92
  "special": true
93
  }
 
86
  "id": 65003,
87
  "content": "<mask>",
88
  "single_word": false,
89
+ "lstrip": true,
90
+ "rstrip": false,
91
  "normalized": true,
92
  "special": true
93
  }
tokenizer_config.json CHANGED
@@ -67,9 +67,9 @@
67
  },
68
  "65003": {
69
  "content": "<mask>",
70
- "lstrip": false,
71
  "normalized": true,
72
- "rstrip": true,
73
  "single_word": false,
74
  "special": true
75
  }
 
67
  },
68
  "65003": {
69
  "content": "<mask>",
70
+ "lstrip": true,
71
  "normalized": true,
72
+ "rstrip": false,
73
  "single_word": false,
74
  "special": true
75
  }