smetan commited on
Commit
1b84041
1 Parent(s): b747576

upd lstrip to default

Browse files
Files changed (1) hide show
  1. special_tokens_map.json +6 -6
special_tokens_map.json CHANGED
@@ -1,21 +1,21 @@
1
  {
2
  "bos_token": {
3
- "content": "<s>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true
4
  },
5
  "eos_token": {
6
- "content": "</s>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true
7
  },
8
  "unk_token": {
9
- "content": "<unk>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true
10
  },
11
  "sep_token": {
12
- "content": "</s>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true
13
  },
14
  "pad_token": {
15
- "content": "<pad>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true
16
  },
17
  "cls_token": {
18
- "content": "<s>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true
19
  },
20
  "mask_token": {
21
  "content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true
 
1
  {
2
  "bos_token": {
3
+ "content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true
4
  },
5
  "eos_token": {
6
+ "content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true
7
  },
8
  "unk_token": {
9
+ "content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true
10
  },
11
  "sep_token": {
12
+ "content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true
13
  },
14
  "pad_token": {
15
+ "content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true
16
  },
17
  "cls_token": {
18
+ "content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true
19
  },
20
  "mask_token": {
21
  "content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true