HugoLaurencon HF staff commited on
Commit
3b387ac
1 Parent(s): 48fbb79

Update special_tokens_map.json

Browse files
Files changed (1) hide show
  1. special_tokens_map.json +6 -6
special_tokens_map.json CHANGED
@@ -3,21 +3,21 @@
3
  {
4
  "content": "<fake_token_around_image>",
5
  "lstrip": false,
6
- "normalized": true,
7
  "rstrip": false,
8
  "single_word": false
9
  },
10
  {
11
  "content": "<image>",
12
  "lstrip": false,
13
- "normalized": true,
14
  "rstrip": false,
15
  "single_word": false
16
  },
17
  {
18
  "content": "<end_of_utterance>",
19
  "lstrip": false,
20
- "normalized": true,
21
  "rstrip": false,
22
  "single_word": false
23
  }
@@ -25,14 +25,14 @@
25
  "bos_token": {
26
  "content": "<s>",
27
  "lstrip": false,
28
- "normalized": true,
29
  "rstrip": false,
30
  "single_word": false
31
  },
32
  "eos_token": {
33
  "content": "</s>",
34
  "lstrip": false,
35
- "normalized": true,
36
  "rstrip": false,
37
  "single_word": false
38
  },
@@ -40,7 +40,7 @@
40
  "unk_token": {
41
  "content": "<unk>",
42
  "lstrip": false,
43
- "normalized": true,
44
  "rstrip": false,
45
  "single_word": false
46
  }
 
3
  {
4
  "content": "<fake_token_around_image>",
5
  "lstrip": false,
6
+ "normalized": false,
7
  "rstrip": false,
8
  "single_word": false
9
  },
10
  {
11
  "content": "<image>",
12
  "lstrip": false,
13
+ "normalized": false,
14
  "rstrip": false,
15
  "single_word": false
16
  },
17
  {
18
  "content": "<end_of_utterance>",
19
  "lstrip": false,
20
+ "normalized": false,
21
  "rstrip": false,
22
  "single_word": false
23
  }
 
25
  "bos_token": {
26
  "content": "<s>",
27
  "lstrip": false,
28
+ "normalized": false,
29
  "rstrip": false,
30
  "single_word": false
31
  },
32
  "eos_token": {
33
  "content": "</s>",
34
  "lstrip": false,
35
+ "normalized": false,
36
  "rstrip": false,
37
  "single_word": false
38
  },
 
40
  "unk_token": {
41
  "content": "<unk>",
42
  "lstrip": false,
43
+ "normalized": false,
44
  "rstrip": false,
45
  "single_word": false
46
  }