ArthurZ HF staff commited on
Commit
e830d5a
1 Parent(s): 90028f4

Update special_tokens_map.json

Browse files

Hey! These should not be normalized! see https://huggingface.co/meta-llama/Llama-2-7b-chat-hf/commit/8cb66adf4be62bc67de0bb7e475efc43ca26167a

Files changed (1) hide show
  1. special_tokens_map.json +5 -5
special_tokens_map.json CHANGED
@@ -3,14 +3,14 @@
3
  {
4
  "content": "<fake_token_around_image>",
5
  "lstrip": false,
6
- "normalized": true,
7
  "rstrip": false,
8
  "single_word": false
9
  },
10
  {
11
  "content": "<image>",
12
  "lstrip": false,
13
- "normalized": true,
14
  "rstrip": false,
15
  "single_word": false
16
  }
@@ -18,14 +18,14 @@
18
  "bos_token": {
19
  "content": "<s>",
20
  "lstrip": false,
21
- "normalized": true,
22
  "rstrip": false,
23
  "single_word": false
24
  },
25
  "eos_token": {
26
  "content": "</s>",
27
  "lstrip": false,
28
- "normalized": true,
29
  "rstrip": false,
30
  "single_word": false
31
  },
@@ -33,7 +33,7 @@
33
  "unk_token": {
34
  "content": "<unk>",
35
  "lstrip": false,
36
- "normalized": true,
37
  "rstrip": false,
38
  "single_word": false
39
  }
 
3
  {
4
  "content": "<fake_token_around_image>",
5
  "lstrip": false,
6
+ "normalized": false,
7
  "rstrip": false,
8
  "single_word": false
9
  },
10
  {
11
  "content": "<image>",
12
  "lstrip": false,
13
+ "normalized": false,
14
  "rstrip": false,
15
  "single_word": false
16
  }
 
18
  "bos_token": {
19
  "content": "<s>",
20
  "lstrip": false,
21
+ "normalized": false,
22
  "rstrip": false,
23
  "single_word": false
24
  },
25
  "eos_token": {
26
  "content": "</s>",
27
  "lstrip": false,
28
+ "normalized": false,
29
  "rstrip": false,
30
  "single_word": false
31
  },
 
33
  "unk_token": {
34
  "content": "<unk>",
35
  "lstrip": false,
36
+ "normalized": false,
37
  "rstrip": false,
38
  "single_word": false
39
  }