TheBloke commited on
Commit
c9da849
1 Parent(s): a3fdd80

Update special_tokens_map.json

Browse files
Files changed (1) hide show
  1. special_tokens_map.json +3 -3
special_tokens_map.json CHANGED
@@ -1,13 +1,13 @@
1
  {
2
  "bos_token": {
3
- "content": "",
4
  "lstrip": false,
5
  "normalized": true,
6
  "rstrip": false,
7
  "single_word": false
8
  },
9
  "eos_token": {
10
- "content": "",
11
  "lstrip": false,
12
  "normalized": true,
13
  "rstrip": false,
@@ -15,7 +15,7 @@
15
  },
16
  "pad_token": "</s>",
17
  "unk_token": {
18
- "content": "",
19
  "lstrip": false,
20
  "normalized": true,
21
  "rstrip": false,
 
1
  {
2
  "bos_token": {
3
+ "content": "<s>",
4
  "lstrip": false,
5
  "normalized": true,
6
  "rstrip": false,
7
  "single_word": false
8
  },
9
  "eos_token": {
10
+ "content": "</s>",
11
  "lstrip": false,
12
  "normalized": true,
13
  "rstrip": false,
 
15
  },
16
  "pad_token": "</s>",
17
  "unk_token": {
18
+ "content": "<unk>",
19
  "lstrip": false,
20
  "normalized": true,
21
  "rstrip": false,