Joseph717171 commited on
Commit
511bd67
1 Parent(s): c392d47

Update added_tokens.json

Browse files
Files changed (1) hide show
  1. added_tokens.json +11 -11
added_tokens.json CHANGED
@@ -1,4 +1,14 @@
1
  {
 
 
 
 
 
 
 
 
 
 
2
  "<pad10>": 32010,
3
  "<pad11>": 32011,
4
  "<pad12>": 32012,
@@ -19,16 +29,6 @@
19
  "<pad27>": 32027,
20
  "<pad28>": 32028,
21
  "<pad29>": 32029,
22
- "<pad2>": 32002,
23
  "<pad30>": 32030,
24
- "<pad31>": 32031,
25
- "<pad3>": 32003,
26
- "<pad4>": 32004,
27
- "<pad5>": 32005,
28
- "<pad6>": 32006,
29
- "<pad7>": 32007,
30
- "<pad8>": 32008,
31
- "<pad9>": 32009,
32
- "<|im_end|>": 32000,
33
- "<|im_start|>": 32001
34
  }
 
1
  {
2
+ "<|im_end|>": 32000,
3
+ "<|im_start|>": 32001,
4
+ "<pad2>": 32002,
5
+ "<pad3>": 32003,
6
+ "<pad4>": 32004,
7
+ "<pad5>": 32005,
8
+ "<pad6>": 32006,
9
+ "<pad7>": 32007,
10
+ "<pad8>": 32008,
11
+ "<pad9>": 32009,
12
  "<pad10>": 32010,
13
  "<pad11>": 32011,
14
  "<pad12>": 32012,
 
29
  "<pad27>": 32027,
30
  "<pad28>": 32028,
31
  "<pad29>": 32029,
 
32
  "<pad30>": 32030,
33
+ "<pad31>": 32031
 
 
 
 
 
 
 
 
 
34
  }