Hugging Face's logo

Xenova
/
Phi-3-mini-4k-instruct

Text Generation
Transformers.js
ONNX
phi3
ONNX
DML
ONNXRuntime
nlp
conversational
custom_code
Model card Files Community
5

Update config.json

#4
by Xenova HF staff - opened
base: refs/heads/main
←
from: refs/pr/4
Discussion Files changed
+40
-660
Files changed (6) hide show
  1. added_tokens.json +8 -35
  2. config.json +15 -115
  3. generation_config.json +1 -1
  4. special_tokens_map.json +1 -4
  5. tokenizer.json +8 -279
  6. tokenizer_config.json +7 -226
added_tokens.json CHANGED
@@ -1,40 +1,13 @@
1
  {
2
- "<|/code|>": 32014,
3
- "<|/data|>": 32033,
4
- "<|/inst|>": 32037,
5
- "<|/query|>": 32031,
6
- "<|/sys|>": 32035,
7
- "<|assistant_mask|>": 32017,
8
- "<|assistant|>": 32001,
9
- "<|calc|>": 32012,
10
- "<|code|>": 32013,
11
- "<|continue|>": 32009,
12
- "<|data|>": 32032,
13
- "<|diff_marker|>": 32025,
14
- "<|disc_sep|>": 32029,
15
- "<|disc_start|>": 32028,
16
- "<|disc_thread|><|query|>": 32030,
17
  "<|endoftext|>": 32000,
18
- "<|end|>": 32007,
19
- "<|fim_middle|>": 32021,
20
- "<|fim_prefix|>": 32020,
21
- "<|fim_suffix|>": 32022,
22
- "<|function_call|>": 32005,
23
- "<|function_list|>": 32011,
24
- "<|function_output|>": 32003,
25
- "<|ghissue|>": 32026,
26
- "<|ghreview|>": 32027,
27
- "<|inst|>": 32036,
28
- "<|ipynb_marker|>": 32024,
29
- "<|message|>": 32019,
30
- "<|meta_start|>": 32023,
31
- "<|raw|>": 32008,
32
- "<|resource|>": 32016,
33
- "<|start|>": 32018,
34
- "<|step|>": 32002,
35
- "<|summary|>": 32015,
36
  "<|system|>": 32006,
37
- "<|sys|>": 32034,
38
- "<|tag|>": 32004,
 
39
  "<|user|>": 32010
40
  }
 
1
  {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
  "<|endoftext|>": 32000,
3
+ "<|assistant|>": 32001,
4
+ "<|placeholder1|>": 32002,
5
+ "<|placeholder2|>": 32003,
6
+ "<|placeholder3|>": 32004,
7
+ "<|placeholder4|>": 32005,
 
 
 
 
 
 
 
 
 
 
 
 
 
8
  "<|system|>": 32006,
9
+ "<|end|>": 32007,
10
+ "<|placeholder5|>": 32008,
11
+ "<|placeholder6|>": 32009,
12
  "<|user|>": 32010
13
  }
config.json CHANGED
@@ -1,138 +1,38 @@
1
  {
2
- "_name_or_path": "microsoft/Phi-3-mini-128k-instruct-onnx",
3
  "architectures": [
4
- "PhiLongRoPEForCausalLM"
5
  ],
6
  "attention_dropout": 0.0,
7
- "xauto_map": {
8
- "AutoConfig": "configuration_phi_longrope.PhiLongRoPEConfig",
9
- "AutoModelForCausalLM": "modeling_phi_longrope.PhiLongRoPEForCausalLM"
10
  },
11
  "bos_token_id": 1,
12
- "eos_token_id": 2,
 
13
  "hidden_act": "silu",
14
  "hidden_size": 3072,
15
  "initializer_range": 0.02,
16
  "intermediate_size": 8192,
17
- "max_position_embeddings": 131072,
18
  "model_type": "phi3",
19
- "xmodel_type": "phi_longrope",
20
  "num_attention_heads": 32,
21
  "num_hidden_layers": 32,
22
  "num_key_value_heads": 32,
23
  "original_max_position_embeddings": 4096,
 
 
24
  "rms_norm_eps": 1e-05,
25
- "rope_scaling": {
26
- "long_factor": [
27
- 1.0,
28
- 1.0,
29
- 1.0,
30
- 1.0,
31
- 1.0,
32
- 1.0,
33
- 1.0,
34
- 1.0,
35
- 1.0,
36
- 3.7391297817230225,
37
- 3.7699999999999756,
38
- 6.679999999999908,
39
- 11.956522941589355,
40
- 14.695652961730957,
41
- 16.649999999999697,
42
- 20.83000000000035,
43
- 22.91304588317871,
44
- 25.652175903320312,
45
- 28.391305923461914,
46
- 31.130435943603516,
47
- 33.869564056396484,
48
- 36.60869598388672,
49
- 39.34782409667969,
50
- 42.08695983886719,
51
- 44.826087951660156,
52
- 47.565216064453125,
53
- 50.30434799194336,
54
- 53.04347610473633,
55
- 55.78260803222656,
56
- 58.5217399597168,
57
- 61.26087188720703,
58
- 64.0,
59
- 64.0,
60
- 64.0,
61
- 64.0,
62
- 64.0,
63
- 64.0,
64
- 64.0,
65
- 64.0,
66
- 64.0,
67
- 64.0,
68
- 64.0,
69
- 64.0,
70
- 64.0,
71
- 64.0,
72
- 64.0,
73
- 64.0,
74
- 64.0
75
- ],
76
- "short_factor": [
77
- 1.0,
78
- 1.0,
79
- 1.0,
80
- 1.0,
81
- 1.05,
82
- 1.05,
83
- 1.1500000000000001,
84
- 1.2000000000000002,
85
- 1.2500000000000002,
86
- 1.5000000000000004,
87
- 1.8000000000000007,
88
- 2.25,
89
- 2.3000000000000007,
90
- 2.3500000000000014,
91
- 2.5,
92
- 2.5,
93
- 2.55,
94
- 2.5999999999999988,
95
- 2.6999999999999993,
96
- 2.6999999999999993,
97
- 2.8499999999999988,
98
- 2.8999999999999986,
99
- 2.9999999999999982,
100
- 2.9999999999999982,
101
- 2.9999999999999982,
102
- 3.099999999999998,
103
- 3.299999999999997,
104
- 3.399999999999997,
105
- 3.599999999999996,
106
- 3.649999999999996,
107
- 3.7999999999999954,
108
- 3.899999999999995,
109
- 3.899999999999995,
110
- 3.899999999999995,
111
- 4.099999999999994,
112
- 4.099999999999994,
113
- 4.099999999999994,
114
- 4.099999999999994,
115
- 4.099999999999994,
116
- 4.149999999999994,
117
- 4.149999999999994,
118
- 4.149999999999994,
119
- 4.149999999999994,
120
- 4.149999999999994,
121
- 4.199999999999994,
122
- 4.249999999999994,
123
- 4.299999999999994,
124
- 4.399999999999993
125
- ],
126
- "type": "longrope"
127
- },
128
  "rope_theta": 10000.0,
129
- "sliding_window": 131072,
130
  "tie_word_embeddings": false,
131
  "torch_dtype": "bfloat16",
132
- "transformers_version": "4.38.1",
133
  "transformers.js_config": {
134
  "kv_cache_dtype": "float16"
135
  },
136
  "use_cache": true,
137
- "vocab_size": 32038
138
- }
 
1
  {
2
+ "_name_or_path": "microsoft/Phi-3-mini-4k-instruct-onnx",
3
  "architectures": [
4
+ "Phi3ForCausalLM"
5
  ],
6
  "attention_dropout": 0.0,
7
+ "auto_map": {
8
+ "AutoConfig": "configuration_phi3.Phi3Config",
9
+ "AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM"
10
  },
11
  "bos_token_id": 1,
12
+ "embd_pdrop": 0.0,
13
+ "eos_token_id": 32000,
14
  "hidden_act": "silu",
15
  "hidden_size": 3072,
16
  "initializer_range": 0.02,
17
  "intermediate_size": 8192,
18
+ "max_position_embeddings": 4096,
19
  "model_type": "phi3",
 
20
  "num_attention_heads": 32,
21
  "num_hidden_layers": 32,
22
  "num_key_value_heads": 32,
23
  "original_max_position_embeddings": 4096,
24
+ "pad_token_id": 32000,
25
+ "resid_pdrop": 0.0,
26
  "rms_norm_eps": 1e-05,
27
+ "rope_scaling": null,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  "rope_theta": 10000.0,
29
+ "sliding_window": 2047,
30
  "tie_word_embeddings": false,
31
  "torch_dtype": "bfloat16",
32
+ "transformers_version": "4.39.3",
33
  "transformers.js_config": {
34
  "kv_cache_dtype": "float16"
35
  },
36
  "use_cache": true,
37
+ "vocab_size": 32064
38
+ }
generation_config.json CHANGED
@@ -8,4 +8,4 @@
8
  ],
9
  "pad_token_id": 32000,
10
  "transformers_version": "4.39.3"
11
- }
 
8
  ],
9
  "pad_token_id": 32000,
10
  "transformers_version": "4.39.3"
11
+ }
special_tokens_map.json CHANGED
@@ -1,7 +1,4 @@
1
  {
2
- "additional_special_tokens": [
3
- "<|/inst|>"
4
- ],
5
  "bos_token": {
6
  "content": "<s>",
7
  "lstrip": false,
@@ -30,4 +27,4 @@
30
  "rstrip": false,
31
  "single_word": false
32
  }
33
- }
 
1
  {
 
 
 
2
  "bos_token": {
3
  "content": "<s>",
4
  "lstrip": false,
 
27
  "rstrip": false,
28
  "single_word": false
29
  }
30
+ }
tokenizer.json CHANGED
@@ -50,7 +50,7 @@
50
  },
51
  {
52
  "id": 32002,
53
- "content": "<|step|>",
54
  "single_word": false,
55
  "lstrip": false,
56
  "rstrip": true,
@@ -59,7 +59,7 @@
59
  },
60
  {
61
  "id": 32003,
62
- "content": "<|function_output|>",
63
  "single_word": false,
64
  "lstrip": false,
65
  "rstrip": true,
@@ -68,7 +68,7 @@
68
  },
69
  {
70
  "id": 32004,
71
- "content": "<|tag|>",
72
  "single_word": false,
73
  "lstrip": false,
74
  "rstrip": true,
@@ -77,7 +77,7 @@
77
  },
78
  {
79
  "id": 32005,
80
- "content": "<|function_call|>",
81
  "single_word": false,
82
  "lstrip": false,
83
  "rstrip": true,
@@ -104,7 +104,7 @@
104
  },
105
  {
106
  "id": 32008,
107
- "content": "<|raw|>",
108
  "single_word": false,
109
  "lstrip": false,
110
  "rstrip": true,
@@ -113,7 +113,7 @@
113
  },
114
  {
115
  "id": 32009,
116
- "content": "<|continue|>",
117
  "single_word": false,
118
  "lstrip": false,
119
  "rstrip": true,
@@ -128,250 +128,7 @@
128
  "rstrip": true,
129
  "normalized": false,
130
  "special": true
131
- },
132
- {
133
- "id": 32011,
134
- "content": "<|function_list|>",
135
- "single_word": false,
136
- "lstrip": false,
137
- "rstrip": true,
138
- "normalized": false,
139
- "special": true
140
- },
141
- {
142
- "id": 32012,
143
- "content": "<|calc|>",
144
- "single_word": false,
145
- "lstrip": false,
146
- "rstrip": true,
147
- "normalized": false,
148
- "special": true
149
- },
150
- {
151
- "id": 32013,
152
- "content": "<|code|>",
153
- "single_word": false,
154
- "lstrip": false,
155
- "rstrip": true,
156
- "normalized": false,
157
- "special": true
158
- },
159
- {
160
- "id": 32014,
161
- "content": "<|/code|>",
162
- "single_word": false,
163
- "lstrip": false,
164
- "rstrip": true,
165
- "normalized": false,
166
- "special": true
167
- },
168
- {
169
- "id": 32015,
170
- "content": "<|summary|>",
171
- "single_word": false,
172
- "lstrip": false,
173
- "rstrip": true,
174
- "normalized": false,
175
- "special": true
176
- },
177
- {
178
- "id": 32016,
179
- "content": "<|resource|>",
180
- "single_word": false,
181
- "lstrip": false,
182
- "rstrip": true,
183
- "normalized": false,
184
- "special": true
185
- },
186
- {
187
- "id": 32017,
188
- "content": "<|assistant_mask|>",
189
- "single_word": false,
190
- "lstrip": false,
191
- "rstrip": true,
192
- "normalized": false,
193
- "special": true
194
- },
195
- {
196
- "id": 32018,
197
- "content": "<|start|>",
198
- "single_word": false,
199
- "lstrip": false,
200
- "rstrip": true,
201
- "normalized": false,
202
- "special": true
203
- },
204
- {
205
- "id": 32019,
206
- "content": "<|message|>",
207
- "single_word": false,
208
- "lstrip": false,
209
- "rstrip": true,
210
- "normalized": false,
211
- "special": true
212
- },
213
- {
214
- "id": 32020,
215
- "content": "<|fim_prefix|>",
216
- "single_word": false,
217
- "lstrip": false,
218
- "rstrip": true,
219
- "normalized": false,
220
- "special": true
221
- },
222
- {
223
- "id": 32021,
224
- "content": "<|fim_middle|>",
225
- "single_word": false,
226
- "lstrip": false,
227
- "rstrip": true,
228
- "normalized": false,
229
- "special": true
230
- },
231
- {
232
- "id": 32022,
233
- "content": "<|fim_suffix|>",
234
- "single_word": false,
235
- "lstrip": false,
236
- "rstrip": true,
237
- "normalized": false,
238
- "special": true
239
- },
240
- {
241
- "id": 32023,
242
- "content": "<|meta_start|>",
243
- "single_word": false,
244
- "lstrip": false,
245
- "rstrip": true,
246
- "normalized": false,
247
- "special": true
248
- },
249
- {
250
- "id": 32024,
251
- "content": "<|ipynb_marker|>",
252
- "single_word": false,
253
- "lstrip": false,
254
- "rstrip": true,
255
- "normalized": false,
256
- "special": true
257
- },
258
- {
259
- "id": 32025,
260
- "content": "<|diff_marker|>",
261
- "single_word": false,
262
- "lstrip": false,
263
- "rstrip": true,
264
- "normalized": false,
265
- "special": true
266
- },
267
- {
268
- "id": 32026,
269
- "content": "<|ghissue|>",
270
- "single_word": false,
271
- "lstrip": false,
272
- "rstrip": true,
273
- "normalized": false,
274
- "special": true
275
- },
276
- {
277
- "id": 32027,
278
- "content": "<|ghreview|>",
279
- "single_word": false,
280
- "lstrip": false,
281
- "rstrip": true,
282
- "normalized": false,
283
- "special": true
284
- },
285
- {
286
- "id": 32028,
287
- "content": "<|disc_start|>",
288
- "single_word": false,
289
- "lstrip": false,
290
- "rstrip": true,
291
- "normalized": false,
292
- "special": true
293
- },
294
- {
295
- "id": 32029,
296
- "content": "<|disc_sep|>",
297
- "single_word": false,
298
- "lstrip": false,
299
- "rstrip": true,
300
- "normalized": false,
301
- "special": true
302
- },
303
- {
304
- "id": 32030,
305
- "content": "<|disc_thread|><|query|>",
306
- "single_word": false,
307
- "lstrip": false,
308
- "rstrip": true,
309
- "normalized": false,
310
- "special": true
311
- },
312
- {
313
- "id": 32031,
314
- "content": "<|/query|>",
315
- "single_word": false,
316
- "lstrip": false,
317
- "rstrip": true,
318
- "normalized": false,
319
- "special": true
320
- },
321
- {
322
- "id": 32032,
323
- "content": "<|data|>",
324
- "single_word": false,
325
- "lstrip": false,
326
- "rstrip": true,
327
- "normalized": false,
328
- "special": true
329
- },
330
- {
331
- "id": 32033,
332
- "content": "<|/data|>",
333
- "single_word": false,
334
- "lstrip": false,
335
- "rstrip": true,
336
- "normalized": false,
337
- "special": true
338
- },
339
- {
340
- "id": 32034,
341
- "content": "<|sys|>",
342
- "single_word": false,
343
- "lstrip": false,
344
- "rstrip": true,
345
- "normalized": false,
346
- "special": true
347
- },
348
- {
349
- "id": 32035,
350
- "content": "<|/sys|>",
351
- "single_word": false,
352
- "lstrip": false,
353
- "rstrip": true,
354
- "normalized": false,
355
- "special": true
356
- },
357
- {
358
- "id": 32036,
359
- "content": "<|inst|>",
360
- "single_word": false,
361
- "lstrip": false,
362
- "rstrip": true,
363
- "normalized": false,
364
- "special": true
365
- },
366
- {
367
- "id": 32037,
368
- "content": "<|/inst|>",
369
- "single_word": false,
370
- "lstrip": false,
371
- "rstrip": true,
372
- "normalized": false,
373
- "special": true
374
- }
375
  ],
376
  "normalizer": {
377
  "type": "Sequence",
@@ -393,12 +150,6 @@
393
  "post_processor": {
394
  "type": "TemplateProcessing",
395
  "single": [
396
- {
397
- "SpecialToken": {
398
- "id": "<s>",
399
- "type_id": 0
400
- }
401
- },
402
  {
403
  "Sequence": {
404
  "id": "A",
@@ -407,24 +158,12 @@
407
  }
408
  ],
409
  "pair": [
410
- {
411
- "SpecialToken": {
412
- "id": "<s>",
413
- "type_id": 0
414
- }
415
- },
416
  {
417
  "Sequence": {
418
  "id": "A",
419
  "type_id": 0
420
  }
421
  },
422
- {
423
- "SpecialToken": {
424
- "id": "<s>",
425
- "type_id": 1
426
- }
427
- },
428
  {
429
  "Sequence": {
430
  "id": "B",
@@ -432,17 +171,7 @@
432
  }
433
  }
434
  ],
435
- "special_tokens": {
436
- "<s>": {
437
- "id": "<s>",
438
- "ids": [
439
- 1
440
- ],
441
- "tokens": [
442
- "<s>"
443
- ]
444
- }
445
- }
446
  },
447
  "decoder": {
448
  "type": "Sequence",
 
50
  },
51
  {
52
  "id": 32002,
53
+ "content": "<|placeholder1|>",
54
  "single_word": false,
55
  "lstrip": false,
56
  "rstrip": true,
 
59
  },
60
  {
61
  "id": 32003,
62
+ "content": "<|placeholder2|>",
63
  "single_word": false,
64
  "lstrip": false,
65
  "rstrip": true,
 
68
  },
69
  {
70
  "id": 32004,
71
+ "content": "<|placeholder3|>",
72
  "single_word": false,
73
  "lstrip": false,
74
  "rstrip": true,
 
77
  },
78
  {
79
  "id": 32005,
80
+ "content": "<|placeholder4|>",
81
  "single_word": false,
82
  "lstrip": false,
83
  "rstrip": true,
 
104
  },
105
  {
106
  "id": 32008,
107
+ "content": "<|placeholder5|>",
108
  "single_word": false,
109
  "lstrip": false,
110
  "rstrip": true,
 
113
  },
114
  {
115
  "id": 32009,
116
+ "content": "<|placeholder6|>",
117
  "single_word": false,
118
  "lstrip": false,
119
  "rstrip": true,
 
128
  "rstrip": true,
129
  "normalized": false,
130
  "special": true
131
+ }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
132
  ],
133
  "normalizer": {
134
  "type": "Sequence",
 
150
  "post_processor": {
151
  "type": "TemplateProcessing",
152
  "single": [
 
 
 
 
 
 
153
  {
154
  "Sequence": {
155
  "id": "A",
 
158
  }
159
  ],
160
  "pair": [
 
 
 
 
 
 
161
  {
162
  "Sequence": {
163
  "id": "A",
164
  "type_id": 0
165
  }
166
  },
 
 
 
 
 
 
167
  {
168
  "Sequence": {
169
  "id": "B",
 
171
  }
172
  }
173
  ],
174
+ "special_tokens": {}
 
 
 
 
 
 
 
 
 
 
175
  },
176
  "decoder": {
177
  "type": "Sequence",
tokenizer_config.json CHANGED
@@ -43,7 +43,7 @@
43
  "special": true
44
  },
45
  "32002": {
46
- "content": "<|step|>",
47
  "lstrip": false,
48
  "normalized": false,
49
  "rstrip": true,
@@ -51,7 +51,7 @@
51
  "special": true
52
  },
53
  "32003": {
54
- "content": "<|function_output|>",
55
  "lstrip": false,
56
  "normalized": false,
57
  "rstrip": true,
@@ -59,7 +59,7 @@
59
  "special": true
60
  },
61
  "32004": {
62
- "content": "<|tag|>",
63
  "lstrip": false,
64
  "normalized": false,
65
  "rstrip": true,
@@ -67,7 +67,7 @@
67
  "special": true
68
  },
69
  "32005": {
70
- "content": "<|function_call|>",
71
  "lstrip": false,
72
  "normalized": false,
73
  "rstrip": true,
@@ -91,7 +91,7 @@
91
  "special": true
92
  },
93
  "32008": {
94
- "content": "<|raw|>",
95
  "lstrip": false,
96
  "normalized": false,
97
  "rstrip": true,
@@ -99,7 +99,7 @@
99
  "special": true
100
  },
101
  "32009": {
102
- "content": "<|continue|>",
103
  "lstrip": false,
104
  "normalized": false,
105
  "rstrip": true,
@@ -113,229 +113,10 @@
113
  "rstrip": true,
114
  "single_word": false,
115
  "special": true
116
- },
117
- "32011": {
118
- "content": "<|function_list|>",
119
- "lstrip": false,
120
- "normalized": false,
121
- "rstrip": true,
122
- "single_word": false,
123
- "special": true
124
- },
125
- "32012": {
126
- "content": "<|calc|>",
127
- "lstrip": false,
128
- "normalized": false,
129
- "rstrip": true,
130
- "single_word": false,
131
- "special": true
132
- },
133
- "32013": {
134
- "content": "<|code|>",
135
- "lstrip": false,
136
- "normalized": false,
137
- "rstrip": true,
138
- "single_word": false,
139
- "special": true
140
- },
141
- "32014": {
142
- "content": "<|/code|>",
143
- "lstrip": false,
144
- "normalized": false,
145
- "rstrip": true,
146
- "single_word": false,
147
- "special": true
148
- },
149
- "32015": {
150
- "content": "<|summary|>",
151
- "lstrip": false,
152
- "normalized": false,
153
- "rstrip": true,
154
- "single_word": false,
155
- "special": true
156
- },
157
- "32016": {
158
- "content": "<|resource|>",
159
- "lstrip": false,
160
- "normalized": false,
161
- "rstrip": true,
162
- "single_word": false,
163
- "special": true
164
- },
165
- "32017": {
166
- "content": "<|assistant_mask|>",
167
- "lstrip": false,
168
- "normalized": false,
169
- "rstrip": true,
170
- "single_word": false,
171
- "special": true
172
- },
173
- "32018": {
174
- "content": "<|start|>",
175
- "lstrip": false,
176
- "normalized": false,
177
- "rstrip": true,
178
- "single_word": false,
179
- "special": true
180
- },
181
- "32019": {
182
- "content": "<|message|>",
183
- "lstrip": false,
184
- "normalized": false,
185
- "rstrip": true,
186
- "single_word": false,
187
- "special": true
188
- },
189
- "32020": {
190
- "content": "<|fim_prefix|>",
191
- "lstrip": false,
192
- "normalized": false,
193
- "rstrip": true,
194
- "single_word": false,
195
- "special": true
196
- },
197
- "32021": {
198
- "content": "<|fim_middle|>",
199
- "lstrip": false,
200
- "normalized": false,
201
- "rstrip": true,
202
- "single_word": false,
203
- "special": true
204
- },
205
- "32022": {
206
- "content": "<|fim_suffix|>",
207
- "lstrip": false,
208
- "normalized": false,
209
- "rstrip": true,
210
- "single_word": false,
211
- "special": true
212
- },
213
- "32023": {
214
- "content": "<|meta_start|>",
215
- "lstrip": false,
216
- "normalized": false,
217
- "rstrip": true,
218
- "single_word": false,
219
- "special": true
220
- },
221
- "32024": {
222
- "content": "<|ipynb_marker|>",
223
- "lstrip": false,
224
- "normalized": false,
225
- "rstrip": true,
226
- "single_word": false,
227
- "special": true
228
- },
229
- "32025": {
230
- "content": "<|diff_marker|>",
231
- "lstrip": false,
232
- "normalized": false,
233
- "rstrip": true,
234
- "single_word": false,
235
- "special": true
236
- },
237
- "32026": {
238
- "content": "<|ghissue|>",
239
- "lstrip": false,
240
- "normalized": false,
241
- "rstrip": true,
242
- "single_word": false,
243
- "special": true
244
- },
245
- "32027": {
246
- "content": "<|ghreview|>",
247
- "lstrip": false,
248
- "normalized": false,
249
- "rstrip": true,
250
- "single_word": false,
251
- "special": true
252
- },
253
- "32028": {
254
- "content": "<|disc_start|>",
255
- "lstrip": false,
256
- "normalized": false,
257
- "rstrip": true,
258
- "single_word": false,
259
- "special": true
260
- },
261
- "32029": {
262
- "content": "<|disc_sep|>",
263
- "lstrip": false,
264
- "normalized": false,
265
- "rstrip": true,
266
- "single_word": false,
267
- "special": true
268
- },
269
- "32030": {
270
- "content": "<|disc_thread|><|query|>",
271
- "lstrip": false,
272
- "normalized": false,
273
- "rstrip": true,
274
- "single_word": false,
275
- "special": true
276
- },
277
- "32031": {
278
- "content": "<|/query|>",
279
- "lstrip": false,
280
- "normalized": false,
281
- "rstrip": true,
282
- "single_word": false,
283
- "special": true
284
- },
285
- "32032": {
286
- "content": "<|data|>",
287
- "lstrip": false,
288
- "normalized": false,
289
- "rstrip": true,
290
- "single_word": false,
291
- "special": true
292
- },
293
- "32033": {
294
- "content": "<|/data|>",
295
- "lstrip": false,
296
- "normalized": false,
297
- "rstrip": true,
298
- "single_word": false,
299
- "special": true
300
- },
301
- "32034": {
302
- "content": "<|sys|>",
303
- "lstrip": false,
304
- "normalized": false,
305
- "rstrip": true,
306
- "single_word": false,
307
- "special": true
308
- },
309
- "32035": {
310
- "content": "<|/sys|>",
311
- "lstrip": false,
312
- "normalized": false,
313
- "rstrip": true,
314
- "single_word": false,
315
- "special": true
316
- },
317
- "32036": {
318
- "content": "<|inst|>",
319
- "lstrip": false,
320
- "normalized": false,
321
- "rstrip": true,
322
- "single_word": false,
323
- "special": true
324
- },
325
- "32037": {
326
- "content": "<|/inst|>",
327
- "lstrip": false,
328
- "normalized": false,
329
- "rstrip": true,
330
- "single_word": false,
331
- "special": true
332
  }
333
  },
334
- "additional_special_tokens": [
335
- "<|/inst|>"
336
- ],
337
  "bos_token": "<s>",
338
- "chat_template": "{{ bos_token }}{% for message in messages %}{{'<|' + message['role'] + '|>' + '\n' + message['content'] + '<|end|>\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
339
  "clean_up_tokenization_spaces": false,
340
  "eos_token": "<|endoftext|>",
341
  "legacy": false,
 
43
  "special": true
44
  },
45
  "32002": {
46
+ "content": "<|placeholder1|>",
47
  "lstrip": false,
48
  "normalized": false,
49
  "rstrip": true,
 
51
  "special": true
52
  },
53
  "32003": {
54
+ "content": "<|placeholder2|>",
55
  "lstrip": false,
56
  "normalized": false,
57
  "rstrip": true,
 
59
  "special": true
60
  },
61
  "32004": {
62
+ "content": "<|placeholder3|>",
63
  "lstrip": false,
64
  "normalized": false,
65
  "rstrip": true,
 
67
  "special": true
68
  },
69
  "32005": {
70
+ "content": "<|placeholder4|>",
71
  "lstrip": false,
72
  "normalized": false,
73
  "rstrip": true,
 
91
  "special": true
92
  },
93
  "32008": {
94
+ "content": "<|placeholder5|>",
95
  "lstrip": false,
96
  "normalized": false,
97
  "rstrip": true,
 
99
  "special": true
100
  },
101
  "32009": {
102
+ "content": "<|placeholder6|>",
103
  "lstrip": false,
104
  "normalized": false,
105
  "rstrip": true,
 
113
  "rstrip": true,
114
  "single_word": false,
115
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
116
  }
117
  },
 
 
 
118
  "bos_token": "<s>",
119
+ "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}",
120
  "clean_up_tokenization_spaces": false,
121
  "eos_token": "<|endoftext|>",
122
  "legacy": false,