Hugging Face's logo

Xenova
/
Phi-3-mini-4k-instruct

Text Generation
Transformers.js
ONNX
phi3
ONNX
DML
ONNXRuntime
nlp
conversational
custom_code
Model card Files Community
5
Xenova HF staff commited on
Commit
3aa405a
•
1 Parent(s): b107548

Upload 6 files

Browse files
Files changed (5) hide show
  1. added_tokens.json +8 -35
  2. generation_config.json +1 -1
  3. special_tokens_map.json +1 -4
  4. tokenizer.json +8 -279
  5. tokenizer_config.json +1 -1
added_tokens.json CHANGED
@@ -1,40 +1,13 @@
1
  {
2
- "<|/code|>": 32014,
3
- "<|/data|>": 32033,
4
- "<|/inst|>": 32037,
5
- "<|/query|>": 32031,
6
- "<|/sys|>": 32035,
7
- "<|assistant_mask|>": 32017,
8
- "<|assistant|>": 32001,
9
- "<|calc|>": 32012,
10
- "<|code|>": 32013,
11
- "<|continue|>": 32009,
12
- "<|data|>": 32032,
13
- "<|diff_marker|>": 32025,
14
- "<|disc_sep|>": 32029,
15
- "<|disc_start|>": 32028,
16
- "<|disc_thread|><|query|>": 32030,
17
  "<|endoftext|>": 32000,
18
- "<|end|>": 32007,
19
- "<|fim_middle|>": 32021,
20
- "<|fim_prefix|>": 32020,
21
- "<|fim_suffix|>": 32022,
22
- "<|function_call|>": 32005,
23
- "<|function_list|>": 32011,
24
- "<|function_output|>": 32003,
25
- "<|ghissue|>": 32026,
26
- "<|ghreview|>": 32027,
27
- "<|inst|>": 32036,
28
- "<|ipynb_marker|>": 32024,
29
- "<|message|>": 32019,
30
- "<|meta_start|>": 32023,
31
- "<|raw|>": 32008,
32
- "<|resource|>": 32016,
33
- "<|start|>": 32018,
34
- "<|step|>": 32002,
35
- "<|summary|>": 32015,
36
  "<|system|>": 32006,
37
- "<|sys|>": 32034,
38
- "<|tag|>": 32004,
 
39
  "<|user|>": 32010
40
  }
 
1
  {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
  "<|endoftext|>": 32000,
3
+ "<|assistant|>": 32001,
4
+ "<|placeholder1|>": 32002,
5
+ "<|placeholder2|>": 32003,
6
+ "<|placeholder3|>": 32004,
7
+ "<|placeholder4|>": 32005,
 
 
 
 
 
 
 
 
 
 
 
 
 
8
  "<|system|>": 32006,
9
+ "<|end|>": 32007,
10
+ "<|placeholder5|>": 32008,
11
+ "<|placeholder6|>": 32009,
12
  "<|user|>": 32010
13
  }
generation_config.json CHANGED
@@ -8,4 +8,4 @@
8
  ],
9
  "pad_token_id": 32000,
10
  "transformers_version": "4.39.3"
11
- }
 
8
  ],
9
  "pad_token_id": 32000,
10
  "transformers_version": "4.39.3"
11
+ }
special_tokens_map.json CHANGED
@@ -1,7 +1,4 @@
1
  {
2
- "additional_special_tokens": [
3
- "<|/inst|>"
4
- ],
5
  "bos_token": {
6
  "content": "<s>",
7
  "lstrip": false,
@@ -30,4 +27,4 @@
30
  "rstrip": false,
31
  "single_word": false
32
  }
33
- }
 
1
  {
 
 
 
2
  "bos_token": {
3
  "content": "<s>",
4
  "lstrip": false,
 
27
  "rstrip": false,
28
  "single_word": false
29
  }
30
+ }
tokenizer.json CHANGED
@@ -50,7 +50,7 @@
50
  },
51
  {
52
  "id": 32002,
53
- "content": "<|step|>",
54
  "single_word": false,
55
  "lstrip": false,
56
  "rstrip": true,
@@ -59,7 +59,7 @@
59
  },
60
  {
61
  "id": 32003,
62
- "content": "<|function_output|>",
63
  "single_word": false,
64
  "lstrip": false,
65
  "rstrip": true,
@@ -68,7 +68,7 @@
68
  },
69
  {
70
  "id": 32004,
71
- "content": "<|tag|>",
72
  "single_word": false,
73
  "lstrip": false,
74
  "rstrip": true,
@@ -77,7 +77,7 @@
77
  },
78
  {
79
  "id": 32005,
80
- "content": "<|function_call|>",
81
  "single_word": false,
82
  "lstrip": false,
83
  "rstrip": true,
@@ -104,7 +104,7 @@
104
  },
105
  {
106
  "id": 32008,
107
- "content": "<|raw|>",
108
  "single_word": false,
109
  "lstrip": false,
110
  "rstrip": true,
@@ -113,7 +113,7 @@
113
  },
114
  {
115
  "id": 32009,
116
- "content": "<|continue|>",
117
  "single_word": false,
118
  "lstrip": false,
119
  "rstrip": true,
@@ -128,250 +128,7 @@
128
  "rstrip": true,
129
  "normalized": false,
130
  "special": true
131
- },
132
- {
133
- "id": 32011,
134
- "content": "<|function_list|>",
135
- "single_word": false,
136
- "lstrip": false,
137
- "rstrip": true,
138
- "normalized": false,
139
- "special": true
140
- },
141
- {
142
- "id": 32012,
143
- "content": "<|calc|>",
144
- "single_word": false,
145
- "lstrip": false,
146
- "rstrip": true,
147
- "normalized": false,
148
- "special": true
149
- },
150
- {
151
- "id": 32013,
152
- "content": "<|code|>",
153
- "single_word": false,
154
- "lstrip": false,
155
- "rstrip": true,
156
- "normalized": false,
157
- "special": true
158
- },
159
- {
160
- "id": 32014,
161
- "content": "<|/code|>",
162
- "single_word": false,
163
- "lstrip": false,
164
- "rstrip": true,
165
- "normalized": false,
166
- "special": true
167
- },
168
- {
169
- "id": 32015,
170
- "content": "<|summary|>",
171
- "single_word": false,
172
- "lstrip": false,
173
- "rstrip": true,
174
- "normalized": false,
175
- "special": true
176
- },
177
- {
178
- "id": 32016,
179
- "content": "<|resource|>",
180
- "single_word": false,
181
- "lstrip": false,
182
- "rstrip": true,
183
- "normalized": false,
184
- "special": true
185
- },
186
- {
187
- "id": 32017,
188
- "content": "<|assistant_mask|>",
189
- "single_word": false,
190
- "lstrip": false,
191
- "rstrip": true,
192
- "normalized": false,
193
- "special": true
194
- },
195
- {
196
- "id": 32018,
197
- "content": "<|start|>",
198
- "single_word": false,
199
- "lstrip": false,
200
- "rstrip": true,
201
- "normalized": false,
202
- "special": true
203
- },
204
- {
205
- "id": 32019,
206
- "content": "<|message|>",
207
- "single_word": false,
208
- "lstrip": false,
209
- "rstrip": true,
210
- "normalized": false,
211
- "special": true
212
- },
213
- {
214
- "id": 32020,
215
- "content": "<|fim_prefix|>",
216
- "single_word": false,
217
- "lstrip": false,
218
- "rstrip": true,
219
- "normalized": false,
220
- "special": true
221
- },
222
- {
223
- "id": 32021,
224
- "content": "<|fim_middle|>",
225
- "single_word": false,
226
- "lstrip": false,
227
- "rstrip": true,
228
- "normalized": false,
229
- "special": true
230
- },
231
- {
232
- "id": 32022,
233
- "content": "<|fim_suffix|>",
234
- "single_word": false,
235
- "lstrip": false,
236
- "rstrip": true,
237
- "normalized": false,
238
- "special": true
239
- },
240
- {
241
- "id": 32023,
242
- "content": "<|meta_start|>",
243
- "single_word": false,
244
- "lstrip": false,
245
- "rstrip": true,
246
- "normalized": false,
247
- "special": true
248
- },
249
- {
250
- "id": 32024,
251
- "content": "<|ipynb_marker|>",
252
- "single_word": false,
253
- "lstrip": false,
254
- "rstrip": true,
255
- "normalized": false,
256
- "special": true
257
- },
258
- {
259
- "id": 32025,
260
- "content": "<|diff_marker|>",
261
- "single_word": false,
262
- "lstrip": false,
263
- "rstrip": true,
264
- "normalized": false,
265
- "special": true
266
- },
267
- {
268
- "id": 32026,
269
- "content": "<|ghissue|>",
270
- "single_word": false,
271
- "lstrip": false,
272
- "rstrip": true,
273
- "normalized": false,
274
- "special": true
275
- },
276
- {
277
- "id": 32027,
278
- "content": "<|ghreview|>",
279
- "single_word": false,
280
- "lstrip": false,
281
- "rstrip": true,
282
- "normalized": false,
283
- "special": true
284
- },
285
- {
286
- "id": 32028,
287
- "content": "<|disc_start|>",
288
- "single_word": false,
289
- "lstrip": false,
290
- "rstrip": true,
291
- "normalized": false,
292
- "special": true
293
- },
294
- {
295
- "id": 32029,
296
- "content": "<|disc_sep|>",
297
- "single_word": false,
298
- "lstrip": false,
299
- "rstrip": true,
300
- "normalized": false,
301
- "special": true
302
- },
303
- {
304
- "id": 32030,
305
- "content": "<|disc_thread|><|query|>",
306
- "single_word": false,
307
- "lstrip": false,
308
- "rstrip": true,
309
- "normalized": false,
310
- "special": true
311
- },
312
- {
313
- "id": 32031,
314
- "content": "<|/query|>",
315
- "single_word": false,
316
- "lstrip": false,
317
- "rstrip": true,
318
- "normalized": false,
319
- "special": true
320
- },
321
- {
322
- "id": 32032,
323
- "content": "<|data|>",
324
- "single_word": false,
325
- "lstrip": false,
326
- "rstrip": true,
327
- "normalized": false,
328
- "special": true
329
- },
330
- {
331
- "id": 32033,
332
- "content": "<|/data|>",
333
- "single_word": false,
334
- "lstrip": false,
335
- "rstrip": true,
336
- "normalized": false,
337
- "special": true
338
- },
339
- {
340
- "id": 32034,
341
- "content": "<|sys|>",
342
- "single_word": false,
343
- "lstrip": false,
344
- "rstrip": true,
345
- "normalized": false,
346
- "special": true
347
- },
348
- {
349
- "id": 32035,
350
- "content": "<|/sys|>",
351
- "single_word": false,
352
- "lstrip": false,
353
- "rstrip": true,
354
- "normalized": false,
355
- "special": true
356
- },
357
- {
358
- "id": 32036,
359
- "content": "<|inst|>",
360
- "single_word": false,
361
- "lstrip": false,
362
- "rstrip": true,
363
- "normalized": false,
364
- "special": true
365
- },
366
- {
367
- "id": 32037,
368
- "content": "<|/inst|>",
369
- "single_word": false,
370
- "lstrip": false,
371
- "rstrip": true,
372
- "normalized": false,
373
- "special": true
374
- }
375
  ],
376
  "normalizer": {
377
  "type": "Sequence",
@@ -393,12 +150,6 @@
393
  "post_processor": {
394
  "type": "TemplateProcessing",
395
  "single": [
396
- {
397
- "SpecialToken": {
398
- "id": "<s>",
399
- "type_id": 0
400
- }
401
- },
402
  {
403
  "Sequence": {
404
  "id": "A",
@@ -407,24 +158,12 @@
407
  }
408
  ],
409
  "pair": [
410
- {
411
- "SpecialToken": {
412
- "id": "<s>",
413
- "type_id": 0
414
- }
415
- },
416
  {
417
  "Sequence": {
418
  "id": "A",
419
  "type_id": 0
420
  }
421
  },
422
- {
423
- "SpecialToken": {
424
- "id": "<s>",
425
- "type_id": 1
426
- }
427
- },
428
  {
429
  "Sequence": {
430
  "id": "B",
@@ -432,17 +171,7 @@
432
  }
433
  }
434
  ],
435
- "special_tokens": {
436
- "<s>": {
437
- "id": "<s>",
438
- "ids": [
439
- 1
440
- ],
441
- "tokens": [
442
- "<s>"
443
- ]
444
- }
445
- }
446
  },
447
  "decoder": {
448
  "type": "Sequence",
 
50
  },
51
  {
52
  "id": 32002,
53
+ "content": "<|placeholder1|>",
54
  "single_word": false,
55
  "lstrip": false,
56
  "rstrip": true,
 
59
  },
60
  {
61
  "id": 32003,
62
+ "content": "<|placeholder2|>",
63
  "single_word": false,
64
  "lstrip": false,
65
  "rstrip": true,
 
68
  },
69
  {
70
  "id": 32004,
71
+ "content": "<|placeholder3|>",
72
  "single_word": false,
73
  "lstrip": false,
74
  "rstrip": true,
 
77
  },
78
  {
79
  "id": 32005,
80
+ "content": "<|placeholder4|>",
81
  "single_word": false,
82
  "lstrip": false,
83
  "rstrip": true,
 
104
  },
105
  {
106
  "id": 32008,
107
+ "content": "<|placeholder5|>",
108
  "single_word": false,
109
  "lstrip": false,
110
  "rstrip": true,
 
113
  },
114
  {
115
  "id": 32009,
116
+ "content": "<|placeholder6|>",
117
  "single_word": false,
118
  "lstrip": false,
119
  "rstrip": true,
 
128
  "rstrip": true,
129
  "normalized": false,
130
  "special": true
131
+ }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
132
  ],
133
  "normalizer": {
134
  "type": "Sequence",
 
150
  "post_processor": {
151
  "type": "TemplateProcessing",
152
  "single": [
 
 
 
 
 
 
153
  {
154
  "Sequence": {
155
  "id": "A",
 
158
  }
159
  ],
160
  "pair": [
 
 
 
 
 
 
161
  {
162
  "Sequence": {
163
  "id": "A",
164
  "type_id": 0
165
  }
166
  },
 
 
 
 
 
 
167
  {
168
  "Sequence": {
169
  "id": "B",
 
171
  }
172
  }
173
  ],
174
+ "special_tokens": {}
 
 
 
 
 
 
 
 
 
 
175
  },
176
  "decoder": {
177
  "type": "Sequence",
tokenizer_config.json CHANGED
@@ -127,4 +127,4 @@
127
  "tokenizer_class": "LlamaTokenizer",
128
  "unk_token": "<unk>",
129
  "use_default_system_prompt": false
130
- }
 
127
  "tokenizer_class": "LlamaTokenizer",
128
  "unk_token": "<unk>",
129
  "use_default_system_prompt": false
130
+ }