Commit
•
17f3102
1
Parent(s):
d46c317
Upload tokenizer
Browse files- added_tokens.json +1 -3
- special_tokens_map.json +2 -2
- tokenizer.json +0 -18
- tokenizer_config.json +2 -18
added_tokens.json
CHANGED
@@ -36,7 +36,5 @@
|
|
36 |
" ": 50260,
|
37 |
" ": 50259,
|
38 |
" ": 50258,
|
39 |
-
" ": 50257
|
40 |
-
"<|im_end|>": 50296,
|
41 |
-
"<|im_start|>": 50295
|
42 |
}
|
|
|
36 |
" ": 50260,
|
37 |
" ": 50259,
|
38 |
" ": 50258,
|
39 |
+
" ": 50257
|
|
|
|
|
40 |
}
|
special_tokens_map.json
CHANGED
@@ -7,13 +7,13 @@
|
|
7 |
"single_word": false
|
8 |
},
|
9 |
"eos_token": {
|
10 |
-
"content": "<|
|
11 |
"lstrip": false,
|
12 |
"normalized": false,
|
13 |
"rstrip": false,
|
14 |
"single_word": false
|
15 |
},
|
16 |
-
"pad_token": "
|
17 |
"unk_token": {
|
18 |
"content": "<|endoftext|>",
|
19 |
"lstrip": false,
|
|
|
7 |
"single_word": false
|
8 |
},
|
9 |
"eos_token": {
|
10 |
+
"content": "<|endoftext|>",
|
11 |
"lstrip": false,
|
12 |
"normalized": false,
|
13 |
"rstrip": false,
|
14 |
"single_word": false
|
15 |
},
|
16 |
+
"pad_token": "<|endoftext|>",
|
17 |
"unk_token": {
|
18 |
"content": "<|endoftext|>",
|
19 |
"lstrip": false,
|
tokenizer.json
CHANGED
@@ -353,24 +353,6 @@
|
|
353 |
"rstrip": false,
|
354 |
"normalized": true,
|
355 |
"special": false
|
356 |
-
},
|
357 |
-
{
|
358 |
-
"id": 50295,
|
359 |
-
"content": "<|im_start|>",
|
360 |
-
"single_word": false,
|
361 |
-
"lstrip": false,
|
362 |
-
"rstrip": false,
|
363 |
-
"normalized": true,
|
364 |
-
"special": false
|
365 |
-
},
|
366 |
-
{
|
367 |
-
"id": 50296,
|
368 |
-
"content": "<|im_end|>",
|
369 |
-
"single_word": false,
|
370 |
-
"lstrip": false,
|
371 |
-
"rstrip": false,
|
372 |
-
"normalized": false,
|
373 |
-
"special": true
|
374 |
}
|
375 |
],
|
376 |
"normalizer": null,
|
|
|
353 |
"rstrip": false,
|
354 |
"normalized": true,
|
355 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
356 |
}
|
357 |
],
|
358 |
"normalizer": null,
|
tokenizer_config.json
CHANGED
@@ -312,29 +312,13 @@
|
|
312 |
"rstrip": false,
|
313 |
"single_word": false,
|
314 |
"special": false
|
315 |
-
},
|
316 |
-
"50295": {
|
317 |
-
"content": "<|im_start|>",
|
318 |
-
"lstrip": false,
|
319 |
-
"normalized": true,
|
320 |
-
"rstrip": false,
|
321 |
-
"single_word": false,
|
322 |
-
"special": false
|
323 |
-
},
|
324 |
-
"50296": {
|
325 |
-
"content": "<|im_end|>",
|
326 |
-
"lstrip": false,
|
327 |
-
"normalized": false,
|
328 |
-
"rstrip": false,
|
329 |
-
"single_word": false,
|
330 |
-
"special": true
|
331 |
}
|
332 |
},
|
333 |
"bos_token": "<|endoftext|>",
|
334 |
"clean_up_tokenization_spaces": true,
|
335 |
-
"eos_token": "<|
|
336 |
"model_max_length": 2048,
|
337 |
-
"pad_token": "
|
338 |
"tokenizer_class": "CodeGenTokenizer",
|
339 |
"unk_token": "<|endoftext|>"
|
340 |
}
|
|
|
312 |
"rstrip": false,
|
313 |
"single_word": false,
|
314 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
315 |
}
|
316 |
},
|
317 |
"bos_token": "<|endoftext|>",
|
318 |
"clean_up_tokenization_spaces": true,
|
319 |
+
"eos_token": "<|endoftext|>",
|
320 |
"model_max_length": 2048,
|
321 |
+
"pad_token": "<|endoftext|>",
|
322 |
"tokenizer_class": "CodeGenTokenizer",
|
323 |
"unk_token": "<|endoftext|>"
|
324 |
}
|