kalytm commited on
Commit
510af2c
1 Parent(s): 4ff387e

Upload tokenizer

Browse files
Files changed (3) hide show
  1. special_tokens_map.json +3 -3
  2. tokenizer.json +28 -6
  3. tokenizer_config.json +27 -7
special_tokens_map.json CHANGED
@@ -35,14 +35,14 @@
35
  "<|extra0|>"
36
  ],
37
  "bos_token": {
38
- "content": "<|endoftext|>",
39
  "lstrip": false,
40
  "normalized": false,
41
  "rstrip": false,
42
  "single_word": false
43
  },
44
  "eos_token": {
45
- "content": "<|endoftext|>",
46
  "lstrip": false,
47
  "normalized": false,
48
  "rstrip": false,
@@ -56,7 +56,7 @@
56
  "single_word": false
57
  },
58
  "unk_token": {
59
- "content": "<|endoftext|>",
60
  "lstrip": false,
61
  "normalized": false,
62
  "rstrip": false,
 
35
  "<|extra0|>"
36
  ],
37
  "bos_token": {
38
+ "content": "<s>",
39
  "lstrip": false,
40
  "normalized": false,
41
  "rstrip": false,
42
  "single_word": false
43
  },
44
  "eos_token": {
45
+ "content": "</s>",
46
  "lstrip": false,
47
  "normalized": false,
48
  "rstrip": false,
 
56
  "single_word": false
57
  },
58
  "unk_token": {
59
+ "content": "<unk>",
60
  "lstrip": false,
61
  "normalized": false,
62
  "rstrip": false,
tokenizer.json CHANGED
@@ -1,11 +1,6 @@
1
  {
2
  "version": "1.0",
3
- "truncation": {
4
- "direction": "Right",
5
- "max_length": 2048,
6
- "strategy": "LongestFirst",
7
- "stride": 0
8
- },
9
  "padding": null,
10
  "added_tokens": [
11
  {
@@ -304,6 +299,33 @@
304
  "rstrip": false,
305
  "normalized": false,
306
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
307
  }
308
  ],
309
  "normalizer": null,
 
1
  {
2
  "version": "1.0",
3
+ "truncation": null,
 
 
 
 
 
4
  "padding": null,
5
  "added_tokens": [
6
  {
 
299
  "rstrip": false,
300
  "normalized": false,
301
  "special": true
302
+ },
303
+ {
304
+ "id": 100289,
305
+ "content": "<s>",
306
+ "single_word": false,
307
+ "lstrip": false,
308
+ "rstrip": false,
309
+ "normalized": false,
310
+ "special": true
311
+ },
312
+ {
313
+ "id": 100290,
314
+ "content": "</s>",
315
+ "single_word": false,
316
+ "lstrip": false,
317
+ "rstrip": false,
318
+ "normalized": false,
319
+ "special": true
320
+ },
321
+ {
322
+ "id": 100291,
323
+ "content": "<unk>",
324
+ "single_word": false,
325
+ "lstrip": false,
326
+ "rstrip": false,
327
+ "normalized": false,
328
+ "special": true
329
  }
330
  ],
331
  "normalizer": null,
tokenizer_config.json CHANGED
@@ -264,6 +264,30 @@
264
  "rstrip": false,
265
  "single_word": false,
266
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
267
  }
268
  },
269
  "additional_special_tokens": [
@@ -301,16 +325,12 @@
301
  "<|reg7|>",
302
  "<|extra0|>"
303
  ],
304
- "bos_token": "<|endoftext|>",
305
  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
306
  "clean_up_tokenization_spaces": true,
307
- "eos_token": "<|endoftext|>",
308
- "max_length": 2048,
309
  "model_max_length": 2048,
310
  "pad_token": "<|endoftext|>",
311
- "stride": 0,
312
  "tokenizer_class": "GPT2Tokenizer",
313
- "truncation_side": "right",
314
- "truncation_strategy": "longest_first",
315
- "unk_token": "<|endoftext|>"
316
  }
 
264
  "rstrip": false,
265
  "single_word": false,
266
  "special": true
267
+ },
268
+ "100289": {
269
+ "content": "<s>",
270
+ "lstrip": false,
271
+ "normalized": false,
272
+ "rstrip": false,
273
+ "single_word": false,
274
+ "special": true
275
+ },
276
+ "100290": {
277
+ "content": "</s>",
278
+ "lstrip": false,
279
+ "normalized": false,
280
+ "rstrip": false,
281
+ "single_word": false,
282
+ "special": true
283
+ },
284
+ "100291": {
285
+ "content": "<unk>",
286
+ "lstrip": false,
287
+ "normalized": false,
288
+ "rstrip": false,
289
+ "single_word": false,
290
+ "special": true
291
  }
292
  },
293
  "additional_special_tokens": [
 
325
  "<|reg7|>",
326
  "<|extra0|>"
327
  ],
328
+ "bos_token": "<s>",
329
  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
330
  "clean_up_tokenization_spaces": true,
331
+ "eos_token": "</s>",
 
332
  "model_max_length": 2048,
333
  "pad_token": "<|endoftext|>",
 
334
  "tokenizer_class": "GPT2Tokenizer",
335
+ "unk_token": "<unk>"
 
 
336
  }