ykhwang commited on
Commit
6ad0fe6
1 Parent(s): 7474caf

Fixed tokenizer_config.json

Browse files

Fixed tokenizer_config.json's inconsistent BOS, UNK token contents.

Files changed (1) hide show
  1. tokenizer_config.json +2 -2
tokenizer_config.json CHANGED
@@ -3,7 +3,7 @@
3
  "add_prefix_space": false,
4
  "bos_token": {
5
  "__type": "AddedToken",
6
- "content": "<|endoftext|>",
7
  "lstrip": false,
8
  "normalized": true,
9
  "rstrip": false,
@@ -25,7 +25,7 @@
25
  "tokenizer_class": "GPT2Tokenizer",
26
  "unk_token": {
27
  "__type": "AddedToken",
28
- "content": "<|endoftext|>",
29
  "lstrip": false,
30
  "normalized": true,
31
  "rstrip": false,
 
3
  "add_prefix_space": false,
4
  "bos_token": {
5
  "__type": "AddedToken",
6
+ "content": "<||bos||>",
7
  "lstrip": false,
8
  "normalized": true,
9
  "rstrip": false,
 
25
  "tokenizer_class": "GPT2Tokenizer",
26
  "unk_token": {
27
  "__type": "AddedToken",
28
+ "content": "<||unk||>",
29
  "lstrip": false,
30
  "normalized": true,
31
  "rstrip": false,