Commit
•
8bcaaba
1
Parent(s):
3b12d8b
Upload tokenizer_config.json with huggingface_hub
Browse files- tokenizer_config.json +2 -4
tokenizer_config.json
CHANGED
|
@@ -1996,12 +1996,10 @@
|
|
| 1996 |
}
|
| 1997 |
},
|
| 1998 |
"additional_special_tokens": [
|
| 1999 |
-
"<start_of_turn>",
|
| 2000 |
-
"<end_of_turn>"
|
| 2001 |
],
|
| 2002 |
-
"bos_token": "
|
| 2003 |
"clean_up_tokenization_spaces": false,
|
| 2004 |
-
"eos_token": "
|
| 2005 |
"model_max_length": 1000000000000000019884624838656,
|
| 2006 |
"pad_token": "<pad>",
|
| 2007 |
"padding_side": "left",
|
|
|
|
| 1996 |
}
|
| 1997 |
},
|
| 1998 |
"additional_special_tokens": [
|
|
|
|
|
|
|
| 1999 |
],
|
| 2000 |
+
"bos_token": "<|begin_of_text|>",
|
| 2001 |
"clean_up_tokenization_spaces": false,
|
| 2002 |
+
"eos_token": "<|end_of_text|>",
|
| 2003 |
"model_max_length": 1000000000000000019884624838656,
|
| 2004 |
"pad_token": "<pad>",
|
| 2005 |
"padding_side": "left",
|