swap for non-int8 version
Browse files- added_tokens.json +1 -0
- special_tokens_map.json +1 -7
- tokenizer_config.json +0 -0
added_tokens.json
CHANGED
|
@@ -1518,6 +1518,7 @@
|
|
| 1518 |
"<|da|>": 50284,
|
| 1519 |
"<|de|>": 50260,
|
| 1520 |
"<|el|>": 50280,
|
|
|
|
| 1521 |
"<|en|>": 50258,
|
| 1522 |
"<|es|>": 50261,
|
| 1523 |
"<|et|>": 50306,
|
|
|
|
| 1518 |
"<|da|>": 50284,
|
| 1519 |
"<|de|>": 50260,
|
| 1520 |
"<|el|>": 50280,
|
| 1521 |
+
"<|endoftext|>": 50256,
|
| 1522 |
"<|en|>": 50258,
|
| 1523 |
"<|es|>": 50261,
|
| 1524 |
"<|et|>": 50306,
|
special_tokens_map.json
CHANGED
|
@@ -107,13 +107,7 @@
|
|
| 107 |
"<|nocaptions|>",
|
| 108 |
"<|notimestamps|>"
|
| 109 |
],
|
| 110 |
-
"bos_token":
|
| 111 |
-
"content": "<|endoftext|>",
|
| 112 |
-
"lstrip": false,
|
| 113 |
-
"normalized": true,
|
| 114 |
-
"rstrip": false,
|
| 115 |
-
"single_word": false
|
| 116 |
-
},
|
| 117 |
"eos_token": "<|endoftext|>",
|
| 118 |
"pad_token": "<|endoftext|>",
|
| 119 |
"unk_token": "<|endoftext|>"
|
|
|
|
| 107 |
"<|nocaptions|>",
|
| 108 |
"<|notimestamps|>"
|
| 109 |
],
|
| 110 |
+
"bos_token": "<|endoftext|>",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 111 |
"eos_token": "<|endoftext|>",
|
| 112 |
"pad_token": "<|endoftext|>",
|
| 113 |
"unk_token": "<|endoftext|>"
|
tokenizer_config.json
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|