Update preprocessor/tokenizer_config.yaml
Browse files
preprocessor/tokenizer_config.yaml
CHANGED
|
@@ -8,7 +8,7 @@ padding_strategy: longest
|
|
| 8 |
padding_direction: right
|
| 9 |
pad_to_multiple_of: 0
|
| 10 |
pad_token_type_id: 0
|
| 11 |
-
bos_token: <|
|
| 12 |
eos_token: <|endoftext|>
|
| 13 |
unk_token: <|endoftext|>
|
| 14 |
sep_token: <sep>
|
|
|
|
| 8 |
padding_direction: right
|
| 9 |
pad_to_multiple_of: 0
|
| 10 |
pad_token_type_id: 0
|
| 11 |
+
bos_token: <|startoftranscript|>
|
| 12 |
eos_token: <|endoftext|>
|
| 13 |
unk_token: <|endoftext|>
|
| 14 |
sep_token: <sep>
|