Hezar: Upload tokenizer and config

preprocessor/tokenizer_config.yaml

@@ -1,6 +1,6 @@
 name: whisper_bpe_tokenizer
 config_type: preprocessor
-max_length: 512
+max_length: 448
 truncation_strategy: longest_first
 truncation_direction: right
 stride: 0
@@ -8,11 +8,11 @@ padding_strategy: longest
 padding_direction: right
 pad_to_multiple_of: 0
 pad_token_type_id: 0
-bos_token: <|startoftranscript|>
+bos_token: <|endoftext|>
 eos_token: <|endoftext|>
 unk_token: <|endoftext|>
 sep_token: <sep>
-pad_token: <pad>
+pad_token: <|endoftext|>
 cls_token: <cls>
 mask_token: <mask>
 additional_special_tokens:
@@ -127,7 +127,7 @@ additional_special_tokens:
 continuing_subword_prefix: ''
 end_of_word_suffix: ''
 fuse_unk: false
-vocab_size: 30000
+vocab_size: 50364
 min_frequency: 2
 limit_alphabet: 1000
 initial_alphabet: []