Hezar: Upload tokenizer_config.yaml

preprocessor/tokenizer_config.yaml

@@ -1,12 +1,21 @@
 name: wordpiece_tokenizer
 config_type: preprocessor
-model: wordpiece
 max_length: 512
 truncation_strategy: longest_first
 truncation_direction: right
 stride: 0
 padding_strategy: longest
 padding_direction: right
-pad_token_id: 0
+pad_to_multiple_of: 0
+pad_token_type_id: 0
+unk_token: '[UNK]'
+sep_token: '[SEP]'
 pad_token: '[PAD]'
-pad_token_type_id: 0
+cls_token: '[CLS]'
+mask_token: '[MASK]'
+wordpieces_prefix: '##'
+vocab_size: 30000
+min_frequency: 2
+limit_alphabet: 1000
+initial_alphabet: []
+show_progress: true