arxyzan commited on
Commit
2133341
1 Parent(s): 2d150b3

Hezar: Upload tokenizer and config

Browse files
Files changed (1) hide show
  1. preprocessor/tokenizer_config.yaml +33 -0
preprocessor/tokenizer_config.yaml ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: bpe_tokenizer
2
+ config_type: preprocessor
3
+ max_length: 512
4
+ truncation_strategy: longest_first
5
+ truncation_direction: right
6
+ stride: 0
7
+ padding_strategy: longest
8
+ padding_direction: right
9
+ pad_to_multiple_of: 0
10
+ pad_token_id: 0
11
+ pad_token: <pad>
12
+ pad_token_type_id: 0
13
+ unk_token: <unk>
14
+ special_tokens:
15
+ - <s>
16
+ - <pad>
17
+ - </s>
18
+ - <unk>
19
+ - <mask>
20
+ - <|endoftext|>
21
+ - <|startoftext|>
22
+ - <nl>
23
+ - <hs>
24
+ - <sep>
25
+ - <cls>
26
+ continuing_subword_prefix: ''
27
+ end_of_word_suffix: ''
28
+ fuse_unk: false
29
+ vocab_size: 42000
30
+ min_frequency: 2
31
+ limit_alphabet: 1000
32
+ initial_alphabet: []
33
+ show_progress: true