arxyzan commited on
Commit
f5fa9c5
1 Parent(s): 039ce81

Hezar: Upload tokenizer_config.yaml

Browse files
Files changed (1) hide show
  1. preprocessor/tokenizer_config.yaml +0 -12
preprocessor/tokenizer_config.yaml CHANGED
@@ -15,18 +15,6 @@ sep_token: <sep>
15
  pad_token: </s>
16
  cls_token: <cls>
17
  mask_token: <mask>
18
- special_tokens:
19
- - <s>
20
- - <pad>
21
- - </s>
22
- - <unk>
23
- - <mask>
24
- - <|endoftext|>
25
- - <|startoftext|>
26
- - <nl>
27
- - <hs>
28
- - <sep>
29
- - <cls>
30
  continuing_subword_prefix: ''
31
  end_of_word_suffix: ''
32
  fuse_unk: false
 
15
  pad_token: </s>
16
  cls_token: <cls>
17
  mask_token: <mask>
 
 
 
 
 
 
 
 
 
 
 
 
18
  continuing_subword_prefix: ''
19
  end_of_word_suffix: ''
20
  fuse_unk: false