arxyzan commited on
Commit
3121bf3
1 Parent(s): 63325a1

Remove deprecated tokenizer config fields

Browse files
Files changed (1) hide show
  1. preprocessor/tokenizer_config.yaml +1 -4
preprocessor/tokenizer_config.yaml CHANGED
@@ -1,11 +1,8 @@
1
  name: bpe_tokenizer
2
  config_type: preprocessor
3
- max_length: 512
4
- truncation: longest_first
5
  truncation_side: right
6
- stride: 0
7
- padding: longest
8
  padding_side: right
 
9
  pad_to_multiple_of: 0
10
  pad_token_type_id: 0
11
  bos_token: <s>
 
1
  name: bpe_tokenizer
2
  config_type: preprocessor
 
 
3
  truncation_side: right
 
 
4
  padding_side: right
5
+ stride: 0
6
  pad_to_multiple_of: 0
7
  pad_token_type_id: 0
8
  bos_token: <s>