huseinzol05 commited on
Commit
883b280
1 Parent(s): 1c0c561

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +4 -0
tokenizer_config.json CHANGED
@@ -42,6 +42,10 @@
42
  "bos_token": "<s>",
43
  "clean_up_tokenization_spaces": true,
44
  "eos_token": "</s>",
 
 
 
 
45
  "model_max_length": 1000000000000000019884624838656,
46
  "pad_token": "<pad>",
47
  "tokenizer_class": "PreTrainedTokenizerFast",
 
42
  "bos_token": "<s>",
43
  "clean_up_tokenization_spaces": true,
44
  "eos_token": "</s>",
45
+ "model_input_names": [
46
+ "input_ids",
47
+ "attention_mask"
48
+ ],
49
  "model_max_length": 1000000000000000019884624838656,
50
  "pad_token": "<pad>",
51
  "tokenizer_class": "PreTrainedTokenizerFast",