imvladikon commited on
Commit
97d18de
1 Parent(s): 1d2027f

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +6 -0
tokenizer_config.json CHANGED
@@ -110,6 +110,12 @@
110
  "sp_model_kwargs": {},
111
  "special_tokens_map_file": null,
112
  "tokenizer_class": "EncT5Tokenizer",
 
 
 
 
 
 
113
  "unk_token": "<unk>",
114
  "use_fast": true
115
  }
 
110
  "sp_model_kwargs": {},
111
  "special_tokens_map_file": null,
112
  "tokenizer_class": "EncT5Tokenizer",
113
+ "auto_map": {
114
+ "AutoTokenizer": [
115
+ "configuration_enc_t5.EncT5Tokenizer",
116
+ null
117
+ ]
118
+ }
119
  "unk_token": "<unk>",
120
  "use_fast": true
121
  }