nielsr HF staff commited on
Commit
907fce7
1 Parent(s): 4d1625b

Fix tokenizer files

Browse files
Files changed (3) hide show
  1. .gitignore +1 -0
  2. special_tokens_map.json +1 -0
  3. tokenizer_config.json +3 -1
.gitignore ADDED
@@ -0,0 +1 @@
 
1
+ .DS_Store
special_tokens_map.json CHANGED
@@ -1204,5 +1204,6 @@
1204
  ],
1205
  "eos_token": "</s>",
1206
  "pad_token": "<pad>",
 
1207
  "unk_token": "<unk>"
1208
  }
1204
  ],
1205
  "eos_token": "</s>",
1206
  "pad_token": "<pad>",
1207
+ "sep_token": "</s>",
1208
  "unk_token": "<unk>"
1209
  }
tokenizer_config.json CHANGED
@@ -1204,8 +1204,9 @@
1204
  ],
1205
  "eos_token": "</s>",
1206
  "extra_ids": 100,
 
1207
  "loc_extra_ids": 501,
1208
- "model_max_length": 1000000000000000019884624838656,
1209
  "only_label_first_subword": true,
1210
  "other_extra_ids": 200,
1211
  "pad_token": "<pad>",
@@ -1217,6 +1218,7 @@
1217
  ],
1218
  "pad_token_label": -100,
1219
  "processor_class": "UdopProcessor",
 
1220
  "sep_token_box": [
1221
  1000,
1222
  1000,
1204
  ],
1205
  "eos_token": "</s>",
1206
  "extra_ids": 100,
1207
+ "from_slow": true,
1208
  "loc_extra_ids": 501,
1209
+ "model_max_length": 512,
1210
  "only_label_first_subword": true,
1211
  "other_extra_ids": 200,
1212
  "pad_token": "<pad>",
1218
  ],
1219
  "pad_token_label": -100,
1220
  "processor_class": "UdopProcessor",
1221
+ "sep_token": "</s>",
1222
  "sep_token_box": [
1223
  1000,
1224
  1000,