Matthijs commited on
Commit
18285f9
1 Parent(s): eb2cf91

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +3 -2
tokenizer_config.json CHANGED
@@ -1,9 +1,10 @@
1
  {
2
  "add_blank": true,
3
  "clean_up_tokenization_spaces": true,
4
- "language": null,
5
  "model_max_length": 1000000000000000019884624838656,
6
  "pad_token": "<pad>",
7
- "tokenizer_class": "VitsMmsTokenizer",
 
8
  "unk_token": "<unk>"
9
  }
 
1
  {
2
  "add_blank": true,
3
  "clean_up_tokenization_spaces": true,
4
+ "language": "ara",
5
  "model_max_length": 1000000000000000019884624838656,
6
  "pad_token": "<pad>",
7
+ "phonemize": false,
8
+ "tokenizer_class": "VitsTokenizer",
9
  "unk_token": "<unk>"
10
  }