Matthijs commited on
Commit
1bba128
1 Parent(s): 6280a5c

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer_config.json +2 -1
  2. vocab.json +4 -3
tokenizer_config.json CHANGED
@@ -4,6 +4,7 @@
4
  "language": null,
5
  "model_max_length": 1000000000000000019884624838656,
6
  "pad_token": "<pad>",
7
- "tokenizer_class": "VitsMmsTokenizer",
 
8
  "unk_token": "<unk>"
9
  }
 
4
  "language": null,
5
  "model_max_length": 1000000000000000019884624838656,
6
  "pad_token": "<pad>",
7
+ "phonemize": true,
8
+ "tokenizer_class": "VitsTokenizer",
9
  "unk_token": "<unk>"
10
  }
vocab.json CHANGED
@@ -1,13 +1,14 @@
1
  {
2
- "": 16,
3
  "!": 5,
4
  "\"": 11,
5
  "'": 176,
 
6
  ".": 4,
7
  ":": 2,
8
  ";": 1,
9
- "<pad>": 176,
10
- "<unk>": 177,
11
  "?": 6,
12
  "A": 17,
13
  "B": 18,
 
1
  {
2
+ " ": 16,
3
  "!": 5,
4
  "\"": 11,
5
  "'": 176,
6
+ ",": 3,
7
  ".": 4,
8
  ":": 2,
9
  ";": 1,
10
+ "<pad>": 177,
11
+ "<unk>": 178,
12
  "?": 6,
13
  "A": 17,
14
  "B": 18,