AidenWilliams commited on
Commit
5cfba2e
1 Parent(s): 242bcb2

Upload processor

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +28 -4
  2. vocab.json +48 -1
special_tokens_map.json CHANGED
@@ -1,6 +1,30 @@
1
  {
2
- "bos_token": "<s>",
3
- "eos_token": "</s>",
4
- "pad_token": "[PAD]",
5
- "unk_token": "[UNK]"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6
  }
 
1
  {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": {
17
+ "content": "[PAD]",
18
+ "lstrip": true,
19
+ "normalized": false,
20
+ "rstrip": true,
21
+ "single_word": false
22
+ },
23
+ "unk_token": {
24
+ "content": "[UNK]",
25
+ "lstrip": true,
26
+ "normalized": false,
27
+ "rstrip": true,
28
+ "single_word": false
29
+ }
30
  }
vocab.json CHANGED
@@ -1 +1,48 @@
1
- {"g": 0, "k": 1, "\u00ec": 2, "w": 3, "d": 4, "p": 5, "\u00f3": 6, "o": 7, "n": 8, "\u0127": 9, "x": 10, "\u0121": 11, "\u00f9": 12, "b": 13, "f": 14, "\u00e9": 15, "j": 16, "4": 17, "a": 18, "m": 19, "e": 20, "\u0107": 21, "\u017c": 22, "q": 23, "s": 24, "y": 25, "\u010b": 26, "\u00ed": 27, "-": 28, "t": 29, "h": 30, "r": 31, "i": 32, "\u00e1": 33, "u": 34, "v": 35, "c": 37, "\u00f2": 38, "`": 39, "l": 40, "\u00e0": 41, "\u00e8": 42, "z": 43, "|": 36, "[UNK]": 44, "[PAD]": 45}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "-": 28,
3
+ "4": 17,
4
+ "[PAD]": 45,
5
+ "[UNK]": 44,
6
+ "`": 39,
7
+ "a": 18,
8
+ "b": 13,
9
+ "c": 37,
10
+ "d": 4,
11
+ "e": 20,
12
+ "f": 14,
13
+ "g": 0,
14
+ "h": 30,
15
+ "i": 32,
16
+ "j": 16,
17
+ "k": 1,
18
+ "l": 40,
19
+ "m": 19,
20
+ "n": 8,
21
+ "o": 7,
22
+ "p": 5,
23
+ "q": 23,
24
+ "r": 31,
25
+ "s": 24,
26
+ "t": 29,
27
+ "u": 34,
28
+ "v": 35,
29
+ "w": 3,
30
+ "x": 10,
31
+ "y": 25,
32
+ "z": 43,
33
+ "|": 36,
34
+ "à": 41,
35
+ "á": 33,
36
+ "è": 42,
37
+ "é": 15,
38
+ "ì": 2,
39
+ "í": 27,
40
+ "ò": 38,
41
+ "ó": 6,
42
+ "ù": 12,
43
+ "ć": 21,
44
+ "ċ": 26,
45
+ "ġ": 11,
46
+ "ħ": 9,
47
+ "ż": 22
48
+ }