chaos6749 commited on
Commit
0219516
β€’
1 Parent(s): 06f3a04

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +49 -49
vocab.json CHANGED
@@ -1,53 +1,53 @@
1
  {
2
  "[PAD]": 50,
3
  "[UNK]": 49,
4
- "|": 37,
5
- "γ„±": 17,
6
- "γ„²": 19,
7
- "γ„³": 34,
8
- "γ„΄": 25,
9
- "γ„΅": 47,
10
- "γ„Ά": 26,
11
- "γ„·": 44,
12
- "γ„Έ": 7,
13
- "γ„Ή": 27,
14
- "γ„Ί": 33,
15
- "γ„»": 12,
16
- "γ„Ό": 29,
17
- "γ…€": 45,
18
- "ㅁ": 18,
19
- "γ…‚": 4,
20
- "γ…ƒ": 1,
21
- "γ…„": 11,
22
- "γ……": 43,
23
- "γ…†": 13,
24
- "γ…‡": 2,
25
- "γ…ˆ": 41,
26
- "γ…‰": 46,
27
- "γ…Š": 0,
28
- "γ…‹": 42,
29
- "γ…Œ": 40,
30
- "ㅍ": 32,
31
- "γ…Ž": 15,
32
- "ㅏ": 5,
33
- "ㅐ": 21,
34
- "γ…‘": 23,
35
- "γ…’": 8,
36
- "γ…“": 9,
37
- "γ…”": 36,
38
- "γ…•": 39,
39
- "γ…–": 3,
40
- "γ…—": 14,
41
- "γ…˜": 22,
42
- "γ…™": 28,
43
- "γ…š": 6,
44
- "γ…›": 31,
45
- "γ…œ": 10,
46
- "ㅝ": 24,
47
- "γ…ž": 16,
48
- "γ…Ÿ": 35,
49
- "γ… ": 20,
50
- "γ…‘": 30,
51
- "γ…’": 48,
52
- "γ…£": 38
53
  }
 
1
  {
2
  "[PAD]": 50,
3
  "[UNK]": 49,
4
+ "|": 31,
5
+ "γ„±": 12,
6
+ "γ„²": 14,
7
+ "γ„³": 11,
8
+ "γ„΄": 30,
9
+ "γ„΅": 26,
10
+ "γ„Ά": 21,
11
+ "γ„·": 45,
12
+ "γ„Έ": 46,
13
+ "γ„Ή": 28,
14
+ "γ„Ί": 24,
15
+ "γ„»": 1,
16
+ "γ„Ό": 5,
17
+ "γ…€": 40,
18
+ "ㅁ": 4,
19
+ "γ…‚": 0,
20
+ "γ…ƒ": 18,
21
+ "γ…„": 41,
22
+ "γ……": 38,
23
+ "γ…†": 10,
24
+ "γ…‡": 8,
25
+ "γ…ˆ": 33,
26
+ "γ…‰": 6,
27
+ "γ…Š": 48,
28
+ "γ…‹": 22,
29
+ "γ…Œ": 7,
30
+ "ㅍ": 34,
31
+ "γ…Ž": 37,
32
+ "ㅏ": 32,
33
+ "ㅐ": 36,
34
+ "γ…‘": 25,
35
+ "γ…’": 2,
36
+ "γ…“": 27,
37
+ "γ…”": 15,
38
+ "γ…•": 47,
39
+ "γ…–": 42,
40
+ "γ…—": 39,
41
+ "γ…˜": 20,
42
+ "γ…™": 17,
43
+ "γ…š": 9,
44
+ "γ…›": 29,
45
+ "γ…œ": 3,
46
+ "ㅝ": 44,
47
+ "γ…ž": 35,
48
+ "γ…Ÿ": 19,
49
+ "γ… ": 16,
50
+ "γ…‘": 43,
51
+ "γ…’": 13,
52
+ "γ…£": 23
53
  }