f77777 commited on
Commit
035b913
1 Parent(s): 0d30e6b

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +60 -60
vocab.json CHANGED
@@ -1,64 +1,64 @@
1
  {
2
  "[PAD]": 61,
3
  "[UNK]": 60,
4
- "|": 22,
5
- "ँ": 43,
6
- "ं": 4,
7
- "ः": 39,
8
- "अ": 19,
9
- "आ": 5,
10
- "इ": 26,
11
- "ई": 16,
12
- "उ": 14,
13
- "ऊ": 52,
14
- "ए": 58,
15
- "ओ": 32,
16
- "औ": 48,
17
- "क": 37,
18
- "ख": 7,
19
- "ग": 17,
20
- "घ": 13,
21
- "ङ": 50,
22
- "च": 6,
23
- "छ": 40,
24
- "ज": 12,
25
- "झ": 30,
26
- "ञ": 42,
27
- "ट": 20,
28
- "ठ": 21,
29
- "ड": 41,
30
- "ढ": 23,
31
- "ण": 3,
32
- "त": 35,
33
- "थ": 10,
34
- "द": 25,
35
- "ध": 28,
36
- "न": 53,
37
- "प": 0,
38
- "फ": 33,
39
- "ब": 45,
40
- "भ": 2,
41
- "म": 57,
42
- "य": 44,
43
- "र": 9,
44
- "ल": 59,
45
- "व": 47,
46
- "श": 18,
47
- "ष": 8,
48
- "स": 34,
49
- "ह": 49,
50
- "ा": 54,
51
- "ि": 55,
52
- "ी": 36,
53
- "ु": 15,
54
- "ू": 56,
55
- "ृ": 27,
56
- "े": 38,
57
- "ै": 11,
58
- "ो": 46,
59
- "ौ": 31,
60
- "्": 1,
61
- "।": 24,
62
- "–": 29,
63
- "’": 51
64
  }
 
1
  {
2
  "[PAD]": 61,
3
  "[UNK]": 60,
4
+ "|": 12,
5
+ "ँ": 3,
6
+ "ं": 31,
7
+ "ः": 20,
8
+ "अ": 44,
9
+ "आ": 48,
10
+ "इ": 42,
11
+ "ई": 1,
12
+ "उ": 11,
13
+ "ऊ": 6,
14
+ "ए": 0,
15
+ "ओ": 5,
16
+ "औ": 59,
17
+ "क": 51,
18
+ "ख": 8,
19
+ "ग": 18,
20
+ "घ": 16,
21
+ "ङ": 57,
22
+ "च": 24,
23
+ "छ": 37,
24
+ "ज": 17,
25
+ "झ": 25,
26
+ "ञ": 58,
27
+ "ट": 21,
28
+ "ठ": 2,
29
+ "ड": 49,
30
+ "ढ": 10,
31
+ "ण": 35,
32
+ "त": 27,
33
+ "थ": 46,
34
+ "द": 47,
35
+ "ध": 45,
36
+ "न": 29,
37
+ "प": 23,
38
+ "फ": 38,
39
+ "ब": 55,
40
+ "भ": 9,
41
+ "म": 26,
42
+ "य": 43,
43
+ "र": 36,
44
+ "ल": 7,
45
+ "व": 53,
46
+ "श": 4,
47
+ "ष": 19,
48
+ "स": 22,
49
+ "ह": 33,
50
+ "ा": 34,
51
+ "ि": 15,
52
+ "ी": 32,
53
+ "ु": 56,
54
+ "ू": 41,
55
+ "ृ": 54,
56
+ "े": 28,
57
+ "ै": 40,
58
+ "ो": 52,
59
+ "ौ": 50,
60
+ "्": 14,
61
+ "।": 30,
62
+ "–": 13,
63
+ "’": 39
64
  }