Fredium commited on
Commit
1335178
1 Parent(s): 38a08a8

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +62 -62
vocab.json CHANGED
@@ -1,66 +1,66 @@
1
  {
2
  "[PAD]": 63,
3
  "[UNK]": 62,
4
- "|": 3,
5
- "ਂ": 40,
6
- "ਅ": 31,
7
- "ਆ": 51,
8
- "ਇ": 33,
9
- "ਈ": 12,
10
- "ਉ": 29,
11
- "ਊ": 18,
12
- "ਏ": 47,
13
- "ਐ": 5,
14
- "ਓ": 35,
15
- "ਔ": 42,
16
- "ਕ": 60,
17
- "ਖ": 58,
18
- "ਗ": 25,
19
- "ਘ": 10,
20
- "ਚ": 61,
21
- "ਛ": 0,
22
- "ਜ": 53,
23
- "ਝ": 23,
24
- "ਟ": 2,
25
- "ਠ": 43,
26
- "ਡ": 4,
27
- "ਢ": 6,
28
- "ਣ": 26,
29
- "ਤ": 38,
30
- "ਥ": 14,
31
- "ਦ": 54,
32
- "ਧ": 45,
33
- "ਨ": 21,
34
- "ਪ": 17,
35
- "ਫ": 24,
36
- "ਬ": 36,
37
- "ਭ": 49,
38
- "ਮ": 1,
39
- "ਯ": 50,
40
- "ਰ": 22,
41
- "ਲ": 28,
42
- "ਲ਼": 15,
43
- "ਵ": 48,
44
- "ਸ਼": 16,
45
- "ਸ": 41,
46
- "ਹ": 27,
47
- "਼": 20,
48
- "ਾ": 59,
49
- "ਿ": 30,
50
- "ੀ": 46,
51
- "ੁ": 37,
52
- "ੂ": 19,
53
- "ੇ": 9,
54
- "ੈ": 57,
55
- "ੋ": 7,
56
- "ੌ": 39,
57
- "੍": 55,
58
- "ਖ਼": 56,
59
- "ਗ਼": 44,
60
- "ਜ਼": 32,
61
- "ੜ": 52,
62
- "ਫ਼": 34,
63
- "ੰ": 11,
64
- "ੱ": 13,
65
- "…": 8
66
  }
 
1
  {
2
  "[PAD]": 63,
3
  "[UNK]": 62,
4
+ "|": 43,
5
+ "ਂ": 32,
6
+ "ਅ": 57,
7
+ "ਆ": 1,
8
+ "ਇ": 55,
9
+ "ਈ": 44,
10
+ "ਉ": 24,
11
+ "ਊ": 56,
12
+ "ਏ": 51,
13
+ "ਐ": 35,
14
+ "ਓ": 27,
15
+ "ਔ": 16,
16
+ "ਕ": 50,
17
+ "ਖ": 25,
18
+ "ਗ": 17,
19
+ "ਘ": 3,
20
+ "ਚ": 20,
21
+ "ਛ": 15,
22
+ "ਜ": 7,
23
+ "ਝ": 52,
24
+ "ਟ": 8,
25
+ "ਠ": 12,
26
+ "ਡ": 38,
27
+ "ਢ": 26,
28
+ "ਣ": 31,
29
+ "ਤ": 46,
30
+ "ਥ": 4,
31
+ "ਦ": 18,
32
+ "ਧ": 34,
33
+ "ਨ": 40,
34
+ "ਪ": 45,
35
+ "ਫ": 58,
36
+ "ਬ": 14,
37
+ "ਭ": 21,
38
+ "ਮ": 42,
39
+ "ਯ": 30,
40
+ "ਰ": 54,
41
+ "ਲ": 11,
42
+ "ਲ਼": 37,
43
+ "ਵ": 9,
44
+ "ਸ਼": 28,
45
+ "ਸ": 49,
46
+ "ਹ": 19,
47
+ "਼": 61,
48
+ "ਾ": 22,
49
+ "ਿ": 33,
50
+ "ੀ": 10,
51
+ "ੁ": 47,
52
+ "ੂ": 13,
53
+ "ੇ": 39,
54
+ "ੈ": 23,
55
+ "ੋ": 29,
56
+ "ੌ": 48,
57
+ "੍": 0,
58
+ "ਖ਼": 6,
59
+ "ਗ਼": 53,
60
+ "ਜ਼": 36,
61
+ "ੜ": 2,
62
+ "ਫ਼": 5,
63
+ "ੰ": 41,
64
+ "ੱ": 59,
65
+ "…": 60
66
  }