Commit
•
6250973
1
Parent(s):
ff43006
add tokenizer
Browse files- added_tokens.json +1 -1
- vocab.json +1 -1
added_tokens.json
CHANGED
@@ -1 +1 @@
|
|
1 |
-
{"<s>":
|
|
|
1 |
+
{"<s>": 91, "</s>": 92}
|
vocab.json
CHANGED
@@ -1 +1 @@
|
|
1 |
-
{"
|
|
|
1 |
+
{"&": 1, "b": 2, "c": 3, "d": 4, "e": 5, "g": 6, "h": 7, "j": 8, "k": 9, "m": 10, "n": 11, "o": 12, "p": 13, "r": 14, "s": 15, "t": 16, "u": 17, "v": 18, "w": 19, "x": 20, "y": 21, "z": 22, "|": 0, "ँ": 24, "ं": 25, "ः": 26, "अ": 27, "आ": 28, "इ": 29, "ई": 30, "उ": 31, "ऊ": 32, "ऋ": 33, "ए": 34, "ऐ": 35, "ऑ": 36, "ओ": 37, "औ": 38, "क": 39, "ख": 40, "ग": 41, "घ": 42, "च": 43, "छ": 44, "ज": 45, "झ": 46, "ञ": 47, "ट": 48, "ठ": 49, "ड": 50, "ढ": 51, "ण": 52, "त": 53, "थ": 54, "द": 55, "ध": 56, "न": 57, "प": 58, "फ": 59, "ब": 60, "भ": 61, "म": 62, "य": 63, "र": 64, "ल": 65, "व": 66, "श": 67, "ष": 68, "स": 69, "ह": 70, "़": 71, "ा": 72, "ि": 73, "ी": 74, "ु": 75, "ू": 76, "ृ": 77, "ॅ": 78, "े": 79, "ै": 80, "ॉ": 81, "ो": 82, "ौ": 83, "्": 84, "क़": 85, "ग़": 86, "ज़": 87, "ड़": 88, "ढ़": 89, "’": 90, "[UNK]": 90, "[PAD]": 91}
|