Commit
•
36e6811
1
Parent(s):
8cde8bf
Create tokenizer.ckpt
Browse files- tokenizer.ckpt +70 -0
tokenizer.ckpt
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
'l' => 66
|
| 2 |
+
'e' => 1
|
| 3 |
+
'c' => 2
|
| 4 |
+
'i' => 3
|
| 5 |
+
'n' => 4
|
| 6 |
+
'q' => 5
|
| 7 |
+
't' => 6
|
| 8 |
+
'_' => 7
|
| 9 |
+
'b' => 8
|
| 10 |
+
'j' => 9
|
| 11 |
+
'a' => 10
|
| 12 |
+
'*' => 11
|
| 13 |
+
's' => 12
|
| 14 |
+
'-' => 13
|
| 15 |
+
'o' => 14
|
| 16 |
+
'u' => 15
|
| 17 |
+
'à' => 16
|
| 18 |
+
'h' => 17
|
| 19 |
+
'P' => 18
|
| 20 |
+
'r' => 19
|
| 21 |
+
'x' => 20
|
| 22 |
+
'm' => 21
|
| 23 |
+
'p' => 22
|
| 24 |
+
'd' => 23
|
| 25 |
+
"'" => 24
|
| 26 |
+
'ô' => 25
|
| 27 |
+
'f' => 26
|
| 28 |
+
'k' => 27
|
| 29 |
+
'è' => 28
|
| 30 |
+
'î' => 29
|
| 31 |
+
'M' => 30
|
| 32 |
+
'v' => 31
|
| 33 |
+
'ç' => 32
|
| 34 |
+
'D' => 33
|
| 35 |
+
'é' => 34
|
| 36 |
+
'g' => 35
|
| 37 |
+
'z' => 36
|
| 38 |
+
'S' => 37
|
| 39 |
+
'C' => 38
|
| 40 |
+
'L' => 39
|
| 41 |
+
'R' => 40
|
| 42 |
+
'I' => 41
|
| 43 |
+
'N' => 42
|
| 44 |
+
'û' => 43
|
| 45 |
+
'y' => 44
|
| 46 |
+
'J' => 45
|
| 47 |
+
'B' => 46
|
| 48 |
+
'O' => 47
|
| 49 |
+
'A' => 48
|
| 50 |
+
'E' => 49
|
| 51 |
+
'H' => 50
|
| 52 |
+
'F' => 51
|
| 53 |
+
'U' => 52
|
| 54 |
+
'ê' => 53
|
| 55 |
+
'T' => 54
|
| 56 |
+
'V' => 55
|
| 57 |
+
'G' => 56
|
| 58 |
+
'w' => 57
|
| 59 |
+
'ï' => 58
|
| 60 |
+
'â' => 59
|
| 61 |
+
'É' => 60
|
| 62 |
+
'ù' => 61
|
| 63 |
+
'ë' => 62
|
| 64 |
+
'W' => 63
|
| 65 |
+
'Y' => 64
|
| 66 |
+
'K' => 65
|
| 67 |
+
'<blank>' => 0
|
| 68 |
+
================
|
| 69 |
+
'starting_index' => 0
|
| 70 |
+
'blank_label' => '<blank>'
|