File size: 5,269 Bytes
810ba42
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
{
    "version": "1.0",
    "truncation": null,
    "padding": null,
    "added_tokens": [
        {
            "id": 0,
            "content": "<s>",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": false,
            "special": true
        },
        {
            "id": 1,
            "content": "<pad>",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": false,
            "special": true
        },
        {
            "id": 2,
            "content": "</s>",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": false,
            "special": true
        },
        {
            "id": 3,
            "content": "<unk>",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": false,
            "special": true
        },
        {
            "id": 79,
            "content": "<mask>",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": false,
            "special": true
        },
        {
            "id": 80,
            "content": "<ctc_blank>",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": false,
            "special": true
        }
    ],
    "normalizer": {
        "type": "Precompiled",
        "precompiled_charsmap": null
    },
    "pre_tokenizer": {
        "type": "Sequence",
        "pretokenizers": [
            {
                "type": "WhitespaceSplit"
            },
            {
                "type": "Metaspace",
                "replacement": "\u2581",
                "add_prefix_space": true
            },
            {
                "type": "Split",
                "pattern": {
                    "Regex": ""
                },
                "behavior": "Isolated",
                "invert": false
            }
        ]
    },
    "post_processor": {
        "type": "TemplateProcessing",
        "single": [
            {
                "Sequence": {
                    "id": "A",
                    "type_id": 0
                }
            },
            {
                "SpecialToken": {
                    "id": "</s>",
                    "type_id": 0
                }
            }
        ],
        "pair": [
            {
                "Sequence": {
                    "id": "A",
                    "type_id": 0
                }
            },
            {
                "SpecialToken": {
                    "id": "</s>",
                    "type_id": 0
                }
            },
            {
                "Sequence": {
                    "id": "B",
                    "type_id": 0
                }
            },
            {
                "SpecialToken": {
                    "id": "</s>",
                    "type_id": 0
                }
            }
        ],
        "special_tokens": {
            "</s>": {
                "id": "</s>",
                "ids": [
                    2
                ],
                "tokens": [
                    "</s>"
                ]
            }
        }
    },
    "decoder": {
        "type": "Metaspace",
        "replacement": "\u2581",
        "add_prefix_space": true
    },
    "model": {
        "unk_id": 2,
        "vocab": {
            "<s>": 0,
            "<pad>": 1,
            "</s>": 2,
            "<unk>": 3,
            "\u2581": 4,
            "e": 5,
            "t": 6,
            "a": 7,
            "o": 8,
            "n": 9,
            "i": 10,
            "h": 11,
            "s": 12,
            "r": 13,
            "d": 14,
            "l": 15,
            "u": 16,
            "c": 17,
            "m": 18,
            "f": 19,
            "w": 20,
            "g": 21,
            "y": 22,
            ",": 23,
            "p": 24,
            "b": 25,
            ".": 26,
            "v": 27,
            "k": 28,
            "\"": 29,
            "I": 30,
            "'": 31,
            "T": 32,
            "A": 33,
            "S": 34,
            "H": 35,
            ";": 36,
            "x": 37,
            "W": 38,
            "-": 39,
            "B": 40,
            "?": 41,
            "C": 42,
            "M": 43,
            "!": 44,
            "q": 45,
            "j": 46,
            "E": 47,
            "N": 48,
            "P": 49,
            "O": 50,
            "D": 51,
            "L": 52,
            "G": 53,
            "R": 54,
            "F": 55,
            "Y": 56,
            "z": 57,
            "J": 58,
            ":": 59,
            "K": 60,
            "U": 61,
            "V": 62,
            ")": 63,
            "(": 64,
            "Q": 65,
            "Z": 66,
            "]": 67,
            "[": 68,
            "X": 69,
            "\u2014": 70,
            "/": 71,
            "\u00e6": 72,
            "\u00e9": 73,
            "{": 74,
            "}": 75,
            "\u00ea": 76,
            "\u0153": 77,
            "\u0304": 78,
            "<mask>": 79,
            "<ctc_blank>": 80
        }
    }
}