File size: 3,141 Bytes
d112d51
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
{
    "version": "1.0",
    "truncation": null,
    "padding": null,
    "added_tokens": [
        {
            "id": 60,
            "content": "<unk>",
            "single_word": false,
            "lstrip": false,
            "rstrip": false,
            "normalized": false,
            "special": true
        }
    ],
    "normalizer": {
        "type": "Sequence",
        "normalizers": [
            {
                "type": "Lowercase"
            },
            {
                "type": "Replace",
                "pattern": {
                    "Regex": "[^|\u0abe\u0ac7\u0aa4\u0aa8\u0ab0\u0acd\u0aae\u0ac0\u0a95\u0a82\u0acb\u0aaa\u0ab5\u0ac1\u0ab8\u0aaf\u0ab9\u0a9c\u0aa5\u0abf\u0ab6\u0aa3\u0a9b\u0ab2\u0aa6\u0a86\u0a93\u0a85\u0a8f\u0a88\u0aac\u0a97\u0a96\u0aa7\u0aad\u0a9f\u0ab3\u0ac2\u0a9a\u0aa1\u0ab7\u0a89\u0aab\u0a98\u0aa0\u0ac3'\u0a8a\u0a87\u0a9e\u0a9d\u0aa2\u0ac8\u0a83\u0acc\\-\u0a8b\u0a90 ]"
                },
                "content": ""
            },
            {
                "type": "Strip",
                "strip_left": true,
                "strip_right": true
            },
            {
                "type": "Replace",
                "pattern": {
                    "Regex": "(?=.)|(?<!^)$"
                },
                "content": "|"
            }
        ]
    },
    "pre_tokenizer": {
        "type": "Split",
        "pattern": {
            "Regex": ""
        },
        "behavior": "Isolated",
        "invert": false
    },
    "post_processor": null,
    "decoder": null,
    "model": {
        "vocab": {
            "|": 0,
            "\u0abe": 1,
            "\u0ac7": 2,
            "\u0aa4": 3,
            "\u0aa8": 4,
            "\u0ab0": 5,
            "\u0acd": 6,
            "\u0aae": 7,
            "\u0ac0": 8,
            "\u0a95": 9,
            "\u0a82": 10,
            "\u0acb": 11,
            "\u0aaa": 12,
            "\u0ab5": 13,
            "\u0ac1": 14,
            "\u0ab8": 15,
            "\u0aaf": 16,
            "\u0ab9": 17,
            "\u0a9c": 18,
            "\u0aa5": 19,
            "\u0abf": 20,
            "\u0ab6": 21,
            "\u0aa3": 22,
            "\u0a9b": 23,
            "\u0ab2": 24,
            "\u0aa6": 25,
            "\u0a86": 26,
            "\u0a93": 27,
            "\u0a85": 28,
            "\u0a8f": 29,
            "\u0a88": 30,
            "\u0aac": 31,
            "\u0a97": 32,
            "\u0a96": 33,
            "\u0aa7": 34,
            "\u0aad": 35,
            "\u0a9f": 36,
            "\u0ab3": 37,
            "\u0ac2": 38,
            "\u0a9a": 39,
            "\u0aa1": 40,
            "\u0ab7": 41,
            "\u0a89": 42,
            "\u0aab": 43,
            "\u0a98": 44,
            "\u0aa0": 45,
            "\u0ac3": 46,
            "'": 47,
            "\u0a8a": 48,
            "\u0a87": 49,
            "\u0a9e": 50,
            "\u0a9d": 51,
            "\u0aa2": 52,
            "\u0ac8": 53,
            "\u0a83": 54,
            "\u0acc": 55,
            "-": 56,
            "\u0a8b": 57,
            "\u0a90": 58,
            " ": 59,
            "<unk>": 60
        }
    }
}