unijoh commited on
Commit
4484de8
·
verified ·
1 Parent(s): 0fe550c

Upload 8 files

Browse files
Files changed (3) hide show
  1. constraint_mask.json +175 -175
  2. tag_mappings.json +0 -0
  3. tokenizer.json +16 -2
constraint_mask.json CHANGED
@@ -1,176 +1,176 @@
1
- {
2
- "0": [
3
- [
4
- 30,
5
- 33
6
- ],
7
- [
8
- 34,
9
- 36
10
- ],
11
- [
12
- 37,
13
- 41
14
- ],
15
- [
16
- 42,
17
- 43
18
- ],
19
- [
20
- 44,
21
- 45
22
- ]
23
- ],
24
- "1": [
25
- [
26
- 30,
27
- 33
28
- ],
29
- [
30
- 34,
31
- 36
32
- ],
33
- [
34
- 37,
35
- 41
36
- ],
37
- [
38
- 46,
39
- 50
40
- ],
41
- [
42
- 51,
43
- 53
44
- ]
45
- ],
46
- "2": [
47
- [
48
- 15,
49
- 29
50
- ],
51
- [
52
- 30,
53
- 33
54
- ],
55
- [
56
- 34,
57
- 36
58
- ],
59
- [
60
- 37,
61
- 41
62
- ],
63
- [
64
- 67,
65
- 70
66
- ]
67
- ],
68
- "3": [
69
- [
70
- 15,
71
- 29
72
- ],
73
- [
74
- 30,
75
- 33
76
- ],
77
- [
78
- 34,
79
- 36
80
- ],
81
- [
82
- 37,
83
- 41
84
- ]
85
- ],
86
- "4": [
87
- [
88
- 34,
89
- 36
90
- ],
91
- [
92
- 54,
93
- 60
94
- ],
95
- [
96
- 61,
97
- 63
98
- ],
99
- [
100
- 64,
101
- 66
102
- ],
103
- [
104
- 67,
105
- 70
106
- ]
107
- ],
108
- "5": [
109
- [
110
- 30,
111
- 33
112
- ],
113
- [
114
- 34,
115
- 36
116
- ],
117
- [
118
- 37,
119
- 41
120
- ],
121
- [
122
- 51,
123
- 53
124
- ]
125
- ],
126
- "6": [
127
- [
128
- 15,
129
- 29
130
- ],
131
- [
132
- 46,
133
- 50
134
- ]
135
- ],
136
- "7": [
137
- [
138
- 15,
139
- 29
140
- ]
141
- ],
142
- "8": [],
143
- "9": [],
144
- "10": [
145
- [
146
- 15,
147
- 29
148
- ]
149
- ],
150
- "11": [],
151
- "12": [
152
- [
153
- 15,
154
- 29
155
- ]
156
- ],
157
- "13": [],
158
- "14": [
159
- [
160
- 30,
161
- 33
162
- ],
163
- [
164
- 34,
165
- 36
166
- ],
167
- [
168
- 37,
169
- 41
170
- ],
171
- [
172
- 71,
173
- 72
174
- ]
175
- ]
176
  }
 
1
+ {
2
+ "0": [
3
+ [
4
+ 30,
5
+ 33
6
+ ],
7
+ [
8
+ 34,
9
+ 36
10
+ ],
11
+ [
12
+ 37,
13
+ 41
14
+ ],
15
+ [
16
+ 42,
17
+ 43
18
+ ],
19
+ [
20
+ 44,
21
+ 45
22
+ ]
23
+ ],
24
+ "1": [
25
+ [
26
+ 30,
27
+ 33
28
+ ],
29
+ [
30
+ 34,
31
+ 36
32
+ ],
33
+ [
34
+ 37,
35
+ 41
36
+ ],
37
+ [
38
+ 46,
39
+ 50
40
+ ],
41
+ [
42
+ 51,
43
+ 53
44
+ ]
45
+ ],
46
+ "2": [
47
+ [
48
+ 15,
49
+ 29
50
+ ],
51
+ [
52
+ 30,
53
+ 33
54
+ ],
55
+ [
56
+ 34,
57
+ 36
58
+ ],
59
+ [
60
+ 37,
61
+ 41
62
+ ],
63
+ [
64
+ 67,
65
+ 70
66
+ ]
67
+ ],
68
+ "3": [
69
+ [
70
+ 15,
71
+ 29
72
+ ],
73
+ [
74
+ 30,
75
+ 33
76
+ ],
77
+ [
78
+ 34,
79
+ 36
80
+ ],
81
+ [
82
+ 37,
83
+ 41
84
+ ]
85
+ ],
86
+ "4": [
87
+ [
88
+ 34,
89
+ 36
90
+ ],
91
+ [
92
+ 54,
93
+ 60
94
+ ],
95
+ [
96
+ 61,
97
+ 63
98
+ ],
99
+ [
100
+ 64,
101
+ 66
102
+ ],
103
+ [
104
+ 67,
105
+ 70
106
+ ]
107
+ ],
108
+ "5": [
109
+ [
110
+ 30,
111
+ 33
112
+ ],
113
+ [
114
+ 34,
115
+ 36
116
+ ],
117
+ [
118
+ 37,
119
+ 41
120
+ ],
121
+ [
122
+ 51,
123
+ 53
124
+ ]
125
+ ],
126
+ "6": [
127
+ [
128
+ 15,
129
+ 29
130
+ ],
131
+ [
132
+ 46,
133
+ 50
134
+ ]
135
+ ],
136
+ "7": [
137
+ [
138
+ 15,
139
+ 29
140
+ ]
141
+ ],
142
+ "8": [],
143
+ "9": [],
144
+ "10": [
145
+ [
146
+ 15,
147
+ 29
148
+ ]
149
+ ],
150
+ "11": [],
151
+ "12": [
152
+ [
153
+ 15,
154
+ 29
155
+ ]
156
+ ],
157
+ "13": [],
158
+ "14": [
159
+ [
160
+ 30,
161
+ 33
162
+ ],
163
+ [
164
+ 34,
165
+ 36
166
+ ],
167
+ [
168
+ 37,
169
+ 41
170
+ ],
171
+ [
172
+ 71,
173
+ 72
174
+ ]
175
+ ]
176
  }
tag_mappings.json CHANGED
The diff for this file is too large to render. See raw diff
 
tokenizer.json CHANGED
@@ -1,7 +1,21 @@
1
  {
2
  "version": "1.0",
3
- "truncation": null,
4
- "padding": null,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5
  "added_tokens": [
6
  {
7
  "id": 0,
 
1
  {
2
  "version": "1.0",
3
+ "truncation": {
4
+ "direction": "Right",
5
+ "max_length": 128,
6
+ "strategy": "LongestFirst",
7
+ "stride": 0
8
+ },
9
+ "padding": {
10
+ "strategy": {
11
+ "Fixed": 128
12
+ },
13
+ "direction": "Right",
14
+ "pad_to_multiple_of": null,
15
+ "pad_id": 1,
16
+ "pad_type_id": 0,
17
+ "pad_token": "<pad>"
18
+ },
19
  "added_tokens": [
20
  {
21
  "id": 0,