Spaces:
Running
Running
mathemakitten
commited on
Commit
•
037729c
1
Parent(s):
d085278
gloo
Browse files- glue-suite-v2.py +130 -21
glue-suite-v2.py
CHANGED
@@ -7,36 +7,145 @@ class Suite(evaluate.EvaluationSuite):
|
|
7 |
def __init__(self, name):
|
8 |
super().__init__(name)
|
9 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
10 |
def setup(self):
|
11 |
self.preprocessor = lambda x: {"text": x["text"].lower()}
|
12 |
self.suite = [
|
13 |
SubTask(
|
14 |
task_type="text-classification",
|
15 |
-
data="
|
16 |
-
|
17 |
-
|
18 |
args_for_task={
|
19 |
"metric": "accuracy",
|
20 |
-
"input_column": "
|
|
|
21 |
"label_column": "label",
|
22 |
"label_mapping": {
|
23 |
-
"LABEL_0": 0
|
24 |
-
"LABEL_1": 1
|
25 |
-
}
|
26 |
-
}
|
27 |
-
),
|
28 |
-
SubTask(
|
29 |
-
task_type="text-classification",
|
30 |
-
data="sst2",
|
31 |
-
split="test[:10]",
|
32 |
-
data_preprocessor=lambda x: {"sentence": x["sentence"].lower()},
|
33 |
-
args_for_task={
|
34 |
-
"metric": "accuracy",
|
35 |
-
"input_column": "sentence",
|
36 |
-
"label_column": "label",
|
37 |
-
"label_mapping": {
|
38 |
-
"LABEL_0": 0.0,
|
39 |
-
"LABEL_1": 1.0
|
40 |
}
|
41 |
}
|
42 |
)
|
|
|
7 |
def __init__(self, name):
|
8 |
super().__init__(name)
|
9 |
|
10 |
+
"""
|
11 |
+
{
|
12 |
+
"data": "glue",
|
13 |
+
"name": "cola",
|
14 |
+
"split": "test[:10]",
|
15 |
+
"args_for_task": {
|
16 |
+
"metric": "accuracy",
|
17 |
+
"input_column": "sentence",
|
18 |
+
"label_column": "label",
|
19 |
+
"label_mapping": {
|
20 |
+
"LABEL_0": 0.0,
|
21 |
+
"LABEL_1": 1.0
|
22 |
+
}
|
23 |
+
}
|
24 |
+
},
|
25 |
+
{
|
26 |
+
"data": "glue",
|
27 |
+
"name": "sst2",
|
28 |
+
"split": "validation[:10]",
|
29 |
+
"args_for_task": {
|
30 |
+
"metric": "accuracy",
|
31 |
+
"input_column": "sentence",
|
32 |
+
"label_column": "label",
|
33 |
+
"label_mapping": {
|
34 |
+
"LABEL_0": 0.0,
|
35 |
+
"LABEL_1": 1.0
|
36 |
+
}
|
37 |
+
}
|
38 |
+
},
|
39 |
+
{
|
40 |
+
"data": "glue",
|
41 |
+
"name": "mnli",
|
42 |
+
"split": "validation_mismatched[:10]",
|
43 |
+
"args_for_task": {
|
44 |
+
"metric": "accuracy",
|
45 |
+
"input_column": "premise",
|
46 |
+
"second_input_column": "hypothesis",
|
47 |
+
"label_mapping": {
|
48 |
+
"LABEL_0": 0,
|
49 |
+
"LABEL_1": 1,
|
50 |
+
"LABEL_2": 2
|
51 |
+
},
|
52 |
+
"label_column": "label"
|
53 |
+
}
|
54 |
+
},
|
55 |
+
{
|
56 |
+
"data": "glue",
|
57 |
+
"name": "mrpc",
|
58 |
+
"split": "validation[:10]",
|
59 |
+
"args_for_task": {
|
60 |
+
"metric": "accuracy",
|
61 |
+
"input_column": "sentence1",
|
62 |
+
"second_input_column": "sentence2",
|
63 |
+
"label_mapping": {
|
64 |
+
"LABEL_0": 0,
|
65 |
+
"LABEL_1": 1
|
66 |
+
},
|
67 |
+
"label_column": "label"
|
68 |
+
}
|
69 |
+
},
|
70 |
+
{
|
71 |
+
"data": "glue",
|
72 |
+
"name": "qqp",
|
73 |
+
"split": "validation[:10]",
|
74 |
+
"args_for_task": {
|
75 |
+
"metric": "accuracy",
|
76 |
+
"input_column": "question1",
|
77 |
+
"second_input_column": "question2",
|
78 |
+
"label_mapping": {
|
79 |
+
"LABEL_0": 0,
|
80 |
+
"LABEL_1": 1
|
81 |
+
},
|
82 |
+
"label_column": "label"
|
83 |
+
}
|
84 |
+
},
|
85 |
+
{
|
86 |
+
"data": "glue",
|
87 |
+
"name": "qnli",
|
88 |
+
"split": "validation[:10]",
|
89 |
+
"args_for_task": {
|
90 |
+
"metric": "accuracy",
|
91 |
+
"input_column": "question",
|
92 |
+
"second_input_column": "sentence",
|
93 |
+
"label_mapping": {
|
94 |
+
"LABEL_0": 0,
|
95 |
+
"LABEL_1": 1
|
96 |
+
},
|
97 |
+
"label_column": "label"
|
98 |
+
}
|
99 |
+
},
|
100 |
+
{
|
101 |
+
"data": "glue",
|
102 |
+
"name": "rte",
|
103 |
+
"split": "validation[:10]",
|
104 |
+
"args_for_task": {
|
105 |
+
"metric": "accuracy",
|
106 |
+
"input_column": "sentence1",
|
107 |
+
"second_input_column": "sentence2",
|
108 |
+
"label_mapping": {
|
109 |
+
"LABEL_0": 0,
|
110 |
+
"LABEL_1": 1
|
111 |
+
},
|
112 |
+
"label_column": "label"
|
113 |
+
}
|
114 |
+
},
|
115 |
+
{
|
116 |
+
"data": "glue",
|
117 |
+
"name": "wnli",
|
118 |
+
"split": "validation[:10]",
|
119 |
+
"args_for_task": {
|
120 |
+
"metric": "accuracy",
|
121 |
+
"input_column": "sentence1",
|
122 |
+
"second_input_column": "sentence2",
|
123 |
+
"label_mapping": {
|
124 |
+
"LABEL_0": 0,
|
125 |
+
"LABEL_1": 1
|
126 |
+
},
|
127 |
+
"label_column": "label"
|
128 |
+
}
|
129 |
+
}
|
130 |
+
]
|
131 |
+
}
|
132 |
+
"""
|
133 |
def setup(self):
|
134 |
self.preprocessor = lambda x: {"text": x["text"].lower()}
|
135 |
self.suite = [
|
136 |
SubTask(
|
137 |
task_type="text-classification",
|
138 |
+
data="glue",
|
139 |
+
subset="wnli",
|
140 |
+
split="validation[:10]",
|
141 |
args_for_task={
|
142 |
"metric": "accuracy",
|
143 |
+
"input_column": "sentence1",
|
144 |
+
"second_input_column": "sentence2",
|
145 |
"label_column": "label",
|
146 |
"label_mapping": {
|
147 |
+
"LABEL_0": 0,
|
148 |
+
"LABEL_1": 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
149 |
}
|
150 |
}
|
151 |
)
|