Hugging Face's logo

Spaces:
mathemakitten
/
glue-suite-v2
Running

App Files Community
mathemakitten commited on
Commit
ec0b829
•
1 Parent(s): 037729c

gloo fr

Browse files
Files changed (1) hide show
  1. glue-suite-v2.py +111 -124
glue-suite-v2.py CHANGED
@@ -7,132 +7,119 @@ class Suite(evaluate.EvaluationSuite):
7
  def __init__(self, name):
8
  super().__init__(name)
9
 
10
- """
11
- {
12
- "data": "glue",
13
- "name": "cola",
14
- "split": "test[:10]",
15
- "args_for_task": {
16
- "metric": "accuracy",
17
- "input_column": "sentence",
18
- "label_column": "label",
19
- "label_mapping": {
20
- "LABEL_0": 0.0,
21
- "LABEL_1": 1.0
22
- }
23
- }
24
- },
25
- {
26
- "data": "glue",
27
- "name": "sst2",
28
- "split": "validation[:10]",
29
- "args_for_task": {
30
- "metric": "accuracy",
31
- "input_column": "sentence",
32
- "label_column": "label",
33
- "label_mapping": {
34
- "LABEL_0": 0.0,
35
- "LABEL_1": 1.0
36
- }
37
- }
38
- },
39
- {
40
- "data": "glue",
41
- "name": "mnli",
42
- "split": "validation_mismatched[:10]",
43
- "args_for_task": {
44
- "metric": "accuracy",
45
- "input_column": "premise",
46
- "second_input_column": "hypothesis",
47
- "label_mapping": {
48
- "LABEL_0": 0,
49
- "LABEL_1": 1,
50
- "LABEL_2": 2
51
- },
52
- "label_column": "label"
53
- }
54
- },
55
- {
56
- "data": "glue",
57
- "name": "mrpc",
58
- "split": "validation[:10]",
59
- "args_for_task": {
60
- "metric": "accuracy",
61
- "input_column": "sentence1",
62
- "second_input_column": "sentence2",
63
- "label_mapping": {
64
- "LABEL_0": 0,
65
- "LABEL_1": 1
66
- },
67
- "label_column": "label"
68
- }
69
- },
70
- {
71
- "data": "glue",
72
- "name": "qqp",
73
- "split": "validation[:10]",
74
- "args_for_task": {
75
- "metric": "accuracy",
76
- "input_column": "question1",
77
- "second_input_column": "question2",
78
- "label_mapping": {
79
- "LABEL_0": 0,
80
- "LABEL_1": 1
81
- },
82
- "label_column": "label"
83
- }
84
- },
85
- {
86
- "data": "glue",
87
- "name": "qnli",
88
- "split": "validation[:10]",
89
- "args_for_task": {
90
- "metric": "accuracy",
91
- "input_column": "question",
92
- "second_input_column": "sentence",
93
- "label_mapping": {
94
- "LABEL_0": 0,
95
- "LABEL_1": 1
96
- },
97
- "label_column": "label"
98
- }
99
- },
100
- {
101
- "data": "glue",
102
- "name": "rte",
103
- "split": "validation[:10]",
104
- "args_for_task": {
105
- "metric": "accuracy",
106
- "input_column": "sentence1",
107
- "second_input_column": "sentence2",
108
- "label_mapping": {
109
- "LABEL_0": 0,
110
- "LABEL_1": 1
111
- },
112
- "label_column": "label"
113
- }
114
- },
115
- {
116
- "data": "glue",
117
- "name": "wnli",
118
- "split": "validation[:10]",
119
- "args_for_task": {
120
- "metric": "accuracy",
121
- "input_column": "sentence1",
122
- "second_input_column": "sentence2",
123
- "label_mapping": {
124
- "LABEL_0": 0,
125
- "LABEL_1": 1
126
- },
127
- "label_column": "label"
128
- }
129
- }
130
- ]
131
- }
132
- """
133
  def setup(self):
134
  self.preprocessor = lambda x: {"text": x["text"].lower()}
135
  self.suite = [
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
136
  SubTask(
137
  task_type="text-classification",
138
  data="glue",
@@ -149,4 +136,4 @@ class Suite(evaluate.EvaluationSuite):
149
  }
150
  }
151
  )
152
- ]
 
7
  def __init__(self, name):
8
  super().__init__(name)
9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
10
  def setup(self):
11
  self.preprocessor = lambda x: {"text": x["text"].lower()}
12
  self.suite = [
13
+ SubTask(
14
+ task_type="text-classification",
15
+ data="glue",
16
+ subset="cola",
17
+ split="test[:10]",
18
+ args_for_task={
19
+ "metric": "accuracy",
20
+ "input_column": "sentence",
21
+ "label_column": "label",
22
+ "label_mapping": {
23
+ "LABEL_0": 0.0,
24
+ "LABEL_1": 1.0
25
+ }
26
+ }
27
+ ),
28
+ SubTask(
29
+ task_type="text-classification",
30
+ data="glue",
31
+ subset="sst2",
32
+ split="validation[:10]",
33
+ args_for_task={
34
+ "metric": "accuracy",
35
+ "input_column": "sentence",
36
+ "label_column": "label",
37
+ "label_mapping": {
38
+ "LABEL_0": 0.0,
39
+ "LABEL_1": 1.0
40
+ }
41
+ }
42
+ ),
43
+ SubTask(
44
+ task_type="text-classification",
45
+ data="glue",
46
+ subset="qqp",
47
+ split="validation[:10]",
48
+ args_for_task={
49
+ "metric": "accuracy",
50
+ "input_column": "question1",
51
+ "second_input_column": "question2",
52
+ "label_column": "label",
53
+ "label_mapping": {
54
+ "LABEL_0": 0,
55
+ "LABEL_1": 1
56
+ }
57
+ }
58
+ ),
59
+ SubTask(
60
+ task_type="text-classification",
61
+ data="glue",
62
+ subset="mrpc",
63
+ split="validation[:10]",
64
+ args_for_task={
65
+ "metric": "accuracy",
66
+ "input_column": "sentence1",
67
+ "second_input_column": "sentence2",
68
+ "label_column": "label",
69
+ "label_mapping": {
70
+ "LABEL_0": 0,
71
+ "LABEL_1": 1
72
+ }
73
+ }
74
+ ),
75
+ SubTask(
76
+ task_type="text-classification",
77
+ data="glue",
78
+ subset="mnli",
79
+ split="validation_mismatched[:10]",
80
+ args_for_task={
81
+ "metric": "accuracy",
82
+ "input_column": "premise",
83
+ "second_input_column": "hypothesis",
84
+ "label_mapping": {
85
+ "LABEL_0": 0,
86
+ "LABEL_1": 1,
87
+ "LABEL_2": 2
88
+ }
89
+ }
90
+ ),
91
+ SubTask(
92
+ task_type="text-classification",
93
+ data="glue",
94
+ subset="qnli",
95
+ split="validation[:10]",
96
+ args_for_task={
97
+ "metric": "accuracy",
98
+ "input_column": "question",
99
+ "second_input_column": "sentence",
100
+ "label_column": "label",
101
+ "label_mapping": {
102
+ "LABEL_0": 0,
103
+ "LABEL_1": 1
104
+ }
105
+ }
106
+ ),
107
+ SubTask(
108
+ task_type="text-classification",
109
+ data="glue",
110
+ subset="rte",
111
+ split="validation[:10]",
112
+ args_for_task={
113
+ "metric": "accuracy",
114
+ "input_column": "sentence1",
115
+ "second_input_column": "sentence2",
116
+ "label_column": "label",
117
+ "label_mapping": {
118
+ "LABEL_0": 0,
119
+ "LABEL_1": 1
120
+ }
121
+ }
122
+ ),
123
  SubTask(
124
  task_type="text-classification",
125
  data="glue",
 
136
  }
137
  }
138
  )
139
+ ]