Hugging Face's logo

Spaces:
mathemakitten
/
glue-suite-v2
Running

App Files Community
mathemakitten commited on
Commit
037729c
•
1 Parent(s): d085278

gloo

Browse files
Files changed (1) hide show
  1. glue-suite-v2.py +130 -21
glue-suite-v2.py CHANGED
@@ -7,36 +7,145 @@ class Suite(evaluate.EvaluationSuite):
7
  def __init__(self, name):
8
  super().__init__(name)
9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
10
  def setup(self):
11
  self.preprocessor = lambda x: {"text": x["text"].lower()}
12
  self.suite = [
13
  SubTask(
14
  task_type="text-classification",
15
- data="imdb",
16
- split="test[:10]",
17
- data_preprocessor=self.preprocessor,
18
  args_for_task={
19
  "metric": "accuracy",
20
- "input_column": "text",
 
21
  "label_column": "label",
22
  "label_mapping": {
23
- "LABEL_0": 0.0,
24
- "LABEL_1": 1.0
25
- }
26
- }
27
- ),
28
- SubTask(
29
- task_type="text-classification",
30
- data="sst2",
31
- split="test[:10]",
32
- data_preprocessor=lambda x: {"sentence": x["sentence"].lower()},
33
- args_for_task={
34
- "metric": "accuracy",
35
- "input_column": "sentence",
36
- "label_column": "label",
37
- "label_mapping": {
38
- "LABEL_0": 0.0,
39
- "LABEL_1": 1.0
40
  }
41
  }
42
  )
 
7
  def __init__(self, name):
8
  super().__init__(name)
9
 
10
+ """
11
+ {
12
+ "data": "glue",
13
+ "name": "cola",
14
+ "split": "test[:10]",
15
+ "args_for_task": {
16
+ "metric": "accuracy",
17
+ "input_column": "sentence",
18
+ "label_column": "label",
19
+ "label_mapping": {
20
+ "LABEL_0": 0.0,
21
+ "LABEL_1": 1.0
22
+ }
23
+ }
24
+ },
25
+ {
26
+ "data": "glue",
27
+ "name": "sst2",
28
+ "split": "validation[:10]",
29
+ "args_for_task": {
30
+ "metric": "accuracy",
31
+ "input_column": "sentence",
32
+ "label_column": "label",
33
+ "label_mapping": {
34
+ "LABEL_0": 0.0,
35
+ "LABEL_1": 1.0
36
+ }
37
+ }
38
+ },
39
+ {
40
+ "data": "glue",
41
+ "name": "mnli",
42
+ "split": "validation_mismatched[:10]",
43
+ "args_for_task": {
44
+ "metric": "accuracy",
45
+ "input_column": "premise",
46
+ "second_input_column": "hypothesis",
47
+ "label_mapping": {
48
+ "LABEL_0": 0,
49
+ "LABEL_1": 1,
50
+ "LABEL_2": 2
51
+ },
52
+ "label_column": "label"
53
+ }
54
+ },
55
+ {
56
+ "data": "glue",
57
+ "name": "mrpc",
58
+ "split": "validation[:10]",
59
+ "args_for_task": {
60
+ "metric": "accuracy",
61
+ "input_column": "sentence1",
62
+ "second_input_column": "sentence2",
63
+ "label_mapping": {
64
+ "LABEL_0": 0,
65
+ "LABEL_1": 1
66
+ },
67
+ "label_column": "label"
68
+ }
69
+ },
70
+ {
71
+ "data": "glue",
72
+ "name": "qqp",
73
+ "split": "validation[:10]",
74
+ "args_for_task": {
75
+ "metric": "accuracy",
76
+ "input_column": "question1",
77
+ "second_input_column": "question2",
78
+ "label_mapping": {
79
+ "LABEL_0": 0,
80
+ "LABEL_1": 1
81
+ },
82
+ "label_column": "label"
83
+ }
84
+ },
85
+ {
86
+ "data": "glue",
87
+ "name": "qnli",
88
+ "split": "validation[:10]",
89
+ "args_for_task": {
90
+ "metric": "accuracy",
91
+ "input_column": "question",
92
+ "second_input_column": "sentence",
93
+ "label_mapping": {
94
+ "LABEL_0": 0,
95
+ "LABEL_1": 1
96
+ },
97
+ "label_column": "label"
98
+ }
99
+ },
100
+ {
101
+ "data": "glue",
102
+ "name": "rte",
103
+ "split": "validation[:10]",
104
+ "args_for_task": {
105
+ "metric": "accuracy",
106
+ "input_column": "sentence1",
107
+ "second_input_column": "sentence2",
108
+ "label_mapping": {
109
+ "LABEL_0": 0,
110
+ "LABEL_1": 1
111
+ },
112
+ "label_column": "label"
113
+ }
114
+ },
115
+ {
116
+ "data": "glue",
117
+ "name": "wnli",
118
+ "split": "validation[:10]",
119
+ "args_for_task": {
120
+ "metric": "accuracy",
121
+ "input_column": "sentence1",
122
+ "second_input_column": "sentence2",
123
+ "label_mapping": {
124
+ "LABEL_0": 0,
125
+ "LABEL_1": 1
126
+ },
127
+ "label_column": "label"
128
+ }
129
+ }
130
+ ]
131
+ }
132
+ """
133
  def setup(self):
134
  self.preprocessor = lambda x: {"text": x["text"].lower()}
135
  self.suite = [
136
  SubTask(
137
  task_type="text-classification",
138
+ data="glue",
139
+ subset="wnli",
140
+ split="validation[:10]",
141
  args_for_task={
142
  "metric": "accuracy",
143
+ "input_column": "sentence1",
144
+ "second_input_column": "sentence2",
145
  "label_column": "label",
146
  "label_mapping": {
147
+ "LABEL_0": 0,
148
+ "LABEL_1": 1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
149
  }
150
  }
151
  )