mathemakitten commited on
Commit
140e94b
1 Parent(s): efa4e1c
Files changed (1) hide show
  1. glue-suite-v2.py +31 -30
glue-suite-v2.py CHANGED
@@ -13,34 +13,35 @@ class Suite(evaluate.EvaluationSuite):
13
  citation="insert citation here",
14
  features=Features({"predictions": Value("int64"), "references": Value("int64")}))
15
 
16
- preprocessor = None #lambda x: x["text"].lower()
17
- suite = [
18
- SubTask(
19
- data="imdb",
20
- split="test",
21
- data_preprocessor=lambda x: x["text"].lower(),#self.preprocessor,
22
- args_for_task={
23
- "metric": "accuracy",
24
- "input_column": "text",
25
- "label_column": "label",
26
- "label_mapping": {
27
- "LABEL_0": 0.0,
28
- "LABEL_1": 1.0
 
 
29
  }
30
- }
31
- ),
32
- SubTask(
33
- data="sst2",
34
- split="test[:10]",
35
- data_preprocessor=self.preprocessor,
36
- # args_for_task={
37
- # "metric": "accuracy",
38
- # "input_column": "sentence",
39
- # "label_column": "label",
40
- # "label_mapping": {
41
- # "LABEL_0": 0.0,
42
- # "LABEL_1": 1.0
43
- # }
44
- # }
45
- )
46
- ]
 
13
  citation="insert citation here",
14
  features=Features({"predictions": Value("int64"), "references": Value("int64")}))
15
 
16
+ def setup(self):
17
+ self.preprocessor = None #lambda x: x["text"].lower()
18
+ self.suite = [
19
+ SubTask(
20
+ data="imdb",
21
+ split="test",
22
+ data_preprocessor=lambda x: x["text"].lower(),#self.preprocessor,
23
+ args_for_task={
24
+ "metric": "accuracy",
25
+ "input_column": "text",
26
+ "label_column": "label",
27
+ "label_mapping": {
28
+ "LABEL_0": 0.0,
29
+ "LABEL_1": 1.0
30
+ }
31
  }
32
+ ),
33
+ SubTask(
34
+ data="sst2",
35
+ split="test[:10]",
36
+ data_preprocessor=self.preprocessor,
37
+ # args_for_task={
38
+ # "metric": "accuracy",
39
+ # "input_column": "sentence",
40
+ # "label_column": "label",
41
+ # "label_mapping": {
42
+ # "LABEL_0": 0.0,
43
+ # "LABEL_1": 1.0
44
+ # }
45
+ # }
46
+ )
47
+ ]