ugh

glue-suite-v2.py

@@ -13,36 +13,34 @@ class Suite(evaluate.EvaluationSuite):
             citation="insert citation here",
             features=Features({"predictions": Value("int64"), "references": Value("int64")}))
 
-    def setup(self):
-        self.preprocessor = None #lambda x: x["text"].lower()
-
-        print(SubTask)
-        self.suite = [
-            SubTask(
-                data="imdb",
-                split="test",
-                data_preprocessor=lambda x: x["text"].lower(),#self.preprocessor,
-                args_for_task={"metric": "accuracy",
-                    "input_column": "text",
-                    "label_column": "label",
-                    "label_mapping": {
-                        "LABEL_0": 0.0,
-                        "LABEL_1": 1.0
-                    }
+    preprocessor = None #lambda x: x["text"].lower()
+    suite = [
+        SubTask(
+            data="imdb",
+            split="test",
+            data_preprocessor=lambda x: x["text"].lower(),#self.preprocessor,
+            args_for_task={
+                "metric": "accuracy",
+                "input_column": "text",
+                "label_column": "label",
+                "label_mapping": {
+                    "LABEL_0": 0.0,
+                    "LABEL_1": 1.0
                 }
-            ),
-            SubTask(
-                data="sst2",
-                split="test[:10]",
-                data_preprocessor=self.preprocessor,
-                # args_for_task={
-                #     "metric": "accuracy",
-                #     "input_column": "sentence",
-                #     "label_column": "label",
-                #     "label_mapping": {
-                #         "LABEL_0": 0.0,
-                #         "LABEL_1": 1.0
-                #     }
-                # }
-            )
-        ]
+            }
+        ),
+        SubTask(
+            data="sst2",
+            split="test[:10]",
+            data_preprocessor=self.preprocessor,
+            # args_for_task={
+            #     "metric": "accuracy",
+            #     "input_column": "sentence",
+            #     "label_column": "label",
+            #     "label_mapping": {
+            #         "LABEL_0": 0.0,
+            #         "LABEL_1": 1.0
+            #     }
+            # }
+        )
+    ]