WIP

glue-suite-v2.py

@@ -1,4 +1,17 @@
-from evaluate.evaluation_suite import SubTask
+
+from typing import Optional, Union, Callable
+from dataclasses import dataclass
+from datasets import Dataset
+
+@dataclass
+class SubTask:
+    model_or_pipeline: Optional[Union[str, "Pipeline", Callable, "PreTrainedModel", "TFPreTrainedModel"]] = None
+    data: Optional[Union[str, Dataset]] = None
+    subset: Optional[str] = None
+    split: Optional[str] = None
+    data_preprocessor: Optional[Callable] = None,
+    args_for_task: Optional[dict] = None
+
 
 preprocessor = lambda x: x["text"].lower()