Repository: flairNLP/flair

Issue:
Add SentEval classification tasks
[SentEval](https://github.com/facebookresearch/SentEval) is a toolkit for evaluating general-purpose sentence representations. They define a set of benchmark tasks for which they learn standard classifiers on top of sentence representations (representations are fixed so there is no task-specific fine-tuning or learning of representations). Details are provided in [their paper](https://arxiv.org/abs/1803.05449). 

We want to add the same evaluation datasets to Flair to enable the same type of evaluation, closely integrated into Flair. We start with text classification since this is already supported in Flair, so we add the following datasets: 

- [x] MR : binary sentiment in movie reviews
- [x] CR : binary sentiment in customer reviews
- [x] SUBJ : binary classification whether a text is 'subjective' or 'objective'
- [x] MPQA : binary sentiment in phrasal constructions
- [x] SST (binary) : binary sentiment in movie reviews (stanford sentiment treebank)
- [x] SST (fine-grained) : 5-class sentiment in movie reviews (stanford sentiment treebank)









Discussion:

Resolving patch:
diff --git a/flair/data.py b/flair/data.py
--- a/flair/data.py
+++ b/flair/data.py
@@ -220,7 +220,7 @@ def set_label(self, label_type: str, value: str, score: float = 1.):
         return self
 
     def get_labels(self, label_type: str = None):
-        if label_type == None:
+        if label_type is None:
             return self.labels
 
         return self.annotation_layers[label_type] if label_type in self.annotation_layers else []
diff --git a/flair/datasets.py b/flair/datasets.py
--- a/flair/datasets.py
+++ b/flair/datasets.py
@@ -246,6 +246,7 @@ def __init__(
             self,
             data_folder: Union[str, Path],
             column_name_map: Dict[int, str],
+            label_type: str = 'class',
             train_file=None,
             test_file=None,
             dev_file=None,
@@ -279,6 +280,7 @@ def __init__(
         train: FlairDataset = CSVClassificationDataset(
             train_file,
             column_name_map,
+            label_type=label_type,
             tokenizer=tokenizer,
             max_tokens_per_doc=max_tokens_per_doc,
             max_chars_per_doc=max_chars_per_doc,
@@ -290,6 +292,7 @@ def __init__(
         test: FlairDataset = CSVClassificationDataset(
             test_file,
             column_name_map,
+            label_type=label_type,
             tokenizer=tokenizer,
             max_tokens_per_doc=max_tokens_per_doc,
             max_chars_per_doc=max_chars_per_doc,
@@ -301,6 +304,7 @@ def __init__(
         dev: FlairDataset = CSVClassificationDataset(
             dev_file,
             column_name_map,
+            label_type=label_type,
             tokenizer=tokenizer,
             max_tokens_per_doc=max_tokens_per_doc,
             max_chars_per_doc=max_chars_per_doc,
@@ -778,6 +782,7 @@ def __init__(
             self,
             path_to_file: Union[str, Path],
             column_name_map: Dict[int, str],
+            label_type: str = "class",
             max_tokens_per_doc: int = -1,
             max_chars_per_doc: int = -1,
             tokenizer=segtok_tokenizer,
@@ -870,7 +875,7 @@ def __init__(
                                 self.column_name_map[column].startswith("label")
                                 and row[column]
                         ):
-                            sentence.add_label(row[column])
+                            sentence.add_label(label_type, row[column])
 
                     if 0 < self.max_tokens_per_doc < len(sentence):
                         sentence.tokens = sentence.tokens[: self.max_tokens_per_doc]
@@ -1526,6 +1531,229 @@ def __init__(
         )
 
 
+class SENTEVAL_CR(ClassificationCorpus):
+    def __init__(
+            self,
+            in_memory: bool = True,
+    ):
+        # this dataset name
+        dataset_name = self.__class__.__name__.lower()
+
+        # default dataset folder is the cache root
+        data_folder = Path(flair.cache_root) / "datasets" / dataset_name
+
+        # download data if necessary
+        if not (data_folder / "train.txt").is_file():
+
+            # download senteval datasets if necessary und unzip
+            senteval_path = "https://dl.fbaipublicfiles.com/senteval/senteval_data/datasmall_NB_ACL12.zip"
+            cached_path(senteval_path, Path("datasets") / "senteval")
+            senteval_folder = Path(flair.cache_root) / "datasets" / "senteval"
+            unzip_file(senteval_folder / "datasmall_NB_ACL12.zip", senteval_folder)
+
+            # create dataset directory if necessary
+            if not os.path.exists(data_folder):
+                os.makedirs(data_folder)
+
+            # create train.txt file by iterating over pos and neg file
+            with open(data_folder / "train.txt", "a") as train_file:
+
+                with open(senteval_folder / "data" / "customerr" / "custrev.pos", encoding="latin1") as file:
+                    for line in file:
+                        train_file.write(f"__label__POSITIVE {line}")
+
+                with open(senteval_folder / "data" / "customerr" / "custrev.neg", encoding="latin1") as file:
+                    for line in file:
+                        train_file.write(f"__label__NEGATIVE {line}")
+
+        super(SENTEVAL_CR, self).__init__(
+            data_folder, label_type='sentiment', tokenizer=segtok_tokenizer, in_memory=in_memory
+        )
+
+
+class SENTEVAL_MR(ClassificationCorpus):
+    def __init__(
+            self,
+            in_memory: bool = True,
+    ):
+        # this dataset name
+        dataset_name = self.__class__.__name__.lower()
+
+        # default dataset folder is the cache root
+        data_folder = Path(flair.cache_root) / "datasets" / dataset_name
+
+        # download data if necessary
+        if not (data_folder / "train.txt").is_file():
+
+            # download senteval datasets if necessary und unzip
+            senteval_path = "https://dl.fbaipublicfiles.com/senteval/senteval_data/datasmall_NB_ACL12.zip"
+            cached_path(senteval_path, Path("datasets") / "senteval")
+            senteval_folder = Path(flair.cache_root) / "datasets" / "senteval"
+            unzip_file(senteval_folder / "datasmall_NB_ACL12.zip", senteval_folder)
+
+            # create dataset directory if necessary
+            if not os.path.exists(data_folder):
+                os.makedirs(data_folder)
+
+            # create train.txt file by iterating over pos and neg file
+            with open(data_folder / "train.txt", "a") as train_file:
+
+                with open(senteval_folder / "data" / "rt10662" / "rt-polarity.pos", encoding="latin1") as file:
+                    for line in file:
+                        train_file.write(f"__label__POSITIVE {line}")
+
+                with open(senteval_folder / "data" / "rt10662" / "rt-polarity.neg", encoding="latin1") as file:
+                    for line in file:
+                        train_file.write(f"__label__NEGATIVE {line}")
+
+        super(SENTEVAL_MR, self).__init__(
+            data_folder, label_type='sentiment', tokenizer=segtok_tokenizer, in_memory=in_memory
+        )
+
+
+class SENTEVAL_SUBJ(ClassificationCorpus):
+    def __init__(
+            self,
+            in_memory: bool = True,
+    ):
+        # this dataset name
+        dataset_name = self.__class__.__name__.lower()
+
+        # default dataset folder is the cache root
+        data_folder = Path(flair.cache_root) / "datasets" / dataset_name
+
+        # download data if necessary
+        if not (data_folder / "train.txt").is_file():
+
+            # download senteval datasets if necessary und unzip
+            senteval_path = "https://dl.fbaipublicfiles.com/senteval/senteval_data/datasmall_NB_ACL12.zip"
+            cached_path(senteval_path, Path("datasets") / "senteval")
+            senteval_folder = Path(flair.cache_root) / "datasets" / "senteval"
+            unzip_file(senteval_folder / "datasmall_NB_ACL12.zip", senteval_folder)
+
+            # create dataset directory if necessary
+            if not os.path.exists(data_folder):
+                os.makedirs(data_folder)
+
+            # create train.txt file by iterating over pos and neg file
+            with open(data_folder / "train.txt", "a") as train_file:
+
+                with open(senteval_folder / "data" / "subj" / "subj.subjective", encoding="latin1") as file:
+                    for line in file:
+                        train_file.write(f"__label__SUBJECTIVE {line}")
+
+                with open(senteval_folder / "data" / "subj" / "subj.objective", encoding="latin1") as file:
+                    for line in file:
+                        train_file.write(f"__label__OBJECTIVE {line}")
+
+        super(SENTEVAL_SUBJ, self).__init__(
+            data_folder, label_type='objectivity', tokenizer=segtok_tokenizer, in_memory=in_memory
+        )
+
+
+class SENTEVAL_MPQA(ClassificationCorpus):
+    def __init__(
+            self,
+            in_memory: bool = True,
+    ):
+        # this dataset name
+        dataset_name = self.__class__.__name__.lower()
+
+        # default dataset folder is the cache root
+        data_folder = Path(flair.cache_root) / "datasets" / dataset_name
+
+        # download data if necessary
+        if not (data_folder / "train.txt").is_file():
+
+            # download senteval datasets if necessary und unzip
+            senteval_path = "https://dl.fbaipublicfiles.com/senteval/senteval_data/datasmall_NB_ACL12.zip"
+            cached_path(senteval_path, Path("datasets") / "senteval")
+            senteval_folder = Path(flair.cache_root) / "datasets" / "senteval"
+            unzip_file(senteval_folder / "datasmall_NB_ACL12.zip", senteval_folder)
+
+            # create dataset directory if necessary
+            if not os.path.exists(data_folder):
+                os.makedirs(data_folder)
+
+            # create train.txt file by iterating over pos and neg file
+            with open(data_folder / "train.txt", "a") as train_file:
+
+                with open(senteval_folder / "data" / "mpqa" / "mpqa.pos", encoding="latin1") as file:
+                    for line in file:
+                        train_file.write(f"__label__POSITIVE {line}")
+
+                with open(senteval_folder / "data" / "mpqa" / "mpqa.neg", encoding="latin1") as file:
+                    for line in file:
+                        train_file.write(f"__label__NEGATIVE {line}")
+
+        super(SENTEVAL_MPQA, self).__init__(
+            data_folder, label_type='sentiment', tokenizer=segtok_tokenizer, in_memory=in_memory
+        )
+
+
+class SENTEVAL_SST_BINARY(CSVClassificationCorpus):
+    def __init__(
+            self,
+            in_memory: bool = True,
+    ):
+        # this dataset name
+        dataset_name = self.__class__.__name__.lower()
+
+        # default dataset folder is the cache root
+        data_folder = Path(flair.cache_root) / "datasets" / dataset_name
+
+        # download data if necessary
+        if not (data_folder / "train.txt").is_file():
+
+            # download senteval datasets if necessary und unzip
+            cached_path('https://raw.githubusercontent.com/PrincetonML/SIF/master/data/sentiment-train', Path("datasets") / dataset_name)
+            cached_path('https://raw.githubusercontent.com/PrincetonML/SIF/master/data/sentiment-test', Path("datasets") / dataset_name)
+            cached_path('https://raw.githubusercontent.com/PrincetonML/SIF/master/data/sentiment-dev', Path("datasets") / dataset_name)
+
+        super(SENTEVAL_SST_BINARY, self).__init__(
+            data_folder,
+            column_name_map={0: 'text', 1: 'label'},
+            tokenizer=segtok_tokenizer,
+            in_memory=in_memory,
+            delimiter='\t',
+            quotechar=None,
+        )
+
+
+class SENTEVAL_SST_GRANULAR(ClassificationCorpus):
+    def __init__(
+            self,
+            in_memory: bool = True,
+    ):
+        # this dataset name
+        dataset_name = self.__class__.__name__.lower()
+
+        # default dataset folder is the cache root
+        data_folder = Path(flair.cache_root) / "datasets" / dataset_name
+
+        # download data if necessary
+        if not (data_folder / "train.txt").is_file():
+
+            # download senteval datasets if necessary und unzip
+            cached_path('https://raw.githubusercontent.com/AcademiaSinicaNLPLab/sentiment_dataset/master/data/stsa.fine.train', Path("datasets") / dataset_name / 'raw')
+            cached_path('https://raw.githubusercontent.com/AcademiaSinicaNLPLab/sentiment_dataset/master/data/stsa.fine.test', Path("datasets") / dataset_name / 'raw')
+            cached_path('https://raw.githubusercontent.com/AcademiaSinicaNLPLab/sentiment_dataset/master/data/stsa.fine.dev', Path("datasets") / dataset_name / 'raw')
+
+            # convert to FastText format
+            for split in ['train', 'dev', 'test']:
+                with open(data_folder / f"{split}.txt", "w") as train_file:
+
+                    with open(data_folder / 'raw' / f'stsa.fine.{split}', encoding="latin1") as file:
+                        for line in file:
+                            train_file.write(f"__label__{line[0]} {line[2:]}")
+
+        super(SENTEVAL_SST_GRANULAR, self).__init__(
+            data_folder,
+            tokenizer=segtok_tokenizer,
+            in_memory=in_memory,
+        )
+
+
 class GERMEVAL(ColumnCorpus):
     def __init__(
             self,
diff --git a/flair/models/text_classification_model.py b/flair/models/text_classification_model.py
--- a/flair/models/text_classification_model.py
+++ b/flair/models/text_classification_model.py
@@ -34,7 +34,7 @@ def __init__(
         self,
         document_embeddings: flair.embeddings.DocumentEmbeddings,
         label_dictionary: Dictionary,
-        label_type: str = "class",
+        label_type: str = None,
         multi_label: bool = None,
         multi_label_threshold: float = 0.5,
         beta: float = 1.0,
@@ -121,10 +121,12 @@ def _get_state_dict(self):
     def _init_model_with_state_dict(state):
         beta = 1.0 if "beta" not in state.keys() else state["beta"]
         weights = None if "weight_dict" not in state.keys() else state["weight_dict"]
+        label_type = None if "label_type" not in state.keys() else state["label_type"]
 
         model = TextClassifier(
             document_embeddings=state["document_embeddings"],
             label_dictionary=state["label_dictionary"],
+            label_type=label_type,
             multi_label=state["multi_label"],
             beta=beta,
             loss_weights=weights,
@@ -168,6 +170,8 @@ def predict(
         :param use_tokenizer: a custom tokenizer when string are provided (default is space based tokenizer).
         :return: the list of sentences containing the labels
         """
+        predicted_label_type = self.label_type if self.label_type is not None else 'class'
+
         with torch.no_grad():
             if not sentences:
                 return sentences
@@ -231,7 +235,7 @@ def predict(
 
                 for (sentence, labels) in zip(batch, predicted_labels):
                     for label in labels:
-                        sentence.add_label(self.label_type, label.value, label.score)
+                        sentence.add_label(predicted_label_type, label.value, label.score)
 
                 # clearing token embeddings to save memory
                 store_embeddings(batch, storage_mode=embedding_storage_mode)


Test patch: