Repository: flairNLP/flair

Issue:
Label cleaning heuristic does not consider `Sentence` labels

In #2472 , a change was introduced that replaces the internal label dictionary with one that replaces underscores in labels.

This causes crashes if you use labels with underscores, because here:
https://github.com/flairNLP/flair/blob/master/flair/models/tars_model.py#L61

when the sentences are prepared in the tars format, using the labels as keys does not exist.

stack trace:

```
  File "finetune.py", line 130, in main
    finetune_tars(tars_format_sampled, tars_format_sampled_dev, tars_format_test, dataset, output_dir)
  File "finetune.py", line 46, in finetune_tars
    results = trainer.train(base_path=output_path,
  File "/Users/markn/anaconda3/envs/annotate/lib/python3.8/site-packages/flair/trainers/trainer.py", line 467, in train
    loss = self.model.forward_loss(batch_step)
  File "/Users/markn/anaconda3/envs/annotate/lib/python3.8/site-packages/flair/models/tars_model.py", line 41, in forward_loss
    sentences = self._get_tars_formatted_sentences(data_points)
  File "/Users/markn/anaconda3/envs/annotate/lib/python3.8/site-packages/flair/models/tars_model.py", line 64, in _get_tars_formatted_sentences
    sampled_negative_labels = self._get_nearest_labels_for(positive_labels)
  File "/Users/markn/anaconda3/envs/annotate/lib/python3.8/site-packages/flair/models/tars_model.py", line 95, in _get_nearest_labels_for
    for plausible_label in self.label_nearest_map[label]:
KeyError: 'search_creative_work'
```



Discussion:

Resolving patch:
diff --git a/flair/models/tars_model.py b/flair/models/tars_model.py
--- a/flair/models/tars_model.py
+++ b/flair/models/tars_model.py
@@ -26,7 +26,6 @@ def __init__(self):
         self._current_task = None
         self._task_specific_attributes = {}
         self.label_nearest_map = None
-        self.clean_up_labels: bool = True
 
         super(FewshotClassifier, self).__init__()
 
@@ -164,14 +163,7 @@ def _compute_label_similarity_for_current_epoch(self):
 
     def get_current_label_dictionary(self):
         label_dictionary = self._task_specific_attributes[self._current_task]['label_dictionary']
-        if self.clean_up_labels:
-            # default: make new dictionary with modified labels (no underscores)
-            dictionary = Dictionary(add_unk=False)
-            for label in label_dictionary.get_items():
-                dictionary.add_item(label.replace("_", " "))
-            return dictionary
-        else:
-            return label_dictionary
+        return label_dictionary
 
     def get_current_label_type(self):
         return self._task_specific_attributes[self._current_task]['label_type']
@@ -678,14 +670,24 @@ def __init__(
             log.info("TARS initialized without a task. You need to call .add_and_switch_to_new_task() "
                      "before training this model")
 
+        self.clean_up_labels = True
+
+    def _clean(self, label_value: str) -> str:
+        if self.clean_up_labels:
+            return label_value.replace("_", " ")
+        else:
+            return label_value
+
     def _get_tars_formatted_sentence(self, label, sentence):
 
+        label = self._clean(label)
+
         original_text = sentence.to_tokenized_string()
 
         label_text_pair = f"{label} {self.separator} {original_text}" if self.prefix \
             else f"{original_text} {self.separator} {label}"
 
-        sentence_labels = [label.value for label in sentence.get_labels(self.get_current_label_type())]
+        sentence_labels = [self._clean(label.value) for label in sentence.get_labels(self.get_current_label_type())]
 
         tars_label = self.LABEL_MATCH if label in sentence_labels else self.LABEL_NO_MATCH
 
@@ -853,7 +855,6 @@ def predict(
                     # only use label with highest confidence if enforcing single-label predictions
                     if not multi_label:
                         if len(sentence.get_labels(label_name)) > 0:
-
                             # get all label scores and do an argmax to get the best label
                             label_scores = torch.tensor([label.score for label in sentence.get_labels(label_name)],
                                                         dtype=torch.float)


Test patch:
diff --git a/tests/resources/tasks/imdb_underscore/README.md b/tests/resources/tasks/imdb_underscore/README.md
new file mode 100644
--- /dev/null
+++ b/tests/resources/tasks/imdb_underscore/README.md
@@ -0,0 +1,13 @@
+## IMDB
+
+Data is taken from [here](http://ai.stanford.edu/~amaas/data/sentiment/).
+
+The dataset contains data for a binary sentiment classification. 
+We took a small random sample and converted it to the expected format of our data fetcher:
+```
+__label__<class_name> <text>
+```
+
+#### Publications Using the Dataset
+
+* Andrew L. Maas, Raymond E. Daly, Peter T. Pham, Dan Huang, Andrew Y. Ng, and Christopher Potts. (2011). Learning Word Vectors for Sentiment Analysis. The 49th Annual Meeting of the Association for Computational Linguistics (ACL 2011).
\ No newline at end of file
diff --git a/tests/resources/tasks/imdb_underscore/dev.txt b/tests/resources/tasks/imdb_underscore/dev.txt
new file mode 100644
--- /dev/null
+++ b/tests/resources/tasks/imdb_underscore/dev.txt
@@ -0,0 +1,5 @@
+__label__POSITIVE_Sentiment I rented this movie for two reasons. The first was all of the good things that I read about it. I definately was impressed, and couldn't agree more with all of the reviews I read. The second reason is because I know these guys. I don't know Mark as well as I do Mike. He hasn't changed much from the years we knew each other. I know him as "the reaper" from a local Milwaukee radio station (WMSE). The way he is in this film is genuinely Mike 100%. He once gave me a table that he made. It was a little one, about 15" high, that said Metallica on it. The odd thing about it was it only had two legs. Sometimes it's the thought that counts, and Mike always has thought about others first.
+__label__POSITIVE_Sentiment Julie Brown hilariously demolishes Madonna's attempt at a rockumentary with gut ripping humor and truly original and catchy songs that rival Madonna's own. Cinematography and sets are top notch.<br /><br />Kathy Griffin and Chris Elliott offer their own injections of comedy that enhances and compliments this film. Appearances by Bobcat Goldthwait and Wink Martindale, as themselves, is an added bonus.<br /><br />It's hard to tell if Brown's performance is meant to insult or playfully tease Madonna, though I hardly think the Material Girl would find humor in it.<br /><br />My Favorite line: "Why don't you come here (to the Phillipines); all they eat is dog and I'm a vegetarian."
+__label__NEGATIVE_Sentiment This is very much overrated. I guess it carries some nostalgic value for many people. It has its moments, but every scene is heavily overacted and the plot is quite shallow. With this cast it could have been much better.
+__label__NEGATIVE_Sentiment Sometimes when a film is panned by the critics one just has to see it to see if it is really as bad as the claim, well in this case it was. I believe it was meant to be a sex comedy but it was neither sexy nor funny, I think I chuckled once when Johnny Vegas failed to vault a roadside fence and that was it.<br /><br />The film follows the lives of four men who deliver potatoes for a living, each of them has a problem with their sex lives; Dave has left his wife and seeks out an old flame who once invited him for a threesome a few years before, Ferris is living with his mother in law who treats him as a sex object, Tolly has an obsession for involving strawberry jam in his encounters and Jeremy claims to be in a loving relationship but is in fact stalking some poor woman.<br /><br />The tone of the jokes varies from tasteless to "I don't believe that anybody could write that" an example of the the latter being when they tell a small child his father has died in a car crash, then for the punch line when the boy asks how his mum is they tell him she is in a coma. The blame for this all lands squarely at the feet of the writers, the actors obviously made a poor choice agreeing to be in this film but I didn't think their acting was all that bad... not that it was all that good either. Surprisingly for a fairly sleazy film about sex there is no nudity and the sex is never erotic.<br /><br />I certainly wouldn't recommend seeing this for any reason other than morbid curiosity although judging from other reviews this seems to be a love it or loathe it film, I suspect it would be better watched after a few beers and with your mates.
+__label__NEGATIVE_Sentiment This movie has some good performances, as others have pointed out, but suffers (as others have pointed out, except for the people who apparently are either friends of the filmmaker or the cast to otherwise explain why they would deem this a "10") from some self-conscious and self-absorbed film school padding and excess plots. This is the type of plot that Sex In The City could handle in a half hour episode, so there was no reason for it to be even an 88 minute movie. A perfect example of wasted footage is the fast forwarding montage in the first third of the movie. Some of the back story is merited, but too much time, for example, was spent on Daria character with the anal sex boyfriend and on the back story for Paulie, who was not a realistic character, although the actor did a decent job with the lines he was given. <br /><br />The worst aspect of the movie was the level of amateurish parts: from typos in the typed material, to bad jumps and edits, poor camera positions, angles, lighting problems throughout and, most glaringly, a poorly written script with a badly developed concept. If the writer (also the director and lead) had collaborated with someone, he might not have ended up with a 100% rotten score on Rotten Tomatoes, which further belays the ability of anyone to truly believe the people who gave it a 10 on the rating system here.
\ No newline at end of file
diff --git a/tests/resources/tasks/imdb_underscore/test.txt b/tests/resources/tasks/imdb_underscore/test.txt
new file mode 100644
--- /dev/null
+++ b/tests/resources/tasks/imdb_underscore/test.txt
@@ -0,0 +1,5 @@
+__label__NEGATIVE_Sentiment Thomas Hardy is one of my favorite authors. Some truly wonderful movies have been made from his novels ("Far From the Madding Crowd," "Tess of the D'Urbervilles," "The Mayor of Casterbridge"), and I had high hopes for this one. The Hallmark-Hall-of-Fame-ification of "Return of the Native" totally wrecked it. The cast was terrific, the photography excellent, but the script was dismal and the direction positively ruinous. People walked up to people, said lines, walked away. A meager excitement developed when Clive Owen and Catherine Zeta Jones (very young, very beautiful) exchanged a bit of flesh-pressing, but even Clive, who is a superb actor, couldn't save it. It was awash with the usual Hallmark "romantic" strings background music and pretend bumpkins offering plot exposition, and what could have been dynamite turned out to be awful. The richness of the above three movies was commpletely absent.
+__label__POSITIVE_Sentiment It finally hit me watching my VHS of Christmas in Connecticut what other film this one reminded me of. If it weren't for the fact that the other was done 20 years later, I'd say it was a remake.<br /><br />Just as Rock Hudson was a phony fishing expert for Abercrombie&Fitch who had to get some on the job training at a fishing tournament, Barbara Stanwyck plays an forties version of Martha Stewart.<br /><br />Stanwyck's a cooking columnist who's built up this whole image of living on a small Connecticut farm with husband and baby cooking all these marvelous delicacies. Trouble is she's unmarried, childless, writes her column from her apartment in New York and doesn't know how to boil water. But her writing is a hit with the public.<br /><br />Trouble comes when she's hijacked into cooking a home Christmas dinner for a war hero sailor played by Dennis Morgan who gets to sing a couple of songs as well. Got to keep up the image at any cost. And her publisher Sidney Greenstreet likes the idea so well that he invites himself to the dinner.<br /><br />So with borrowed farm, baby, and Reginald Gardiner who'd like to make it real with Stanwyck she tries to brazen it through. <br /><br />Christmas in Connecticut's now a Yuletide classic and deservedly so. The leads are warm and human and they get great support from the assembled players. S.Z. Sakall as the Hungarian restaurant owner/friend of Stanwyck from whom she gets her cooking information and Una O'Connor as the housekeeper have a nice chemistry between them. Reginald Gardiner and Stanwyck have no chemistry at all, obvious to all but Reggie and he's funny in his stuffed shirt way.<br /><br />Most people remember this film as one of Sidney Greenstreet's few ventures into comedy. If he's not an outright villain, a cynical observer of life or a tyrannical tycoon, Greenstreet is few other things on screen. Christmas in Connecticut gave him a rare opportunity to burlesque his own image and he made the most of it.<br /><br />In a biography of Barbara Stanwyck, she mentions she enjoyed making Christmas in Connecticut as a welcome change from some villainous parts like Double Indemnity she'd been doing recently. One of the things that made doing the film so enjoyable was that between takes, director Peter Godfrey and Greenstreet would do some impromptu entertaining of cast and crew with English Music Hall numbers. Made for a relaxed and warm set and the cast responded accordingly.<br /><br />Now if only someone had been filming those numbers.
+__label__NEGATIVE_Sentiment I saw this one on the late, late, late, late show back when MST3K was just a twinkle in Joel Hodgson's eye. I had the privilege of seeing it without knowing exactly how bad it would be. I didn't know that a woman with an arrow in her chest could run "to the ends of the earth" in about an hour and a half (thank God her father gave such specific directions) and then take days to make it back to the castle with help. I didn't know that a sword wielding barbarian-type could run into the forest and create a hanglider and flash powder bombs in under 30 seconds. I had no idea how disjointed a movie could be.<br /><br />It's a bad thing children, a very bad thing. If you enjoy bad movies, go for it.
+__label__POSITIVE_Sentiment When I first heard about "Greek," I figured I would watch it because it sounded ridiculous. Another of ABC Family's so-bad-they're-almost-good shows. But tuning in with a friend from college found us both enjoying the pilot episode a lot more than we had expected.<br /><br />As a member of a Greek society, I can say that a lot of the stereotypes that are brought up here are ones that come up almost every time someone starts talking about the sororities and fraternities on a campus. And are also very fun to play with just on are own, let alone to watch on a TV screen. The opening scene harkened to an only-slightly-dramatized version of preparing for an actual formal rush in some sororities and it continued on from there.<br /><br />This isn't a show for over-sensitive Greeks. If you get offended even at jokes about things that aren't-so-great about Greek life, then you'll spend the entire first episode, and probably many other, cringing and yelling. But everyone else should have a ton of fun watching it. It's nothing new, but when it comes to college, nothing ever is.
+__label__NEGATIVE_Sentiment Anyone who will pay to see Troma movies knows, and appreciates, what they are going to get. Having said that, I didn't think it was possible to make a movie this bad, and still be compelling. I found myself watching just to see how much worse it could get before the end. First off, it's an Indonesian action movie with an American main character who looks and acts like the bastard son of "Taxi"'s Christopher Lloyd and Rambo. He puts posters of himself dressed up like Sly's "Cobra" all over the place and even has a custom built firing range (with action-posed cutouts of his greatest enemies)in Jakarta although he's in the CIA and has just arrived days earlier. There is a lot of action involving gun-play(no muzzle-flashes on those M-16s, only sound effects), motorcycles(that bust through walls), karate(where no one makes physical contact) and even some sex(where all the actors are ugly). The main plot of an epic like this should at least be reasonably plausible, but not here. It involves the world's most dangerous drug cartel going all out to find a "drug detector device". Why would they need it? That is never revealed, why not kill drug-sniffing dogs? Makes no sense, but, it is taken seriously. The actors are to be commended because they really seemed to think this movie would make them all famous and tried hard to "act". Best line? "Now dance to your grave you dirty whore!" Best scene? Rambo jumps onto flying helicopter, pulls machine gun out of baddie's hand, let's go, falls, shoots helicopter as he's falling, helicopter blows up, cut to mannequin thrown in water. F**king genius! If you can't appreciate trash, don't watch it. If you can, it's awesome. One last thing, did I mention it was directed by the three Punjabi brothers?
diff --git a/tests/resources/tasks/imdb_underscore/train.txt b/tests/resources/tasks/imdb_underscore/train.txt
new file mode 100644
--- /dev/null
+++ b/tests/resources/tasks/imdb_underscore/train.txt
@@ -0,0 +1,5 @@
+__label__POSITIVE_Sentiment Water Lilies is a well-made first film from France about young female sexuality and friendship. Sciamma works with specialized, slightly sanitized material that is as off-putting to some as it is alluring to others. The film focuses exclusively on three middle-class teenage girls in a tidy new Paris suburb. Their lives revolve around a big indoor swimming pool where two of the three are part of a synchronized water ballet team.<br /><br />Such distractions as parents, siblings, work and school have been neatly excised from the equation. The central sensibility belongs to the attractively sullen but skinny Marie (Pauline Acquart), who is not on the team, but thinks she would like to be. Marie worships Floriane (Adèle Haenel), an alluring blonde and team standout whom the boys are after. This takes Marie away from her former best friend, also a member of the water ballet team, the somewhat plump Anne (Louise Blachère). Being less special Anne is more truly accessible to the boys. Floriane, like this film, promises a bit more then she truly offers. Marie has the more essential quality for a teenage girl: she suffers inwardly. Flroiane doesn't so much suffer as jump into situations and then bolt.<br /><br />Marie is dazzled by the glamor of the water ballet as well as Floriane. Floriane takes advantage of this to make Marie first her slave and a cover for her assignations, then, lacking any other friends, her confidante. All the other girls think Floriane a slut, an illusion she encourages in the men and boys she teases, because it leads them on. She suffers the pretty girl's fate of being not a person but an object, and she can't resist the validation the boys give her by wanting to kiss her and bed her, but she doesn't really care about any of them and knows her involvements with them are a trap. Enlisting Marie to act as her pal so her (unseen) mother won't know she's going out to meet boys, she also gets Marie to rescue her from the boys later. It looked the opposite at first, but Floriane needs Marie as much as Marie thinks she needs her. Anne is left with her discomfort with her body and a desire to get laid that's earthier and more real than the other girls'.<br /><br />Keeping all external context at bay, Sciamma can highlight subtle shifts in the delicate equation of the three girls' goals and interactions. On the other hand the film's water madness, which includes lots of showering and spitting as well as underwater swimming shots, makes it feel completely airless at times and some of its 95 minutes do not pass so quickly. Luckily the film has a sense of humor and lets the trio sometimes forget their ever-present goals and avoidances and just do silly, pointless girl things. It's the offbeat moments that give the film life; too bad in a way that there aren't more of them. But Sciamma has the courage of her obsessions and what remains as one walks out of the theater is the personalities and their dynamics. Along the way of course it is pleasant to watch the swimming and to gaze at the girls, who understandably love to gaze at themselves. <br /><br />There's no great revelation or drama on the way, but things get a bit more interesting when it emerges that Marie doesn't just admire but truly desires Floriane and is jealous of her boyfriends--whom Floriane always stops before they go all the way. In a typical irony of this kind of plot, Floriane actually decides she wants to have her first real sex with Marie--but Marie is the one who holds off, because she knows it won't have the significance to Floriane that it will have to her. When it happens, it's a timid, mechanical affair. Meanwhile Anne has a huge crush on Francois (Warren Jacquin), a male swimmer, but of course he is after Floriane. Boys are not an element that's been subtracted and there always seem to be several dozen ready at poolside or on the dance floor, but they are just bodies and faces, available studs.
+__label__POSITIVE_Sentiment Alive<br /><br />Alive is a very entertaining SCI-FI movie from Japan. I have noticed a lot of disappointed film geeks who loved Versus this director's debut film or his third film Azumi. I have heard they are blood drenched films with swords and zombies and all kinds of goodies. Frankly I went to the video store to get Versus but I am just fine with Alive.<br /><br />If you are looking for beginning to end wall to wall action then Alive is not your pick. There is plenty of action however it comes as pay-off for a whole hour of character driven build-up. Personally I think it is well done and worth it.<br /><br />Of course some of the plot is silly as with many SCI-Fi action films and I think the subtitles using the term foreign object could have replaced with parasite for greater effect. This film is brutal when it needs to be so faint of heart need not apply.<br /><br />They kept the budget down by for the most part confining all the action to one underground building(taking a cue from the cube) but the film doesn't suffer for it. Another bonus for this film is intense gothic imagines that are done with great artistic flair during the many Flashbacks and dream sequences. <br /><br />Rent this!
+__label__POSITIVE_Sentiment FORBIDDEN PLANET is the best SF film from the golden age of SF cinema and what makes it a great film is its sense of wonder . As soon as the spaceship lands the audience - via the ships human crew - travels through an intelligent and sometimes terrifying adventure . We meet the unforgetable Robbie , the mysterious Dr Morbuis , his beautiful and innocent daughter Altair and we learn about the former inhabitants of the planet - The Krell who died out overnight . Or did they ? <br /><br />You can nitpick and say the planet is obviously filmed in a movie studio with painted backdrops but that adds to a sense of menace of claustraphobia I feel and Bebe and Louis Barron`s electronic music adds even more atmosphere <br /><br />I`m shocked this film isn`t in the top 250 IMDB films .
+__label__POSITIVE_Sentiment Maybe some people may consider this a slow movie. However, it's precisely this "slow burning" that allows it to profoundly affect the viewer. Like that marvelous first scene: first, we can hear the voices; then, we are allowed to see the characters; finally, the camera slowly pans back and to the side and we see another character, a young man who is just sitting there, apparently not even listening to the conversation or, at least, not really caring. The fact that this young man will be the leading role shows the movie's strategy: allowing the character to develop and be determined by the environment surrounding him but without remarking the points more than necessary. People come and go, flow in and out of his life. Although most of the movie follows him (and most of the supporting characters are only seen when close to him, defined according to the relationship maintained with him), we are induced to believe that we are being objective, so subtle is the director touch.
+__label__NEGATIVE_Sentiment The world may have ended. Unfortunately this film survived as yet another testament to Canada's inability to make movies that go beyond the execrable. Maybe it's because all our really good people (Norman Jewison, Martin Short et al) go to Hollywood.) In fact it's too bad Short wasn't cast in this apallingly pretentious and banal film. He might have given it some credibility. The Canadian government should realize --- and this movie is a magnificent example --- that shovelling money into the trough does not result in good cinema. If the people lapping up these public funds had had to compete, they might have been forced to come up with something worthwhile. As it is they have produced yet another snickering embarassment.
diff --git a/tests/test_tars.py b/tests/test_tars.py
--- a/tests/test_tars.py
+++ b/tests/test_tars.py
@@ -1,10 +1,10 @@
-from flair.data import Corpus
+from flair.data import Sentence
 from flair.datasets import ClassificationCorpus
 from flair.models import TARSClassifier
+from flair.trainers import ModelTrainer
 
 
 def test_init_tars_and_switch(tasks_base_path):
-
     # test corpus
     corpus = ClassificationCorpus(tasks_base_path / "imdb")
 
@@ -14,28 +14,56 @@ def test_init_tars_and_switch(tasks_base_path):
                           label_type='class')
 
     # check if right number of classes
-    assert(len(tars.get_current_label_dictionary()) == 2)
+    assert (len(tars.get_current_label_dictionary()) == 2)
 
     # switch to task with only one label
     tars.add_and_switch_to_new_task('1_CLASS', 'one class', "testlabel")
 
     # check if right number of classes
-    assert(len(tars.get_current_label_dictionary()) == 1)
+    assert (len(tars.get_current_label_dictionary()) == 1)
 
     # switch to task with three labels provided as list
     tars.add_and_switch_to_new_task('3_CLASS', ['list 1', 'list 2', 'list 3'], "testlabel")
 
     # check if right number of classes
-    assert(len(tars.get_current_label_dictionary()) == 3)
+    assert (len(tars.get_current_label_dictionary()) == 3)
 
     # switch to task with four labels provided as set
     tars.add_and_switch_to_new_task('4_CLASS', {'set 1', 'set 2', 'set 3', 'set 4'}, "testlabel")
 
     # check if right number of classes
-    assert(len(tars.get_current_label_dictionary()) == 4)
+    assert (len(tars.get_current_label_dictionary()) == 4)
 
     # switch to task with two labels provided as Dictionary
     tars.add_and_switch_to_new_task('2_CLASS_AGAIN', corpus.make_label_dictionary(label_type='class'), "testlabel")
 
     # check if right number of classes
-    assert(len(tars.get_current_label_dictionary()) == 2)
\ No newline at end of file
+    assert (len(tars.get_current_label_dictionary()) == 2)
+
+
+def test_train_tars(tasks_base_path):
+    # test corpus
+    corpus = ClassificationCorpus(tasks_base_path / "imdb_underscore")
+
+    # create a TARS classifier
+    tars = TARSClassifier(embeddings="sshleifer/tiny-distilroberta-base")
+
+    # switch to a new task (TARS can do multiple tasks so you must define one)
+    tars.add_and_switch_to_new_task(task_name="question 2_CLASS",
+                                    label_dictionary=corpus.make_label_dictionary(label_type='class'),
+                                    label_type='class',
+                                    )
+
+    # initialize the text classifier trainer
+    trainer = ModelTrainer(tars, corpus)
+
+    # start the training
+    trainer.train(base_path='resources/taggers/trec',  # path to store the model artifacts
+                  learning_rate=0.02,  # use very small learning rate
+                  mini_batch_size=1,
+                  # mini_batch_chunk_size=4,  # optionally set this if transformer is too much for your machine
+                  max_epochs=1,  # terminate after 10 epochs
+                  )
+
+    sentence = Sentence("This is great!")
+    tars.predict(sentence)