Repository: flairNLP/flair

Issue:
Several errors with BERT embeddings in flair 0.5
Hello,
I've been trying to replicate some experiments I did in the past with BERT embeddings on the new 0.5 release, without success.
The only update I did to the code was to replace BertEmbeddings with the new TransformerWordEmbeddings, as as I understood this is the recommended way to proceed. The embeddings I specify are 'bert-base-multilingual-cased', but I've been tried also with distilbert and gpt2.
In commit 63aeabf9a18bdf53af3bcba5bd80f43ac717656e, I met the error specified in #1672 , invalid shape. So I tried a more recent commit to see if this was solved.
Now I'm in 3bf14fd13c3833394c7283132148261f937c0a83, which fails due to the following error:
```python
Traceback (most recent call last):
  File "train_flair.py", line 187, in <module>
    sys.exit(main(sys.argv[1:]))
  File "train_flair.py", line 182, in main
    trainer.train(args.outfolder, checkpoint=True, **hp)
  File "/opt/conda/lib/python3.6/site-packages/flair/trainers/trainer.py", line 364, in train
    loss = self.model.forward_loss(batch_step)
  File "/opt/conda/lib/python3.6/site-packages/flair/models/text_classification_model.py", line 146, in forward_loss
    scores = self.forward(data_points)
  File "/opt/conda/lib/python3.6/site-packages/flair/models/text_classification_model.py", line 100, in forward
    self.document_embeddings.embed(sentences)
  File "/opt/conda/lib/python3.6/site-packages/flair/embeddings/base.py", line 61, in embed
    self._add_embeddings_internal(sentences)
  File "/opt/conda/lib/python3.6/site-packages/flair/embeddings/document.py", line 369, in _add_embeddings_internal
    self.embeddings.embed(sentences)
  File "/opt/conda/lib/python3.6/site-packages/flair/embeddings/token.py", line 71, in embed
    embedding.embed(sentences)
  File "/opt/conda/lib/python3.6/site-packages/flair/embeddings/base.py", line 61, in embed
    self._add_embeddings_internal(sentences)
  File "/opt/conda/lib/python3.6/site-packages/flair/embeddings/token.py", line 892, in _add_embeddings_internal
    self._add_embeddings_to_sentences(batch)
  File "/opt/conda/lib/python3.6/site-packages/flair/embeddings/token.py", line 965, in _add_embeddings_to_sentences
    longest_sequence_in_batch: int = len(max(subtokenized_sentences, key=len))
ValueError: max() arg is an empty sequence
```
I thought it could be because of some empty sentence, but I always call `corpus.filter_empty_sentences()` before training, to be safe.
If I switch to turian embeddings, everything goes well. Is this a known issue?


Discussion:
@lucaventurini haven't seen this error yet. Could you provide a minimal code example to reproduce, for instance like embedding a single sentence that throws this error?
Ok, it took me a while to find the offending sentence, but I think it has to due with newer emojis, like 🤟, and especially if the sentence is made only of these and does not contain any other token.
Ah interesting, can you paste the sentence?
Yes. It is just "🤟".

Resolving patch:
diff --git a/flair/embeddings/token.py b/flair/embeddings/token.py
--- a/flair/embeddings/token.py
+++ b/flair/embeddings/token.py
@@ -927,6 +927,9 @@ def _add_embeddings_to_sentences(self, sentences: List[Sentence]):
             self.allow_long_sentences = False
             self.stride = 0
 
+        non_empty_sentences = []
+        empty_sentences = []
+
         for sentence in sentences:
             tokenized_string = sentence.to_tokenized_string()
 
@@ -936,6 +939,12 @@ def _add_embeddings_to_sentences(self, sentences: List[Sentence]):
             # method 2:
             # transformer specific tokenization
             subtokenized_sentence = self.tokenizer.tokenize(tokenized_string)
+            if len(subtokenized_sentence) == 0:
+                empty_sentences.append(sentence)
+                continue
+            else:
+                non_empty_sentences.append(sentence)
+
             token_subtoken_lengths = self.reconstruct_tokens_from_subtokens(sentence, subtokenized_sentence)
             subtokenized_sentences_token_lengths.append(token_subtoken_lengths)
 
@@ -953,7 +962,6 @@ def _add_embeddings_to_sentences(self, sentences: List[Sentence]):
                 subtoken_ids_split_sentence = encoded_inputs['input_ids']
                 subtokenized_sentences.append(torch.tensor(subtoken_ids_split_sentence, dtype=torch.long))
 
-
                 if 'overflowing_tokens' in encoded_inputs:
                     subtoken_ids_sentence = encoded_inputs['overflowing_tokens']
                 else:
@@ -961,6 +969,15 @@ def _add_embeddings_to_sentences(self, sentences: List[Sentence]):
 
             sentence_parts_lengths.append(nr_sentence_parts)
 
+        # empty sentences get zero embeddings
+        for sentence in empty_sentences:
+            for token in sentence:
+                token.set_embedding(self.name, torch.zeros(self.embedding_length))
+
+        # only embed non-empty sentences and if there is at least one
+        sentences = non_empty_sentences
+        if len(sentences) == 0: return
+
         # find longest sentence in batch
         longest_sequence_in_batch: int = len(max(subtokenized_sentences, key=len))
 
@@ -1074,9 +1091,12 @@ def reconstruct_tokens_from_subtokens(self, sentence, subtokens):
 
             # some BERT tokenizers somehow omit words - in such cases skip to next token
             if subtoken_count == 0 and not token_text.startswith(subtoken.lower()):
-                token_subtoken_lengths.append(0)
-                token = next(word_iterator)
-                token_text = self._get_processed_token_text(token)
+
+                while True:
+                    token_subtoken_lengths.append(0)
+                    token = next(word_iterator)
+                    token_text = self._get_processed_token_text(token)
+                    if token_text.startswith(subtoken.lower()): break
 
             subtoken_count += 1
 
@@ -1099,6 +1119,13 @@ def reconstruct_tokens_from_subtokens(self, sentence, subtokens):
                     token_text = self._get_processed_token_text(token)
                 else:
                     break
+
+        # if tokens are unaccounted for
+        while len(token_subtoken_lengths) < len(sentence) and len(token.text) == 1:
+            token_subtoken_lengths.append(0)
+            if len(token_subtoken_lengths) == len(sentence): break
+            token = next(word_iterator)
+
         # check if all tokens were matched to subtokens
         if token != sentence[-1]:
             log.error(f"Tokenization MISMATCH in sentence '{sentence.to_tokenized_string()}'")


Test patch:
diff --git a/tests/test_embeddings.py b/tests/test_embeddings.py
--- a/tests/test_embeddings.py
+++ b/tests/test_embeddings.py
@@ -117,6 +117,33 @@ def test_transformer_weird_sentences():
     for token in sentence:
         assert len(token.get_embedding()) == 768
 
+    sentence = Sentence("🤟 🤟  🤟 hüllo")
+    embeddings.embed(sentence)
+    for token in sentence:
+        assert len(token.get_embedding()) == 768
+
+    sentence = Sentence("🤟hallo 🤟 🤟 🤟 🤟")
+    embeddings.embed(sentence)
+    for token in sentence:
+        assert len(token.get_embedding()) == 768
+
+    sentence = Sentence("🤟hallo 🤟 🤟 🤟 🤟")
+    embeddings.embed(sentence)
+    for token in sentence:
+        assert len(token.get_embedding()) == 768
+
+    sentence = Sentence("🤟")
+    embeddings.embed(sentence)
+    for token in sentence:
+        assert len(token.get_embedding()) == 768
+
+    sentence = Sentence("🤟")
+    sentence_2 = Sentence("second sentence")
+    embeddings.embed([sentence, sentence_2])
+    for token in sentence:
+        assert len(token.get_embedding()) == 768
+    for token in sentence_2:
+        assert len(token.get_embedding()) == 768
 
 def test_fine_tunable_flair_embedding():
     language_model_forward = LanguageModel(