Repository: flairNLP/flair

Issue:
ValueError when input has double spaces and flair does not use tokenizer
I found that flair will crash when using Sentence() with use_tokenizer=False (its default value) and the sentence string has a double-space, which causes an empty token and the following error on PyTorch:

(To reproduce the following, I changed the input text in predict.py to have a double-space.)

```
Traceback (most recent call last):
  File "predict.py", line 8, in <module>
    tagger.predict(sentence)
  File "/home/projects/ner/flair/flair/models/sequence_tagger_model.py", line 371, in predict
    score, tag_seq = self._predict_scores_batch(batch)
  File "/home/projects/ner/flair/flair/models/sequence_tagger_model.py", line 386, in _predict_scores_batch
    all_feats, tags = self.forward(sentences)
  File "/home/projects/ner/flair/flair/models/sequence_tagger_model.py", line 157, in forward
    self.embeddings.embed(sentences)
  File "/home/projects/ner/flair/flair/embeddings.py", line 115, in embed
    embedding.embed(sentences)
  File "/home/projects/ner/flair/flair/embeddings.py", line 48, in embed
    self._add_embeddings_internal(sentences)
  File "/home/projects/ner/flair/flair/embeddings.py", line 268, in _add_embeddings_internal
    tokens_sorted_by_length = sorted(tokens_char_indices, key=lambda p: len(p), reverse=True)
  File "/home/miniconda3/lib/python3.6/site-packages/torch/onnx/__init__.py", line 68, in wrapper
    return fn(*args, **kwargs)
  File "/home/miniconda3/lib/python3.6/site-packages/torch/nn/utils/rnn.py", line 141, in pack_padded_sequence
    data, batch_sizes = PackPadded.apply(input, lengths, batch_first)
  File "/home/miniconda3/lib/python3.6/site-packages/torch/nn/_functions/packing.py", line 12, in forward
    raise ValueError("Length of all samples has to be greater than 0, "
ValueError: Length of all samples has to be greater than 0, but found an element in 'lengths' that is <= 0
```



Discussion:
Thanks for reporting this. I'm going to fix it.

Resolving patch:
diff --git a/flair/data.py b/flair/data.py
--- a/flair/data.py
+++ b/flair/data.py
@@ -236,8 +236,9 @@ def __init__(self, text: str = None, use_tokenizer: bool = False, labels: List[L
             else:
                 # add each word in tokenized string as Token object to Sentence
                 for word in text.split(' '):
-                    token = Token(word)
-                    self.add_token(token)
+                    if word:
+                        token = Token(word)
+                        self.add_token(token)
 
     def _infer_space_after(self):
         """


Test patch:
diff --git a/tests/test_data.py b/tests/test_data.py
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -81,6 +81,7 @@ def test_sentence_infer_tokenization():
     assert ('xyz " abc "' == sentence.to_tokenized_string())
     assert ('xyz "abc"' == sentence.to_plain_string())
 
+
 def test_sentence_get_item():
     sentence: Sentence = Sentence('I love Berlin.', use_tokenizer=True)
 
@@ -91,6 +92,16 @@ def test_sentence_get_item():
         token = sentence[4]
 
 
+def test_sentence_whitespace_tokenization():
+    sentence: Sentence = Sentence('I  love Berlin .')
+
+    assert (4 == len(sentence.tokens))
+    assert ('I' == sentence.get_token(1).text)
+    assert ('love' == sentence.get_token(2).text)
+    assert ('Berlin' == sentence.get_token(3).text)
+    assert ('.' == sentence.get_token(4).text)
+
+
 def test_sentence_to_tagged_string():
     token1 = Token('I', 0)
     token2 = Token('love', 1, 0)