Repository: deepset-ai/haystack

Issue:
PreProcessor produces empty Document during splitting
**Describe the bug**
If you use a PreProcessor with
1. small split_length
2. split_respect_sentence_boundary=True
3. a very long first sentence

then the first document is just empty text, resulting in errors further down the processing.

Lets remove empty documents inside the preprocessor.


Answer Text and Answer Position mismatch during retriever query bencharmking
When running the retriever query benchmarks, we get the following warning

```
01/26/2021 10:59:15 - WARNING - haystack.preprocessor.utils -   Answer Text and Answer position mismatch. Skipping Answer
```

We should investigate how many samples trigger this warning, whether this has an impact on the benchmarking and whether there is a simple way to fix it.



Discussion:

Hey @brandenchan 
I have implemented a check if the answer string corresponds to the answer offsets, see [here](https://github.com/deepset-ai/haystack/blob/master/haystack/preprocessor/utils.py#L141). (This check is similar to what is in FARM QA preprocessing). I need this check at this position because there are bugs later on in the pipeline when answer text and position do not match.

I see a couple of ways to improve the situation:
1. clean the input data, so that answer text and offsets correspond
2. have a more informative error message pointing to the direct example (this might spoil the logger output)
3. implement problematic_ids that are filled with the erroneous ids and just return statistics about it.

What do you say? 
I think a more informative error messaging would be great. I imagine both having statistics and IDs would be really useful. I would imagine it being something like this:

```
Answer text and answer positions were mismatched in 45 samples so they have been skipped. Their IDs are:
[9asf7gadsg, 089as0dgaf, jk1245jk ...] 
```

Resolving patch:
diff --git a/haystack/document_store/base.py b/haystack/document_store/base.py
--- a/haystack/document_store/base.py
+++ b/haystack/document_store/base.py
@@ -165,6 +165,9 @@ def add_eval_data(self, filename: str, doc_index: str = "eval_document", label_i
         if preprocessor is not None:
             assert preprocessor.split_by != "sentence", f"Split by sentence not supported.\n" \
                                                     f"Please set 'split_by' to either 'word' or 'passage' in the supplied PreProcessor."
+            assert preprocessor.split_respect_sentence_boundary == False, \
+                f"split_respect_sentence_boundary not supported yet.\n" \
+                f"Please set 'split_respect_sentence_boundary' to False in the supplied PreProcessor."
             assert preprocessor.split_overlap == 0, f"Overlapping documents are currently not supported when adding eval data.\n" \
                                                     f"Please set 'split_overlap=0' in the supplied PreProcessor."
             assert preprocessor.clean_empty_lines == False, f"clean_empty_lines currently not supported when adding eval data.\n" \
diff --git a/haystack/preprocessor/preprocessor.py b/haystack/preprocessor/preprocessor.py
--- a/haystack/preprocessor/preprocessor.py
+++ b/haystack/preprocessor/preprocessor.py
@@ -128,7 +128,12 @@ def split(self, document: dict) -> List[dict]:
                 word_count += len(sen.split(" "))
             if current_slice:
                 list_splits.append(current_slice)
-            text_splits = [' '.join(sl) for sl in list_splits]
+
+            text_splits = []
+            for sl in list_splits:
+                txt = ' '.join(sl)
+                if len(txt) > 0:
+                    text_splits.append(txt)
         else:
             # create individual "elements" of passage, sentence, or word
             if self.split_by == "passage":
@@ -138,7 +143,7 @@ def split(self, document: dict) -> List[dict]:
             elif self.split_by == "word":
                 elements = text.split(" ")
             else:
-                raise NotImplementedError("PreProcessor only supports 'passage' or 'sentence' split_by options.")
+                raise NotImplementedError("PreProcessor only supports 'passage', 'sentence' or 'word' split_by options.")
 
             # concatenate individual elements based on split_length & split_stride
             if self.split_overlap:
@@ -148,7 +153,8 @@ def split(self, document: dict) -> List[dict]:
             text_splits = []
             for seg in segments:
                 txt = " ".join([t for t in seg if t])
-                text_splits.append(txt)
+                if len(txt) > 0:
+                    text_splits.append(txt)
 
         # create new document dicts for each text split
         documents = []
diff --git a/haystack/preprocessor/utils.py b/haystack/preprocessor/utils.py
--- a/haystack/preprocessor/utils.py
+++ b/haystack/preprocessor/utils.py
@@ -34,6 +34,7 @@ def eval_data_from_json(filename: str, max_docs: Union[int, bool] = None, prepro
 
     docs: List[Document] = []
     labels = []
+    problematic_ids = []
 
     with open(filename, "r") as file:
         data = json.load(file)
@@ -45,10 +46,13 @@ def eval_data_from_json(filename: str, max_docs: Union[int, bool] = None, prepro
                 if len(docs) > max_docs:
                     break
             # Extracting paragraphs and their labels from a SQuAD document dict
-            cur_docs, cur_labels = _extract_docs_and_labels_from_dict(document, preprocessor)
+            cur_docs, cur_labels, cur_problematic_ids = _extract_docs_and_labels_from_dict(document, preprocessor)
             docs.extend(cur_docs)
             labels.extend(cur_labels)
-
+            problematic_ids.extend(cur_problematic_ids)
+    if len(problematic_ids) > 0:
+        logger.warning(f"Could not convert an answer for {len(problematic_ids)} questions.\n"
+                       f"There were conversion errors for question ids: {problematic_ids}")
     return docs, labels
 
 
@@ -69,6 +73,7 @@ def eval_data_from_jsonl(filename: str, batch_size: Optional[int] = None,
 
     docs: List[Document] = []
     labels = []
+    problematic_ids = []
 
     with open(filename, "r") as file:
         for document in file:
@@ -77,15 +82,20 @@ def eval_data_from_jsonl(filename: str, batch_size: Optional[int] = None,
                     break
             # Extracting paragraphs and their labels from a SQuAD document dict
             document_dict = json.loads(document)
-            cur_docs, cur_labels = _extract_docs_and_labels_from_dict(document_dict, preprocessor)
+            cur_docs, cur_labels, cur_problematic_ids = _extract_docs_and_labels_from_dict(document_dict, preprocessor)
             docs.extend(cur_docs)
             labels.extend(cur_labels)
+            problematic_ids.extend(cur_problematic_ids)
 
             if batch_size is not None:
                 if len(docs) >= batch_size:
+                    if len(problematic_ids) > 0:
+                        logger.warning(f"Could not convert an answer for {len(problematic_ids)} questions.\n"
+                                       f"There were conversion errors for question ids: {problematic_ids}")
                     yield docs, labels
                     docs = []
                     labels = []
+                    problematic_ids = []
 
     yield docs, labels
 
@@ -93,6 +103,7 @@ def eval_data_from_jsonl(filename: str, batch_size: Optional[int] = None,
 def _extract_docs_and_labels_from_dict(document_dict: Dict, preprocessor: PreProcessor = None):
     docs = []
     labels = []
+    problematic_ids = []
 
     # get all extra fields from document level (e.g. title)
     meta_doc = {k: v for k, v in document_dict.items() if k not in ("paragraphs", "title")}
@@ -139,7 +150,8 @@ def _extract_docs_and_labels_from_dict(document_dict: Dict, preprocessor: PrePro
                     ans = answer["text"]
                     ans_position = cur_doc.text[answer["answer_start"]:answer["answer_start"]+len(ans)]
                     if ans != ans_position:
-                        logger.warning(f"Answer Text and Answer position mismatch. Skipping Answer")
+                        # do not use answer
+                        problematic_ids.append(qa.get("id","missing"))
                         break
                     # find corresponding document or split
                     if len(splits) == 1:
@@ -181,7 +193,7 @@ def _extract_docs_and_labels_from_dict(document_dict: Dict, preprocessor: PrePro
                     )
                     labels.append(label)
 
-    return docs, labels
+    return docs, labels, problematic_ids
 
 
 def convert_files_to_dicts(dir_path: str, clean_func: Optional[Callable] = None, split_paragraphs: bool = False) -> \


Test patch:
diff --git a/test/test_preprocessor.py b/test/test_preprocessor.py
--- a/test/test_preprocessor.py
+++ b/test/test_preprocessor.py
@@ -51,6 +51,10 @@ def test_preprocess_word_split():
     documents = preprocessor.process(document)
     assert len(documents) == 5
 
+    preprocessor = PreProcessor(split_length=5, split_overlap=0, split_by="word", split_respect_sentence_boundary=True)
+    documents = preprocessor.process(document)
+    assert len(documents) == 15
+
 
 @pytest.mark.tika
 def test_preprocess_passage_split():