Repository: deepset-ai/haystack

Issue:
evaluation breaks if doc content is no string e.g. for Tables
**Describe the bug**
Evaluation breaks when the document content is no string. It is checked for deduplication whether the content is empty string, via .strip, which is a string function

`label.document.content.strip() != ""`
reader/base.py


PR with fix will follow this issue

**Error message**
``` 
haystack/haystack/nodes/reader/base.py", line 187, in <dictcomp>
    if label.document.content.strip() != ""

Exception: Exception while running node 'Reader': 'DataFrame' object has no attribute 'strip'
Enable debug logging to see the data that was passed when the pipeline failed.
```

**Expected behavior**
Working evaluation, check for df and str


cc: @bglearning @sjrl 


**FAQ Check**
- [ ] Have you had a look at [our new FAQ page](https://docs.haystack.deepset.ai/docs/faq)?

**System:**
 - OS: 
 - GPU/CPU:
 - Haystack version (commit or version number): 
 - DocumentStore:
 - Reader:
 - Retriever:



Discussion:

Resolving patch:
diff --git a/haystack/nodes/reader/base.py b/haystack/nodes/reader/base.py
--- a/haystack/nodes/reader/base.py
+++ b/haystack/nodes/reader/base.py
@@ -122,7 +122,9 @@ def run(self, query: str, documents: List[Document], top_k: Optional[int] = None
             # filters out empty documents
             relevant_documents = list(
                 {
-                    label.document.id: label.document for label in labels.labels if label.document.content.strip() != ""
+                    label.document.id: label.document
+                    for label in labels.labels
+                    if not isinstance(label.document.content, str) or label.document.content.strip() != ""
                 }.values()
             )
             results_label_input = predict(query=query, documents=relevant_documents, top_k=top_k)
@@ -184,7 +186,7 @@ def run_batch(  # type: ignore
                     {
                         label.document.id: label.document
                         for label in labelx.labels
-                        if label.document.content.strip() != ""
+                        if not isinstance(label.document.content, str) or label.document.content.strip() != ""
                     }.values()
                 )
                 relevant_documents.append(relevant_docs_labelx)


Test patch:
diff --git a/test/pipelines/test_eval.py b/test/pipelines/test_eval.py
--- a/test/pipelines/test_eval.py
+++ b/test/pipelines/test_eval.py
@@ -3,6 +3,7 @@
 from pathlib import Path
 import pytest
 import sys
+import pandas as pd
 from copy import deepcopy
 from haystack.document_stores.memory import InMemoryDocumentStore
 from haystack.document_stores.elasticsearch import ElasticsearchDocumentStore
@@ -407,6 +408,117 @@ def test_eval_data_split_passage(document_store):
     ),
 ]
 
+EVAL_TABLE_LABELS = [
+    MultiLabel(
+        labels=[
+            Label(
+                query="How old is Brad Pitt?",
+                answer=Answer(answer="56", offsets_in_context=[Span(1, 2)]),
+                document=Document(
+                    id="a044cf3fb8aade03a12399c7a2fe9a6b",
+                    content_type="table",
+                    content=pd.DataFrame(
+                        columns=["Actors", "Age", "Number of movies"],
+                        data=[
+                            ["Brad Pitt", "56", "87"],
+                            ["Leonardo Di Caprio", "45", "53"],
+                            ["George Clooney", "59", "69"],
+                        ],
+                    ),
+                ),
+                is_correct_answer=True,
+                is_correct_document=True,
+                origin="gold-label",
+            ),
+            Label(  # Label with different doc but same answer and query
+                query="How old is Brad Pitt?",
+                answer=Answer(answer="56", offsets_in_context=[Span(4, 5)]),
+                document=Document(
+                    id="a044cf3fb8aade03a12399c7a2fe9a6b",
+                    content_type="table",
+                    content=pd.DataFrame(
+                        columns=["Actors", "Age", "Number of movies"],
+                        data=[["Beyonce", "45", "53"], ["Brad Pitt", "56", "87"], ["Jane Doe", "59", "69"]],
+                    ),
+                ),
+                is_correct_answer=True,
+                is_correct_document=True,
+                origin="gold-label",
+            ),
+        ]
+    ),
+    MultiLabel(
+        labels=[
+            Label(
+                query="To which state does Spikeroog belong?",
+                answer=Answer(answer="Lower Saxony", offsets_in_context=[Span(7, 8)]),
+                document=Document(
+                    id="b044cf3fb8aade03a12399c7a2fe9a6c",
+                    content_type="table",
+                    content=pd.DataFrame(
+                        columns=["0", "1"],
+                        data=[
+                            ["Area", "18.25 km2 (7.05 sq mi)"],
+                            ["Population", "794"],
+                            ["Country", "Germany"],
+                            ["State", "Lower Saxony"],
+                            ["District", "Wittmund"],
+                        ],
+                    ),
+                ),
+                is_correct_answer=True,
+                is_correct_document=True,
+                origin="gold-label",
+            )
+        ]
+    ),
+]
+
+
+@pytest.mark.integration
+@pytest.mark.parametrize("document_store", ["memory"], indirect=True)
+@pytest.mark.parametrize("retriever", ["table_text_retriever"], indirect=True)
+@pytest.mark.parametrize("table_reader_and_param", ["tapas_small"], indirect=True)
+@pytest.mark.embedding_dim(512)
+def test_table_qa_eval(table_reader_and_param, document_store, retriever):
+    docs = []
+    for multi_label in EVAL_TABLE_LABELS:
+        for label in multi_label.labels:
+            docs.append(label.document)
+
+    assert len(docs) == 3
+
+    document_store.write_documents(docs)
+    document_store.update_embeddings(retriever=retriever)
+
+    table_reader, _ = table_reader_and_param
+    p = Pipeline()
+    p.add_node(component=retriever, name="TableRetriever", inputs=["Query"])
+    p.add_node(component=table_reader, name="TableReader", inputs=["TableRetriever"])
+
+    eval_result = p.eval(labels=EVAL_TABLE_LABELS, params={"TableRetriever": {"top_k": 2}})
+    table_reader_results = eval_result.node_results["TableReader"]
+
+    assert set(table_reader_results["query"].tolist()) == {
+        "How old is Brad Pitt?",
+        "To which state does Spikeroog belong?",
+    }
+
+    metrics = eval_result.calculate_metrics(document_scope="document_id_or_answer")
+    assert metrics["TableRetriever"]["recall_single_hit"] == 1.0
+    assert metrics["TableRetriever"]["recall_multi_hit"] == 1.0
+    assert metrics["TableRetriever"]["precision"] == 0.5
+    assert metrics["TableRetriever"]["mrr"] == 1.0
+    assert metrics["TableRetriever"]["map"] == 1.0
+    assert metrics["TableRetriever"]["ndcg"] == 1.0
+    assert metrics["TableReader"]["exact_match"] == 1.0
+    assert metrics["TableReader"]["f1"] == 1.0
+
+    # assert metrics are floats
+    for node_metrics in metrics.values():
+        for value in node_metrics.values():
+            assert isinstance(value, float)
+
 
 @pytest.mark.parametrize("retriever_with_docs", ["tfidf"], indirect=True)
 @pytest.mark.parametrize("document_store_with_docs", ["memory"], indirect=True)