Repository: deepset-ai/haystack

Issue:
MAP and MRR wrong for multiple gold documents
**Describe the bug**
Both the MAP and the MRR show wrong values. It seems we calculate the score for single gold documents and then override it after each instead of calculating it for the whole batch of gold documents. 

**Expected behavior**
correct values

**To Reproduce**
```
retrieved_docs = ["one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"]
gold_docs = ["one", "two", "three","four","seven"]

from haystack.components.evaluators import DocumentMAPEvaluator, DocumentMRREvaluator
from haystack import Document

mapevaluator = DocumentMAPEvaluator()
mrrevaluator = DocumentMRREvaluator()
mapresult = mapevaluator.run(
    ground_truth_documents=[[Document(content=content) for content in gold_docs]],
    retrieved_documents=[[Document(content=content) for content in retrieved_docs]])

mrrresult = mrrevaluator.run(
    ground_truth_documents=[[Document(content=content) for content in gold_docs]],
    retrieved_documents=[[Document(content=content) for content in retrieved_docs]])

print(mapresult["individual_scores"])
print(mrrresult["individual_scores"])
print(mapresult["score"])
print(mrrresult["score"])
```



Discussion:
Hi @ju-gu I'm trying to understand the example above. You are providing 10 "retrievals" but have only 5 gold truth docs? In other words, you have:

| Actual retrieved | True doc |
| --------------- | --------------- | 
|  one |  one  |
|  two |  two  |
|  three |  three  |
|  four |  four  |
|  five |  seven  |
|  six |  ?  |
|  seven |  ?  |
|  eight |  ?  |
|  nine |  ?  |
|  ten |  ?  |

I don't think this is a typical use case, are you sure this is what you're trying to do?

Or are you trying to do this instead:

| Actual retrieved | True doc |
| --------------- | --------------- | 
|  ["one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten"] |  ["one", "two", "three","four","seven"]  |
I am doing the latter. The two lists retrieved docs and true docs become a list of lists in these lines: 

ground_truth_documents=[[Document(content=content) for content in gold_docs]]
retrieved_documents=[[Document(content=content) for content in retrieved_docs]])

So it is just one set of retrieved docs and one set of gold documents


Resolving patch:
diff --git a/haystack/components/evaluators/document_map.py b/haystack/components/evaluators/document_map.py
--- a/haystack/components/evaluators/document_map.py
+++ b/haystack/components/evaluators/document_map.py
@@ -43,6 +43,7 @@ class DocumentMAPEvaluator:
     ```
     """
 
+    # Refer to https://www.pinecone.io/learn/offline-evaluation/ for the algorithm.
     @component.output_types(score=float, individual_scores=List[float])
     def run(
         self, ground_truth_documents: List[List[Document]], retrieved_documents: List[List[Document]]
@@ -68,25 +69,21 @@ def run(
         individual_scores = []
 
         for ground_truth, retrieved in zip(ground_truth_documents, retrieved_documents):
-            score = 0.0
-            for ground_document in ground_truth:
-                if ground_document.content is None:
-                    continue
-
-                average_precision = 0.0
-                relevant_documents = 0
+            average_precision = 0.0
+            average_precision_numerator = 0.0
+            relevant_documents = 0
 
-                for rank, retrieved_document in enumerate(retrieved):
-                    if retrieved_document.content is None:
-                        continue
-
-                    if ground_document.content in retrieved_document.content:
-                        relevant_documents += 1
-                        average_precision += relevant_documents / (rank + 1)
-                if relevant_documents > 0:
-                    score = average_precision / relevant_documents
-            individual_scores.append(score)
+            ground_truth_contents = [doc.content for doc in ground_truth if doc.content is not None]
+            for rank, retrieved_document in enumerate(retrieved):
+                if retrieved_document.content is None:
+                    continue
 
-        score = sum(individual_scores) / len(retrieved_documents)
+                if retrieved_document.content in ground_truth_contents:
+                    relevant_documents += 1
+                    average_precision_numerator += relevant_documents / (rank + 1)
+            if relevant_documents > 0:
+                average_precision = average_precision_numerator / relevant_documents
+            individual_scores.append(average_precision)
 
+        score = sum(individual_scores) / len(ground_truth_documents)
         return {"score": score, "individual_scores": individual_scores}
diff --git a/haystack/components/evaluators/document_mrr.py b/haystack/components/evaluators/document_mrr.py
--- a/haystack/components/evaluators/document_mrr.py
+++ b/haystack/components/evaluators/document_mrr.py
@@ -41,6 +41,7 @@ class DocumentMRREvaluator:
     ```
     """
 
+    # Refer to https://www.pinecone.io/learn/offline-evaluation/ for the algorithm.
     @component.output_types(score=float, individual_scores=List[float])
     def run(
         self, ground_truth_documents: List[List[Document]], retrieved_documents: List[List[Document]]
@@ -66,20 +67,17 @@ def run(
         individual_scores = []
 
         for ground_truth, retrieved in zip(ground_truth_documents, retrieved_documents):
-            score = 0.0
-            for ground_document in ground_truth:
-                if ground_document.content is None:
-                    continue
-
-                for rank, retrieved_document in enumerate(retrieved):
-                    if retrieved_document.content is None:
-                        continue
+            reciprocal_rank = 0.0
 
-                    if ground_document.content in retrieved_document.content:
-                        score = 1 / (rank + 1)
-                        break
-            individual_scores.append(score)
+            ground_truth_contents = [doc.content for doc in ground_truth if doc.content is not None]
+            for rank, retrieved_document in enumerate(retrieved):
+                if retrieved_document.content is None:
+                    continue
+                if retrieved_document.content in ground_truth_contents:
+                    reciprocal_rank = 1 / (rank + 1)
+                    break
+            individual_scores.append(reciprocal_rank)
 
-        score = sum(individual_scores) / len(retrieved_documents)
+        score = sum(individual_scores) / len(ground_truth_documents)
 
         return {"score": score, "individual_scores": individual_scores}


Test patch:
diff --git a/test/components/evaluators/test_document_map.py b/test/components/evaluators/test_document_map.py
--- a/test/components/evaluators/test_document_map.py
+++ b/test/components/evaluators/test_document_map.py
@@ -62,7 +62,17 @@ def test_run_with_complex_data():
             ],
         ],
     )
-    assert result == {"individual_scores": [1.0, 0.8333333333333333, 1.0, 0.5, 0.0, 1.0], "score": 0.7222222222222222}
+    assert result == {
+        "individual_scores": [
+            1.0,
+            pytest.approx(0.8333333333333333),
+            1.0,
+            pytest.approx(0.5833333333333333),
+            0.0,
+            pytest.approx(0.8055555555555555),
+        ],
+        "score": pytest.approx(0.7037037037037037),
+    }
 
 
 def test_run_with_different_lengths():