Repository: deepset-ai/haystack

Issue:
Implement function to calculate Mean Reciprocal Rank metric
As specified in proposal #5794 we need to implement a function to calculate the Mean Reciprocal Rank metric.

Ideally the function should be part of the private interface and called only through the `calculate_metrics` function (see #6063). `_calculate_mrr()` could be a nice name.

For more detailed information check out the original proposal.

Discussion:

Patch:
diff --git a/haystack/components/evaluators/answer_exact_match.py b/haystack/components/evaluators/answer_exact_match.py
index f6292f32..dcd44408 100644
--- a/haystack/components/evaluators/answer_exact_match.py
+++ b/haystack/components/evaluators/answer_exact_match.py
@@ -7,9 +7,9 @@ from haystack.core.component import component
 class AnswerExactMatchEvaluator:
     """
     Evaluator that checks if the predicted answers matches any of the ground truth answers exactly.
-    The result is a number from 0.0 to 1.0, it represents the proportion of questions where any predicted answer
-    matched one of the ground truth answers.
-    Each question can have multiple ground truth answers and multiple predicted answers.
+    The result is a number from 0.0 to 1.0, it represents the proportion any predicted answer
+    that matched one of the ground truth answers.
+    There can be multiple ground truth answers and multiple predicted answers as input.
 
     Usage example:
     ```python
@@ -17,7 +17,6 @@ class AnswerExactMatchEvaluator:
 
     evaluator = AnswerExactMatchEvaluator()
     result = evaluator.run(
-        questions=["What is the capital of Germany?", "What is the capital of France?"],
         ground_truth_answers=[["Berlin"], ["Paris"]],
         predicted_answers=[["Berlin"], ["Lyon"]],
     )
@@ -30,15 +29,11 @@ class AnswerExactMatchEvaluator:
     """
 
     @component.output_types(individual_scores=List[int], score=float)
-    def run(
-        self, questions: List[str], ground_truth_answers: List[List[str]], predicted_answers: List[List[str]]
-    ) -> Dict[str, Any]:
+    def run(self, ground_truth_answers: List[List[str]], predicted_answers: List[List[str]]) -> Dict[str, Any]:
         """
         Run the AnswerExactMatchEvaluator on the given inputs.
-        All lists must have the same length.
+        `ground_truth_answers` and `retrieved_answers` must have the same length.
 
-        :param questions:
-            A list of questions.
         :param ground_truth_answers:
             A list of expected answers for each question.
         :param predicted_answers:
@@ -49,8 +44,8 @@ class AnswerExactMatchEvaluator:
             - `score` - A number from 0.0 to 1.0 that represents the proportion of questions where any predicted
                          answer matched one of the ground truth answers.
         """
-        if not len(questions) == len(ground_truth_answers) == len(predicted_answers):
-            raise ValueError("The length of questions, ground_truth_answers, and predicted_answers must be the same.")
+        if not len(ground_truth_answers) == len(predicted_answers):
+            raise ValueError("The length of ground_truth_answers and predicted_answers must be the same.")
 
         matches = []
         for truths, extracted in zip(ground_truth_answers, predicted_answers):
@@ -60,6 +55,6 @@ class AnswerExactMatchEvaluator:
                 matches.append(0)
 
         # The proportion of questions where any predicted answer matched one of the ground truth answers
-        average = sum(matches) / len(questions)
+        average = sum(matches) / len(predicted_answers)
 
         return {"individual_scores": matches, "score": average}
diff --git a/haystack/components/evaluators/document_map.py b/haystack/components/evaluators/document_map.py
new file mode 100644
index 00000000..483a7699
--- /dev/null
+++ b/haystack/components/evaluators/document_map.py
@@ -0,0 +1,84 @@
+from typing import Any, Dict, List
+
+from haystack import Document, component
+
+
+@component
+class DocumentMeanAveragePrecision:
+    """
+    Evaluator that calculates the mean average precision of the retrieved documents, a metric
+    that measures how high retrieved documents are ranked.
+    Each question can have multiple ground truth documents and multiple retrieved documents.
+
+    `DocumentMeanAveragePrecision` doesn't normalize its inputs, the `DocumentCleaner` component
+    should be used to clean and normalize the documents before passing them to this evaluator.
+
+    Usage example:
+    ```python
+    from haystack.components.evaluators import AnswerExactMatchEvaluator
+
+    evaluator = DocumentMeanAveragePrecision()
+    result = evaluator.run(
+        ground_truth_documents=[
+            [Document(content="France")],
+            [Document(content="9th century"), Document(content="9th")],
+        ],
+        retrieved_documents=[
+            [Document(content="France")],
+            [Document(content="9th century"), Document(content="10th century"), Document(content="9th")],
+        ],
+    )
+
+    print(result["individual_scores"])
+    # [1.0, 0.8333333333333333]
+    print(result["score"])
+    # 0.9166666666666666
+    ```
+    """
+
+    @component.output_types(score=float, individual_scores=List[float])
+    def run(
+        self, ground_truth_documents: List[List[Document]], retrieved_documents: List[List[Document]]
+    ) -> Dict[str, Any]:
+        """
+        Run the DocumentMeanAveragePrecision on the given inputs.
+        All lists must have the same length.
+
+        :param ground_truth_documents:
+            A list of expected documents for each question.
+        :param retrieved_documents:
+            A list of retrieved documents for each question.
+        :returns:
+            A dictionary with the following outputs:
+            - `score` - The average of calculated scores.
+            - `invididual_scores` - A list of numbers from 0.0 to 1.0 that represents how high retrieved documents are ranked.
+        """
+        if len(ground_truth_documents) != len(retrieved_documents):
+            msg = "The length of ground_truth_documents and retrieved_documents must be the same."
+            raise ValueError(msg)
+
+        individual_scores = []
+
+        for ground_truth, retrieved in zip(ground_truth_documents, retrieved_documents):
+            score = 0.0
+            for ground_document in ground_truth:
+                if ground_document.content is None:
+                    continue
+
+                average_precision = 0.0
+                relevant_documents = 0
+
+                for rank, retrieved_document in enumerate(retrieved):
+                    if retrieved_document.content is None:
+                        continue
+
+                    if ground_document.content in retrieved_document.content:
+                        relevant_documents += 1
+                        average_precision += relevant_documents / (rank + 1)
+                if relevant_documents > 0:
+                    score = average_precision / relevant_documents
+            individual_scores.append(score)
+
+        score = sum(individual_scores) / len(retrieved_documents)
+
+        return {"score": score, "individual_scores": individual_scores}
diff --git a/haystack/components/evaluators/document_mrr.py b/haystack/components/evaluators/document_mrr.py
new file mode 100644
index 00000000..d0194902
--- /dev/null
+++ b/haystack/components/evaluators/document_mrr.py
@@ -0,0 +1,79 @@
+from typing import Any, Dict, List
+
+from haystack import Document, component
+
+
+@component
+class DocumentMeanReciprocalRank:
+    """
+    Evaluator that calculates the mean reciprocal rank of the retrieved documents.
+
+    MRR measures how high the first retrieved document is ranked.
+    Each question can have multiple ground truth documents and multiple retrieved documents.
+
+    `DocumentMeanReciprocalRank` doesn't normalize its inputs, the `DocumentCleaner` component
+    should be used to clean and normalize the documents before passing them to this evaluator.
+
+    Usage example:
+    ```python
+    from haystack.components.evaluators import AnswerExactMatchEvaluator
+    evaluator = DocumentMeanReciprocalRank()
+    result = evaluator.run(
+        ground_truth_documents=[
+            [Document(content="France")],
+            [Document(content="9th century"), Document(content="9th")],
+        ],
+        retrieved_documents=[
+            [Document(content="France")],
+            [Document(content="9th century"), Document(content="10th century"), Document(content="9th")],
+        ],
+    )
+    print(result["individual_scores"])
+    # [1.0, 0.8333333333333333]
+    print(result["score"])
+    # 0.9166666666666666
+    ```
+    """
+
+    @component.output_types(score=float, individual_scores=List[float])
+    def run(
+        self, ground_truth_documents: List[List[Document]], retrieved_documents: List[List[Document]]
+    ) -> Dict[str, Any]:
+        """
+        Run the DocumentMeanReciprocalRank on the given inputs.
+
+        `ground_truth_documents` and `retrieved_documents` must have the same length.
+
+        :param ground_truth_documents:
+            A list of expected documents for each question.
+        :param retrieved_documents:
+            A list of retrieved documents for each question.
+        :returns:
+            A dictionary with the following outputs:
+            - `score` - The average of calculated scores.
+            - `invididual_scores` - A list of numbers from 0.0 to 1.0 that represents how high the first retrieved document is ranked.
+        """
+        if len(ground_truth_documents) != len(retrieved_documents):
+            msg = "The length of ground_truth_documents and retrieved_documents must be the same."
+            raise ValueError(msg)
+
+        individual_scores = []
+
+        for ground_truth, retrieved in zip(ground_truth_documents, retrieved_documents):
+            score = 0.0
+            for ground_document in ground_truth:
+                if ground_document.content is None:
+                    continue
+
+                for rank, retrieved_document in enumerate(retrieved):
+                    if retrieved_document.content is None:
+                        continue
+
+                    if ground_document.content in retrieved_document.content:
+                        score = 1 / (rank + 1)
+                        break
+            individual_scores.append(score)
+
+        score = sum(individual_scores) / len(retrieved_documents)
+
+        return {"score": score, "individual_scores": individual_scores}
diff --git a/haystack/components/evaluators/document_recall.py b/haystack/components/evaluators/document_recall.py
index 0aaa2bd1..4102aa1f 100644
--- a/haystack/components/evaluators/document_recall.py
+++ b/haystack/components/evaluators/document_recall.py
@@ -31,16 +31,15 @@ class RecallMode(Enum):
 @component
 class DocumentRecallEvaluator:
     """
-    Evaluator that calculates the Recall score for a list of questions.
+    Evaluator that calculates the Recall score for a list of documents.
     Returns both a list of scores for each question and the average.
-    Each question can have multiple ground truth documents and multiple predicted documents.
+    There can be multiple ground truth documents and multiple predicted documents as input.
 
     Usage example:
     ```python
     from haystack.components.evaluators import DocumentRecallEvaluator
     evaluator = DocumentRecallEvaluator()
     result = evaluator.run(
-        questions=["What is the capital of Germany?", "What is the capital of France?"],
         ground_truth_answers=[["Berlin"], ["Paris"]],
         predicted_answers=[["Paris"], ["London"]],
     )
@@ -80,17 +79,12 @@ class DocumentRecallEvaluator:
 
     @component.output_types(score=float, individual_scores=List[float])
     def run(
-        self,
-        questions: List[str],
-        ground_truth_documents: List[List[Document]],
-        retrieved_documents: List[List[Document]],
+        self, ground_truth_documents: List[List[Document]], retrieved_documents: List[List[Document]]
     ) -> Dict[str, Any]:
         """
         Run the DocumentRecallEvaluator on the given inputs.
-        All lists must have the same length.
+        `ground_truth_documents` and `retrieved_documents` must have the same length.
 
-        :param questions:
-            A list of questions.
         :param ground_truth_documents:
             A list of expected documents for each question.
         :param retrieved_documents:
@@ -100,8 +94,8 @@ class DocumentRecallEvaluator:
             - `invididual_scores` - A list of numbers from 0.0 to 1.0 that represents the proportion of matching documents retrieved.
                                     If the mode is `single_hit`, the individual scores are True or False.
         """
-        if not len(questions) == len(ground_truth_documents) == len(retrieved_documents):
-            msg = "The length of questions, ground_truth_documents, and predicted_documents must be the same."
+        if len(ground_truth_documents) != len(retrieved_documents):
+            msg = "The length of ground_truth_documents and retrieved_documents must be the same."
             raise ValueError(msg)
 
         scores = []
@@ -109,4 +103,4 @@ class DocumentRecallEvaluator:
             score = self.mode_function(ground_truth, retrieved)
             scores.append(score)
 
-        return {"score": sum(scores) / len(questions), "individual_scores": scores}
+        return {"score": sum(scores) / len(retrieved_documents), "individual_scores": scores}
diff --git a/releasenotes/notes/document-map-evaluator-de896c94b54fe3fa.yaml b/releasenotes/notes/document-map-evaluator-de896c94b54fe3fa.yaml
new file mode 100644
index 00000000..36e28b73
--- /dev/null
+++ b/releasenotes/notes/document-map-evaluator-de896c94b54fe3fa.yaml
@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Add DocumentMeanAveragePrecision, it can be used to calculate mean average precision of retrieved documents.
diff --git a/releasenotes/notes/document-mrr-evaluator-fa7c266cc91201a7.yaml b/releasenotes/notes/document-mrr-evaluator-fa7c266cc91201a7.yaml
new file mode 100644
index 00000000..7e56e948
--- /dev/null
+++ b/releasenotes/notes/document-mrr-evaluator-fa7c266cc91201a7.yaml
@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Add DocumentMeanReciprocalRank, it can be used to calculate mean reciprocal rank of retrieved documents.


Test patch:
diff --git a/test/components/evaluators/test_answer_exact_match.py b/test/components/evaluators/test_answer_exact_match.py
index 91e4647a..9c7b395b 100644
--- a/test/components/evaluators/test_answer_exact_match.py
+++ b/test/components/evaluators/test_answer_exact_match.py
@@ -5,33 +5,21 @@ from haystack.components.evaluators import AnswerExactMatchEvaluator
 
 def test_run_with_all_matching():
     evaluator = AnswerExactMatchEvaluator()
-    result = evaluator.run(
-        questions=["What is the capital of Germany?", "What is the capital of France?"],
-        ground_truth_answers=[["Berlin"], ["Paris"]],
-        predicted_answers=[["Berlin"], ["Paris"]],
-    )
+    result = evaluator.run(ground_truth_answers=[["Berlin"], ["Paris"]], predicted_answers=[["Berlin"], ["Paris"]])
 
     assert result == {"individual_scores": [1, 1], "score": 1.0}
 
 
 def test_run_with_no_matching():
     evaluator = AnswerExactMatchEvaluator()
-    result = evaluator.run(
-        questions=["What is the capital of Germany?", "What is the capital of France?"],
-        ground_truth_answers=[["Berlin"], ["Paris"]],
-        predicted_answers=[["Paris"], ["London"]],
-    )
+    result = evaluator.run(ground_truth_answers=[["Berlin"], ["Paris"]], predicted_answers=[["Paris"], ["London"]])
 
     assert result == {"individual_scores": [0, 0], "score": 0.0}
 
 
 def test_run_with_partial_matching():
     evaluator = AnswerExactMatchEvaluator()
-    result = evaluator.run(
-        questions=["What is the capital of Germany?", "What is the capital of France?"],
-        ground_truth_answers=[["Berlin"], ["Paris"]],
-        predicted_answers=[["Berlin"], ["London"]],
-    )
+    result = evaluator.run(ground_truth_answers=[["Berlin"], ["Paris"]], predicted_answers=[["Berlin"], ["London"]])
 
     assert result == {"individual_scores": [1, 0], "score": 0.5}
 
@@ -39,14 +27,6 @@ def test_run_with_partial_matching():
 def test_run_with_complex_data():
     evaluator = AnswerExactMatchEvaluator()
     result = evaluator.run(
-        questions=[
-            "In what country is Normandy located?",
-            "When was the Latin version of the word Norman first recorded?",
-            "What developed in Normandy during the 1100s?",
-            "In what century did important classical music developments occur in Normandy?",
-            "From which countries did the Norse originate?",
-            "What century did the Normans first gain their separate identity?",
-        ],
         ground_truth_answers=[
             ["France"],
             ["9th century", "9th"],
@@ -71,22 +51,7 @@ def test_run_with_different_lengths():
     evaluator = AnswerExactMatchEvaluator()
 
     with pytest.raises(ValueError):
-        evaluator.run(
-            questions=["What is the capital of Germany?"],
-            ground_truth_answers=[["Berlin"], ["Paris"]],
-            predicted_answers=[["Berlin"], ["London"]],
-        )
-
-    with pytest.raises(ValueError):
-        evaluator.run(
-            questions=["What is the capital of Germany?", "What is the capital of France?"],
-            ground_truth_answers=[["Berlin"]],
-            predicted_answers=[["Berlin"], ["London"]],
-        )
+        evaluator.run(ground_truth_answers=[["Berlin"]], predicted_answers=[["Berlin"], ["London"]])
 
     with pytest.raises(ValueError):
-        evaluator.run(
-            questions=["What is the capital of Germany?", "What is the capital of France?"],
-            ground_truth_answers=[["Berlin"], ["Paris"]],
-            predicted_answers=[["Berlin"]],
-        )
+        evaluator.run(ground_truth_answers=[["Berlin"], ["Paris"]], predicted_answers=[["Berlin"]])
diff --git a/test/components/evaluators/test_document_map.py b/test/components/evaluators/test_document_map.py
new file mode 100644
index 00000000..f203dd01
--- /dev/null
+++ b/test/components/evaluators/test_document_map.py
@@ -0,0 +1,78 @@
+import pytest
+
+from haystack import Document
+from haystack.components.evaluators.document_map import DocumentMeanAveragePrecision
+
+
+def test_run_with_all_matching():
+    evaluator = DocumentMeanAveragePrecision()
+    result = evaluator.run(
+        ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+        retrieved_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+    )
+
+    assert result == {"individual_scores": [1.0, 1.0], "score": 1.0}
+
+
+def test_run_with_no_matching():
+    evaluator = DocumentMeanAveragePrecision()
+    result = evaluator.run(
+        ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+        retrieved_documents=[[Document(content="Paris")], [Document(content="London")]],
+    )
+
+    assert result == {"individual_scores": [0.0, 0.0], "score": 0.0}
+
+
+def test_run_with_partial_matching():
+    evaluator = DocumentMeanAveragePrecision()
+    result = evaluator.run(
+        ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+        retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
+    )
+
+    assert result == {"individual_scores": [1.0, 0.0], "score": 0.5}
+
+
+def test_run_with_complex_data():
+    evaluator = DocumentMeanAveragePrecision()
+    result = evaluator.run(
+        ground_truth_documents=[
+            [Document(content="France")],
+            [Document(content="9th century"), Document(content="9th")],
+            [Document(content="classical music"), Document(content="classical")],
+            [Document(content="11th century"), Document(content="the 11th")],
+            [Document(content="Denmark, Iceland and Norway")],
+            [Document(content="10th century"), Document(content="10th")],
+        ],
+        retrieved_documents=[
+            [Document(content="France")],
+            [Document(content="9th century"), Document(content="10th century"), Document(content="9th")],
+            [Document(content="classical"), Document(content="rock music"), Document(content="dubstep")],
+            [Document(content="11th"), Document(content="the 11th"), Document(content="11th century")],
+            [Document(content="Denmark"), Document(content="Norway"), Document(content="Iceland")],
+            [
+                Document(content="10th century"),
+                Document(content="the first half of the 10th century"),
+                Document(content="10th"),
+                Document(content="10th"),
+            ],
+        ],
+    )
+    assert result == {"individual_scores": [1.0, 0.8333333333333333, 1.0, 0.5, 0.0, 1.0], "score": 0.7222222222222222}
+
+
+def test_run_with_different_lengths():
+    with pytest.raises(ValueError):
+        evaluator = DocumentMeanAveragePrecision()
+        evaluator.run(
+            ground_truth_documents=[[Document(content="Berlin")]],
+            retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
+        )
+
+    with pytest.raises(ValueError):
+        evaluator = DocumentMeanAveragePrecision()
+        evaluator.run(
+            ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+            retrieved_documents=[[Document(content="Berlin")]],
+        )
diff --git a/test/components/evaluators/test_document_mrr.py b/test/components/evaluators/test_document_mrr.py
new file mode 100644
index 00000000..959492c6
--- /dev/null
+++ b/test/components/evaluators/test_document_mrr.py
@@ -0,0 +1,82 @@
+import pytest
+
+from haystack import Document
+from haystack.components.evaluators.document_mrr import DocumentMeanReciprocalRank
+
+
+def test_run_with_all_matching():
+    evaluator = DocumentMeanReciprocalRank()
+    result = evaluator.run(
+        ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+        retrieved_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+    )
+
+    assert result == {"individual_scores": [1.0, 1.0], "score": 1.0}
+
+
+def test_run_with_no_matching():
+    evaluator = DocumentMeanReciprocalRank()
+    result = evaluator.run(
+        ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+        retrieved_documents=[[Document(content="Paris")], [Document(content="London")]],
+    )
+
+    assert result == {"individual_scores": [0.0, 0.0], "score": 0.0}
+
+
+def test_run_with_partial_matching():
+    evaluator = DocumentMeanReciprocalRank()
+    result = evaluator.run(
+        ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+        retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
+    )
+
+    assert result == {"individual_scores": [1.0, 0.0], "score": 0.5}
+
+
+def test_run_with_complex_data():
+    evaluator = DocumentMeanReciprocalRank()
+    result = evaluator.run(
+        ground_truth_documents=[
+            [Document(content="France")],
+            [Document(content="9th century"), Document(content="9th")],
+            [Document(content="classical music"), Document(content="classical")],
+            [Document(content="11th century"), Document(content="the 11th")],
+            [Document(content="Denmark, Iceland and Norway")],
+            [Document(content="10th century"), Document(content="10th")],
+        ],
+        retrieved_documents=[
+            [Document(content="France")],
+            [Document(content="10th century"), Document(content="9th century"), Document(content="9th")],
+            [Document(content="rock music"), Document(content="dubstep"), Document(content="classical")],
+            [Document(content="11th"), Document(content="the 11th"), Document(content="11th century")],
+            [Document(content="Denmark"), Document(content="Norway"), Document(content="Iceland")],
+            [
+                Document(content="10th century"),
+                Document(content="the first half of the 10th century"),
+                Document(content="10th"),
+                Document(content="10th"),
+            ],
+        ],
+    )
+
+    assert result == {
+        "individual_scores": [1.0, 0.5, 0.3333333333333333, 0.5, 0.0, 1.0],
+        "score": pytest.approx(0.555555555555555),
+    }
+
+
+def test_run_with_different_lengths():
+    with pytest.raises(ValueError):
+        evaluator = DocumentMeanReciprocalRank()
+        evaluator.run(
+            ground_truth_documents=[[Document(content="Berlin")]],
+            retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
+        )
+
+    with pytest.raises(ValueError):
+        evaluator = DocumentMeanReciprocalRank()
+        evaluator.run(
+            ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+            retrieved_documents=[[Document(content="Berlin")]],
+        )
diff --git a/test/components/evaluators/test_document_recall.py b/test/components/evaluators/test_document_recall.py
index d73406df..56e77f02 100644
--- a/test/components/evaluators/test_document_recall.py
+++ b/test/components/evaluators/test_document_recall.py
@@ -16,7 +16,6 @@ class TestDocumentRecallEvaluatorSingleHit:
 
     def test_run_with_all_matching(self, evaluator):
         result = evaluator.run(
-            questions=["What is the capital of Germany?", "What is the capital of France?"],
             ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
             retrieved_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
         )
@@ -25,7 +24,6 @@ class TestDocumentRecallEvaluatorSingleHit:
 
     def test_run_with_no_matching(self, evaluator):
         result = evaluator.run(
-            questions=["What is the capital of Germany?", "What is the capital of France?"],
             ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
             retrieved_documents=[[Document(content="Paris")], [Document(content="London")]],
         )
@@ -34,7 +32,6 @@ class TestDocumentRecallEvaluatorSingleHit:
 
     def test_run_with_partial_matching(self, evaluator):
         result = evaluator.run(
-            questions=["What is the capital of Germany?", "What is the capital of France?"],
             ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
             retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
         )
@@ -43,14 +40,6 @@ class TestDocumentRecallEvaluatorSingleHit:
 
     def test_run_with_complex_data(self, evaluator):
         result = evaluator.run(
-            questions=[
-                "In what country is Normandy located?",
-                "When was the Latin version of the word Norman first recorded?",
-                "What developed in Normandy during the 1100s?",
-                "In what century did important classical music developments occur in Normandy?",
-                "From which countries did the Norse originate?",
-                "What century did the Normans first gain their separate identity?",
-            ],
             ground_truth_documents=[
                 [Document(content="France")],
                 [Document(content="9th century"), Document(content="9th")],
@@ -78,21 +67,12 @@ class TestDocumentRecallEvaluatorSingleHit:
     def test_run_with_different_lengths(self, evaluator):
         with pytest.raises(ValueError):
             evaluator.run(
-                questions=["What is the capital of Germany?"],
-                ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
-                retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
-            )
-
-        with pytest.raises(ValueError):
-            evaluator.run(
-                questions=["What is the capital of Germany?", "What is the capital of France?"],
                 ground_truth_documents=[[Document(content="Berlin")]],
                 retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
             )
 
         with pytest.raises(ValueError):
             evaluator.run(
-                questions=["What is the capital of Germany?", "What is the capital of France?"],
                 ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
                 retrieved_documents=[[Document(content="Berlin")]],
             )
@@ -105,7 +85,6 @@ class TestDocumentRecallEvaluatorMultiHit:
 
     def test_run_with_all_matching(self, evaluator):
         result = evaluator.run(
-            questions=["What is the capital of Germany?", "What is the capital of France?"],
             ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
             retrieved_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
         )
@@ -114,7 +93,6 @@ class TestDocumentRecallEvaluatorMultiHit:
 
     def test_run_with_no_matching(self, evaluator):
         result = evaluator.run(
-            questions=["What is the capital of Germany?", "What is the capital of France?"],
             ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
             retrieved_documents=[[Document(content="Paris")], [Document(content="London")]],
         )
@@ -123,7 +101,6 @@ class TestDocumentRecallEvaluatorMultiHit:
 
     def test_run_with_partial_matching(self, evaluator):
         result = evaluator.run(
-            questions=["What is the capital of Germany?", "What is the capital of France?"],
             ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
             retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
         )
@@ -132,14 +109,6 @@ class TestDocumentRecallEvaluatorMultiHit:
 
     def test_run_with_complex_data(self, evaluator):
         result = evaluator.run(
-            questions=[
-                "In what country is Normandy located?",
-                "When was the Latin version of the word Norman first recorded?",
-                "What developed in Normandy during the 1100s?",
-                "In what century did important classical music developments occur in Normandy?",
-                "From which countries did the Norse originate?",
-                "What century did the Normans first gain their separate identity?",
-            ],
             ground_truth_documents=[
                 [Document(content="France")],
                 [Document(content="9th century"), Document(content="9th")],
@@ -172,21 +141,12 @@ class TestDocumentRecallEvaluatorMultiHit:
     def test_run_with_different_lengths(self, evaluator):
         with pytest.raises(ValueError):
             evaluator.run(
-                questions=["What is the capital of Germany?"],
-                ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
-                retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
-            )
-
-        with pytest.raises(ValueError):
-            evaluator.run(
-                questions=["What is the capital of Germany?", "What is the capital of France?"],
                 ground_truth_documents=[[Document(content="Berlin")]],
                 retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
             )
 
         with pytest.raises(ValueError):
             evaluator.run(
-                questions=["What is the capital of Germany?", "What is the capital of France?"],
                 ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
                 retrieved_documents=[[Document(content="Berlin")]],
             )