Repository: deepset-ai/haystack

Issue:
Implement function to calculate Recall metric
As specified in proposal #5794 we need to implement a function to calculate the Recall metric.

Ideally the function should be part of the private interface and called only through the `calculate_metrics` function (see #6063). `_calculate_recall()` could be a nice name.

For more detailed information check out the original proposal.


Discussion:

Resolving patch:
diff --git a/haystack/components/evaluators/document_recall.py b/haystack/components/evaluators/document_recall.py
new file mode 100644
--- /dev/null
+++ b/haystack/components/evaluators/document_recall.py
@@ -0,0 +1,112 @@
+from enum import Enum
+from typing import Any, Dict, List, Union
+
+from haystack.core.component import component
+from haystack.dataclasses import Document
+
+
+class RecallMode(Enum):
+    """
+    Enum for the mode to use for calculating the recall score.
+    """
+
+    # Score is based on whether any document is retrieved.
+    SINGLE_HIT = "single_hit"
+    # Score is based on how many documents were retrieved.
+    MULTI_HIT = "multi_hit"
+
+    def __str__(self):
+        return self.value
+
+    @staticmethod
+    def from_str(string: str) -> "RecallMode":
+        enum_map = {e.value: e for e in RecallMode}
+        mode = enum_map.get(string)
+        if mode is None:
+            msg = f"Unknown recall mode '{string}'. Supported modes are: {list(enum_map.keys())}"
+            raise ValueError(msg)
+        return mode
+
+
+@component
+class DocumentRecallEvaluator:
+    """
+    Evaluator that calculates the Recall score for a list of questions.
+    Returns both a list of scores for each question and the average.
+    Each question can have multiple ground truth documents and multiple predicted documents.
+
+    Usage example:
+    ```python
+    from haystack.components.evaluators import DocumentRecallEvaluator
+    evaluator = DocumentRecallEvaluator()
+    result = evaluator.run(
+        questions=["What is the capital of Germany?", "What is the capital of France?"],
+        ground_truth_answers=[["Berlin"], ["Paris"]],
+        predicted_answers=[["Paris"], ["London"]],
+    )
+    print(result["individual_scores"])
+    # [0.0, 0.0]
+    print(result["score"])
+    # 0.0
+    ```
+    """
+
+    def __init__(self, mode: Union[str, RecallMode] = RecallMode.SINGLE_HIT):
+        """
+        Create a DocumentRecallEvaluator component.
+
+        :param mode:
+            Mode to use for calculating the recall score.
+        """
+        if isinstance(mode, str):
+            mode = RecallMode.from_str(mode)
+
+        mode_functions = {RecallMode.SINGLE_HIT: self._recall_single_hit, RecallMode.MULTI_HIT: self._recall_multi_hit}
+        self.mode_function = mode_functions[mode]
+
+    def _recall_single_hit(self, ground_truth_documents: List[Document], retrieved_documents: List[Document]) -> bool:
+        unique_truths = {g.content for g in ground_truth_documents}
+        unique_retrievals = {p.content for p in retrieved_documents}
+        retrieved_ground_truths = unique_truths.intersection(unique_retrievals)
+
+        return len(retrieved_ground_truths) > 0
+
+    def _recall_multi_hit(self, ground_truth_documents: List[Document], retrieved_documents: List[Document]) -> float:
+        unique_truths = {g.content for g in ground_truth_documents}
+        unique_retrievals = {p.content for p in retrieved_documents}
+        retrieved_ground_truths = unique_truths.intersection(unique_retrievals)
+
+        return len(retrieved_ground_truths) / len(ground_truth_documents)
+
+    @component.output_types(score=float, individual_scores=List[float])
+    def run(
+        self,
+        questions: List[str],
+        ground_truth_documents: List[List[Document]],
+        retrieved_documents: List[List[Document]],
+    ) -> Dict[str, Any]:
+        """
+        Run the DocumentRecallEvaluator on the given inputs.
+        All lists must have the same length.
+
+        :param questions:
+            A list of questions.
+        :param ground_truth_documents:
+            A list of expected documents for each question.
+        :param retrieved_documents:
+            A list of retrieved documents for each question.
+        A dictionary with the following outputs:
+            - `score` - The average of calculated scores.
+            - `invididual_scores` - A list of numbers from 0.0 to 1.0 that represents the proportion of matching documents retrieved.
+                                    If the mode is `single_hit`, the individual scores are True or False.
+        """
+        if not len(questions) == len(ground_truth_documents) == len(retrieved_documents):
+            msg = "The length of questions, ground_truth_documents, and predicted_documents must be the same."
+            raise ValueError(msg)
+
+        scores = []
+        for ground_truth, retrieved in zip(ground_truth_documents, retrieved_documents):
+            score = self.mode_function(ground_truth, retrieved)
+            scores.append(score)
+
+        return {"score": sum(scores) / len(questions), "individual_scores": scores}


Test patch:
diff --git a/test/components/evaluators/test_document_recall.py b/test/components/evaluators/test_document_recall.py
new file mode 100644
--- /dev/null
+++ b/test/components/evaluators/test_document_recall.py
@@ -0,0 +1,192 @@
+import pytest
+
+from haystack.components.evaluators.document_recall import DocumentRecallEvaluator, RecallMode
+from haystack.dataclasses import Document
+
+
+def test_init_with_unknown_mode_string():
+    with pytest.raises(ValueError):
+        DocumentRecallEvaluator(mode="unknown_mode")
+
+
+class TestDocumentRecallEvaluatorSingleHit:
+    @pytest.fixture
+    def evaluator(self):
+        return DocumentRecallEvaluator(mode=RecallMode.SINGLE_HIT)
+
+    def test_run_with_all_matching(self, evaluator):
+        result = evaluator.run(
+            questions=["What is the capital of Germany?", "What is the capital of France?"],
+            ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+            retrieved_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+        )
+
+        assert result == {"individual_scores": [1.0, 1.0], "score": 1.0}
+
+    def test_run_with_no_matching(self, evaluator):
+        result = evaluator.run(
+            questions=["What is the capital of Germany?", "What is the capital of France?"],
+            ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+            retrieved_documents=[[Document(content="Paris")], [Document(content="London")]],
+        )
+
+        assert result == {"individual_scores": [0.0, 0.0], "score": 0.0}
+
+    def test_run_with_partial_matching(self, evaluator):
+        result = evaluator.run(
+            questions=["What is the capital of Germany?", "What is the capital of France?"],
+            ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+            retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
+        )
+
+        assert result == {"individual_scores": [1.0, 0.0], "score": 0.5}
+
+    def test_run_with_complex_data(self, evaluator):
+        result = evaluator.run(
+            questions=[
+                "In what country is Normandy located?",
+                "When was the Latin version of the word Norman first recorded?",
+                "What developed in Normandy during the 1100s?",
+                "In what century did important classical music developments occur in Normandy?",
+                "From which countries did the Norse originate?",
+                "What century did the Normans first gain their separate identity?",
+            ],
+            ground_truth_documents=[
+                [Document(content="France")],
+                [Document(content="9th century"), Document(content="9th")],
+                [Document(content="classical music"), Document(content="classical")],
+                [Document(content="11th century"), Document(content="the 11th")],
+                [Document(content="Denmark, Iceland and Norway")],
+                [Document(content="10th century"), Document(content="10th")],
+            ],
+            retrieved_documents=[
+                [Document(content="France")],
+                [Document(content="9th century"), Document(content="10th century"), Document(content="9th")],
+                [Document(content="classical"), Document(content="rock music"), Document(content="dubstep")],
+                [Document(content="11th"), Document(content="the 11th"), Document(content="11th century")],
+                [Document(content="Denmark"), Document(content="Norway"), Document(content="Iceland")],
+                [
+                    Document(content="10th century"),
+                    Document(content="the first half of the 10th century"),
+                    Document(content="10th"),
+                    Document(content="10th"),
+                ],
+            ],
+        )
+        assert result == {"individual_scores": [True, True, True, True, False, True], "score": 0.8333333333333334}
+
+    def test_run_with_different_lengths(self, evaluator):
+        with pytest.raises(ValueError):
+            evaluator.run(
+                questions=["What is the capital of Germany?"],
+                ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+                retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
+            )
+
+        with pytest.raises(ValueError):
+            evaluator.run(
+                questions=["What is the capital of Germany?", "What is the capital of France?"],
+                ground_truth_documents=[[Document(content="Berlin")]],
+                retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
+            )
+
+        with pytest.raises(ValueError):
+            evaluator.run(
+                questions=["What is the capital of Germany?", "What is the capital of France?"],
+                ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+                retrieved_documents=[[Document(content="Berlin")]],
+            )
+
+
+class TestDocumentRecallEvaluatorMultiHit:
+    @pytest.fixture
+    def evaluator(self):
+        return DocumentRecallEvaluator(mode=RecallMode.MULTI_HIT)
+
+    def test_run_with_all_matching(self, evaluator):
+        result = evaluator.run(
+            questions=["What is the capital of Germany?", "What is the capital of France?"],
+            ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+            retrieved_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+        )
+
+        assert result == {"individual_scores": [1.0, 1.0], "score": 1.0}
+
+    def test_run_with_no_matching(self, evaluator):
+        result = evaluator.run(
+            questions=["What is the capital of Germany?", "What is the capital of France?"],
+            ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+            retrieved_documents=[[Document(content="Paris")], [Document(content="London")]],
+        )
+
+        assert result == {"individual_scores": [0.0, 0.0], "score": 0.0}
+
+    def test_run_with_partial_matching(self, evaluator):
+        result = evaluator.run(
+            questions=["What is the capital of Germany?", "What is the capital of France?"],
+            ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+            retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
+        )
+
+        assert result == {"individual_scores": [1.0, 0.0], "score": 0.5}
+
+    def test_run_with_complex_data(self, evaluator):
+        result = evaluator.run(
+            questions=[
+                "In what country is Normandy located?",
+                "When was the Latin version of the word Norman first recorded?",
+                "What developed in Normandy during the 1100s?",
+                "In what century did important classical music developments occur in Normandy?",
+                "From which countries did the Norse originate?",
+                "What century did the Normans first gain their separate identity?",
+            ],
+            ground_truth_documents=[
+                [Document(content="France")],
+                [Document(content="9th century"), Document(content="9th")],
+                [Document(content="classical music"), Document(content="classical")],
+                [Document(content="11th century"), Document(content="the 11th")],
+                [
+                    Document(content="Denmark"),
+                    Document(content="Iceland"),
+                    Document(content="Norway"),
+                    Document(content="Denmark, Iceland and Norway"),
+                ],
+                [Document(content="10th century"), Document(content="10th")],
+            ],
+            retrieved_documents=[
+                [Document(content="France")],
+                [Document(content="9th century"), Document(content="10th century"), Document(content="9th")],
+                [Document(content="classical"), Document(content="rock music"), Document(content="dubstep")],
+                [Document(content="11th"), Document(content="the 11th"), Document(content="11th century")],
+                [Document(content="Denmark"), Document(content="Norway"), Document(content="Iceland")],
+                [
+                    Document(content="10th century"),
+                    Document(content="the first half of the 10th century"),
+                    Document(content="10th"),
+                    Document(content="10th"),
+                ],
+            ],
+        )
+        assert result == {"individual_scores": [1.0, 1.0, 0.5, 1.0, 0.75, 1.0], "score": 0.875}
+
+    def test_run_with_different_lengths(self, evaluator):
+        with pytest.raises(ValueError):
+            evaluator.run(
+                questions=["What is the capital of Germany?"],
+                ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+                retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
+            )
+
+        with pytest.raises(ValueError):
+            evaluator.run(
+                questions=["What is the capital of Germany?", "What is the capital of France?"],
+                ground_truth_documents=[[Document(content="Berlin")]],
+                retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
+            )
+
+        with pytest.raises(ValueError):
+            evaluator.run(
+                questions=["What is the capital of Germany?", "What is the capital of France?"],
+                ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+                retrieved_documents=[[Document(content="Berlin")]],
+            )