Repository: deepset-ai/haystack

Issue:
BM25Retriever scores <0 filtered out even with algorithms that allow it
**Describe the bug**
Recently this PR was merged that set `scale_scores` to False by default: https://github.com/deepset-ai/haystack/pull/6717
Tagging you here @shadeMe since you were involved in that PR. 

However, some BM25 algorithms, including our default one Okapi, allow for negative scores and we end up filtering those too.

This was discovered in the solution of the advent challenge for day 5 where a community member reported a new kind ex out of bounds error showing up, which was caused because the retriever stopped returning anything: https://github.com/deepset-ai/haystack/discussions/6441#discussioncomment-8194537

**Expected behavior**
If the algorithm allows for negative scores in the first place, maybe we shouldn't filter them out? 

**To Reproduce**
Run the solution colab by removing the fix. I fixed the colab by setting the BM25 algorithm to "BM2Plus". Remove this and you will get an empty list as a result: https://colab.research.google.com/drive/1JzO8lkrb4DtnkPgKnaZkngZeHcfAWQQ5?usp=sharing#scrollTo=WOTZW1ja_8L5



Discussion:
- To be honest, I set `scale_score` to False some time ago in #6276 (for the reasons pointed out in #6271).
- In #6717, docs with scores <=0 were dropped. This does not play well with the fact that some algorithms allow scores <=0 (and we don't scale them)



Patch:
diff --git a/haystack/document_stores/in_memory/document_store.py b/haystack/document_stores/in_memory/document_store.py
index 44b2f6d552..a155e2f3c1 100644
--- a/haystack/document_stores/in_memory/document_store.py
+++ b/haystack/document_stores/in_memory/document_store.py
@@ -34,7 +34,7 @@ class InMemoryDocumentStore:
     def __init__(
         self,
         bm25_tokenization_regex: str = r"(?u)\b\w\w+\b",
-        bm25_algorithm: Literal["BM25Okapi", "BM25L", "BM25Plus"] = "BM25Okapi",
+        bm25_algorithm: Literal["BM25Okapi", "BM25L", "BM25Plus"] = "BM25L",
         bm25_parameters: Optional[Dict] = None,
         embedding_similarity_function: Literal["dot_product", "cosine"] = "dot_product",
     ):
@@ -208,12 +208,17 @@ def bm25_retrieval(
         # get the last top_k indexes and reverse them
         top_docs_positions = np.argsort(docs_scores)[-top_k:][::-1]
 
+        # BM25Okapi can return meaningful negative values, so they should not be filtered out when scale_score is False.
+        # It's the only algorithm supported by rank_bm25 at the time of writing (2024) that can return negative scores.
+        # see https://github.com/deepset-ai/haystack/pull/6889 for more context.
+        negatives_are_valid = self.bm25_algorithm is rank_bm25.BM25Okapi and not scale_score
+
         # Create documents with the BM25 score to return them
         return_documents = []
         for i in top_docs_positions:
             doc = all_documents[i]
             score = docs_scores[i]
-            if score <= 0.0:
+            if not negatives_are_valid and score <= 0.0:
                 continue
             doc_fields = doc.to_dict()
             doc_fields["score"] = score


Test patch:
diff --git a/test/components/retrievers/test_in_memory_bm25_retriever.py b/test/components/retrievers/test_in_memory_bm25_retriever.py
index 4c1df2f22a..c1b16e8526 100644
--- a/test/components/retrievers/test_in_memory_bm25_retriever.py
+++ b/test/components/retrievers/test_in_memory_bm25_retriever.py
@@ -120,7 +120,7 @@ def test_retriever_valid_run(self, mock_docs):
         result = retriever.run(query="PHP")
 
         assert "documents" in result
-        assert len(result["documents"]) == 1
+        assert len(result["documents"]) == 5
         assert result["documents"][0].content == "PHP is a popular programming language"
 
     def test_invalid_run_wrong_store_type(self):
@@ -173,5 +173,5 @@ def test_run_with_pipeline_and_top_k(self, mock_docs, query: str, query_result:
         assert "retriever" in result
         results_docs = result["retriever"]["documents"]
         assert results_docs
-        assert len(results_docs) == 1
+        assert len(results_docs) == top_k
         assert results_docs[0].content == query_result
diff --git a/test/document_stores/test_in_memory.py b/test/document_stores/test_in_memory.py
index 9ebcae4156..3b31c13db3 100644
--- a/test/document_stores/test_in_memory.py
+++ b/test/document_stores/test_in_memory.py
@@ -3,6 +3,7 @@
 
 import pandas as pd
 import pytest
+from haystack_bm25 import rank_bm25
 
 from haystack import Document
 from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
@@ -26,7 +27,7 @@ def test_to_dict(self):
             "type": "haystack.document_stores.in_memory.document_store.InMemoryDocumentStore",
             "init_parameters": {
                 "bm25_tokenization_regex": r"(?u)\b\w\w+\b",
-                "bm25_algorithm": "BM25Okapi",
+                "bm25_algorithm": "BM25L",
                 "bm25_parameters": {},
                 "embedding_similarity_function": "dot_product",
             },
@@ -165,6 +166,39 @@ def test_bm25_retrieval_with_scale_score(self, document_store: InMemoryDocumentS
         results = document_store.bm25_retrieval(query="Python", top_k=1, scale_score=False)
         assert results[0].score != results1[0].score
 
+    def test_bm25_retrieval_with_non_scaled_BM25Okapi(self, document_store: InMemoryDocumentStore):
+        # Highly repetitive documents make BM25Okapi return negative scores, which should not be filtered if the
+        # scores are not scaled
+        docs = [
+            Document(
+                content="""Use pip to install a basic version of Haystack's latest release: pip install
+                farm-haystack. All the core Haystack components live in the haystack repo. But there's also the
+                haystack-extras repo which contains components that are not as widely used, and you need to
+                install them separately."""
+            ),
+            Document(
+                content="""Use pip to install a basic version of Haystack's latest release: pip install
+                farm-haystack[inference]. All the core Haystack components live in the haystack repo. But there's
+                also the haystack-extras repo which contains components that are not as widely used, and you need
+                to install them separately."""
+            ),
+            Document(
+                content="""Use pip to install only the Haystack 2.0 code: pip install haystack-ai. The haystack-ai
+                package is built on the main branch which is an unstable beta version, but it's useful if you want
+                to try the new features as soon as they are merged."""
+            ),
+        ]
+        document_store.write_documents(docs)
+
+        document_store.bm25_algorithm = rank_bm25.BM25Okapi
+        results1 = document_store.bm25_retrieval(query="Haystack installation", top_k=10, scale_score=False)
+        assert len(results1) == 3
+        assert all(res.score < 0.0 for res in results1)
+
+        results2 = document_store.bm25_retrieval(query="Haystack installation", top_k=10, scale_score=True)
+        assert len(results2) == 3
+        assert all(0.0 <= res.score <= 1.0 for res in results2)
+
     def test_bm25_retrieval_with_table_content(self, document_store: InMemoryDocumentStore):
         # Tests if the bm25_retrieval method correctly returns a dataframe when the content_type is table.
         table_content = pd.DataFrame({"language": ["Python", "Java"], "use": ["Data Science", "Web Development"]})