Repository: deepset-ai/haystack

Issue:
MemoryBM25Retriever returns non relevant documents
**Describe the bug**

`MemoryBM25Retriever` currently always returns `top_k` number of `Document` even if they're not relevant.

The [`test_retriever_valid_run`](https://github.com/deepset-ai/haystack/blob/ccc9f010bbdc75e1c5f1d269a43a5f06f36acba1/test/preview/components/retrievers/test_memory_bm25_retriever.py#L119-L129) is a good example of the current behavior.

A simple fix would be to filter out `Document`s with `score==0.0` before scaling the score.

**Expected behavior**

Non relevant `Document`, those with score `0.0`, are not returned.

**Additional context**

@tstadel confirmed that it's unexpected we're returning `Document`s with a score of `0.0`.

**To Reproduce**

```
from haystack.preview.document_stores import MemoryDocumentStore
from haystack.preview.dataclasses import Document
from haystack.preview.components.retrievers import MemoryBM25Retriever

docs = [
    Document(text="Javascript is a popular programming language"),
    Document(text="Java is a popular programming language"),
    Document(text="Python is a popular programming language"),
    Document(text="Ruby is a popular programming language"),
    Document(text="PHP is a popular programming language"),
]

ds = MemoryDocumentStore()
ds.write_documents(docs)

retriever = MemoryBM25Retriever(ds, top_k=5, scale_score=False)
result = retriever.run(query="PHP")

# This is wrong
assert len(result["documents"]) == 5
assert result["documents"][0].text == "PHP is a popular programming language"
assert result["documents"][1].score == 0.0
assert result["documents"][2].score == 0.0
assert result["documents"][3].score == 0.0
assert result["documents"][4].score == 0.0

# This is the expected behaviour
assert len(result["documents"]) == 1
assert result["documents"][0].text == "PHP is a popular programming language"
assert result["documents"][0].score > 0.0
```



Discussion:

Resolving patch:
diff --git a/haystack/document_stores/in_memory/document_store.py b/haystack/document_stores/in_memory/document_store.py
--- a/haystack/document_stores/in_memory/document_store.py
+++ b/haystack/document_stores/in_memory/document_store.py
@@ -212,8 +212,11 @@ def bm25_retrieval(
         return_documents = []
         for i in top_docs_positions:
             doc = all_documents[i]
+            score = docs_scores[i]
+            if score <= 0.0:
+                continue
             doc_fields = doc.to_dict()
-            doc_fields["score"] = docs_scores[i]
+            doc_fields["score"] = score
             return_document = Document.from_dict(doc_fields)
             return_documents.append(return_document)
         return return_documents


Test patch:
diff --git a/test/components/retrievers/test_in_memory_bm25_retriever.py b/test/components/retrievers/test_in_memory_bm25_retriever.py
--- a/test/components/retrievers/test_in_memory_bm25_retriever.py
+++ b/test/components/retrievers/test_in_memory_bm25_retriever.py
@@ -113,15 +113,14 @@ def test_from_dict_nonexisting_docstore(self):
             InMemoryBM25Retriever.from_dict(data)
 
     def test_retriever_valid_run(self, mock_docs):
-        top_k = 5
         ds = InMemoryDocumentStore()
         ds.write_documents(mock_docs)
 
-        retriever = InMemoryBM25Retriever(ds, top_k=top_k)
+        retriever = InMemoryBM25Retriever(ds, top_k=5)
         result = retriever.run(query="PHP")
 
         assert "documents" in result
-        assert len(result["documents"]) == top_k
+        assert len(result["documents"]) == 1
         assert result["documents"][0].content == "PHP is a popular programming language"
 
     def test_invalid_run_wrong_store_type(self):
@@ -174,5 +173,5 @@ def test_run_with_pipeline_and_top_k(self, mock_docs, query: str, query_result:
         assert "retriever" in result
         results_docs = result["retriever"]["documents"]
         assert results_docs
-        assert len(results_docs) == top_k
+        assert len(results_docs) == 1
         assert results_docs[0].content == query_result
diff --git a/test/document_stores/test_in_memory.py b/test/document_stores/test_in_memory.py
--- a/test/document_stores/test_in_memory.py
+++ b/test/document_stores/test_in_memory.py
@@ -5,8 +5,8 @@
 import pytest
 
 from haystack import Document
-from haystack.document_stores.in_memory import InMemoryDocumentStore
 from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
+from haystack.document_stores.in_memory import InMemoryDocumentStore
 from haystack.testing.document_store import DocumentStoreBaseTests
 
 
@@ -17,7 +17,7 @@ class TestMemoryDocumentStore(DocumentStoreBaseTests):  # pylint: disable=R0904
 
     @pytest.fixture
     def document_store(self) -> InMemoryDocumentStore:
-        return InMemoryDocumentStore()
+        return InMemoryDocumentStore(bm25_algorithm="BM25L")
 
     def test_to_dict(self):
         store = InMemoryDocumentStore()
@@ -73,7 +73,6 @@ def test_write_documents(self, document_store):
             document_store.write_documents(docs)
 
     def test_bm25_retrieval(self, document_store: InMemoryDocumentStore):
-        document_store = InMemoryDocumentStore()
         # Tests if the bm25_retrieval method returns the correct document based on the input query.
         docs = [Document(content="Hello world"), Document(content="Haystack supports multiple languages")]
         document_store.write_documents(docs)
@@ -106,7 +105,7 @@ def test_bm25_retrieval_with_different_top_k(self, document_store: InMemoryDocum
         document_store.write_documents(docs)
 
         # top_k = 2
-        results = document_store.bm25_retrieval(query="languages", top_k=2)
+        results = document_store.bm25_retrieval(query="language", top_k=2)
         assert len(results) == 2
 
         # top_k = 3
@@ -141,7 +140,7 @@ def test_bm25_retrieval_with_updated_docs(self, document_store: InMemoryDocument
         document_store.write_documents(docs)
 
         results = document_store.bm25_retrieval(query="Python", top_k=1)
-        assert len(results) == 1
+        assert len(results) == 0
 
         document_store.write_documents([Document(content="Python is a popular programming language")])
         results = document_store.bm25_retrieval(query="Python", top_k=1)
@@ -199,10 +198,10 @@ def test_bm25_retrieval_default_filter_for_text_and_dataframes(self, document_st
         docs = [Document(), Document(content="Gardening"), Document(content="Bird watching")]
         document_store.write_documents(docs)
         results = document_store.bm25_retrieval(query="doesn't matter, top_k is 10", top_k=10)
-        assert len(results) == 2
+        assert len(results) == 0
 
     def test_bm25_retrieval_with_filters(self, document_store: InMemoryDocumentStore):
-        selected_document = Document(content="Gardening", meta={"selected": True})
+        selected_document = Document(content="Java is, well...", meta={"selected": True})
         docs = [Document(), selected_document, Document(content="Bird watching")]
         document_store.write_documents(docs)
         results = document_store.bm25_retrieval(query="Java", top_k=10, filters={"selected": True})
@@ -224,10 +223,10 @@ def test_bm25_retrieval_with_filters_on_text_or_dataframe(self, document_store:
         assert results[0].id == document.id
 
     def test_bm25_retrieval_with_documents_with_mixed_content(self, document_store: InMemoryDocumentStore):
-        double_document = Document(content="Gardening", embedding=[1.0, 2.0, 3.0])
+        double_document = Document(content="Gardening is a hobby", embedding=[1.0, 2.0, 3.0])
         docs = [Document(embedding=[1.0, 2.0, 3.0]), double_document, Document(content="Bird watching")]
         document_store.write_documents(docs)
-        results = document_store.bm25_retrieval(query="Java", top_k=10, filters={"embedding": {"$not": None}})
+        results = document_store.bm25_retrieval(query="Gardening", top_k=10, filters={"embedding": {"$not": None}})
         assert len(results) == 1
         assert results[0].id == double_document.id