Repository: deepset-ai/haystack

Issue:
Hybrid doc search e2e fails with DuplicateDocumentError
Here is the test run https://github.com/deepset-ai/haystack/actions/runs/9342231146/job/25710027363
which fails with a DuplicteDocumentError.

We are calling `write_documents` twice, writing documents with the same content once without embeddings and once with embeddings:
```python
hybrid_pipeline.get_component("bm25_retriever").document_store.write_documents(documents)
        doc_embedder = SentenceTransformersDocumentEmbedder(model="sentence-transformers/all-MiniLM-L6-v2")
        doc_embedder.warm_up()
        embedded_documents = doc_embedder.run(documents=documents)["documents"]
>       hybrid_pipeline.get_component("embedding_retriever").document_store.write_documents(embedded_documents)
```

The error occurs since we merged [feat: Add memory sharing between different instances of InMemoryDocumentStore](https://github.com/deepset-ai/haystack/pull/7781)



Discussion:

Resolving patch:
diff --git a/haystack/document_stores/in_memory/document_store.py b/haystack/document_stores/in_memory/document_store.py
--- a/haystack/document_stores/in_memory/document_store.py
+++ b/haystack/document_stores/in_memory/document_store.py
@@ -46,6 +46,8 @@ class BM25DocumentStats:
 # Global storage for all InMemoryDocumentStore instances, indexed by the index name.
 _STORAGES: Dict[str, Dict[str, Document]] = {}
 _BM25_STATS_STORAGES: Dict[str, Dict[str, BM25DocumentStats]] = {}
+_AVERAGE_DOC_LEN_STORAGES: Dict[str, float] = {}
+_FREQ_VOCAB_FOR_IDF_STORAGES: Dict[str, Counter] = {}
 
 
 class InMemoryDocumentStore:
@@ -91,14 +93,16 @@ def __init__(
         self.bm25_parameters = bm25_parameters or {}
         self.embedding_similarity_function = embedding_similarity_function
 
-        # Global BM25 statistics
-        self._avg_doc_len: float = 0.0
-        self._freq_vocab_for_idf: Counter = Counter()
-
         # Per-document statistics
         if self.index not in _BM25_STATS_STORAGES:
             _BM25_STATS_STORAGES[self.index] = {}
 
+        if self.index not in _AVERAGE_DOC_LEN_STORAGES:
+            _AVERAGE_DOC_LEN_STORAGES[self.index] = 0.0
+
+        if self.index not in _FREQ_VOCAB_FOR_IDF_STORAGES:
+            _FREQ_VOCAB_FOR_IDF_STORAGES[self.index] = Counter()
+
     @property
     def storage(self) -> Dict[str, Document]:
         """
@@ -110,6 +114,18 @@ def storage(self) -> Dict[str, Document]:
     def _bm25_attr(self) -> Dict[str, BM25DocumentStats]:
         return _BM25_STATS_STORAGES.get(self.index, {})
 
+    @property
+    def _avg_doc_len(self) -> float:
+        return _AVERAGE_DOC_LEN_STORAGES.get(self.index, 0.0)
+
+    @_avg_doc_len.setter
+    def _avg_doc_len(self, value: float):
+        _AVERAGE_DOC_LEN_STORAGES[self.index] = value
+
+    @property
+    def _freq_vocab_for_idf(self) -> Counter:
+        return _FREQ_VOCAB_FOR_IDF_STORAGES.get(self.index, Counter())
+
     def _dispatch_bm25(self):
         """
         Select the correct BM25 algorithm based on user specification.


Test patch:
diff --git a/e2e/pipelines/test_hybrid_doc_search_pipeline.py b/e2e/pipelines/test_hybrid_doc_search_pipeline.py
--- a/e2e/pipelines/test_hybrid_doc_search_pipeline.py
+++ b/e2e/pipelines/test_hybrid_doc_search_pipeline.py
@@ -2,7 +2,6 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-import json
 
 from haystack import Document, Pipeline
 from haystack.components.embedders import SentenceTransformersDocumentEmbedder, SentenceTransformersTextEmbedder
@@ -10,7 +9,6 @@
 from haystack.components.rankers import TransformersSimilarityRanker
 from haystack.components.retrievers.in_memory import InMemoryBM25Retriever, InMemoryEmbeddingRetriever
 from haystack.document_stores.in_memory import InMemoryDocumentStore
-from haystack.document_stores.types import DuplicatePolicy
 
 
 def test_hybrid_doc_search_pipeline(tmp_path):
@@ -50,7 +48,6 @@ def test_hybrid_doc_search_pipeline(tmp_path):
         Document(content="My name is Mario and I live in the capital of Italy."),
         Document(content="My name is Giorgio and I live in Rome."),
     ]
-    hybrid_pipeline.get_component("bm25_retriever").document_store.write_documents(documents)
     doc_embedder = SentenceTransformersDocumentEmbedder(model="sentence-transformers/all-MiniLM-L6-v2")
     doc_embedder.warm_up()
     embedded_documents = doc_embedder.run(documents=documents)["documents"]