Repository: deepset-ai/haystack

Issue:
Incremental update of FAISSDocumentStore leads to incomplete query results
**Describe the bug**
I'm using the FAISSDocumentStore and its index to handle incremental updates of the document base. For that, I must be able to delete old documents and add new documents. Since updating the embeddings takes some time, I want to only update embeddings for newly added documents. However, after deleting a document from the FAISSDocumentStore and adding a new document, querying the FAISSDocumentStore leads to incomplete query results. Instead of returning the top_k documents, less documents get returned although there are enough documents in the FAISSDocumentStore


**To Reproduce**
As a working example, I have changed the covid-FAQ-example a bit:
```python
import logging
import pandas as pd

logging.basicConfig(format="%(levelname)s - %(name)s -  %(message)s", level=logging.WARNING)
logging.getLogger("haystack").setLevel(logging.INFO)

from haystack.telemetry import disable_telemetry

from haystack.document_stores import FAISSDocumentStore


from haystack.nodes import EmbeddingRetriever
from haystack.utils import fetch_archive_from_http

from haystack.pipelines import FAQPipeline
from haystack.utils import print_answers


disable_telemetry()

document_store = FAISSDocumentStore(
    faiss_index_factory_str="Flat",
    embedding_field="embedding",
    embedding_dim=384,
    similarity="cosine",
    duplicate_documents="overwrite" #using skip does not work neither as then meta-data of the documents is not properly stored
)


retriever = EmbeddingRetriever(
    document_store=document_store,
    embedding_model="sentence-transformers/all-MiniLM-L6-v2",
    scale_score=False,
)

# Download
doc_dir = "data/covid"
s3_url = "https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/documents/small_faq_covid.csv.zip"
fetch_archive_from_http(url=s3_url, output_dir=doc_dir, proxies={'http': '127.0.0.1:3129', 'https': '127.0.0.1:3129'})

# Get dataframe with columns "question", "answer" and some custom metadata
df = pd.read_csv(f"{doc_dir}/small_faq_covid.csv")
# Minimal cleaning
df.fillna(value="", inplace=True)
df["question"] = df["question"].apply(lambda x: x.strip())
print(df.head())

df = df.rename(columns={"question": "content"})

# Convert Dataframe to list of dicts and index them in our DocumentStore
docs_to_index = df.to_dict(orient="records")
document_store.write_documents(docs_to_index)

document_store.update_embeddings(retriever=retriever, update_existing_embeddings=False)

pipe = FAQPipeline(retriever=retriever)

prediction = pipe.run(query="How is the virus spreading?", params={"Retriever": {"top_k": 10}})

#expected result: 209, returned result: 209
print(len(document_store.get_all_documents()))
#expected result: 10, returned result: 10
print("Result count: " + str(len(prediction['answers'])))

#delete first found document
idToDelete = prediction["documents"][0].id
document_store.delete_documents(ids=[idToDelete])
#expected result: 208, returned result: 208
print(len(document_store.get_all_documents()))

#re-add all documents and update embeddings for new documents
document_store.write_documents(docs_to_index)

document_store.update_embeddings(retriever=retriever, update_existing_embeddings=False)

prediction = pipe.run(query="How is the virus spreading?", params={"Retriever": {"top_k": 10}})
#expected result: 10, returned result: 6
print("Result count: " + str(len(prediction['answers'])))
```
`

**Expected behavior**
Both prints of the number of returned query results should return 10 as specified by the top_k parameter. However, the last one returns only 6.

**FAQ Check**
- [X] Have you had a look at [our new FAQ page](https://docs.haystack.deepset.ai/docs/faq)?

**System:**
 - OS: Windows 10
 - GPU/CPU: Intel Core i7-6600, Haystack run as CPU-only
 - Haystack version (commit or version number): 1.16.1
 - DocumentStore:
 - Reader:
 - Retriever:



Discussion:
Btw, if you change the parameter update_existing_embeddings to True, it works flawlessly. However, I do not consider it as a promising workaround as it leads to recalculating the embeddings of the whole document store. In my case, this takes way too long and breaks the idea of incremental updates/deletes of the document store.
Ok so the issue is reproducible and seems to be strictly related to the use of `overwrite`. Here is the reduced minimal example:

```python
import logging

logging.basicConfig(format="%(levelname)s - %(name)s -  %(message)s", level=logging.WARNING)
logging.getLogger("haystack").setLevel(logging.INFO)

from haystack import Document
from haystack.document_stores import FAISSDocumentStore
from haystack.nodes import EmbeddingRetriever
from haystack.pipelines import DocumentSearchPipeline


document_store = FAISSDocumentStore(
    faiss_index_factory_str="Flat",
    embedding_field="embedding",
    embedding_dim=384,
    similarity="cosine",
    duplicate_documents="overwrite",  # using skip works, but the meta-data of the documents is not properly stored
)
retriever = EmbeddingRetriever(
    document_store=document_store, embedding_model="sentence-transformers/all-MiniLM-L6-v2", scale_score=False
)
pipe = DocumentSearchPipeline(retriever=retriever)
docs = [Document(f"doc {i}") for i in range(100)]

# First write
print("Docs to write: ", len(docs))
document_store.write_documents(docs)
document_store.update_embeddings(retriever=retriever, update_existing_embeddings=False)
print("Docs count: ", len(document_store.get_all_documents()))  # expected 100, returned 100

prediction = pipe.run(query="doc 30", params={"Retriever": {"top_k": 10}})
print("Result count: ", len(prediction["documents"]))  # expected 10, returned 10

# document_store.delete_documents()
# print("Docs count: ", len(document_store.get_all_documents()))  # expected 0, returned 0

# Second write
print("Docs to write: ", len(docs))
document_store.write_documents(docs)
document_store.update_embeddings(retriever=retriever, update_existing_embeddings=False)
print("Docs count: ", len(document_store.get_all_documents()))  # expected 100, returned 100

prediction = pipe.run(query="doc 30", params={"Retriever": {"top_k": 10}})
print("Result count: ", len(prediction["documents"]))  # expected 10, returned 5
```

Changing `overwrite` to `skip` solves the issue, a sign that there is definitely a bug on how `overwrite` works. I haven't changed any other faiss parameter, which might be also involved in the failure, nor the model used.

Interestingly, emptying the document store in between the two writes also fixes the issue (see commented code).
However, when using skip, there is also a bug and that's why I changed my
code. If I remember correctly, meta data was not stored properly when using
skip.

But this is a different story for another bug report.

ZanSara ***@***.***> schrieb am Mi., 21. Juni 2023, 12:56:

> Ok so the issue is reproducible and seems to be strictly related to the
> use of overwrite. Here is the reduced minimal example:
>
> import logging
> logging.basicConfig(format="%(levelname)s - %(name)s -  %(message)s", level=logging.WARNING)logging.getLogger("haystack").setLevel(logging.INFO)
> from haystack import Documentfrom haystack.document_stores import FAISSDocumentStorefrom haystack.nodes import EmbeddingRetrieverfrom haystack.pipelines import DocumentSearchPipeline
>
> document_store = FAISSDocumentStore(
>     faiss_index_factory_str="Flat",
>     embedding_field="embedding",
>     embedding_dim=384,
>     similarity="cosine",
>     duplicate_documents="overwrite",  # using skip works, but the meta-data of the documents is not properly stored
> )retriever = EmbeddingRetriever(
>     document_store=document_store, embedding_model="sentence-transformers/all-MiniLM-L6-v2", scale_score=False
> )pipe = DocumentSearchPipeline(retriever=retriever)docs = [Document(f"doc {i}") for i in range(100)]
> # First writeprint("Docs to write: ", len(docs))document_store.write_documents(docs)document_store.update_embeddings(retriever=retriever, update_existing_embeddings=False)print("Docs count: ", len(document_store.get_all_documents()))  # expected 209, returned 209
> prediction = pipe.run(query="doc 30", params={"Retriever": {"top_k": 10}})print("Result count: ", len(prediction["documents"]))  # expected 10, returned 10
> # Second writeprint("Docs to write: ", len(docs))document_store.write_documents(docs)document_store.update_embeddings(retriever=retriever, update_existing_embeddings=False)print("Docs count: ", len(document_store.get_all_documents()))  # expected 209, returned 209
> prediction = pipe.run(query="doc 30", params={"Retriever": {"top_k": 10}})print("Result count: ", len(prediction["documents"]))  # expected 10, returned 5
>
> Changing overwrite to skip solves the issue, a sign that there is
> definitely a bug on how overwrite works. I haven't changed any other
> faiss parameter, which might be also involved in the failure, nor the model
> used.
>
> —
> Reply to this email directly, view it on GitHub
> <https://github.com/deepset-ai/haystack/issues/5071#issuecomment-1600622745>,
> or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/AIUZ7HM6RO5QE55T4K7ZDDTXMLHMLANCNFSM6AAAAAAYY2LNL4>
> .
> You are receiving this because you authored the thread.Message ID:
> ***@***.***>
>

Hey @ZanSara...
It seems very similar to https://github.com/deepset-ai/haystack/issues/2834.



Resolving patch:
diff --git a/haystack/document_stores/faiss.py b/haystack/document_stores/faiss.py
--- a/haystack/document_stores/faiss.py
+++ b/haystack/document_stores/faiss.py
@@ -257,54 +257,63 @@ def write_documents(
         document_objects = self._handle_duplicate_documents(
             documents=document_objects, index=index, duplicate_documents=duplicate_documents
         )
-        if len(document_objects) > 0:
-            add_vectors = all(doc.embedding is not None for doc in document_objects)
-
-            if self.duplicate_documents == "overwrite" and add_vectors:
-                logger.warning(
-                    "You have to provide `duplicate_documents = 'overwrite'` arg and "
-                    "`FAISSDocumentStore` does not support update in existing `faiss_index`.\n"
-                    "Please call `update_embeddings` method to repopulate `faiss_index`"
-                )
 
-            vector_id = self.faiss_indexes[index].ntotal
-            with tqdm(
-                total=len(document_objects), disable=not self.progress_bar, position=0, desc="Writing Documents"
-            ) as progress_bar:
-                for i in range(0, len(document_objects), batch_size):
+        if len(document_objects) == 0:
+            return
+
+        vector_id = self.faiss_indexes[index].ntotal
+        add_vectors = all(doc.embedding is not None for doc in document_objects)
+
+        if vector_id > 0 and self.duplicate_documents == "overwrite" and add_vectors:
+            logger.warning(
+                "`FAISSDocumentStore` is adding new vectors to an existing `faiss_index`.\n"
+                "Please call `update_embeddings` method to correctly repopulate `faiss_index`"
+            )
+
+        with tqdm(
+            total=len(document_objects), disable=not self.progress_bar, position=0, desc="Writing Documents"
+        ) as progress_bar:
+            for i in range(0, len(document_objects), batch_size):
+                batch_documents = document_objects[i : i + batch_size]
+                if add_vectors:
+                    if not self.faiss_indexes[index].is_trained:
+                        raise ValueError(
+                            f"FAISS index of type {self.faiss_index_factory_str} must be trained before adding vectors. Call `train_index()` "
+                            "method before adding the vectors. For details, refer to the documentation: "
+                            "[FAISSDocumentStore API](https://docs.haystack.deepset.ai/reference/document-store-api#faissdocumentstoretrain_index)."
+                        )
+
+                    embeddings = [doc.embedding for doc in batch_documents]
+                    embeddings_to_index = np.array(embeddings, dtype="float32")
+
+                    if self.similarity == "cosine":
+                        self.normalize_embedding(embeddings_to_index)
+
+                    self.faiss_indexes[index].add(embeddings_to_index)
+
+                # write_documents method (duplicate_documents="overwrite") should properly work in combination with
+                # update_embeddings method (update_existing_embeddings=False).
+                # If no new embeddings are provided, we save the existing FAISS vector ids
+                elif self.duplicate_documents == "overwrite":
+                    existing_docs = self.get_documents_by_id(ids=[doc.id for doc in batch_documents], index=index)
+                    existing_docs_vector_ids = {
+                        doc.id: doc.meta["vector_id"] for doc in existing_docs if doc.meta and "vector_id" in doc.meta
+                    }
+
+                docs_to_write_in_sql = []
+                for doc in batch_documents:
+                    meta = doc.meta
                     if add_vectors:
-                        if not self.faiss_indexes[index].is_trained:
-                            raise ValueError(
-                                "FAISS index of type {} must be trained before adding vectors. Call `train_index()` "
-                                "method before adding the vectors. For details, refer to the documentation: "
-                                "[FAISSDocumentStore API](https://docs.haystack.deepset.ai/reference/document-store-api#faissdocumentstoretrain_index)."
-                                "".format(self.faiss_index_factory_str)
-                            )
-
-                        embeddings = [doc.embedding for doc in document_objects[i : i + batch_size]]
-                        embeddings_to_index = np.array(embeddings, dtype="float32")
-
-                        if self.similarity == "cosine":
-                            self.normalize_embedding(embeddings_to_index)
-
-                        self.faiss_indexes[index].add(embeddings_to_index)
-
-                    docs_to_write_in_sql = []
-                    for doc in document_objects[i : i + batch_size]:
-                        meta = doc.meta
-                        if add_vectors:
-                            meta["vector_id"] = vector_id
-                            vector_id += 1
-                        docs_to_write_in_sql.append(doc)
-
-                    super(FAISSDocumentStore, self).write_documents(
-                        docs_to_write_in_sql,
-                        index=index,
-                        duplicate_documents=duplicate_documents,
-                        batch_size=batch_size,
-                    )
-                    progress_bar.update(batch_size)
-            progress_bar.close()
+                        meta["vector_id"] = vector_id
+                        vector_id += 1
+                    elif self.duplicate_documents == "overwrite" and doc.id in existing_docs_vector_ids:
+                        meta["vector_id"] = existing_docs_vector_ids[doc.id]
+                    docs_to_write_in_sql.append(doc)
+
+                super(FAISSDocumentStore, self).write_documents(
+                    docs_to_write_in_sql, index=index, duplicate_documents=duplicate_documents, batch_size=batch_size
+                )
+                progress_bar.update(batch_size)
 
     def _create_document_field_map(self) -> Dict:
         return {self.index: self.embedding_field}


Test patch:
diff --git a/test/document_stores/test_faiss.py b/test/document_stores/test_faiss.py
--- a/test/document_stores/test_faiss.py
+++ b/test/document_stores/test_faiss.py
@@ -204,6 +204,25 @@ def test_update_docs_different_indexes(self, ds, documents_with_embeddings):
         assert len(docs_from_index_b) == len(docs_b)
         assert {int(doc.meta["vector_id"]) for doc in docs_from_index_b} == {0, 1, 2, 3}
 
+    @pytest.mark.integration
+    def test_dont_update_existing_embeddings(self, ds, docs):
+        retriever = MockDenseRetriever(document_store=ds)
+        first_doc_id = docs[0].id
+
+        for i in range(1, 4):
+            ds.write_documents(docs[:i])
+            ds.update_embeddings(retriever=retriever, update_existing_embeddings=False)
+
+            assert ds.get_document_count() == i
+            assert ds.get_embedding_count() == i
+            assert ds.get_document_by_id(id=first_doc_id).meta["vector_id"] == "0"
+
+            # Check if the embeddings of the first document remain unchanged after multiple updates
+            if i == 1:
+                first_doc_embedding = ds.get_document_by_id(id=first_doc_id).embedding
+            else:
+                assert np.array_equal(ds.get_document_by_id(id=first_doc_id).embedding, first_doc_embedding)
+
     @pytest.mark.integration
     def test_passing_index_from_outside(self, documents_with_embeddings, tmp_path):
         d = 768