Repository: imartinez/privateGPT
File: private_gpt/server/ingest/ingest_service.py (Python)

Code before:
            for doc_id in ingested_docs_ids:
                ref_doc_info = docstore.get_ref_doc_info(ref_doc_id=doc_id)
                doc_metadata = None
                if ref_doc_info is not None and ref_doc_info.metadata is not None:
                    doc_metadata = IngestedDoc.curate_metadata(ref_doc_info.metadata)
                ingested_docs.append(
                    IngestedDoc(
                        object="ingest.document",
                        doc_id=doc_id,
                        doc_metadata=doc_metadata,
                    )
                )
        except ValueError:
            logger.warning("Got an exception when getting list of docs", exc_info=True)
            pass
        logger.debug("Found count=%s ingested documents", len(ingested_docs))
        return ingested_docs

    def delete(self, doc_id: str) -> None:
        """Delete an ingested document.

        :raises ValueError: if the document does not exist
        """
        logger.info("Deleting the ingested document=%s in the doc store", doc_id)
        self.storage_context.docstore.delete_ref_doc(doc_id)
        # FIXME the documents are only deleted, and not in the vector store
        #  or index store
        # self.storage_context.vector_store.delete(doc_id)
        # self.storage_context.index_store.delete_index_struct(doc_id)


Review comment (suggestion):
Here is the way to fix it. 
LlamaIndex has a built-in function to delete docs from the index (including the index_store, vector_store and doc_store). 

Documentation: https://docs.llamaindex.ai/en/stable/module_guides/indexing/document_management.html#deletion

The working code is (you'll need to import load_index_from_storage from llama_index, and store self.vector_store = vector_store_component.vector_store in the IngestService constructor)

```
def delete(self, doc_id: str) -> None:
        """Delete an ingested document.

        :raises ValueError: if the document does not exist
        """
        logger.info("Deleting the ingested document=%s in the doc store", doc_id)

        index = load_index_from_storage(self.storage_context, store_nodes_override=True)
        index.delete_ref_doc(doc_id, delete_from_docstore=True)

        # persist the index and nodes
        self.storage_context.persist(persist_dir=local_data_path)
```

There are two subtle details in the code to make it work.
store_nodes_override=True -> this is key to force the deletion update the doc_store and index_store
self.storage_context.persist() -> calling this is key to persist the changes

Code after:

            for doc_id in ingested_docs_ids:
                ref_doc_info = docstore.get_ref_doc_info(ref_doc_id=doc_id)
                doc_metadata = None
                if ref_doc_info is not None and ref_doc_info.metadata is not None:
                    doc_metadata = IngestedDoc.curate_metadata(ref_doc_info.metadata)
                ingested_docs.append(
                    IngestedDoc(
                        object="ingest.document",
                        doc_id=doc_id,
                        doc_metadata=doc_metadata,
                    )
                )
        except ValueError:
            logger.warning("Got an exception when getting list of docs", exc_info=True)
            pass
        logger.debug("Found count=%s ingested documents", len(ingested_docs))
        return ingested_docs

    def delete(self, doc_id: str) -> None:
        """Delete an ingested document.

        :raises ValueError: if the document does not exist
        """
        logger.info(
            "Deleting the ingested document=%s in the doc and index store", doc_id
        )

        # Load the index with store_nodes_override=True to be able to delete them
        index = load_index_from_storage(self.storage_context, store_nodes_override=True)

        # Delete the document from the index
        index.delete_ref_doc(doc_id, delete_from_docstore=True)

        # Save the index
        self.storage_context.persist(persist_dir=local_data_path)