Repository: deepset-ai/haystack

Issue:
Change the default `DuplicatePolicy` value in `write_documents`
**Describe the bug**

Currently the default value for the duplication policy in [write_documents](https://github.com/deepset-ai/haystack/blob/main/haystack/document_stores/protocols.py#L118) is `DuplicatePolicy.FAIL`.

This makes impossible to change the default to a different value while respecting the protocol.

**Expected behavior**

The method shouldn't be opinionated and the default value should be something like `None` or an explicit `DuplicatePolicy.NONE`.


Discussion:

Resolving patch:
diff --git a/haystack/document_stores/in_memory/document_store.py b/haystack/document_stores/in_memory/document_store.py
--- a/haystack/document_stores/in_memory/document_store.py
+++ b/haystack/document_stores/in_memory/document_store.py
@@ -103,18 +103,11 @@ def filter_documents(self, filters: Optional[Dict[str, Any]] = None) -> List[Doc
             return [doc for doc in self.storage.values() if document_matches_filter(filters=filters, document=doc)]
         return list(self.storage.values())
 
-    def write_documents(self, documents: List[Document], policy: DuplicatePolicy = DuplicatePolicy.FAIL) -> int:
+    def write_documents(self, documents: List[Document], policy: DuplicatePolicy = DuplicatePolicy.NONE) -> int:
         """
-        Writes (or overwrites) documents into the DocumentStore.
-
-        :param documents: A list of documents.
-        :param policy: Documents with the same ID count as duplicates. When duplicates are met,
-            the DocumentStore can:
-             - skip: keep the existing document and ignore the new one.
-             - overwrite: remove the old document and write the new one.
-             - fail: an error is raised.
-        :raises DuplicateError: Exception trigger on duplicate document if `policy=DuplicatePolicy.FAIL`
-        :return: None
+        Refer to the DocumentStore.write_documents() protocol documentation.
+
+        If `policy` is set to `DuplicatePolicy.NONE` defaults to `DuplicatePolicy.FAIL`.
         """
         if (
             not isinstance(documents, Iterable)
@@ -123,6 +116,9 @@ def write_documents(self, documents: List[Document], policy: DuplicatePolicy = D
         ):
             raise ValueError("Please provide a list of Documents.")
 
+        if policy == DuplicatePolicy.NONE:
+            policy = DuplicatePolicy.FAIL
+
         written_documents = len(documents)
         for document in documents:
             if policy != DuplicatePolicy.OVERWRITE and document.id in self.storage.keys():
diff --git a/haystack/document_stores/protocols.py b/haystack/document_stores/protocols.py
--- a/haystack/document_stores/protocols.py
+++ b/haystack/document_stores/protocols.py
@@ -12,6 +12,7 @@
 
 
 class DuplicatePolicy(Enum):
+    NONE = "none"
     SKIP = "skip"
     OVERWRITE = "overwrite"
     FAIL = "fail"
@@ -115,20 +116,20 @@ def filter_documents(self, filters: Optional[Dict[str, Any]] = None) -> List[Doc
         """
         ...
 
-    def write_documents(self, documents: List[Document], policy: DuplicatePolicy = DuplicatePolicy.FAIL) -> int:
+    def write_documents(self, documents: List[Document], policy: DuplicatePolicy = DuplicatePolicy.NONE) -> int:
         """
-        Writes (or overwrites) documents into the DocumentStore.
-
-        :param documents: a list of documents.
-        :param policy: documents with the same ID count as duplicates. When duplicates are met,
-            the DocumentStore can:
-             - skip: keep the existing document and ignore the new one.
-             - overwrite: remove the old document and write the new one.
-             - fail: an error is raised
-        :raises DuplicateError: Exception trigger on duplicate document if `policy=DuplicatePolicy.FAIL`
-        :return: The number of documents that was written.
-            If DuplicatePolicy.OVERWRITE is used, this number is always equal to the number of documents in input.
-            If DuplicatePolicy.SKIP is used, this number can be lower than the number of documents in the input list.
+        Writes Documents into the DocumentStore.
+
+        :param documents: a list of Document objects.
+        :param policy: the policy to apply when a Document with the same id already exists in the DocumentStore.
+            - `DuplicatePolicy.NONE`: Default policy, behaviour depends on the Document Store.
+            - `DuplicatePolicy.SKIP`: If a Document with the same id already exists, it is skipped and not written.
+            - `DuplicatePolicy.OVERWRITE`: If a Document with the same id already exists, it is overwritten.
+            - `DuplicatePolicy.FAIL`: If a Document with the same id already exists, an error is raised.
+        :raises DuplicateError: If `policy` is set to `DuplicatePolicy.FAIL` and a Document with the same id already exists.
+        :return: The number of Documents written.
+            If `DuplicatePolicy.OVERWRITE` is used, this number is always equal to the number of documents in input.
+            If `DuplicatePolicy.SKIP` is used, this number can be lower than the number of documents in the input list.
         """
         ...
 


Test patch:
diff --git a/haystack/testing/document_store.py b/haystack/testing/document_store.py
--- a/haystack/testing/document_store.py
+++ b/haystack/testing/document_store.py
@@ -70,11 +70,14 @@ def document_store(self):
     @pytest.mark.unit
     def test_write_documents(self, document_store: DocumentStore):
         """
-        Test write_documents() normal behaviour.
+        Test write_documents() default behaviour.
         """
-        doc = Document(content="test doc")
-        assert document_store.write_documents([doc]) == 1
-        assert document_store.filter_documents() == [doc]
+        msg = (
+            "Default write_documents() behaviour depends on the Document Store implementantion, "
+            "as we don't enforce a default behaviour when no policy is set. "
+            "Override this test in your custom test class."
+        )
+        raise NotImplementedError(msg)
 
     @pytest.mark.unit
     def test_write_documents_duplicate_fail(self, document_store: DocumentStore):
@@ -83,7 +86,7 @@ def test_write_documents_duplicate_fail(self, document_store: DocumentStore):
         using DuplicatePolicy.FAIL.
         """
         doc = Document(content="test doc")
-        assert document_store.write_documents([doc]) == 1
+        assert document_store.write_documents([doc], policy=DuplicatePolicy.FAIL) == 1
         with pytest.raises(DuplicateDocumentError):
             document_store.write_documents(documents=[doc], policy=DuplicatePolicy.FAIL)
         assert document_store.filter_documents() == [doc]
@@ -95,7 +98,7 @@ def test_write_documents_duplicate_skip(self, document_store: DocumentStore):
         using DuplicatePolicy.SKIP.
         """
         doc = Document(content="test doc")
-        assert document_store.write_documents([doc]) == 1
+        assert document_store.write_documents([doc], policy=DuplicatePolicy.SKIP) == 1
         assert document_store.write_documents(documents=[doc], policy=DuplicatePolicy.SKIP) == 0
 
     @pytest.mark.unit
@@ -107,7 +110,7 @@ def test_write_documents_duplicate_overwrite(self, document_store: DocumentStore
         doc1 = Document(id="1", content="test doc 1")
         doc2 = Document(id="1", content="test doc 2")
 
-        assert document_store.write_documents([doc2]) == 1
+        assert document_store.write_documents([doc2], policy=DuplicatePolicy.OVERWRITE) == 1
         assert document_store.filter_documents() == [doc2]
         assert document_store.write_documents(documents=[doc1], policy=DuplicatePolicy.OVERWRITE) == 1
         assert document_store.filter_documents() == [doc1]
diff --git a/test/document_stores/test_in_memory.py b/test/document_stores/test_in_memory.py
--- a/test/document_stores/test_in_memory.py
+++ b/test/document_stores/test_in_memory.py
@@ -5,7 +5,7 @@
 import pytest
 
 from haystack import Document
-from haystack.document_stores import InMemoryDocumentStore, DocumentStoreError, DuplicatePolicy
+from haystack.document_stores import InMemoryDocumentStore, DocumentStoreError, DuplicatePolicy, DuplicateDocumentError
 
 
 from haystack.testing.document_store import DocumentStoreBaseTests
@@ -70,17 +70,11 @@ def test_from_dict(self, mock_regex):
         assert store.bm25_algorithm.__name__ == "BM25Plus"
         assert store.bm25_parameters == {"key": "value"}
 
-    @pytest.mark.unit
-    def test_written_documents_count(self, document_store: InMemoryDocumentStore):
-        # FIXME Remove after the document store base tests have been rewritten
-        documents = [Document(content=f"Hello world #{i}") for i in range(10)]
-        docs_written = document_store.write_documents(documents[0:2])
-        assert docs_written == 2
-        assert document_store.filter_documents() == documents[0:2]
-
-        docs_written = document_store.write_documents(documents, DuplicatePolicy.SKIP)
-        assert docs_written == len(documents) - 2
-        assert document_store.filter_documents() == documents
+    def test_write_documents(self, document_store):
+        docs = [Document(id="1")]
+        assert document_store.write_documents(docs) == 1
+        with pytest.raises(DuplicateDocumentError):
+            document_store.write_documents(docs)
 
     @pytest.mark.unit
     def test_bm25_retrieval(self, document_store: InMemoryDocumentStore):