Repository: deepset-ai/haystack

Issue:
Split overlap metadata array can't be added to Pinecone and Weaviate
**Describe the bug**
Using haystack from master with Pinecone and Weaviate i get error about not supported arrays in metada.
weaviate.exceptions.UnexpectedStatusCodeException: Add property to class! Unexpected status code: 422, with response body: {'error': [{'message': "property '_split_overlap': invalid dataType: '[]' is not a valid class name"}]}.
Overlap metadata is very useful, i'm using it at the moment, so would like to keep it.

**Error message**
raceback (most recent call last):
  File "/mnt/c/work/ai/haystack/examples/preprocess_and_index_qa_weaviate.py", line 75, in <module>
    basic_qa_pipeline()
  File "/mnt/c/work/ai/haystack/examples/preprocess_and_index_qa_weaviate.py", line 36, in basic_qa_pipeline
    document_store.write_documents(docs)
  File "/mnt/c/work/ai/haystack/haystack/document_stores/weaviate.py", line 567, in write_documents
    self._update_schema(property, _doc[property], index)
  File "/mnt/c/work/ai/haystack/haystack/document_stores/weaviate.py", line 426, in _update_schema
    self.weaviate_client.schema.property.create(index, property_dict)
  File "/root/.virtualenvs/haystack/lib/python3.8/site-packages/weaviate/schema/properties/crud_properties.py", line 80, in create
    raise UnexpectedStatusCodeException("Add property to class", response)
weaviate.exceptions.UnexpectedStatusCodeException: Add property to class! Unexpected status code: 422, with response body: {'error': [{'message': "property '_split_overlap': invalid dataType: '[]' is not a valid class name"}]}.

**Expected behavior**
Proposed solution is to use dict instead of table with entries: split_start, split_end.

**Additional context**
Add any other context about the problem here, like document types / preprocessing steps / settings of reader etc.

**To Reproduce**
Run indexing with overlapping from Haystack main with Weaviate or Pinecone.

**FAQ Check**
- [n] Have you had a look at [our new FAQ page](https://docs.haystack.deepset.ai/docs/faq)?

**System:**
 - OS: 
 - GPU/CPU:
 - Haystack version (commit or version number): 1.16.0-rc0
 - DocumentStore:
 - Reader:
 - Retriever:



Discussion:
Hey @ldebowczyk, can you share the code that generates this error? It can help us figure out where the issue is.
Hi, 

Sure, here you are. You will need locally running weaviate instance.

`
import logging

logging.basicConfig(format="%(levelname)s - %(name)s -  %(message)s", level=logging.WARNING)
logging.getLogger("haystack").setLevel(logging.INFO)

def basic_qa_pipeline():
    OPEN_AI_API_KEY = "Open-AI-Key"
    doc_dir = "data"

    from haystack.utils import convert_files_to_docs
    from haystack.nodes import PreProcessor

    all_docs = convert_files_to_docs(dir_path=doc_dir)
    preprocessor = PreProcessor(
        clean_empty_lines=True,
        clean_whitespace=True,
        clean_header_footer=False,
        split_by="word",
        split_length=100,
        split_overlap=20,
        split_respect_sentence_boundary=True,
        add_page_number=True
    )
    docs = preprocessor.process(all_docs)

    print(f"n_files_input: {len(all_docs)}\nn_docs_output: {len(docs)}")

    from haystack.document_stores import WeaviateDocumentStore
    document_store = WeaviateDocumentStore(host="http://localhost", port=8080, index="test", embedding_dim=1536)

    from haystack.nodes import EmbeddingRetriever

    retriever = EmbeddingRetriever(
        document_store=document_store,
        embedding_model="text-embedding-ada-002",
        batch_size=32,
        api_key=OPEN_AI_API_KEY,
        max_seq_len=1024
    )

    document_store.write_documents(docs)
    document_store.update_embeddings(retriever, batch_size=16)

if __name__ == "__main__":
    basic_qa_pipeline()
`

Resolving patch:
diff --git a/haystack/document_stores/pinecone.py b/haystack/document_stores/pinecone.py
--- a/haystack/document_stores/pinecone.py
+++ b/haystack/document_stores/pinecone.py
@@ -1,3 +1,4 @@
+import json
 from typing import Set, Union, List, Optional, Dict, Generator, Any
 
 import logging
@@ -1217,6 +1218,8 @@ def _get_documents_by_meta(
         for _id, meta in zip(ids, metadata):
             content = meta.pop("content")
             content_type = meta.pop("content_type")
+            if "_split_overlap" in meta:
+                meta["_split_overlap"] = json.loads(meta["_split_overlap"])
             doc = Document(id=_id, content=content, content_type=content_type, meta=meta)
             documents.append(doc)
         if return_embedding:
@@ -1366,6 +1369,8 @@ def _meta_for_pinecone(self, meta: Dict[str, Any], parent_key: str = "", labels:
                 # Replace any None values with empty strings
                 if value is None:
                     value = ""
+                if key == "_split_overlap":
+                    value = json.dumps(value)
                 # format key
                 new_key = f"{parent_key}.{key}" if parent_key else key
                 # if value is dict, expand
diff --git a/haystack/document_stores/weaviate.py b/haystack/document_stores/weaviate.py
--- a/haystack/document_stores/weaviate.py
+++ b/haystack/document_stores/weaviate.py
@@ -243,13 +243,19 @@ def _create_schema_and_index(self, index: Optional[str] = None, recreate_index:
                 )
 
     def _convert_weaviate_result_to_document(
-        self, result: dict, return_embedding: bool, scale_score: bool = True
+        self,
+        result: dict,
+        return_embedding: bool,
+        scale_score: bool = True,
+        json_properties: Optional[List[str]] = None,
     ) -> Document:
         """
         Convert weaviate result dict into haystack document object. This is more involved because
         weaviate search result dict varies between get and query interfaces.
         Weaviate get methods return the data items in properties key, whereas the query doesn't.
         """
+        json_properties = json_properties or []
+
         score = None
         content = ""
 
@@ -315,6 +321,12 @@ def _convert_weaviate_result_to_document(
                 continue
             if v is None:
                 continue
+            # Weaviate doesn't support dict as a property type. We store them as JSON strings and convert them back here.
+            if k in json_properties:
+                if isinstance(v, list):
+                    v = [json.loads(str(item)) for item in v]
+                else:
+                    v = json.loads(str(v))
             meta_data[k] = v
 
         document = Document.from_dict(
@@ -353,7 +365,11 @@ def get_document_by_id(
         except weaviate.exceptions.UnexpectedStatusCodeException as usce:
             logger.debug("Weaviate could not get the document requested: %s", usce)
         if result:
-            document = self._convert_weaviate_result_to_document(result, return_embedding=True)
+            # We retrieve the JSON properties from the schema and convert them back to the Python dicts
+            json_properties = self._get_json_properties(index=index)
+            document = self._convert_weaviate_result_to_document(
+                result, return_embedding=True, json_properties=json_properties
+            )
         return document
 
     def get_documents_by_id(
@@ -370,6 +386,8 @@ def get_documents_by_id(
             raise NotImplementedError("WeaviateDocumentStore does not support headers.")
 
         index = self._sanitize_index_name(index) or self.index
+        # We retrieve the JSON properties from the schema and convert them back to the Python dicts
+        json_properties = self._get_json_properties(index=index)
         documents = []
         # TODO: better implementation with multiple where filters instead of chatty call below?
         for id in ids:
@@ -380,7 +398,9 @@ def get_documents_by_id(
             except weaviate.exceptions.UnexpectedStatusCodeException as usce:
                 logger.debug("Weaviate could not get the document requested: %s", usce)
             if result:
-                document = self._convert_weaviate_result_to_document(result, return_embedding=True)
+                document = self._convert_weaviate_result_to_document(
+                    result, return_embedding=True, json_properties=json_properties
+                )
                 documents.append(document)
         return documents
 
@@ -435,6 +455,22 @@ def _get_date_properties(self, index: Optional[str] = None) -> List[str]:
 
         return cur_properties
 
+    def _get_json_properties(self, index: Optional[str] = None) -> List[str]:
+        """
+        Get all existing properties that store JSON strings in the indexes' schema.
+        """
+        index = self._sanitize_index_name(index) or self.index
+        cur_properties = []
+        for class_item in self.weaviate_client.schema.get()["classes"]:
+            if class_item["class"] == index:
+                cur_properties = [
+                    item["name"]
+                    for item in class_item["properties"]
+                    if item["description"].startswith("JSON dynamic property")
+                ]
+
+        return cur_properties
+
     def _update_schema(
         self, new_prop: str, property_value: Union[List, str, int, float, bool], index: Optional[str] = None
     ):
@@ -444,7 +480,13 @@ def _update_schema(
         index = self._sanitize_index_name(index) or self.index
         data_type = self._get_weaviate_type_of_value(property_value)
 
-        property_dict = {"dataType": [data_type], "description": f"dynamic property {new_prop}", "name": new_prop}
+        description = f"dynamic property {new_prop}"
+        if data_type.startswith("dict"):
+            # Weaviate does not support dict type, so we use string or list of string instead
+            data_type = "string" + data_type[4:]
+            description = f"JSON dynamic property {new_prop}"
+
+        property_dict = {"dataType": [data_type], "description": description, "name": new_prop}
         self.weaviate_client.schema.property.create(index, property_dict)
 
     @staticmethod
@@ -472,6 +514,9 @@ def _get_weaviate_type_of_value(value: Union[List, str, int, float, bool]) -> st
             data_type = "number"
         elif isinstance(value, bool):
             data_type = "boolean"
+        # Weaviate doesn't support dictionaries as property types, we'll convert them to JSON strings.
+        elif isinstance(value, dict):
+            data_type = "dict"
 
         if list_of_values:
             data_type += "[]"
@@ -557,6 +602,7 @@ def write_documents(
         batched_documents = get_batches_from_generator(document_objects, batch_size)
         with tqdm(total=len(document_objects), disable=not self.progress_bar) as progress_bar:
             for document_batch in batched_documents:
+                json_fields = []
                 for doc in document_batch:
                     _doc = {**doc.to_dict(field_map=self._create_document_field_map())}
                     _ = _doc.pop("score", None)
@@ -569,6 +615,14 @@ def write_documents(
                                 raise ValueError(
                                     f'"meta" info contains duplicate key "{k}" with the top-level document structure'
                                 )
+                            # Weaviate doesn't support dictionaries as property types, so we convert them to JSON strings
+                            if isinstance(v, dict):
+                                json_fields.append(k)
+                                v = json.dumps(v)
+                            elif isinstance(v, list):
+                                if len(v) > 0 and isinstance(v[0], dict):
+                                    json_fields.append(k)
+                                    v = [json.dumps(item) for item in v]
                             _doc[k] = v
                         _doc.pop("meta")
 
@@ -586,7 +640,10 @@ def write_documents(
                     missing_props = self._check_document(current_properties, _doc)
                     if missing_props:
                         for property in missing_props:
-                            self._update_schema(property, _doc[property], index)
+                            property_value = _doc[property]
+                            if property in json_fields:
+                                property_value = doc.meta[property]
+                            self._update_schema(property, property_value, index)
                             current_properties.append(property)
 
                     # Weaviate requires dates to be in RFC3339 format
@@ -891,8 +948,12 @@ def get_all_documents_generator(
             return_embedding = self.return_embedding
 
         results = self._get_all_documents_in_index(index=index, filters=filters, batch_size=batch_size)
+        # We retrieve the JSON properties from the schema and convert them back to the Python dicts
+        json_properties = self._get_json_properties(index=index)
         for result in results:
-            document = self._convert_weaviate_result_to_document(result, return_embedding=return_embedding)
+            document = self._convert_weaviate_result_to_document(
+                result, return_embedding=return_embedding, json_properties=json_properties
+            )
             yield document
 
     def query(
@@ -1048,9 +1109,13 @@ def query(
             if query_output.get("data").get("Get").get(index):
                 results = query_output.get("data").get("Get").get(index)
 
+        # We retrieve the JSON properties from the schema and convert them back to the Python dicts
+        json_properties = self._get_json_properties(index=index)
         documents = []
         for result in results:
-            doc = self._convert_weaviate_result_to_document(result, return_embedding=True, scale_score=scale_score)
+            doc = self._convert_weaviate_result_to_document(
+                result, return_embedding=True, scale_score=scale_score, json_properties=json_properties
+            )
             documents.append(doc)
 
         return documents
@@ -1312,10 +1377,12 @@ def query_by_embedding(
             if query_output.get("data").get("Get").get(index):
                 results = query_output.get("data").get("Get").get(index)
 
+        # We retrieve the JSON properties from the schema and convert them back to the Python dicts
+        json_properties = self._get_json_properties(index=index)
         documents = []
         for result in results:
             doc = self._convert_weaviate_result_to_document(
-                result, return_embedding=return_embedding, scale_score=scale_score
+                result, return_embedding=return_embedding, scale_score=scale_score, json_properties=json_properties
             )
             documents.append(doc)
 
@@ -1383,10 +1450,13 @@ def update_embeddings(
             )
 
         result = self._get_all_documents_in_index(index=index, filters=filters, batch_size=batch_size)
+        # We retrieve the JSON properties from the schema and convert them back to the Python dicts
+        json_properties = self._get_json_properties(index=index)
 
         for result_batch in get_batches_from_generator(result, batch_size):
             document_batch = [
-                self._convert_weaviate_result_to_document(hit, return_embedding=False) for hit in result_batch
+                self._convert_weaviate_result_to_document(hit, return_embedding=False, json_properties=json_properties)
+                for hit in result_batch
             ]
             embeddings = retriever.embed_documents(document_batch)
             self._validate_embeddings_shape(


Test patch:
diff --git a/test/document_stores/test_pinecone.py b/test/document_stores/test_pinecone.py
--- a/test/document_stores/test_pinecone.py
+++ b/test/document_stores/test_pinecone.py
@@ -3,10 +3,11 @@
 import os
 import numpy as np
 from inspect import getmembers, isclass, isfunction
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, ANY
 
 import pytest
 
+from haystack.document_stores.pinecone import pinecone
 from haystack.document_stores.pinecone import PineconeDocumentStore
 from haystack.schema import Document
 from haystack.errors import FilterError, PineconeDocumentStoreError
@@ -52,6 +53,17 @@ def doc_store_with_docs(self, ds: PineconeDocumentStore, documents: List[Documen
         ds.write_documents(documents)
         return ds
 
+    @pytest.fixture
+    def mocked_ds(self):
+        class DSMock(PineconeDocumentStore):
+            pass
+
+        pinecone.init = MagicMock()
+        DSMock._create_index = MagicMock()
+        mocked_ds = DSMock(api_key="MOCK")
+
+        return mocked_ds
+
     @pytest.fixture
     def docs_all_formats(self) -> List[Union[Document, Dict[str, Any]]]:
         return [
@@ -515,3 +527,34 @@ def test_get_all_labels_legacy_document_id(self, ds, monkeypatch):
         labels = ds.get_all_labels()
 
         assert labels[0].answer.document_ids == ["a0747b83aea0b60c4b114b15476dd32d"]
+
+    @pytest.mark.unit
+    def test_split_overlap_meta(self, mocked_ds):
+        """
+        Tests that we can upload Docs with a _split_overlap_meta field to Pinecone as a JSON string
+        and that the field is parsed correctly as dictionary when retrieved.
+        """
+        doc = Document(content="test", meta={"_split_overlap": [{"doc_id": "test_id", "range": (0, 10)}]}, id="test_id")
+        # Test writing as JSON string
+        mocked_ds.write_documents([doc])
+        call_args = mocked_ds.pinecone_indexes["document"].upsert.call_args.kwargs
+        assert list(call_args["vectors"])[0][2] == {
+            "content": "test",
+            "content_type": "text",
+            "_split_overlap": '[{"doc_id": "test_id", "range": [0, 10]}]',
+        }
+        # Test retrieving as dict
+        mocked_ds._get_all_document_ids = MagicMock(return_value=["test_id"])
+        mocked_ds.pinecone_indexes["document"].fetch.return_value = {
+            "vectors": {
+                "test_id": {
+                    "metadata": {
+                        "_split_overlap": '[{"doc_id": "test_id", "range": [0, 10]}]',
+                        "content": "test",
+                        "content_type": "text",
+                    }
+                }
+            }
+        }
+        retrieved_docs = mocked_ds.get_all_documents()
+        assert retrieved_docs[0].meta["_split_overlap"] == [{"doc_id": "test_id", "range": [0, 10]}]
diff --git a/test/document_stores/test_weaviate.py b/test/document_stores/test_weaviate.py
--- a/test/document_stores/test_weaviate.py
+++ b/test/document_stores/test_weaviate.py
@@ -8,7 +8,7 @@
 from haystack.document_stores.weaviate import WeaviateDocumentStore
 from haystack.schema import Document
 from haystack.testing import DocumentStoreBaseTestAbstract
-
+from haystack.document_stores import weaviate
 
 embedding_dim = 768
 
@@ -59,6 +59,23 @@ def documents(self):
 
         return documents
 
+    @pytest.fixture()
+    def mocked_ds(self):
+        """
+        This fixture provides an instance of the WeaviateDocumentStore equipped with a mocked Weaviate client.
+        """
+
+        class DSMock(WeaviateDocumentStore):
+            pass
+
+        mocked_client = mock.MagicMock()
+        mocked_client.Client().is_ready.return_value = True
+        mocked_client.Client().schema.contains.return_value = False
+        weaviate.client = mocked_client
+        mocked_ds = DSMock()
+
+        return mocked_ds
+
     @pytest.mark.skip(reason="Weaviate does not support labels")
     @pytest.mark.integration
     def test_write_labels(self):
@@ -258,12 +275,9 @@ def test_get_embedding_count(self, ds, documents):
         assert ds.get_embedding_count() == 9
 
     @pytest.mark.unit
-    def test__get_current_properties(self):
-        with mock.patch("haystack.document_stores.weaviate.client") as mocked_client:
-            mocked_client.Client().is_ready.return_value = True
-            mocked_client.Client().schema.contains.return_value = False
-            mocked_client.Client().schema.get.return_value = json.loads(
-                """
+    def test__get_current_properties(self, mocked_ds):
+        mocked_ds.weaviate_client.schema.get.return_value = json.loads(
+            """
 {
   "classes": [{
     "class": "Document",
@@ -279,7 +293,146 @@ def test__get_current_properties(self):
     ]
   }]
 } """
-            )
-            ds = WeaviateDocumentStore()
-            # Ensure we dropped the cross-reference property
-            assert ds._get_current_properties() == ["hitCounter"]
+        )
+        # Ensure we dropped the cross-reference property
+        assert mocked_ds._get_current_properties() == ["hitCounter"]
+
+    @pytest.mark.unit
+    def test_dict_metadata(self, mocked_ds):
+        """
+        Tests that metadata of type dict is converted to JSON string when writing to Weaviate and converted
+        back to dict when reading from Weaviate.
+        """
+        doc = Document(content="test", meta={"dict_field": {"key": "value"}})
+        # Test writing as JSON string
+        mocked_ds.write_documents([doc])
+        mocked_ds.weaviate_client.batch.add_data_object.assert_called_with(
+            data_object={
+                "content": json.dumps(doc.content),
+                "content_type": doc.content_type,
+                "id_hash_keys": doc.id_hash_keys,
+                "dict_field": json.dumps(doc.meta["dict_field"]),
+            },
+            class_name=mock.ANY,
+            uuid=mock.ANY,
+            vector=mock.ANY,
+        )
+
+        # Test retrieving as dict
+        mocked_ds.weaviate_client.query.get().do.return_value = json.loads(
+            """
+            {
+              "data": {
+                "Get": {
+                  "Document": [
+                    {
+                      "content": "\\"test\\"",
+                      "content_type": "text",
+                      "dict_field": "{\\"key\\": \\"value\\"}",
+                      "id_hash_keys": ["content"]
+                    }
+                  ]
+                }
+              }
+            }
+            """
+        )
+        mocked_ds.get_document_count = mock.MagicMock(return_value=1)
+        mocked_ds.weaviate_client.schema.get.return_value = json.loads(
+            """
+            {
+              "classes": [
+                {
+                  "class": "Document",
+                  "description": "Haystack index, it's a class in Weaviate",
+                  "properties": [
+                    {
+                      "dataType": ["text"],
+                      "description": "Document Content (e.g. the text)",
+                      "name": "content",
+                      "tokenization": "word"
+                    },
+                    {
+                      "dataType": ["text"],
+                      "description": "JSON dynamic property dict_field",
+                      "name": "dict_field",
+                      "tokenization": "whitespace"
+                    }
+                  ]
+                }
+              ]
+            }
+            """
+        )
+        retrieved_docs = mocked_ds.get_all_documents()
+        assert retrieved_docs[0].meta["dict_field"] == {"key": "value"}
+
+    @pytest.mark.unit
+    def test_list_of_dict_metadata(self, mocked_ds):
+        """
+        Tests that metadata of type list of dict is converted to list of JSON string when writing to Weaviate and
+        converted back to list of dict when reading from Weaviate.
+        """
+        doc = Document(content="test", meta={"list_dict_field": [{"key": "value"}, {"key": "value"}]})
+        # Test writing as list of JSON strings
+        mocked_ds.write_documents([doc])
+        mocked_ds.weaviate_client.batch.add_data_object.assert_called_with(
+            data_object={
+                "content": json.dumps(doc.content),
+                "content_type": doc.content_type,
+                "id_hash_keys": doc.id_hash_keys,
+                "list_dict_field": [json.dumps(item) for item in doc.meta["list_dict_field"]],
+            },
+            class_name=mock.ANY,
+            uuid=mock.ANY,
+            vector=mock.ANY,
+        )
+
+        # Test retrieving as list of dict
+        mocked_ds.weaviate_client.query.get().do.return_value = json.loads(
+            """
+            {
+              "data": {
+                "Get": {
+                  "Document": [
+                    {
+                      "content": "\\"test\\"",
+                      "content_type": "text",
+                      "list_dict_field": ["{\\"key\\": \\"value\\"}", "{\\"key\\": \\"value\\"}"],
+                      "id_hash_keys": ["content"]
+                    }
+                  ]
+                }
+              }
+            }
+            """
+        )
+        mocked_ds.get_document_count = mock.MagicMock(return_value=1)
+        mocked_ds.weaviate_client.schema.get.return_value = json.loads(
+            """
+            {
+              "classes": [
+                {
+                  "class": "Document",
+                  "description": "Haystack index, it's a class in Weaviate",
+                  "properties": [
+                    {
+                      "dataType": ["text"],
+                      "description": "Document Content (e.g. the text)",
+                      "name": "content",
+                      "tokenization": "word"
+                    },
+                    {
+                      "dataType": ["text[]"],
+                      "description": "JSON dynamic property dict_field",
+                      "name": "list_dict_field",
+                      "tokenization": "whitespace"
+                    }
+                  ]
+                }
+              ]
+            }
+            """
+        )
+        retrieved_docs = mocked_ds.get_all_documents()
+        assert retrieved_docs[0].meta["list_dict_field"] == [{"key": "value"}, {"key": "value"}]