Repository: deepset-ai/haystack

Issue:
Remove pin from pydantic
**Is your feature request related to a problem? Please describe.**
We had several tests failing after the release of pydantic 1.10 and we [pinned the version](https://github.com/deepset-ai/haystack/pull/3126) to an older one but we should stay up to date as that's an important dependency for Haystack.

**Describe the solution you'd like**
Remove the pin and fix the test failures (those are mostly related to the new `dataclass` decorator that we use e.g. in `Document`)



Discussion:

Resolving patch:
diff --git a/haystack/nodes/file_converter/pdf.py b/haystack/nodes/file_converter/pdf.py
--- a/haystack/nodes/file_converter/pdf.py
+++ b/haystack/nodes/file_converter/pdf.py
@@ -1,4 +1,4 @@
-from typing import List, Optional, Dict
+from typing import List, Optional, Dict, Any
 
 import os
 import logging
@@ -74,7 +74,7 @@ def __init__(
     def convert(
         self,
         file_path: Path,
-        meta: Optional[Dict[str, str]] = None,
+        meta: Optional[Dict[str, Any]] = None,
         remove_numeric_tables: Optional[bool] = None,
         valid_languages: Optional[List[str]] = None,
         encoding: Optional[str] = None,
@@ -212,7 +212,7 @@ def __init__(
     def convert(
         self,
         file_path: Path,
-        meta: Optional[Dict[str, str]] = None,
+        meta: Optional[Dict[str, Any]] = None,
         remove_numeric_tables: Optional[bool] = None,
         valid_languages: Optional[List[str]] = None,
         encoding: Optional[str] = None,
diff --git a/haystack/nodes/summarizer/transformers.py b/haystack/nodes/summarizer/transformers.py
--- a/haystack/nodes/summarizer/transformers.py
+++ b/haystack/nodes/summarizer/transformers.py
@@ -28,7 +28,7 @@ class TransformersSummarizer(BaseSummarizer):
     **Example**
 
     ```python
-    |     docs = [Document(text="PG&E stated it scheduled the blackouts in response to forecasts for high winds amid dry conditions."
+    |     docs = [Document(content="PG&E stated it scheduled the blackouts in response to forecasts for high winds amid dry conditions."
     |            "The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were scheduled to be affected by"
     |            "the shutoffs which were expected to last through at least midday tomorrow.")]
     |
diff --git a/haystack/schema.py b/haystack/schema.py
--- a/haystack/schema.py
+++ b/haystack/schema.py
@@ -2,11 +2,10 @@
 import csv
 import hashlib
 
-import typing
 from typing import Any, Optional, Dict, List, Union
 
 try:
-    from typing import Literal
+    from typing import Literal  # type: ignore
 except ImportError:
     from typing_extensions import Literal  # type: ignore
 
@@ -16,21 +15,18 @@
 import time
 import json
 import ast
-from dataclasses import asdict
+from dataclasses import asdict, InitVar
 
 import mmh3
 import numpy as np
 import pandas as pd
 
-from pydantic import BaseConfig
+from pydantic import BaseConfig, Field
 from pydantic.json import pydantic_encoder
 
-if not typing.TYPE_CHECKING:
-    # We are using Pydantic dataclasses instead of vanilla Python's
-    # See #1598 for the reasons behind this choice & performance considerations
-    from pydantic.dataclasses import dataclass
-else:
-    from dataclasses import dataclass  # type: ignore  # pylint: disable=ungrouped-imports
+# We are using Pydantic dataclasses instead of vanilla Python's
+# See #1598 for the reasons behind this choice & performance considerations
+from pydantic.dataclasses import dataclass
 
 
 logger = logging.getLogger(__name__)
@@ -41,12 +37,13 @@
 
 @dataclass
 class Document:
-    content: Union[str, pd.DataFrame]
-    content_type: Literal["text", "table", "image", "audio"]
     id: str
-    meta: Dict[str, Any]
+    content: Union[str, pd.DataFrame]
+    content_type: Literal["text", "table", "image", "audio"] = Field(default="text")
+    meta: Dict[str, Any] = Field(default={})
     score: Optional[float] = None
     embedding: Optional[np.ndarray] = None
+    id_hash_keys: InitVar[Optional[List[str]]] = None
 
     # We use a custom init here as we want some custom logic. The annotations above are however still needed in order
     # to use some dataclass magic like "asdict()". See https://www.python.org/dev/peps/pep-0557/#custom-init-method
@@ -58,7 +55,7 @@ def __init__(
         content_type: Literal["text", "table", "image", "audio"] = "text",
         id: Optional[str] = None,
         score: Optional[float] = None,
-        meta: Dict[str, Any] = None,
+        meta: Optional[Dict[str, Any]] = None,
         embedding: Optional[np.ndarray] = None,
         id_hash_keys: Optional[List[str]] = None,
     ):
@@ -66,15 +63,11 @@ def __init__(
         One of the core data classes in Haystack. It's used to represent documents / passages in a standardized way within Haystack.
         Documents are stored in DocumentStores, are returned by Retrievers, are the input for Readers and are used in
         many other places that manipulate or interact with document-level data.
-
         Note: There can be multiple Documents originating from one file (e.g. PDF), if you split the text
         into smaller passages. We'll have one Document per passage in this case.
-
         Each document has a unique ID. This can be supplied by the user or generated automatically.
         It's particularly helpful for handling of duplicates and referencing documents in other objects (e.g. Labels)
-
         There's an easy option to convert from/to dicts via `from_dict()` and `to_dict`.
-
         :param content: Content of the document. For most cases, this will be text, but it can be a table or image.
         :param content_type: One of "text", "table" or "image". Haystack components can use this to adjust their
                              handling of Documents and check compatibility.
@@ -154,6 +147,9 @@ def to_dict(self, field_map={}) -> Dict:
         inv_field_map = {v: k for k, v in field_map.items()}
         _doc: Dict[str, str] = {}
         for k, v in self.__dict__.items():
+            # Exclude internal fields (Pydantic, ...) fields from the conversion process
+            if k.startswith("__"):
+                continue
             if k == "content":
                 # Convert pd.DataFrame to list of rows for serialization
                 if self.content_type == "table" and isinstance(self.content, pd.DataFrame):
@@ -184,6 +180,9 @@ def from_dict(
             _doc["meta"] = {}
         # copy additional fields into "meta"
         for k, v in _doc.items():
+            # Exclude internal fields (Pydantic, ...) fields from the conversion process
+            if k.startswith("__"):
+                continue
             if k not in init_args and k not in field_map:
                 _doc["meta"][k] = v
         # remove additional fields from top level
@@ -615,6 +614,8 @@ class MultiLabel:
     contexts: List[str]
     offsets_in_contexts: List[Dict]
     offsets_in_documents: List[Dict]
+    drop_negative_labels: InitVar[bool] = False
+    drop_no_answer: InitVar[bool] = False
 
     def __init__(self, labels: List[Label], drop_negative_labels=False, drop_no_answers=False, **kwargs):
         """
@@ -676,6 +677,7 @@ def __init__(self, labels: List[Label], drop_negative_labels=False, drop_no_answ
         # as separate no_answer labels, and thus with document.id but without answer.document_id.
         # If we do not exclude them from document_ids this would be problematic for retriever evaluation as they do not contain the answer.
         # Hence, we exclude them here as well.
+
         self.document_ids = [l.document.id for l in self.labels if not l.no_answer]
         self.contexts = [l.document.content for l in self.labels if not l.no_answer]
 


Test patch:
diff --git a/test/nodes/test_generator.py b/test/nodes/test_generator.py
--- a/test/nodes/test_generator.py
+++ b/test/nodes/test_generator.py
@@ -2,7 +2,6 @@
 import sys
 from typing import List
 
-import numpy as np
 import pytest
 
 from haystack.schema import Document
@@ -64,8 +63,8 @@ def test_generator_pipeline(document_store, retriever, rag_generator, docs_with_
 def test_lfqa_pipeline(document_store, retriever, lfqa_generator, docs_with_true_emb):
     # reuse existing DOCS but regenerate embeddings with retribert
     docs: List[Document] = []
-    for idx, d in enumerate(docs_with_true_emb):
-        docs.append(Document(d.content, str(idx)))
+    for d in docs_with_true_emb:
+        docs.append(Document(content=d.content))
     document_store.write_documents(docs)
     document_store.update_embeddings(retriever)
     query = "Tell me about Berlin?"
@@ -84,8 +83,8 @@ def test_lfqa_pipeline(document_store, retriever, lfqa_generator, docs_with_true
 def test_lfqa_pipeline_unknown_converter(document_store, retriever, docs_with_true_emb):
     # reuse existing DOCS but regenerate embeddings with retribert
     docs: List[Document] = []
-    for idx, d in enumerate(docs_with_true_emb):
-        docs.append(Document(d.content, str(idx)))
+    for d in docs_with_true_emb:
+        docs.append(Document(content=d.content))
     document_store.write_documents(docs)
     document_store.update_embeddings(retriever)
     seq2seq = Seq2SeqGenerator(model_name_or_path="patrickvonplaten/t5-tiny-random")
@@ -106,8 +105,8 @@ def test_lfqa_pipeline_unknown_converter(document_store, retriever, docs_with_tr
 def test_lfqa_pipeline_invalid_converter(document_store, retriever, docs_with_true_emb):
     # reuse existing DOCS but regenerate embeddings with retribert
     docs: List[Document] = []
-    for idx, d in enumerate(docs_with_true_emb):
-        docs.append(Document(d.content, str(idx)))
+    for d in docs_with_true_emb:
+        docs.append(Document(content=d.content))
     document_store.write_documents(docs)
     document_store.update_embeddings(retriever)