Repository: deepset-ai/haystack

Issue:
PDFMinerToDocument updates Document's meta field after initializing it
**Describe the bug**
The PDFMinerToDocument updates meta data of a Document after initializing it instead of setting the meta data at initialization: https://github.com/deepset-ai/haystack/blob/dd9660f90d8cd074ac420139e0f78fa3970b162e/haystack/components/converters/pdfminer.py#L177

This can lead to two documents with the same content but different metadata to be assigned the same document id. When these documents are written to a DocumentStore, they will be handled as duplicates although the aren't.

**Expected behavior**
Set meta data of Document in PDFMinerToDocument when Document is initialized instead of updating meta data later.

**Additional context**
A similar issue was fixed for PyPDFToDocument in https://github.com/deepset-ai/haystack/pull/8698

**To Reproduce**
Use PDFMinerToDocument with an empty PDF, which returns a Document with empty content with filepath stored in meta. Then compare the id that document to the id of Document with empty document and no meta data. They IDs are currently the same but should differ as in this test: https://github.com/deepset-ai/haystack/blob/dd9660f90d8cd074ac420139e0f78fa3970b162e/test/components/converters/test_pypdf_to_document.py#L206

**FAQ Check**
- [x] Have you had a look at [our new FAQ page](https://docs.haystack.deepset.ai/docs/faq)?

Discussion:

Solution patch:
diff --git a/haystack/components/converters/pdfminer.py b/haystack/components/converters/pdfminer.py
index 8642447816..c5f2415685 100644
--- a/haystack/components/converters/pdfminer.py
+++ b/haystack/components/converters/pdfminer.py
@@ -98,15 +98,15 @@ def __init__(  # pylint: disable=too-many-positional-arguments
         )
         self.store_full_path = store_full_path
 
-    def _converter(self, extractor) -> Document:
+    def _converter(self, extractor) -> str:
         """
-        Extracts text from PDF pages then convert the text into Documents
+        Extracts text from PDF pages then converts the text into a single str
 
         :param extractor:
             Python generator that yields PDF pages.
 
         :returns:
-            PDF text converted to Haystack Document
+            PDF text converted to single str
         """
         pages = []
         for page in extractor:
@@ -118,9 +118,9 @@ def _converter(self, extractor) -> Document:
             pages.append(text)
 
         # Add a page delimiter
-        concat = "\f".join(pages)
+        delimited_pages = "\f".join(pages)
 
-        return Document(content=concat)
+        return delimited_pages
 
     @component.output_types(documents=List[Document])
     def run(
@@ -157,14 +157,14 @@ def run(
                 continue
             try:
                 pdf_reader = extract_pages(io.BytesIO(bytestream.data), laparams=self.layout_params)
-                document = self._converter(pdf_reader)
+                text = self._converter(pdf_reader)
             except Exception as e:
                 logger.warning(
                     "Could not read {source} and convert it to Document, skipping. {error}", source=source, error=e
                 )
                 continue
 
-            if document.content is None or document.content.strip() == "":
+            if text is None or text.strip() == "":
                 logger.warning(
                     "PDFMinerToDocument could not extract text from the file {source}. Returning an empty document.",
                     source=source,
@@ -174,7 +174,7 @@ def run(
 
             if not self.store_full_path and (file_path := bytestream.meta.get("file_path")):
                 merged_metadata["file_path"] = os.path.basename(file_path)
-            document.meta = merged_metadata
+            document = Document(content=text, meta=merged_metadata)
             documents.append(document)
 
         return {"documents": documents}
diff --git a/haystack/components/embedders/azure_document_embedder.py b/haystack/components/embedders/azure_document_embedder.py
index b28fc1fda1..2223c82bf1 100644
--- a/haystack/components/embedders/azure_document_embedder.py
+++ b/haystack/components/embedders/azure_document_embedder.py
@@ -51,6 +51,8 @@ def __init__(  # noqa: PLR0913 (too-many-arguments) # pylint: disable=too-many-p
         embedding_separator: str = "\n",
         timeout: Optional[float] = None,
         max_retries: Optional[int] = None,
+        *,
+        default_headers: Optional[Dict[str, str]] = None,
     ):
         """
         Creates an AzureOpenAIDocumentEmbedder component.
@@ -95,6 +97,7 @@ def __init__(  # noqa: PLR0913 (too-many-arguments) # pylint: disable=too-many-p
             `OPENAI_TIMEOUT` environment variable, or 30 seconds.
         :param max_retries: Maximum number of retries to contact AzureOpenAI after an internal error.
             If not set, defaults to either the `OPENAI_MAX_RETRIES` environment variable or to 5 retries.
+        :param default_headers: Default headers to send to the AzureOpenAI client.
         """
         # if not provided as a parameter, azure_endpoint is read from the env var AZURE_OPENAI_ENDPOINT
         azure_endpoint = azure_endpoint or os.environ.get("AZURE_OPENAI_ENDPOINT")
@@ -119,6 +122,7 @@ def __init__(  # noqa: PLR0913 (too-many-arguments) # pylint: disable=too-many-p
         self.embedding_separator = embedding_separator
         self.timeout = timeout or float(os.environ.get("OPENAI_TIMEOUT", 30.0))
         self.max_retries = max_retries or int(os.environ.get("OPENAI_MAX_RETRIES", 5))
+        self.default_headers = default_headers or {}
 
         self._client = AzureOpenAI(
             api_version=api_version,
@@ -129,6 +133,7 @@ def __init__(  # noqa: PLR0913 (too-many-arguments) # pylint: disable=too-many-p
             organization=organization,
             timeout=self.timeout,
             max_retries=self.max_retries,
+            default_headers=self.default_headers,
         )
 
     def _get_telemetry_data(self) -> Dict[str, Any]:
@@ -161,6 +166,7 @@ def to_dict(self) -> Dict[str, Any]:
             azure_ad_token=self.azure_ad_token.to_dict() if self.azure_ad_token is not None else None,
             timeout=self.timeout,
             max_retries=self.max_retries,
+            default_headers=self.default_headers,
         )
 
     @classmethod
diff --git a/haystack/components/embedders/azure_text_embedder.py b/haystack/components/embedders/azure_text_embedder.py
index bef34d6c3f..c37bf2338d 100644
--- a/haystack/components/embedders/azure_text_embedder.py
+++ b/haystack/components/embedders/azure_text_embedder.py
@@ -46,6 +46,8 @@ def __init__(  # pylint: disable=too-many-positional-arguments
         max_retries: Optional[int] = None,
         prefix: str = "",
         suffix: str = "",
+        *,
+        default_headers: Optional[Dict[str, str]] = None,
     ):
         """
         Creates an AzureOpenAITextEmbedder component.
@@ -82,6 +84,7 @@ def __init__(  # pylint: disable=too-many-positional-arguments
             A string to add at the beginning of each text.
         :param suffix:
             A string to add at the end of each text.
+        :param default_headers: Default headers to send to the AzureOpenAI client.
         """
         # Why is this here?
         # AzureOpenAI init is forcing us to use an init method that takes either base_url or azure_endpoint as not
@@ -105,6 +108,7 @@ def __init__(  # pylint: disable=too-many-positional-arguments
         self.max_retries = max_retries or int(os.environ.get("OPENAI_MAX_RETRIES", 5))
         self.prefix = prefix
         self.suffix = suffix
+        self.default_headers = default_headers or {}
 
         self._client = AzureOpenAI(
             api_version=api_version,
@@ -115,6 +119,7 @@ def __init__(  # pylint: disable=too-many-positional-arguments
             organization=organization,
             timeout=self.timeout,
             max_retries=self.max_retries,
+            default_headers=self.default_headers,
         )
 
     def _get_telemetry_data(self) -> Dict[str, Any]:
@@ -143,6 +148,7 @@ def to_dict(self) -> Dict[str, Any]:
             azure_ad_token=self.azure_ad_token.to_dict() if self.azure_ad_token is not None else None,
             timeout=self.timeout,
             max_retries=self.max_retries,
+            default_headers=self.default_headers,
         )
 
     @classmethod
diff --git a/releasenotes/notes/add-default-headers-azure-embedders-6ffd24ec1502c5e4.yaml b/releasenotes/notes/add-default-headers-azure-embedders-6ffd24ec1502c5e4.yaml
new file mode 100644
index 0000000000..f8a7401c39
--- /dev/null
+++ b/releasenotes/notes/add-default-headers-azure-embedders-6ffd24ec1502c5e4.yaml
@@ -0,0 +1,4 @@
+---
+enhancements:
+  - |
+    Added `default_headers` parameter to `AzureOpenAIDocumentEmbedder` and `AzureOpenAITextEmbedder`.
diff --git a/releasenotes/notes/pdfminer-docid-b9f1b2f1b936b228.yaml b/releasenotes/notes/pdfminer-docid-b9f1b2f1b936b228.yaml
new file mode 100644
index 0000000000..2497c6048e
--- /dev/null
+++ b/releasenotes/notes/pdfminer-docid-b9f1b2f1b936b228.yaml
@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    PDFMinerToDocument now creates documents with id based on converted text and meta data. Before, PDFMinerToDocument did not consider the document's meta field when generating the document's id.


Test patch:
diff --git a/test/components/converters/test_pdfminer_to_document.py b/test/components/converters/test_pdfminer_to_document.py
index 4b30f2819a..92aeb2dcd1 100644
--- a/test/components/converters/test_pdfminer_to_document.py
+++ b/test/components/converters/test_pdfminer_to_document.py
@@ -5,6 +5,7 @@
 
 import pytest
 
+from haystack import Document
 from haystack.dataclasses import ByteStream
 from haystack.components.converters.pdfminer import PDFMinerToDocument
 
@@ -150,3 +151,7 @@ def test_run_empty_document(self, caplog, test_files_path):
             results = converter.run(sources=sources)
             assert "PDFMinerToDocument could not extract text from the file" in caplog.text
             assert results["documents"][0].content == ""
+
+            # Check that not only content is used when the returned document is initialized and doc id is generated
+            assert results["documents"][0].meta["file_path"] == "non_text_searchable.pdf"
+            assert results["documents"][0].id != Document(content="").id
diff --git a/test/components/embedders/test_azure_document_embedder.py b/test/components/embedders/test_azure_document_embedder.py
index 354f35a0fc..033ed36122 100644
--- a/test/components/embedders/test_azure_document_embedder.py
+++ b/test/components/embedders/test_azure_document_embedder.py
@@ -22,6 +22,7 @@ def test_init_default(self, monkeypatch):
         assert embedder.progress_bar is True
         assert embedder.meta_fields_to_embed == []
         assert embedder.embedding_separator == "\n"
+        assert embedder.default_headers == {}
 
     def test_to_dict(self, monkeypatch):
         monkeypatch.setenv("AZURE_OPENAI_API_KEY", "fake-api-key")
@@ -45,9 +46,43 @@ def test_to_dict(self, monkeypatch):
                 "embedding_separator": "\n",
                 "max_retries": 5,
                 "timeout": 30.0,
+                "default_headers": {},
             },
         }
 
+    def test_from_dict(self, monkeypatch):
+        monkeypatch.setenv("AZURE_OPENAI_API_KEY", "fake-api-key")
+        data = {
+            "type": "haystack.components.embedders.azure_document_embedder.AzureOpenAIDocumentEmbedder",
+            "init_parameters": {
+                "api_key": {"env_vars": ["AZURE_OPENAI_API_KEY"], "strict": False, "type": "env_var"},
+                "azure_ad_token": {"env_vars": ["AZURE_OPENAI_AD_TOKEN"], "strict": False, "type": "env_var"},
+                "api_version": "2023-05-15",
+                "azure_deployment": "text-embedding-ada-002",
+                "dimensions": None,
+                "azure_endpoint": "https://example-resource.azure.openai.com/",
+                "organization": None,
+                "prefix": "",
+                "suffix": "",
+                "batch_size": 32,
+                "progress_bar": True,
+                "meta_fields_to_embed": [],
+                "embedding_separator": "\n",
+                "max_retries": 5,
+                "timeout": 30.0,
+                "default_headers": {},
+            },
+        }
+        component = AzureOpenAIDocumentEmbedder.from_dict(data)
+        assert component.azure_deployment == "text-embedding-ada-002"
+        assert component.azure_endpoint == "https://example-resource.azure.openai.com/"
+        assert component.api_version == "2023-05-15"
+        assert component.max_retries == 5
+        assert component.timeout == 30.0
+        assert component.prefix == ""
+        assert component.suffix == ""
+        assert component.default_headers == {}
+
     @pytest.mark.integration
     @pytest.mark.skipif(
         not os.environ.get("AZURE_OPENAI_API_KEY", None) and not os.environ.get("AZURE_OPENAI_ENDPOINT", None),
diff --git a/test/components/embedders/test_azure_text_embedder.py b/test/components/embedders/test_azure_text_embedder.py
index 5f1f82e3d8..f4aab7edfe 100644
--- a/test/components/embedders/test_azure_text_embedder.py
+++ b/test/components/embedders/test_azure_text_embedder.py
@@ -19,6 +19,7 @@ def test_init_default(self, monkeypatch):
         assert embedder.organization is None
         assert embedder.prefix == ""
         assert embedder.suffix == ""
+        assert embedder.default_headers == {}
 
     def test_to_dict(self, monkeypatch):
         monkeypatch.setenv("AZURE_OPENAI_API_KEY", "fake-api-key")
@@ -38,9 +39,39 @@ def test_to_dict(self, monkeypatch):
                 "timeout": 30.0,
                 "prefix": "",
                 "suffix": "",
+                "default_headers": {},
             },
         }
 
+    def test_from_dict(self, monkeypatch):
+        monkeypatch.setenv("AZURE_OPENAI_API_KEY", "fake-api-key")
+        data = {
+            "type": "haystack.components.embedders.azure_text_embedder.AzureOpenAITextEmbedder",
+            "init_parameters": {
+                "api_key": {"env_vars": ["AZURE_OPENAI_API_KEY"], "strict": False, "type": "env_var"},
+                "azure_ad_token": {"env_vars": ["AZURE_OPENAI_AD_TOKEN"], "strict": False, "type": "env_var"},
+                "azure_deployment": "text-embedding-ada-002",
+                "dimensions": None,
+                "organization": None,
+                "azure_endpoint": "https://example-resource.azure.openai.com/",
+                "api_version": "2023-05-15",
+                "max_retries": 5,
+                "timeout": 30.0,
+                "prefix": "",
+                "suffix": "",
+                "default_headers": {},
+            },
+        }
+        component = AzureOpenAITextEmbedder.from_dict(data)
+        assert component.azure_deployment == "text-embedding-ada-002"
+        assert component.azure_endpoint == "https://example-resource.azure.openai.com/"
+        assert component.api_version == "2023-05-15"
+        assert component.max_retries == 5
+        assert component.timeout == 30.0
+        assert component.prefix == ""
+        assert component.suffix == ""
+        assert component.default_headers == {}
+
     @pytest.mark.integration
     @pytest.mark.skipif(
         not os.environ.get("AZURE_OPENAI_API_KEY", None) and not os.environ.get("AZURE_OPENAI_ENDPOINT", None),