Repository: deepset-ai/haystack

Issue:
Improve PDF converter performance and simplify installation
**Is your feature request related to a problem? Please describe.**
As a user, I want to use the PDF converter without extra installation steps. And I would like the performance to be reasonable, specially when processing large PDF files.

**Describe the solution you'd like**
Implement a new PDF converter base class, which uses more performant code and simplify installation process.

**Describe alternatives you've considered**
- xpdf
- PyPDF2
- PDFMiner
- Tika

**Additional context**




Discussion:

Resolving patch:
diff --git a/haystack/nodes/file_converter/__init__.py b/haystack/nodes/file_converter/__init__.py
--- a/haystack/nodes/file_converter/__init__.py
+++ b/haystack/nodes/file_converter/__init__.py
@@ -9,7 +9,7 @@
 from haystack.nodes.file_converter.txt import TextConverter
 from haystack.nodes.file_converter.azure import AzureConverter
 from haystack.nodes.file_converter.parsr import ParsrConverter
-from haystack.nodes.file_converter.pdf import PDFToTextConverter
+
 
 MarkdownConverter = safe_import(
     "haystack.nodes.file_converter.markdown", "MarkdownConverter", "preprocessing"
@@ -20,3 +20,6 @@
 PDFToTextOCRConverter = safe_import(
     "haystack.nodes.file_converter.pdf_ocr", "PDFToTextOCRConverter", "ocr"
 )  # Has optional dependencies
+PDFToTextConverter = safe_import(
+    "haystack.nodes.file_converter.pdf", "PDFToTextConverter", "pdf"
+)  # Has optional dependencies
diff --git a/haystack/nodes/file_converter/pdf.py b/haystack/nodes/file_converter/pdf.py
--- a/haystack/nodes/file_converter/pdf.py
+++ b/haystack/nodes/file_converter/pdf.py
@@ -1,7 +1,18 @@
 import logging
-import subprocess
+import warnings
+from concurrent.futures import ProcessPoolExecutor
+from multiprocessing import cpu_count
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
+
+from more_itertools import divide
+
+try:
+    import fitz
+except (ImportError, ModuleNotFoundError) as ie:
+    from haystack.utils.import_utils import _optional_component_not_installed
+
+    _optional_component_not_installed(__name__, "pdf", ie)
 
 from haystack.nodes.file_converter.base import BaseConverter
 from haystack.schema import Document
@@ -15,9 +26,11 @@ def __init__(
         remove_numeric_tables: bool = False,
         valid_languages: Optional[List[str]] = None,
         id_hash_keys: Optional[List[str]] = None,
-        encoding: Optional[str] = "UTF-8",
-        keep_physical_layout: bool = False,
-    ):
+        encoding: Optional[str] = None,
+        keep_physical_layout: Optional[bool] = None,
+        sort_by_position: bool = False,
+        multiprocessing: Union[bool, int] = True,
+    ) -> None:
         """
         :param remove_numeric_tables: This option uses heuristics to remove numeric rows from the tables.
                                       The tabular structures in documents might be noise for the reader model if it
@@ -33,36 +46,30 @@ def __init__(
             attributes. If you want to ensure you don't have duplicate documents in your DocumentStore but texts are
             not unique, you can modify the metadata and pass e.g. `"meta"` to this field (e.g. [`"content"`, `"meta"`]).
             In this case the id will be generated by using the content and the defined metadata.
-        :param encoding: Encoding that will be passed as `-enc` parameter to `pdftotext`.
-                         Defaults to "UTF-8" in order to support special characters (e.g. German Umlauts, Cyrillic ...).
-                         (See list of available encodings, such as "Latin1", by running `pdftotext -listenc` in the terminal)
-        :param keep_physical_layout: This option will maintain original physical layout on the extracted text.
-            It works by passing the `-layout` parameter to `pdftotext`. When disabled, PDF is read in the stream order.
+        :param encoding: This parameter is being deprecated.
+                         It will be automatically detected by PyMuPDF.
+        :param keep_physical_layout: This parameter is being deprecated.
+        :param sort_by_position: Specifies whether to sort the extracted text by positional coordinates or logical reading order.
+                        If set to True, the text is sorted first by vertical position, and then by horizontal position.
+                        If set to False (default), the logical reading order in the PDF is used.
+        :param multiprocessing: We use multiprocessing to speed up PyMuPDF conversion, you can disable it by setting it to False.
+                                If set to True (the default value), the total number of cores is used. To specify the number of cores to use, set it to an integer.
         """
         super().__init__(
             remove_numeric_tables=remove_numeric_tables, valid_languages=valid_languages, id_hash_keys=id_hash_keys
         )
-        try:
-            subprocess.run(
-                ["pdftotext", "-v"], shell=False, check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
-            )
-        except FileNotFoundError:
-            raise FileNotFoundError(
-                """pdftotext is not installed. It is part of xpdf or poppler-utils software suite.
 
-                   Installation on Linux:
-                   wget --no-check-certificate https://dl.xpdfreader.com/xpdf-tools-linux-4.04.tar.gz &&
-                   tar -xvf xpdf-tools-linux-4.04.tar.gz && sudo cp xpdf-tools-linux-4.04/bin64/pdftotext /usr/local/bin
+        self.sort_by_position = sort_by_position
+        self.multiprocessing = multiprocessing
 
-                   Installation on MacOS:
-                   brew install xpdf
-
-                   You can find more details here: https://www.xpdfreader.com
-                """
+        if encoding:
+            warnings.warn(
+                "The encoding parameter is being deprecated. It will be automatically detected by PyMuPDF.",
+                DeprecationWarning,
             )
 
-        self.encoding = encoding
-        self.keep_physical_layout = keep_physical_layout
+        if keep_physical_layout:
+            warnings.warn("The keep_physical_layout parameter is being deprecated.", DeprecationWarning)
 
     def convert(
         self,
@@ -74,10 +81,12 @@ def convert(
         id_hash_keys: Optional[List[str]] = None,
         start_page: Optional[int] = None,
         end_page: Optional[int] = None,
+        keep_physical_layout: Optional[bool] = None,
+        sort_by_position: Optional[bool] = None,
+        multiprocessing: Optional[Union[bool, int]] = None,
     ) -> List[Document]:
         """
-        Extract text from a .pdf file using the pdftotext library (https://www.xpdfreader.com/pdftotext-man.html)
-
+        Extract text from a PDF file and convert it to a Document.
         :param file_path: Path to the .pdf file you want to convert
         :param meta: Optional dictionary with metadata that shall be attached to all resulting documents.
                      Can be any custom keys and values.
@@ -91,16 +100,21 @@ def convert(
                                 This option can be used to add test for encoding errors. If the extracted text is
                                 not one of the valid languages, then it might likely be encoding error resulting
                                 in garbled text.
-        :param encoding: Encoding that overwrites self.encoding and will be passed as `-enc` parameter to `pdftotext`.
-                         (See list of available encodings by running `pdftotext -listenc` in the terminal)
-        :param keep_physical_layout: This option will maintain original physical layout on the extracted text.
-            It works by passing the `-layout` parameter to `pdftotext`. When disabled, PDF is read in the stream order.
+        :param encoding: This parameter is being deprecated.
+                         It will be automatically detected by PyMuPDF.
+        :param keep_physical_layout: This parameter is being deprecated.
+        :param sort_by_position: Specifies whether to sort the extracted text by positional coordinates or logical reading order.
+                        If set to True, the text is sorted first by vertical position, and then by horizontal position.
+                        If set to False (default), the logical reading order in the PDF is used.
         :param id_hash_keys: Generate the document id from a custom list of strings that refer to the document's
             attributes. If you want to ensure you don't have duplicate documents in your DocumentStore but texts are
             not unique, you can modify the metadata and pass e.g. `"meta"` to this field (e.g. [`"content"`, `"meta"`]).
             In this case the id will be generated by using the content and the defined metadata.
         :param start_page: The page number where to start the conversion
         :param end_page: The page number where to end the conversion.
+        :param multiprocessing: We use multiprocessing to speed up PyMuPDF conversion, you can disable it by setting it to False.
+                                If set to None (the default value), the value defined in the class initialization is used.
+                                If set to True, the total number of cores is used. To specify the number of cores to use, set it to an integer.
         """
         if remove_numeric_tables is None:
             remove_numeric_tables = self.remove_numeric_tables
@@ -108,25 +122,30 @@ def convert(
             valid_languages = self.valid_languages
         if id_hash_keys is None:
             id_hash_keys = self.id_hash_keys
+        if multiprocessing is None:
+            multiprocessing = self.multiprocessing
+        if sort_by_position is None:
+            sort_by_position = self.sort_by_position
+
+        if encoding:
+            warnings.warn(
+                "The encoding parameter is being deprecated. It will be automatically detected by PyMuPDF.",
+                DeprecationWarning,
+            )
 
-        keep_physical_layout = self.keep_physical_layout
+        if keep_physical_layout:
+            warnings.warn("The keep_physical_layout parameter is being deprecated.", DeprecationWarning)
 
         pages = self._read_pdf(
-            file_path, layout=keep_physical_layout, encoding=encoding, start_page=start_page, end_page=end_page
+            file_path,
+            sort_by_position=sort_by_position,
+            start_page=start_page,
+            end_page=end_page,
+            multiprocessing=multiprocessing,
         )
 
         cleaned_pages = []
         for page in pages:
-            # pdftotext tool provides an option to retain the original physical layout of a PDF page. This behaviour
-            # can be toggled by using the layout param.
-            #  layout=True
-            #      + table structures get retained better
-            #      - multi-column pages(eg, research papers) gets extracted with text from multiple columns on same line
-            #  layout=False
-            #      + keeps strings in content stream order, hence multi column layout works well
-            #      - cells of tables gets split across line
-            #
-            #  Here, as a "safe" default, layout is turned off.
             lines = page.splitlines()
             cleaned_lines = []
             for line in lines:
@@ -157,40 +176,71 @@ def convert(
         document = Document(content=text, meta=meta, id_hash_keys=id_hash_keys)
         return [document]
 
+    def _get_text_parallel(self, page_mp):
+        idx, filename, parts, sort_by_position = page_mp
+
+        doc = fitz.open(filename)
+
+        text = ""
+        for i in parts[idx]:
+            page = doc[i]
+            text += page.get_text("text", sort=sort_by_position) + "\f"
+
+        return text
+
     def _read_pdf(
         self,
         file_path: Path,
-        layout: bool,
-        encoding: Optional[str] = None,
+        sort_by_position: bool = False,
         start_page: Optional[int] = None,
         end_page: Optional[int] = None,
+        multiprocessing: Optional[Union[bool, int]] = None,
     ) -> List[str]:
         """
         Extract pages from the pdf file at file_path.
 
         :param file_path: path of the pdf file
-        :param layout: whether to retain the original physical layout for a page. If disabled, PDF pages are read in
-                       the content stream order.
-        :param encoding: Encoding that overwrites self.encoding and will be passed as `-enc` parameter to `pdftotext`.
-                         (See list of available encodings by running `pdftotext -listenc` in the terminal)
-        :param start_page: The page number where to start the conversion
+        :param sort_by_position: Specifies whether to sort the extracted text by positional coordinates or logical reading order.
+                        If set to True, the text is sorted first by vertical position, and then by horizontal position.
+                        If set to False (default), the logical reading order in the PDF is used.
+        :param start_page: The page number where to start the conversion, starting from 1.
         :param end_page: The page number where to end the conversion.
+        :param encoding: This parameter is being deprecated.
+                         It will be automatically detected by PyMuPDF.
+        :param multiprocessing: We use multiprocessing to speed up PyMuPDF conversion, you can disable it by setting it to False.
+                                If set to None (the default value), the value defined in the class initialization is used.
+                                If set to True, the total number of cores is used. To specify the number of cores to use, set it to an integer.
         """
-        if not encoding:
-            encoding = self.encoding
-
-        start_page = start_page or 1
-
-        command = ["pdftotext", "-enc", str(encoding), "-layout" if layout else "-raw", "-f", str(start_page)]
-
-        if end_page is not None:
-            command.extend(["-l", str(end_page)])
-
-        command.extend([str(file_path), "-"])
-
-        output = subprocess.run(command, stdout=subprocess.PIPE, shell=False, check=False)
-        document = output.stdout.decode(errors="ignore")
-        document = "\f" * (start_page - 1) + document  # tracking skipped pages for correct page numbering
+        if start_page is None:
+            start_page = 0
+        else:
+            start_page = start_page - 1
+
+        doc = fitz.open(file_path)
+        page_count = int(doc.page_count)
+
+        if end_page is None or (end_page is not None and end_page > page_count):
+            end_page = page_count
+
+        document = ""
+
+        if not multiprocessing:
+            for i in range(start_page, end_page):
+                page = doc[i]
+                document += page.get_text("text", sort=sort_by_position) + "\f"
+        else:
+            cpu = cpu_count() if isinstance(multiprocessing, bool) else multiprocessing
+            page_list = [i for i in range(start_page, end_page)]
+            cpu = cpu if len(page_list) > cpu else len(page_list)
+            parts = divide(cpu, page_list)
+            pages_mp = [(i, file_path, parts, sort_by_position) for i in range(cpu)]
+
+            with ProcessPoolExecutor(max_workers=cpu) as pool:
+                results = pool.map(self._get_text_parallel, pages_mp)
+                for page in results:
+                    document += page
+
+        document = "\f" * start_page + document  # tracking skipped pages for correct page numbering
         pages = document.split("\f")
         pages = pages[:-1]  # the last page in the split is always empty.
 


Test patch:
diff --git a/test/nodes/test_file_converter.py b/test/nodes/test_file_converter.py
--- a/test/nodes/test_file_converter.py
+++ b/test/nodes/test_file_converter.py
@@ -25,7 +25,7 @@
     PreProcessor,
 )
 
-from ..conftest import SAMPLES_PATH
+from ..conftest import SAMPLES_PATH, fail_at_version
 
 
 @pytest.mark.tika
@@ -65,6 +65,7 @@ def test_pdftoppm_command_format():
     ), 'Your installation of poppler is incompatible with Haystack. Try installing via "conda install -c conda-forge poppler"'
 
 
+@pytest.mark.unit
 @pytest.mark.parametrize("Converter", [PDFToTextConverter])
 def test_pdf_command_whitespaces(Converter):
     converter = Converter()
@@ -75,25 +76,28 @@ def test_pdf_command_whitespaces(Converter):
     assert "ɪ" in document.content
 
 
+@pytest.mark.unit
 @pytest.mark.parametrize("Converter", [PDFToTextConverter])
 def test_pdf_encoding(Converter):
     converter = Converter()
 
+    document = converter.run(file_paths=SAMPLES_PATH / "pdf" / "sample_pdf_5.pdf")[0]["documents"][0]
+    assert "Ж" in document.content
+
     document = converter.run(file_paths=SAMPLES_PATH / "pdf" / "sample_pdf_2.pdf")[0]["documents"][0]
     assert "ɪ" in document.content
 
-    document = converter.run(file_paths=SAMPLES_PATH / "pdf" / "sample_pdf_2.pdf", encoding="Latin1")[0]["documents"][0]
-    assert "ɪ" not in document.content
-
 
+@pytest.mark.unit
 @pytest.mark.parametrize("Converter", [PDFToTextConverter])
-def test_pdf_layout(Converter):
-    converter = Converter(keep_physical_layout=True)
+def test_pdf_sort_by_position(Converter):
+    converter = Converter(sort_by_position=True)
 
     document = converter.convert(file_path=SAMPLES_PATH / "pdf" / "sample_pdf_3.pdf")[0]
     assert str(document.content).startswith("This is the second test sentence.")
 
 
+@pytest.mark.unit
 @pytest.mark.parametrize("Converter", [PDFToTextConverter])
 def test_pdf_ligatures(Converter):
     converter = Converter()
@@ -115,8 +119,9 @@ def test_pdf_ligatures(Converter):
     assert "ɪ" not in document.content
 
 
+@pytest.mark.unit
 @pytest.mark.parametrize("Converter", [PDFToTextConverter])
-def test_page_range(Converter):
+def test_pdf_page_range(Converter):
     converter = Converter()
     document = converter.convert(file_path=SAMPLES_PATH / "pdf" / "sample_pdf_1.pdf", start_page=2)[0]
     pages = document.content.split("\f")
@@ -129,8 +134,9 @@ def test_page_range(Converter):
     assert pages[2] == ""  # the page 3 is empty.
 
 
+@pytest.mark.unit
 @pytest.mark.parametrize("Converter", [PDFToTextConverter])
-def test_page_range_numbers(Converter):
+def test_pdf_page_range_numbers(Converter):
     converter = Converter()
     document = converter.convert(file_path=SAMPLES_PATH / "pdf" / "sample_pdf_1.pdf", start_page=2)[0]
 
@@ -142,6 +148,68 @@ def test_page_range_numbers(Converter):
     assert documents[1].meta["page"] == 4
 
 
+@pytest.mark.unit
+@pytest.mark.parametrize("Converter", [PDFToTextConverter])
+def test_pdf_parallel(Converter):
+    converter = Converter(multiprocessing=True)
+    document = converter.convert(file_path=SAMPLES_PATH / "pdf" / "sample_pdf_6.pdf")[0]
+
+    pages = document.content.split("\f")
+
+    assert pages[0] == "This is the page 1 of the document."
+    assert pages[-1] == "This is the page 50 of the document."
+
+
+@pytest.mark.unit
+@pytest.mark.parametrize("Converter", [PDFToTextConverter])
+def test_pdf_parallel_page_range(Converter):
+    converter = Converter(multiprocessing=True)
+    document = converter.convert(file_path=SAMPLES_PATH / "pdf" / "sample_pdf_6.pdf", start_page=2)[0]
+
+    pages = document.content.split("\f")
+
+    assert pages[0] == ""
+    assert len(pages) == 50
+
+
+@pytest.mark.unit
+@pytest.mark.parametrize("Converter", [PDFToTextConverter])
+def test_pdf_parallel_sort_by_position(Converter):
+    converter = Converter(multiprocessing=True, sort_by_position=True)
+    document = converter.convert(file_path=SAMPLES_PATH / "pdf" / "sample_pdf_6.pdf")[0]
+
+    pages = document.content.split("\f")
+
+    assert pages[0] == "This is the page 1 of the document."
+    assert pages[-1] == "This is the page 50 of the document."
+
+
+@fail_at_version(1, 17)
+def test_deprecated_encoding():
+    with pytest.warns(DeprecationWarning):
+        converter = PDFToTextConverter(encoding="utf-8")
+
+
+@fail_at_version(1, 17)
+def test_deprecated_encoding_in_convert_method():
+    converter = PDFToTextConverter()
+    with pytest.warns(DeprecationWarning):
+        converter.convert(file_path=SAMPLES_PATH / "pdf" / "sample_pdf_1.pdf", encoding="utf-8")
+
+
+@fail_at_version(1, 17)
+def test_deprecated_keep_physical_layout():
+    with pytest.warns(DeprecationWarning):
+        converter = PDFToTextConverter(keep_physical_layout=True)
+
+
+@fail_at_version(1, 17)
+def test_deprecated_keep_physical_layout_in_convert_method():
+    converter = PDFToTextConverter()
+    with pytest.warns(DeprecationWarning):
+        converter.convert(file_path=SAMPLES_PATH / "pdf" / "sample_pdf_1.pdf", keep_physical_layout=True)
+
+
 @pytest.mark.tika
 @pytest.mark.parametrize("Converter", [PDFToTextConverter, TikaConverter])
 def test_table_removal(Converter):
diff --git a/test/samples/pdf/sample_pdf_5.pdf b/test/samples/pdf/sample_pdf_5.pdf
new file mode 100644
Binary files /dev/null and b/test/samples/pdf/sample_pdf_5.pdf differ
diff --git a/test/samples/pdf/sample_pdf_6.pdf b/test/samples/pdf/sample_pdf_6.pdf
new file mode 100644
Binary files /dev/null and b/test/samples/pdf/sample_pdf_6.pdf differ