Repository: deepset-ai/haystack

Issue:
`HTMLToDocument` sometimes skips some relevant text
**Describe the bug**
`HTMLToDocument` uses `ArticleExtractor` from [boilerpy3](https://github.com/jmriebold/BoilerPy3).
This extractor is tuned towards news articles and sometimes skips some relevant text.

**To Reproduce**
```python
from haystack.components.fetchers import LinkContentFetcher
from haystack.components.converters import HTMLToDocument

lc = LinkContentFetcher()
hd = HTMLToDocument()

out=lc.run(urls=["https://www.rainforest-alliance.org/species/capybara/"])

print('largest rodents' in str(out['streams'][0]))
# True

doc=hd.run(sources=out['streams'])['documents'][0]

print('largest rodents' in doc.content)
# False
```

**Expected behavior**
- I would like to configure the best [boilerpy3 extractor](https://github.com/jmriebold/BoilerPy3?tab=readme-ov-file#extractors) for my use case
- The default extractor should probably be more generic (e.g., `DefaultExtractor`)

**System:**
 - Haystack version (commit or version number): 2.0.0b3


Discussion:

Resolving patch:
diff --git a/haystack/components/converters/html.py b/haystack/components/converters/html.py
--- a/haystack/components/converters/html.py
+++ b/haystack/components/converters/html.py
@@ -1,6 +1,6 @@
 import logging
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union, Literal
 from boilerpy3 import extractors
 
 from haystack import Document, component
@@ -28,6 +28,27 @@ class HTMLToDocument:
 
     """
 
+    def __init__(
+        self,
+        extractor_type: Literal[
+            "DefaultExtractor",
+            "ArticleExtractor",
+            "ArticleSentencesExtractor",
+            "LargestContentExtractor",
+            "CanolaExtractor",
+            "KeepEverythingExtractor",
+            "NumWordsRulesExtractor",
+        ] = "DefaultExtractor",
+    ):
+        """
+        Create an HTMLToDocument component.
+
+        :param extractor_type: The type of boilerpy3 extractor to use. Defaults to `DefaultExtractor`.
+          For more information on the different types of extractors,
+          see [boilerpy3 documentation](https://github.com/jmriebold/BoilerPy3?tab=readme-ov-file#extractors).
+        """
+        self.extractor_type = extractor_type
+
     @component.output_types(documents=List[Document])
     def run(self, sources: List[Union[str, Path, ByteStream]], meta: Optional[List[Dict[str, Any]]] = None):
         """
@@ -46,7 +67,8 @@ def run(self, sources: List[Union[str, Path, ByteStream]], meta: Optional[List[D
         elif len(sources) != len(meta):
             raise ValueError("The length of the metadata list must match the number of sources.")
 
-        extractor = extractors.ArticleExtractor(raise_on_failure=False)
+        extractor_class = getattr(extractors, self.extractor_type)
+        extractor = extractor_class(raise_on_failure=False)
 
         for source, metadata in zip(sources, meta):
             try:


Test patch:
diff --git a/test/components/converters/test_html_to_document.py b/test/components/converters/test_html_to_document.py
--- a/test/components/converters/test_html_to_document.py
+++ b/test/components/converters/test_html_to_document.py
@@ -18,6 +18,20 @@ def test_run(self, test_files_path):
         assert len(docs) == 1
         assert "Haystack" in docs[0].content
 
+    def test_run_different_extractors(self, test_files_path):
+        """
+        Test if the component runs correctly with different boilrepy3 extractors.
+        """
+        sources = [test_files_path / "html" / "what_is_haystack.html"]
+
+        converter_article = HTMLToDocument(extractor_type="ArticleExtractor")
+        converter_keep_everything = HTMLToDocument(extractor_type="KeepEverythingExtractor")
+
+        doc_article = converter_article.run(sources=sources)["documents"][0]
+        doc_keep_everything = converter_keep_everything.run(sources=sources)["documents"][0]
+
+        assert len(doc_keep_everything.content) > len(doc_article.content)
+
     def test_run_doc_metadata(self, test_files_path):
         """
         Test if the component runs correctly when metadata is supplied by the user.