Repository: deepset-ai/haystack

Issue:
Crawler should accept a webdriver instead of configuring it internally
The automatic installation of Chrome webdriver that's invoked as a part of the `Crawler.__init__` fails on my system and I have found no way of influencing it to install the correct chrome driver.

At the moment `Crawler` configures its selenium webdriver in the constructor based on the provided `webdriver_options` depending on the environment it is in. However it will always default to the Chrome webdriver, which might not be available to everyone and there is no clean way around that right now.

**Describe the solution you'd like**
I'd like the `Crawler` constructor to accept an argument like `webdriver: Optional[WebDriver] = None` and use this driver instead of setting up its own if provided. This is generally a better design and doesn't just solve my problem, but can be useful to anyone wanting to configure the webdriver further than what's possible with the `webdriver_options`.

**Describe alternatives you've considered**
I tried to get the webdriver installed locally, but without success, because https://github.com/deepset-ai/haystack/blob/5f013918272402934a1d5ccd85c14370dc4f0f3d/haystack/nodes/connector/crawler.py#L164 this line will always try to install it from scratch.

**Additional context**
I can make the PR if it helps, its a trivial change!



Discussion:
Hey, @mirosval!

Thanks for reporting this issue...
 
Your idea sounds good to me, so feel free to open a PR!

If you need help with the process, you can refer to the [guidelines for contributors](https://github.com/deepset-ai/haystack/blob/main/CONTRIBUTING.md).
I had this same issue on Ubuntu running under WSL2. The workaround I found was to downgrade google-chrome-stable to 114.0.5735.90, and then selenium's webdriver manager could download the correct chromedriver, and the Crawler worked again. AFAIK 114.0.5735.90 is the last version that there is a chromedriver for in the list that the manager checks; everything for 115 is being tracked elsewhere.

Resolving patch:
diff --git a/haystack/nodes/connector/crawler.py b/haystack/nodes/connector/crawler.py
--- a/haystack/nodes/connector/crawler.py
+++ b/haystack/nodes/connector/crawler.py
@@ -6,15 +6,16 @@
 import sys
 import time
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional, Tuple, Union, Set
+from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
 from urllib.parse import urlparse
 
+from haystack.lazy_imports import LazyImport
 from haystack.nodes.base import BaseComponent
 from haystack.schema import Document
-from haystack.lazy_imports import LazyImport
 
 with LazyImport("Run 'pip install farm-haystack[crawler]'") as selenium_import:
-    from selenium import webdriver
+    from selenium import webdriver as selenium_webdriver
+    from selenium.webdriver.remote.webdriver import WebDriver
     from selenium.common.exceptions import StaleElementReferenceException
     from selenium.webdriver.chrome.options import Options
     from selenium.webdriver.chrome.service import Service
@@ -53,6 +54,7 @@ def __init__(
         file_path_meta_field_name: Optional[str] = None,
         crawler_naming_function: Optional[Callable[[str, str], str]] = None,
         webdriver_options: Optional[List[str]] = None,
+        webdriver: Optional["WebDriver"] = None,
     ):
         """
         Init object with basic params for crawling (can be overwritten later).
@@ -96,10 +98,27 @@ def __init__(
                     This option enables remote debug over HTTP.
             See [Chromium Command Line Switches](https://peter.sh/experiments/chromium-command-line-switches/) for more details on the available options.
             If your crawler fails, raising a `selenium.WebDriverException`, this [Stack Overflow thread](https://stackoverflow.com/questions/50642308/webdriverexception-unknown-error-devtoolsactiveport-file-doesnt-exist-while-t) can be helpful. Contains useful suggestions for webdriver_options.
+        :param webdriver: A pre-configured Selenium WebDriver.
+            When webdriver_options is not sufficient, use this parameter to override the whole web driver. This lets you use different engines other than the default Chrome.
         """
         selenium_import.check()
         super().__init__()
 
+        self.urls = urls
+        self.crawler_depth = crawler_depth
+        self.filter_urls = filter_urls
+        self.overwrite_existing_files = overwrite_existing_files
+        self.id_hash_keys = id_hash_keys
+        self.extract_hidden_text = extract_hidden_text
+        self.loading_wait_time = loading_wait_time
+        self.crawler_naming_function = crawler_naming_function
+        self.output_dir = output_dir
+        self.file_path_meta_field_name = file_path_meta_field_name
+
+        if webdriver is not None:
+            self.driver = webdriver
+            return
+
         IN_COLAB = "google.colab" in sys.modules
         IN_AZUREML = os.environ.get("AZUREML_ENVIRONMENT_IMAGE", None) == "True"
         IN_WINDOWS = sys.platform in ["win32", "cygwin"]
@@ -119,17 +138,7 @@ def __init__(
         for option in set(webdriver_options):
             options.add_argument(option)
 
-        self.driver = webdriver.Chrome(service=Service(), options=options)
-        self.urls = urls
-        self.crawler_depth = crawler_depth
-        self.filter_urls = filter_urls
-        self.overwrite_existing_files = overwrite_existing_files
-        self.id_hash_keys = id_hash_keys
-        self.extract_hidden_text = extract_hidden_text
-        self.loading_wait_time = loading_wait_time
-        self.crawler_naming_function = crawler_naming_function
-        self.output_dir = output_dir
-        self.file_path_meta_field_name = file_path_meta_field_name
+        self.driver = selenium_webdriver.Chrome(service=Service(), options=options)
 
     def __del__(self):
         self.driver.quit()
@@ -337,6 +346,7 @@ def _crawl_urls(
             if loading_wait_time is not None:
                 time.sleep(loading_wait_time)
             el = self.driver.find_element(by=By.TAG_NAME, value="body")
+
             if extract_hidden_text:
                 text = el.get_attribute("textContent")
             else:
@@ -469,9 +479,11 @@ def _extract_sublinks_from_url(
         loading_wait_time: Optional[int] = None,
     ) -> Set[str]:
         self.driver.get(base_url)
+
         if loading_wait_time is not None:
             time.sleep(loading_wait_time)
         a_elements = self.driver.find_elements(by=By.XPATH, value="//a[@href]")
+
         sub_links = set()
 
         filter_pattern = re.compile("|".join(filter_urls)) if filter_urls is not None else None


Test patch:
diff --git a/test/nodes/test_connector.py b/test/nodes/test_connector.py
--- a/test/nodes/test_connector.py
+++ b/test/nodes/test_connector.py
@@ -1,14 +1,13 @@
-from typing import List
-
-import json
-from pathlib import Path
-import re
 import hashlib
+import json
 import os
-from unittest.mock import patch
+import re
+from pathlib import Path
+from typing import List
+from unittest.mock import Mock, patch
 
 import pytest
-
+from selenium.webdriver import Chrome
 from selenium.webdriver.common.by import By
 
 from haystack.nodes.connector.crawler import Crawler
@@ -54,8 +53,9 @@ def content_in_results(crawler: Crawler, url: str, results: List[Path], expected
 
 
 @pytest.mark.unit
-@patch("haystack.nodes.connector.crawler.webdriver")
-def test_crawler_url_none_exception(webdriver):
+@patch("haystack.nodes.connector.crawler.Service")
+@patch("haystack.nodes.connector.crawler.selenium_webdriver")
+def test_crawler_url_none_exception(service, webdriver):
     crawler = Crawler()
     with pytest.raises(ValueError):
         crawler.crawl()
@@ -258,3 +258,11 @@ def test_crawler_depth_2_multiple_urls(test_url, tmp_path):
     assert content_in_results(crawler, test_url + "/page1_subpage1.html", paths)
     assert content_in_results(crawler, test_url + "/page1_subpage2.html", paths)
     assert content_in_results(crawler, test_url + "/page2_subpage1.html", paths)
+
+
+@pytest.mark.unit
+def test_crawler_custom_webdriver():
+    webdriver = Mock(Chrome)
+    crawler = Crawler(webdriver=webdriver)
+
+    assert webdriver is crawler.driver