Repository: deepset-ai/haystack

Issue:
Crawler does not work with selenium>=4.11.0
**Describe the bug**
Due to recent changes in selenium, the Chromium driver can't be found though it is properly installed.
Related issue: https://github.com/SeleniumHQ/selenium/issues/12466

**Error message**
> NoSuchDriverException                     Traceback (most recent call last)
> [/usr/local/lib/python3.10/dist-packages/haystack/nodes/connector/crawler.py](https://localhost:8080/#) in __init__(self, urls, crawler_depth, filter_urls, id_hash_keys, extract_hidden_text, loading_wait_time, output_dir, overwrite_existing_files, file_path_meta_field_name, crawler_naming_function, webdriver_options)
>     122             try:
> --> 123                 self.driver = webdriver.Chrome(service=Service("chromedriver"), options=options)
>     124             except WebDriverException as exc:
> 
> 5 frames
> NoSuchDriverException: Message: Unable to locate or obtain driver for chrome; For documentation on this error, please visit: https://www.selenium.dev/documentation/webdriver/troubleshooting/errors/driver_location
> 
> 
> The above exception was the direct cause of the following exception:
> 
> NodeError                                 Traceback (most recent call last)
> [/usr/local/lib/python3.10/dist-packages/haystack/nodes/connector/crawler.py](https://localhost:8080/#) in __init__(self, urls, crawler_depth, filter_urls, id_hash_keys, extract_hidden_text, loading_wait_time, output_dir, overwrite_existing_files, file_path_meta_field_name, crawler_naming_function, webdriver_options)
>     123                 self.driver = webdriver.Chrome(service=Service("chromedriver"), options=options)
>     124             except WebDriverException as exc:
> --> 125                 raise NodeError(
>     126                     """
>     127         \'chromium-driver\' needs to be installed manually when running colab. Follow the below given commands:
> 
> NodeError: 
>         'chromium-driver' needs to be installed manually when running colab. Follow the below given commands: ...

**Expected behavior**
The crawler should find the driver and properly work.

**Additional context**
As a workaround, selenium can be downgraded: `pip install "selenium<4.11.0"`
**We should support the new version and make it work.** (It doesn't seem like a difficult change...)

**To Reproduce**
```python
from haystack.nodes import Crawler

crawler = Crawler(output_dir="crawled_files")
```

**System:**
 - OS: Ubuntu 22.04
 - Haystack version (commit or version number): 1.19.0



Discussion:

Resolving patch:
diff --git a/haystack/nodes/connector/crawler.py b/haystack/nodes/connector/crawler.py
--- a/haystack/nodes/connector/crawler.py
+++ b/haystack/nodes/connector/crawler.py
@@ -9,19 +9,16 @@
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union, Set
 from urllib.parse import urlparse
 
-from haystack.errors import NodeError
 from haystack.nodes.base import BaseComponent
 from haystack.schema import Document
 from haystack.lazy_imports import LazyImport
 
 with LazyImport("Run 'pip install farm-haystack[crawler]'") as selenium_import:
     from selenium import webdriver
-    from selenium.common.exceptions import StaleElementReferenceException, WebDriverException
+    from selenium.common.exceptions import StaleElementReferenceException
     from selenium.webdriver.chrome.options import Options
     from selenium.webdriver.chrome.service import Service
     from selenium.webdriver.common.by import By
-    from webdriver_manager.chrome import ChromeDriverManager
-
 
 logger = logging.getLogger(__name__)
 
@@ -109,8 +106,10 @@ def __init__(
         if webdriver_options is None:
             webdriver_options = ["--headless", "--disable-gpu", "--disable-dev-shm-usage", "--single-process"]
         webdriver_options.append("--headless")
+        if IS_ROOT or IN_WINDOWS or IN_COLAB:
+            webdriver_options.append("--no-sandbox")
         if IS_ROOT or IN_WINDOWS:
-            webdriver_options.extend(["--no-sandbox", "--remote-debugging-port=9222"])
+            webdriver_options.append("--remote-debugging-port=9222")
         if IN_COLAB or IN_AZUREML:
             webdriver_options.append("--disable-dev-shm-usage")
 
@@ -118,50 +117,7 @@ def __init__(
         for option in set(webdriver_options):
             options.add_argument(option)
 
-        if IN_COLAB:
-            try:
-                self.driver = webdriver.Chrome(service=Service("chromedriver"), options=options)
-            except WebDriverException as exc:
-                raise NodeError(
-                    """
-        \'chromium-driver\' needs to be installed manually when running colab. Follow the below given commands:
-                        %%shell
-                        cat > /etc/apt/sources.list.d/debian.list <<'EOF'
-                        deb [arch=amd64 signed-by=/usr/share/keyrings/debian-buster.gpg] http://deb.debian.org/debian buster main
-                        deb [arch=amd64 signed-by=/usr/share/keyrings/debian-buster-updates.gpg] http://deb.debian.org/debian buster-updates main
-                        deb [arch=amd64 signed-by=/usr/share/keyrings/debian-security-buster.gpg] http://deb.debian.org/debian-security buster/updates main
-                        EOF
-
-                        apt-key adv --keyserver keyserver.ubuntu.com --recv-keys DCC9EFBF77E11517
-                        apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 648ACFD622F3D138
-                        apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 112695A0E562B32A
-                        apt-key export 77E11517 | gpg --dearmour -o /usr/share/keyrings/debian-buster.gpg
-                        apt-key export 22F3D138 | gpg --dearmour -o /usr/share/keyrings/debian-buster-updates.gpg
-                        apt-key export E562B32A | gpg --dearmour -o /usr/share/keyrings/debian-security-buster.gpg
-
-                        cat > /etc/apt/preferences.d/chromium.pref << 'EOF'
-                        Package: *
-                        Pin: release a=eoan
-                        Pin-Priority: 500
-
-
-                        Package: *
-                        Pin: origin "deb.debian.org"
-                        Pin-Priority: 300
-
-
-                        Package: chromium*
-                        Pin: origin "deb.debian.org"
-                        Pin-Priority: 700
-                        EOF
-
-                        apt-get update
-                        apt-get install chromium chromium-driver
-        If it has already been installed, please check if it has been copied to the right directory i.e. to \'/usr/bin\'"""
-                ) from exc
-        else:
-            logger.info("'chrome-driver' will be automatically installed.")
-            self.driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)
+        self.driver = webdriver.Chrome(service=Service(), options=options)
         self.urls = urls
         self.crawler_depth = crawler_depth
         self.filter_urls = filter_urls


Test patch:
diff --git a/test/nodes/test_connector.py b/test/nodes/test_connector.py
--- a/test/nodes/test_connector.py
+++ b/test/nodes/test_connector.py
@@ -54,9 +54,8 @@ def content_in_results(crawler: Crawler, url: str, results: List[Path], expected
 
 
 @pytest.mark.unit
-@patch("haystack.nodes.connector.crawler.ChromeDriverManager")
 @patch("haystack.nodes.connector.crawler.webdriver")
-def test_crawler_url_none_exception(webdriver, manager):
+def test_crawler_url_none_exception(webdriver):
     crawler = Crawler()
     with pytest.raises(ValueError):
         crawler.crawl()