Repository: robocorp/rpaframework

Issue:
Enabling Truststore in Python 3.10
- conditional package installation
- checking python and pip versions
- early implementation into `core` package


Discussion:

Resolving patch:
diff --git a/packages/core/src/RPA/core/__init__.py b/packages/core/src/RPA/core/__init__.py
--- a/packages/core/src/RPA/core/__init__.py
+++ b/packages/core/src/RPA/core/__init__.py
@@ -0,0 +1,4 @@
+from .certificates import use_system_certificates
+
+
+use_system_certificates()
diff --git a/packages/core/src/RPA/core/certificates.py b/packages/core/src/RPA/core/certificates.py
new file mode 100644
--- /dev/null
+++ b/packages/core/src/RPA/core/certificates.py
@@ -0,0 +1,60 @@
+import logging
+import sys
+from importlib.metadata import PackageNotFoundError, version as get_version
+
+from packaging import version as version_parser
+
+
+LOGGER = logging.getLogger(__name__)
+
+
+def _check_pip_version(min_version: str) -> bool:
+    try:
+        pip_version = get_version("pip")
+    except PackageNotFoundError:
+        LOGGER.debug("'pip' is not installed!")
+        return False
+
+    if version_parser.parse(pip_version) >= version_parser.parse(min_version):
+        LOGGER.debug(
+            "Current 'pip' version %s satisfies minimum of %s.",
+            pip_version,
+            min_version,
+        )
+        return True
+
+    LOGGER.debug(
+        "Current 'pip' version %s doesn't satisfy minimum of %s.",
+        pip_version,
+        min_version,
+    )
+    return False
+
+
+def use_system_certificates():
+    """Exposes native system certificate stores.
+
+    Call this before importing anything else.
+    """
+    # Works with Python 3.10.12, pip 23.2.1 and above.
+    py_version = (3, 10, 12)
+    pip_version_str = "23.2.1"
+    if sys.version_info >= py_version and _check_pip_version(pip_version_str):
+        try:
+            import truststore  # pylint: disable=import-outside-toplevel
+        except ImportError:
+            LOGGER.debug("Dependency `truststore` is not installed!")
+        else:
+            truststore.inject_into_ssl()
+            LOGGER.info(
+                "Truststore injection done, using system certificate store to validate"
+                " HTTPS."
+            )
+            return
+
+    LOGGER.info(
+        "Truststore not in use, HTTPS traffic validated against `certifi` package."
+        " (requires Python %s and 'pip' %s at minimum)",
+        ".".join(str(nr) for nr in py_version),
+        pip_version_str,
+    )
diff --git a/packages/core/src/RPA/core/webdriver.py b/packages/core/src/RPA/core/webdriver.py
--- a/packages/core/src/RPA/core/webdriver.py
+++ b/packages/core/src/RPA/core/webdriver.py
@@ -6,6 +6,7 @@
 import stat
 from pathlib import Path
 from typing import Dict, List, Optional
+from urllib.parse import urljoin
 
 from packaging import version as version_parser
 from selenium import webdriver
@@ -13,7 +14,9 @@
 from selenium.webdriver.remote.webdriver import WebDriver
 from webdriver_manager.chrome import ChromeDriverManager as _ChromeDriverManager
 from webdriver_manager.core.download_manager import DownloadManager
-from webdriver_manager.core.driver_cache import DriverCacheManager
+from webdriver_manager.core.driver_cache import (
+    DriverCacheManager as _DriverCacheManager,
+)
 from webdriver_manager.core.logger import log
 from webdriver_manager.core.manager import DriverManager
 from webdriver_manager.core.os_manager import ChromeType, OperationSystemManager
@@ -28,12 +31,29 @@
 from RPA.core.robocorp import robocorp_home
 
 
+# FIXME(cmin764; 6 Sep 2023): Remove this once the following issue is solved:
+#  https://github.com/SergeyPirogov/webdriver_manager/issues/618
+class DriverCacheManager(_DriverCacheManager):
+    """Fixes caching when retrieving an existing already downloaded webdriver."""
+
+    # pylint: disable=unused-private-member
+    def __get_metadata_key(self, *args, **kwargs) -> str:
+        # pylint: disable=super-with-arguments
+        get_metadata_key = functools.partial(
+            super(DriverCacheManager, self)._DriverCacheManager__get_metadata_key,
+            *args,
+            **kwargs,
+        )
+        return get_metadata_key() or get_metadata_key()
+
+
 class ChromeDriver(_ChromeDriver):
     """Custom class which correctly obtains the chromedriver download URL."""
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args, versions_url: str, **kwargs):
         super().__init__(*args, **kwargs)
 
+        self._versions_url = versions_url
         self._resolve_version = False
 
     def _get_resolved_version(self, determined_browser_version: Optional[str]) -> str:
@@ -55,10 +75,17 @@ def _get_resolved_version(self, determined_browser_version: Optional[str]) -> st
         resp = self._http_client.get(url=latest_release_url)
         return resp.text.rstrip()
 
-    def get_latest_release_version(self) -> str:
+    def get_driver_version_to_download(self) -> str:
+        if self._resolve_version:
+            return self.get_latest_release_version(resolve_version=True)
+
+        return super().get_driver_version_to_download()
+
+    # pylint: disable=arguments-differ
+    def get_latest_release_version(self, resolve_version: bool = False) -> str:
         # This is activated for any chromedriver version.
         determined_browser_version = self.get_browser_version_from_os()
-        if determined_browser_version and not self._resolve_version:
+        if determined_browser_version and not resolve_version:
             parts = version_parser.parse(determined_browser_version).release
             if len(parts) == 4:
                 # Got a fully downloadable version that MAY be available, but we are
@@ -101,11 +128,7 @@ def get_url_for_version_and_platform(
             self._resolve_modern_url, driver_platform=driver_platform
         )
 
-        versions_url = (
-            "https://googlechromelabs.github.io/chrome-for-testing/"
-            "known-good-versions-with-downloads.json"
-        )
-        response = self._http_client.get(versions_url)
+        response = self._http_client.get(self._versions_url)
         data = response.json()
         versions = data["versions"]
 
@@ -135,7 +158,7 @@ def get_driver_download_url(self, os_type: str, resolve: bool = False) -> str:
             # This time we want a resolved version based on the previously parsed
             #  non-existing one on the server.
             self._resolve_version = True  # force resolve
-            self._driver_version_to_download = None  # resets cache
+            self._driver_version_to_download = None  # ensures resolving
 
         return super().get_driver_download_url(os_type)
 
@@ -155,6 +178,10 @@ def __init__(
         download_manager: Optional[DownloadManager] = None,
         cache_manager: Optional[DriverCacheManager] = None,
         os_system_manager: Optional[OperationSystemManager] = None,
+        versions_url: str = (
+            "https://googlechromelabs.github.io/chrome-for-testing/"
+            "known-good-versions-with-downloads.json"
+        ),
     ):
         super().__init__(
             driver_version=driver_version,
@@ -177,6 +204,7 @@ def __init__(
             chrome_type=chrome_type,
             http_client=self.http_client,
             os_system_manager=os_system_manager,
+            versions_url=versions_url,
         )
 
     def _get_driver_binary_path(self, driver: ChromeDriver) -> str:
@@ -196,9 +224,9 @@ class IEDriverManager(_IEDriverManager):
 
     # Forcefully download the 32bit version of the webdriver no matter the architecture
     #  of the Windows system since it is known that the 64bit version is limited and
-    #  also creates issues, like freezing browser automation with Selenium.
+    #  also creates issues, like freezing browser automation with Selenium sometimes.
     # https://www.selenium.dev/documentation/webdriver/browsers/internet_explorer/
-    FORCE_32BIT = True
+    FORCE_32BIT = not os.getenv("RPA_ALLOW_64BIT_IE")
 
     def get_os_type(self) -> str:
         return "Win32" if self.FORCE_32BIT else super().get_os_type()
@@ -223,7 +251,39 @@ def get_os_type(self) -> str:
     # NOTE: IE is discontinued and not supported/encouraged anymore.
     "ie": IEDriverManager,
 }
-# Available `WebDriver` classes in Selenium.
+# Control Room decides from where the webdrivers should be downloaded, but the user
+#  can opt out from using our own trusted internal source and default to
+#  `webdriver-manager`'s implicit locations.
+_USE_EXTERNAL_WEBDRIVERS = bool(os.getenv("RPA_EXTERNAL_WEBDRIVERS"))
+_SOURCE_BASE = os.getenv(
+    "RC_WEBDRIVER_SOURCE", "https://downloads.robocorp.com/ext/webdrivers/"
+)
+_join_base = functools.partial(urljoin, _SOURCE_BASE)
+_DRIVER_SOURCES = {
+    "chrome": {
+        "url": _join_base("chrome/v1"),
+        "latest_release_url": _join_base("chrome/v1/LATEST_RELEASE"),
+        "versions_url": _join_base("chrome/v2/known-good-versions-with-downloads.json"),
+    },
+    "firefox": {
+        "url": _join_base("firefox/download"),
+        "latest_release_url": _join_base("firefox/releases/latest"),
+        "mozila_release_tag": _join_base("firefox/releases/tags/{0}"),
+    },
+    "edge": {
+        "url": _join_base("edge"),
+        "latest_release_url": _join_base("edge/LATEST_RELEASE"),
+    },
+    "ie": {
+        "url": _join_base("ie/download"),
+        "latest_release_url": _join_base("ie/releases"),
+        "ie_release_tag": _join_base("ie/releases/tags/selenium-{0}"),
+    },
+}
+_DRIVER_SOURCES["gecko"] = _DRIVER_SOURCES["mozilla"] = _DRIVER_SOURCES["firefox"]
+_DRIVER_SOURCES["chromiumedge"] = _DRIVER_SOURCES["edge"]
+# Available `WebDriver` classes in Selenium which also support automatic webdriver
+#  download with `webdriver-manager`.
 SUPPORTED_BROWSERS = dict(
     {name: name.capitalize() for name in AVAILABLE_DRIVERS},
     **{"chromiumedge": "ChromiumEdge"},
@@ -293,7 +353,8 @@ def _is_chromium() -> bool:
 
 def _to_manager(browser: str, *, root: Path) -> DriverManager:
     browser = browser.strip()
-    manager_factory = AVAILABLE_DRIVERS.get(browser.lower())
+    browser_lower = browser.lower()
+    manager_factory = AVAILABLE_DRIVERS.get(browser_lower)
     if not manager_factory:
         raise ValueError(
             f"Unsupported browser {browser!r} for webdriver download!"
@@ -304,8 +365,18 @@ def _to_manager(browser: str, *, root: Path) -> DriverManager:
         manager_factory = functools.partial(
             manager_factory, chrome_type=ChromeType.CHROMIUM
         )
+    if not _USE_EXTERNAL_WEBDRIVERS:
+        url_kwargs = _DRIVER_SOURCES.get(browser_lower)
+        if url_kwargs:
+            manager_factory = functools.partial(manager_factory, **url_kwargs)
+        else:
+            LOGGER.warning("Can't set an internal webdriver source for %r!", browser)
+
     cache_manager = DriverCacheManager(root_dir=str(root))
     manager = manager_factory(cache_manager=cache_manager)
+    driver = manager.driver
+    cache = getattr(functools, "cache", functools.lru_cache(maxsize=None))
+    driver.get_latest_release_version = cache(driver.get_latest_release_version)
     return manager
 
 


Test patch:
diff --git a/packages/core/tests/python/test_webdriver.py b/packages/core/tests/python/test_webdriver.py
--- a/packages/core/tests/python/test_webdriver.py
+++ b/packages/core/tests/python/test_webdriver.py
@@ -1,27 +1,50 @@
-import mock
+import unittest.mock as mock
+
 import pytest
+from webdriver_manager.core.os_manager import ChromeType
 
 from RPA.core import webdriver
 
 from . import RESULTS_DIR
 
 
-def _test_chrome_download():
-    path = webdriver.download("Chrome", root=RESULTS_DIR)
-    assert "chromedriver" in path
+@pytest.fixture(autouse=True, scope="module")
+def disable_caching():
+    with mock.patch(
+        "webdriver_manager.core.driver_cache.DriverCacheManager.find_driver",
+        new=mock.Mock(return_value=None),
+    ):
+        yield
 
 
+# Tests with different Chrome versions, which trigger different download approaches.
 @pytest.mark.parametrize("version_override", [None, "114.0.5735.198", "115.0.5790.110"])
-@mock.patch(
-    "webdriver_manager.core.driver_cache.DriverCacheManager.find_driver",
-    new=mock.Mock(return_value=None),
-)  # disable caching
 def test_chrome_download(version_override):
-    if not version_override:
-        _test_chrome_download()
-        return
-
-    get_version_location = "RPA.core.webdriver.ChromeDriver.get_browser_version_from_os"
-    with mock.patch(get_version_location) as get_version:
-        get_version.return_value = version_override
-        _test_chrome_download()
+    get_version_target = "RPA.core.webdriver.ChromeDriver.get_browser_version_from_os"
+    browser_type = (
+        ChromeType.CHROMIUM if webdriver._is_chromium() else ChromeType.GOOGLE
+    )
+    get_version = lambda: webdriver._OPS_MANAGER.get_browser_version_from_os(
+        browser_type
+    )
+    with mock.patch(get_version_target, wraps=get_version) as mock_get_version:
+        if version_override:
+            mock_get_version.return_value = version_override
+
+        path = webdriver.download("Chrome", root=RESULTS_DIR)
+        assert "chromedriver" in path
+
+
+def test_firefox_download():
+    path = webdriver.download("Firefox", root=RESULTS_DIR)
+    assert "geckodriver" in path
+
+
+def test_edge_download():
+    path = webdriver.download("Edge", root=RESULTS_DIR)
+    assert "msedgedriver" in path
+
+
+def test_ie_download():
+    path = webdriver.download("Ie", root=RESULTS_DIR)
+    assert "IEDriverServer.exe" in path