Repository: robocorp/rpaframework

Issue:
Treat browser window maximization under headless mode (Selenium)
There's a misconception when developing browser automation with `Selenium`, where maximizing the window makes sense on your local UI-enabled machine, but once that's deployed under our Linux cloud container it doesn't make sense and calling [Maximize Browser Window](https://robocorp.com/docs/libraries/rpa-framework/rpa-browser-selenium/keywords#maximize-browser-window) won't know to which size to enlarge since there's no known resolution in a headless environment. (Stackoverflow [context](https://stackoverflow.com/questions/44796194/how-to-set-window-size-to-fullscreen-for-headless-chrome-using-chrome-options/44827872#44827872))

A couple of improvements can be added:
- Issue a warning when trying to maximize under a headless scenario. (and eventually disable the command if not forced through a parameter)
- Log/docs-driven recommendation on using [Set Window Size](https://robocorp.com/docs/libraries/rpa-framework/rpa-browser-selenium/keywords#set-window-size) in such scenarios. (if the default 1440x900 is not enough)


Discussion:

Resolving patch:
diff --git a/packages/main/src/RPA/Browser/Selenium.py b/packages/main/src/RPA/Browser/Selenium.py
--- a/packages/main/src/RPA/Browser/Selenium.py
+++ b/packages/main/src/RPA/Browser/Selenium.py
@@ -7,6 +7,7 @@
 import os
 import platform
 import shutil
+import subprocess
 import time
 import traceback
 import urllib.parse
@@ -38,8 +39,9 @@
 from SeleniumLibrary.errors import ElementNotFound
 from SeleniumLibrary.keywords import (
     AlertKeywords,
-    BrowserManagementKeywords,
+    BrowserManagementKeywords as _BrowserManagementKeywords,
     ScreenshotKeywords,
+    WindowKeywords as _WindowKeywords,
 )
 from SeleniumLibrary.keywords.webdrivertools import SeleniumOptions, WebDriverCreator
 from SeleniumLibrary.locators import ElementFinder
@@ -106,7 +108,7 @@ def binary_location(self) -> Optional[str]:
         return self.binary._start_cmd if self.binary else None
 
 
-class BrowserManagementKeywordsOverride(BrowserManagementKeywords):
+class BrowserManagementKeywords(_BrowserManagementKeywords):
     """Overridden keywords for browser management."""
 
     def __init__(self, ctx):
@@ -127,7 +129,7 @@ def go_to(self, url: str) -> None:
         url = ensure_scheme(url, default=self._default_scheme)
         super().go_to(url)
 
-    go_to.__doc__ = BrowserManagementKeywords.go_to.__doc__
+    go_to.__doc__ = _BrowserManagementKeywords.go_to.__doc__
 
     @keyword
     def open_browser(
@@ -158,7 +160,35 @@ def open_browser(
             executable_path=executable_path,
         )
 
-    open_browser.__doc__ = BrowserManagementKeywords.open_browser.__doc__
+    open_browser.__doc__ = _BrowserManagementKeywords.open_browser.__doc__
+
+
+# pylint: disable=missing-class-docstring
+class WindowKeywords(_WindowKeywords):
+
+    # pylint: disable=arguments-differ
+    @keyword
+    def maximize_browser_window(self, *args, force: bool = False, **kwargs):
+        """Maximizes current browser window.
+
+        The window won't be maximized in headless mode since there's no way to know the
+        screen size to set the window size to in the absence of an UI. Use the
+        ``Set Window Size`` keyword with a specific side or set the `force` param to
+        `True` if you still want to enforce this undefined behaviour.
+        """
+        if self.ctx.headless:
+            decision = (
+                "continuing with maximization" if force else "ignoring maximization"
+            )
+            self.ctx.logger.warning(
+                "Attempting to maximize browser in headless mode without knowning the"
+                " screen resolution. (%s)",
+                decision,
+            )
+            if not force:
+                return
+
+        super().maximize_browser_window(*args, **kwargs)
 
 
 class Selenium(SeleniumLibrary):
@@ -207,12 +237,13 @@ class Selenium(SeleniumLibrary):
         **{"chromiumedge": "ChromiumEdge"},
     )
     # Both driver and browser lower-case names.
-    CHROMIUM_BROWSERS = ["chrome", "edge", "chromiumedge", "msedge"]
+    CHROMIUM_BROWSERS = ["chrome", "edge", "chromiumedge", "msedge", "ie"]
 
     def __init__(self, *args, **kwargs):
         # We need to pop our kwargs before passing kwargs to SeleniumLibrary
         self.auto_close = kwargs.pop("auto_close", True)
         self.locators_path = kwargs.pop("locators_path", None)
+        self.headless: Optional[bool] = None  # will be set once the browser starts
 
         # Parse user-given plugins
         plugins = kwargs.get("plugins", "")
@@ -230,8 +261,9 @@ def __init__(self, *args, **kwargs):
         self._element_finder = RobocorpElementFinder(self)
 
         # Add inherit/overridden library keywords.
-        self.browser_management = BrowserManagementKeywordsOverride(self)
-        override_plugins = [self.browser_management]
+        self.browser_management = BrowserManagementKeywords(self)
+        window_kw = WindowKeywords(self)
+        override_plugins = [self.browser_management, window_kw]
         self.add_library_components(override_plugins)
 
         self.logger = logging.getLogger(__name__)
@@ -475,6 +507,7 @@ def open_available_browser(
 
         attempts = []
         index_or_alias = None
+        self.headless: bool = headless  # it's resolved through the decorator
 
         # Try all browsers in preferred order
         for browser, download in product(browsers, downloads):
@@ -560,23 +593,9 @@ def _arg_download(self, download: Any) -> List:
         else:
             return [bool(download)]
 
-    def _set_chromium_options(
-        self,
-        browser_lower: str,
-        kwargs: dict,
-        options: ChromiumOptions,
-        use_profile: bool = False,
-        profile_name: Optional[str] = None,
-        profile_path: Optional[str] = None,
-        preferences: Optional[dict] = None,
-        proxy: str = None,
+    def _set_experimental_options(
+        self, browser_lower: str, options: ArgOptions, preferences: Optional[dict]
     ):
-        if proxy:
-            options.add_argument(f"--proxy-server={proxy}")
-        options.add_argument("--disable-dev-shm-usage")
-        options.add_argument("--disable-web-security")
-        options.add_argument("--allow-running-insecure-content")
-        options.add_argument("--no-sandbox")
         default_preferences = {
             "safebrowsing.enabled": True,
             "credentials_enable_service": False,
@@ -596,7 +615,32 @@ def _set_chromium_options(
         if not self.auto_close:
             # Leave the browser window open if auto-closing is disabled.
             options.add_experimental_option("detach", True)
+
+    def _set_chromium_options(
+        self,
+        browser_lower: str,
+        kwargs: dict,
+        options: ArgOptions,
+        use_profile: bool = False,
+        profile_name: Optional[str] = None,
+        profile_path: Optional[str] = None,
+        preferences: Optional[dict] = None,
+        proxy: str = None,
+    ):
+        if proxy:
+            options.add_argument(f"--proxy-server={proxy}")
+        options.add_argument("--disable-dev-shm-usage")
+        options.add_argument("--disable-web-security")
+        options.add_argument("--allow-running-insecure-content")
+        options.add_argument("--no-sandbox")
+
+        if browser_lower != "ie":
+            # While Edge supports experimental options, Edge in IE mode doesn't.
+            self._set_experimental_options(browser_lower, options, preferences)
         if use_profile:
+            # NOTE(cmin764; 14 Sep 2023): This doesn't currently work with IE due to a
+            #  bug in the webdriver.
+            # Issue: https://github.com/MicrosoftEdge/EdgeWebDriver/issues/29
             self._set_user_profile(browser_lower, options, profile_path, profile_name)
         if self.logger.isEnabledFor(logging.DEBUG):
             # Deprecated params, but no worries as they get popped then bundled in a
@@ -608,7 +652,7 @@ def _set_ie_options(self, options: IeOptions, *, url: Optional[str]):
         binary_location = getattr(options, "binary_location", None)
         if binary_location:
             options.edge_executable_path = binary_location
-        # An invalid default URL will make the automation freeze.
+        # An invalid default URL might make the automation freeze.
         options.initial_browser_url = (
             options.initial_browser_url or url or "https://robocorp.com/"
         )
@@ -722,7 +766,7 @@ def _get_driver_args(  # noqa: C901
         options: ArgOptions = self.normalize_options(options, browser=browser)
         if headless:
             self._set_headless_options(browser_lower, options)
-        if maximized:
+        if maximized and not headless:
             options.add_argument("--start-maximized")
         if user_agent:
             options.add_argument(f"user-agent={user_agent}")
@@ -742,8 +786,8 @@ def _get_driver_args(  # noqa: C901
                 preferences=preferences,
                 proxy=proxy,
             )
-        elif browser_lower == "ie":
-            self._set_ie_options(options, url=url)
+            if browser_lower == "ie":
+                self._set_ie_options(options, url=url)
         elif browser_lower == "firefox":
             self._set_firefox_options(options, preferences=preferences)
         if self.download_preferences and browser_lower not in self.download_preferences:
@@ -770,9 +814,8 @@ def _get_driver_args(  # noqa: C901
     def _set_headless_options(self, browser_lower: str, options: ArgOptions) -> None:
         """Set headless mode for the browser, if possible.
 
-        ``browser`` string name of the browser
-
-        ``options`` browser options class instance
+        ``browser_lower`` is the lower-case string name of the browser.
+        ``options`` are the browser options to be updated to match the headless config.
         """
         if browser_lower == "safari":
             self.logger.warning(
@@ -804,16 +847,16 @@ def _set_headless_options(self, browser_lower: str, options: ArgOptions) -> None
     def _set_user_profile(
         self,
         browser_lower: str,
-        options: ChromiumOptions,
+        options: ArgOptions,
         profile_path: Optional[str] = None,
         profile_name: Optional[str] = None,
     ) -> None:
-        """Set user profile configuration into browser options
+        """Set user profile configuration into the browser options.
 
-        Requires environment variable ``RPA_CHROME_USER_PROFILE_DIR``
-        to point into user profile directory.
-
-        ``options`` dictionary of browser options
+        In the absence of an explicit `profile_path` an `RPA_CHROME_USER_PROFILE_DIR`
+        environment variable is required to point into the user profile directory
+        instead. Optionally, if a `profile_name` is passed, then we'll be using that
+        instead of the default one.
         """
         data_dir = profile_path or os.getenv("RPA_CHROME_USER_PROFILE_DIR")
         system = platform.system()
@@ -874,12 +917,13 @@ def _get_service_class(self, lower_browser: str) -> type:
         class BrowserService(Service):
             """Custom service class wrapping the picked browser's one."""
 
+            # pylint: disable=no-self-argument
             def __init__(
-                self, *args, executable_path: str = default_webdriver, **kwargs
+                this, *args, executable_path: str = default_webdriver, **kwargs
             ):
                 # NOTE(cmin764): Starting with Selenium 4.9.1, we have to block their
                 #  `SeleniumManager` from early stage, otherwise it will be activated
-                #  when the `WebDriver` class itself is instantiated with a null
+                #  when the `WebDriver` class itself if instantiated with a null
                 #  executable path.
                 resolved_path = (
                     shutil.which(executable_path) if executable_path else None
@@ -890,8 +934,26 @@ def __init__(
                         " (with disabled Selenium Manager)"
                     )
 
+                this._log_webdriver_version(resolved_path)
                 super().__init__(*args, executable_path=resolved_path, **kwargs)
 
+            @staticmethod
+            def _log_webdriver_version(path: str):
+                args = [path, "--version"]
+                try:
+                    result = subprocess.run(
+                        args, check=True, capture_output=True, text=True
+                    )
+                # pylint: disable=broad-except
+                except Exception as exc:
+                    self.logger.warning(
+                        "Webdriver version couldn't be read due to: %s", exc
+                    )
+                else:
+                    self.logger.info(
+                        "Webdriver version taken into use: %s", result.stdout
+                    )
+
             # pylint: disable=no-self-argument
             def __del__(this) -> None:
                 # With auto-close disabled, we shouldn't call the object's cleanup
@@ -902,6 +964,11 @@ def __del__(this) -> None:
 
         return BrowserService
 
+    def _log_browser_version(self, browser: str, *, options: ArgOptions):
+        binary_location = getattr(options, "binary_location", None)
+        version = core_webdriver.get_browser_version(browser, path=binary_location)
+        self.logger.info("Targeted browser version: %s", version)
+
     def _create_webdriver(
         self, browser: str, alias: Optional[str], download: bool, **kwargs
     ) -> AliasType:
@@ -937,6 +1004,7 @@ def _create_driver(path: Optional[str] = None) -> AliasType:
             # NOTE: But don't break a browser name like "ChromiumEdge".
             cap_browser = browser[0].upper() + browser[1:]
             kwargs["service"] = service
+            self._log_browser_version(cap_browser, options=kwargs["options"])
             return self.browser_management.create_webdriver(
                 cap_browser, alias, **kwargs
             )


Test patch:
diff --git a/packages/main/tests/robot/test_browser.robot b/packages/main/tests/robot/test_browser.robot
--- a/packages/main/tests/robot/test_browser.robot
+++ b/packages/main/tests/robot/test_browser.robot
@@ -28,6 +28,7 @@ My Custom Keyword
 Create Browser Data Directory
     ${data_dir} =    Absolute Path    ${BROWSER_DATA}
     RPA.FileSystem.Create Directory    ${data_dir}    parents=${True}
+    RPA.FileSystem.Empty Directory     ${data_dir}
     RETURN    ${data_dir}
 
 Download With Specific Browser
@@ -48,6 +49,7 @@ Download With Specific Browser
     ...    RPA.FileSystem.Remove File    ${file_path}
 
 
+# Tasks which rely on the already open browser in headless mode.
 *** Tasks ***
 Does alert contain
     Go To    ${ALERT_HTML}
@@ -171,12 +173,19 @@ Test Shadow Root
     ${text} =    Get Text    ${elem}
     Should Be Equal    ${text}    some text
 
+Maximize window in headless mode
+    Maximize Browser Window     force=${False}  # forcing it works locally only
+
+
+# Tasks which close the already open browser and opens a new one, most probably in
+#  non-headless mode.
+*** Tasks ***
 Download PDF in custom Firefox directory
-    [Tags]    manual  # no support for the Firefox browser in CI (or on dev machine)
+    [Tags]    manual  # no guaranteed support for the Firefox browser in CI/dev env
     Download With Specific Browser    Firefox
 
 Download PDF in custom Chrome directory
-    [Tags]    skip  # flaky test in CI, mainly on Windows with Python 3.7, 3.8
+    [Tags]    skip  # flaky test in CI, mainly on Windows with Python 3.8
     Download With Specific Browser    Chrome
 
 Open Browser With Dict Options
@@ -193,8 +202,9 @@ Open Browser With Dict Options
     ...    modules=RPA.core.webdriver
     Log To Console    Downloaded webdriver path: ${driver_path}
 
+    ${log_path} =   Set Variable    ${BROWSER_DATA}${/}browser.log
     Open Browser    https://robocorp.com/docs    browser=Chrome    options=${options}
-    ...    executable_path=${driver_path}
+    ...    executable_path=${driver_path}   service_log_path=${log_path}
     ${visible} =    Is Element Visible    xpath://button[contains(@class, "desktop")]
     Should Be True    ${visible}
 
@@ -212,7 +222,7 @@ Open In Incognito With Custom Options
     Open Available Browser    https://robocorp.com/docs    browser_selection=Chrome
     ...    headless=${True}    options=${options}    port=${18888}
     # Custom profile usage now works in headless mode as well. (but not guaranteed
-    #    with older browser versions)
+    #  with older browser versions)
     ...    use_profile=${True}    profile_path=${data_dir}
 
     ${visible} =    Is Element Visible    xpath://button[contains(@class, "desktop")]
@@ -222,10 +232,28 @@ Open In Incognito With Custom Options
     Close Browser
     [Teardown]    RPA.FileSystem.Remove Directory    ${data_dir}    recursive=${True}
 
+Open Edge in IE mode with profile
+    [Documentation]     Prove that we have support for profile usage since the IE
+    ...    webdriver still opens Edge but in IE mode and still supports setting an user
+    ...    data dir.
+    [Setup]     Close Browser
+    [Tags]      windows     skip  # windows specific test without headless support
+
+    ${data_dir} =    Create Browser Data Directory
+    Open Available Browser    https://robocorp.com/docs    browser_selection=Ie
+    ...    use_profile=${True}    profile_path=${data_dir}    profile_name=Default
+    # NOTE(cmin764; 14 Sep 2023): Currently there's a patch in the webdriver binary not
+    #  overriding the user data dir we're trying to set.
+    Run Keyword And Ignore Error    Directory Should Not Be Empty    ${data_dir}
+
+    [Teardown]      RPA.FileSystem.Remove Directory    ${data_dir}    recursive=${True}
+
 Open Edge in normal and IE mode without closing
     [Documentation]     Downloads fresh webdrivers and starts Edge in normal and IE
-    ...     mode on a Windows machine.
-    [Tags]  skip    windows  # requires Windows OS with UI
+    ...    mode on a Windows machine without auto-closing the browser at the end of
+    ...    the execution. (on Windows the webdrivers will be closed but the browser
+    ...    stays open)
+    [Tags]      windows     skip  # requires Windows OS with UI
     [Setup]    Close Browser
 
     ${webdrivers_dir} =     Evaluate    RPA.core.webdriver.DRIVER_ROOT
@@ -239,10 +267,19 @@ Open Edge in normal and IE mode without closing
     Import Library      RPA.Browser.Selenium    auto_close=${False}
     ...     WITH NAME   Selenium
 
-    ${url} =    Set Variable    https://robocorp.com/docs
-    @{browsers} =   Create List     Edge    Ie
-    FOR    ${browser}    IN    @{browsers}
-        Selenium.Open Available Browser   ${url}   browser_selection=${browser}
+    &{edge_check} =  Create Dictionary
+    ...    browser     Edge
+    ...    url   https://robocorp.com/docs
+    ...    text     Portal
+    &{ie_check} =  Create Dictionary
+    ...    browser     Ie
+    ...    url   https://demos.telerik.com/aspnet-ajax/salesdashboard/views/about.aspx
+    ...    text     Telerik
+    @{checks} =   Create List    ${edge_check}     ${ie_check}
+
+    FOR    ${check}    IN    @{checks}
+        Selenium.Open Available Browser   ${check}[url]
+        ...     browser_selection=${check}[browser]
         ...     headless=${False}   download=${True}
-        Selenium.Page Should Contain Link   Portal
+        Selenium.Page Should Contain   ${check}[text]
     END