Repository: robocorp/rpaframework

Issue:
RPA.Windows locator offset does not work as expected
I have tried making a robot which uses `RPA.Windows` the `offset`in a locator.
The offset is way off.

For an example i used the calculator.

```
Control Window    name:Calculator
Click    name:Calculator> id:136    # Does as expected. Clicks center of element
Sleep    2s
Click    name:Calculator> id:136 offset:0,0    # Not as expected. Clicks way out of window
Sleep    2s
Click    name:Calculator> id:136 offset:-300,-300    # Not as expected. Clicks way out of window
```


Discussion:
I can confirm, even our [test](https://github.com/robocorp/rpaframework/blob/master/packages/windows/tests/robot/test_windows.robot#L127-L128) is highlighting the fact that there's something shady out there.

Working on a fix!

Resolving patch:
diff --git a/packages/windows/src/RPA/Windows/keywords/action.py b/packages/windows/src/RPA/Windows/keywords/action.py
--- a/packages/windows/src/RPA/Windows/keywords/action.py
+++ b/packages/windows/src/RPA/Windows/keywords/action.py
@@ -124,46 +124,63 @@ def middle_click(
         """
         return self._mouse_click(locator, "MiddleClick", wait_time, timeout)
 
-    def _mouse_click(self, element, click_type, wait_time, timeout):
-        click_wait_time = wait_time or self.ctx.wait_time
+    def _mouse_click(
+        self,
+        locator: Locator,
+        click_type: str,
+        wait_time: Optional[float],
+        timeout: Optional[float],
+    ) -> WindowsElement:
+        click_wait_time: float = wait_time or self.ctx.wait_time
         with self.set_timeout(timeout):
-            element = self.ctx.get_element(element)
-            if element.item.robocorp_click_offset:
-                self.ctx.logger.debug("Click element with offset")
-                self._click_element_coordinates(element, click_type, click_wait_time)
-            else:
-                self.ctx.logger.debug("Click element")
-                self._click_element(element, click_type, click_wait_time)
+            element = self.ctx.get_element(locator)
+            self._click_element(element, click_type, click_wait_time)
         return element
 
-    def _click_element_coordinates(self, element, click_type, click_wait_time):
-        callable_attribute = hasattr(element.item, click_type)
-        if callable_attribute:
-            rect = element.item.BoundingRectangle
-            offset_x, offset_y = [
-                int(v) for v in element.item.robocorp_click_offset.split(",")
-            ]
-            getattr(element.item, click_type)(
-                rect.xcenter() + offset_x,
-                rect.ycenter() + offset_y,
-                waitTime=click_wait_time,
-            )
-        else:
+    def _click_element(
+        self, element: WindowsElement, click_type: str, click_wait_time: float
+    ):
+        item = element.item
+        click_function = getattr(item, click_type, None)
+        if not click_function:
             raise ActionNotPossible(
                 f"Element {element!r} does not have {click_type!r} attribute"
             )
 
-    def _click_element(self, element, click_type, click_wait_time):
-        attr = hasattr(element.item, click_type)
-        if attr:
-            getattr(element.item, click_type)(
-                waitTime=click_wait_time,
-                simulateMove=False,
-            )
-        else:
-            raise ActionNotPossible(
-                f"Element {element!r} does not have {click_type!r} attribute"
-            )
+        # Attribute added in `RPA.core.windows.locators.LocatorMethods`.
+        offset: Optional[str] = item.robocorp_click_offset
+        offset_x: Optional[int] = None
+        offset_y: Optional[int] = None
+        log_message = f"{click_type}-ing element"
+        if offset:
+            # Get a new fresh bounding box each time, since the element might have been
+            #  moved from its initial spot.
+            rect = item.BoundingRectangle
+            # Now compute the new coordinates starting from the element center.
+            dist_x, dist_y = (int(dist.strip()) for dist in offset.split(","))
+            pos_x, pos_y = rect.xcenter() + dist_x, rect.ycenter() + dist_y
+            # If we pass the newly obtained absolute position to the clicking function
+            #  that won't work as expected. You see (`help(item.Click)`), if the passed
+            #  offset is positive then it gets relative to the left-top corner and if
+            #  is negative then the right-bottom corner is used.
+            # Let's assume we end up with a positive relative offset. (using left-top
+            #  fixed corner)
+            offset_x, offset_y = pos_x - rect.left, pos_y - rect.top
+            # If by any chance an offset is negative, it gets relative to the
+            #  right-bottom corner, therefore adjust it accordingly.
+            if offset_x < 0:
+                offset_x -= rect.width()
+            if offset_y < 0:
+                offset_y -= rect.height()
+            log_message += f" with offset: {offset_x}, {offset_y}"
+
+        self.logger.debug(log_message)
+        click_function(
+            x=offset_x,
+            y=offset_y,
+            simulateMove=self.ctx.SIMULATE_MOVE,
+            waitTime=click_wait_time,
+        )
 
     @keyword(tags=["action"])
     def select(self, locator: Locator, value: str) -> WindowsElement:
@@ -199,7 +216,7 @@ def send_keys(
         interval: float = 0.01,
         wait_time: Optional[float] = None,
         send_enter: bool = False,
-    ) -> WindowsElement:
+    ):
         """Send keys to desktop, current window or to Control element
         defined by given locator.
 
diff --git a/packages/windows/tasks.py b/packages/windows/tasks.py
--- a/packages/windows/tasks.py
+++ b/packages/windows/tasks.py
@@ -99,15 +99,23 @@ def typecheck(ctx):
     poetry(ctx, "run mypy src")
 
 
-@task(install)
+@task
 def test(ctx):
-    """Run unittests"""
+    """Run Python unit tests and Robot Framework tests"""
+    testpython(ctx)
+    # NOTE: Windows robot tests disabled in CI for now. (requires VM with UI support)
+    # testrobot(ctx)
+
+
+@task(install)
+def testpython(ctx):
+    """Run Python unit-tests."""
     poetry(ctx, "run pytest")
 
 
 @task(install)
 def testrobot(ctx, ci=False, task_robot=None):
-    """Run Robot Framework tests"""
+    """Run Robot Framework tests."""
     exclude = []
     if not task_robot:
         exclude.append("manual")
@@ -121,8 +129,7 @@ def testrobot(ctx, ci=False, task_robot=None):
     poetry(ctx, run_cmd)
 
 
-# add test
-@task(cleanlibspec, lint, libspec)
+@task(cleanlibspec, lint, libspec, test)
 def build(ctx):
     """Build distributable python package"""
     poetry(ctx, "build -vv -f sdist")


Test patch:
diff --git a/packages/windows/tests/python/test_windows.py b/packages/windows/tests/python/test_windows.py
--- a/packages/windows/tests/python/test_windows.py
+++ b/packages/windows/tests/python/test_windows.py
@@ -1,19 +1,19 @@
-# import pytest
+import pytest
+from unittest import mock
+
 from RPA.Windows import Windows
+from RPA.Windows.utils import IS_WINDOWS
 
-#
-# Problem with pytest
-# https://github.com/pywinauto/pywinauto/issues/858
-#
-# @pytest.fixture
-# def library():
-#     library = Windows()
-#     return library
 
-library = Windows(locators_path="tests/python/locators.json")
+@pytest.fixture
+def library():
+    lib = Windows(locators_path="tests/python/locators.json")
+    return lib
 
 
-def test_do_some_calculations():  # (library):
+@pytest.mark.skipif(not IS_WINDOWS, reason="Windows required")
+@pytest.mark.xfail(reason="UI rendering required")
+def test_do_some_calculations(library):
     try:
         library.windows_run("calc.exe")
         library.control_window("name:Calculator")
@@ -28,5 +28,35 @@ def test_do_some_calculations():  # (library):
         library.close_current_window()
 
 
-if __name__ == "__main__":
-    test_do_some_calculations()
+@pytest.mark.parametrize(
+    "offset, x, y",
+    [
+        (None, None, None),  # no offset provided
+        ("0,0", 100, 50),  # right in the center
+        ("10,30", 110, 80),  # a little bit SE
+        ("-100,-50", 0, 0),  # exactly on the left-top corner
+        ("100, 50", 200, 100),  # exactly on the right-bottom corner
+        # Outside the bounding-box.
+        ("-110,-60", -210, -110),  # (NW) -- right-bottom corner relative
+        ("110,60", 210, 110),  # (SE) -- left-top corner relative
+    ],
+)
+@mock.patch("RPA.core.windows.locators.LocatorMethods.get_element")
+def test_coordinates_clicking(mock_get_element, library, offset, x, y):
+    # Let's imagine a 200x100 button positioned on the upper left side of the screen.
+    mock_element = mock.Mock()
+    item = mock_element.item
+    rect = item.BoundingRectangle
+    rect.left = 100
+    rect.top = 200
+    rect.right = 300
+    rect.bottom = 300
+    rect.xcenter.return_value = (rect.left + rect.right) // 2
+    rect.ycenter.return_value = (rect.top + rect.bottom) // 2
+    rect.width.return_value = rect.right - rect.left
+    rect.height.return_value = rect.bottom - rect.top
+    mock_get_element.return_value = mock_element
+
+    item.robocorp_click_offset = offset
+    library.click("MyButton")
+    item.Click.assert_called_once_with(x=x, y=y, simulateMove=False, waitTime=0.5)
diff --git a/packages/windows/tests/robot/test_windows.robot b/packages/windows/tests/robot/test_windows.robot
--- a/packages/windows/tests/robot/test_windows.robot
+++ b/packages/windows/tests/robot/test_windows.robot
@@ -1,7 +1,8 @@
 *** Settings ***
+Library           Process
 Library           RPA.Windows
 Library           String
-Library           Process
+
 Task Setup        Set Wait Time    0.7
 
 
@@ -28,6 +29,7 @@ Calculator Teardown
     Send Keys    keys={Alt}1
     Sleep    1s
 
+
 Select Temperature Unit
     [Arguments]    ${unit}    ${degrees}
     # Select Unit
@@ -37,6 +39,7 @@ Select Temperature Unit
     Log To Console    Set ${unit}: ${degrees}
     Send Keys    keys={Esc}
 
+
 Get Temperature Values
     [Arguments]    ${keys}
     ${values}=    Split String    ${keys}    ,
@@ -45,6 +48,7 @@ Get Temperature Values
         Log To Console    ${temperature}
     END
 
+
 Calculator button actions
     Control Window    Calculator type:Window
     Click    id:num9Button
@@ -52,10 +56,11 @@ Calculator button actions
     Click    id:plusButton
     Click    id:num4Button
     Click    id:equalButton
-    ${result}=    Get Attribute    id:CalculatorResults    Name
+    ${result} =    Get Attribute    id:CalculatorResults    Name
     Log To Console    \n${result}
     Send Keys    keys={Esc}
 
+
 Calculator with keys
     Control Window    Calculator
     Click    id:clearButton
@@ -68,6 +73,7 @@ Calculator with keys
     END
     Length Should Be    ${buttons}      11      msg=From 0 to 9 and decimlar separator
 
+
 Keep open a single Notepad
     Set Global Timeout    ${TIMEOUT}
     ${closed} =     Set Variable    0
@@ -78,6 +84,7 @@ Keep open a single Notepad
     Log    Closed Notepads: ${closed}
     Windows Run   Notepad
 
+
 Kill app by name
     [Arguments]     ${app_name}
 
@@ -92,6 +99,7 @@ Kill app by name
         END
     END
 
+
 Close Current Window And Sleep
     Close Current Window
     Sleep   1s
@@ -103,38 +111,55 @@ Windows search Calculator by clicking buttons
     Calculator button actions
     [Teardown]    Close Current Window And Sleep
 
+
 Calculator by clicking buttons already running
     [Tags]    skip      manual
     Calculator button actions
 
+
 Windows run Do some calculations
     Windows Run    calc.exe     wait_time=1
     Calculator with keys
     [Teardown]    Close Current Window And Sleep
 
+
 Windows run Do some calculations already running
     [Tags]    skip      manual
     Calculator with keys
 
+
 Play Task Calculator
+    [Documentation]     Checks if 34+6=40
+
     Windows Search    Calculator    wait_time=1
     Control Window    Calculator
     Click    id:clearButton
-    Click    type:Group and name:"Number pad" > type:Button and index:4
-    # It is optional to use "and" in the locator syntax.
+
+    ${button_locator} =     Set Variable
+    ...     type:Group and name:"Number pad" > type:Button
+    # It is optional to use "and" in the locator syntax. (only "and"s are assumed)
+    Click    ${button_locator} and index:4  # "3"
+    Click    ${button_locator} index:5 offset:0,0  # "4"
+    # NOTE(cmin764): On different Windows versions the app UI differs so the offsets.
+    #  (might be also resolution bound)
     ${ver} =    Get OS Version
     IF    "${ver}" == "11"
-        # FIXME(cmiN): On Windows 11 this offset minimizes the window. (might be resolution bound)
-        ${locator} =    Set Variable    type:Group and name:"Number pad" > type:Button index:5
+        ${offset} =    Set Variable     480,100
     ELSE
-        ${locator} =    Set Variable    type:Group and name:"Number pad" > type:Button index:5 offset:370,0
+        ${offset} =    Set Variable     370,0  # FIXME on Windows 10
     END
-    Click    ${locator}
+    # Click the "+" button relative to "4" by offset.
+    Click    ${button_locator} index:5 offset:${offset}  # "+"
     # "control" maps to same thing as "type" -> "ControlType".
-    Click    control:Group and name:"Number pad" > control:Button index:7
-    Click    id:equalButton
+    Click    control:Group and name:"Number pad" > control:Button index:7  # 6
+    Click    id:equalButton  # "="
+
+    ${result} =    Get Attribute    id:CalculatorResults    Name
+    Should Be Equal     ${result}       Display is 40
+
     [Teardown]    Close Current Window
 
+
 Play Task Temperature
     Windows Search    Calculator    wait_time=1
     Control Window    Calculator
@@ -167,6 +192,7 @@ Play Task Temperature
 
     [Teardown]    Close Current Window
 
+
 Play Task UIDemo
     [Tags]    skip
 
@@ -199,6 +225,7 @@ Play Task UIDemo
     # TODO: Add more actions to the task: slider, checkboxes, table (get/set vals).
     [Teardown]    Close Current Window
 
+
 Resize window with Spotify
     [Tags]    skip    manual
 
@@ -219,6 +246,7 @@ Resize window with Spotify
 
     [Teardown]    Close Current Window
 
+
 Notepad write text into a file
     Windows Search    notepad   wait_time=1
     Control Window    subname:"- Notepad"
@@ -263,6 +291,7 @@ Notepad write text into a file
     Minimize Window    subname:"- Notepad"
     [Teardown]      Close Current Window
 
+
 Control Window by handle
     Log To Console    \nList Windows
     ${win} =    List Windows
@@ -272,6 +301,7 @@ Control Window by handle
     ${win} =    Control Window    handle:${win}[0][handle]    # handle of the first window in the list
     Log To Console      Controlled window: ${win}
 
+
 Calculator result from recording
     [Tags]    skip    manual
     Windows Run     Calc
@@ -288,6 +318,7 @@ Calculator result from recording
 
     [Teardown]    Close Current Window
 
+
 Write to Notepad in the background
     [Tags]    skip
     Windows Run     Notepad
@@ -316,6 +347,7 @@ Write to Notepad in the background
 
     [Teardown]    Close Window      subname:Notepad  # finally Notepad is closed too
 
+
 Test getting elements
     [Tags]    skip
     Clear Anchor
@@ -333,6 +365,7 @@ Test getting elements
         Log To Console    App: ${button.name}
     END
 
+
 Control window after closing linked root element
     [Setup]    Keep open a single Notepad
     ${window} =     Control Window   subname:Notepad control:WindowControl
@@ -347,6 +380,7 @@ Control window after closing linked root element
 
     [Teardown]    Close Current Window  # closes Calculator (last active window)
 
+
 Tree printing and controlled anchor cleanup
     Print Tree     #capture_image_folder=output${/}controls