Repository: opsdroid/opsdroid

Issue:
Handle multiple events in a single sync response in the matrix connector
<!-- Before you post an issue or if you are unsure about something join our matrix channel https://riot.im/app/#/room/#opsdroid-general:matrix.org and ask away! We are more than happy to help you. -->
# Description

Currently, when parsing a sync response, the matrix connector returns out of the method when it encounters the first event in the timeline.

https://github.com/opsdroid/opsdroid/blob/3fe47d89df719e22861352fc3806d4093832c024/opsdroid/connector/matrix/connector.py#L284-L286

This is then subsequently passed to `opsdroid.parse` in the `listen` method:

https://github.com/opsdroid/opsdroid/blob/3fe47d89df719e22861352fc3806d4093832c024/opsdroid/connector/matrix/connector.py#L306-L309

The fix for this is to allow returning a list of `Event`s from the `_parse_sync_response` method and then iterating over them in `listen`.


Discussion:
An alternative (probably preferable) solution is to move the call to `opsdroid.parse` into the `_parse_sync_response` method.
Since I am a beginner, I am not sure if what I understood was what you were referring to. Just wanted to ask is it just moving the call for `opsdorid.parse` in the `_parse_sync_response` method instead of this [return statement](https://github.com/opsdroid/opsdroid/blob/3fe47d89df719e22861352fc3806d4093832c024/opsdroid/connector/matrix/connector.py#L284) where each event will be passed in the `opsdorid.parse` method inside the `for` loop.
This is approximately @Cadair's solution in #1715.

```python
    async def _parse_sync_response(self, response):
        # ... snip
        for roomid, roomInfo in response.rooms.join.items():
            if roomInfo.timeline:
                for event in roomInfo.timeline.events:
                    if event.sender != self.mxid:
                        # ... snip
                        await self.opsdroid.parse(
                            self._event_creator.create_event(event.source, roomid)
                        )

    async def listen(self):
        # ... snip
        while True:
            response = await self.connection.sync(
                # ... snip
            )
            # ... snip
            await self._parse_sync_response(response)
```

I think this might be a bit cleaner (which I've recommended in #1715):

```python
    async def _parse_sync_response(self, response):
        # ... snip
        for roomid, roomInfo in response.rooms.join.items():
            if roomInfo.timeline:
                for event in roomInfo.timeline.events:
                    if event.sender != self.mxid:
                        # ... snip
                        yield await self._event_creator.create_event(
                            event.source, roomid
                        )
        return

    async def listen(self):
        # ... snip
        while True:
            response = await self.connection.sync(
                # ... snip
            )
            # ... snip
            async for message in self._parse_sync_response(response):
                if message:
                    await self.opsdroid.parse(message)
```
I was thinking more like this, similar to @Cadair :
> 
> ```python
>     async def _parse_sync_response(self, response):
>         # ... snip
>         for roomid, roomInfo in response.rooms.join.items():
>             if roomInfo.timeline:
>                 for event in roomInfo.timeline.events:
>                     if event.sender != self.mxid:
>                         # ... snip
>                         await self.opsdroid.parse(
>                             self._event_creator.create_event(event.source, roomid)
>                         )
> 
>     async def listen(self):
>         # ... snip
>         while True:
>             response = await self.connection.sync(
>                 # ... snip
>             )
>             # ... snip
>             await self._parse_sync_response(response)
> ```
> 
@cognifloyd, yours is also a beautiful solution:
> 
> ```python
>     async def _parse_sync_response(self, response):
>         # ... snip
>         for roomid, roomInfo in response.rooms.join.items():
>             if roomInfo.timeline:
>                 for event in roomInfo.timeline.events:
>                     if event.sender != self.mxid:
>                         # ... snip
>                         yield await self._event_creator.create_event(
>                             event.source, roomid
>                         )
>         return
> 
>     async def listen(self):
>         # ... snip
>         while True:
>             response = await self.connection.sync(
>                 # ... snip
>             )
>             # ... snip
>             async for message in self._parse_sync_response(response):
>                 if message:
>                     await self.opsdroid.parse(message)
> ```


@Cadair, @cognifloyd Do you want me to make the change as the one stated above?

Resolving patch:
diff --git a/opsdroid/connector/matrix/connector.py b/opsdroid/connector/matrix/connector.py
--- a/opsdroid/connector/matrix/connector.py
+++ b/opsdroid/connector/matrix/connector.py
@@ -182,7 +182,6 @@ async def exchange_keys(self, initial_sync=False):
 
     async def connect(self):
         """Create connection object with chat library."""
-
         if self._allow_encryption:
             _LOGGER.debug(f"Using {self.store_path} for the matrix client store.")
             Path(self.store_path).mkdir(exist_ok=True)
@@ -299,14 +298,12 @@ async def _parse_sync_response(self, response):
             ][0]
             sender = await self.get_nick(None, invite_event.sender)
 
-            await self.opsdroid.parse(
-                events.UserInvite(
-                    target=roomid,
-                    user_id=invite_event.sender,
-                    user=sender,
-                    connector=self,
-                    raw_event=invite_event,
-                )
+            yield events.UserInvite(
+                target=roomid,
+                user_id=invite_event.sender,
+                user=sender,
+                connector=self,
+                raw_event=invite_event,
             )
 
         for roomid, roomInfo in response.rooms.join.items():
@@ -320,7 +317,7 @@ async def _parse_sync_response(self, response):
                                 event = self.connection.decrypt_event(event)
                             except nio.exceptions.EncryptionError:  # pragma: no cover
                                 _LOGGER.exception(f"Failed to decrypt event {event}")
-                        return await self._event_creator.create_event(
+                        yield await self._event_creator.create_event(
                             event.source, roomid
                         )
 
@@ -342,10 +339,8 @@ async def listen(self):  # pragma: no cover
 
             await self.exchange_keys()
 
-            message = await self._parse_sync_response(response)
-
-            if message:
-                await self.opsdroid.parse(message)
+            async for event in self._parse_sync_response(response):
+                await self.opsdroid.parse(event)
 
     def lookup_target(self, room):
         """Convert name or alias of a room to the corresponding room ID."""


Test patch:
diff --git a/opsdroid/connector/matrix/tests/conftest.py b/opsdroid/connector/matrix/tests/conftest.py
--- a/opsdroid/connector/matrix/tests/conftest.py
+++ b/opsdroid/connector/matrix/tests/conftest.py
@@ -2,6 +2,7 @@
 from pathlib import Path
 
 import pytest
+import nio
 
 from opsdroid.connector.matrix import ConnectorMatrix
 
@@ -35,7 +36,7 @@ def token_config(mock_api_obj):
 
 
 @pytest.fixture
-async def connector(opsdroid, request, mock_api_obj):
+async def connector(opsdroid, request, mock_api_obj, mocker):
     if hasattr(request, "param"):
         fix_name = request.param
     else:
@@ -55,11 +56,31 @@ async def connector(opsdroid, request, mock_api_obj):
         config["homeserver"] = mock_api_obj.base_url
 
     conn = ConnectorMatrix(config, opsdroid=opsdroid)
+    conn.connection = mocker.MagicMock()
     yield conn
-    if conn.connection is not None:
+    if isinstance(conn.connection, nio.AsyncClient):
         await conn.disconnect()
 
 
+@pytest.fixture
+async def connector_connected(
+    opsdroid,
+    double_filter_response,
+    single_message_sync_response,
+    mock_whoami_join,
+    mock_api,
+):
+    config = {
+        "access_token": "token",
+        "homeserver": mock_api.base_url,
+        "rooms": {"main": "#test:localhost"},
+    }
+    conn = ConnectorMatrix(config, opsdroid=opsdroid)
+    await conn.connect()
+    yield conn
+    await conn.disconnect()
+
+
 ################################################################################
 # Sync Factory
 ################################################################################
@@ -148,3 +169,15 @@ def single_message_sync_response(mock_api_obj):
     event = message_factory("Test", "m.text", "@stuart:localhost")
     sync = sync_response([event])
     mock_api_obj.add_response("/_matrix/client/r0/sync", "GET", sync)
+
+
+@pytest.fixture
+def mock_whoami_join(mock_api_obj):
+    mock_api_obj.add_response(
+        "/_matrix/client/r0/account/whoami", "GET", {"user_id": "@opsdroid:localhost"}
+    )
+    mock_api_obj.add_response(
+        "/_matrix/client/r0/join/#test:localhost",
+        "POST",
+        {"room_id": "!12355:localhost"},
+    )
diff --git a/opsdroid/connector/matrix/tests/test_connector_connect.py b/opsdroid/connector/matrix/tests/test_connector_connect.py
--- a/opsdroid/connector/matrix/tests/test_connector_connect.py
+++ b/opsdroid/connector/matrix/tests/test_connector_connect.py
@@ -9,18 +9,6 @@ def test_constructor(opsdroid, default_config):
     assert isinstance(connector, ConnectorMatrix)
 
 
-@pytest.fixture
-def mock_whoami_join(mock_api_obj):
-    mock_api_obj.add_response(
-        "/_matrix/client/r0/account/whoami", "GET", {"user_id": "@opsdroid:localhost"}
-    )
-    mock_api_obj.add_response(
-        "/_matrix/client/r0/join/#test:localhost",
-        "POST",
-        {"room_id": "!12355:localhost"},
-    )
-
-
 @pytest.mark.matrix_connector_config("token_config")
 @pytest.mark.asyncio
 async def test_connect_access_token(
diff --git a/opsdroid/connector/matrix/tests/test_event_parsing.py b/opsdroid/connector/matrix/tests/test_event_parsing.py
new file mode 100644
--- /dev/null
+++ b/opsdroid/connector/matrix/tests/test_event_parsing.py
@@ -0,0 +1,56 @@
+from opsdroid.connector.matrix.tests.conftest import message_factory, sync_response
+import pytest
+
+from nio.responses import SyncResponse
+
+
+async def events_from_sync(events, connector):
+    response = SyncResponse.from_dict(sync_response(events))
+    return [m async for m in connector._parse_sync_response(response)]
+
+
+def assert_event_properties(event, **kwargs):
+    for attr, value in kwargs.items():
+        assert getattr(event, attr) == value
+
+
+@pytest.mark.add_response(
+    "/_matrix/client/r0/profile/@test:localhost/displayname",
+    "GET",
+    {"displayname": "test"},
+)
+@pytest.mark.add_response(
+    "/_matrix/client/r0/profile/@test:localhost/displayname",
+    "GET",
+    {"displayname": "test"},
+)
+@pytest.mark.matrix_connector_config(
+    {"access_token": "hello", "rooms": {"main": "#test:localhost"}}
+)
+@pytest.mark.asyncio
+async def test_receive_message(opsdroid, connector_connected, mock_api, caplog):
+    events = await events_from_sync(
+        [
+            message_factory("Hello", "m.text", "@test:localhost"),
+            message_factory("Hello2", "m.text", "@test:localhost"),
+        ],
+        connector_connected,
+    )
+
+    assert_event_properties(
+        events[0],
+        text="Hello",
+        user_id="@test:localhost",
+        user="test",
+        target="!12345:localhost",
+    )
+
+    assert_event_properties(
+        events[1],
+        text="Hello2",
+        user_id="@test:localhost",
+        user="test",
+        target="!12345:localhost",
+    )
+
+    assert len(caplog.records) == 0, caplog.records
diff --git a/tests/test_connector_matrix.py b/tests/test_connector_matrix.py
--- a/tests/test_connector_matrix.py
+++ b/tests/test_connector_matrix.py
@@ -295,7 +295,12 @@ async def test_parse_sync_response(self, connector):
                 nio.ProfileGetDisplayNameResponse("SomeUsersName")
             )
 
-            returned_message = await connector._parse_sync_response(self.sync_return)
+            returned_message = [
+                m async for m in connector._parse_sync_response(self.sync_return)
+            ]
+            assert len(returned_message) == 1
+
+            returned_message = returned_message[0]
 
             assert isinstance(returned_message, events.Message)
             assert returned_message.text == "LOUD NOISES"
@@ -309,9 +314,9 @@ async def test_parse_sync_response(self, connector):
             )
             assert returned_message.raw_event == raw_message
 
-            returned_message = await connector._parse_sync_response(
-                self.sync_return_join
-            )
+            returned_message = [
+                m async for m in connector._parse_sync_response(self.sync_return_join)
+            ][0]
 
             assert isinstance(returned_message, events.JoinRoom)
             assert returned_message.user == "SomeUsersName"
@@ -327,17 +332,14 @@ async def test_parse_sync_response(self, connector):
 
     async def test_sync_parse_invites(self, connector):
         with amock.patch(api_string.format("get_displayname")) as patched_name:
-            connector.opsdroid = amock.MagicMock()
-            connector.opsdroid.parse.return_value = asyncio.Future()
-            connector.opsdroid.parse.return_value.set_result("")
             patched_name.return_value = asyncio.Future()
             patched_name.return_value.set_result(
                 nio.ProfileGetDisplayNameResponse("SomeUsersName")
             )
 
-            await connector._parse_sync_response(self.sync_invite)
-
-            (invite,), _ = connector.opsdroid.parse.call_args
+            (invite,) = [
+                m async for m in connector._parse_sync_response(self.sync_invite)
+            ]
 
             assert invite.target == "!AWtmOvkBPTCSPbdaHn:localhost"
             assert invite.user == "SomeUsersName"
@@ -477,7 +479,8 @@ async def _get_message(self, connector):
             patched_nick.return_value = asyncio.Future()
             patched_nick.return_value.set_result("Neo")
 
-            return await connector._parse_sync_response(self.sync_return)
+            async for x in connector._parse_sync_response(self.sync_return):
+                return x
 
     async def test_send_edited_message(self, connector):
         message = events.EditedMessage(
@@ -498,10 +501,7 @@ def expected_content(message):
                 "msgtype": "m.text",
                 "m.new_content": new_content,
                 "body": f"* {new_content['body']}",
-                "m.relates_to": {
-                    "rel_type": "m.replace",
-                    "event_id": event_id,
-                },
+                "m.relates_to": {"rel_type": "m.replace", "event_id": event_id},
             }
 
         with amock.patch(
@@ -616,10 +616,7 @@ def test_get_roomname(self, connector):
         assert connector.get_roomname("someroom") == "someroom"
 
     def test_lookup_target(self, connector):
-        connector.room_ids = {
-            "main": "!aroomid:localhost",
-            "test": "#test:localhost",
-        }
+        connector.room_ids = {"main": "!aroomid:localhost", "test": "#test:localhost"}
 
         assert connector.lookup_target("main") == "!aroomid:localhost"
         assert connector.lookup_target("#test:localhost") == "!aroomid:localhost"
@@ -1011,10 +1008,7 @@ async def test_respond_user_role(self, connector):
 
     async def test_send_reaction(self, connector):
         message = events.Message(
-            "hello",
-            event_id="$11111",
-            connector=connector,
-            target="!test:localhost",
+            "hello", event_id="$11111", connector=connector, target="!test:localhost"
         )
         reaction = events.Reaction("⭕")
         with OpsDroid() as _:
@@ -1038,10 +1032,7 @@ async def test_send_reaction(self, connector):
 
     async def test_send_reply(self, connector):
         message = events.Message(
-            "hello",
-            event_id="$11111",
-            connector=connector,
-            target="!test:localhost",
+            "hello", event_id="$11111", connector=connector, target="!test:localhost"
         )
         reply = events.Reply("reply")
         with OpsDroid() as _:
@@ -1169,10 +1160,7 @@ async def test_no_user_id(self, connector):
             )
 
     def test_m_notice(self, connector):
-        connector.rooms["test"] = {
-            "alias": "#test:localhost",
-            "send_m_notice": True,
-        }
+        connector.rooms["test"] = {"alias": "#test:localhost", "send_m_notice": True}
 
         assert connector.message_type("main") == "m.text"
         assert connector.message_type("test") == "m.notice"