Repository: matrix-org/synapse

Issue:
populate_stats_process_rooms failing with an unknown room
As part of #14643 we are re-populating room and user stats, this is currently failing with an error:

```
Room !XXX for event $YYY is unknown
```

<details>
<summary>Stack trace</summary>

```
2023-01-06 17:45:04,582 - synapse.storage.background_updates - 431 - INFO - background_updates-0 - Starting update batch on background update 'populate_stats_process_rooms'
2023-01-06 17:45:04,725 - synapse.storage.background_updates - 302 - ERROR - background_updates-0 - Error doing update
Capture point (most recent call last):
  File "venv/lib/python3.8-pyston2.3/runpy.py", line 194, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "venv/lib/python3.8-pyston2.3/runpy.py", line 87, in _run_code
    exec(code, run_globals)
  File "src/synapse/app/homeserver.py", line 386, in <module>
    main()
  File "src/synapse/app/homeserver.py", line 382, in main
    run(hs)
  File "src/synapse/app/homeserver.py", line 361, in run
    _base.start_reactor(
  File "src/synapse/app/_base.py", line 191, in start_reactor
    run()
  File "src/synapse/app/_base.py", line 173, in run
    run_command()
  File "src/synapse/app/_base.py", line 148, in <lambda>
    run_command: Callable[[], None] = lambda: reactor.run(),
  File "venv/site-packages/twisted/internet/base.py", line 1318, in run
    self.mainLoop()
  File "venv/site-packages/twisted/internet/base.py", line 1328, in mainLoop
    reactorBaseSelf.runUntilCurrent()
  File "venv/site-packages/twisted/internet/base.py", line 967, in runUntilCurrent
    f(*a, **kw)
  File "src/synapse/storage/databases/main/events_worker.py", line 1185, in fire
    d.callback(row_dict)
  File "venv/site-packages/twisted/internet/defer.py", line 696, in callback
    self._startRunCallbacks(result)
  File "venv/site-packages/twisted/internet/defer.py", line 798, in _startRunCallbacks
    self._runCallbacks()
  File "venv/site-packages/twisted/internet/defer.py", line 892, in _runCallbacks
    current.result = callback(  # type: ignore[misc]
  File "venv/site-packages/twisted/internet/defer.py", line 1792, in gotResult
    _inlineCallbacks(r, gen, status, context)
  File "venv/site-packages/twisted/internet/defer.py", line 1775, in _inlineCallbacks
    status.deferred.errback()
  File "venv/site-packages/twisted/internet/defer.py", line 735, in errback
    self._startRunCallbacks(fail)
  File "venv/site-packages/twisted/internet/defer.py", line 798, in _startRunCallbacks
    self._runCallbacks()
  File "venv/site-packages/twisted/internet/defer.py", line 892, in _runCallbacks
    current.result = callback(  # type: ignore[misc]
  File "venv/site-packages/twisted/internet/defer.py", line 735, in errback
    self._startRunCallbacks(fail)
  File "venv/site-packages/twisted/internet/defer.py", line 798, in _startRunCallbacks
    self._runCallbacks()
  File "venv/site-packages/twisted/internet/defer.py", line 892, in _runCallbacks
    current.result = callback(  # type: ignore[misc]
  File "venv/site-packages/twisted/internet/defer.py", line 1792, in gotResult
    _inlineCallbacks(r, gen, status, context)
  File "venv/site-packages/twisted/internet/defer.py", line 1693, in _inlineCallbacks
    result = context.run(
  File "venv/site-packages/twisted/python/failure.py", line 518, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
Traceback (most recent call last):
  File "src/synapse/storage/background_updates.py", line 294, in run_background_updates
    result = await self.do_next_background_update(sleep)
  File "src/synapse/storage/background_updates.py", line 424, in do_next_background_update
    await self._do_background_update(desired_duration_ms)
  File "src/synapse/storage/background_updates.py", line 467, in _do_background_update
    items_updated = await update_handler(progress, batch_size)
  File "src/synapse/storage/databases/main/stats.py", line 206, in _populate_stats_process_rooms
    await self._calculate_and_set_initial_state_for_room(room_id)
  File "src/synapse/storage/databases/main/stats.py", line 557, in _calculate_and_set_initial_state_for_room
    state_event_map = await self.get_events(event_ids, get_prev_content=False)  # type: ignore[attr-defined]
  File "src/synapse/storage/databases/main/events_worker.py", line 536, in get_events
    events = await self.get_events_as_list(
  File "src/synapse/logging/opentracing.py", line 896, in _wrapper
    return await func(*args, **kwargs)  # type: ignore[misc]
  File "src/synapse/logging/opentracing.py", line 896, in _wrapper
    return await func(*args, **kwargs)  # type: ignore[misc]
  File "src/synapse/storage/databases/main/events_worker.py", line 586, in get_events_as_list
    event_entry_map = await self.get_unredacted_events_from_cache_or_db(
  File "src/synapse/storage/databases/main/events_worker.py", line 818, in get_unredacted_events_from_cache_or_db
    missing_events: Dict[str, EventCacheEntry] = await delay_cancellation(
  File "venv/site-packages/twisted/internet/defer.py", line 1697, in _inlineCallbacks
    result = context.run(gen.send, result)
  File "src/synapse/storage/databases/main/events_worker.py", line 804, in get_missing_events_from_cache_or_db
    raise e
  File "src/synapse/storage/databases/main/events_worker.py", line 797, in get_missing_events_from_cache_or_db
    db_missing_events = await self._get_events_from_db(
  File "src/synapse/storage/databases/main/events_worker.py", line 1297, in _get_events_from_db
    raise Exception(
Exception: Room !XXX for event $YYY is unknown
```

</details>

This exception is raised at:

https://github.com/matrix-org/synapse/blob/db1cfe9c80a707995fcad8f3faa839acb247068a/synapse/storage/databases/main/events_worker.py#L1302-L1304

It happens when attempting to fetch a non-membership event from a room whose room version is unknown (`room_version IS NULL` in the `rooms` table).

Some history:

* The `room_version` column (and populating it) was added in #6729 ~3 years ago, but this skips any room that doesn't have a `m.room.create` event in the `current_state_events` table.
* #6874 then handles using the `room_version` column when reading events.
* There was then a fix-up for this in #7070 where we also try to pull the room version from `state_events` (as opposed to `current_state_events`).
	* @richvdh noticed in review that this will null the `room_version` for rooms that don't have a create event: https://github.com/matrix-org/synapse/pull/7070/files#r393343338, but notes it should be OK since #6874 handles this case (of out-of-band memberships).


Discussion:
To see how many rooms might be affected I ran:

```sql
SELECT room_id FROM (
    SELECT room_id, COUNT(*) FROM current_state_events
    WHERE
        room_id IN (
            SELECT room_id FROM rooms WHERE room_version IS NULL
        )
        AND type != 'm.room.member'
    GROUP BY room_id
) AS cse
WHERE cse.count > 0;
```

This only result in two rooms, including the room it is bailing on.
> This only result in two rooms, including the room it is bailing on.

The 2nd room has a `m.room.create` event in `current_state_events` and `state_events`, but that event doesn't exist in the `events` or `event_json` table.

It actually seems to have *no* events in the `events` / `event_json` tables which makes me think this was a failed purge?
@erikjohnston and I discussed this and we think that these two rooms are so broken that we should see if there's a way for stats processing to skip them -- they're not usable anyway.

Resolving patch:
diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py
--- a/synapse/storage/databases/main/events_worker.py
+++ b/synapse/storage/databases/main/events_worker.py
@@ -110,6 +110,10 @@
 )
 
 
+class InvalidEventError(Exception):
+    """The event retrieved from the database is invalid and cannot be used."""
+
+
 @attr.s(slots=True, auto_attribs=True)
 class EventCacheEntry:
     event: EventBase
@@ -1299,7 +1303,7 @@ async def _fetch_event_ids_and_get_outstanding_redactions(
                 # invites, so just accept it for all membership events.
                 #
                 if d["type"] != EventTypes.Member:
-                    raise Exception(
+                    raise InvalidEventError(
                         "Room %s for event %s is unknown" % (d["room_id"], event_id)
                     )
 
diff --git a/synapse/storage/databases/main/stats.py b/synapse/storage/databases/main/stats.py
--- a/synapse/storage/databases/main/stats.py
+++ b/synapse/storage/databases/main/stats.py
@@ -29,6 +29,7 @@
     LoggingDatabaseConnection,
     LoggingTransaction,
 )
+from synapse.storage.databases.main.events_worker import InvalidEventError
 from synapse.storage.databases.main.state_deltas import StateDeltasStore
 from synapse.types import JsonDict
 from synapse.util.caches.descriptors import cached
@@ -554,7 +555,17 @@ def _fetch_current_state_stats(
             "get_initial_state_for_room", _fetch_current_state_stats
         )
 
-        state_event_map = await self.get_events(event_ids, get_prev_content=False)  # type: ignore[attr-defined]
+        try:
+            state_event_map = await self.get_events(event_ids, get_prev_content=False)  # type: ignore[attr-defined]
+        except InvalidEventError as e:
+            # If an exception occurs fetching events then the room is broken;
+            # skip process it to avoid being stuck on a room.
+            logger.warning(
+                "Failed to fetch events for room %s, skipping stats calculation: %r.",
+                room_id,
+                e,
+            )
+            return
 
         room_state: Dict[str, Union[None, bool, str]] = {
             "join_rules": None,


Test patch:
diff --git a/tests/storage/databases/main/test_room.py b/tests/storage/databases/main/test_room.py
--- a/tests/storage/databases/main/test_room.py
+++ b/tests/storage/databases/main/test_room.py
@@ -40,9 +40,23 @@ def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
         self.token = self.login("foo", "pass")
 
     def _generate_room(self) -> str:
-        room_id = self.helper.create_room_as(self.user_id, tok=self.token)
+        """Create a room and return the room ID."""
+        return self.helper.create_room_as(self.user_id, tok=self.token)
 
-        return room_id
+    def run_background_updates(self, update_name: str) -> None:
+        """Insert and run the background update."""
+        self.get_success(
+            self.store.db_pool.simple_insert(
+                "background_updates",
+                {"update_name": update_name, "progress_json": "{}"},
+            )
+        )
+
+        # ... and tell the DataStore that it hasn't finished all updates yet
+        self.store.db_pool.updates._all_done = False
+
+        # Now let's actually drive the updates to completion
+        self.wait_for_background_updates()
 
     def test_background_populate_rooms_creator_column(self) -> None:
         """Test that the background update to populate the rooms creator column
@@ -71,22 +85,7 @@ def test_background_populate_rooms_creator_column(self) -> None:
         )
         self.assertEqual(room_creator_before, None)
 
-        # Insert and run the background update.
-        self.get_success(
-            self.store.db_pool.simple_insert(
-                "background_updates",
-                {
-                    "update_name": _BackgroundUpdates.POPULATE_ROOMS_CREATOR_COLUMN,
-                    "progress_json": "{}",
-                },
-            )
-        )
-
-        # ... and tell the DataStore that it hasn't finished all updates yet
-        self.store.db_pool.updates._all_done = False
-
-        # Now let's actually drive the updates to completion
-        self.wait_for_background_updates()
+        self.run_background_updates(_BackgroundUpdates.POPULATE_ROOMS_CREATOR_COLUMN)
 
         # Make sure the background update filled in the room creator
         room_creator_after = self.get_success(
@@ -137,22 +136,7 @@ def test_background_add_room_type_column(self) -> None:
             )
         )
 
-        # Insert and run the background update
-        self.get_success(
-            self.store.db_pool.simple_insert(
-                "background_updates",
-                {
-                    "update_name": _BackgroundUpdates.ADD_ROOM_TYPE_COLUMN,
-                    "progress_json": "{}",
-                },
-            )
-        )
-
-        # ... and tell the DataStore that it hasn't finished all updates yet
-        self.store.db_pool.updates._all_done = False
-
-        # Now let's actually drive the updates to completion
-        self.wait_for_background_updates()
+        self.run_background_updates(_BackgroundUpdates.ADD_ROOM_TYPE_COLUMN)
 
         # Make sure the background update filled in the room type
         room_type_after = self.get_success(
@@ -164,3 +148,39 @@ def test_background_add_room_type_column(self) -> None:
             )
         )
         self.assertEqual(room_type_after, RoomTypes.SPACE)
+
+    def test_populate_stats_broken_rooms(self) -> None:
+        """Ensure that re-populating room stats skips broken rooms."""
+
+        # Create a good room.
+        good_room_id = self._generate_room()
+
+        # Create a room and then break it by having no room version.
+        room_id = self._generate_room()
+        self.get_success(
+            self.store.db_pool.simple_update(
+                table="rooms",
+                keyvalues={"room_id": room_id},
+                updatevalues={"room_version": None},
+                desc="test",
+            )
+        )
+
+        # Nuke any current stats in the database.
+        self.get_success(
+            self.store.db_pool.simple_delete(
+                table="room_stats_state", keyvalues={"1": 1}, desc="test"
+            )
+        )
+
+        self.run_background_updates("populate_stats_process_rooms")
+
+        # Only the good room appears in the stats tables.
+        results = self.get_success(
+            self.store.db_pool.simple_select_onecol(
+                table="room_stats_state",
+                keyvalues={},
+                retcol="room_id",
+            )
+        )
+        self.assertEqual(results, [good_room_id])