Repository: matrix-org/synapse

Issue:
state resolution failing with `KeyError: '$S38_aRG4Qz8eIdrTyzJuQdH3Ieus1Y1dhczoFKnOB7M'`
### Description

My client is getting continuous 500's from my synchrotrons due to a `KeyError` from `_get_event`, and thus I'm unable to sync. Restarting does not help:

```
2019-12-30 17:33:07,540 - synapse.http.server - 109 - ERROR - GET-18029- Failed handle request via 'SyncRestServlet': <SynapseRequest at 0x7f272183c5f8 method='GET' uri='/_matrix/client/r0/sync?filter=7&timeout=...
Traceback (most recent call last):
  File "/home/synapse/env/lib/python3.6/site-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks
    result = g.send(result)
StopIteration: {}

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/synapse/env/lib/python3.6/site-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks
    result = g.send(result)
StopIteration: {}

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/synapse/env/lib/python3.6/site-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks
    result = g.send(result)
StopIteration: {}

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/synapse/env/lib/python3.6/site-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks
    result = g.send(result)
StopIteration: []

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/synapse/env/lib/python3.6/site-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks
    result = g.send(result)
StopIteration: {}

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/synapse/env/lib/python3.6/site-packages/synapse/http/server.py", line 77, in wrapped_request_handler
    await h(self, request)
  File "/home/synapse/env/lib/python3.6/site-packages/synapse/http/server.py", line 326, in _async_render
    callback_return = await callback_return
  File "/home/synapse/env/lib/python3.6/site-packages/synapse/rest/client/v2_alpha/sync.py", line 178, in on_GET
    full_state=full_state,
  File "/home/synapse/env/lib/python3.6/site-packages/synapse/handlers/sync.py", line 263, in wait_for_sync_for_user
    full_state,
  File "/home/synapse/env/lib/python3.6/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/synapse/env/lib/python3.6/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/home/synapse/env/lib/python3.6/site-packages/synapse/handlers/sync.py", line 285, in _wait_for_sync_for_user
    sync_config, since_token, full_state=full_state
  File "/home/synapse/env/lib/python3.6/site-packages/synapse/handlers/sync.py", line 956, in generate_sync_result
    sync_result_builder, account_data_by_room
  File "/home/synapse/env/lib/python3.6/site-packages/synapse/handlers/sync.py", line 1367, in _generate_sync_entry_for_rooms
    await concurrently_execute(handle_room_entries, room_entries, 10)
  File "/home/synapse/env/lib/python3.6/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/synapse/env/lib/python3.6/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/home/synapse/env/lib/python3.6/site-packages/synapse/util/async_helpers.py", line 155, in _concurrently_execute_inner
    await maybe_awaitable(func(next(it)))
  File "/home/synapse/env/lib/python3.6/site-packages/synapse/handlers/sync.py", line 1762, in _generate_room_entry
    newly_joined_room=newly_joined,
  File "/home/synapse/env/lib/python3.6/site-packages/synapse/handlers/sync.py", line 413, in _load_filtered_recents
    current_state_ids = await self.state.get_current_state_ids(room_id)
  File "/home/synapse/env/lib/python3.6/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/synapse/env/lib/python3.6/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/home/synapse/env/lib/python3.6/site-packages/synapse/state/__init__.py", line 171, in get_current_state_ids
    ret = yield self.resolve_state_groups_for_events(room_id, latest_event_ids)
  File "/home/synapse/env/lib/python3.6/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/synapse/env/lib/python3.6/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/home/synapse/env/lib/python3.6/site-packages/synapse/util/metrics.py", line 82, in measured_func
    r = yield func(self, *args, **kwargs)
  File "/home/synapse/env/lib/python3.6/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/synapse/env/lib/python3.6/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/home/synapse/env/lib/python3.6/site-packages/synapse/state/__init__.py", line 402, in resolve_state_groups_for_events
    state_res_store=StateResolutionStore(self.store),
  File "/home/synapse/env/lib/python3.6/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/synapse/env/lib/python3.6/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/home/synapse/env/lib/python3.6/site-packages/synapse/state/__init__.py", line 525, in resolve_state_groups
    state_res_store=state_res_store,
  File "/home/synapse/env/lib/python3.6/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/synapse/env/lib/python3.6/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/home/synapse/env/lib/python3.6/site-packages/synapse/state/v2.py", line 116, in resolve_events_with_store
    room_id, power_events, event_map, state_res_store, full_conflicted_set
  File "/home/synapse/env/lib/python3.6/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/synapse/env/lib/python3.6/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/home/synapse/env/lib/python3.6/site-packages/synapse/state/v2.py", line 364, in _reverse_topological_power_sort
    room_id, event_id, event_map, state_res_store
  File "/home/synapse/env/lib/python3.6/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/synapse/env/lib/python3.6/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/home/synapse/env/lib/python3.6/site-packages/synapse/state/v2.py", line 186, in _get_power_level_for_sender
    aev = yield _get_event(room_id, aid, event_map, state_res_store)
  File "/home/synapse/env/lib/python3.6/site-packages/twisted/internet/defer.py", line 1418, in _inlineCallbacks
    result = g.send(result)
  File "/home/synapse/env/lib/python3.6/site-packages/synapse/state/v2.py", line 535, in _get_event
    event = event_map[event_id]
KeyError: '$S38_aRG4Qz8eIdrTyzJuQdH3Ieus1Y1dhczoFKnOB7M'
2019-12-30 17:33:07,544 - synapse.access.http.9183 - 302 - INFO - GET-18029- 127.0.0.1 - 9183 - {@andrewm:amorgan.xyz} Processed request: 16.942sec/0.001sec (6.428sec, 0.142sec) (9.334sec/43.300sec/259) 55B 500
```

The event ID is `$S38_aRG4Qz8eIdrTyzJuQdH3Ieus1Y1dhczoFKnOB7M`, in room `!BAXLHOFjvDKUeLafmO:matrix.org` (IRC Matrix bridges). I don't see any event with that ID in my `events` table, nor can I see it in the content of any event in my db.

### Version information

Synapse v1.7.2


Discussion:
Likewise for t2bot.io
Riot can no longer sync with my server; the event ID is the same. I'm not sure the room belongs to it, but it does appear in the logs preceding each error:

```
synapse.state: [GET-1676] Resolving state for !BAXLHOFjvDKUeLafmO:matrix.org with 2 groups
synapse.state: [GET-1676] Resolving conflicted state for '!BAXLHOFjvDKUeLafmO:matrix.org'
```


@heftig does your server use workers at all or is it one giant synapse?
No workers here.

Resolving patch:
diff --git a/synapse/state/v2.py b/synapse/state/v2.py
--- a/synapse/state/v2.py
+++ b/synapse/state/v2.py
@@ -183,16 +183,20 @@ def _get_power_level_for_sender(room_id, event_id, event_map, state_res_store):
 
     pl = None
     for aid in event.auth_event_ids():
-        aev = yield _get_event(room_id, aid, event_map, state_res_store)
-        if (aev.type, aev.state_key) == (EventTypes.PowerLevels, ""):
+        aev = yield _get_event(
+            room_id, aid, event_map, state_res_store, allow_none=True
+        )
+        if aev and (aev.type, aev.state_key) == (EventTypes.PowerLevels, ""):
             pl = aev
             break
 
     if pl is None:
         # Couldn't find power level. Check if they're the creator of the room
         for aid in event.auth_event_ids():
-            aev = yield _get_event(room_id, aid, event_map, state_res_store)
-            if (aev.type, aev.state_key) == (EventTypes.Create, ""):
+            aev = yield _get_event(
+                room_id, aid, event_map, state_res_store, allow_none=True
+            )
+            if aev and (aev.type, aev.state_key) == (EventTypes.Create, ""):
                 if aev.content.get("creator") == event.sender:
                     return 100
                 break
@@ -403,10 +407,17 @@ def _iterative_auth_checks(
 
         auth_events = {}
         for aid in event.auth_event_ids():
-            ev = yield _get_event(room_id, aid, event_map, state_res_store)
+            ev = yield _get_event(
+                room_id, aid, event_map, state_res_store, allow_none=True
+            )
 
-            if ev.rejected_reason is None:
-                auth_events[(ev.type, ev.state_key)] = ev
+            if not ev:
+                logger.warning(
+                    "auth_event id %s for event %s is missing", aid, event_id
+                )
+            else:
+                if ev.rejected_reason is None:
+                    auth_events[(ev.type, ev.state_key)] = ev
 
         for key in event_auth.auth_types_for_event(event):
             if key in resolved_state:
@@ -457,8 +468,10 @@ def _mainline_sort(
         auth_events = pl_ev.auth_event_ids()
         pl = None
         for aid in auth_events:
-            ev = yield _get_event(room_id, aid, event_map, state_res_store)
-            if (ev.type, ev.state_key) == (EventTypes.PowerLevels, ""):
+            ev = yield _get_event(
+                room_id, aid, event_map, state_res_store, allow_none=True
+            )
+            if ev and (ev.type, ev.state_key) == (EventTypes.PowerLevels, ""):
                 pl = aid
                 break
 
@@ -506,8 +519,10 @@ def _get_mainline_depth_for_event(event, mainline_map, event_map, state_res_stor
         event = None
 
         for aid in auth_events:
-            aev = yield _get_event(room_id, aid, event_map, state_res_store)
-            if (aev.type, aev.state_key) == (EventTypes.PowerLevels, ""):
+            aev = yield _get_event(
+                room_id, aid, event_map, state_res_store, allow_none=True
+            )
+            if aev and (aev.type, aev.state_key) == (EventTypes.PowerLevels, ""):
                 event = aev
                 break
 
@@ -516,7 +531,7 @@ def _get_mainline_depth_for_event(event, mainline_map, event_map, state_res_stor
 
 
 @defer.inlineCallbacks
-def _get_event(room_id, event_id, event_map, state_res_store):
+def _get_event(room_id, event_id, event_map, state_res_store, allow_none=False):
     """Helper function to look up event in event_map, falling back to looking
     it up in the store
 
@@ -525,15 +540,22 @@ def _get_event(room_id, event_id, event_map, state_res_store):
         event_id (str)
         event_map (dict[str,FrozenEvent])
         state_res_store (StateResolutionStore)
+        allow_none (bool): if the event is not found, return None rather than raising
+            an exception
 
     Returns:
-        Deferred[FrozenEvent]
+        Deferred[Optional[FrozenEvent]]
     """
     if event_id not in event_map:
         events = yield state_res_store.get_events([event_id], allow_rejected=True)
         event_map.update(events)
-    event = event_map[event_id]
-    assert event is not None
+    event = event_map.get(event_id)
+
+    if event is None:
+        if allow_none:
+            return None
+        raise Exception("Unknown event %s" % (event_id,))
+
     if event.room_id != room_id:
         raise Exception(
             "In state res for room %s, event %s is in %s"


Test patch: