Repository: matrix-org/synapse

Issue:
Failed to handle PDU: NoneType is not iterable
This appears to have happened when receiving a transaction (from matrix.org in this case), and did cause calls to get_missing_events which appears to be a more complicated way to get the event. It does appear to eventually have received the event.

Homeserver has been based off of develop with some additional patches that should not affect the problem here: https://github.com/matrix-org/synapse/compare/develop...turt2live:travis/t2bot.io

```
homeserver_1 - 2018-09-16 03:40:03,006 - synapse.storage.events - 441 - INFO - Calculating state delta for room !redacted_MUyU:matrix.org
homeserver_1 - 2018-09-16 03:40:03,007 - synapse.util.async_helpers - 218 - INFO - Releasing linearizer lock 'fed_room_pdu' for key '!redacted_MUyU:matrix.org'
homeserver_1 - 2018-09-16 03:40:03,009 - synapse.federation.federation_server - 203 - ERROR - Failed to handle PDU $redacted_NfGx:matrix.org: Traceback (most recent call last):
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 501, in errback
    self._startRunCallbacks(fail)
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 568, in _startRunCallbacks
    self._runCallbacks()
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 654, in _runCallbacks
    current.result = callback(current.result, *args, **kw)
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1475, in gotResult
    _inlineCallbacks(r, g, status)
--- <exception caught here> ---
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/federation/federation_server.py", line 192, in process_pdus_for_room
    origin, pdu
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/federation/federation_server.py", line 621, in _handle_received_pdu
    origin, pdu, get_missing=True, sent_to_us_directly=True,
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/handlers/federation.py", line 235, in on_receive_pdu
    origin, pdu, prevs, min_depth
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/handlers/federation.py", line 389, in _get_missing_events_for_pdu
    get_missing=False
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/handlers/federation.py", line 313, in on_receive_pdu
    auth_chain=auth_chain,
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/handlers/federation.py", line 473, in _process_received_pdu
    state=state,
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/handlers/federation.py", line 1465, in _handle_new_event
    six.reraise(tp, value, tb)
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/handlers/federation.py", line 1455, in _handle_new_event
    backfilled=backfilled,
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/federation/federation_server.py", line 621, in _handle_received_pdu
    origin, pdu, get_missing=True, sent_to_us_directly=True,
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/handlers/federation.py", line 235, in on_receive_pdu
    origin, pdu, prevs, min_depth
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/handlers/federation.py", line 389, in _get_missing_events_for_pdu
    get_missing=False
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/handlers/federation.py", line 313, in on_receive_pdu
    auth_chain=auth_chain,
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/handlers/federation.py", line 473, in _process_received_pdu
    state=state,
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
    result = result.throwExceptionIntoGenerator(g)
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/python/failure.py", line 491, in throwExceptionIntoGenerator
    return g.throw(self.type, self.value, self.tb)
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/handlers/federation.py", line 1465, in _handle_new_event
    six.reraise(tp, value, tb)
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/handlers/federation.py", line 1455, in _handle_new_event
    backfilled=backfilled,
twisted.internet.defer.FirstError: FirstError[#0, [Failure instance: Traceback: <type 'exceptions.TypeError'>: 'NoneType' object is not iterable
/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/internet/defer.py:654:_runCallbacks
/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/internet/defer.py:1475:gotResult
/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/internet/defer.py:1416:_inlineCallbacks
/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/python/failure.py:491:throwExceptionIntoGenerator
--- <exception caught here> ---
/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/storage/events.py:154:handle_queue_loop
/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/internet/defer.py:1416:_inlineCallbacks
/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/python/failure.py:491:throwExceptionIntoGenerator
/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/storage/events.py:312:persisting_queue
/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/internet/defer.py:1416:_inlineCallbacks
/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/python/failure.py:491:throwExceptionIntoGenerator
/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/storage/events.py:195:f
/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/internet/defer.py:1418:_inlineCallbacks
/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/storage/events.py:453:_persist_events
]]
homeserver_1 - 2018-09-16 03:40:03,018 - synapse.handlers.federation - 226 - INFO - Acquiring lock for room '!redacted_MUyU:matrix.org' to fetch 1 missing events: [u'$redacted_NfGx:matrix.org']...
homeserver_1 - 2018-09-16 03:40:03,018 - synapse.util.async_helpers - 205 - INFO - Acquired uncontended linearizer lock 'fed_room_pdu' for key '!redacted_MUyU:matrix.org'
```


Discussion:
This is still happening:

```
homeserver_1 - 2018-09-27 16:09:35,629 - synapse.federation.federation_server - 205 - ERROR - PUT-4013-$redacted_dvww:matrix.org - Failed to handle PDU $redacted_dvww:matrix.org: Traceback (most recent call last):
--- <exception caught here> ---
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/federation/federation_server.py", line 194, in process_pdus_for_room
    origin, pdu
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/federation/federation_server.py", line 623, in _handle_received_pdu
    origin, pdu, sent_to_us_directly=True,
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/handlers/federation.py", line 268, in on_receive_pdu
    origin, pdu, prevs, min_depth
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/handlers/federation.py", line 548, in _get_missing_events_for_pdu
    sent_to_us_directly=False,
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/handlers/federation.py", line 438, in on_receive_pdu
    auth_chain=auth_chain,
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/handlers/federation.py", line 637, in _process_received_pdu
    yield self._handle_new_events(origin, event_infos)
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/handlers/federation.py", line 1672, in _handle_new_events
    backfilled=backfilled,
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/handlers/federation.py", line 2531, in persist_events_and_notify
    backfilled=backfilled,
  File "/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/storage/events.py", line 273, in persist_events
    defer.gatherResults(deferreds, consumeErrors=True)
twisted.internet.defer.FirstError: FirstError[#0, [Failure instance: Traceback: <type 'exceptions.TypeError'>: 'NoneType' object is not iterable
/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/internet/defer.py:654:_runCallbacks
/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/internet/defer.py:1475:gotResult
/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/internet/defer.py:1416:_inlineCallbacks
/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/python/failure.py:491:throwExceptionIntoGenerator
--- <exception caught here> ---
/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/storage/events.py:154:handle_queue_loop
/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/internet/defer.py:1416:_inlineCallbacks
/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/python/failure.py:491:throwExceptionIntoGenerator
/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/storage/events.py:312:persisting_queue
/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/internet/defer.py:1416:_inlineCallbacks
/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/python/failure.py:491:throwExceptionIntoGenerator
/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/storage/events.py:195:f
/home/matrix/.synapse/local/lib/python2.7/site-packages/twisted/internet/defer.py:1418:_inlineCallbacks
/home/matrix/.synapse/local/lib/python2.7/site-packages/synapse/storage/events.py:453:_persist_events
]]

```

Resolving patch:
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -38,6 +38,7 @@
 from synapse.storage.event_federation import EventFederationStore
 from synapse.storage.events_worker import EventsWorkerStore
 from synapse.types import RoomStreamToken, get_domain_from_id
+from synapse.util import batch_iter
 from synapse.util.async_helpers import ObservableDeferred
 from synapse.util.caches.descriptors import cached, cachedInlineCallbacks
 from synapse.util.frozenutils import frozendict_json_encoder
@@ -386,12 +387,10 @@ def _persist_events(self, events_and_contexts, backfilled=False,
                             )
 
                         for room_id, ev_ctx_rm in iteritems(events_by_room):
-                            # Work out new extremities by recursively adding and removing
-                            # the new events.
                             latest_event_ids = yield self.get_latest_event_ids_in_room(
                                 room_id
                             )
-                            new_latest_event_ids = yield self._calculate_new_extremeties(
+                            new_latest_event_ids = yield self._calculate_new_extremities(
                                 room_id, ev_ctx_rm, latest_event_ids
                             )
 
@@ -400,6 +399,12 @@ def _persist_events(self, events_and_contexts, backfilled=False,
                                 # No change in extremities, so no change in state
                                 continue
 
+                            # there should always be at least one forward extremity.
+                            # (except during the initial persistence of the send_join
+                            # results, in which case there will be no existing
+                            # extremities, so we'll `continue` above and skip this bit.)
+                            assert new_latest_event_ids, "No forward extremities left!"
+
                             new_forward_extremeties[room_id] = new_latest_event_ids
 
                             len_1 = (
@@ -517,44 +522,79 @@ def _persist_events(self, events_and_contexts, backfilled=False,
                     )
 
     @defer.inlineCallbacks
-    def _calculate_new_extremeties(self, room_id, event_contexts, latest_event_ids):
-        """Calculates the new forward extremeties for a room given events to
+    def _calculate_new_extremities(self, room_id, event_contexts, latest_event_ids):
+        """Calculates the new forward extremities for a room given events to
         persist.
 
         Assumes that we are only persisting events for one room at a time.
         """
-        new_latest_event_ids = set(latest_event_ids)
-        # First, add all the new events to the list
-        new_latest_event_ids.update(
-            event.event_id for event, ctx in event_contexts
+
+        # we're only interested in new events which aren't outliers and which aren't
+        # being rejected.
+        new_events = [
+            event for event, ctx in event_contexts
             if not event.internal_metadata.is_outlier() and not ctx.rejected
+        ]
+
+        # start with the existing forward extremities
+        result = set(latest_event_ids)
+
+        # add all the new events to the list
+        result.update(
+            event.event_id for event in new_events
         )
-        # Now remove all events that are referenced by the to-be-added events
-        new_latest_event_ids.difference_update(
+
+        # Now remove all events which are prev_events of any of the new events
+        result.difference_update(
             e_id
-            for event, ctx in event_contexts
+            for event in new_events
             for e_id, _ in event.prev_events
-            if not event.internal_metadata.is_outlier() and not ctx.rejected
         )
 
-        # And finally remove any events that are referenced by previously added
-        # events.
-        rows = yield self._simple_select_many_batch(
-            table="event_edges",
-            column="prev_event_id",
-            iterable=list(new_latest_event_ids),
-            retcols=["prev_event_id"],
-            keyvalues={
-                "is_state": False,
-            },
-            desc="_calculate_new_extremeties",
-        )
+        # Finally, remove any events which are prev_events of any existing events.
+        existing_prevs = yield self._get_events_which_are_prevs(result)
+        result.difference_update(existing_prevs)
 
-        new_latest_event_ids.difference_update(
-            row["prev_event_id"] for row in rows
-        )
+        defer.returnValue(result)
 
-        defer.returnValue(new_latest_event_ids)
+    @defer.inlineCallbacks
+    def _get_events_which_are_prevs(self, event_ids):
+        """Filter the supplied list of event_ids to get those which are prev_events of
+        existing (non-outlier/rejected) events.
+
+        Args:
+            event_ids (Iterable[str]): event ids to filter
+
+        Returns:
+            Deferred[List[str]]: filtered event ids
+        """
+        results = []
+
+        def _get_events(txn, batch):
+            sql = """
+            SELECT prev_event_id
+            FROM event_edges
+                INNER JOIN events USING (event_id)
+                LEFT JOIN rejections USING (event_id)
+            WHERE
+                prev_event_id IN (%s)
+                AND NOT events.outlier
+                AND rejections.event_id IS NULL
+            """ % (
+                ",".join("?" for _ in batch),
+            )
+
+            txn.execute(sql, batch)
+            results.extend(r[0] for r in txn)
+
+        for chunk in batch_iter(event_ids, 100):
+            yield self.runInteraction(
+                "_get_events_which_are_prevs",
+                _get_events,
+                chunk,
+            )
+
+        defer.returnValue(results)
 
     @defer.inlineCallbacks
     def _get_new_state_after_events(self, room_id, events_context, old_latest_event_ids,
@@ -586,10 +626,6 @@ def _get_new_state_after_events(self, room_id, events_context, old_latest_event_
             the new current state is only returned if we've already calculated
             it.
         """
-
-        if not new_latest_event_ids:
-            return
-
         # map from state_group to ((type, key) -> event_id) state map
         state_groups_map = {}
 


Test patch: