# Buggy code
def _get_events(txn, batch):
    sql = """
            SELECT prev_event_id
            FROM event_edges
                INNER JOIN events USING (event_id)
                LEFT JOIN rejections USING (event_id)
            WHERE
                prev_event_id IN (%s)
                AND NOT events.outlier
                AND rejections.event_id IS NULL
            """ % (",".join("?" for _ in batch),)

    txn.execute(sql, batch)
    results.extend(r[0] for r in txn)


# Traceback (twisted.internet.defer.FirstError)
2019-04-22 01:18:17,013 - synapse.http.server - 112 - ERROR - POST-16014877 - Failed handle request via 'ReplicationFederationSendEventsRestServlet': <SynapseRequest at 0x7fc87ed2abe0 method='POST' uri='/_synapse/replication/fed_send_events/TloUSGYPDO' clientproto='HTTP/1.1' site=9092>
Capture point (most recent call last):
File "/usr/local/lib/python3.7/runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "/usr/local/lib/python3.7/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/home/synapse/src/synapse/app/homeserver.py", line 661, in <module>
main()
File "/home/synapse/src/synapse/app/homeserver.py", line 657, in main
run(hs)
File "/home/synapse/src/synapse/app/homeserver.py", line 648, in run
logger=logger,
File "/home/synapse/src/synapse/app/_base.py", line 141, in start_reactor
daemon.start()
File "/home/synapse/env-py37/lib/python3.7/site-packages/daemonize.py", line 248, in start
self.action(*privileged_action_result)
File "/home/synapse/src/synapse/app/_base.py", line 127, in run
reactor.run()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/base.py", line 1272, in run
self.mainLoop()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/base.py", line 1281, in mainLoop
self.runUntilCurrent()
File "/home/synapse/src/synapse/metrics/__init__.py", line 347, in f
ret = func(*args, **kwargs)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/base.py", line 875, in runUntilCurrent
f(*a, **kw)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 460, in callback
self._startRunCallbacks(result)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 568, in _startRunCallbacks
self._runCallbacks()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1475, in gotResult
_inlineCallbacks(r, g, status)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1461, in _inlineCallbacks
status.deferred.callback(e.value)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 460, in callback
self._startRunCallbacks(result)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 568, in _startRunCallbacks
self._runCallbacks()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1475, in gotResult
_inlineCallbacks(r, g, status)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1461, in _inlineCallbacks
status.deferred.callback(e.value)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 460, in callback
self._startRunCallbacks(result)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 568, in _startRunCallbacks
self._runCallbacks()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1475, in gotResult
_inlineCallbacks(r, g, status)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1461, in _inlineCallbacks
status.deferred.callback(e.value)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 460, in callback
self._startRunCallbacks(result)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 568, in _startRunCallbacks
self._runCallbacks()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 654, in _runCallbacks
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1475, in gotResult
_inlineCallbacks(r, g, status)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1461, in _inlineCallbacks
status.deferred.callback(e.value)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 460, in callback
self._startRunCallbacks(result)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 568, in _startRunCallbacks
self._runCallbacks()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1475, in gotResult
_inlineCallbacks(r, g, status)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1464, in _inlineCallbacks
status.deferred.errback()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 501, in errback
self._startRunCallbacks(fail)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 568, in _startRunCallbacks
self._runCallbacks()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1475, in gotResult
_inlineCallbacks(r, g, status)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1464, in _inlineCallbacks
status.deferred.errback()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 501, in errback
self._startRunCallbacks(fail)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 568, in _startRunCallbacks
self._runCallbacks()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1475, in gotResult
_inlineCallbacks(r, g, status)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1464, in _inlineCallbacks
status.deferred.errback()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 501, in errback
self._startRunCallbacks(fail)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 568, in _startRunCallbacks
self._runCallbacks()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1475, in gotResult
_inlineCallbacks(r, g, status)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/home/synapse/src/synapse/storage/events.py", line 166, in handle_queue_loop
item.deferred.errback()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 501, in errback
self._startRunCallbacks(fail)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 568, in _startRunCallbacks
self._runCallbacks()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/home/synapse/src/synapse/util/async_helpers.py", line 75, in errback
self._observers.pop().errback(f)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 501, in errback
self._startRunCallbacks(fail)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 568, in _startRunCallbacks
self._runCallbacks()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1114, in _cbDeferred
self.errback(failure.Failure(FirstError(result, index)))
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 501, in errback
self._startRunCallbacks(fail)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 568, in _startRunCallbacks
self._runCallbacks()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1475, in gotResult
_inlineCallbacks(r, g, status)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1464, in _inlineCallbacks
status.deferred.errback()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 501, in errback
self._startRunCallbacks(fail)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 568, in _startRunCallbacks
self._runCallbacks()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1475, in gotResult
_inlineCallbacks(r, g, status)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1464, in _inlineCallbacks
status.deferred.errback()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 501, in errback
self._startRunCallbacks(fail)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 568, in _startRunCallbacks
self._runCallbacks()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1475, in gotResult
_inlineCallbacks(r, g, status)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1464, in _inlineCallbacks
status.deferred.errback()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 501, in errback
self._startRunCallbacks(fail)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 568, in _startRunCallbacks
self._runCallbacks()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/home/synapse/src/synapse/util/async_helpers.py", line 75, in errback
self._observers.pop().errback(f)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 501, in errback
self._startRunCallbacks(fail)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 568, in _startRunCallbacks
self._runCallbacks()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1475, in gotResult
_inlineCallbacks(r, g, status)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1464, in _inlineCallbacks
status.deferred.errback()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 501, in errback
self._startRunCallbacks(fail)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 568, in _startRunCallbacks
self._runCallbacks()
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1475, in gotResult
_inlineCallbacks(r, g, status)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/home/synapse/env-py37/lib/python3.7/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/home/synapse/src/synapse/http/server.py", line 112, in wrapped_request_handler
exc_info=(f.type, f.value, f.getTracebackObject()),
File "/usr/local/lib/python3.7/logging/__init__.py", line 1412, in error
self._log(ERROR, msg, args, **kwargs)
File "/usr/local/lib/python3.7/logging/__init__.py", line 1519, in _log
self.handle(record)
File "/usr/local/lib/python3.7/logging/__init__.py", line 1529, in handle
self.callHandlers(record)
File "/usr/local/lib/python3.7/logging/__init__.py", line 1591, in callHandlers
hdlr.handle(record)
File "/usr/local/lib/python3.7/logging/__init__.py", line 905, in handle
self.emit(record)
File "/usr/local/lib/python3.7/logging/handlers.py", line 1234, in emit
self.flush()
File "/usr/local/lib/python3.7/logging/handlers.py", line 1310, in flush
self.target.handle(record)
File "/usr/local/lib/python3.7/logging/__init__.py", line 905, in handle
self.emit(record)
File "/usr/local/lib/python3.7/logging/handlers.py", line 70, in emit
logging.FileHandler.emit(self, record)
File "/usr/local/lib/python3.7/logging/__init__.py", line 1132, in emit
StreamHandler.emit(self, record)
File "/usr/local/lib/python3.7/logging/__init__.py", line 1034, in emit
msg = self.format(record)
File "/usr/local/lib/python3.7/logging/__init__.py", line 880, in format
return fmt.format(record)
File "/usr/local/lib/python3.7/logging/__init__.py", line 627, in format
record.exc_text = self.formatException(record.exc_info)
File "/home/synapse/src/synapse/util/logformatter.py", line 45, in formatException
traceback.print_stack(tb.tb_frame.f_back, None, sio)
Traceback (most recent call last):
File "/home/synapse/src/synapse/http/server.py", line 81, in wrapped_request_handler
yield h(self, request)
File "/home/synapse/src/synapse/http/server.py", line 316, in _async_render
callback_return = yield callback(request, **kwargs)
File "/home/synapse/src/synapse/replication/http/federation.py", line 117, in _handle_request
event_and_contexts, backfilled,
File "/home/synapse/src/synapse/handlers/federation.py", line 2662, in persist_events_and_notify
backfilled=backfilled,
File "/home/synapse/src/synapse/storage/events.py", line 287, in persist_events
defer.gatherResults(deferreds, consumeErrors=True)
twisted.internet.defer.FirstError: FirstError[#0, [Failure instance: Traceback: <class 'AssertionError'>: No forward extremities left!
/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py:654:_runCallbacks
/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py:1475:gotResult
/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py:1416:_inlineCallbacks
/home/synapse/env-py37/lib/python3.7/site-packages/twisted/python/failure.py:512:throwExceptionIntoGenerator
--- <exception caught here> ---
/home/synapse/src/synapse/storage/events.py:163:handle_queue_loop
/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py:1416:_inlineCallbacks
/home/synapse/env-py37/lib/python3.7/site-packages/twisted/python/failure.py:512:throwExceptionIntoGenerator
/home/synapse/src/synapse/storage/events.py:324:persisting_queue
/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py:1416:_inlineCallbacks
/home/synapse/env-py37/lib/python3.7/site-packages/twisted/python/failure.py:512:throwExceptionIntoGenerator
/home/synapse/src/synapse/storage/events.py:205:f
/home/synapse/env-py37/lib/python3.7/site-packages/twisted/internet/defer.py:1418:_inlineCallbacks
/home/synapse/src/synapse/storage/events.py:416:_persist_events
]]

# Fixed code
def _get_events(txn, batch):
    sql = """
            SELECT prev_event_id, internal_metadata
            FROM event_edges
                INNER JOIN events USING (event_id)
                LEFT JOIN rejections USING (event_id)
                LEFT JOIN event_json USING (event_id)
            WHERE
                prev_event_id IN (%s)
                AND NOT events.outlier
                AND rejections.event_id IS NULL
            """ % (",".join("?" for _ in batch),)

    txn.execute(sql, batch)
    results.extend(r[0] for r in txn if not json.loads(r[1]).get("soft_failed"))