# Buggy code
def _purge_history_txn(self, txn, room_id, token_str, delete_local_events):
    token = RoomStreamToken.parse(token_str)

    # Tables that should be pruned:
    #     event_auth
    #     event_backward_extremities
    #     event_edges
    #     event_forward_extremities
    #     event_json
    #     event_push_actions
    #     event_reference_hashes
    #     event_search
    #     event_to_state_groups
    #     events
    #     rejections
    #     room_depth
    #     state_groups
    #     state_groups_state

    # we will build a temporary table listing the events so that we don't
    # have to keep shovelling the list back and forth across the
    # connection. Annoyingly the python sqlite driver commits the
    # transaction on CREATE, so let's do this first.
    #
    # furthermore, we might already have the table from a previous (failed)
    # purge attempt, so let's drop the table first.

    txn.execute("DROP TABLE IF EXISTS events_to_purge")

    txn.execute(
        "CREATE TEMPORARY TABLE events_to_purge ("
        "    event_id TEXT NOT NULL,"
        "    should_delete BOOLEAN NOT NULL"
        ")"
    )

    # First ensure that we're not about to delete all the forward extremeties
    txn.execute(
        "SELECT e.event_id, e.depth FROM events as e "
        "INNER JOIN event_forward_extremities as f "
        "ON e.event_id = f.event_id "
        "AND e.room_id = f.room_id "
        "WHERE f.room_id = ?",
        (room_id,),
    )
    rows = txn.fetchall()
    max_depth = max(row[1] for row in rows)

    if max_depth < token.topological:
        # We need to ensure we don't delete all the events from the database
        # otherwise we wouldn't be able to send any events (due to not
        # having any backwards extremeties)
        raise SynapseError(
            400, "topological_ordering is greater than forward extremeties"
        )

    logger.info("[purge] looking for events to delete")

    should_delete_expr = "state_key IS NULL"
    should_delete_params = ()  # type: Tuple[Any, ...]
    if not delete_local_events:
        should_delete_expr += " AND event_id NOT LIKE ?"

        # We include the parameter twice since we use the expression twice
        should_delete_params += ("%:" + self.hs.hostname, "%:" + self.hs.hostname)

    should_delete_params += (room_id, token.topological)

    # Note that we insert events that are outliers and aren't going to be
    # deleted, as nothing will happen to them.
    txn.execute(
        "INSERT INTO events_to_purge"
        " SELECT event_id, %s"
        " FROM events AS e LEFT JOIN state_events USING (event_id)"
        " WHERE (NOT outlier OR (%s)) AND e.room_id = ? AND topological_ordering < ?"
        % (should_delete_expr, should_delete_expr),
        should_delete_params,
    )

    # We create the indices *after* insertion as that's a lot faster.

    # create an index on should_delete because later we'll be looking for
    # the should_delete / shouldn't_delete subsets
    txn.execute(
        "CREATE INDEX events_to_purge_should_delete ON events_to_purge(should_delete)"
    )

    # We do joins against events_to_purge for e.g. calculating state
    # groups to purge, etc., so lets make an index.
    txn.execute("CREATE INDEX events_to_purge_id ON events_to_purge(event_id)")

    txn.execute("SELECT event_id, should_delete FROM events_to_purge")
    event_rows = txn.fetchall()
    logger.info(
        "[purge] found %i events before cutoff, of which %i can be deleted",
        len(event_rows),
        sum(1 for e in event_rows if e[1]),
    )

    logger.info("[purge] Finding new backward extremities")

    # We calculate the new entries for the backward extremeties by finding
    # events to be purged that are pointed to by events we're not going to
    # purge.
    txn.execute(
        "SELECT DISTINCT e.event_id FROM events_to_purge AS e"
        " INNER JOIN event_edges AS ed ON e.event_id = ed.prev_event_id"
        " LEFT JOIN events_to_purge AS ep2 ON ed.event_id = ep2.event_id"
        " WHERE ep2.event_id IS NULL"
    )
    new_backwards_extrems = txn.fetchall()

    logger.info("[purge] replacing backward extremities: %r", new_backwards_extrems)

    txn.execute("DELETE FROM event_backward_extremities WHERE room_id = ?", (room_id,))

    # Update backward extremeties
    txn.executemany(
        "INSERT INTO event_backward_extremities (room_id, event_id) VALUES (?, ?)",
        [(room_id, event_id) for (event_id,) in new_backwards_extrems],
    )

    logger.info("[purge] finding state groups referenced by deleted events")

    # Get all state groups that are referenced by events that are to be
    # deleted.
    txn.execute(
        """
            SELECT DISTINCT state_group FROM events_to_purge
            INNER JOIN event_to_state_groups USING (event_id)
        """
    )

    referenced_state_groups = {sg for (sg,) in txn}
    logger.info(
        "[purge] found %i referenced state groups", len(referenced_state_groups)
    )

    logger.info("[purge] removing events from event_to_state_groups")
    txn.execute(
        "DELETE FROM event_to_state_groups "
        "WHERE event_id IN (SELECT event_id from events_to_purge)"
    )
    for event_id, _ in event_rows:
        txn.call_after(self._get_state_group_for_event.invalidate, (event_id,))

    # Delete all remote non-state events
    for table in (
        "events",
        "event_json",
        "event_auth",
        "event_edges",
        "event_forward_extremities",
        "event_reference_hashes",
        "event_search",
        "rejections",
    ):
        logger.info("[purge] removing events from %s", table)

        txn.execute(
            "DELETE FROM %s WHERE event_id IN ("
            "    SELECT event_id FROM events_to_purge WHERE should_delete"
            ")" % (table,)
        )

    # event_push_actions lacks an index on event_id, and has one on
    # (room_id, event_id) instead.
    for table in ("event_push_actions",):
        logger.info("[purge] removing events from %s", table)

        txn.execute(
            "DELETE FROM %s WHERE room_id = ? AND event_id IN ("
            "    SELECT event_id FROM events_to_purge WHERE should_delete"
            ")" % (table,),
            (room_id,),
        )

    # Mark all state and own events as outliers
    logger.info("[purge] marking remaining events as outliers")
    txn.execute(
        "UPDATE events SET outlier = ?"
        " WHERE event_id IN ("
        "    SELECT event_id FROM events_to_purge "
        "    WHERE NOT should_delete"
        ")",
        (True,),
    )

    # synapse tries to take out an exclusive lock on room_depth whenever it
    # persists events (because upsert), and once we run this update, we
    # will block that for the rest of our transaction.
    #
    # So, let's stick it at the end so that we don't block event
    # persistence.
    #
    # We do this by calculating the minimum depth of the backwards
    # extremities. However, the events in event_backward_extremities
    # are ones we don't have yet so we need to look at the events that
    # point to it via event_edges table.
    txn.execute(
        """
            SELECT COALESCE(MIN(depth), 0)
            FROM event_backward_extremities AS eb
            INNER JOIN event_edges AS eg ON eg.prev_event_id = eb.event_id
            INNER JOIN events AS e ON e.event_id = eg.event_id
            WHERE eb.room_id = ?
        """,
        (room_id,),
    )
    (min_depth,) = txn.fetchone()

    logger.info("[purge] updating room_depth to %d", min_depth)

    txn.execute(
        "UPDATE room_depth SET min_depth = ? WHERE room_id = ?",
        (min_depth, room_id),
    )

    # finally, drop the temp table. this will commit the txn in sqlite,
    # so make sure to keep this actually last.
    txn.execute("DROP TABLE events_to_purge")

    logger.info("[purge] done")

    return referenced_state_groups


# Traceback (twisted.internet.defer.FirstError)
2020-06-13 15:25:39,124 - synapse.storage.data_stores.main.events - 95 - ERROR - persist_events-9 - IntegrityError, retrying.
Traceback (most recent call last):
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/data_stores/main/events.py", line 93, in f
res = yield func(self, *args, delete_existing=False, **kwargs)
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/data_stores/main/events.py", line 211, in _persist_events_and_state_updates
new_forward_extremeties=new_forward_extremeties,
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/database.py", line 527, in runInteraction
**kwargs
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/internet/defer.py", line 1416, in _inlineCallbacks
result = result.throwExceptionIntoGenerator(g)
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/failure.py", line 512, in throwExceptionIntoGenerator
return g.throw(self.type, self.value, self.tb)
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/database.py", line 575, in runWithConnection
self._db_pool.runWithConnection(inner_func, *args, **kwargs)
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/threadpool.py", line 250, in inContext
result = inContext.theWork()
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/threadpool.py", line 266, in <lambda>
inContext.theWork = lambda: context.call(ctx, func, *args, **kw)
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/context.py", line 122, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/context.py", line 85, in callWithContext
return func(*args,**kw)
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/enterprise/adbapi.py", line 306, in _runWithConnection
compat.reraise(excValue, excTraceback)
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/compat.py", line 464, in reraise
raise exception.with_traceback(traceback)
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/enterprise/adbapi.py", line 297, in _runWithConnection
result = func(conn, *args, **kw)
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/database.py", line 572, in inner_func
return func(conn, *args, **kwargs)
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/database.py", line 418, in new_transaction
r = func(cursor, *args, **kwargs)
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/logging/utils.py", line 73, in wrapped
return f(*args, **kwargs)
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/data_stores/main/events.py", line 451, in _persist_events_txn
backfilled=backfilled,
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/data_stores/main/events.py", line 1001, in _update_metadata_tables_txn
self._handle_event_relations(txn, event)
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/data_stores/main/events.py", line 1299, in _handle_event_relations
"aggregation_key": aggregation_key,
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/database.py", line 653, in simple_insert_txn
txn.execute(sql, vals)
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/database.py", line 210, in execute
self._do_execute(self.txn.execute, sql, *args)
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/database.py", line 236, in _do_execute
return func(sql, *args)
psycopg2.errors.UniqueViolation: duplicate key value violates unique constraint "event_relations_id"
DETAIL:  Key (event_id)=($hnVIuNuExbqJ-9iXkDpaRAGdKdeDue4YI8w9G1qOSHc) already exists.

2020-06-13 15:25:39,130 - synapse.storage.data_stores.main.events - 815 - INFO - persist_events-9 - Deleting existing
2020-06-13 15:25:39,147 - synapse.handlers.federation - 1064 - ERROR - GET-1637 - Failed to backfill from matrix.org because FirstError[#0, [Failure instance: Traceback: <class 'psycopg2.errors.UniqueViolation'>: duplicate key value violates unique constraint "event_relations_id"
DETAIL:  Key (event_id)=($hnVIuNuExbqJ-9iXkDpaRAGdKdeDue4YI8w9G1qOSHc) already exists.

/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/internet/defer.py:654:_runCallbacks
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/internet/defer.py:1475:gotResult
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/internet/defer.py:1416:_inlineCallbacks
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/failure.py:512:throwExceptionIntoGenerator
--- <exception caught here> ---
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/persist_events.py:152:handle_queue_loop
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/persist_events.py:262:persisting_queue
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/persist_events.py:438:_persist_events
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/internet/defer.py:1416:_inlineCallbacks
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/failure.py:512:throwExceptionIntoGenerator
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/data_stores/main/events.py:96:f
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/internet/defer.py:1416:_inlineCallbacks
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/failure.py:512:throwExceptionIntoGenerator
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/data_stores/main/events.py:211:_persist_events_and_state_updates
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/internet/defer.py:1416:_inlineCallbacks
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/failure.py:512:throwExceptionIntoGenerator
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/database.py:527:runInteraction
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/internet/defer.py:1416:_inlineCallbacks
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/failure.py:512:throwExceptionIntoGenerator
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/database.py:575:runWithConnection
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/threadpool.py:250:inContext
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/threadpool.py:266:<lambda>
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/context.py:122:callWithContext
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/context.py:85:callWithContext
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/enterprise/adbapi.py:306:_runWithConnection
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/compat.py:464:reraise
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/enterprise/adbapi.py:297:_runWithConnection
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/database.py:572:inner_func
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/database.py:418:new_transaction
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/logging/utils.py:73:wrapped
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/data_stores/main/events.py:451:_persist_events_txn
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/data_stores/main/events.py:1001:_update_metadata_tables_txn
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/data_stores/main/events.py:1299:_handle_event_relations
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/database.py:653:simple_insert_txn
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/database.py:210:execute
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/database.py:236:_do_execute
]]
Traceback (most recent call last):
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/handlers/federation.py", line 1033, in try_backfill
dom, room_id, limit=100, extremities=extremities
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/handlers/federation.py", line 909, in backfill
await self._handle_new_event(dest, event, backfilled=True)
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/handlers/federation.py", line 1845, in _handle_new_event
[(event, context)], backfilled=backfilled
File "/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/handlers/federation.py", line 2870, in persist_events_and_notify
event_and_contexts, backfilled=backfilled
twisted.internet.defer.FirstError: FirstError[#0, [Failure instance: Traceback: <class 'psycopg2.errors.UniqueViolation'>: duplicate key value violates unique constraint "event_relations_id"
DETAIL:  Key (event_id)=($hnVIuNuExbqJ-9iXkDpaRAGdKdeDue4YI8w9G1qOSHc) already exists.

/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/internet/defer.py:654:_runCallbacks
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/internet/defer.py:1475:gotResult
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/internet/defer.py:1416:_inlineCallbacks
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/failure.py:512:throwExceptionIntoGenerator
--- <exception caught here> ---
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/persist_events.py:152:handle_queue_loop
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/persist_events.py:262:persisting_queue
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/persist_events.py:438:_persist_events
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/internet/defer.py:1416:_inlineCallbacks
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/failure.py:512:throwExceptionIntoGenerator
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/data_stores/main/events.py:96:f
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/internet/defer.py:1416:_inlineCallbacks
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/failure.py:512:throwExceptionIntoGenerator
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/data_stores/main/events.py:211:_persist_events_and_state_updates
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/internet/defer.py:1416:_inlineCallbacks
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/failure.py:512:throwExceptionIntoGenerator
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/database.py:527:runInteraction
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/internet/defer.py:1416:_inlineCallbacks
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/failure.py:512:throwExceptionIntoGenerator
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/database.py:575:runWithConnection
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/threadpool.py:250:inContext
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/threadpool.py:266:<lambda>
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/context.py:122:callWithContext
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/context.py:85:callWithContext
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/enterprise/adbapi.py:306:_runWithConnection
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/python/compat.py:464:reraise
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/twisted/enterprise/adbapi.py:297:_runWithConnection
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/database.py:572:inner_func
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/database.py:418:new_transaction
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/logging/utils.py:73:wrapped
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/data_stores/main/events.py:451:_persist_events_txn
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/data_stores/main/events.py:1001:_update_metadata_tables_txn
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/data_stores/main/events.py:1299:_handle_event_relations
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/database.py:653:simple_insert_txn
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/database.py:210:execute
/opt/venvs/matrix-synapse/lib/python3.7/site-packages/synapse/storage/database.py:236:_do_execute
]]

# Fixed code
def _purge_history_txn(self, txn, room_id, token_str, delete_local_events):
    token = RoomStreamToken.parse(token_str)

    # Tables that should be pruned:
    #     event_auth
    #     event_backward_extremities
    #     event_edges
    #     event_forward_extremities
    #     event_json
    #     event_push_actions
    #     event_reference_hashes
    #     event_relations
    #     event_search
    #     event_to_state_groups
    #     events
    #     rejections
    #     room_depth
    #     state_groups
    #     state_groups_state

    # we will build a temporary table listing the events so that we don't
    # have to keep shovelling the list back and forth across the
    # connection. Annoyingly the python sqlite driver commits the
    # transaction on CREATE, so let's do this first.
    #
    # furthermore, we might already have the table from a previous (failed)
    # purge attempt, so let's drop the table first.

    txn.execute("DROP TABLE IF EXISTS events_to_purge")

    txn.execute(
        "CREATE TEMPORARY TABLE events_to_purge ("
        "    event_id TEXT NOT NULL,"
        "    should_delete BOOLEAN NOT NULL"
        ")"
    )

    # First ensure that we're not about to delete all the forward extremeties
    txn.execute(
        "SELECT e.event_id, e.depth FROM events as e "
        "INNER JOIN event_forward_extremities as f "
        "ON e.event_id = f.event_id "
        "AND e.room_id = f.room_id "
        "WHERE f.room_id = ?",
        (room_id,),
    )
    rows = txn.fetchall()
    max_depth = max(row[1] for row in rows)

    if max_depth < token.topological:
        # We need to ensure we don't delete all the events from the database
        # otherwise we wouldn't be able to send any events (due to not
        # having any backwards extremeties)
        raise SynapseError(
            400, "topological_ordering is greater than forward extremeties"
        )

    logger.info("[purge] looking for events to delete")

    should_delete_expr = "state_key IS NULL"
    should_delete_params = ()  # type: Tuple[Any, ...]
    if not delete_local_events:
        should_delete_expr += " AND event_id NOT LIKE ?"

        # We include the parameter twice since we use the expression twice
        should_delete_params += ("%:" + self.hs.hostname, "%:" + self.hs.hostname)

    should_delete_params += (room_id, token.topological)

    # Note that we insert events that are outliers and aren't going to be
    # deleted, as nothing will happen to them.
    txn.execute(
        "INSERT INTO events_to_purge"
        " SELECT event_id, %s"
        " FROM events AS e LEFT JOIN state_events USING (event_id)"
        " WHERE (NOT outlier OR (%s)) AND e.room_id = ? AND topological_ordering < ?"
        % (should_delete_expr, should_delete_expr),
        should_delete_params,
    )

    # We create the indices *after* insertion as that's a lot faster.

    # create an index on should_delete because later we'll be looking for
    # the should_delete / shouldn't_delete subsets
    txn.execute(
        "CREATE INDEX events_to_purge_should_delete ON events_to_purge(should_delete)"
    )

    # We do joins against events_to_purge for e.g. calculating state
    # groups to purge, etc., so lets make an index.
    txn.execute("CREATE INDEX events_to_purge_id ON events_to_purge(event_id)")

    txn.execute("SELECT event_id, should_delete FROM events_to_purge")
    event_rows = txn.fetchall()
    logger.info(
        "[purge] found %i events before cutoff, of which %i can be deleted",
        len(event_rows),
        sum(1 for e in event_rows if e[1]),
    )

    logger.info("[purge] Finding new backward extremities")

    # We calculate the new entries for the backward extremeties by finding
    # events to be purged that are pointed to by events we're not going to
    # purge.
    txn.execute(
        "SELECT DISTINCT e.event_id FROM events_to_purge AS e"
        " INNER JOIN event_edges AS ed ON e.event_id = ed.prev_event_id"
        " LEFT JOIN events_to_purge AS ep2 ON ed.event_id = ep2.event_id"
        " WHERE ep2.event_id IS NULL"
    )
    new_backwards_extrems = txn.fetchall()

    logger.info("[purge] replacing backward extremities: %r", new_backwards_extrems)

    txn.execute("DELETE FROM event_backward_extremities WHERE room_id = ?", (room_id,))

    # Update backward extremeties
    txn.executemany(
        "INSERT INTO event_backward_extremities (room_id, event_id) VALUES (?, ?)",
        [(room_id, event_id) for (event_id,) in new_backwards_extrems],
    )

    logger.info("[purge] finding state groups referenced by deleted events")

    # Get all state groups that are referenced by events that are to be
    # deleted.
    txn.execute(
        """
            SELECT DISTINCT state_group FROM events_to_purge
            INNER JOIN event_to_state_groups USING (event_id)
        """
    )

    referenced_state_groups = {sg for (sg,) in txn}
    logger.info(
        "[purge] found %i referenced state groups", len(referenced_state_groups)
    )

    logger.info("[purge] removing events from event_to_state_groups")
    txn.execute(
        "DELETE FROM event_to_state_groups "
        "WHERE event_id IN (SELECT event_id from events_to_purge)"
    )
    for event_id, _ in event_rows:
        txn.call_after(self._get_state_group_for_event.invalidate, (event_id,))

    # Delete all remote non-state events
    for table in (
        "events",
        "event_json",
        "event_auth",
        "event_edges",
        "event_forward_extremities",
        "event_reference_hashes",
        "event_relations",
        "event_search",
        "rejections",
    ):
        logger.info("[purge] removing events from %s", table)

        txn.execute(
            "DELETE FROM %s WHERE event_id IN ("
            "    SELECT event_id FROM events_to_purge WHERE should_delete"
            ")" % (table,)
        )

    # event_push_actions lacks an index on event_id, and has one on
    # (room_id, event_id) instead.
    for table in ("event_push_actions",):
        logger.info("[purge] removing events from %s", table)

        txn.execute(
            "DELETE FROM %s WHERE room_id = ? AND event_id IN ("
            "    SELECT event_id FROM events_to_purge WHERE should_delete"
            ")" % (table,),
            (room_id,),
        )

    # Mark all state and own events as outliers
    logger.info("[purge] marking remaining events as outliers")
    txn.execute(
        "UPDATE events SET outlier = ?"
        " WHERE event_id IN ("
        "    SELECT event_id FROM events_to_purge "
        "    WHERE NOT should_delete"
        ")",
        (True,),
    )

    # synapse tries to take out an exclusive lock on room_depth whenever it
    # persists events (because upsert), and once we run this update, we
    # will block that for the rest of our transaction.
    #
    # So, let's stick it at the end so that we don't block event
    # persistence.
    #
    # We do this by calculating the minimum depth of the backwards
    # extremities. However, the events in event_backward_extremities
    # are ones we don't have yet so we need to look at the events that
    # point to it via event_edges table.
    txn.execute(
        """
            SELECT COALESCE(MIN(depth), 0)
            FROM event_backward_extremities AS eb
            INNER JOIN event_edges AS eg ON eg.prev_event_id = eb.event_id
            INNER JOIN events AS e ON e.event_id = eg.event_id
            WHERE eb.room_id = ?
        """,
        (room_id,),
    )
    (min_depth,) = txn.fetchone()

    logger.info("[purge] updating room_depth to %d", min_depth)

    txn.execute(
        "UPDATE room_depth SET min_depth = ? WHERE room_id = ?",
        (min_depth, room_id),
    )

    # finally, drop the temp table. this will commit the txn in sqlite,
    # so make sure to keep this actually last.
    txn.execute("DROP TABLE events_to_purge")

    logger.info("[purge] done")

    return referenced_state_groups