Repository: matrix-org/synapse

Issue:
Running under sqlite, Synapse incorrectly populates the to-device messages current stream ID
Spotted by @kegsay, diagnosed by me.

Synapse writes to the device inbox and outbox tables here:

https://github.com/matrix-org/synapse/blob/3e8531d3baf205733693f9ae8b43aa0b4c82b744/synapse/storage/databases/main/deviceinbox.py#L804-L814

https://github.com/matrix-org/synapse/blob/3e8531d3baf205733693f9ae8b43aa0b4c82b744/synapse/storage/databases/main/deviceinbox.py#L741-L802

Suppose that:

1. Synapse calls add_messages_txn with 0 local events (for the inbox) and at least 1 remote event (for the outbox). This is stored normally in the DB.
2. The event fails to send, or otherwise does not get its chance to be sent out to the destination.
3. Synapse shuts down.

At this point, 

```sql
SELECT MAX(stream_id) FROM device_inbox
```

is strictly smaller than

```sql
SELECT MAX(stream_id) FROM device_federation_outbox
```

(the latter is the stream ID used in step 1.)

Now Synapse restarts. On sqlite, it populates the to-device stream using

https://github.com/matrix-org/synapse/blob/3e8531d3baf205733693f9ae8b43aa0b4c82b744/synapse/storage/databases/main/deviceinbox.py#L104-L106

which will end up calling

https://github.com/matrix-org/synapse/blob/91587d4cf903c926a29ed70cecd63549271dc3b9/synapse/storage/util/id_generators.py#L78-L93

which is the _first_ query, which is smaller.

This means the stream_id has jumped backwards.

I don't think this is necessarily the end of the world, since it looks like the tables don't expect the stream ID to be unique (i.e. they're fine with it being reused). But it is a surprise and makes debugging more confusing.


Discussion:

Resolving patch:
diff --git a/synapse/replication/tcp/streams/_base.py b/synapse/replication/tcp/streams/_base.py
--- a/synapse/replication/tcp/streams/_base.py
+++ b/synapse/replication/tcp/streams/_base.py
@@ -621,7 +621,7 @@ def __init__(self, hs: "HomeServer"):
         super().__init__(
             hs.get_instance_name(),
             store.get_all_new_device_messages,
-            store._device_inbox_id_gen,
+            store._to_device_msg_id_gen,
         )
 
 
diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py
--- a/synapse/storage/databases/main/deviceinbox.py
+++ b/synapse/storage/databases/main/deviceinbox.py
@@ -87,25 +87,32 @@ def __init__(
                 self._instance_name in hs.config.worker.writers.to_device
             )
 
-            self._device_inbox_id_gen: AbstractStreamIdGenerator = (
+            self._to_device_msg_id_gen: AbstractStreamIdGenerator = (
                 MultiWriterIdGenerator(
                     db_conn=db_conn,
                     db=database,
                     notifier=hs.get_replication_notifier(),
                     stream_name="to_device",
                     instance_name=self._instance_name,
-                    tables=[("device_inbox", "instance_name", "stream_id")],
+                    tables=[
+                        ("device_inbox", "instance_name", "stream_id"),
+                        ("device_federation_outbox", "instance_name", "stream_id"),
+                    ],
                     sequence_name="device_inbox_sequence",
                     writers=hs.config.worker.writers.to_device,
                 )
             )
         else:
             self._can_write_to_device = True
-            self._device_inbox_id_gen = StreamIdGenerator(
-                db_conn, hs.get_replication_notifier(), "device_inbox", "stream_id"
+            self._to_device_msg_id_gen = StreamIdGenerator(
+                db_conn,
+                hs.get_replication_notifier(),
+                "device_inbox",
+                "stream_id",
+                extra_tables=[("device_federation_outbox", "stream_id")],
             )
 
-        max_device_inbox_id = self._device_inbox_id_gen.get_current_token()
+        max_device_inbox_id = self._to_device_msg_id_gen.get_current_token()
         device_inbox_prefill, min_device_inbox_id = self.db_pool.get_cache_dict(
             db_conn,
             "device_inbox",
@@ -145,8 +152,8 @@ def process_replication_rows(
     ) -> None:
         if stream_name == ToDeviceStream.NAME:
             # If replication is happening than postgres must be being used.
-            assert isinstance(self._device_inbox_id_gen, MultiWriterIdGenerator)
-            self._device_inbox_id_gen.advance(instance_name, token)
+            assert isinstance(self._to_device_msg_id_gen, MultiWriterIdGenerator)
+            self._to_device_msg_id_gen.advance(instance_name, token)
             for row in rows:
                 if row.entity.startswith("@"):
                     self._device_inbox_stream_cache.entity_has_changed(
@@ -162,11 +169,11 @@ def process_replication_position(
         self, stream_name: str, instance_name: str, token: int
     ) -> None:
         if stream_name == ToDeviceStream.NAME:
-            self._device_inbox_id_gen.advance(instance_name, token)
+            self._to_device_msg_id_gen.advance(instance_name, token)
         super().process_replication_position(stream_name, instance_name, token)
 
     def get_to_device_stream_token(self) -> int:
-        return self._device_inbox_id_gen.get_current_token()
+        return self._to_device_msg_id_gen.get_current_token()
 
     async def get_messages_for_user_devices(
         self,
@@ -801,7 +808,7 @@ def add_messages_txn(
                                 msg.get(EventContentFields.TO_DEVICE_MSGID),
                             )
 
-        async with self._device_inbox_id_gen.get_next() as stream_id:
+        async with self._to_device_msg_id_gen.get_next() as stream_id:
             now_ms = self._clock.time_msec()
             await self.db_pool.runInteraction(
                 "add_messages_to_device_inbox", add_messages_txn, now_ms, stream_id
@@ -813,7 +820,7 @@ def add_messages_txn(
                     destination, stream_id
                 )
 
-        return self._device_inbox_id_gen.get_current_token()
+        return self._to_device_msg_id_gen.get_current_token()
 
     async def add_messages_from_remote_to_device_inbox(
         self,
@@ -857,7 +864,7 @@ def add_messages_txn(
                 txn, stream_id, local_messages_by_user_then_device
             )
 
-        async with self._device_inbox_id_gen.get_next() as stream_id:
+        async with self._to_device_msg_id_gen.get_next() as stream_id:
             now_ms = self._clock.time_msec()
             await self.db_pool.runInteraction(
                 "add_messages_from_remote_to_device_inbox",


Test patch: